You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

6313 lines
303 KiB
Python

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

"""Browser setup, authenticated API, and VM console for one remote host."""
from __future__ import annotations
import asyncio
import hashlib
import hmac
import ipaddress
import json
import logging
import os
import re
import secrets
import subprocess
import tempfile
import threading
import time
import uuid
import xml.etree.ElementTree as ET
from concurrent.futures import ThreadPoolExecutor
from contextlib import asynccontextmanager, suppress
from pathlib import Path
from typing import Any, Callable, Literal
from urllib.parse import urlsplit
from fastapi import BackgroundTasks, Depends, FastAPI, HTTPException, Query, Request, WebSocket
from fastapi.responses import FileResponse, JSONResponse
from fastapi.staticfiles import StaticFiles
from pydantic import BaseModel, ConfigDict, Field, SecretStr, model_validator
from .auth import DatabaseSessionStore, LoginLimiter, Session, SessionStore, verify_password
from .cloud_init import (
_validate_guest_auth, _validate_ssh_public_key, generate_root_password,
hash_root_password, validate_cloud_network, validate_guest_hostname,
)
from .cloud_templates import (
TemplateError, TemplateStore, download_verified_image, image_metadata_for,
template_id_for, template_scope, template_volume_name, validate_template_name,
)
from .config import Settings
from .console import ConsoleRelayError, relay_console
from .database import DatabaseError, make_components
from .database_bootstrap import DatabaseBootstrap, DatabaseSetupError
from .libvirt_service import (
FIREWALL_FILTER_COMPILER_VERSION,
REINSTALL_CLEANUP_PENDING_ERROR_CODE,
REINSTALL_RESULT_UNCERTAIN_ERROR_CODE,
SUPPORTED_FIREWALL_FILTER_COMPILER_VERSIONS,
SNAPSHOT_KEY_RE, VMError, VMService, VMSnapshotUncertainError,
build_firewall_filter_xml, firewall_policy_digest, snapshot_volume_name,
)
from .metrics import (MAX_QUERY_POINTS, RETENTION_SECONDS, RANGES, MetricsCollector,
host_scope, query_interval_seconds)
from .panel_state import PanelState, SetupError
from .platform_network_policy import (
default_platform_policy_snapshot, normalize_platform_policy_snapshot,
)
from .remote_host import (
MAX_PUBLIC_NETWORK_POOLS,
PUBLIC_POOL_KEY,
HostProfile,
RemoteHostError,
RemoteHostManager,
validate_public_network_pool,
validate_public_network_pools,
)
from .rescue_templates import (
MAX_RESCUE_ISO_BYTES, RescueTemplateStore, rescue_scope,
rescue_volume_name, verify_bootable_iso,
)
from .ssh_keys import SSHKeyError
from .smtp_service import SMTPError, SMTPSettingsStore, send_verification_code
from .updater import BRANCH, REPOSITORY_URL, UpdateError, Updater
from .user_isos import (
UserISOFiles, UserISOStore, download_user_iso, user_iso_scope,
user_iso_volume_name, validate_user_iso_name, validate_user_iso_url,
)
from .users import UserError, normalize_email
from .engine_context import EngineComponents, TRUSTED_ACTOR, actor, actor_account, actor_session
COOKIE_NAME = "virsh_panel_session"
CONSOLE_ACCESS_CHECK_SECONDS = 15
LOG = logging.getLogger(__name__)
STARTUP_LOG = logging.getLogger("uvicorn.error")
SNAPSHOT_PUBLIC_ERROR_CODES = {
"snapshot_guest_agent_unavailable",
"live_snapshot_unsupported",
"snapshot_pool_unsupported",
"snapshot_result_uncertain",
}
REINSTALL_PUBLIC_ERROR_CODES = {
REINSTALL_CLEANUP_PENDING_ERROR_CODE,
REINSTALL_RESULT_UNCERTAIN_ERROR_CODE,
}
# These phases are emitted only after the replacement definition has been
# created or the old-volume cleanup boundary has been crossed. Keep this
# compatibility list for rows written by releases that did not persist the
# explicit ``reinstall_committed`` marker before a process restart.
REINSTALL_COMMITTED_PHASES = frozenset({
"cleaning_up", "cleanup_pending", "result_uncertain", "completed",
"重装已完成,但凭据或任务记录保存失败",
"重装完成,但旧系统盘清理待核对", "重装结果待核对",
})
def reinstall_phase_committed(value: Any) -> bool:
return isinstance(value, str) and value.strip() in REINSTALL_COMMITTED_PHASES
def reinstall_cleanup_pending(job: dict[str, Any]) -> bool:
phase = job.get("phase")
if isinstance(phase, str):
phase = phase.strip()
error_code = job.get("error_code")
return (
error_code == REINSTALL_CLEANUP_PENDING_ERROR_CODE
or job.get("cleanup_pending") is True
or phase in {"cleanup_pending", "重装完成,但旧系统盘清理待核对"}
or (isinstance(job.get("warning"), str)
and job["warning"].strip().startswith("旧存储卷"))
)
def reinstall_post_commit(job: dict[str, Any]) -> bool:
error_code = job.get("error_code")
return (
job.get("reinstall_committed") is True
or error_code in REINSTALL_PUBLIC_ERROR_CODES
or reinstall_phase_committed(job.get("phase"))
or reinstall_cleanup_pending(job)
or (isinstance(job.get("warning"), str)
and bool(job["warning"].strip()))
)
class PublicIpClaimCleanupPending(VMError):
"""The VM mutation completed but its engine IP claim did not."""
class VMDeleteSnapshotCleanupPending(VMError):
"""The VM and its primary disks are gone but snapshot GC is incomplete."""
def __init__(self, message: str) -> None:
super().__init__(
message, 503,
error_code="vm_delete_snapshot_cleanup_pending",
)
class VMDeleteFirewallCleanupPending(VMError):
"""The VM is gone but its deterministic nwfilter still needs cleanup."""
def __init__(self, message: str) -> None:
super().__init__(
message, 503,
error_code="vm_delete_firewall_cleanup_pending",
)
class SetupBody(BaseModel):
model_config = ConfigDict(extra="forbid")
token: str = Field(min_length=1, max_length=256)
username: str = Field(min_length=3, max_length=32)
password: str = Field(min_length=12, max_length=1024)
class DatabaseSetupBody(BaseModel):
model_config = ConfigDict(extra="forbid")
host: str = Field(min_length=1, max_length=253)
port: int = Field(ge=1, le=65535)
database: str = Field(min_length=1, max_length=64)
username: str = Field(min_length=1, max_length=128)
password: SecretStr
setup_token: str = Field(min_length=1, max_length=256)
class UpdateSettingsBody(BaseModel):
model_config = ConfigDict(extra="forbid")
repository_url: str = Field(default=REPOSITORY_URL, max_length=2048)
branch: str = Field(default=BRANCH, max_length=128)
access_token: SecretStr | None = None
class UpdateApplyBody(BaseModel):
model_config = ConfigDict(extra="forbid")
expected_current_commit: str = Field(pattern=r"^(?:[0-9a-f]{40}|[0-9a-f]{64})$")
expected_remote_commit: str = Field(pattern=r"^(?:[0-9a-f]{40}|[0-9a-f]{64})$")
class LoginBody(BaseModel):
model_config = ConfigDict(extra="forbid")
email: str | None = Field(default=None, min_length=3, max_length=254)
# Only the injected, database-free test harness may use the historical
# username login. Production explicitly rejects this field below.
username: str | None = Field(default=None, min_length=1, max_length=64)
password: str = Field(min_length=1, max_length=1024)
class RegistrationCodeBody(BaseModel):
model_config = ConfigDict(extra="forbid")
email: str = Field(min_length=3, max_length=254)
class RegistrationBody(RegistrationCodeBody):
username: str = Field(min_length=3, max_length=32)
password: str = Field(min_length=12, max_length=1024)
code: str = Field(pattern=r"^[0-9]{6}$")
class SMTPSettingsBody(BaseModel):
model_config = ConfigDict(extra="forbid")
host: str = Field(min_length=1, max_length=253)
port: int = Field(ge=1, le=65535)
tls_mode: Literal["ssl", "starttls", "none"]
username: str = Field(default="", max_length=256)
password: SecretStr | None = None
from_address: str = Field(min_length=3, max_length=254)
class UnlockBody(BaseModel):
model_config = ConfigDict(extra="forbid")
passphrase: str = Field(min_length=1, max_length=1024)
class HostScanBody(BaseModel):
model_config = ConfigDict(extra="forbid")
host: str = Field(min_length=1, max_length=253)
port: int = Field(ge=1, le=65535)
class HostBody(HostScanBody):
private_key: str = Field(default="", max_length=65536)
private_key_passphrase: str = Field(default="", max_length=1024)
host_key: str = Field(min_length=1, max_length=4096)
confirmed_fingerprint: str = Field(min_length=1, max_length=256)
storage_pool: str = Field(default="default", min_length=1, max_length=4096)
iso_pool: str = Field(default="iso", min_length=1, max_length=4096)
rescue_iso_pool: str = Field(default="", max_length=4096)
default_network: str = Field(default="default", min_length=1, max_length=128)
region: str = Field(default="", max_length=64,
pattern=r"^[^\x00-\x1f\x7f]*$")
public_pool_cidr: str = Field(default="", max_length=18)
public_gateway: str = Field(default="", max_length=15)
public_dns_servers: list[str] = Field(default_factory=list, max_length=4)
public_reserved_ips: list[str] = Field(default_factory=list, max_length=254)
public_vm_prefix: int | None = Field(default=None, ge=24, le=30)
def profile(self) -> HostProfile:
return HostProfile(
host=self.host,
port=self.port,
private_key=self.private_key,
host_key=self.host_key,
private_key_passphrase=self.private_key_passphrase,
storage_pool=self.storage_pool,
iso_pool=self.iso_pool,
rescue_iso_pool=self.rescue_iso_pool,
default_network=self.default_network,
region=self.region,
public_pool_cidr=self.public_pool_cidr,
public_gateway=self.public_gateway,
public_dns_servers=self.public_dns_servers,
public_reserved_ips=self.public_reserved_ips,
public_vm_prefix=self.public_vm_prefix,
)
class PublicNetworkPoolBody(BaseModel):
model_config = ConfigDict(extra="forbid")
revision: int = Field(ge=1, le=(1 << 63) - 1)
state: Literal["active", "draining", "disabled"] = "active"
network_selector: str = Field(min_length=1, max_length=128)
cidr: str = Field(min_length=7, max_length=18)
gateway: str = Field(min_length=7, max_length=15)
dns_servers: list[str] = Field(min_length=1, max_length=4)
reserved_ips: list[str] = Field(default_factory=list, max_length=254)
guest_prefix_length: int = Field(ge=24, le=30)
class CreateISOBody(BaseModel):
model_config = ConfigDict(extra="forbid")
source: Literal["iso"] = "iso"
name: str = Field(min_length=1, max_length=63)
vcpu: int = Field(ge=1, le=64)
memory_mib: int = Field(ge=512, le=262144)
disk_gib: int = Field(ge=1, le=4096)
bandwidth_mbps: int | None = Field(default=None, ge=1, le=100000)
network: str = Field(min_length=1, max_length=128)
iso: str | None = Field(default=None, min_length=1, max_length=255)
user_iso_id: str | None = Field(default=None, pattern=r"^[0-9a-f]{32}$")
@model_validator(mode="after")
def exactly_one_iso_source(self):
if (self.iso is None) == (self.user_iso_id is None):
raise ValueError("iso 与 user_iso_id 必须且只能选择一个")
return self
class CreateCloudBody(BaseModel):
model_config = ConfigDict(extra="forbid")
source: Literal["cloud"]
name: str = Field(min_length=1, max_length=63)
vcpu: int = Field(ge=1, le=64)
memory_mib: int = Field(ge=512, le=262144)
disk_gib: int = Field(ge=1, le=4096)
bandwidth_mbps: int | None = Field(default=None, ge=1, le=100000)
network: str = Field(min_length=1, max_length=128)
template_id: str = Field(min_length=1, max_length=80)
public_pool_key: str = Field(default="", max_length=32,
pattern=r"^(?:|[0-9a-f]{32})$")
public_ip_cidr: str = Field(default="", max_length=64)
gateway: str = Field(default="", max_length=64)
dns_servers: list[str] = Field(default_factory=list, max_length=4)
guest_ssh_public_key: str = Field(default="", max_length=16384)
guest_ssh_key_id: int | None = Field(default=None, ge=1)
root_password: SecretStr | None = None
platform_network_policy_snapshot: dict[str, Any] = Field(
default_factory=default_platform_policy_snapshot)
@model_validator(mode="after")
def validate_platform_network_policy(self):
try:
self.platform_network_policy_snapshot = (
normalize_platform_policy_snapshot(
self.platform_network_policy_snapshot))
except ValueError as exc:
raise ValueError("平台网络策略快照无效") from exc
return self
class ChangeNetworkBody(BaseModel):
model_config = ConfigDict(extra="forbid")
public_ip_cidr: str = Field(min_length=7, max_length=64)
public_pool_key: str = Field(default="", max_length=32,
pattern=r"^(?:|[0-9a-f]{32})$")
confirm_restart: bool = False
class VMSettingsBody(BaseModel):
model_config = ConfigDict(extra="forbid")
# Every field is optional so an administrator can change one setting at a
# time. The validator below rejects an empty patch; omitted credentials
# are preserved from the existing cloud-init seed.
hostname: str | None = Field(default=None, max_length=253)
root_password: SecretStr | None = Field(default=None, min_length=12, max_length=128)
ssh_public_key: str | None = Field(default=None, max_length=16384)
confirm_restart: bool = False
@model_validator(mode="after")
def require_setting(self):
if self.hostname is None and self.root_password is None and self.ssh_public_key is None:
raise ValueError("至少需要修改一项虚拟机设置")
return self
class SSHKeyBody(BaseModel):
model_config = ConfigDict(extra="forbid")
name: str = Field(min_length=1, max_length=80)
public_key: str = Field(min_length=1, max_length=16384)
class ImportTemplateBody(BaseModel):
model_config = ConfigDict(extra="forbid")
url: str = Field(min_length=16, max_length=2048)
sha256: str = Field(default="", max_length=64)
name: str = Field(min_length=1, max_length=64)
class ImportUserISOBody(BaseModel):
model_config = ConfigDict(extra="forbid")
url: str = Field(min_length=1, max_length=2048)
sha256: str = Field(default="", max_length=64)
name: str = Field(min_length=1, max_length=64)
class RenameTemplateBody(BaseModel):
model_config = ConfigDict(extra="forbid")
name: str = Field(min_length=1, max_length=64)
class DeleteTemplateBody(BaseModel):
model_config = ConfigDict(extra="forbid")
created_at: int = Field(ge=1)
class ReinstallBody(BaseModel):
model_config = ConfigDict(extra="forbid")
template_id: str = Field(min_length=1, max_length=80)
ssh_public_key: str = Field(default="", max_length=16384)
ssh_key_id: int | None = Field(default=None, ge=1)
root_password: SecretStr | None = None
confirmation_name: str = Field(min_length=1, max_length=63)
class EnterRescueBody(BaseModel):
model_config = ConfigDict(extra="forbid")
template_id: str = Field(pattern=r"^[0-9a-f]{32}$")
confirmation_name: str = Field(min_length=1, max_length=63)
class ExitRescueBody(BaseModel):
model_config = ConfigDict(extra="forbid")
confirmation_name: str = Field(min_length=1, max_length=63)
# PHP's billing ledger supplies the authoritative target. The original
# XML was saved before rescue mode and may predate a traffic-quota block.
network_suspended: bool = False
class ForceRescueBody(BaseModel):
model_config = ConfigDict(extra="forbid")
confirmation_name: str = Field(min_length=1, max_length=63)
ack_data_loss: Literal[True]
class ActionBody(BaseModel):
model_config = ConfigDict(extra="forbid")
action: Literal["start", "shutdown", "reboot", "poweroff", "reset"]
class CreateSnapshotBody(BaseModel):
model_config = ConfigDict(extra="forbid")
snapshot_key: str = Field(pattern=r"^[0-9a-f]{32}$")
name: str = Field(min_length=1, max_length=64)
consistency_mode: Literal["filesystem", "crash"] = "filesystem"
@model_validator(mode="after")
def validate_name(self):
if (self.name != self.name.strip()
or any(ord(char) < 32 or ord(char) == 127 for char in self.name)):
raise ValueError("快照名称不能包含首尾空白或控制字符")
return self
class EmptySnapshotBody(BaseModel):
model_config = ConfigDict(extra="forbid")
class ServiceResumeBody(BaseModel):
model_config = ConfigDict(extra="forbid")
# Only restart a guest that the expiry policy itself stopped. A tenant
# that had already shut the VM down before expiry must remain shut down
# after renewal.
start_if_suspended: bool = False
class FirewallRuleBody(BaseModel):
model_config = ConfigDict(extra="forbid")
protocol: Literal["tcp", "udp", "icmp"]
source_cidr: str = Field(min_length=1, max_length=18)
port_from: int | None = Field(default=None, ge=1, le=65535)
port_to: int | None = Field(default=None, ge=1, le=65535)
description: str = Field(default="", max_length=120)
@model_validator(mode="after")
def validate_firewall_rule(self):
try:
source = ipaddress.ip_network(self.source_cidr, strict=False)
except ValueError as exc:
raise ValueError("来源必须是 IPv4 CIDR") from exc
if source.version != 4:
raise ValueError("防火墙一期仅支持 IPv4")
if self.source_cidr != str(source):
raise ValueError("来源必须使用规范 IPv4 CIDR")
self.source_cidr = str(source)
if self.protocol in {"tcp", "udp"}:
if (self.port_from is None or self.port_to is None
or self.port_from > self.port_to):
raise ValueError("TCP/UDP 必须提供有效端口范围")
elif self.port_from is not None or self.port_to is not None:
raise ValueError("ICMP 规则不能包含端口")
if (self.description != self.description.strip()
or any(ord(char) < 32 or ord(char) == 127
for char in self.description)):
raise ValueError("规则说明不能包含首尾空白或控制字符")
return self
class FirewallPolicyBody(BaseModel):
model_config = ConfigDict(extra="forbid")
enabled: bool
revision: int = Field(ge=1, le=9223372036854775807)
policy_digest: str = Field(pattern=r"^[0-9a-f]{64}$")
rules: list[FirewallRuleBody] = Field(default_factory=list, max_length=50)
def _public_host(profile: dict | None) -> dict:
if profile is None:
return {"configured": False}
keys = (
"host", "port", "key_type", "fingerprint", "storage_pool",
"iso_pool", "default_network", "private_key_configured",
"private_key_encrypted", "key_locked", "private_key_passphrase_saved",
)
return {
"configured": True,
**{key: profile[key] for key in keys if key in profile},
"rescue_iso_pool": profile.get("rescue_iso_pool", ""),
"region": profile.get("region", ""),
"public_pool_cidr": profile.get("public_pool_cidr", ""),
"public_gateway": profile.get("public_gateway", ""),
"public_dns_servers": profile.get("public_dns_servers", []),
"public_reserved_ips": profile.get("public_reserved_ips", []),
"public_vm_prefix": profile.get("public_vm_prefix"),
}
def _peer(request: Request) -> str:
# Uvicorn has already processed forwarding headers from its trusted local
# reverse proxy. Never read a raw client-supplied X-Real-IP here, because
# that would let a login attacker reset the rate limit.
return request.client.host if request.client else "unknown"
def _origin_matches_host(origin: str | None, host: str | None) -> bool:
if not origin or not host:
return False
try:
parsed = urlsplit(origin)
except ValueError:
return False
if parsed.scheme != "https":
return False
return bool(
parsed.netloc == host
and parsed.hostname
and not parsed.username
and not parsed.password
and not parsed.path
and not parsed.query
and not parsed.fragment
)
def _plain_loopback_allowed(scope: dict, host: str | None) -> bool:
"""Permit local diagnostics only when both peer and requested host are local.
The request scheme is supplied by the ASGI server. Uvicorn's proxy header
handling may upgrade it to HTTPS/WSS only for its explicitly trusted proxy
address; this function never trusts an incoming forwarding header itself.
"""
client = scope.get("client")
if not client or not host:
return False
try:
peer_is_local = ipaddress.ip_address(client[0]).is_loopback
host_is_local = urlsplit("//" + host).hostname in {
"localhost", "127.0.0.1", "::1",
}
except ValueError:
return False
return peer_is_local and host_is_local
def create_app(
settings: Settings | None = None,
*,
panel_state: PanelState | None = None,
remote_manager: RemoteHostManager | None = None,
service_factory: Callable[[dict], VMService] | None = None,
metric_store: Any | None = None,
engine_components: EngineComponents | None = None,
) -> FastAPI:
settings = settings or Settings.from_env()
settings.validate()
# Injected objects keep the existing isolated HTTP test harness working.
# Normal production startup never uses the old JSON-backed stores.
internal_mode = engine_components is not None
database_mode = internal_mode or (panel_state is None and remote_manager is None)
bootstrap = DatabaseBootstrap(settings.data_dir) if database_mode and not internal_mode else None
data_store = None
updater = None
if internal_mode:
state = engine_components.state
manager = engine_components.manager
templates = engine_components.templates
data_store = engine_components.store
metric_store = engine_components.metric_store
setup_token = None
elif database_mode:
configured_database = bootstrap.load_config()
if configured_database is None:
setup_token = bootstrap.ensure_token()
state = None
manager = None
templates = None
else:
state, manager, templates, data_store = make_components(
settings.data_dir, configured_database, bootstrap.master_key(),
settings.ssh_binary,
)
setup_token = (bootstrap.ensure_token()
if state.setup_required() else state.initialize_setup_token())
else:
state = panel_state or PanelState(settings.data_dir)
setup_token = state.initialize_setup_token()
manager = remote_manager or RemoteHostManager(
settings.data_dir, ssh_binary=settings.ssh_binary)
templates = TemplateStore(settings.data_dir)
if setup_token is not None:
# Uvicorn writes this logger to stderr, which the supplied systemd
# service captures in its journal. Never expose the token over HTTP.
STARTUP_LOG.warning("VIRSH_PANEL_SETUP_TOKEN=%s", setup_token)
if database_mode and (settings.data_dir / "generations").exists():
STARTUP_LOG.warning(
"Legacy data/generations detected; its old SSH key files are not migrated "
"or automatically removed. Verify and remove them separately."
)
shared_cloud_template_runtime = (
engine_components.cloud_templates
if internal_mode and engine_components.cloud_templates is not None
else None
)
import_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="cloud-image-import")
import_lock = threading.RLock()
import_jobs: dict[str, dict] = {}
pending_templates: set[tuple[str, str]] = set()
rescue_catalog = RescueTemplateStore(data_store)
rescue_import_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="rescue-iso-import")
rescue_import_lock = threading.RLock()
rescue_import_jobs: dict[str, dict] = {}
pending_rescue_imports: set[str] = set()
shared_user_iso_runtime = (
engine_components.user_isos
if internal_mode and engine_components.user_isos is not None
else None
)
if shared_user_iso_runtime is not None:
user_iso_catalog = shared_user_iso_runtime.catalog
user_iso_files = shared_user_iso_runtime.files
user_iso_import_executor = shared_user_iso_runtime.executor
user_iso_import_lock = shared_user_iso_runtime.lock
user_iso_import_jobs = shared_user_iso_runtime.jobs
pending_user_iso_imports = shared_user_iso_runtime.pending
else:
user_iso_catalog = UserISOStore(data_store)
user_iso_files = UserISOFiles(settings.data_dir)
user_iso_import_executor = ThreadPoolExecutor(
max_workers=2, thread_name_prefix="user-iso-import")
user_iso_import_lock = threading.RLock()
user_iso_import_jobs: dict[str, dict] = {}
pending_user_iso_imports: set[str] = set()
# One child app represents one host. Serialize marker recheck, cache
# transfer and marker commit so concurrent VM creates cannot both act on a
# stale "not ready" observation for the same remote volume.
user_iso_cache_lock = threading.RLock()
cloud_template_cache_lock = threading.RLock()
reinstall_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="vm-reinstall")
reinstall_lock = threading.RLock()
reinstall_jobs: dict[str, dict] = {}
pending_reinstalls: set[tuple[str, str]] = set()
rescue_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="vm-rescue")
rescue_lock = threading.RLock()
rescue_jobs: dict[str, dict] = {}
pending_rescues: set[tuple[str, str]] = set()
# A full local backup can consume all currently allocated guest blocks.
# Serialize snapshot I/O per host so two sparse targets cannot both pass a
# stale free-space observation and then fill the pool together. This is a
# dedicated executor: unrelated VM reads and operations remain concurrent.
snapshot_executor = ThreadPoolExecutor(
max_workers=1, thread_name_prefix="vm-snapshot")
snapshot_lock = threading.RLock()
snapshot_jobs: dict[str, dict] = {}
snapshot_futures: dict[str, Any] = {}
pending_snapshot_jobs: set[tuple[str, str]] = set()
snapshot_suspend_barriers: set[tuple[str, str]] = set()
test_snapshots: dict[str, str] = {}
test_rescue_states: dict[str, str] = {}
test_firewalls: dict[str, str] = {}
# The production store encrypts each credential record with a fresh AES-GCM
# nonce. The in-memory fallback is only for injected, database-free tests.
test_credentials: dict[str, str] = {}
public_pool_catalog_lock = threading.RLock()
public_pool_catalog_kind = "public_pool_catalog"
public_pool_catalog_key = "current"
def legacy_public_network_pool(profile: dict) -> dict | None:
if not profile.get("public_pool_cidr"):
return None
try:
return validate_public_network_pool({
"pool_key": "legacy-default",
"revision": 1,
"state": "active",
"network_selector": profile["default_network"],
"cidr": profile["public_pool_cidr"],
"gateway": profile["public_gateway"],
"dns_servers": profile.get("public_dns_servers", []),
"reserved_ips": profile.get("public_reserved_ips", []),
"guest_prefix_length": (
profile.get("public_vm_prefix")
if profile.get("public_vm_prefix") is not None
else ipaddress.IPv4Network(
profile["public_pool_cidr"], strict=True).prefixlen
),
}, profile["host"], allow_legacy_key=True)
except (KeyError, TypeError, ValueError, RemoteHostError) as exc:
raise VMError(
"宿主机旧公网地址池配置损坏,请管理员重新保存设置", 500,
) from exc
def read_native_public_network_pools(
profile: dict,
) -> tuple[str | None, list[dict]]:
if data_store is None:
return None, []
raw = data_store.get(public_pool_catalog_kind, public_pool_catalog_key)
if raw is None:
return None, []
try:
catalog = json.loads(raw)
if (not isinstance(catalog, dict) or set(catalog) != {"version", "pools"}
or catalog["version"] != 1
or not isinstance(catalog["pools"], list)):
raise ValueError("invalid public pool catalog")
pools = validate_public_network_pools(catalog["pools"], profile["host"])
except (KeyError, TypeError, ValueError, RemoteHostError) as exc:
raise VMError(
"宿主机公网地址池目录损坏,请管理员核查引擎记录", 500,
) from exc
return raw, pools
def public_network_pools(profile: dict) -> tuple[list[dict], bool]:
raw, pools = read_native_public_network_pools(profile)
if raw is not None:
return pools, False
legacy = legacy_public_network_pool(profile)
return ([] if legacy is None else [legacy]), legacy is not None
def public_pool_runtime_fields(pool: dict) -> dict:
return {
key: pool[key] for key in (
"network_selector", "cidr", "gateway", "dns_servers",
"reserved_ips", "guest_prefix_length",
)
}
def public_pool_for_key(
profile: dict, pool_key: str, *, active: bool = False,
) -> dict:
pools, _compatibility = public_network_pools(profile)
pool = next((item for item in pools if item["pool_key"] == pool_key), None)
if pool is None:
raise VMError("公网地址池不存在或尚未同步到宿主机", 409)
if active and pool["state"] != "active":
raise VMError("公网地址池没有开放新地址分配", 409)
return pool
def public_pool_for_ip(
profile: dict,
public_ip_cidr: str,
*,
active: bool = False,
) -> dict:
try:
requested = ipaddress.IPv4Interface(public_ip_cidr)
except (TypeError, ValueError) as exc:
raise VMError("预留公网 IP 格式无效", 422) from exc
matches = []
pools, _compatibility = public_network_pools(profile)
for pool in pools:
network = ipaddress.IPv4Network(pool["cidr"], strict=True)
if (requested.ip in network
and requested.network.prefixlen == pool["guest_prefix_length"]
and (not active or pool["state"] == "active")):
matches.append(pool)
if len(matches) != 1:
raise VMError("预留公网 IP 无法唯一匹配宿主机地址池", 422)
return matches[0]
def ip_claim_key(profile: dict, address: str) -> str:
return f"{host_scope(profile)}.{address}"
def insert_public_ip_claim(
profile: dict, address: str, vm_id: str, pool_key: str | None = None,
) -> bool:
if data_store is None:
return True
claim_value = {"vm_uuid": vm_id, "reserved_at": int(time.time())}
if pool_key is not None:
claim_value["pool_key"] = pool_key
claim = json.dumps(claim_value, separators=(",", ":"))
return data_store.insert("ip_claim", ip_claim_key(profile, address), claim)
def public_ip_claim_addresses(profile: dict) -> set[ipaddress.IPv4Address]:
claimed: set[ipaddress.IPv4Address] = set()
if data_store is None:
return claimed
scope_prefix = f"{host_scope(profile)}."
for key in data_store.list("ip_claim"):
if not key.startswith(scope_prefix):
continue
try:
claimed.add(ipaddress.IPv4Address(key[len(scope_prefix):]))
except ValueError:
continue
return claimed
def public_network_pool_state(
profile: dict,
pool: dict,
*,
claimed_addresses: set[ipaddress.IPv4Address] | None = None,
) -> dict:
"""Build address state from the panel-owned claim ledger only.
Dedicated hosts are configured out of band and the panel owns every
sellable IP. Do not enumerate libvirt domains on this hot path; the
atomic claim below is the engine-side concurrency guard.
"""
try:
network = ipaddress.IPv4Network(pool["cidr"], strict=True)
gateway = ipaddress.IPv4Address(pool["gateway"])
guest_prefix = pool["guest_prefix_length"]
dns = [ipaddress.IPv4Address(value) for value in pool["dns_servers"]]
reserved = [ipaddress.IPv4Address(value) for value in pool["reserved_ips"]]
statically_blocked = {
network.network_address, network.broadcast_address, gateway,
*dns, *reserved,
}
try:
host_address = ipaddress.IPv4Address(profile.get("host", ""))
except (TypeError, ValueError):
host_address = None
if host_address is not None and host_address in network:
statically_blocked.add(host_address)
except (KeyError, TypeError, ValueError) as exc:
raise VMError(
"宿主机公网地址池配置损坏,请管理员修复设置", 500,
) from exc
allocated: set[ipaddress.IPv4Address] = set()
if claimed_addresses is None:
claimed_addresses = public_ip_claim_addresses(profile)
allocated.update(address for address in claimed_addresses
if address in network)
usable = set(network.hosts())
reserved_in_pool = usable & statically_blocked
allocated_in_pool = (usable - reserved_in_pool) & allocated
available = sorted(usable - reserved_in_pool - allocated_in_pool)
return {
**pool,
"configuration_state": "ready",
"pool_cidr": str(network),
"guest_prefix": guest_prefix,
"usable_count": len(usable),
"reserved_count": len(reserved_in_pool),
"allocated_count": len(allocated_in_pool),
"available": available,
"_network": network,
"_usable": usable,
"_reserved": reserved_in_pool,
"_allocated": allocated_in_pool,
}
def allocate_public_ip(profile: dict, pool: dict, vm_id: str) -> str:
"""Choose and reserve a public IP, including across panel processes.
Claims remain after a successful create and are released on panel VM
deletion. A claim from an uncertain create is retained rather than
risking reuse of an address whose guest may still exist.
"""
if pool["state"] != "active":
raise VMError("公网地址池没有开放新地址分配", 409)
state = public_network_pool_state(profile, pool)
for address in state["available"]:
if not insert_public_ip_claim(
profile, str(address), vm_id, pool["pool_key"]):
continue
return f"{address}/{state['guest_prefix']}"
raise VMError("公网地址池中没有可分配的 IP,请检查已用和保留地址", 409)
def claim_scheduled_public_ip(
profile: dict,
pool: dict,
public_ip_cidr: str,
vm_id: str,
) -> str:
"""Atomically claim the exact pool address reserved by PHP.
PHP owns the commercial reservation, while this engine claim closes
the race between concurrent workers and libvirt domain creation. The
host profile remains authoritative for prefix, gateway and DNS.
"""
if pool["state"] != "active":
raise VMError("公网地址池没有开放新地址分配", 409)
state = public_network_pool_state(profile, pool)
try:
requested = ipaddress.IPv4Interface(public_ip_cidr)
except (TypeError, ValueError) as exc:
raise VMError("预留公网 IP 格式无效", 422) from exc
address = requested.ip
if (requested.network.prefixlen != state["guest_prefix"]
or address not in state["_network"]
or address not in state["_usable"]):
raise VMError("预留公网 IP 不属于宿主机可分配地址池", 422)
if address in state["_reserved"]:
raise VMError("预留公网 IP 是宿主机保留地址,不能分配", 409)
if address in state["_allocated"]:
raise VMError("预留公网 IP 已被虚拟机或其他任务占用", 409)
if not insert_public_ip_claim(
profile, str(address), vm_id, pool["pool_key"]):
raise VMError("预留公网 IP 已被其他创建任务占用", 409)
return f"{address}/{state['guest_prefix']}"
def claim_explicit_public_ip(profile: dict, public_ip_cidr: str,
vm_id: str) -> None:
if data_store is None:
return
try:
address = str(ipaddress.IPv4Interface(public_ip_cidr).ip)
except (TypeError, ValueError):
# The libvirt create path returns the detailed input error.
return
if not insert_public_ip_claim(profile, address, vm_id):
raise VMError("该公网 IP 已被面板中的虚拟机或创建任务占用", 409)
def release_public_ip_claim(profile: dict, public_ip_cidr: str,
vm_id: str) -> None:
if data_store is None:
return
try:
address = str(ipaddress.IPv4Interface(public_ip_cidr).ip)
key = ip_claim_key(profile, address)
except (TypeError, ValueError, KeyError):
return
raw = data_store.get("ip_claim", key)
if raw is None:
return
try:
claim = json.loads(raw)
except (TypeError, ValueError):
LOG.error("Could not parse public IP claim for %s", key)
return
if isinstance(claim, dict) and claim.get("vm_uuid") == vm_id:
data_store.delete("ip_claim", key)
def release_vm_public_ip_claims(profile: dict, vm_id: str) -> None:
if data_store is None:
return
prefix = f"{host_scope(profile)}."
for key, raw in data_store.list("ip_claim").items():
if not key.startswith(prefix):
continue
try:
claim = json.loads(raw)
except (TypeError, ValueError):
LOG.error("Could not parse public IP claim for %s", key)
continue
if isinstance(claim, dict) and claim.get("vm_uuid") == vm_id:
data_store.delete("ip_claim", key)
def compare_and_set_public_pool_catalog(
expected: str | None, value: str,
) -> bool:
if data_store is None:
raise VMError("公网地址池目录存储不可用", 409)
compare = getattr(data_store, "compare_and_set", None)
if callable(compare):
return bool(compare(
public_pool_catalog_kind, public_pool_catalog_key,
expected, value,
))
# Database-free tests and the retired single-process browser mode use
# a simple store. Keep the same compare-and-set behavior locally.
with public_pool_catalog_lock:
current = data_store.get(
public_pool_catalog_kind, public_pool_catalog_key)
if current != expected:
return False
data_store.set(public_pool_catalog_kind, public_pool_catalog_key, value)
return True
def upsert_public_network_pool(
profile: dict, pool_key: str, body: PublicNetworkPoolBody,
) -> tuple[dict, bool]:
if not PUBLIC_POOL_KEY.fullmatch(pool_key):
raise VMError("公网地址池标识必须是 32 位小写十六进制", 422)
try:
requested = validate_public_network_pool({
"pool_key": pool_key,
**body.model_dump(),
}, profile["host"])
except RemoteHostError as exc:
raise VMError(str(exc), exc.status) from exc
# The data model stores a selector now so a later multi-line release
# does not need another migration. Network-sensitive VM operations still
# instantiate the host's default VMService, so accepting another bridge
# today would create guests that cannot be safely reinstalled or edited.
if requested["network_selector"] != profile["default_network"]:
raise VMError(
"当前阶段公网地址池必须使用宿主机默认网络;多线路网桥尚未启用", 409,
)
for _attempt in range(8):
raw, native = read_native_public_network_pools(profile)
existing = next(
(item for item in native if item["pool_key"] == pool_key), None)
if existing is not None:
if requested["revision"] < existing["revision"]:
raise VMError("公网地址池版本已更新,请刷新后重试", 409)
if requested["revision"] == existing["revision"]:
if requested == existing:
return existing, True
raise VMError("相同公网地址池版本对应了不同配置", 409)
candidate = [item for item in native if item["pool_key"] != pool_key]
candidate.append(requested)
try:
candidate = validate_public_network_pools(candidate, profile["host"])
except RemoteHostError as exc:
raise VMError(str(exc), exc.status) from exc
claimed_addresses = public_ip_claim_addresses(profile)
if existing is not None and (
public_pool_runtime_fields(existing)
!= public_pool_runtime_fields(requested)):
state = public_network_pool_state(
profile, existing, claimed_addresses=claimed_addresses)
if state["allocated_count"]:
raise VMError(
"公网地址池仍有虚拟机或任务占用,不能修改网段、网关、DNS或保留地址",
409,
)
if raw is None:
legacy = legacy_public_network_pool(profile)
if (legacy is not None
and public_pool_runtime_fields(legacy)
!= public_pool_runtime_fields(requested)):
state = public_network_pool_state(
profile, legacy, claimed_addresses=claimed_addresses)
if state["allocated_count"]:
raise VMError(
"旧公网地址池仍有虚拟机或任务占用;首个新地址池必须沿用原网络配置",
409,
)
encoded = json.dumps(
{"version": 1, "pools": candidate},
ensure_ascii=False, sort_keys=True, separators=(",", ":"),
)
if compare_and_set_public_pool_catalog(raw, encoded):
return requested, False
raise VMError("公网地址池配置正在被其他请求修改,请刷新后重试", 409)
def credential_key(profile: dict, vm_id: str) -> str:
try:
canonical_id = str(uuid.UUID(vm_id))
return f"{host_scope(profile)}.{canonical_id}"
except (TypeError, ValueError, AttributeError, KeyError) as exc:
raise VMError("虚拟机或宿主机身份信息无效", 400) from exc
def save_credential(profile: dict, vm_id: str, auth_mode: str,
generation: str, password: str | None = None) -> None:
key = credential_key(profile, vm_id)
if not isinstance(generation, str) or not generation:
raise ValueError("虚拟机系统配置代号无效")
value = json.dumps({"auth_mode": auth_mode, "password": password,
"generation": generation},
ensure_ascii=False, separators=(",", ":"))
if data_store is not None:
data_store.set("vm_credential", key, value)
else:
test_credentials[key] = value
def delete_credential(profile: dict, vm_id: str) -> None:
key = credential_key(profile, vm_id)
if data_store is not None:
data_store.delete("vm_credential", key)
else:
test_credentials.pop(key, None)
def rescue_state_key(profile: dict, vm_id: str) -> str:
return credential_key(profile, vm_id)
def rescue_xml_matches(first: str, second: str) -> bool:
try:
return ET.canonicalize(first) == ET.canonicalize(second)
except (TypeError, ET.ParseError):
return False
def load_rescue_state(profile: dict, vm_id: str) -> dict | None:
key = rescue_state_key(profile, vm_id)
raw = (data_store.get("vm_rescue", key) if data_store is not None
else test_rescue_states.get(key))
if raw is None:
return None
try:
record = json.loads(raw)
except (TypeError, ValueError) as exc:
raise DatabaseError("虚拟机救援记录损坏") from exc
if (not isinstance(record, dict)
or not isinstance(record.get("original_xml"), str)
or not record["original_xml"].startswith("<domain")
or not isinstance(record.get("template_id"), str)
or record.get("volume_name") != rescue_volume_name(record["template_id"])
or record.get("status") not in {
"queued", "running", "staged", "active", "failed", "exiting",
}):
raise DatabaseError("虚拟机救援记录损坏")
return record
def save_rescue_state(profile: dict, vm_id: str, record: dict) -> None:
key = rescue_state_key(profile, vm_id)
value = json.dumps(record, ensure_ascii=False, separators=(",", ":"))
if data_store is not None:
data_store.set("vm_rescue", key, value)
else:
test_rescue_states[key] = value
def delete_rescue_state(profile: dict, vm_id: str) -> None:
key = rescue_state_key(profile, vm_id)
if data_store is not None:
data_store.delete("vm_rescue", key)
else:
test_rescue_states.pop(key, None)
def current_rescue_state(profile: dict, vm_id: str,
service: VMService | None = None) -> dict | None:
"""Clear a stale record only when the original inactive XML is intact.
An interrupted worker may have stopped before changing the VM. The
database job is marked failed on restart, but its recovery record must
not indefinitely block normal operations in that safe case.
Callers hold rescue_lock so this cannot race a live rescue worker.
"""
record = load_rescue_state(profile, vm_id)
if record is None or (host_scope(profile), vm_id) in pending_rescues:
return record
service = service or new_service(profile)
current_vm = service.get_vm(vm_id)
if current_vm.get("rescue_mode") or current_vm.get("rescue_staged"):
return record
try:
original_still_defined = rescue_xml_matches(
service.prepare_rescue_vm(vm_id, record["volume_name"]),
record["original_xml"],
)
except VMError:
return record
if original_still_defined:
delete_rescue_state(profile, vm_id)
return None
return record
def parse_credential(raw: str | None) -> dict | None:
if raw is None:
return None
try:
value = json.loads(raw)
except (TypeError, ValueError) as exc:
raise DatabaseError("虚拟机登录凭据记录损坏") from exc
if (not isinstance(value, dict)
or value.get("auth_mode") not in ("password", "ssh_key", "ssh_key_password")
or (value["auth_mode"] in ("password", "ssh_key_password")
and (not isinstance(value.get("password"), str)
or not value["password"]))
or not isinstance(value.get("generation"), str)
or not value["generation"]
or (value["auth_mode"] == "ssh_key" and value.get("password") is not None)):
raise DatabaseError("虚拟机登录凭据记录损坏")
return value
def get_credential(profile: dict, vm_id: str) -> dict | None:
key = credential_key(profile, vm_id)
raw = (data_store.get("vm_credential", key) if data_store is not None
else test_credentials.get(key))
return parse_credential(raw)
def firewall_record_key(profile: dict, vm_id: str) -> str:
return credential_key(profile, vm_id)
def load_firewall_record(profile: dict, vm_id: str) -> dict | None:
key = firewall_record_key(profile, vm_id)
raw = (data_store.get("vm_firewall", key) if data_store is not None
else test_firewalls.get(key))
if raw is None:
return None
try:
record = json.loads(raw)
policy = FirewallPolicyBody.model_validate({
"enabled": record["enabled"],
"revision": record["revision"],
"policy_digest": record["policy_digest"],
"rules": record["rules"],
})
except (KeyError, TypeError, ValueError) as exc:
raise DatabaseError("虚拟机防火墙记录损坏") from exc
record_version = record.get("version") if isinstance(record, dict) else None
if (not isinstance(record, dict) or record_version not in {1, 2}
or record.get("vm_id") != str(uuid.UUID(vm_id))
or not re.fullmatch(r"[0-9a-f]{64}",
record.get("policy_digest", ""))
or not re.fullmatch(r"[0-9a-f]{64}",
record.get("filter_digest", ""))
or isinstance(record.get("applied_at"), bool)
or not isinstance(record.get("applied_at"), int)):
raise DatabaseError("虚拟机防火墙记录损坏")
filter_version = record.get("filter_version", 1)
if (isinstance(filter_version, bool)
or not isinstance(filter_version, int)
or filter_version not in SUPPORTED_FIREWALL_FILTER_COMPILER_VERSIONS):
raise DatabaseError("虚拟机防火墙记录损坏")
try:
platform_policy = (
normalize_platform_policy_snapshot(
record["platform_policy_snapshot"])
if record_version == 2
else default_platform_policy_snapshot()
)
except (KeyError, TypeError, ValueError) as exc:
raise DatabaseError("虚拟机防火墙记录损坏") from exc
if (record_version == 2
and (not isinstance(record.get("platform_policy_digest"), str)
or not hmac.compare_digest(
record["platform_policy_digest"],
platform_policy["digest"],
))):
raise DatabaseError("虚拟机防火墙记录损坏")
if record_version == 1 and filter_version >= 4:
raise DatabaseError("虚拟机防火墙记录损坏")
normalized_policy = policy.model_dump()
if not hmac.compare_digest(
normalized_policy["policy_digest"],
firewall_policy_digest(
normalized_policy["enabled"],
normalized_policy["rules"],
)):
raise DatabaseError("虚拟机防火墙记录损坏")
expected_filter_digest = build_firewall_filter_xml(
vm_id,
normalized_policy["enabled"],
normalized_policy["rules"],
compiler_version=filter_version,
platform_policy_snapshot=platform_policy,
)[3]
if not hmac.compare_digest(
record["filter_digest"], expected_filter_digest):
raise DatabaseError("虚拟机防火墙记录损坏")
return {
**record,
**normalized_policy,
"version": record_version,
"filter_version": filter_version,
"platform_policy_snapshot": platform_policy,
"platform_policy_digest": platform_policy["digest"],
}
def save_firewall_record(profile: dict, vm_id: str, record: dict) -> None:
key = firewall_record_key(profile, vm_id)
encoded = json.dumps(
record, ensure_ascii=False, separators=(",", ":"))
if data_store is not None:
data_store.set("vm_firewall", key, encoded)
else:
test_firewalls[key] = encoded
def delete_firewall_record(profile: dict, vm_id: str) -> None:
key = firewall_record_key(profile, vm_id)
if data_store is not None:
data_store.delete("vm_firewall", key)
else:
test_firewalls.pop(key, None)
def is_default_platform_firewall(
vm_id: str, observed: dict,
) -> bool:
"""Recognize the exact record-free wrapper installed at VM creation."""
observed_digest = observed.get("filter_digest")
platform_policy = observed.get("platform_policy_snapshot")
expected_digest = build_firewall_filter_xml(
vm_id, False, [],
platform_policy_snapshot=platform_policy)[3]
return bool(
observed.get("filter_mode") == "managed"
and observed.get("sync_state") == "active"
and observed.get("coherent") is True
and isinstance(observed_digest, str)
and hmac.compare_digest(observed_digest, expected_digest)
)
def upgrade_firewall_filter_if_needed(
profile: dict, vm_id: str, service: VMService,
) -> bool:
"""Verify the host filter and recompile trusted legacy policy only."""
record = load_firewall_record(profile, vm_id)
observed = service.get_firewall_policy(vm_id)
observed_filter_digest = observed.get("filter_digest")
observed_mode = observed.get("filter_mode")
try:
observed_platform_policy = normalize_platform_policy_snapshot(
observed.get("platform_policy_snapshot")
or default_platform_policy_snapshot())
except ValueError as exc:
raise VMError(
"虚拟机平台网络策略证据无效", 409,
error_code="firewall_drift_detected",
) from exc
observed_active = (
observed.get("sync_state") == "active"
and observed.get("coherent") is True)
if record is None:
if (observed_active and observed_mode == "baseline"
and observed_filter_digest is None):
return False
if is_default_platform_firewall(vm_id, observed):
return False
raise VMError(
"防火墙实际状态与已保存版本不一致,请先核查漂移",
409,
error_code="firewall_drift_detected",
)
observation_active = (
observed_active
and observed_mode == "managed"
and isinstance(observed_filter_digest, str))
if not hmac.compare_digest(
record["platform_policy_digest"],
observed_platform_policy["digest"]):
raise VMError(
"虚拟机平台网络策略与引擎记录不一致",
409,
error_code="firewall_drift_detected",
)
if record["filter_version"] >= FIREWALL_FILTER_COMPILER_VERSION:
if (observation_active and hmac.compare_digest(
observed_filter_digest, record["filter_digest"])):
return False
raise VMError(
"防火墙实际状态与已保存版本不一致,请先核查漂移",
409,
error_code="firewall_drift_detected",
)
expected_filter_digest = build_firewall_filter_xml(
vm_id, record["enabled"], record["rules"],
platform_policy_snapshot=observed_platform_policy)[3]
if not observation_active:
raise VMError(
"防火墙实际状态与已保存版本不一致,请先核查漂移",
409,
error_code="firewall_drift_detected",
)
if hmac.compare_digest(
observed_filter_digest, expected_filter_digest):
upgraded = {
**record,
"version": 2,
"filter_version": FIREWALL_FILTER_COMPILER_VERSION,
"filter_digest": expected_filter_digest,
"platform_policy_snapshot": observed_platform_policy,
"platform_policy_digest": observed_platform_policy["digest"],
"applied_at": int(time.time()),
}
try:
save_firewall_record(profile, vm_id, upgraded)
except Exception as exc:
raise VMError(
"防火墙已升级但本地证据未确认,已停止修改网络",
503,
error_code="firewall_result_uncertain",
) from exc
return True
if not hmac.compare_digest(
observed_filter_digest, record["filter_digest"]):
raise VMError(
"防火墙实际状态与已保存版本不一致,请先核查漂移",
409,
error_code="firewall_drift_detected",
)
result = service.apply_firewall_policy(
vm_id, enabled=record["enabled"], rules=record["rules"])
result_policy_digest = result.get("policy_digest")
result_filter_digest = result.get("filter_digest")
result_platform_digest = result.get("platform_policy_digest")
if (not isinstance(result_policy_digest, str)
or not isinstance(result_filter_digest, str)
or not hmac.compare_digest(
result_policy_digest, record["policy_digest"])
or not hmac.compare_digest(
result_filter_digest, expected_filter_digest)
or not isinstance(result_platform_digest, str)
or not hmac.compare_digest(
result_platform_digest,
observed_platform_policy["digest"])):
raise VMError(
"防火墙编译器升级结果不确定,已停止修改网络",
503,
error_code="firewall_result_uncertain",
)
upgraded = {
**record,
"version": 2,
"filter_version": FIREWALL_FILTER_COMPILER_VERSION,
"filter_digest": expected_filter_digest,
"platform_policy_snapshot": observed_platform_policy,
"platform_policy_digest": observed_platform_policy["digest"],
"applied_at": int(time.time()),
}
try:
save_firewall_record(profile, vm_id, upgraded)
except Exception as exc:
raise VMError(
"防火墙已升级但本地证据未确认,已停止修改网络",
503,
error_code="firewall_result_uncertain",
) from exc
return True
def snapshot_record_key(vm_id: str, snapshot_key: str) -> str:
try:
canonical_id = str(uuid.UUID(vm_id))
except (TypeError, ValueError, AttributeError) as exc:
raise VMError("虚拟机 UUID 无效", 400) from exc
if not isinstance(snapshot_key, str) or not SNAPSHOT_KEY_RE.fullmatch(snapshot_key):
raise VMError("快照标识无效", 422)
return f"{canonical_id}:{snapshot_key}"
def parse_snapshot_record(key: str, raw: str) -> dict:
try:
record = json.loads(raw)
vm_id, snapshot_key = key.split(":", 1)
canonical_id = str(uuid.UUID(vm_id))
except (TypeError, ValueError, AttributeError) as exc:
raise DatabaseError("虚拟机快照记录损坏") from exc
allowed_states = {
"creating", "ready", "restoring", "deleting", "failed", "uncertain",
}
if (not isinstance(record, dict)
or record.get("vm_id") != canonical_id
or record.get("snapshot_key") != snapshot_key
or not SNAPSHOT_KEY_RE.fullmatch(snapshot_key)
or record.get("volume_name") != snapshot_volume_name(
canonical_id, snapshot_key)
or record.get("state") not in allowed_states
or record.get("consistency") not in {"filesystem", "crash"}
or not isinstance(record.get("name"), str)
or not 1 <= len(record["name"]) <= 64
or not isinstance(record.get("created_at"), int)
or record["created_at"] < 1):
raise DatabaseError("虚拟机快照记录损坏")
size_bytes = record.get("size_bytes")
if (size_bytes is not None
and (isinstance(size_bytes, bool) or not isinstance(size_bytes, int)
or size_bytes < 0)):
raise DatabaseError("虚拟机快照记录损坏")
return record
def list_snapshot_records(vm_id: str) -> list[dict]:
canonical_id = str(uuid.UUID(vm_id))
prefix = canonical_id + ":"
rows = (data_store.list("vm_snapshot") if data_store is not None
else test_snapshots.copy())
records = [
parse_snapshot_record(key, raw)
for key, raw in rows.items()
if key.startswith(prefix)
]
return sorted(records, key=lambda row: (row["created_at"], row["snapshot_key"]),
reverse=True)
def has_retained_snapshots() -> bool:
rows = (data_store.list("vm_snapshot") if data_store is not None
else test_snapshots.copy())
return any(
parse_snapshot_record(key, raw)["state"] in {
"ready", "creating", "restoring", "deleting", "uncertain",
}
for key, raw in rows.items()
)
def get_snapshot_record(vm_id: str, snapshot_key: str) -> dict:
key = snapshot_record_key(vm_id, snapshot_key)
raw = (data_store.get("vm_snapshot", key) if data_store is not None
else test_snapshots.get(key))
if raw is None:
raise VMError("快照不存在", 404)
return parse_snapshot_record(key, raw)
def save_snapshot_record(record: dict) -> None:
key = snapshot_record_key(record["vm_id"], record["snapshot_key"])
value = json.dumps(record, ensure_ascii=False, separators=(",", ":"))
if data_store is not None:
data_store.set("vm_snapshot", key, value)
else:
test_snapshots[key] = value
def insert_snapshot_record(record: dict) -> bool:
key = snapshot_record_key(record["vm_id"], record["snapshot_key"])
value = json.dumps(record, ensure_ascii=False, separators=(",", ":"))
if data_store is not None:
return bool(data_store.insert("vm_snapshot", key, value))
if key in test_snapshots:
return False
test_snapshots[key] = value
return True
def delete_snapshot_record(vm_id: str, snapshot_key: str) -> None:
key = snapshot_record_key(vm_id, snapshot_key)
if data_store is not None:
data_store.delete("vm_snapshot", key)
else:
test_snapshots.pop(key, None)
def public_snapshot_record(record: dict) -> dict:
return {
key: record[key]
for key in (
"snapshot_key", "name", "state", "consistency", "created_at",
"completed_at", "size_bytes", "last_restored_at",
)
if key in record
}
def forbid_pending_snapshot(
profile: dict, vm_id: str, label: str, *, allow_uncertain: bool = False,
ignore_suspend_barrier: bool = False,
) -> None:
key = (host_scope(profile), str(uuid.UUID(vm_id)))
with snapshot_lock:
if key in snapshot_suspend_barriers and not ignore_suspend_barrier:
raise VMError(
f"虚拟机正在执行到期暂停策略,暂时不能{label}", 409)
if key in pending_snapshot_jobs:
raise VMError(f"虚拟机正在执行快照任务,暂时不能{label}", 409)
blocked_states = {"creating", "restoring", "deleting"}
if not allow_uncertain:
blocked_states.add("uncertain")
if any(row["state"] in blocked_states
for row in list_snapshot_records(key[1])):
raise VMError(
"虚拟机存在进行中或结果不确定的快照任务,必须先完成安全核对",
409,
)
def forbid_snapshot_suspend_barrier(
profile: dict, vm_id: str, label: str,
) -> None:
key = (host_scope(profile), str(uuid.UUID(vm_id)))
with snapshot_lock:
if key in snapshot_suspend_barriers:
raise VMError(
f"虚拟机正在执行到期暂停策略,暂时不能{label}", 409)
def forbid_retained_snapshot(profile: dict, vm_id: str, label: str) -> None:
"""Prevent replacing vda while a local recovery point is retained.
Instance settings may safely rotate the current cloud-init seed: a
later restore intentionally applies that current seed to the restored
disk. Reinstall is different because it replaces the disk baseline
that every retained local snapshot belongs to.
"""
canonical_id = str(uuid.UUID(vm_id))
forbid_pending_snapshot(profile, canonical_id, label)
with snapshot_lock:
if any(row["state"] != "failed"
for row in list_snapshot_records(canonical_id)):
raise VMError(
f"虚拟机仍保留本地快照,请删除全部快照后再{label}",
409,
)
def rollback_cancelled_snapshot_catalog(job: dict) -> None:
"""Roll back a snapshot row after its Future was cancelled queued.
The durable ``cancelled_before_start`` marker proves that the executor
accepted ``Future.cancel()`` before this helper runs. Consequently no
libvirt mutation can have started, and the admission-state row can be
restored without inspecting or changing the host.
"""
vm_id = job.get("vm_id")
snapshot_key = job.get("snapshot_key")
action = job.get("action")
if (not isinstance(vm_id, str) or not isinstance(snapshot_key, str)
or action not in {"create", "restore", "delete"}):
raise DatabaseError("取消的快照任务身份无效")
try:
record = get_snapshot_record(vm_id, snapshot_key)
except VMError as exc:
if action == "create" and exc.status == 404:
return
raise DatabaseError("取消的快照登记记录不存在") from exc
expected_state = {
"create": "creating",
"restore": "restoring",
"delete": "deleting",
}[action]
if action == "create":
if record["state"] != expected_state:
raise DatabaseError("取消的快照创建登记状态已变化")
delete_snapshot_record(vm_id, snapshot_key)
try:
get_snapshot_record(vm_id, snapshot_key)
except VMError as exc:
if exc.status == 404:
return
raise
raise DatabaseError("取消的快照创建登记未能删除")
if record["state"] not in {expected_state, "ready"}:
raise DatabaseError("取消的快照登记状态已变化")
if record["state"] != "ready":
record["state"] = "ready"
save_snapshot_record(record)
if get_snapshot_record(vm_id, snapshot_key)["state"] != "ready":
raise DatabaseError("取消的快照登记未能恢复")
def finish_cancelled_snapshot_job(job_id: str, job: dict) -> dict:
"""Finish the second phase of a durable pre-start cancellation."""
if (job.get("cancelled_before_start") is not True
or job.get("cancel_reason") != "network_suspend"):
raise DatabaseError("快照取消凭据无效")
rollback_cancelled_snapshot_catalog(job)
terminal = dict(job)
terminal.update(
status="failed",
phase="排队快照已由到期网络暂停策略取消",
error="快照任务在开始前已取消,未调用虚拟化操作",
cancelled_at=int(time.time()),
finished_at=int(time.time()),
)
terminal.pop("result", None)
terminal.pop("error_code", None)
# The terminal job is persisted only after catalog rollback. If this
# write fails, the durable phase-one tombstone remains queued and both
# startup recovery and operation polling can retry this exact step.
save_job("snapshot_job", job_id, terminal)
snapshot_jobs[job_id] = terminal
pending_snapshot_jobs.discard((terminal["scope"], terminal["vm_id"]))
snapshot_futures.pop(job_id, None)
return dict(terminal)
def cancel_queued_snapshot_for_network_suspend(
profile: dict, vm_id: str,
) -> None:
"""Cancel exactly one queued same-VM job without touching libvirt."""
scope = host_scope(profile)
matches = [
(job_id, job)
for job_id, job in snapshot_jobs.items()
if (job.get("scope") == scope and job.get("vm_id") == vm_id
and job.get("status") in {"queued", "running"})
]
if len(matches) > 1:
raise VMError("虚拟机存在多个快照任务,拒绝自动取消", 409)
if matches:
job_id, job = matches[0]
if job.get("status") != "queued":
raise VMError("虚拟机快照任务已经开始,暂时不能暂停网络", 409)
if job.get("cancelled_before_start") is not True:
future = snapshot_futures.get(job_id)
if future is None or not future.cancel():
# A Future becomes RUNNING before its callable acquires
# snapshot_lock. Never infer safety from the still-queued
# job row when the executor has already claimed it.
raise VMError(
"虚拟机快照任务已经由执行器领取,暂时不能暂停网络", 409)
tombstone = dict(job)
tombstone.update(
cancelled_before_start=True,
cancel_reason="network_suspend",
cancel_requested_at=int(time.time()),
phase="正在取消尚未开始的排队快照",
)
# Future.cancel() is already definitive process-local proof.
# Publish the tombstone in memory first so a transient phase-
# one database failure remains retryable by operation polling
# without ever letting the cancelled callable reach libvirt.
snapshot_jobs[job_id] = tombstone
job = tombstone
# Persist proof of Future.cancel() before changing the catalog.
try:
save_job("snapshot_job", job_id, tombstone)
except Exception as exc:
LOG.exception(
"Could not persist queued snapshot cancellation: job=%s uuid=%s",
job_id, vm_id,
)
raise VMError(
"排队快照已停止,但取消凭据尚未持久化,暂时不能暂停网络",
503,
) from exc
try:
finish_cancelled_snapshot_job(job_id, job)
except Exception as exc:
LOG.exception(
"Could not finish queued snapshot cancellation: job=%s uuid=%s",
job_id, vm_id,
)
raise VMError(
"排队快照已停止,但登记回滚尚未完成,暂时不能暂停网络",
503,
) from exc
if (scope, vm_id) in pending_snapshot_jobs:
raise VMError("虚拟机快照任务状态尚未收敛,暂时不能暂停网络", 409)
def decorate_vm(vm: dict, credential: dict | None, profile: dict) -> dict:
cloud = bool(vm.get("managed") and vm.get("os_source") == "cloud_template")
current_credential = (credential if cloud and credential
and credential["generation"] == vm.get("credential_generation")
else None)
return {
**{key: value for key, value in vm.items() if key != "credential_generation"},
"region": profile.get("region", ""),
"login_name": "root" if cloud else None,
"auth_mode": current_credential["auth_mode"] if current_credential else "unknown",
"password_available": bool(current_credential and
current_credential["auth_mode"] in (
"password", "ssh_key_password")),
}
def save_job(kind: str, job_id: str, job: dict) -> None:
delegated = TRUSTED_ACTOR.get() if internal_mode else None
if delegated and delegated.get("operation_id"):
job.setdefault("operation_id", delegated["operation_id"])
if data_store is not None:
data_store.set(kind, job_id, json.dumps(
job, ensure_ascii=False, separators=(",", ":")))
def delete_job(kind: str, job_id: str) -> None:
if data_store is not None:
data_store.delete(kind, job_id)
def save_user_iso_job(job_id: str, job: dict) -> None:
if shared_user_iso_runtime is not None:
shared_user_iso_runtime.save_job(job_id, job)
else:
save_job("user_iso_import_job", job_id, job)
def delete_user_iso_job(job_id: str) -> None:
if shared_user_iso_runtime is not None:
shared_user_iso_runtime.delete_job(job_id)
else:
delete_job("user_iso_import_job", job_id)
def restore_jobs() -> None:
"""Restore interrupted jobs without pretending mutations resumed."""
if data_store is None:
return
pending_reinstalls.clear()
destinations = [
("reinstall_job", reinstall_jobs),
("rescue_import_job", rescue_import_jobs),
("rescue_job", rescue_jobs),
("snapshot_job", snapshot_jobs),
]
if shared_cloud_template_runtime is None:
destinations.append(("import_job", import_jobs))
if shared_user_iso_runtime is None:
destinations.append(("user_iso_import_job", user_iso_import_jobs))
for kind, destination in destinations:
for job_id, raw in data_store.list(kind).items():
try:
job = json.loads(raw)
except (TypeError, ValueError) as exc:
raise DatabaseError("数据库任务记录损坏") from exc
if not isinstance(job, dict):
raise DatabaseError("数据库任务记录损坏")
if (kind == "reinstall_job"
and job.get("status") in {
"queued", "running", "succeeded", "completed", "done",
"failed", "cancelled", "canceled", "uncertain",
}
and reinstall_post_commit(job)):
# Releases before the cleanup boundary could persist a
# warning/phase without the explicit marker. Upgrade any
# such durable shape on startup so a direct engine poll
# cannot present it as a completed or retryable reinstall.
cleanup_pending = reinstall_cleanup_pending(job)
job["status"] = "uncertain"
job["phase"] = (
"重装完成,但旧系统盘清理待核对"
if cleanup_pending else "重装结果待核对"
)
job["error_code"] = (
REINSTALL_CLEANUP_PENDING_ERROR_CODE
if cleanup_pending else REINSTALL_RESULT_UNCERTAIN_ERROR_CODE
)
job["error"] = (
"重装已完成,但旧系统盘清理待核对"
if cleanup_pending else "重装结果待核对"
)
job["reinstall_committed"] = True
if cleanup_pending:
job["cleanup_pending"] = True
job.pop("finished_at", None)
save_job(kind, job_id, job)
if (kind == "snapshot_job"
and job.get("status") == "queued"
and job.get("cancelled_before_start") is True):
# Phase one proves the previous process cancelled the
# Future before its callable began. Finish only the local
# catalog rollback; never reinterpret this as an
# interrupted libvirt mutation.
try:
job = finish_cancelled_snapshot_job(job_id, job)
except Exception:
LOG.exception(
"Could not finish durable queued snapshot cancellation: %s",
job_id,
)
destination[job_id] = job
continue
if (kind == "snapshot_job"
and job.get("action") == "create"
and job.get("consistency") == "filesystem"
and job.get("guest_freeze_state") == "attempted"):
try:
profile = manager.current()
if profile is None or profile.get("key_locked"):
raise VMError(
"宿主机未配置或密钥已锁定,无法恢复访客文件系统")
recovery_service = (
service_factory(profile)
if service_factory is not None
else VMService(
profile["libvirt_uri"],
profile["storage_pool"],
profile["iso_pool"],
profile["default_network"],
rescue_iso_pool=profile.get(
"rescue_iso_pool", ""),
)
)
recovery_service.thaw_guest_filesystems(job["vm_id"])
except Exception:
job["freeze_recovery"] = "failed"
LOG.exception(
"Could not recover interrupted guest freeze: "
"job=%s uuid=%s",
job_id, job.get("vm_id"),
)
else:
job["guest_freeze_state"] = "recovered"
job["freeze_recovery"] = "confirmed"
job["freeze_recovered_at"] = int(time.time())
save_job(kind, job_id, job)
if (kind == "reinstall_job"
and job.get("status") == "running"):
# The replacement domain was durably observed as started,
# or may have crossed that boundary before the process
# ended. Preserve an uncertain result in either case;
# a durable post-commit phase or marker authorizes PHP
# to copy the replacement credential/spec into its cache.
committed = reinstall_post_commit(job)
cleanup_pending = reinstall_cleanup_pending(job)
job["status"] = "uncertain"
job["phase"] = (
"重装完成,但旧系统盘清理待核对"
if cleanup_pending else "重装结果待核对"
)
# A stale terminal timestamp must not make the recovered
# operation look settled while its physical result is
# still awaiting inspection.
job.pop("finished_at", None)
if committed:
job["error_code"] = (
REINSTALL_CLEANUP_PENDING_ERROR_CODE
if cleanup_pending else REINSTALL_RESULT_UNCERTAIN_ERROR_CODE
)
job["error"] = (
"重装已完成,但旧系统盘清理待核对"
if cleanup_pending
else "重装结果待核对;面板重启后未能确认旧存储卷清理状态"
)
job["reinstall_committed"] = True
if cleanup_pending:
job["cleanup_pending"] = True
else:
for key in ("error_code", "cleanup_pending", "warning",
"generated_password", "reinstall_committed"):
job.pop(key, None)
job["error"] = "重装结果待核对;面板重启后无法确认新系统是否已启动"
save_job(kind, job_id, job)
elif job.get("status") in {"queued", "running"}:
if kind == "snapshot_job":
# A push backup or disk switch can continue or commit
# after the HTTP process disappears. Never convert that
# ambiguity into a retryable terminal failure.
job["status"] = "uncertain"
job["phase"] = "引擎重启,快照结果待核查"
job["error"] = "引擎在快照任务期间重启;必须核查存储卷和虚拟机定义,不能重复提交"
try:
record = get_snapshot_record(
job["vm_id"], job["snapshot_key"])
record["state"] = "uncertain"
save_snapshot_record(record)
except (KeyError, VMError, DatabaseError, ValueError):
LOG.exception(
"Could not mark interrupted snapshot record uncertain: %s",
job_id)
else:
job["status"] = "failed"
job["phase"] = "面板重启,任务已中断"
job["error"] = "面板在任务执行期间重启;请检查宿主机实际状态,勿直接重复提交"
for key in ("generated_password", "reinstall_committed",
"cleanup_pending", "error_code", "warning"):
job.pop(key, None)
save_job(kind, job_id, job)
elif kind == "snapshot_job" and job.get("status") == "uncertain":
# A previous process may have persisted the job outcome
# before it could mirror that state into the catalog.
# The durable operation identity exists, so preserve the
# row for evidence-only reconciliation rather than
# treating it as the pre-queue crash window below.
try:
record = get_snapshot_record(
job["vm_id"], job["snapshot_key"])
if record["state"] in {
"creating", "restoring", "deleting"}:
record["state"] = "uncertain"
save_snapshot_record(record)
except (KeyError, VMError, DatabaseError, ValueError):
LOG.exception(
"Could not align uncertain snapshot catalog row: %s",
job_id,
)
if (kind == "reinstall_job"
and job.get("status") in {"queued", "running", "uncertain"}
and isinstance(job.get("scope"), str)
and isinstance(job.get("vm_id"), str)):
# Rebuild the in-process conflict barrier after a panel
# restart. Other VM mutation endpoints consult this set
# before touching the same guest.
pending_reinstalls.add((job["scope"], job["vm_id"]))
destination[job_id] = job
# The registry row is reserved before the durable job row. A process
# loss in that narrow window otherwise leaves a permanent
# creating/restoring/deleting record with no operation to reconcile.
# No libvirt mutation is invoked until queue_snapshot_job has durably
# saved its job row. Therefore a transitional catalog row with no
# corresponding durable job proves the process died before the worker
# could cross a physical mutation boundary.
durable_snapshot_records = {
(job.get("vm_id"), job.get("snapshot_key"))
for job in snapshot_jobs.values()
}
for key, raw in data_store.list("vm_snapshot").items():
record = parse_snapshot_record(key, raw)
identity = (record["vm_id"], record["snapshot_key"])
if (record["state"] in {"creating", "restoring", "deleting"}
and identity not in durable_snapshot_records):
if record["state"] == "creating":
delete_snapshot_record(*identity)
else:
record["state"] = "ready"
record["interrupted_at"] = int(time.time())
save_snapshot_record(record)
LOG.warning(
"Rolled back snapshot registry row created before durable job: "
"uuid=%s snapshot=%s",
*identity,
)
restore_jobs()
sessions = DatabaseSessionStore(data_store) if data_store is not None else SessionStore()
login_limiter = LoginLimiter()
registration_limiter = LoginLimiter()
code_send_limiter = LoginLimiter()
setup_limiter = LoginLimiter()
unlock_limiter = LoginLimiter()
def metric_profile():
if manager is None:
return None
background = getattr(manager, "current_for_background", None)
return background() if callable(background) else manager.current()
metric_collector = MetricsCollector(
lambda: metric_store if metric_store is not None else data_store,
metric_profile,
lambda profile: new_service(profile),
owner_getter=lambda scope, vm_id: state.users.owner_of_vm(scope, vm_id),
) if database_mode else None
@asynccontextmanager
async def lifespan(_app: FastAPI):
if metric_collector is not None:
metric_collector.start()
try:
yield
finally:
if metric_collector is not None:
metric_collector.close()
reinstall_executor.shutdown(wait=True, cancel_futures=False)
snapshot_executor.shutdown(wait=True, cancel_futures=False)
import_executor.shutdown(wait=True, cancel_futures=False)
rescue_executor.shutdown(wait=True, cancel_futures=False)
rescue_import_executor.shutdown(wait=True, cancel_futures=False)
if shared_user_iso_runtime is None:
user_iso_import_executor.shutdown(wait=True, cancel_futures=False)
close_manager = getattr(manager, "close", None) if manager is not None else None
if callable(close_manager):
close_manager()
app = FastAPI(title="Virsh Panel API", docs_url=None, redoc_url=None, lifespan=lifespan)
app.mount("/static", StaticFiles(directory=str(settings.static_dir)), name="static")
app.mount(
"/vendor/novnc",
StaticFiles(directory=str(settings.static_dir / "vendor" / "novnc")),
name="novnc",
)
@app.middleware("http")
async def security_headers(request: Request, call_next):
if not internal_mode and request.url.scheme != "https" and not (
request.url.path == "/api/v1/health"
and request.url.scheme == "http"
and _plain_loopback_allowed(request.scope, request.headers.get("host"))
):
return JSONResponse(status_code=400, content={"error": "请使用 HTTPS 访问面板"})
response = await call_next(request)
response.headers["X-Content-Type-Options"] = "nosniff"
response.headers["Referrer-Policy"] = "no-referrer"
response.headers["X-Frame-Options"] = "DENY"
response.headers["Content-Security-Policy"] = (
"default-src 'self'; script-src 'self'; style-src 'self'; "
"img-src 'self' data:; connect-src 'self' wss:; "
"object-src 'none'; frame-ancestors 'none'; base-uri 'self'"
)
if (request.url.path == "/" or request.url.path.startswith("/api/")
or request.url.path in ("/static/app.js", "/static/styles.css")):
response.headers["Cache-Control"] = "no-store"
return response
@app.exception_handler(VMError)
async def handle_vm_error(_request: Request, exc: VMError):
content = {"error": str(exc)}
if exc.error_code:
content["error_code"] = exc.error_code
return JSONResponse(status_code=exc.status, content=content)
@app.exception_handler(PublicIpClaimCleanupPending)
async def handle_public_ip_claim_cleanup(
_request: Request, exc: PublicIpClaimCleanupPending,
):
return JSONResponse(status_code=exc.status, content={
"error": str(exc),
"error_code": "public_ip_claim_cleanup_pending",
})
@app.exception_handler(RemoteHostError)
async def handle_remote_error(_request: Request, exc: RemoteHostError):
content = {"error": str(exc), "error_code": exc.error_code}
if exc.status == 423:
content["error_code"] = "key_locked"
return JSONResponse(status_code=exc.status, content=content)
@app.exception_handler(SetupError)
async def handle_setup_error(_request: Request, exc: SetupError):
return JSONResponse(status_code=exc.status, content={"error": str(exc)})
@app.exception_handler(DatabaseSetupError)
async def handle_database_setup_error(_request: Request, exc: DatabaseSetupError):
return JSONResponse(status_code=exc.status, content={"error": str(exc)})
@app.exception_handler(DatabaseError)
async def handle_database_error(_request: Request, exc: DatabaseError):
return JSONResponse(status_code=exc.status, content={"error": str(exc)})
@app.exception_handler(UpdateError)
async def handle_update_error(_request: Request, exc: UpdateError):
return JSONResponse(status_code=exc.status, content={"error": str(exc)})
@app.exception_handler(TemplateError)
async def handle_template_error(_request: Request, exc: TemplateError):
return JSONResponse(status_code=exc.status, content={"error": str(exc)})
@app.exception_handler(UserError)
async def handle_user_error(_request: Request, exc: UserError):
return JSONResponse(status_code=exc.status, content={"error": str(exc)})
@app.exception_handler(SMTPError)
async def handle_smtp_error(_request: Request, exc: SMTPError):
return JSONResponse(status_code=exc.status, content={"error": str(exc)})
@app.exception_handler(SSHKeyError)
async def handle_ssh_key_error(_request: Request, exc: SSHKeyError):
return JSONResponse(status_code=exc.status, content={"error": str(exc)})
def account_for_session(session: Session) -> dict:
if internal_mode:
return actor_account(session)
if database_mode:
if state is None:
raise HTTPException(status_code=401, detail="请先登录")
user_id = getattr(session, "user_id", None)
if isinstance(user_id, bool) or not isinstance(user_id, int) or user_id < 1:
raise HTTPException(status_code=401, detail="账号不可用,请重新登录")
account = state.users.get_by_id(user_id)
if (account is None or account["username"] != session.username
or not account["email_verified"]):
raise HTTPException(status_code=401, detail="账号不可用,请重新登录")
return account
# The injected JSON state is retained solely for the isolated HTTP
# test harness; production authorization always uses the users table.
return {"id": 0, "username": session.username, "admin": True,
"approved": True, "email_verified": True}
def host_ready() -> bool:
if manager is None:
return False
profile = manager.current()
return profile is not None and not profile.get("key_locked", False)
def require_session(request: Request) -> Session:
if internal_mode:
return actor_session()
session = sessions.get(request.cookies.get(COOKIE_NAME))
if session is None:
raise HTTPException(status_code=401, detail="请先登录")
account_for_session(session)
return session
def require_write(request: Request) -> Session:
session = require_session(request)
if internal_mode:
if not actor()["can_manage"]:
raise HTTPException(status_code=403, detail="当前账户不能执行虚拟机操作")
return session
csrf = request.headers.get("X-CSRF-Token", "")
if not hmac.compare_digest(csrf, session.csrf_token):
raise HTTPException(status_code=403, detail="请求验证失败")
return session
def require_admin(request: Request) -> Session:
session = require_session(request)
if not account_for_session(session)["admin"]:
raise HTTPException(status_code=403, detail="仅管理员可以执行此操作")
return session
def require_admin_write(request: Request) -> Session:
session = require_write(request)
if internal_mode and not actor()["is_super_admin"]:
raise HTTPException(status_code=403, detail="只有超级管理员可以修改宿主机或镜像")
if not account_for_session(session)["admin"]:
raise HTTPException(status_code=403, detail="仅管理员可以执行此操作")
return session
def require_approved(request: Request) -> Session:
session = require_session(request)
account = account_for_session(session)
if not account["admin"] and not account["approved"]:
raise HTTPException(status_code=403, detail="账号等待管理员批准")
return session
def require_approved_write(request: Request) -> Session:
session = require_write(request)
account = account_for_session(session)
if not account["admin"] and not account["approved"]:
raise HTTPException(status_code=403, detail="账号等待管理员批准")
return session
def require_internal_service_write(request: Request) -> Session:
if not internal_mode:
raise HTTPException(status_code=404, detail="Not Found")
return require_approved_write(request)
def require_vm_access(session: Session, profile: dict, vm_id: str) -> str:
if not database_mode:
return vm_id
try:
canonical_id = str(uuid.UUID(vm_id))
except (TypeError, ValueError, AttributeError) as exc:
raise VMError("虚拟机 UUID 无效", 400) from exc
account = account_for_session(session)
if not account["admin"] and not state.users.owns_vm(
host_scope(profile), canonical_id, account["id"]):
raise VMError("虚拟机不存在", 404)
return canonical_id
def smtp_store() -> SMTPSettingsStore:
if data_store is None:
raise SMTPError("请先完成数据库配置", 409)
return SMTPSettingsStore(data_store)
def ssh_key_store():
if not database_mode or state is None:
raise SSHKeyError("请先完成数据库配置", 409)
return state.ssh_keys
def current_updater() -> Updater:
nonlocal updater
if data_store is None:
raise UpdateError("请先完成数据库配置", 409)
if updater is None:
updater = Updater(settings.static_dir.parent, data_store)
return updater
def forbid_long_job_during_update() -> None:
if data_store is None:
return
active_id = data_store.get_value("update.active_job")
if active_id:
job = current_updater().get_job(active_id)
if job is not None and job.get("status") in {"queued", "running"}:
raise HTTPException(status_code=409, detail="面板正在更新,暂时不能开始新任务")
def current_profile() -> dict:
profile = manager.current()
if profile is None:
raise HTTPException(status_code=409, detail="请先在面板中配置宿主机")
if profile.get("key_locked"):
raise RemoteHostError("宿主机 SSH 私钥已锁定,请先在面板解锁", 423)
return profile
def new_service(profile: dict) -> VMService:
if service_factory is not None:
return service_factory(profile)
return VMService(
profile["libvirt_uri"],
profile["storage_pool"],
profile["iso_pool"],
profile["default_network"],
rescue_iso_pool=profile.get("rescue_iso_pool", ""),
)
def verified_created_vm(profile: dict, result: dict, requested_uuid: str) -> dict:
if result.get("uuid") == requested_uuid:
return result
# Keep the preallocated owner/credential records if libvirt may have
# created the requested guest. Never delete a returned, mismatched UUID:
# it may refer to a different existing VM.
LOG.error("VM create returned a mismatched UUID: expected=%s", requested_uuid)
try:
verified = new_service(profile).get_vm(requested_uuid)
except VMError:
verified = None
if verified is not None and verified.get("uuid") == requested_uuid:
LOG.warning("Recovered VM result by querying the requested UUID: %s", requested_uuid)
return verified
raise VMError("虚拟机创建结果与预分配 UUID 不一致;请由管理员核查宿主机及归属记录", 502)
def current_service() -> VMService:
return new_service(current_profile())
def validate_staged_libvirt(
staged, body: HostBody, *, ensure_paths: bool = False,
) -> tuple[dict | None, str | None]:
service = new_service({
"libvirt_uri": staged.libvirt_uri,
"storage_pool": body.storage_pool,
"iso_pool": body.iso_pool,
"rescue_iso_pool": body.rescue_iso_pool,
"default_network": body.default_network,
})
# Validate only host-owned storage/network resources here. Host save
# and connection testing must never enumerate every guest domain;
# capacity and IP scheduling are panel-owned and VM operations use
# targeted UUID reads when they need a particular guest.
if ensure_paths and (body.rescue_iso_pool or any(path.startswith("/") for path in (
body.storage_pool, body.iso_pool))):
service.ensure_storage_pools()
try:
return service.resources(), None
except VMError as exc:
return None, str(exc)
def session_response(username: str) -> JSONResponse:
if database_mode:
account = state.users.get_by_username(username)
if account is None:
raise HTTPException(status_code=401, detail="账号不可用,请重新登录")
session = sessions.issue(username, user_id=account["id"])
else:
session = sessions.issue(username)
account = account_for_session(session)
response = JSONResponse({
"authenticated": True,
"username": session.username,
"csrf_token": session.csrf_token,
"admin": account["admin"],
"approved": account["approved"],
"host_ready": host_ready(),
})
response.set_cookie(
COOKIE_NAME,
session.token,
max_age=8 * 60 * 60,
httponly=True,
secure=settings.secure_cookie,
samesite="strict",
path="/",
)
return response
@app.get("/", include_in_schema=False)
def index():
return FileResponse(settings.static_dir / "index.html")
@app.get("/api/v1/health")
def health():
if data_store is not None:
# A running HTTP listener alone is not a usable post-update panel.
# The updater treats this endpoint as readiness for the configured DB.
if database_mode:
state.users.has_users()
else:
data_store.get("admin", "singleton")
return {"status": "ok"}
@app.get("/api/v1/database/status")
def database_status():
if bootstrap is None:
return {"configured": True, "setup_required": False}
return {"configured": bootstrap.configured(),
"setup_required": not bootstrap.configured()}
@app.post("/api/v1/database/setup")
def setup_database(body: DatabaseSetupBody, request: Request):
nonlocal state, manager, templates, data_store, updater, sessions, rescue_catalog
nonlocal user_iso_catalog
if bootstrap is None:
raise DatabaseSetupError("数据库初始化仅供正常部署使用", 409)
peer = _peer(request)
if not setup_limiter.allowed(peer):
raise HTTPException(status_code=429, detail="初始化尝试过多,请稍后再试")
def connect(config: dict, master_key: bytes):
try:
return make_components(
settings.data_dir, config, master_key, settings.ssh_binary)
except Exception as exc:
# Connector errors may include database host details. Never
# reflect the exception, credentials or driver SQL in HTTP.
LOG.warning("database setup connection failed: %s", type(exc).__name__)
raise DatabaseSetupError(
"无法连接或初始化 MariaDB;请检查地址、账号、数据库和建表权限", 502,
) from exc
try:
components = bootstrap.configure({
"host": body.host,
"port": body.port,
"database": body.database,
"username": body.username,
"password": body.password.get_secret_value(),
}, body.setup_token, connect)
except DatabaseSetupError:
setup_limiter.failed(peer)
raise
state, manager, templates, data_store = components
rescue_catalog = RescueTemplateStore(data_store)
user_iso_catalog = UserISOStore(data_store)
sessions = DatabaseSessionStore(data_store)
restore_jobs()
setup_limiter.succeeded(peer)
LOG.info("panel database configured")
return {"configured": True}
@app.get("/api/v1/setup/status")
def setup_status():
if state is None:
raise HTTPException(status_code=409, detail="请先配置数据库")
return {"setup_required": state.setup_required()}
@app.post("/api/v1/setup/admin")
def setup_admin(body: SetupBody, request: Request):
if state is None:
raise HTTPException(status_code=409, detail="请先配置数据库")
peer = _peer(request)
if not setup_limiter.allowed(peer):
raise HTTPException(status_code=429, detail="初始化尝试过多,请稍后再试")
try:
username = state.create_admin(body.token, body.username, body.password)
except SetupError:
setup_limiter.failed(peer)
raise
setup_limiter.succeeded(peer)
LOG.info("panel initial user created: username=%s", username)
return session_response(username)
@app.get("/api/v1/session")
def session_info(request: Request):
session = sessions.get(request.cookies.get(COOKIE_NAME))
if session is None:
return {"authenticated": False, "username": None, "csrf_token": None}
try:
account = account_for_session(session)
except HTTPException:
return {"authenticated": False, "username": None, "csrf_token": None}
return {
"authenticated": True,
"username": session.username,
"csrf_token": session.csrf_token,
"admin": account["admin"],
"approved": account["approved"],
"host_ready": host_ready(),
}
@app.post("/api/v1/login")
def login(body: LoginBody, request: Request):
if state is None:
raise HTTPException(status_code=409, detail="请先配置数据库")
peer = _peer(request)
if not login_limiter.allowed(peer):
raise HTTPException(status_code=429, detail="登录尝试过多,请稍后再试")
if database_mode:
if body.email is None or body.username is not None:
raise HTTPException(status_code=422, detail="请使用邮箱登录")
try:
email = normalize_email(body.email)
except UserError:
login_limiter.failed(peer)
raise HTTPException(status_code=401, detail="邮箱或密码错误") from None
if not state.users.has_users():
raise HTTPException(status_code=409, detail="请先完成初始化")
account = state.users.get_by_email(email)
encoded = account["password_hash"] if account else None
username = account["username"] if account else ""
identity_ok = account is not None
else:
legacy_admin = state.admin()
if legacy_admin is None:
raise HTTPException(status_code=409, detail="请先完成初始化")
username, encoded = legacy_admin
account = {"email_verified": True}
identity_ok = hmac.compare_digest(body.username or "", username)
password_ok = verify_password(body.password, encoded) if encoded else False
if not password_ok or not identity_ok or not account["email_verified"]:
login_limiter.failed(peer)
raise HTTPException(status_code=401, detail="邮箱或密码错误")
login_limiter.succeeded(peer)
return session_response(username)
def deliver_registration_code(config, email: str, code: str, users) -> None:
try:
send_verification_code(config, email, code)
except Exception as exc:
# Background delivery must not expose whether an email exists or
# relay SMTP server diagnostics to an unauthenticated client.
LOG.warning("Registration email delivery failed: %s", type(exc).__name__)
try:
users.revoke_registration_code(email, code)
except DatabaseError:
LOG.exception("Could not revoke undelivered registration code")
@app.post("/api/v1/register/code")
def send_registration_code(body: RegistrationCodeBody, request: Request,
background_tasks: BackgroundTasks):
if not database_mode or state is None or not state.users.has_users():
raise HTTPException(status_code=409, detail="请先完成面板初始化")
peer = _peer(request)
if not code_send_limiter.allowed(peer):
raise HTTPException(status_code=429, detail="验证码请求过多,请稍后再试")
email = normalize_email(body.email)
config = smtp_store().get()
if config is None:
raise SMTPError("管理员尚未配置 SMTP,暂不能注册", 503)
code = state.users.generate_code()
try:
should_send = state.users.issue_registration_code(email, code)
except UserError as exc:
if exc.status != 429:
raise
should_send = False
code_send_limiter.failed(peer)
if should_send:
# Sending after the HTTP response avoids exposing whether an
# address is registered via SMTP latency or transport failures.
background_tasks.add_task(
deliver_registration_code, config, email, code, state.users,
)
# Identical public response for existing addresses and cooldowns.
return {"sent": True}
@app.post("/api/v1/register", status_code=201)
def register(body: RegistrationBody, request: Request):
if not database_mode or state is None or not state.users.has_users():
raise HTTPException(status_code=409, detail="请先完成面板初始化")
peer = _peer(request)
if not registration_limiter.allowed(peer):
raise HTTPException(status_code=429, detail="注册尝试过多,请稍后再试")
try:
state.users.register_with_code(body.username, body.email,
body.password, body.code)
except UserError:
registration_limiter.failed(peer)
raise
registration_limiter.succeeded(peer)
return {"registered": True, "approved": False}
@app.post("/api/v1/logout")
def logout(request: Request, _session: Session = Depends(require_write)):
sessions.revoke(request.cookies.get(COOKIE_NAME))
response = JSONResponse({"ok": True})
response.delete_cookie(COOKIE_NAME, path="/")
return response
@app.get("/api/v1/ssh-keys")
def list_ssh_keys(_session: Session = Depends(require_session)):
account = account_for_session(_session)
return {"items": ssh_key_store().list_for_user(account["id"])}
@app.post("/api/v1/ssh-keys", status_code=201)
def create_ssh_key(body: SSHKeyBody,
_session: Session = Depends(require_write)):
account = account_for_session(_session)
return ssh_key_store().create_for_user(
account["id"], body.name, body.public_key,
)
@app.delete("/api/v1/ssh-keys/{key_id}")
def delete_ssh_key(key_id: int, _session: Session = Depends(require_write)):
if key_id < 1:
raise SSHKeyError("公钥不存在", 404)
account = account_for_session(_session)
if not ssh_key_store().delete_for_user(account["id"], key_id):
raise SSHKeyError("公钥不存在", 404)
return {"deleted": True}
@app.get("/api/v1/smtp/settings")
def get_smtp_settings(_session: Session = Depends(require_admin)):
return smtp_store().public_status()
@app.put("/api/v1/smtp/settings")
def save_smtp_settings(body: SMTPSettingsBody,
_session: Session = Depends(require_admin_write)):
settings_data = body.model_dump(exclude={"password"})
settings_data["password"] = (body.password.get_secret_value()
if body.password is not None else "")
config = smtp_store().save(settings_data)
LOG.info("SMTP settings changed: admin=%s", _session.username)
return config.public_status()
@app.get("/api/v1/users/pending")
def pending_users(_session: Session = Depends(require_admin)):
if not database_mode:
return {"items": []}
return {"items": state.users.list_pending_users()}
@app.post("/api/v1/users/{user_id}/approve")
def approve_user(user_id: int, _session: Session = Depends(require_admin_write)):
if not database_mode:
raise HTTPException(status_code=409, detail="数据库用户功能不可用")
account = state.users.approve_user(user_id)
if account is None:
raise HTTPException(status_code=404, detail="用户不存在")
LOG.info("user approved: id=%s admin=%s", user_id, _session.username)
return {"approved": True}
def public_update_settings() -> dict:
info = current_updater().settings()
return {
"repository_url": info["repository"],
"branch": info["branch"],
"token_configured": info["token_configured"],
"checkout_ready": info["checkout_ready"],
"configured": info["token_configured"] and info["checkout_ready"],
"current_version": info["current_commit"],
"active_job_id": info.get("active_job_id"),
"last_job_id": info.get("last_job_id"),
"checkout_error": info.get("checkout_error"),
}
@app.get("/api/v1/update/settings")
def update_settings(_session: Session = Depends(require_admin)):
return public_update_settings()
@app.put("/api/v1/update/settings")
def save_update_settings(body: UpdateSettingsBody,
_session: Session = Depends(require_admin_write)):
if body.repository_url != REPOSITORY_URL or body.branch != BRANCH:
raise UpdateError("更新仓库和分支固定为面板指定的 Gitea main 分支", 422)
service = current_updater()
if body.access_token is not None:
token = body.access_token.get_secret_value()
if token:
service.set_token(token)
if not service.settings()["token_configured"]:
raise UpdateError("请填写 Gitea 私有仓库访问令牌", 422)
LOG.info("panel update access token configured: admin=%s", _session.username)
return public_update_settings()
@app.post("/api/v1/update/check")
def check_update(_session: Session = Depends(require_admin_write)):
result = current_updater().check()
return {
"repository_url": result["repository"],
"branch": result["branch"],
"current_version": result["current_commit"],
"latest_version": result["remote_commit"],
"update_available": result["status"] == "update_available",
}
@app.post("/api/v1/update/apply", status_code=202)
def apply_update(body: UpdateApplyBody, _session: Session = Depends(require_admin_write)):
with (import_lock, reinstall_lock, rescue_lock, snapshot_lock,
rescue_import_lock):
if (pending_templates or pending_reinstalls or pending_rescues
or pending_snapshot_jobs or pending_rescue_imports):
raise UpdateError("镜像导入或虚拟机任务正在执行,请等待完成后再更新面板", 409)
try:
unit = subprocess.run(
["/usr/bin/systemctl", "is-active", "virsh-panel-update.service"],
capture_output=True, text=True, timeout=5, check=False,
)
except (OSError, subprocess.TimeoutExpired):
unit = None
if unit is not None and unit.stdout.strip() in {
"activating", "active", "deactivating", "reloading",
}:
raise UpdateError("独立更新服务仍在运行,请等待其完全结束后再提交", 409)
service = current_updater()
job = service.create_job(
expected_current_commit=body.expected_current_commit,
expected_remote_commit=body.expected_remote_commit,
)
try:
# The web service has a read-only code tree. A separate one-shot
# unit performs the Git update and restarts this service.
trigger = subprocess.run(
["/usr/bin/systemctl", "--no-block", "start", "virsh-panel-update.service"],
capture_output=True, text=True, timeout=10, check=False,
)
except (OSError, subprocess.TimeoutExpired):
trigger = None
if trigger is None or trigger.returncode != 0:
service.fail_job(job["job_id"],
"无法启动独立更新服务;请检查 virsh-panel-update.service 是否已安装")
raise UpdateError("无法启动独立更新服务;请检查 systemd 配置", 503)
LOG.info("panel update queued: job=%s admin=%s", job["job_id"], _session.username)
return {"job_id": job["job_id"], "status": "queued"}
@app.get("/api/v1/update/jobs/{job_id}")
def update_job(job_id: str, _session: Session = Depends(require_admin)):
job = current_updater().get_job(job_id)
if job is None:
raise UpdateError("更新任务不存在", 404)
return job
@app.get("/api/v1/host")
def host(_session: Session = Depends(require_admin)):
return _public_host(manager.current())
@app.get("/api/v1/host/network-pools")
def host_network_pools(_session: Session = Depends(require_admin)):
profile = current_profile()
pools, compatibility_mode = public_network_pools(profile)
return {
"items": pools,
"count": len(pools),
"maximum": MAX_PUBLIC_NETWORK_POOLS,
"compatibility_mode": compatibility_mode,
}
@app.put("/api/v1/host/network-pools/{pool_key}")
def save_host_network_pool(
pool_key: str,
body: PublicNetworkPoolBody,
_session: Session = Depends(require_admin_write),
):
# Host saves and pool mutations share this lock so a concurrent bridge
# change cannot commit between catalog validation and installation.
# Production runs one engine worker; the catalog CAS still protects
# independent pool writers and test/in-process concurrency.
with public_pool_catalog_lock:
profile = current_profile()
pool, idempotent = upsert_public_network_pool(profile, pool_key, body)
LOG.info(
"host public network pool saved: host=%s pool=%s revision=%s admin=%s",
profile.get("host"), pool_key, pool["revision"], _session.username,
)
return {
"pool": pool,
"applied": not idempotent,
"idempotent": idempotent,
"compatibility_mode": False,
}
@app.post("/api/v1/host/scan")
def scan_host(body: HostScanBody, _session: Session = Depends(require_admin_write)):
return manager.scan(body.host, body.port)
@app.post("/api/v1/host/unlock")
def unlock_host(body: UnlockBody, request: Request, _session: Session = Depends(require_admin_write)):
peer = _peer(request)
if not unlock_limiter.allowed(peer):
raise HTTPException(status_code=429, detail="私钥口令尝试过多,请稍后再试")
try:
profile = manager.unlock_current(body.passphrase)
except RemoteHostError as exc:
if exc.status == 400:
unlock_limiter.failed(peer)
raise
unlock_limiter.succeeded(peer)
LOG.info("remote host SSH key unlocked: host=%s admin=%s", profile.get("host"), _session.username)
return _public_host(profile)
@app.post("/api/v1/host/test")
def test_host(body: HostBody, _session: Session = Depends(require_admin_write)):
staged = manager.stage(body.profile(), confirmed_fingerprint=body.confirmed_fingerprint)
try:
result = manager.test(staged)
resources, resource_warning = validate_staged_libvirt(staged, body)
return {
**result,
"resources": resources,
"warnings": [resource_warning] if resource_warning else [],
}
finally:
manager.abort(staged)
@app.put("/api/v1/host")
def save_host(body: HostBody, _session: Session = Depends(require_admin_write)):
# A running reinstall uses the saved SSH host identity and paths. Do
# not replace them while its libvirt operation is in flight.
with (reinstall_lock, rescue_lock, snapshot_lock, rescue_import_lock,
public_pool_catalog_lock):
forbid_long_job_during_update()
if (pending_reinstalls or pending_rescues or pending_snapshot_jobs
or pending_rescue_imports
or any(job.get("status") in {"queued", "running", "uncertain"}
for job in reinstall_jobs.values())
or (data_store is not None and data_store.list("vm_rescue"))):
raise VMError("请先完成救援或重装任务并退出救援模式,再修改宿主机配置", 409)
if any(
job.get("status") in {"succeeded", "uncertain"}
and "generated_password" in job
for job in reinstall_jobs.values()):
raise VMError("请先完成重装任务并确认保存自动生成的密码,再修改宿主机配置", 409)
current = manager.current()
if current is not None:
remote_identity_changed = (
current.get("host") != body.host.lower()
or current.get("port") != body.port
or current.get("fingerprint") != body.confirmed_fingerprint
or current.get("storage_pool") != body.storage_pool
)
if remote_identity_changed and has_retained_snapshots():
raise VMError(
"宿主机仍保留本地快照;更换远端宿主身份或存储池前请先删除全部快照",
409,
)
_raw_catalog, native_pools = read_native_public_network_pools(current)
if any(
pool["network_selector"] != body.default_network
for pool in native_pools):
raise VMError(
"宿主机已有公网地址池;请先在公网地址池管理中处理网络接入,再修改默认网络",
409,
)
staged = manager.stage(body.profile(), confirmed_fingerprint=body.confirmed_fingerprint)
committed = False
try:
manager.test(staged)
_resources, resource_warning = validate_staged_libvirt(staged, body, ensure_paths=True)
profile = manager.commit(staged)
committed = True
finally:
if not committed:
manager.abort(staged)
LOG.info("remote host configured: host=%s port=%s admin=%s", body.host, body.port, _session.username)
return {
**_public_host(profile),
"warnings": [resource_warning] if resource_warning else [],
}
@app.get("/api/v1/resources")
def resources(_session: Session = Depends(require_approved)):
profile = current_profile()
values = new_service(profile).resources()
# Root-pool ISO filenames are retained only for legacy/admin clients.
# Ordinary users select owner-scoped opaque IDs from /user-isos.
if database_mode and not account_for_session(_session)["admin"]:
values["isos"] = []
pools, compatibility_mode = public_network_pools(profile)
primary = next(
(pool for pool in pools if pool["state"] == "active"), None)
return {
**values,
# Keep the original scalar fields during the PHP rollout. They
# describe only the deterministic primary active pool; new code
# uses host/network-pools for the configured catalog.
"public_pool_cidr": "" if primary is None else primary["cidr"],
"public_pool_enabled": primary is not None,
"public_vm_prefix": (
None if primary is None else primary["guest_prefix_length"]),
"public_network_pool_count": len(pools),
"public_network_pool_compatibility_mode": compatibility_mode,
}
def cloud_template_record(profile: dict, template_id: str) -> dict:
if shared_cloud_template_runtime is not None:
try:
return shared_cloud_template_runtime.get(template_id)
except TemplateError as exc:
if exc.status != 404:
raise
# Host-scoped template IDs were used before the catalog became
# global. Keep already-paid/queued operations executable on
# their original host while administrators re-import those
# images into the new global catalog. They cannot be promoted
# automatically because the old flow intentionally discarded
# the verified local source after uploading it to libvirt.
return templates.get(template_scope(profile), template_id)
return templates.get(template_scope(profile), template_id)
def cloud_template_replica_scope(profile: dict) -> str:
"""Bind a ready marker to the actual remote host configuration.
``template_scope`` deliberately keeps a stable logical host identity.
That is useful for legacy catalogs, but is too weak as proof that a
cached volume still belongs to the same physical hypervisor. A host
rebind or SSH host-key change must invalidate every ready marker.
"""
identity = [
"virt-cloud-template-replica-v2",
profile.get("host_id"),
profile.get("host"),
profile.get("port"),
profile.get("fingerprint"),
profile.get("storage_pool"),
]
return hashlib.sha256(json.dumps(
identity, ensure_ascii=False, separators=(",", ":"),
).encode("utf-8")).hexdigest()
def cloud_template_replica_volume_name(record: dict) -> str:
source_name = record["volume_name"]
if not isinstance(source_name, str) or not source_name.endswith(".qcow2"):
raise DatabaseError("云镜像源卷名无效")
return f"{source_name[:-6]}-{secrets.token_hex(8)}.qcow2"
def valid_cloud_template_replica_volume_name(
record: dict, volume_name: object) -> bool:
source_name = record.get("volume_name")
if (not isinstance(source_name, str)
or not source_name.endswith(".qcow2")
or not isinstance(volume_name, str)):
return False
prefix = re.escape(source_name[:-6])
return re.fullmatch(
rf"{prefix}-[0-9a-f]{{16}}\.qcow2", volume_name,
) is not None
def cloud_template_replica_payload(
profile: dict, record: dict,
status: Literal["syncing", "ready"],
replica_volume_name: str,
replica_identity: str | None = None) -> dict:
if not valid_cloud_template_replica_volume_name(
record, replica_volume_name):
raise DatabaseError("云镜像宿主机缓存卷名无效")
result = {
"status": status,
"replica_scope": cloud_template_replica_scope(profile),
"source_volume_name": record["volume_name"],
"replica_volume_name": replica_volume_name,
"sha256": record["sha256"],
"virtual_size": record["virtual_size"],
"image_bytes": record["image_bytes"],
}
if status == "ready":
if (not isinstance(replica_identity, str)
or re.fullmatch(r"[0-9a-f]{64}", replica_identity) is None):
raise DatabaseError("云镜像宿主机缓存身份无效")
result["replica_identity"] = replica_identity
return result
def cloud_template_replica_marker(
profile: dict, record: dict) -> dict | None:
if data_store is None:
return None
raw = data_store.get("cloud_template_replica", record["id"])
if raw is None:
return None
try:
value = json.loads(raw)
except (TypeError, ValueError) as exc:
raise DatabaseError("云镜像宿主机缓存记录损坏") from exc
if not isinstance(value, dict):
raise DatabaseError("云镜像宿主机缓存记录损坏")
status = value.get("status")
replica_volume_name = value.get("replica_volume_name")
if (status not in {"syncing", "ready"}
or not valid_cloud_template_replica_volume_name(
record, replica_volume_name)):
# Older/incomplete marker schemas are evidence only that a cache
# may exist, never proof that it is complete or belongs to this
# physical host generation.
return None
replica_identity = value.get("replica_identity")
if status == "ready" and (
not isinstance(replica_identity, str)
or re.fullmatch(r"[0-9a-f]{64}", replica_identity) is None):
return None
expected = cloud_template_replica_payload(
profile, record, status, replica_volume_name,
replica_identity if status == "ready" else None,
)
return value if value == expected else None
def save_cloud_template_replica(
profile: dict, record: dict,
status: Literal["syncing", "ready"],
replica_volume_name: str,
replica_identity: str | None = None) -> None:
if data_store is None:
return
data_store.set(
"cloud_template_replica", record["id"],
json.dumps(
cloud_template_replica_payload(
profile, record, status, replica_volume_name,
replica_identity),
separators=(",", ":"),
),
)
def ensure_cloud_template_replica(
profile: dict, service: VMService, template_id: str) -> dict:
record = cloud_template_record(profile, template_id)
if shared_cloud_template_runtime is None:
return record
with cloud_template_cache_lock:
# Re-read after entering the per-host critical section. Another
# create may have completed this same first-use transfer while the
# request was waiting for the lock.
try:
record = shared_cloud_template_runtime.get(template_id)
except TemplateError as exc:
if exc.status != 404:
raise
return templates.get(template_scope(profile), template_id)
marker = cloud_template_replica_marker(profile, record)
trusted_volume_name = None
trusted_identity = None
if marker is not None and marker["status"] == "ready":
trusted_volume_name = marker["replica_volume_name"]
trusted_identity = marker["replica_identity"]
# Persist a different, never-before-used fallback generation
# before asking libvirt whether the ready volume is still the
# exact volume proven by the marker. If it is not, the old
# name is never overwritten or silently reused.
desired_volume_name = cloud_template_replica_volume_name(record)
elif marker is not None:
# A crash after writing ``syncing`` may have left a partial
# upload. Retry that planned generation as untrusted; the
# service safely replaces it only when no VM references it.
desired_volume_name = marker["replica_volume_name"]
else:
desired_volume_name = cloud_template_replica_volume_name(record)
# Keep a proven ready marker until the global source has itself
# been verified. A damaged source must not downgrade evidence of
# an already complete host replica.
with shared_cloud_template_runtime.source(template_id) as (
record, source_path):
# Once source verification succeeds, always downgrade the
# durable marker before touching libvirt. Even a previously
# ready replica may have disappeared or need replacement; a
# power loss during that upload must never leave ready proof
# pointing at a partially streamed qcow2 volume.
save_cloud_template_replica(
profile, record, "syncing", desired_volume_name)
cached = service.cache_template(
source_path, desired_volume_name,
record["virtual_size"],
trusted_replica_volume_name=trusted_volume_name,
trusted_replica_identity=trusted_identity,
)
cached_volume_name = cached.get("volume_name")
accepted_names = {desired_volume_name}
if trusted_volume_name is not None:
accepted_names.add(trusted_volume_name)
if cached_volume_name not in accepted_names:
raise DatabaseError("云镜像宿主机缓存返回了无效卷")
save_cloud_template_replica(
profile, record, "ready", cached_volume_name,
cached.get("replica_identity"))
return {**record, "volume_name": cached_volume_name}
def user_iso_context(session: Session) -> tuple[int, str]:
owner_id = account_for_session(session)["id"]
return owner_id, user_iso_scope(owner_id)
def user_iso_replica_key(owner_id: int, iso_id: str) -> str:
return f"{owner_id}:{iso_id}"
def trusted_user_iso_replica(owner_id: int, record: dict) -> bool:
if data_store is None:
return False
raw = data_store.get(
"user_iso_replica", user_iso_replica_key(owner_id, record["id"]))
if raw is None:
return False
try:
replica = json.loads(raw)
except (TypeError, ValueError) as exc:
raise DatabaseError("用户 ISO 宿主机缓存记录损坏") from exc
if not isinstance(replica, dict):
raise DatabaseError("用户 ISO 宿主机缓存记录损坏")
return replica == {
"status": "ready",
"volume_name": record["volume_name"],
"size_bytes": record["size_bytes"],
"sha256": record["sha256"],
}
def save_user_iso_replica(owner_id: int, record: dict, *,
status: Literal["syncing", "ready"]) -> None:
if data_store is None:
return
data_store.set(
"user_iso_replica", user_iso_replica_key(owner_id, record["id"]),
json.dumps({
"status": status,
"volume_name": record["volume_name"],
"size_bytes": record["size_bytes"],
"sha256": record["sha256"],
}, separators=(",", ":")),
)
def prune_user_iso_replica_records(owner_id: int,
active_ids: set[str]) -> None:
if data_store is None:
return
prefix = f"{owner_id}:"
for key in data_store.list("user_iso_replica"):
if key.startswith(prefix) and key[len(prefix):] not in active_ids:
data_store.delete("user_iso_replica", key)
def public_user_iso(record: dict) -> dict:
return {key: value for key, value in record.items()
if key in {"id", "name", "size_bytes", "sha256", "created_at"}}
def public_user_iso_error(value: Any) -> str:
safe = {
"面板在任务执行期间重启;请检查宿主机实际状态,勿直接重复提交",
"ISO 下载地址无效", "ISO 下载地址无法解析",
"ISO 下载地址的文件名必须以 .iso 结尾",
"ISO 下载地址解析结果无效",
"ISO 下载地址不能指向内网、回环或保留地址",
"ISO 下载地址发生重定向,请使用最终 HTTPS 直链",
"无法连接 ISO 下载地址", "ISO 下载响应长度无效",
"ISO 下载响应不能使用内容压缩", "ISO 下载超时",
"ISO 下载内容不完整", "ISO 的 SHA-256 校验值不匹配",
"ISO 大小必须在 1 MiB 至可用配额之间",
"下载文件不是有效的 ISO 9660 镜像",
"ISO 未包含可识别的 El Torito 启动记录",
"ISO 超过用户目录的 10 GiB 总配额或剩余容量", "用户 ISO 可用配额不足",
"每个宿主机最多保存 5 个用户 ISO",
"用户 ISO 目录的 10 GiB 存储配额已用完",
"用户 ISO 目录总容量不能超过 10 GiB",
"宿主机用户 ISO 存储空间不足",
"用户 ISO 上传到宿主机失败",
"用户 ISO 导入失败,请查看引擎日志",
}
if isinstance(value, str) and value in safe:
return value
if isinstance(value, str) and re.fullmatch(
r"ISO 下载服务器返回 HTTP [1-5][0-9]{2}", value):
return value
return "ISO 下载或导入失败,请联系管理员查看日志"
# Async job records are persisted by the engine and may contain text from
# libvirt, QEMU, a remote host, or a future implementation. The browser
# endpoints below deliberately expose a small, operation-specific DTO;
# never return a job dictionary wholesale. Keep these sets local to the
# app factory so an engine instance cannot accidentally share mutable
# presentation state with another host.
public_job_statuses = frozenset({
"queued", "pending", "running", "succeeded", "completed", "done",
"failed", "uncertain", "cancelled", "canceled", "blocked", "unknown",
})
public_iso_phases = frozenset({
"等待下载", "正在安全下载 ISO", "正在保存到面板 ISO 目录",
"ISO 已保存,可用于所有宿主机", "导入失败", "引擎重启,任务已中断",
"面板在 ISO 下载期间重启;请重新提交导入",
})
public_reinstall_phases = frozenset({
"等待重装任务开始", "正在检查虚拟机", "重装完成", "重装失败",
"重装完成,但旧系统盘清理待核对", "重装结果待核对",
"重装已完成,但凭据或任务记录保存失败", "面板重启,任务已中断",
# These are stable progress identifiers consumed by the standalone
# engine UI; they are not arbitrary text from libvirt.
"shutting_down", "cloning", "starting", "cleaning_up",
"cleanup_pending", "result_uncertain", "completed",
})
public_rescue_phases = frozenset({
"等待救援任务开始", "等待手动确认的救援冷启动", "等待退出救援模式",
"正在预挂载救援 ISO 并等待正常关机", "正在强制断电并从救援 ISO 冷启动",
"正在正常关机并恢复原系统启动配置",
"救援 ISO 已预挂载,等待手动强制断电后开机", "救援系统已启动",
"已退出救援模式,原系统已启动", "救援操作失败",
"面板重启,任务已中断",
})
# Canonical messages retain useful, already-reviewed user guidance while
# removing any suffix supplied by a host-controlled exception. Prefixes
# are matched only to select a fixed replacement; the original value is
# never returned.
public_reinstall_errors = (
("重装失败且自动回滚不完整", "重装失败且自动回滚不完整;旧系统盘已保留,请检查虚拟机与新建存储卷"),
("重装失败,已恢复原虚拟机定义及旧系统盘", "重装失败,已恢复原虚拟机定义及旧系统盘"),
("正常关机超时,重装已停止", "正常关机超时,重装已停止;旧系统盘未改变"),
("重装系统失败", "重装系统失败,请查看面板日志"),
("虚拟机配置无效", "虚拟机配置无效,重装已停止"),
("虚拟机磁盘或网卡配置已变化", "虚拟机配置已变化,重装已停止"),
("虚拟机网桥或 MAC 已变化", "虚拟机网桥或 MAC 已变化,不能自动沿用原网络"),
("关机后虚拟机配置已变化", "关机后虚拟机配置已变化,重装已停止"),
("关机后存储卷已变化", "关机后存储卷已变化,重装已停止"),
("原配置盘的公网 IP 或 MAC 与虚拟机不一致", "原配置盘的公网 IP 或 MAC 与虚拟机不一致,已停止重装"),
("无法校验原 cloud-init 网络配置", "无法校验原 cloud-init 网络配置,已停止重装"),
("模板卷在宿主机上不存在", "模板卷在宿主机上不存在,请重新导入"),
("模板卷不是 qcow2 格式", "模板卷格式不受支持,已停止重装"),
("模板容量超过原系统盘", "所选模板容量超过原系统盘,无法保留原磁盘大小"),
("存储池空间不足", "存储池空间不足,无法创建新系统盘"),
("新系统盘或配置盘名称冲突", "新系统盘或配置盘名称冲突,已停止重装"),
("宿主机未能克隆新系统盘", "宿主机未能克隆新系统盘,重装已停止"),
("宿主机未能创建新配置盘", "宿主机未能创建新配置盘,重装已停止"),
("宿主机未能切换虚拟机定义", "宿主机未能切换虚拟机定义,重装已停止"),
("宿主机未能保存虚拟机网络配置", "宿主机未能保存新的虚拟机配置,重装已停止"),
("重装已完成,但旧系统盘清理待核对", "重装已完成,但旧系统盘清理待核对,请管理员检查宿主机"),
("重装结果待核对", "重装结果待核对,请检查虚拟机状态和存储卷"),
)
public_rescue_errors = (
("救援 ISO 预挂载失败且原虚拟机持久配置未能恢复", "救援 ISO 预挂载失败且原虚拟机持久配置未能恢复;没有强制断电,请检查宿主机"),
("救援 ISO 预挂载失败", "救援 ISO 预挂载失败,原虚拟机未受影响"),
("救援启动失败,已恢复原虚拟机配置及运行状态", "救援启动失败,已恢复原虚拟机配置及运行状态"),
("救援系统强制重启后未能启动", "救援系统强制重启后未能启动;救援 ISO 配置已保留,请检查控制台"),
("已尝试强制启动,但救援系统未进入运行状态", "已尝试强制启动,但救援系统未进入运行状态;救援 ISO 配置已保留,请检查控制台或重试开机"),
("退出救援失败且救援虚拟机未能完全恢复", "退出救援失败且救援虚拟机未能完全恢复;没有强制断电,请检查宿主机"),
("退出救援失败,已恢复救援启动配置及运行状态", "退出救援失败,已恢复救援启动配置及运行状态"),
("原系统已启动但仍显示救援模式", "原系统已启动但仍显示救援模式,请检查虚拟机配置"),
("关机后救援启动配置已变化", "关机后救援启动配置已变化,救援已停止;请检查虚拟机状态"),
("强制断电失败", "强制断电失败;救援配置已保留,请检查虚拟机状态"),
("救援 ISO 启动设备与预挂载记录不匹配", "救援 ISO 启动设备与预挂载记录不匹配,已拒绝强制重启"),
("救援恢复记录丢失", "救援恢复记录丢失,已停止操作"),
("救援操作失败", "紧急救援操作失败,请查看面板和宿主机日志"),
("正常关机超时", "正常关机超时,救援未完成,请核对救援状态后再继续"),
("无法请求虚拟机正常关机", "无法请求虚拟机正常关机,请核对救援状态"),
)
public_reinstall_warnings = (
("新系统可能已启动,但启动返回错误", "新系统可能已启动,但启动结果需要通过控制台确认;旧系统盘已保留"),
("新系统已启动,但面板未能读取完整虚拟机状态", "新系统已启动,但面板未能读取完整虚拟机状态,请检查宿主机"),
("新系统已启动,但面板无法确认初始密码已保存", "新系统已启动,但面板无法确认初始密码已保存;请立即保存结果页显示的密码"),
("旧存储卷", "重装已完成,但旧系统盘清理需要管理员检查"),
)
public_rescue_warnings = (
("救援配置已运行,但宿主机关机状态返回异常", "救援配置已运行,但宿主机关机状态需要通过 VNC 确认"),
("救援系统可能已启动,但宿主机启动返回错误", "救援系统可能已启动,请通过控制台确认"),
("救援系统可能已启动,但宿主机返回错误", "救援系统可能已启动,请通过控制台确认"),
("原系统可能已启动,但宿主机启动返回错误", "原系统可能已启动,请通过控制台确认"),
("救援 ISO 已配置为下次冷启动", "救援 ISO 已配置为下次冷启动,请手动强制断电后开机"),
("正常关机超时", "正常关机超时;救援 ISO 可能已预挂载,请核对救援状态"),
("正常关机", "正常关机未完成;请核对救援状态后再继续"),
)
def public_status(value: Any) -> str:
return value if isinstance(value, str) and value in public_job_statuses else "unknown"
def public_timestamp(value: Any) -> int | None:
if isinstance(value, bool):
return None
try:
result = int(value)
except (TypeError, ValueError, OverflowError):
return None
return result if result >= 0 else None
def public_identifier(value: Any, *, minimum: int = 1, maximum: int = 128) -> str | None:
if not isinstance(value, str) or not (minimum <= len(value) <= maximum):
return None
return value if re.fullmatch(r"[A-Za-z0-9_-]+", value) else None
def canonical_job_text(value: Any, mappings: tuple[tuple[str, str], ...], fallback: str) -> str | None:
if not isinstance(value, str) or not value.strip():
return None
text = value.strip()
for prefix, canonical in mappings:
if text.startswith(prefix):
return canonical
return fallback
def public_phase(value: Any, status: str, allowed: frozenset[str], *, kind: str) -> str:
if kind == "reinstall" and status == "uncertain":
if isinstance(value, str) and value in {
"重装完成,但旧系统盘清理待核对", "重装结果待核对"}:
return value
return "等待核对结果"
if isinstance(value, str) and value in allowed:
return value
if kind == "reinstall":
return {
"queued": "等待重装任务开始", "pending": "等待重装任务开始",
"running": "正在执行重装任务", "succeeded": "重装完成",
"completed": "重装完成", "done": "重装完成",
"failed": "重装失败", "uncertain": "等待核对结果",
}.get(status, "重装任务状态待核对")
if kind == "rescue":
return {
"queued": "等待救援任务开始", "pending": "等待救援任务开始",
"running": "正在执行救援任务", "succeeded": "救援操作已完成",
"completed": "救援操作已完成", "done": "救援操作已完成",
"failed": "救援操作失败", "uncertain": "等待核对结果",
}.get(status, "救援任务状态待核对")
return {
"queued": "等待下载", "pending": "等待下载", "running": "正在处理",
"succeeded": "已完成", "completed": "已完成", "done": "已完成",
"failed": "导入失败", "uncertain": "等待核对结果",
}.get(status, "任务状态待核对")
def public_warning(value: Any, mappings: tuple[tuple[str, str], ...], fallback: str) -> str | None:
return canonical_job_text(value, mappings, fallback)
def public_user_iso_job(job: dict) -> dict:
status = public_status(job.get("status"))
result: dict[str, Any] = {}
job_id = public_identifier(job.get("job_id"), minimum=8)
if job_id is not None:
result["job_id"] = job_id
result["status"] = status
result["phase"] = public_phase(
job.get("phase"), status, public_iso_phases, kind="iso")
for key in ("created_at", "started_at", "finished_at"):
timestamp = public_timestamp(job.get(key))
if timestamp is not None:
result[key] = timestamp
if isinstance(job.get("iso"), dict):
result["iso"] = public_user_iso(job["iso"])
if "error" in job:
result["error"] = public_user_iso_error(job.get("error"))
return result
def public_user_iso_listing(value: Any) -> dict:
"""Re-sanitize shared-runtime output at this app's HTTP boundary."""
if not isinstance(value, dict):
return {"isos": []}
result: dict[str, Any] = {
"isos": [public_user_iso(item) for item in value.get("isos", [])
if isinstance(item, dict)],
}
active = value.get("active_import_job")
if isinstance(active, dict):
result["active_import_job"] = public_user_iso_job(active)
return result
@app.get("/api/v1/user-isos")
def list_user_isos(_session: Session = Depends(require_approved)):
owner_id, scope = user_iso_context(_session)
if shared_user_iso_runtime is not None:
return public_user_iso_listing(shared_user_iso_runtime.list_for(owner_id))
result = {
"isos": [public_user_iso(row) for row in user_iso_catalog.list(scope)],
}
with user_iso_import_lock:
active = [job for job in user_iso_import_jobs.values()
if job.get("scope") == scope
and job.get("owner_user_id") == owner_id
and job.get("status") in {"queued", "running"}]
if active:
# Only one import per owner/host can be submitted. Sorting is
# defensive for an upgraded database that contains an older
# duplicate; never expose either owner's private scope field.
current = max(active, key=lambda job: (
int(job.get("created_at", 0)), str(job.get("job_id", ""))))
result["active_import_job"] = public_user_iso_job(current)
return result
@app.get("/api/v1/user-isos/import-jobs/{job_id}")
def user_iso_import_job(job_id: str,
_session: Session = Depends(require_approved)):
owner_id, scope = user_iso_context(_session)
if shared_user_iso_runtime is not None:
return public_user_iso_job(
shared_user_iso_runtime.get_job_for(owner_id, job_id))
with user_iso_import_lock:
job = user_iso_import_jobs.get(job_id)
if (job is None or job.get("scope") != scope
or job.get("owner_user_id") != owner_id):
raise TemplateError("ISO 导入任务不存在", 404)
return public_user_iso_job(job)
@app.get("/api/v1/user-isos/{iso_id}")
def get_user_iso(iso_id: str,
_session: Session = Depends(require_approved)):
owner_id, scope = user_iso_context(_session)
if shared_user_iso_runtime is not None:
return {"iso": public_user_iso(
shared_user_iso_runtime.get_for(owner_id, iso_id))}
return {"iso": public_user_iso(user_iso_catalog.get(scope, iso_id))}
def run_user_iso_import(job_id: str, owner_id: int,
scope: str, body: ImportUserISOBody) -> None:
image_path: Path | None = None
iso_id = ""
published = False
cataloged = False
try:
with user_iso_import_lock:
job = user_iso_import_jobs[job_id]
job.update(status="running", phase="正在安全下载 ISO",
started_at=int(time.time()))
save_user_iso_job(job_id, job)
remaining = user_iso_catalog.remaining_bytes(scope)
available = user_iso_files.available_bytes(owner_id)
maximum = min(remaining, available)
if maximum < 1024 * 1024:
raise TemplateError("面板用户 ISO 持久目录空间不足", 409)
image_path, size, digest = download_user_iso(
body.url, body.sha256.lower(),
user_iso_files.owner_directory(owner_id), max_bytes=maximum,
)
iso_id = uuid.uuid4().hex
record = {
"id": iso_id,
"name": body.name,
"volume_name": user_iso_volume_name(iso_id),
"size_bytes": size,
"sha256": digest,
"created_at": int(time.time()),
}
with user_iso_import_lock:
job = user_iso_import_jobs[job_id]
job["phase"] = "正在保存到面板 ISO 目录"
save_user_iso_job(job_id, job)
user_iso_files.publish(
image_path, owner_id, iso_id, expected_size=size)
image_path = None
published = True
user_iso_catalog.add(scope, record)
cataloged = True
user_iso_import_jobs[job_id] = {
"job_id": job_id, "scope": scope, "owner_user_id": owner_id,
"status": "completed", "phase": "ISO 已保存,可用于所有宿主机",
"created_at": user_iso_import_jobs[job_id]["created_at"],
"started_at": user_iso_import_jobs[job_id].get("started_at"),
"finished_at": int(time.time()), "iso": record,
}
save_user_iso_job(job_id, user_iso_import_jobs[job_id])
except Exception as exc:
if isinstance(exc, TemplateError):
error = str(exc)
else:
LOG.exception("User ISO import failed: job=%s owner=%s", job_id, owner_id)
error = "用户 ISO 导入失败,请查看引擎日志"
if cataloged:
try:
user_iso_catalog.delete(scope, iso_id)
except Exception:
LOG.exception("Could not clean failed user ISO catalog: job=%s", job_id)
if published and iso_id:
try:
user_iso_files.delete(owner_id, iso_id)
except Exception:
LOG.exception("Could not clean failed user ISO source: job=%s", job_id)
with user_iso_import_lock:
previous = user_iso_import_jobs.get(job_id, {})
user_iso_import_jobs[job_id] = {
"job_id": job_id, "scope": scope, "owner_user_id": owner_id,
"status": "failed", "phase": "导入失败", "error": error,
"created_at": previous.get("created_at", int(time.time())),
"started_at": previous.get("started_at"),
"finished_at": int(time.time()),
}
try:
save_user_iso_job(job_id, user_iso_import_jobs[job_id])
except DatabaseError:
LOG.exception("Could not persist user ISO import failure: job=%s", job_id)
finally:
if image_path is not None:
image_path.unlink(missing_ok=True)
with user_iso_import_lock:
pending_user_iso_imports.discard(scope)
@app.post("/api/v1/user-isos/import", status_code=202)
def import_user_iso(body: ImportUserISOBody,
_session: Session = Depends(require_approved_write)):
owner_id, scope = user_iso_context(_session)
if shared_user_iso_runtime is not None:
return shared_user_iso_runtime.queue_import(
owner_id, body.url, body.name, body.sha256)
validate_user_iso_url(body.url)
if body.sha256 and (len(body.sha256) != 64
or not all(char in "0123456789abcdefABCDEF"
for char in body.sha256)):
raise TemplateError(
"SHA-256 可留空;填写时必须为 64 位十六进制校验值", 422)
name = validate_user_iso_name(body.name)
body = body.model_copy(update={"name": name, "sha256": body.sha256.lower()})
with user_iso_import_lock:
forbid_long_job_during_update()
existing = user_iso_catalog.list(scope)
user_iso_catalog.remaining_bytes(scope)
if any(row["name"].casefold() == name.casefold() for row in existing):
raise TemplateError("当前用户已有同名 ISO", 409)
if scope in pending_user_iso_imports:
raise TemplateError("已有用户 ISO 正在导入,请等待完成", 409)
job_id = str(uuid.uuid4())
pending_user_iso_imports.add(scope)
user_iso_import_jobs[job_id] = {
"job_id": job_id, "scope": scope, "owner_user_id": owner_id,
"status": "queued", "phase": "等待下载", "created_at": int(time.time()),
}
try:
# Persist before handing work to the executor. Both steps
# share one rollback path so a database failure cannot leave
# this owner permanently stuck in the in-memory pending set.
save_user_iso_job(job_id, user_iso_import_jobs[job_id])
user_iso_import_executor.submit(
run_user_iso_import, job_id, owner_id, scope, body)
except Exception as exc:
pending_user_iso_imports.discard(scope)
user_iso_import_jobs.pop(job_id, None)
with suppress(DatabaseError):
delete_user_iso_job(job_id)
if isinstance(exc, RuntimeError):
raise TemplateError(
"引擎正在停止,无法开始 ISO 导入", 503) from exc
raise
LOG.info("user ISO import queued: job=%s owner=%s", job_id, owner_id)
return {"job_id": job_id, "status": "queued"}
@app.delete("/api/v1/user-isos/{iso_id}")
def delete_user_iso(iso_id: str,
_session: Session = Depends(require_approved_write)):
owner_id, scope = user_iso_context(_session)
if shared_user_iso_runtime is not None:
shared_user_iso_runtime.delete_for(owner_id, iso_id)
LOG.info("user ISO deleted: id=%s owner=%s", iso_id, owner_id)
return {"deleted": True}
with user_iso_import_lock:
forbid_long_job_during_update()
if scope in pending_user_iso_imports:
raise TemplateError("用户 ISO 正在导入,请稍后删除", 409)
user_iso_catalog.get(scope, iso_id)
user_iso_files.delete(owner_id, iso_id)
user_iso_catalog.delete(scope, iso_id)
LOG.info("user ISO deleted: id=%s owner=%s", iso_id, owner_id)
return {"deleted": True}
@app.get("/api/v1/vm-templates")
def list_templates(_session: Session = Depends(require_approved)):
if shared_cloud_template_runtime is not None:
return {"templates": shared_cloud_template_runtime.list()}
profile = current_profile()
return {"templates": templates.list(template_scope(profile))}
def run_template_import(job_id: str, profile: dict, scope: str,
body: ImportTemplateBody, pending_key: str) -> None:
image_path = None
volume_name = ""
uploaded = False
try:
image_path, virtual_size, actual_sha256 = download_verified_image(
body.url, body.sha256.lower(), settings.data_dir)
os_family, version = image_metadata_for(body.url)
template_id = template_id_for(
scope, actual_sha256, version, os_family
)
volume_name = template_volume_name(template_id)
with import_lock:
if any(row["id"] == template_id for row in templates.list(scope)):
raise TemplateError("此云镜像已经导入", 409)
result = new_service(profile).import_template(image_path, volume_name, virtual_size)
uploaded = True
record = {
"id": template_id,
"name": body.name,
"os_family": os_family,
"os_version": version,
"sha256": actual_sha256,
"volume_name": volume_name,
"virtual_size": virtual_size,
"image_bytes": result["image_bytes"],
}
templates.add(scope, record)
with import_lock:
import_jobs[job_id] = {"scope": scope, "status": "completed",
"template": record}
save_job("import_job", job_id, import_jobs[job_id])
except Exception as exc:
if isinstance(exc, (TemplateError, VMError)):
error = str(exc)
else:
LOG.exception("Cloud template import failed: job=%s", job_id)
error = "云镜像导入失败,请查看面板日志"
if uploaded and volume_name:
error += f";宿主机模板卷 {volume_name} 可能需要人工检查"
with import_lock:
import_jobs[job_id] = {"scope": scope, "status": "failed", "error": error}
save_job("import_job", job_id, import_jobs[job_id])
finally:
if image_path is not None:
image_path.unlink(missing_ok=True)
with import_lock:
pending_templates.discard((scope, pending_key))
@app.post("/api/v1/vm-templates/import", status_code=202)
def import_template(body: ImportTemplateBody, _session: Session = Depends(require_admin_write)):
if shared_cloud_template_runtime is not None:
forbid_long_job_during_update()
return shared_cloud_template_runtime.queue_import(
body.url, body.name, body.sha256)
profile = current_profile()
if body.sha256 and (len(body.sha256) != 64
or not all(char in "0123456789abcdefABCDEF"
for char in body.sha256)):
raise TemplateError(
"SHA-256 可留空;填写时必须为 64 位十六进制校验值", 422
)
os_family, version = image_metadata_for(body.url)
name = validate_template_name(body.name)
body = body.model_copy(update={"name": name})
scope = template_scope(profile)
expected_sha256 = body.sha256.lower()
pending_key = expected_sha256 or hashlib.sha256(body.url.encode("utf-8")).hexdigest()
with import_lock:
forbid_long_job_during_update()
existing_templates = templates.list(scope)
if expected_sha256 and any(
row["id"] == template_id_for(
scope, expected_sha256, version, os_family
)
for row in existing_templates):
raise TemplateError("此云镜像已经导入", 409)
if any(row["name"].casefold() == name.casefold()
for row in existing_templates):
raise TemplateError("当前宿主机已有同名云镜像模板", 409)
if (scope, pending_key) in pending_templates:
raise TemplateError("该云镜像正在导入", 409)
if pending_templates:
raise TemplateError("已有云镜像正在导入,请等待完成后再提交", 409)
job_id = str(uuid.uuid4())
pending_templates.add((scope, pending_key))
import_jobs[job_id] = {"scope": scope, "status": "running"}
save_job("import_job", job_id, import_jobs[job_id])
try:
import_executor.submit(run_template_import, job_id, profile,
scope, body, pending_key)
except RuntimeError as exc:
pending_templates.discard((scope, pending_key))
import_jobs.pop(job_id, None)
delete_job("import_job", job_id)
raise TemplateError("面板正在停止,无法开始导入任务", 503) from exc
LOG.info("cloud template import queued: job=%s admin=%s", job_id, _session.username)
return {"job_id": job_id, "status": "running"}
@app.patch("/api/v1/vm-templates/{template_id}")
def rename_template(template_id: str, body: RenameTemplateBody,
_session: Session = Depends(require_admin_write)):
if shared_cloud_template_runtime is not None:
forbid_long_job_during_update()
return {"template": shared_cloud_template_runtime.rename(
template_id, body.name)}
scope = template_scope(current_profile())
with import_lock:
if pending_templates:
raise TemplateError("云镜像正在导入,请稍后修改模板名称", 409)
return {"template": templates.rename(scope, template_id, body.name)}
@app.get("/api/v1/vm-templates/import-jobs/{job_id}")
def import_job(job_id: str, _session: Session = Depends(require_admin)):
if shared_cloud_template_runtime is not None:
return shared_cloud_template_runtime.get_job(job_id)
scope = template_scope(current_profile())
with import_lock:
job = import_jobs.get(job_id)
if job is None or job["scope"] != scope:
raise TemplateError("导入任务不存在", 404)
return {key: value for key, value in job.items() if key != "scope"}
@app.get("/api/v1/rescue-templates")
def list_rescue_templates(_session: Session = Depends(require_approved)):
return {"templates": rescue_catalog.list(rescue_scope(current_profile()))}
def run_rescue_import(job_id: str, profile: dict, scope: str,
record: dict, image_path: Path) -> None:
uploaded = False
cataloged = False
try:
with rescue_import_lock:
rescue_import_jobs[job_id]["status"] = "running"
rescue_import_jobs[job_id]["phase"] = "正在传输 ISO 到宿主机"
save_job("rescue_import_job", job_id, rescue_import_jobs[job_id])
new_service(profile).import_rescue_iso(image_path, record["volume_name"])
uploaded = True
rescue_catalog.add(scope, record)
cataloged = True
with rescue_import_lock:
rescue_import_jobs[job_id] = {
"scope": scope, "status": "completed", "phase": "救援模版已导入",
"template": record,
}
save_job("rescue_import_job", job_id, rescue_import_jobs[job_id])
except Exception as exc:
if isinstance(exc, (TemplateError, VMError)):
error = str(exc)
else:
LOG.exception("Rescue ISO import failed: job=%s", job_id)
error = "救援 ISO 导入失败,请查看面板日志"
if cataloged:
try:
rescue_catalog.delete(scope, record["id"])
except Exception:
LOG.exception("Could not clean failed rescue catalog entry: job=%s", job_id)
error += ";数据库中可能留有救援模版记录,请管理员检查"
if uploaded:
try:
new_service(profile).delete_rescue_iso(record["volume_name"])
except Exception:
LOG.exception("Could not clean failed rescue ISO import: job=%s", job_id)
error += ";宿主机上可能留有救援 ISO 卷,请管理员检查"
with rescue_import_lock:
rescue_import_jobs[job_id] = {
"scope": scope, "status": "failed", "phase": "导入失败", "error": error,
}
try:
save_job("rescue_import_job", job_id, rescue_import_jobs[job_id])
except DatabaseError:
LOG.exception("Could not persist rescue import failure: job=%s", job_id)
finally:
image_path.unlink(missing_ok=True)
with rescue_import_lock:
pending_rescue_imports.discard(scope)
@app.post("/api/v1/rescue-templates", status_code=202)
async def upload_rescue_template(
request: Request,
name: str = Query(min_length=1, max_length=64),
_session: Session = Depends(require_admin_write),
):
name = validate_template_name(name)
profile = current_profile()
scope = rescue_scope(profile)
with rescue_import_lock:
forbid_long_job_during_update()
if scope in pending_rescue_imports:
raise TemplateError("已有救援 ISO 正在导入,请等待完成", 409)
if any(row["name"].casefold() == name.casefold()
for row in rescue_catalog.list(scope)):
raise TemplateError("当前宿主机已有同名救援模版", 409)
pending_rescue_imports.add(scope)
image_path: Path | None = None
handed_off = False
try:
declared_size = request.headers.get("content-length")
if declared_size is not None:
try:
if int(declared_size) > MAX_RESCUE_ISO_BYTES:
raise TemplateError("救援 ISO 超过 8 GiB 限制", 413)
except ValueError as exc:
raise TemplateError("上传长度无效", 400) from exc
fd, raw_path = tempfile.mkstemp(prefix=".rescue-upload-", suffix=".iso",
dir=settings.data_dir)
image_path = Path(raw_path)
os.fchmod(fd, 0o600)
checksum = hashlib.sha256()
size = 0
with os.fdopen(fd, "wb") as output:
async for chunk in request.stream():
size += len(chunk)
if size > MAX_RESCUE_ISO_BYTES:
raise TemplateError("救援 ISO 超过 8 GiB 限制", 413)
output.write(chunk)
checksum.update(chunk)
output.flush()
os.fsync(output.fileno())
if internal_mode:
validation = request.scope.get("engine_upload_validation", {})
if not validation.get("complete") or not validation.get("valid"):
raise HTTPException(status_code=401, detail="救援 ISO 摘要与签名不匹配")
verify_bootable_iso(image_path)
template_id = uuid.uuid4().hex
record = {
"id": template_id, "name": name,
"volume_name": rescue_volume_name(template_id),
"size_bytes": size, "sha256": checksum.hexdigest(),
}
job_id = str(uuid.uuid4())
with rescue_import_lock:
rescue_import_jobs[job_id] = {
"scope": scope, "status": "queued", "phase": "等待传输到宿主机",
}
save_job("rescue_import_job", job_id, rescue_import_jobs[job_id])
try:
rescue_import_executor.submit(
run_rescue_import, job_id, profile, scope, record, image_path,
)
except RuntimeError as exc:
rescue_import_jobs.pop(job_id, None)
delete_job("rescue_import_job", job_id)
raise TemplateError("面板正在停止,无法导入救援 ISO", 503) from exc
LOG.info("rescue ISO import queued: job=%s admin=%s", job_id, _session.username)
handed_off = True # The worker now owns and removes the temporary file.
return {"job_id": job_id, "status": "queued"}
finally:
if not handed_off:
if image_path is not None:
image_path.unlink(missing_ok=True)
with rescue_import_lock:
pending_rescue_imports.discard(scope)
@app.get("/api/v1/rescue-templates/import-jobs/{job_id}")
def rescue_import_job(job_id: str, _session: Session = Depends(require_admin)):
scope = rescue_scope(current_profile())
with rescue_import_lock:
job = rescue_import_jobs.get(job_id)
if job is None or job["scope"] != scope:
raise TemplateError("救援模版导入任务不存在", 404)
return {key: value for key, value in job.items() if key != "scope"}
@app.delete("/api/v1/rescue-templates/{template_id}")
def delete_rescue_template(template_id: str,
_session: Session = Depends(require_admin_write)):
profile = current_profile()
scope = rescue_scope(profile)
with rescue_lock, rescue_import_lock:
forbid_long_job_during_update()
if scope in pending_rescue_imports:
raise TemplateError("救援 ISO 正在导入,请稍后删除", 409)
record = rescue_catalog.get(scope, template_id)
states = (data_store.list("vm_rescue") if data_store is not None
else test_rescue_states.copy())
for raw in states.values():
try:
state_record = json.loads(raw)
except (TypeError, ValueError) as exc:
raise DatabaseError("虚拟机救援记录损坏") from exc
if not isinstance(state_record, dict):
raise DatabaseError("虚拟机救援记录损坏")
if state_record.get("volume_name") == record["volume_name"]:
raise TemplateError("此救援模版正被虚拟机使用,须先退出救援模式", 409)
try:
new_service(profile).delete_rescue_iso(record["volume_name"])
except VMError as exc:
# Only the target volume's precise absence is idempotent.
# A missing/misconfigured pool can also be a 404, but must
# preserve the catalog entry so the administrator can retry.
if (exc.status != 404
or exc.error_code != "rescue_volume_not_found"):
raise
rescue_catalog.delete(scope, template_id)
LOG.info("rescue ISO deleted: id=%s admin=%s", template_id, _session.username)
return {"deleted": True}
@app.get("/api/v1/vms")
def list_vms(_session: Session = Depends(require_approved)):
profile = current_profile()
vms = new_service(profile).list_vms()
if database_mode:
account = account_for_session(_session)
if not account["admin"]:
owned_ids = state.users.list_owned_vm_ids(host_scope(profile), account["id"])
vms = [vm for vm in vms if vm["uuid"] in owned_ids]
records = (data_store.list("vm_credential") if data_store is not None
else test_credentials.copy())
return {"items": [decorate_vm(
vm, parse_credential(records.get(credential_key(profile, vm["uuid"]))), profile,
) for vm in vms]}
@app.get("/api/v1/vm-states")
def list_vm_states(
ids: str | None = Query(None, max_length=4096),
_session: Session = Depends(require_approved),
):
profile = current_profile()
if ids is None:
vms = new_service(profile).list_vm_states()
if database_mode:
account = account_for_session(_session)
if not account["admin"]:
owned_ids = state.users.list_owned_vm_ids(host_scope(profile), account["id"])
vms = [vm for vm in vms if vm["uuid"] in owned_ids]
return {"items": vms}
raw_ids = ids.split(",")
if not 1 <= len(raw_ids) <= 100 or any(not value for value in raw_ids):
raise VMError("虚拟机状态查询数量必须为 1 到 100", 400)
canonical_ids: list[str] = []
seen: set[str] = set()
for value in raw_ids:
try:
canonical = str(uuid.UUID(value))
except (TypeError, ValueError, AttributeError) as exc:
raise VMError("虚拟机 UUID 无效", 400) from exc
if canonical not in seen:
canonical_ids.append(canonical)
seen.add(canonical)
if database_mode:
account = account_for_session(_session)
if not account["admin"]:
owned_ids = state.users.list_owned_vm_ids(
host_scope(profile), account["id"]
)
if any(vm_id not in owned_ids for vm_id in canonical_ids):
raise VMError("虚拟机不存在", 404)
return {"items": new_service(profile).list_vm_states(canonical_ids)}
@app.get("/api/v1/vm-metadata")
def list_vm_metadata(
ids: str = Query(..., min_length=1, max_length=4096),
_session: Session = Depends(require_approved),
):
raw_ids = ids.split(",")
if not 1 <= len(raw_ids) <= 100 or any(not value for value in raw_ids):
raise VMError("虚拟机元数据查询数量必须为 1 到 100", 400)
canonical_ids: list[str] = []
seen: set[str] = set()
for value in raw_ids:
try:
canonical = str(uuid.UUID(value))
except (TypeError, ValueError, AttributeError) as exc:
raise VMError("虚拟机 UUID 无效", 400) from exc
if canonical not in seen:
canonical_ids.append(canonical)
seen.add(canonical)
profile = current_profile()
if database_mode:
account = account_for_session(_session)
if not account["admin"]:
owned_ids = state.users.list_owned_vm_ids(
host_scope(profile), account["id"]
)
if any(vm_id not in owned_ids for vm_id in canonical_ids):
# Reject the complete request rather than disclosing which
# UUID in a mixed-owner batch exists on this host.
raise VMError("虚拟机不存在", 404)
return {"items": new_service(profile).list_vm_metadata(canonical_ids)}
@app.get("/api/v1/vms/{vm_id}/state")
def get_vm_state(vm_id: str, _session: Session = Depends(require_approved)):
profile = current_profile()
require_vm_access(_session, profile, vm_id)
return new_service(profile).get_vm_state(vm_id)
@app.get("/api/v1/vms/{vm_id}")
def get_vm(vm_id: str, _session: Session = Depends(require_approved)):
profile = current_profile()
require_vm_access(_session, profile, vm_id)
vm = new_service(profile).get_vm(vm_id)
return decorate_vm(vm, get_credential(profile, vm["uuid"]), profile)
@app.get("/api/v1/vms/{vm_id}/firewall")
def get_vm_firewall(
vm_id: str,
_session: Session = Depends(require_approved),
):
profile = current_profile()
canonical_id = require_vm_access(_session, profile, vm_id)
observed = new_service(profile).get_firewall_policy(canonical_id)
record = load_firewall_record(profile, canonical_id)
record_free_default = (
record is None
and (
observed.get("filter_mode") == "baseline"
or is_default_platform_firewall(canonical_id, observed)
)
)
if record_free_default:
response = {
"enabled": False,
"revision": 0,
"policy_digest": firewall_policy_digest(False, []),
"filter_digest": observed.get("filter_digest"),
"rules": [],
"sync_state": observed.get("sync_state", "active"),
}
if observed.get("filter_mode") == "managed":
response["filter_version"] = FIREWALL_FILTER_COMPILER_VERSION
return response
applied = bool(
record is not None
and observed.get("filter_mode") == "managed"
and observed.get("sync_state") == "active"
and observed.get("filter_digest") == record["filter_digest"]
and observed.get("platform_policy_digest")
== record["platform_policy_digest"]
)
return {
"enabled": record["enabled"] if applied else observed.get("enabled"),
"revision": record["revision"] if applied else None,
"policy_digest": record["policy_digest"] if applied else None,
"filter_digest": observed.get("filter_digest"),
"rules": record["rules"] if applied else [],
"sync_state": "active" if applied else "drifted",
}
@app.put("/api/v1/vms/{vm_id}/firewall")
def put_vm_firewall(
vm_id: str,
body: FirewallPolicyBody,
_session: Session = Depends(require_internal_service_write),
):
"""Apply one complete typed IPv4 policy; raw nwfilter XML is forbidden."""
forbid_long_job_during_update()
profile = current_profile()
canonical_id = require_vm_access(_session, profile, vm_id)
vm_scope = host_scope(profile)
# These locks protect only the short conflict check. The remote
# libvirt call runs without any host-global lock; EngineApplication's
# per-VM target lock serializes conflicting mutations for this VM.
with reinstall_lock, rescue_lock:
if (template_scope(profile), canonical_id) in pending_reinstalls:
raise VMError("虚拟机正在重装系统,暂时不能修改防火墙", 409)
if ((vm_scope, canonical_id) in pending_rescues
or load_rescue_state(profile, canonical_id) is not None):
raise VMError("虚拟机处于救援模式或切换中,暂时不能修改防火墙", 409)
requested = body.model_dump()
requested_rules = requested["rules"]
requested_digest = firewall_policy_digest(
requested["enabled"], requested_rules)
service = new_service(profile)
observed = service.get_firewall_policy(canonical_id)
try:
platform_policy = normalize_platform_policy_snapshot(
observed.get("platform_policy_snapshot")
or default_platform_policy_snapshot())
except ValueError as exc:
raise VMError(
"虚拟机平台网络策略证据无效", 409,
error_code="firewall_drift_detected",
) from exc
current_filter_digest = build_firewall_filter_xml(
canonical_id, requested["enabled"], requested_rules,
platform_policy_snapshot=platform_policy)[3]
if not hmac.compare_digest(
requested["policy_digest"], requested_digest):
raise VMError(
"防火墙策略摘要与规范化规则不一致", 422,
error_code="firewall_rule_invalid",
)
previous = load_firewall_record(profile, canonical_id)
if previous is not None:
if not hmac.compare_digest(
previous["platform_policy_digest"],
platform_policy["digest"]):
raise VMError(
"虚拟机平台网络策略与引擎记录不一致",
409,
error_code="firewall_drift_detected",
)
if requested["revision"] < previous["revision"]:
raise VMError("防火墙策略版本已更新,请刷新后重试", 409)
if (requested["revision"] == previous["revision"]
and requested_digest != previous["policy_digest"]):
raise VMError("相同防火墙版本对应了不同规则", 409)
if requested["revision"] == previous["revision"]:
observed_filter_digest = observed.get("filter_digest")
if (observed.get("filter_mode") == "managed"
and observed.get("sync_state") == "active"
and isinstance(observed_filter_digest, str)
and hmac.compare_digest(
observed_filter_digest, current_filter_digest)):
if (previous["filter_version"]
!= FIREWALL_FILTER_COMPILER_VERSION
or not hmac.compare_digest(
previous["filter_digest"],
current_filter_digest,
)):
previous = {
**previous,
"version": 2,
"filter_version": FIREWALL_FILTER_COMPILER_VERSION,
"filter_digest": current_filter_digest,
"platform_policy_snapshot": platform_policy,
"platform_policy_digest": platform_policy["digest"],
"applied_at": int(time.time()),
}
save_firewall_record(profile, canonical_id, previous)
return {
"enabled": previous["enabled"],
"revision": previous["revision"],
"policy_digest": previous["policy_digest"],
"filter_digest": current_filter_digest,
"filter_version": FIREWALL_FILTER_COMPILER_VERSION,
"rules": previous["rules"],
"sync_state": "active",
}
legacy_compiler_match = (
previous["filter_version"]
< FIREWALL_FILTER_COMPILER_VERSION
and observed.get("filter_mode") == "managed"
and observed.get("sync_state") == "active"
and isinstance(observed_filter_digest, str)
and hmac.compare_digest(
observed_filter_digest, previous["filter_digest"])
)
if not legacy_compiler_match:
raise VMError(
"防火墙实际状态与已保存版本不一致,请先核查漂移",
409,
error_code="firewall_drift_detected",
)
result = service.apply_firewall_policy(
canonical_id,
enabled=requested["enabled"],
rules=requested_rules,
)
if result.get("policy_digest") != requested_digest:
raise VMError(
"防火墙策略摘要校验失败,结果待核对", 503,
error_code="firewall_result_uncertain",
)
if result.get("platform_policy_digest") != platform_policy["digest"]:
raise VMError(
"平台网络策略摘要校验失败,结果待核对", 503,
error_code="firewall_result_uncertain",
)
record = {
"version": 2,
"filter_version": FIREWALL_FILTER_COMPILER_VERSION,
"vm_id": canonical_id,
"enabled": requested["enabled"],
"revision": requested["revision"],
"rules": result["rules"],
"policy_digest": result["policy_digest"],
"filter_digest": result["filter_digest"],
"platform_policy_snapshot": platform_policy,
"platform_policy_digest": platform_policy["digest"],
"applied_at": int(time.time()),
}
try:
save_firewall_record(profile, canonical_id, record)
except Exception as exc:
LOG.exception(
"Firewall applied but local evidence could not be saved: uuid=%s",
canonical_id,
)
raise VMError(
"防火墙已下发但引擎记录未确认,请查询原操作", 503,
error_code="firewall_result_uncertain",
) from exc
LOG.info(
"vm firewall applied: uuid=%s revision=%s enabled=%s actor=%s",
canonical_id, body.revision, body.enabled, _session.username,
)
return {
"enabled": record["enabled"],
"revision": record["revision"],
"policy_digest": record["policy_digest"],
"filter_digest": record["filter_digest"],
"filter_version": record["filter_version"],
"rules": record["rules"],
"sync_state": "active",
}
def run_snapshot_job(job_id: str, profile: dict) -> None:
"""Run one host-local snapshot mutation outside the HTTP request."""
job = None
try:
with snapshot_lock:
job = snapshot_jobs[job_id]
if job.get("cancelled_before_start") is True:
try:
finish_cancelled_snapshot_job(job_id, job)
except Exception:
LOG.exception(
"Could not finish cancelled snapshot in runner: %s",
job_id,
)
snapshot_futures.pop(job_id, None)
return
if job.get("status") != "queued":
# A submitted callable may only cross the libvirt boundary
# after this queued -> running transition. Terminal or
# otherwise claimed jobs are never replayed here.
if job.get("status") in {
"failed", "succeeded", "completed", "uncertain"}:
pending_snapshot_jobs.discard(
(host_scope(profile), job.get("vm_id")))
snapshot_futures.pop(job_id, None)
return
job["status"] = "running"
job["phase"] = {
"create": "正在建立在线快照时间点",
"restore": "正在构建快照恢复系统盘",
"delete": "正在安全删除快照",
}[job["action"]]
job["started_at"] = int(time.time())
save_job("snapshot_job", job_id, job)
except Exception:
LOG.exception("Could not start snapshot job safely: %s", job_id)
rollback_complete = False
try:
if job is None:
raise KeyError(job_id)
record = get_snapshot_record(job["vm_id"], job["snapshot_key"])
if job["action"] == "create":
delete_snapshot_record(job["vm_id"], job["snapshot_key"])
else:
record["state"] = "ready"
save_snapshot_record(record)
rollback_complete = True
except Exception:
LOG.exception("Could not roll back unstarted snapshot job: %s", job_id)
with snapshot_lock:
current = snapshot_jobs.get(job_id)
if current is not None:
current.update(
status=("failed" if rollback_complete else "uncertain"),
phase=("快照任务未启动" if rollback_complete
else "快照登记回滚结果待核查"),
error=(
"快照任务状态无法持久化,尚未调用虚拟化操作"
if rollback_complete else
"快照任务尚未调用虚拟化操作,但登记状态回滚无法确认"
),
finished_at=int(time.time()),
)
try:
save_job("snapshot_job", job_id, current)
except DatabaseError:
LOG.exception("Could not persist unstarted snapshot failure: %s", job_id)
if isinstance(job, dict) and isinstance(job.get("vm_id"), str):
pending_snapshot_jobs.discard(
(host_scope(profile), job["vm_id"]))
snapshot_futures.pop(job_id, None)
return
def report_progress(phase: str) -> None:
labels = {
"freezing": "正在通过 QEMU Guest Agent 冻结访客文件系统",
"thawed": "访客文件系统已确认解冻",
"copying": "正在复制在线快照数据",
"verifying": "正在验证独立快照卷",
"cloning": "正在构建快照恢复系统盘",
"shutting_down": "正在正常关闭虚拟机以切换系统盘",
"switching": "正在切换快照恢复系统盘",
"starting": "正在从快照恢复系统盘启动",
}
label = labels.get(phase)
if label is None:
return
with snapshot_lock:
current = snapshot_jobs.get(job_id)
if current is not None and current.get("status") == "running":
current["phase"] = label
if phase == "freezing":
current["guest_freeze_state"] = "attempted"
elif phase == "thawed":
current["guest_freeze_state"] = "thawed"
save_job("snapshot_job", job_id, current)
action = job["action"]
vm_id = job["vm_id"]
snapshot_key = job["snapshot_key"]
uncertain = False
physical_committed = False
committed_record = None
try:
service = new_service(profile)
if action == "create":
result = service.create_live_snapshot(
vm_id, snapshot_key,
consistency=job["consistency"], progress=report_progress)
physical_committed = True
verified_capacity = result.get("size_bytes")
verified_identity = result.get("volume_identity")
if (isinstance(verified_capacity, bool)
or not isinstance(verified_capacity, int)
or verified_capacity <= 0
or not isinstance(verified_identity, str)
or re.fullmatch(r"[0-9a-f]{64}", verified_identity) is None):
raise VMSnapshotUncertainError(
"快照复制完成,但卷验证标记无效")
with snapshot_lock:
current = snapshot_jobs[job_id]
current["backup_verified_capacity"] = verified_capacity
current["backup_verified_identity"] = verified_identity
save_job("snapshot_job", job_id, current)
try:
service.release_live_snapshot_completed_evidence(vm_id)
except Exception:
# The durable marker is now authoritative. Failure to
# prune retained libvirt stats must not erase success.
LOG.exception(
"Could not release retained snapshot evidence: job=%s",
job_id,
)
record = get_snapshot_record(vm_id, snapshot_key)
committed_record = record
if result.get("volume_name") != record["volume_name"]:
raise VMSnapshotUncertainError(
"快照复制完成,但存储卷身份与登记记录不一致")
now = int(time.time())
record.update(
state="ready",
completed_at=now,
size_bytes=int(result["size_bytes"]),
)
save_snapshot_record(record)
public_result = {
"snapshot_key": snapshot_key,
"created": True,
"consistency": record["consistency"],
"created_at": record["created_at"],
"size_bytes": record["size_bytes"],
"power_state": result.get("power_state", "running"),
}
elif action == "restore":
record = get_snapshot_record(vm_id, snapshot_key)
committed_record = record
plan = service.snapshot_restore_plan(
vm_id, job["operation_id"])
with snapshot_lock:
current = snapshot_jobs[job_id]
current["restore_old_volume"] = plan["old_volume_name"]
current["restore_replacement_volume"] = (
plan["replacement_volume_name"])
current["restore_was_running"] = plan["was_running"]
save_job("snapshot_job", job_id, current)
result = service.restore_live_snapshot(
vm_id, snapshot_key, record["volume_name"],
job["operation_id"],
expected_old_volume=plan["old_volume_name"],
progress=report_progress)
physical_committed = True
now = int(time.time())
record.update(state="ready", last_restored_at=now)
save_snapshot_record(record)
public_result = {
"snapshot_key": snapshot_key,
"restored": True,
"consistency": record["consistency"],
"power_state": result.get("power_state", "stopped"),
}
if isinstance(result.get("warning"), str):
public_result["warning"] = result["warning"]
else:
record = get_snapshot_record(vm_id, snapshot_key)
committed_record = record
service.delete_snapshot_volume(
vm_id, snapshot_key, record["volume_name"])
physical_committed = True
# Physical deletion commits before the encrypted registry row
# is removed. A registry failure therefore remains uncertain,
# never a retryable failed DELETE.
delete_snapshot_record(vm_id, snapshot_key)
public_result = {
"snapshot_key": snapshot_key,
"deleted": True,
"consistency": record["consistency"],
}
with snapshot_lock:
current = snapshot_jobs[job_id]
current.update(
status="succeeded",
phase={
"create": "在线快照创建完成",
"restore": "快照恢复完成",
"delete": "快照删除完成",
}[action],
result=public_result,
finished_at=int(time.time()),
)
save_job("snapshot_job", job_id, current)
LOG.info(
"vm snapshot %s succeeded: job=%s uuid=%s snapshot=%s",
action, job_id, vm_id, snapshot_key)
except Exception as exc:
uncertain = (physical_committed
or isinstance(exc, VMSnapshotUncertainError))
if not isinstance(exc, (VMError, DatabaseError, ValueError)):
uncertain = True
LOG.exception(
"VM snapshot %s failed unexpectedly: job=%s uuid=%s",
action, job_id, vm_id)
error = (str(exc) if isinstance(exc, (VMError, DatabaseError, ValueError))
else "快照任务出现未确认错误,请管理员核查引擎日志和宿主机存储")
error_code = (exc.error_code if isinstance(exc, VMError)
and exc.error_code in SNAPSHOT_PUBLIC_ERROR_CODES
else None)
try:
record = (committed_record if committed_record is not None
else get_snapshot_record(vm_id, snapshot_key))
if uncertain:
record["state"] = "uncertain"
save_snapshot_record(record)
elif action == "create":
delete_snapshot_record(vm_id, snapshot_key)
else:
record["state"] = "ready"
save_snapshot_record(record)
except Exception:
uncertain = True
LOG.exception("Could not reconcile snapshot record after job failure: %s", job_id)
with snapshot_lock:
current = snapshot_jobs[job_id]
current.update(
status="uncertain" if uncertain else "failed",
phase="快照结果待核查" if uncertain else "快照任务失败",
error=("快照结果不确定,不能重复提交;请管理员核查宿主机存储和虚拟机定义"
if uncertain else error),
finished_at=int(time.time()),
)
if error_code is not None:
current["error_code"] = error_code
try:
save_job("snapshot_job", job_id, current)
except DatabaseError:
LOG.exception("Could not persist snapshot job failure: %s", job_id)
finally:
with snapshot_lock:
pending_snapshot_jobs.discard((host_scope(profile), vm_id))
snapshot_futures.pop(job_id, None)
def reconcile_snapshot_job(job_id: str) -> dict:
"""Safely converge one uncertain job from read-only host evidence."""
with snapshot_lock:
current = snapshot_jobs.get(job_id)
if current is None and data_store is not None:
raw = data_store.get("snapshot_job", job_id)
if raw is not None:
try:
persisted = json.loads(raw)
except (TypeError, ValueError):
persisted = None
if isinstance(persisted, dict):
with snapshot_lock:
current = snapshot_jobs.setdefault(job_id, persisted)
with snapshot_lock:
current = snapshot_jobs.get(job_id)
if current is None:
raise VMError("快照任务不存在", 404)
if current.get("status") != "uncertain":
return dict(current)
job = dict(current)
profile = manager.current()
if profile is None or profile.get("key_locked"):
return job
try:
record = None
try:
record = get_snapshot_record(
job["vm_id"], job["snapshot_key"])
except VMError as exc:
if job.get("action") != "delete" or exc.status != 404:
raise
service = new_service(profile)
if (job.get("action") == "create"
and job.get("consistency") == "filesystem"
and job.get("guest_freeze_state") == "attempted"):
service.thaw_guest_filesystems(job["vm_id"])
with snapshot_lock:
current = snapshot_jobs.get(job_id)
if current is None or current.get("status") != "uncertain":
return dict(current or job)
current["guest_freeze_state"] = "recovered"
current["freeze_recovery"] = "confirmed"
current["freeze_recovered_at"] = int(time.time())
save_job("snapshot_job", job_id, current)
job = dict(current)
if job["action"] == "create":
if record is None:
return job
evidence = service.reconcile_live_snapshot_create(
job["vm_id"], job["snapshot_key"],
record["volume_name"],
verified_capacity=job.get("backup_verified_capacity"),
verified_identity=job.get("backup_verified_identity"),
)
elif job["action"] == "delete":
snapshot_volume = (
record["volume_name"] if record is not None
else snapshot_volume_name(
job["vm_id"], job["snapshot_key"])
)
evidence = service.reconcile_live_snapshot_delete(
job["vm_id"], job["snapshot_key"],
snapshot_volume)
else:
if record is None:
return job
old_volume = job.get("restore_old_volume")
was_running = job.get("restore_was_running")
replacement_volume = job.get("restore_replacement_volume")
if (old_volume is None and was_running is None
and replacement_volume is None):
# The runner persists all three plan fields atomically
# before calling libvirt. Their complete absence proves
# restore never crossed its physical mutation boundary.
evidence = {
"outcome": "failed",
"snapshot_key": job["snapshot_key"],
}
elif (not isinstance(old_volume, str)
or not isinstance(was_running, bool)
or not isinstance(replacement_volume, str)):
return job
else:
evidence = service.reconcile_live_snapshot_restore(
job["vm_id"], job["snapshot_key"],
record["volume_name"], job["operation_id"], old_volume,
was_running)
except (VMError, DatabaseError, KeyError, ValueError):
# An unavailable host, damaged evidence, or missing private
# restore tombstone is not permission to guess a terminal result.
return job
outcome = evidence.get("outcome")
if outcome not in {"succeeded", "failed"}:
return job
now = int(time.time())
if outcome == "succeeded":
if job["action"] == "create":
record.update(
state="ready",
completed_at=now,
size_bytes=int(evidence["size_bytes"]),
)
save_snapshot_record(record)
public_result = {
"snapshot_key": job["snapshot_key"],
"created": True,
"consistency": record["consistency"],
"created_at": record["created_at"],
"size_bytes": record["size_bytes"],
"power_state": evidence.get("power_state", "stopped"),
}
elif job["action"] == "restore":
record.update(state="ready", last_restored_at=now)
save_snapshot_record(record)
public_result = {
"snapshot_key": job["snapshot_key"],
"restored": True,
"consistency": record["consistency"],
"power_state": evidence.get("power_state", "stopped"),
}
else:
if record is not None:
delete_snapshot_record(job["vm_id"], job["snapshot_key"])
public_result = {
"snapshot_key": job["snapshot_key"],
"deleted": True,
"consistency": job["consistency"],
}
else:
if record is None:
return job
if job["action"] == "create":
# Reconciliation proved that no usable recovery point was
# created. Preserve the durable job audit, but remove the
# hidden catalog row so it cannot accumulate outside the
# two-snapshot UI and later block VM expiry deletion.
delete_snapshot_record(job["vm_id"], job["snapshot_key"])
else:
record["state"] = "ready"
save_snapshot_record(record)
public_result = None
with snapshot_lock:
current = snapshot_jobs.get(job_id)
if current is None or current.get("status") != "uncertain":
return dict(current or job)
current.update(
status=outcome,
phase=("快照自动核对完成" if outcome == "succeeded"
else "快照自动核对确认未生效"),
finished_at=now,
)
if public_result is not None:
current["result"] = public_result
current.pop("error", None)
current.pop("error_code", None)
else:
current.pop("result", None)
current["error"] = "快照操作经宿主机证据确认未生效"
current.pop("error_code", None)
save_job("snapshot_job", job_id, current)
saved = dict(current)
if job["action"] == "create":
try:
service.release_live_snapshot_completed_evidence(job["vm_id"])
except Exception:
LOG.exception(
"Could not release reconciled snapshot evidence: job=%s",
job_id,
)
return saved
app.state.reconcile_snapshot_job = reconcile_snapshot_job
def snapshot_job_state(job_id: str) -> dict | None:
with snapshot_lock:
current = snapshot_jobs.get(job_id)
if (current is not None
and current.get("status") == "queued"
and current.get("cancelled_before_start") is True):
try:
current = finish_cancelled_snapshot_job(job_id, current)
except Exception:
LOG.exception(
"Could not finish cancelled snapshot during poll: %s",
job_id,
)
return dict(current) if current is not None else None
app.state.snapshot_job_state = snapshot_job_state
def queue_snapshot_job(
*, profile: dict, owner: str, vm_id: str, snapshot_key: str,
action: Literal["create", "restore", "delete"], consistency: str,
) -> dict:
job_id = str(uuid.uuid4())
delegated = TRUSTED_ACTOR.get() if internal_mode else None
operation_id = (
delegated.get("operation_id")
if isinstance(delegated, dict) and delegated.get("operation_id")
else job_id
)
job = {
"job_id": job_id,
"operation_id": operation_id,
"scope": host_scope(profile),
"owner": owner,
"vm_id": vm_id,
"snapshot_key": snapshot_key,
"action": action,
"consistency": consistency,
"status": "queued",
"phase": "等待快照任务开始",
"created_at": int(time.time()),
}
# Do not publish an in-memory queued job until its durable identity
# exists. Otherwise an initial database failure leaves a ghost with no
# Future that can permanently block fail-safe network isolation.
save_job("snapshot_job", job_id, job)
snapshot_jobs[job_id] = job
pending_snapshot_jobs.add((job["scope"], vm_id))
try:
future = snapshot_executor.submit(
run_snapshot_job, job_id, profile)
snapshot_futures[job_id] = future
except RuntimeError as exc:
pending_snapshot_jobs.discard((job["scope"], vm_id))
terminal = {
**job,
"status": "failed",
"phase": "快照任务未进入执行器",
"error": "快照任务在开始前失败,未调用虚拟化操作",
"failed_before_start": True,
"finished_at": int(time.time()),
}
snapshot_jobs[job_id] = terminal
try:
save_job("snapshot_job", job_id, terminal)
except DatabaseError:
# Keep the in-memory terminal proof. Operation polling compares
# it with the durable queued row and retries persistence; the
# original mutation is never submitted or replayed.
LOG.exception(
"Could not persist pre-start snapshot submit failure: %s",
job_id,
)
raise VMError("引擎正在停止,无法开始快照任务", 503) from exc
return {"job_id": job_id, "status": "queued"}
@app.get("/api/v1/vms/{vm_id}/snapshots")
def list_vm_snapshots(
vm_id: str, _session: Session = Depends(require_approved),
):
profile = current_profile()
canonical_id = require_vm_access(_session, profile, vm_id)
return {"items": [
public_snapshot_record(record)
for record in list_snapshot_records(canonical_id)
], "limit": 2}
@app.get("/api/v1/vms/{vm_id}/snapshots/{snapshot_key}")
def get_vm_snapshot(
vm_id: str, snapshot_key: str,
_session: Session = Depends(require_approved),
):
profile = current_profile()
canonical_id = require_vm_access(_session, profile, vm_id)
return public_snapshot_record(
get_snapshot_record(canonical_id, snapshot_key))
@app.post("/api/v1/vms/{vm_id}/snapshots", status_code=202)
def create_vm_snapshot(
vm_id: str, body: CreateSnapshotBody,
_session: Session = Depends(require_approved_write),
):
preflight_profile = current_profile()
preflight_id = require_vm_access(
_session, preflight_profile, vm_id)
forbid_snapshot_suspend_barrier(
preflight_profile, preflight_id, "创建快照")
with reinstall_lock, rescue_lock, snapshot_lock:
forbid_long_job_during_update()
profile = current_profile()
canonical_id = require_vm_access(_session, profile, vm_id)
scope = host_scope(profile)
forbid_pending_snapshot(profile, canonical_id, "创建快照")
if ((template_scope(profile), canonical_id) in pending_reinstalls
or (scope, canonical_id) in pending_rescues):
raise VMError("虚拟机正在执行重装或救援任务,不能创建快照", 409)
service = new_service(profile)
vm = service.get_vm(canonical_id)
if (not vm.get("managed")
or vm.get("os_source") != "cloud_template"):
raise VMError("只有面板创建的云镜像虚拟机支持本地快照", 409)
if (vm.get("rescue_mode") or vm.get("rescue_staged")):
raise VMError("虚拟机处于救援模式,不能创建快照", 409)
records = list_snapshot_records(canonical_id)
if len([row for row in records if row["state"] != "failed"]) >= 2:
raise VMError("每台虚拟机最多保留两个本地快照", 409)
record = {
"snapshot_key": body.snapshot_key,
"vm_id": canonical_id,
"name": body.name,
"state": "creating",
"consistency": body.consistency_mode,
"volume_name": snapshot_volume_name(
canonical_id, body.snapshot_key),
"created_at": int(time.time()),
}
if not insert_snapshot_record(record):
raise VMError("快照标识已存在,不能重复创建", 409)
try:
queued = queue_snapshot_job(
profile=profile, owner=_session.username,
vm_id=canonical_id, snapshot_key=body.snapshot_key,
action="create", consistency=body.consistency_mode)
except Exception:
delete_snapshot_record(canonical_id, body.snapshot_key)
raise
return queued
@app.post(
"/api/v1/vms/{vm_id}/snapshots/{snapshot_key}/restore",
status_code=202,
)
def restore_vm_snapshot(
vm_id: str, snapshot_key: str, _body: EmptySnapshotBody,
_session: Session = Depends(require_approved_write),
):
preflight_profile = current_profile()
preflight_id = require_vm_access(
_session, preflight_profile, vm_id)
forbid_snapshot_suspend_barrier(
preflight_profile, preflight_id, "恢复快照")
with reinstall_lock, rescue_lock, snapshot_lock:
forbid_long_job_during_update()
profile = current_profile()
canonical_id = require_vm_access(_session, profile, vm_id)
scope = host_scope(profile)
forbid_pending_snapshot(profile, canonical_id, "恢复快照")
if ((template_scope(profile), canonical_id) in pending_reinstalls
or (scope, canonical_id) in pending_rescues):
raise VMError("虚拟机已有快照、重装或救援任务,请等待完成", 409)
record = get_snapshot_record(canonical_id, snapshot_key)
if record["state"] != "ready":
raise VMError("快照当前不可恢复,请刷新状态后重试", 409)
vm = new_service(profile).get_vm(canonical_id)
if vm.get("rescue_mode") or vm.get("rescue_staged"):
raise VMError("虚拟机处于救援模式,不能恢复快照", 409)
record["state"] = "restoring"
save_snapshot_record(record)
try:
queued = queue_snapshot_job(
profile=profile, owner=_session.username,
vm_id=canonical_id, snapshot_key=snapshot_key,
action="restore", consistency=record["consistency"])
except Exception:
record["state"] = "ready"
save_snapshot_record(record)
raise
return queued
@app.delete(
"/api/v1/vms/{vm_id}/snapshots/{snapshot_key}", status_code=202,
)
def delete_vm_snapshot(
vm_id: str, snapshot_key: str, _body: EmptySnapshotBody,
_session: Session = Depends(require_approved_write),
):
preflight_profile = current_profile()
preflight_id = require_vm_access(
_session, preflight_profile, vm_id)
forbid_snapshot_suspend_barrier(
preflight_profile, preflight_id, "删除快照")
with reinstall_lock, rescue_lock, snapshot_lock:
forbid_long_job_during_update()
profile = current_profile()
canonical_id = require_vm_access(_session, profile, vm_id)
scope = host_scope(profile)
forbid_pending_snapshot(profile, canonical_id, "删除快照")
if ((template_scope(profile), canonical_id) in pending_reinstalls
or (scope, canonical_id) in pending_rescues):
raise VMError("虚拟机已有快照、重装或救援任务,请等待完成", 409)
record = get_snapshot_record(canonical_id, snapshot_key)
if record["state"] != "ready":
raise VMError("快照当前不可删除,请刷新状态后重试", 409)
record["state"] = "deleting"
save_snapshot_record(record)
try:
queued = queue_snapshot_job(
profile=profile, owner=_session.username,
vm_id=canonical_id, snapshot_key=snapshot_key,
action="delete", consistency=record["consistency"])
except Exception:
record["state"] = "ready"
save_snapshot_record(record)
raise
return queued
@app.post("/api/v1/vms/{vm_id}/credential")
def reveal_vm_credential(vm_id: str, _session: Session = Depends(require_approved_write)):
profile = current_profile()
require_vm_access(_session, profile, vm_id)
vm = new_service(profile).get_vm(vm_id)
credential = get_credential(profile, vm["uuid"])
decorated = decorate_vm(vm, credential, profile)
LOG.info("vm credential viewed: uuid=%s admin=%s", vm["uuid"], _session.username)
return {
"login_name": decorated["login_name"],
"auth_mode": decorated["auth_mode"],
"password": credential["password"] if decorated["password_available"] else None,
}
@app.get("/api/v1/vms/{vm_id}/metrics")
def get_vm_metrics(
vm_id: str,
range_: Literal["1h", "24h", "7d", "30d"] = Query("1h", alias="range"),
_session: Session = Depends(require_approved),
):
try:
canonical_id = str(uuid.UUID(vm_id))
except (TypeError, ValueError, AttributeError) as exc:
raise VMError("虚拟机 UUID 无效", 400) from exc
profile = current_profile()
require_vm_access(_session, profile, canonical_id)
# Never reveal another host's retained samples for a reused UUID.
new_service(profile).get_vm(canonical_id)
backend = metric_store if metric_store is not None else data_store
if backend is None:
raise HTTPException(status_code=503, detail="指标数据库尚未就绪")
now = int(time.time())
window = RANGES[range_]
interval = query_interval_seconds(window)
since = max(now - window, now - RETENTION_SECONDS)
try:
scope = host_scope(profile)
except (KeyError, ValueError, TypeError) as exc:
raise HTTPException(status_code=409, detail="宿主机身份信息不完整") from exc
points = backend.query_metric_samples(
scope, canonical_id, since=since, until=now,
interval_seconds=interval, limit=MAX_QUERY_POINTS,
)
return {"vm_id": canonical_id, "range": range_,
"interval_seconds": interval, "points": points}
@app.get("/api/v1/vms/{vm_id}/traffic/daily")
def get_vm_daily_traffic(
vm_id: str,
days: int = Query(30, ge=1, le=30),
_session: Session = Depends(require_approved),
):
try:
canonical_id = str(uuid.UUID(vm_id))
except (TypeError, ValueError, AttributeError) as exc:
raise VMError("虚拟机 UUID 无效", 400) from exc
profile = current_profile()
require_vm_access(_session, profile, canonical_id)
# Match the metrics endpoint's live inventory check before consulting
# retained rows. The owner and host filters below isolate old tenants.
new_service(profile).get_vm(canonical_id)
backend = metric_store if metric_store is not None else data_store
if backend is None:
raise HTTPException(status_code=503, detail="指标数据库尚未就绪")
try:
scope = host_scope(profile)
except (KeyError, ValueError, TypeError) as exc:
raise HTTPException(status_code=409, detail="宿主机身份信息不完整") from exc
owner_id = (state.users.owner_of_vm(scope, canonical_id)
if database_mode else None)
configured_host_id = profile.get("host_id")
metric_host_id = configured_host_id if isinstance(configured_host_id, str) else None
now = int(time.time())
day_seconds = 24 * 60 * 60
today_start = now - now % day_seconds
since = max(0, today_start - (days - 1) * day_seconds)
rows = backend.query_daily_traffic(
scope, canonical_id, host_id=metric_host_id, user_id=owner_id,
since=since, until=now, limit=days,
)
by_day = {int(row["day_start"]): row for row in rows}
points = []
for day_start in range(since, today_start + 1, day_seconds):
row = by_day.get(day_start)
points.append({
"date": time.strftime("%Y-%m-%d", time.gmtime(day_start)),
"day_start": day_start,
"download_bytes": row.get("download_bytes") if row else None,
"upload_bytes": row.get("upload_bytes") if row else None,
"total_bytes": row.get("total_bytes") if row else None,
})
return {"vm_id": canonical_id, "days": days, "timezone": "UTC",
"from": since, "until": now, "points": points}
@app.post("/api/v1/vms", status_code=201)
def create_vm(body: CreateISOBody | CreateCloudBody,
_session: Session = Depends(require_approved_write)):
# Keep the update submission lock until libvirt has finished creating
# the VM. Otherwise an update can restart the web process mid-create.
with reinstall_lock:
forbid_long_job_during_update()
profile = current_profile()
account = account_for_session(_session)
owner_id = account["id"] if database_mode and (internal_mode or not account["admin"]) else None
if isinstance(body, CreateCloudBody):
public_key = body.guest_ssh_public_key.strip()
root_password = (body.root_password.get_secret_value()
if body.root_password is not None else "")
if body.guest_ssh_key_id is not None:
if public_key:
raise HTTPException(
status_code=422,
detail="已保存公钥和手动公钥只能选择一种",
)
saved_key = ssh_key_store().get_for_user(
account["id"], body.guest_ssh_key_id,
)
if saved_key is None:
raise SSHKeyError("公钥不存在", 404)
public_key = saved_key["public_key"]
# A cloud VM always has a recoverable initial root password,
# even when a saved or one-off SSH key was selected.
if not root_password:
root_password = generate_root_password()
password_hash = None
if root_password:
try:
password_hash = hash_root_password(root_password)
except ValueError as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc
configured_public_pools, _compatibility_mode = public_network_pools(
profile)
auto_public_ip = bool(configured_public_pools)
scheduled_public_ip = bool(body.public_ip_cidr)
selected_public_pool = None
if auto_public_ip:
if body.gateway or body.dns_servers:
raise HTTPException(
status_code=422,
detail="宿主机已配置公网地址池,网关和 DNS 必须使用宿主机配置",
)
if scheduled_public_ip and not internal_mode:
raise HTTPException(
status_code=422,
detail="宿主机已配置公网地址池,创建时不能手动指定 IP",
)
if scheduled_public_ip and not actor().get("can_provision"):
raise HTTPException(
status_code=403,
detail="只有已确认订单的内部调度任务可以指定预留公网 IP",
)
if body.public_pool_key:
selected_public_pool = public_pool_for_key(
profile, body.public_pool_key, active=True)
elif scheduled_public_ip:
# Old schedulers did not send a pool key. Infer it
# only when the exact reserved address has one
# unambiguous active-pool owner.
selected_public_pool = public_pool_for_ip(
profile, body.public_ip_cidr, active=True)
else:
# Preserve the legacy automatic-allocation request
# during the migration window. Pools are normalized
# by key, so every process makes the same choice.
selected_public_pool = next(
(pool for pool in configured_public_pools
if pool["state"] == "active"), None)
if selected_public_pool is None:
raise HTTPException(
status_code=409,
detail="宿主机没有开放新地址分配的公网地址池",
)
if body.network != selected_public_pool["network_selector"]:
raise HTTPException(
status_code=422,
detail="虚拟机网络与所选公网地址池不一致",
)
elif body.public_pool_key:
raise HTTPException(
status_code=422,
detail="宿主机尚未配置可用的公网地址池",
)
if not auto_public_ip and not (
body.public_ip_cidr and body.gateway and body.dns_servers
):
raise HTTPException(
status_code=422,
detail="宿主机尚未配置公网地址池,请手动填写 IP、网关和 DNS",
)
service = new_service(profile)
# The catalog is engine-global. The selected host receives a
# verified replica only on first use, before IP/credential/VM
# state is mutated.
record = ensure_cloud_template_replica(
profile, service, body.template_id)
spec = body.model_dump(exclude_none=True, exclude={
"source", "template_id", "root_password", "guest_ssh_key_id",
"public_pool_key",
})
spec["os_name"] = record.get("name")
spec["guest_ssh_public_key"] = public_key
if password_hash is not None:
spec["root_password_hash"] = password_hash
requested_uuid = actor()["vm_uuid"] if internal_mode else str(uuid.uuid4())
spec["vm_uuid"] = requested_uuid
allocated_ip = ""
owner_assigned = False
try:
if auto_public_ip:
assert selected_public_pool is not None
if scheduled_public_ip:
allocated_ip = claim_scheduled_public_ip(
profile, selected_public_pool,
body.public_ip_cidr,
requested_uuid,
)
else:
allocated_ip = allocate_public_ip(
profile, selected_public_pool,
requested_uuid,
)
spec["public_ip_cidr"] = allocated_ip
spec["public_pool_key"] = selected_public_pool["pool_key"]
spec["gateway"] = selected_public_pool["gateway"]
spec["dns_servers"] = list(
selected_public_pool["dns_servers"])
else:
claim_explicit_public_ip(profile, body.public_ip_cidr, requested_uuid)
allocated_ip = body.public_ip_cidr
if owner_id is not None:
state.users.assign_vm(host_scope(profile), requested_uuid, owner_id)
owner_assigned = True
save_credential(profile, requested_uuid,
("ssh_key_password" if public_key and root_password
else "ssh_key" if public_key else "password"),
f"panel-seed-{requested_uuid}.iso",
root_password or None)
vm = service.create_cloud_vm(spec, record["volume_name"])
except Exception as exc:
# An incomplete libvirt rollback may have left the guest
# behind. Keep its credential in that exceptional case so
# the operator can still recover access if it exists.
rollback_incomplete = (internal_mode or (isinstance(exc, VMError)
and "回滚不完整" in str(exc)))
if not rollback_incomplete:
try:
delete_credential(profile, requested_uuid)
except DatabaseError:
LOG.exception("Could not remove unused VM credential for %s", requested_uuid)
if owner_assigned:
try:
state.users.delete_vm_owner(host_scope(profile), requested_uuid)
except DatabaseError:
LOG.exception("Could not remove unused VM owner for %s", requested_uuid)
if allocated_ip:
try:
release_public_ip_claim(profile, allocated_ip, requested_uuid)
except DatabaseError:
LOG.exception("Could not release unused public IP claim for %s", requested_uuid)
raise
vm = verified_created_vm(profile, vm, requested_uuid)
else:
user_iso_record = None
if body.user_iso_id is not None:
user_scope = user_iso_scope(account["id"])
user_iso_record = user_iso_catalog.get(
user_scope, body.user_iso_id)
spec = body.model_dump(exclude_none=True, exclude={
"source", "iso", "user_iso_id",
})
else:
spec = body.model_dump(exclude_none=True, exclude={
"source", "user_iso_id",
})
if database_mode:
requested_uuid = actor()["vm_uuid"] if internal_mode else str(uuid.uuid4())
spec["vm_uuid"] = requested_uuid
if owner_id is not None:
state.users.assign_vm(host_scope(profile), requested_uuid, owner_id)
try:
service = new_service(profile)
if user_iso_record is not None:
with user_iso_cache_lock:
# Re-read both catalog and host marker only after
# entering the per-host cache critical section.
# Two first-use VM creates therefore cannot both
# act on the same stale marker=false observation.
user_iso_record = user_iso_catalog.get(
user_iso_scope(account["id"]),
user_iso_record["id"],
)
active_user_iso_records = user_iso_catalog.list(
user_iso_scope(account["id"]))
active_user_iso_volumes = {
row["volume_name"]
for row in active_user_iso_records
}
replica_ready = trusted_user_iso_replica(
account["id"], user_iso_record)
# A ready marker proves a prior transfer completed.
# Keep that durable proof while validating/reusing
# the cache; downgrading it before a fallible source
# check would strand an already referenced volume
# after an engine restart.
if not replica_ready:
save_user_iso_replica(
account["id"], user_iso_record,
status="syncing")
if shared_user_iso_runtime is not None:
with shared_user_iso_runtime.source(
account["id"], user_iso_record["id"]) as (
user_iso_record, source_path):
service.cache_user_iso(
source_path, account["id"],
user_iso_record["volume_name"],
active_user_iso_volumes,
trusted_existing=replica_ready)
save_user_iso_replica(
account["id"], user_iso_record,
status="ready")
prune_user_iso_replica_records(
account["id"],
{row["id"] for row in active_user_iso_records},
)
vm = service.create_user_iso_vm(
spec, account["id"],
user_iso_record["volume_name"],
user_iso_record["id"],
user_iso_record["name"],
)
else:
source_path = user_iso_files.path(
account["id"], user_iso_record["id"],
expected_size=user_iso_record["size_bytes"])
service.cache_user_iso(
source_path, account["id"],
user_iso_record["volume_name"],
active_user_iso_volumes,
trusted_existing=replica_ready)
save_user_iso_replica(
account["id"], user_iso_record,
status="ready")
prune_user_iso_replica_records(
account["id"],
{row["id"] for row in active_user_iso_records},
)
vm = service.create_user_iso_vm(
spec, account["id"],
user_iso_record["volume_name"],
user_iso_record["id"],
user_iso_record["name"],
)
else:
vm = service.create_vm(spec)
except Exception as exc:
rollback_incomplete = (internal_mode or (isinstance(exc, VMError)
and "回滚不完整" in str(exc)))
if owner_id is not None and not rollback_incomplete:
state.users.delete_vm_owner(host_scope(profile), requested_uuid)
raise
if database_mode:
vm = verified_created_vm(profile, vm, requested_uuid)
LOG.info("vm created: uuid=%s admin=%s", vm.get("uuid"), _session.username)
credential = get_credential(profile, vm["uuid"]) if isinstance(body, CreateCloudBody) else None
return decorate_vm(vm, credential, profile)
def run_reinstall(
job_id: str, profile: dict, scope: str, vm_id: str,
template_volume_name: str, template_name: str | None, ssh_public_key: str,
root_password_hash: str | None, generated_password: str | None,
credential_password: str | None,
) -> None:
vm_reinstalled = False
def report_progress(phase: str) -> None:
if not isinstance(phase, str):
return
with reinstall_lock:
job = reinstall_jobs.get(job_id)
if job is not None and job["status"] == "running":
job["phase"] = phase[:200]
# ``cleaning_up`` is emitted only after the replacement
# domain has started. Persist that boundary before
# deleting either old volume, so a process restart or a
# later callback failure cannot turn a committed guest
# into a retryable/failed reinstall.
if phase in {"cleaning_up", "cleanup_pending", "result_uncertain", "completed"}:
job["reinstall_committed"] = True
try:
save_job("reinstall_job", job_id, job)
except Exception:
# Progress persistence is advisory. The mutation must
# continue; the final result or the committed marker
# is persisted by the worker when the callback works.
LOG.exception("Could not persist reinstall progress: job=%s", job_id)
try:
with reinstall_lock:
reinstall_jobs[job_id]["status"] = "running"
reinstall_jobs[job_id]["phase"] = "正在检查虚拟机"
save_job("reinstall_job", job_id, reinstall_jobs[job_id])
result = new_service(profile).reinstall_cloud_vm(
vm_id, template_volume_name,
os_name=template_name,
ssh_public_key=ssh_public_key,
root_password_hash=root_password_hash,
progress=report_progress,
)
if not isinstance(result, dict):
raise VMError("重装结果无效,无法确认虚拟机状态", 502)
credential_generation = result.get("credential_generation")
if not isinstance(credential_generation, str) or not credential_generation:
raise VMError("重装结果缺少新的系统配置代号,无法保存凭据", 502)
result_status = result.get("status")
if result_status is not None and not isinstance(result_status, str):
raise VMError("重装结果状态无效,无法确认虚拟机状态", 502)
if isinstance(result_status, str):
result_status = result_status.strip().lower()
warning = result.get("warning")
error_code = result.get("error_code")
if not isinstance(error_code, str):
error_code = None
result_committed = result.get("reinstall_committed") is True
result_cleanup_pending = result.get("cleanup_pending") is True
result_has_warning = isinstance(warning, str) and bool(warning.strip())
result_has_error_code = isinstance(error_code, str) and bool(error_code.strip())
# A service implementation must not return an explicit failed
# result after crossing the replacement boundary without marking
# it. Only an explicit ``reinstall_committed=false`` is proof of
# a pre-commit failure; every other returned non-success result
# stays uncertain so it cannot authorize a destructive replay.
if (result_status == "failed"
and result.get("reinstall_committed") is False):
raise VMError("重装结果报告失败,未确认新系统已启动", 502)
# The service returns only after the replacement definition has
# started. Mark that commit before touching the credential store,
# so a persistence error cannot be mistaken for a pre-commit
# rollback and leave the panel pointing at the old password.
vm_reinstalled = True
save_credential(profile, vm_id,
("ssh_key_password" if ssh_public_key and credential_password
else "ssh_key" if ssh_public_key else "password"),
credential_generation,
credential_password)
success_statuses = {None, "succeeded", "completed", "done"}
partial = (
result_status not in success_statuses
or result_has_warning
or result_cleanup_pending
or result_has_error_code
or (
result_committed and result_status not in success_statuses
)
)
with reinstall_lock:
job = reinstall_jobs[job_id]
if partial:
if error_code not in REINSTALL_PUBLIC_ERROR_CODES:
error_code = (
REINSTALL_CLEANUP_PENDING_ERROR_CODE
if result.get("cleanup_pending") is True
or (isinstance(warning, str)
and warning.strip().startswith("旧存储卷"))
else REINSTALL_RESULT_UNCERTAIN_ERROR_CODE
)
job["status"] = "uncertain"
job["phase"] = (
"重装完成,但旧系统盘清理待核对"
if error_code == REINSTALL_CLEANUP_PENDING_ERROR_CODE
else "重装结果待核对"
)
job.pop("finished_at", None)
job["error_code"] = error_code
job["error"] = (
"重装已完成,但旧系统盘清理待核对"
if error_code == REINSTALL_CLEANUP_PENDING_ERROR_CODE
else "重装结果待核对"
)
job["reinstall_committed"] = True
if result.get("cleanup_pending") is True:
job["cleanup_pending"] = True
vm = result.get("vm")
if isinstance(vm, dict):
# Keep only the bounded observation needed by the
# PHP runtime cache; never persist disk source paths.
job["vm"] = {
key: vm[key] for key in ("uuid", "name", "state")
if key in vm
}
else:
job["status"] = "succeeded"
job["phase"] = "重装完成"
# A progress marker is needed only while the final result
# is unresolved. Do not let it make a normal completion
# look like a partial commit to the PHP worker.
for key in ("reinstall_committed", "cleanup_pending",
"error_code", "error", "warning"):
job.pop(key, None)
if isinstance(warning, str) and warning:
job["warning"] = warning
if generated_password is not None:
# DatabaseStore encrypts this one-time password. Keep it
# until the operator explicitly acknowledges saving it.
job["generated_password"] = generated_password
save_job("reinstall_job", job_id, job)
LOG.info("vm reinstall %s: job=%s uuid=%s",
"awaits cleanup reconciliation" if partial else "succeeded",
job_id, vm_id)
except Exception as exc:
if isinstance(exc, (VMError, TemplateError, ValueError)):
error = str(exc)
else:
LOG.exception("VM reinstall failed: job=%s uuid=%s", job_id, vm_id)
error = "重装系统失败,请查看面板日志"
with reinstall_lock:
job = reinstall_jobs[job_id]
committed = vm_reinstalled or job.get("reinstall_committed") is True
if committed:
# The replacement is live even when credential/job
# persistence failed. Keep the operation uncertain so it
# cannot be presented as success or replayed blindly.
job["status"] = "uncertain"
job["phase"] = "重装结果待核对"
job.pop("finished_at", None)
job["error_code"] = REINSTALL_RESULT_UNCERTAIN_ERROR_CODE
job["error"] = "重装结果待核对;面板未能确认新凭据已保存"
job["reinstall_committed"] = True
job["warning"] = "新系统已启动,但面板无法确认初始密码已保存;请立即保存结果页显示的密码并检查数据库"
if generated_password is not None:
job["generated_password"] = generated_password
else:
job["status"] = "failed"
job["phase"] = "重装失败"
job["error"] = error
job.pop("generated_password", None)
try:
save_job("reinstall_job", job_id, job)
except Exception:
LOG.exception("Could not persist reinstall status: job=%s", job_id)
# A failed terminal write leaves the durable row at its
# previous phase. Keep the in-memory guard fail-closed
# and make one best-effort retry with an explicit
# uncertain state; a later process restart will normalize
# the old running row the same way.
job["status"] = "uncertain"
job["phase"] = "重装结果待核对"
job.pop("finished_at", None)
if committed:
job["error_code"] = REINSTALL_RESULT_UNCERTAIN_ERROR_CODE
job["error"] = "重装结果待核对;面板未能保存任务状态"
else:
job.pop("error_code", None)
job["error"] = "重装结果待核对;面板未能保存任务状态"
job.pop("generated_password", None)
try:
save_job("reinstall_job", job_id, job)
except Exception:
LOG.exception("Could not persist uncertain reinstall status: job=%s", job_id)
if not committed:
LOG.warning("vm reinstall failed: job=%s uuid=%s", job_id, vm_id)
finally:
with reinstall_lock:
job = reinstall_jobs.get(job_id)
if job is None or job.get("status") not in {
"queued", "running", "uncertain"}:
pending_reinstalls.discard((scope, vm_id))
else:
pending_reinstalls.add((scope, vm_id))
@app.post("/api/v1/vms/{vm_id}/reinstall", status_code=202)
def reinstall_vm(vm_id: str, body: ReinstallBody,
_session: Session = Depends(require_approved_write)):
ssh_public_key = body.ssh_public_key.strip()
supplied_password = (body.root_password.get_secret_value()
if body.root_password is not None else "")
if body.ssh_key_id is not None:
if ssh_public_key:
raise HTTPException(
status_code=422,
detail="已保存公钥和手动公钥只能选择一种",
)
account = account_for_session(_session)
saved_key = ssh_key_store().get_for_user(account["id"], body.ssh_key_id)
if saved_key is None:
raise SSHKeyError("公钥不存在", 404)
ssh_public_key = saved_key["public_key"]
generated_password = None
if not supplied_password and not ssh_public_key:
generated_password = generate_root_password()
supplied_password = generated_password
try:
if ssh_public_key:
ssh_public_key, _unused_hash = _validate_guest_auth(ssh_public_key, None)
root_password_hash = (hash_root_password(supplied_password)
if supplied_password else None)
except ValueError as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc
with reinstall_lock, rescue_lock, snapshot_lock:
forbid_long_job_during_update()
profile = current_profile()
require_vm_access(_session, profile, vm_id)
canonical_id = str(uuid.UUID(vm_id))
forbid_retained_snapshot(profile, canonical_id, "重装系统")
service = new_service(profile)
rescue_record = current_rescue_state(profile, canonical_id, service)
rescue_vm = service.get_vm(canonical_id)
if ((host_scope(profile), canonical_id) in pending_rescues
or rescue_record is not None
or rescue_vm.get("rescue_mode")
or rescue_vm.get("rescue_staged")):
raise VMError("虚拟机处于救援模式或救援任务中,请先退出救援模式", 409)
scope = template_scope(profile)
vm = service.get_vm(vm_id)
if not vm.get("managed") or not vm.get("public_ip_cidr"):
raise VMError("只有面板创建的公网云镜像虚拟机支持自动重装", 409)
if body.confirmation_name != vm["name"]:
raise HTTPException(status_code=422, detail="确认名称与虚拟机名称不一致")
normalized_id = vm["uuid"]
key = (scope, normalized_id)
if key in pending_reinstalls:
raise VMError("该虚拟机正在重装系统", 409)
if any(
job.get("scope") == scope
and job.get("vm_id") == normalized_id
and job.get("status") in {"queued", "running", "uncertain"}
for job in reinstall_jobs.values()):
raise VMError("上一次重装结果待核对,请先由管理员核查虚拟机状态", 409)
# Finish and verify the target-host replica before scheduling any
# task that can shut down the guest or replace its system disk.
record = ensure_cloud_template_replica(
profile, service, body.template_id)
job_id = str(uuid.uuid4())
reinstall_jobs[job_id] = {
"owner": _session.username,
"scope": scope,
"status": "queued",
"phase": "等待重装任务开始",
"vm_id": normalized_id,
"template_id": record["id"],
}
if generated_password is not None:
# The engine record is encrypted at rest by DatabaseStore and
# is the only recovery copy if the process exits after the
# replacement guest starts but before the final result save.
reinstall_jobs[job_id]["generated_password"] = generated_password
save_job("reinstall_job", job_id, reinstall_jobs[job_id])
pending_reinstalls.add(key)
try:
reinstall_executor.submit(
run_reinstall, job_id, profile, scope, normalized_id,
record["volume_name"], record.get("name"), ssh_public_key, root_password_hash,
generated_password, supplied_password or None,
)
except RuntimeError as exc:
pending_reinstalls.discard(key)
reinstall_jobs.pop(job_id, None)
delete_job("reinstall_job", job_id)
raise VMError("面板正在停止,无法开始重装任务", 503) from exc
LOG.info("vm reinstall queued: job=%s uuid=%s admin=%s",
job_id, normalized_id, _session.username)
return {"job_id": job_id, "status": "queued"}
@app.get("/api/v1/reinstall-jobs/{job_id}")
def reinstall_job(job_id: str, _session: Session = Depends(require_approved_write)):
scope = template_scope(current_profile())
with reinstall_lock:
job = reinstall_jobs.get(job_id)
if (job is None or job["scope"] != scope
or job["owner"] != _session.username):
raise VMError("重装任务不存在", 404)
status = public_status(job.get("status"))
result: dict[str, Any] = {"status": status}
for key in ("job_id", "vm_id", "template_id", "operation_id"):
identifier = public_identifier(job.get(key), minimum=1)
if identifier is not None:
result[key] = identifier
if (status == "uncertain"
and job.get("error_code") in REINSTALL_PUBLIC_ERROR_CODES):
result["error_code"] = job["error_code"]
result["phase"] = public_phase(
job.get("phase"), status, public_reinstall_phases, kind="reinstall")
for key in ("created_at", "started_at", "finished_at"):
timestamp = public_timestamp(job.get(key))
if timestamp is not None:
result[key] = timestamp
if "warning" in job:
warning = public_warning(
job.get("warning"), public_reinstall_warnings,
"重装任务已完成,但仍有需要管理员检查的警告")
if warning is not None:
result["warning"] = warning
if "error" in job:
error = canonical_job_text(
job.get("error"), public_reinstall_errors,
"重装系统失败,请查看面板日志")
if error is not None:
result["error"] = error
if status in {"succeeded", "uncertain"} and isinstance(job.get("generated_password"), str):
password = job["generated_password"]
if 1 <= len(password) <= 128 and re.fullmatch(r"[^\x00-\x1f\x7f]+", password):
result["generated_password"] = password
return result
@app.post("/api/v1/reinstall-jobs/{job_id}/ack-password")
def acknowledge_reinstall_password(job_id: str,
_session: Session = Depends(require_approved_write)):
scope = template_scope(current_profile())
with reinstall_lock:
job = reinstall_jobs.get(job_id)
if (job is None or job["scope"] != scope
or job["owner"] != _session.username):
raise VMError("重装任务不存在", 404)
if job["status"] not in {"succeeded", "uncertain"}:
raise VMError("重装任务尚未完成,不能确认密码", 409)
job.pop("generated_password", None)
save_job("reinstall_job", job_id, job)
return {"acknowledged": True}
def run_rescue_job(job_id: str, profile: dict, vm_id: str,
operation: Literal["enter", "exit", "force"],
network_suspended: bool = False) -> None:
vm_scope = host_scope(profile)
key = (vm_scope, vm_id)
completed = False
try:
with rescue_lock:
job = rescue_jobs[job_id]
job["status"] = "running"
job["phase"] = (
"正在预挂载救援 ISO 并等待正常关机" if operation == "enter"
else "正在强制断电并从救援 ISO 冷启动" if operation == "force"
else "正在正常关机并恢复原系统启动配置"
)
save_job("rescue_job", job_id, job)
state_record = load_rescue_state(profile, vm_id)
if state_record is None:
raise VMError("救援恢复记录丢失,已停止操作", 409)
state_record["status"] = "exiting" if operation == "exit" else "running"
save_rescue_state(profile, vm_id, state_record)
service = new_service(profile)
if operation == "enter":
result = service.enter_rescue_vm(vm_id, state_record["volume_name"],
state_record["original_xml"])
elif operation == "force":
result = service.force_start_staged_rescue_vm(
vm_id, state_record["original_xml"])
else:
result = service.exit_rescue_vm(
vm_id, state_record["original_xml"],
network_suspended=network_suspended)
completed = True
staged = (operation == "enter" and isinstance(result, dict)
and result.get("pending_manual_restart") is True)
with rescue_lock:
if operation in {"enter", "force"}:
state_record["status"] = "staged" if staged else "active"
save_rescue_state(profile, vm_id, state_record)
else:
delete_rescue_state(profile, vm_id)
job = rescue_jobs[job_id]
job["status"] = "completed"
job["phase"] = ("救援 ISO 已预挂载,等待手动强制断电后开机" if staged
else "救援系统已启动" if operation in {"enter", "force"}
else "已退出救援模式,原系统已启动")
if staged:
job["pending_manual_restart"] = True
if isinstance(result, dict) and result.get("warning"):
job["warning"] = str(result["warning"])
save_job("rescue_job", job_id, job)
LOG.info("vm rescue %s succeeded: job=%s uuid=%s", operation, job_id, vm_id)
except Exception as exc:
if isinstance(exc, (VMError, TemplateError)):
error = str(exc)
else:
LOG.exception("VM rescue %s failed: job=%s uuid=%s", operation, job_id, vm_id)
error = "紧急救援操作失败,请查看面板和宿主机日志"
with rescue_lock:
try:
state_record = load_rescue_state(profile, vm_id)
if state_record is not None:
# If the original inactive definition survived, the
# failed job cannot strand an otherwise normal VM in
# a database-only "rescue" state.
service = new_service(profile)
actual_vm = service.get_vm(vm_id)
if (not actual_vm.get("rescue_mode")
and rescue_xml_matches(
service.prepare_rescue_vm(
vm_id, state_record["volume_name"]),
state_record["original_xml"],
)):
delete_rescue_state(profile, vm_id)
else:
state_record["status"] = "failed"
save_rescue_state(profile, vm_id, state_record)
except (DatabaseError, VMError, TemplateError):
LOG.exception("Could not save rescue recovery state: job=%s", job_id)
job = rescue_jobs[job_id]
job["status"] = "failed"
job["phase"] = "救援操作失败"
if "正常关机超时" in error or "无法请求虚拟机正常关机" in error:
job["error"] = error
else:
job["error"] = error + ";请确认虚拟机实际运行状态,必要时联系管理员"
if completed:
job["error"] = "虚拟机操作已完成,但数据库状态保存失败;请管理员检查救援记录"
try:
save_job("rescue_job", job_id, job)
except DatabaseError:
LOG.exception("Could not persist rescue job failure: job=%s", job_id)
finally:
with rescue_lock:
pending_rescues.discard(key)
@app.get("/api/v1/vms/{vm_id}/rescue")
def get_vm_rescue(vm_id: str, _session: Session = Depends(require_approved)):
profile = current_profile()
canonical_id = require_vm_access(_session, profile, vm_id)
with rescue_lock:
service = new_service(profile)
vm = service.get_vm(canonical_id)
record = current_rescue_state(profile, vm["uuid"], service)
staged = bool(vm.get("rescue_staged"))
active = bool(vm.get("rescue_mode")) and not staged
if (record is not None
and (host_scope(profile), vm["uuid"]) not in pending_rescues):
if staged and record["status"] in {"queued", "running", "failed"}:
record["status"] = "staged"
save_rescue_state(profile, vm["uuid"], record)
elif active and record["status"] in {"queued", "running", "staged"}:
record["status"] = "active"
save_rescue_state(profile, vm["uuid"], record)
template_name = None
template_id = record["template_id"] if record is not None else None
if record is not None:
try:
template_name = rescue_catalog.get(
rescue_scope(profile), record["template_id"])["name"]
except TemplateError:
pass
return {
"active": active,
"staged": staged,
"power_state": vm.get("state", "unknown"),
"template_id": template_id,
"template_name": template_name,
"status": record["status"] if record is not None else None,
"job_id": record.get("job_id") if record is not None else None,
"can_exit": (active or staged) and record is not None,
}
@app.post("/api/v1/vms/{vm_id}/rescue", status_code=202)
def enter_vm_rescue(vm_id: str, body: EnterRescueBody,
_session: Session = Depends(require_approved_write)):
with reinstall_lock, rescue_lock, snapshot_lock:
forbid_long_job_during_update()
profile = current_profile()
canonical_id = require_vm_access(_session, profile, vm_id)
forbid_pending_snapshot(profile, canonical_id, "进入救援模式")
vm_scope = host_scope(profile)
key = (vm_scope, canonical_id)
if key in pending_rescues or (template_scope(profile), canonical_id) in pending_reinstalls:
raise VMError("该虚拟机已有救援或重装任务,暂时不能重复操作", 409)
service = new_service(profile)
vm = service.get_vm(canonical_id)
if body.confirmation_name != vm["name"]:
raise VMError("确认名称与虚拟机名称不一致", 422)
if vm.get("rescue_mode") or vm.get("rescue_staged"):
raise VMError("虚拟机已预挂载救援 ISO 或处于救援模式,请先退出", 409)
record = rescue_catalog.get(rescue_scope(profile), body.template_id)
original_xml = service.prepare_rescue_vm(canonical_id, record["volume_name"])
prior = load_rescue_state(profile, canonical_id)
if prior is not None:
if not rescue_xml_matches(prior["original_xml"], original_xml):
raise VMError("虚拟机仍有未完成的救援记录,请管理员检查后再操作", 409)
delete_rescue_state(profile, canonical_id)
job_id = str(uuid.uuid4())
state_record = {
"template_id": record["id"], "volume_name": record["volume_name"],
"original_xml": original_xml, "status": "queued", "job_id": job_id,
}
save_rescue_state(profile, canonical_id, state_record)
rescue_jobs[job_id] = {
"scope": vm_scope, "owner": _session.username,
"vm_id": canonical_id, "operation": "enter",
"status": "queued", "phase": "等待救援任务开始",
}
try:
save_job("rescue_job", job_id, rescue_jobs[job_id])
pending_rescues.add(key)
rescue_executor.submit(run_rescue_job, job_id, profile, canonical_id, "enter")
except Exception:
pending_rescues.discard(key)
rescue_jobs.pop(job_id, None)
delete_job("rescue_job", job_id)
delete_rescue_state(profile, canonical_id)
raise
LOG.info("vm rescue enter queued: job=%s uuid=%s user=%s",
job_id, canonical_id, _session.username)
return {"job_id": job_id, "status": "queued"}
@app.post("/api/v1/vms/{vm_id}/rescue/force-start", status_code=202)
def force_start_vm_rescue(vm_id: str, body: ForceRescueBody,
_session: Session = Depends(require_approved_write)):
"""Explicit, confirmed power cycle into an already staged rescue ISO."""
with reinstall_lock, rescue_lock, snapshot_lock:
forbid_long_job_during_update()
profile = current_profile()
canonical_id = require_vm_access(_session, profile, vm_id)
forbid_pending_snapshot(profile, canonical_id, "强制启动救援系统")
vm_scope = host_scope(profile)
key = (vm_scope, canonical_id)
if key in pending_rescues or (template_scope(profile), canonical_id) in pending_reinstalls:
raise VMError("该虚拟机已有救援或重装任务,请等待完成", 409)
record = load_rescue_state(profile, canonical_id)
if record is None:
raise VMError("缺少原始启动配置备份,不能安全强制重启进入救援", 409)
vm = new_service(profile).get_vm(canonical_id)
if body.confirmation_name != vm["name"]:
raise VMError("确认名称与虚拟机名称不一致", 422)
if not vm.get("rescue_staged"):
raise VMError("救援 ISO 尚未预挂载,或虚拟机已经进入救援模式", 409)
job_id = str(uuid.uuid4())
rescue_jobs[job_id] = {
"scope": vm_scope, "owner": _session.username,
"vm_id": canonical_id, "operation": "force",
"status": "queued", "phase": "等待手动确认的救援冷启动",
}
previous_status = record["status"]
previous_job_id = record.get("job_id")
try:
save_job("rescue_job", job_id, rescue_jobs[job_id])
record["status"] = "queued"
record["job_id"] = job_id
save_rescue_state(profile, canonical_id, record)
pending_rescues.add(key)
rescue_executor.submit(run_rescue_job, job_id, profile, canonical_id, "force")
except Exception:
pending_rescues.discard(key)
rescue_jobs.pop(job_id, None)
delete_job("rescue_job", job_id)
record["status"] = previous_status
record["job_id"] = previous_job_id
save_rescue_state(profile, canonical_id, record)
raise
LOG.warning("vm rescue force-start queued: job=%s uuid=%s user=%s",
job_id, canonical_id, _session.username)
return {"job_id": job_id, "status": "queued"}
@app.post("/api/v1/vms/{vm_id}/rescue/exit", status_code=202)
def exit_vm_rescue(vm_id: str, body: ExitRescueBody,
_session: Session = Depends(require_approved_write)):
with reinstall_lock, rescue_lock, snapshot_lock:
forbid_long_job_during_update()
profile = current_profile()
canonical_id = require_vm_access(_session, profile, vm_id)
forbid_pending_snapshot(profile, canonical_id, "退出救援模式")
vm_scope = host_scope(profile)
key = (vm_scope, canonical_id)
if key in pending_rescues or (template_scope(profile), canonical_id) in pending_reinstalls:
raise VMError("该虚拟机已有救援或重装任务,请等待完成", 409)
record = load_rescue_state(profile, canonical_id)
if record is None:
raise VMError("找不到原始启动配置,无法安全退出救援模式", 409)
vm = new_service(profile).get_vm(canonical_id)
if body.confirmation_name != vm["name"]:
raise VMError("确认名称与虚拟机名称不一致", 422)
if not (vm.get("rescue_mode") or vm.get("rescue_staged")):
raise VMError("虚拟机当前并非救援模式,请管理员核对配置", 409)
job_id = str(uuid.uuid4())
rescue_jobs[job_id] = {
"scope": vm_scope, "owner": _session.username,
"vm_id": canonical_id, "operation": "exit",
"status": "queued", "phase": "等待退出救援模式",
}
save_job("rescue_job", job_id, rescue_jobs[job_id])
record["status"] = "exiting"
record["job_id"] = job_id
save_rescue_state(profile, canonical_id, record)
pending_rescues.add(key)
try:
rescue_executor.submit(
run_rescue_job, job_id, profile, canonical_id, "exit",
body.network_suspended)
except RuntimeError as exc:
pending_rescues.discard(key)
record["status"] = "active"
save_rescue_state(profile, canonical_id, record)
rescue_jobs.pop(job_id, None)
delete_job("rescue_job", job_id)
raise VMError("面板正在停止,无法退出救援模式", 503) from exc
LOG.info("vm rescue exit queued: job=%s uuid=%s user=%s",
job_id, canonical_id, _session.username)
return {"job_id": job_id, "status": "queued"}
@app.get("/api/v1/rescue-jobs/{job_id}")
def get_rescue_job(job_id: str, _session: Session = Depends(require_approved)):
profile = current_profile()
vm_scope = host_scope(profile)
with rescue_lock:
job = rescue_jobs.get(job_id)
if (job is None or job["scope"] != vm_scope
or (job["owner"] != _session.username
and not account_for_session(_session)["admin"])):
raise VMError("救援任务不存在", 404)
require_vm_access(_session, profile, job["vm_id"])
status = public_status(job.get("status"))
result: dict[str, Any] = {"status": status}
for key in ("job_id", "operation", "operation_id"):
identifier = public_identifier(job.get(key), minimum=1)
if key == "operation":
identifier = job.get(key) if job.get(key) in {"enter", "force", "exit"} else None
if identifier is not None:
result[key] = identifier
result["phase"] = public_phase(
job.get("phase"), status, public_rescue_phases, kind="rescue")
for key in ("created_at", "started_at", "finished_at"):
timestamp = public_timestamp(job.get(key))
if timestamp is not None:
result[key] = timestamp
if isinstance(job.get("pending_manual_restart"), bool):
result["pending_manual_restart"] = job["pending_manual_restart"]
if "warning" in job:
warning = public_warning(
job.get("warning"), public_rescue_warnings,
"救援任务已完成,但仍有需要管理员检查的警告")
if warning is not None:
result["warning"] = warning
if "error" in job:
error = canonical_job_text(
job.get("error"), public_rescue_errors,
"紧急救援操作失败,请查看面板和宿主机日志")
if error is not None:
result["error"] = error
return result
def apply_private_vm_target(vm_id: str, session: Session,
method: str, label: str, **options) -> dict:
with reinstall_lock, rescue_lock:
forbid_long_job_during_update()
profile = current_profile()
canonical_id = require_vm_access(session, profile, vm_id)
# Network isolation is a fail-safe policy target and cannot remain
# blocked forever by durable snapshot uncertainty. Service stop is
# different: an uncertain restore may later reconcile by restoring
# its recorded running state, so bypassing it here could power a
# purged guest back on without a durable policy tombstone.
fail_safe_suspend = method == "network_suspend"
barrier_key = (host_scope(profile), canonical_id)
barrier_installed = False
with snapshot_lock:
if fail_safe_suspend:
if barrier_key in snapshot_suspend_barriers:
raise VMError(
"虚拟机正在执行到期暂停策略,暂时不能重复暂停网络",
409,
)
snapshot_suspend_barriers.add(barrier_key)
barrier_installed = True
try:
# Only a Future that the executor confirms was still
# pending may be cancelled. Its two-phase tombstone and
# catalog rollback complete inside this same lock.
cancel_queued_snapshot_for_network_suspend(
profile, canonical_id)
forbid_pending_snapshot(
profile, canonical_id, label,
allow_uncertain=True,
ignore_suspend_barrier=True,
)
except Exception:
snapshot_suspend_barriers.discard(barrier_key)
barrier_installed = False
raise
else:
forbid_pending_snapshot(profile, canonical_id, label)
try:
if (template_scope(profile), canonical_id) in pending_reinstalls:
raise VMError(f"虚拟机正在重装系统,暂时不能执行{label}", 409)
if (host_scope(profile), canonical_id) in pending_rescues:
raise VMError(f"虚拟机正在切换救援模式,暂时不能执行{label}", 409)
vm = getattr(new_service(profile), method)(
canonical_id, **options)
finally:
if barrier_installed:
with snapshot_lock:
snapshot_suspend_barriers.discard(barrier_key)
LOG.info("vm target state applied: uuid=%s target=%s actor=%s",
vm.get("uuid"), method, session.username)
return vm
@app.post("/api/v1/vms/{vm_id}/service-suspend")
def service_suspend_vm(
vm_id: str,
_session: Session = Depends(require_internal_service_write),
):
return apply_private_vm_target(
vm_id, _session, "service_suspend", "服务暂停")
@app.post("/api/v1/vms/{vm_id}/service-resume")
def service_resume_vm(
vm_id: str,
body: ServiceResumeBody,
_session: Session = Depends(require_internal_service_write),
):
return apply_private_vm_target(
vm_id, _session, "service_resume", "服务恢复",
start_if_suspended=body.start_if_suspended)
@app.post("/api/v1/vms/{vm_id}/network-suspend")
def network_suspend_vm(
vm_id: str,
_session: Session = Depends(require_internal_service_write),
):
return apply_private_vm_target(
vm_id, _session, "network_suspend", "网络暂停")
@app.post("/api/v1/vms/{vm_id}/network-resume")
def network_resume_vm(
vm_id: str,
_session: Session = Depends(require_internal_service_write),
):
return apply_private_vm_target(
vm_id, _session, "network_resume", "网络恢复")
@app.post("/api/v1/vms/{vm_id}/network")
def change_network_vm(
vm_id: str,
body: ChangeNetworkBody,
request: Request,
_session: Session = Depends(require_internal_service_write),
):
"""Change one managed cloud VM's public IPv4 on the same host.
The engine validates the host pool and holds the new claim before
touching libvirt. A 5xx/transport failure deliberately keeps both IP
claims for reconciliation; only deterministic validation failures
release the new reservation.
"""
with reinstall_lock, rescue_lock, snapshot_lock:
forbid_long_job_during_update()
profile = current_profile()
try:
normalized_id = str(uuid.UUID(vm_id))
except (TypeError, ValueError, AttributeError) as exc:
raise VMError("虚拟机 UUID 无效", 400) from exc
require_vm_access(_session, profile, normalized_id)
forbid_pending_snapshot(profile, normalized_id, "修改网络")
if (template_scope(profile), normalized_id) in pending_reinstalls:
raise VMError("虚拟机正在重装系统,暂时不能修改网络", 409)
if (host_scope(profile), normalized_id) in pending_rescues:
raise VMError("虚拟机正在切换救援模式,请先修改完成后再试", 409)
service = new_service(profile)
vm = service.get_vm(normalized_id)
if (not vm.get("managed") or vm.get("os_source") != "cloud_template"
or not vm.get("public_ip_cidr")):
raise VMError("只有面板创建的公网云镜像虚拟机支持更换 IP", 409)
if not body.confirm_restart:
raise VMError("更换公网 IP 前必须确认会短暂重启虚拟机", 422)
old_ip = str(vm["public_ip_cidr"])
try:
old_pool_key = vm.get("public_pool_key")
# Guests created before pool metadata existed, and guests
# created while the synthetic legacy pool was active, are
# adopted by exact CIDR membership after the native catalog
# is installed.
if (isinstance(old_pool_key, str) and old_pool_key
and old_pool_key != "legacy-default"):
current_pool = public_pool_for_key(
profile, old_pool_key, active=False)
inferred_current_pool = public_pool_for_ip(
profile, old_ip, active=False)
if inferred_current_pool["pool_key"] != current_pool["pool_key"]:
raise VMError(
"虚拟机公网 IP 与记录的地址池不一致,已停止网络修改",
409,
)
else:
current_pool = public_pool_for_ip(
profile, old_ip, active=False)
target_pool = (
public_pool_for_key(profile, body.public_pool_key, active=True)
if body.public_pool_key else
public_pool_for_ip(profile, body.public_ip_cidr, active=True)
)
except VMError:
raise
except (TypeError, ValueError) as exc:
raise VMError("虚拟机当前公网 IP 配置无效,已停止网络修改", 409) from exc
if (target_pool["network_selector"]
!= current_pool["network_selector"]):
raise VMError(
"不允许通过换 IP 接口迁移到不同的物理网络接入",
409,
)
try:
validated = validate_cloud_network(
body.public_ip_cidr,
target_pool["gateway"],
target_pool["dns_servers"],
require_auth=False,
)
except (TypeError, ValueError) as exc:
raise VMError("目标公网 IP 配置无效", 422) from exc
if validated["public_ip_cidr"] == old_ip:
return {
"vm": vm,
"public_pool_key": target_pool["pool_key"],
"public_ip_cidr": old_ip,
"gateway": validated["gateway"],
"dns_servers": list(validated["dns_servers"]),
"network_changed": False,
}
upgrade_firewall_filter_if_needed(
profile, normalized_id, service)
mark_network_mutation_started = request.scope.get(
"virt_cloud_mark_network_mutation_started")
if callable(mark_network_mutation_started):
mark_network_mutation_started()
claim_scheduled_public_ip(
profile, target_pool,
validated["public_ip_cidr"], normalized_id,
)
claim_reserved = True
try:
network = {
**validated,
"public_pool_key": target_pool["pool_key"],
"source_public_pool_key": current_pool["pool_key"],
}
result = service.change_cloud_vm_network(
normalized_id, network,
confirm_restart=body.confirm_restart,
)
except Exception as exc:
# A successful libvirt request can lose its response. Never
# release a new claim on a 5xx/unknown result; doing so could
# allocate the same address to a second guest.
if (claim_reserved and isinstance(exc, VMError)
and exc.status < 500):
release_public_ip_claim(
profile, validated["public_ip_cidr"], normalized_id)
raise
try:
release_public_ip_claim(profile, old_ip, normalized_id)
except DatabaseError as exc:
LOG.exception("Could not release old public IP claim for %s", normalized_id)
# The guest may already use the target address, but PHP must
# keep both ledger rows unavailable until the engine claim
# swap is complete. A 5xx makes the durable operation
# uncertain instead of silently terminal-successful.
raise PublicIpClaimCleanupPending(
"新公网 IP 已生效,但旧地址占用清理失败,必须人工核查",
503,
) from exc
return result
@app.post("/api/v1/vms/{vm_id}/settings")
def change_vm_settings(
vm_id: str,
body: VMSettingsBody,
_session: Session = Depends(require_approved_write),
):
"""Change guest hostname/root credentials from the authenticated panel boundary.
The public-key value is validated directly from this request. This
endpoint never accepts an SSH-key ID and never reads the tenant key
catalog. Password hashing and credential persistence happen inside
the authenticated engine so plaintext is not written to operation
records or logs.
"""
with reinstall_lock, rescue_lock, snapshot_lock:
forbid_long_job_during_update()
profile = current_profile()
canonical_id = require_vm_access(_session, profile, vm_id)
forbid_pending_snapshot(profile, canonical_id, "修改设置")
if (template_scope(profile), canonical_id) in pending_reinstalls:
raise VMError("虚拟机正在重装系统,暂时不能修改设置", 409)
if (host_scope(profile), canonical_id) in pending_rescues:
raise VMError("虚拟机正在切换救援模式,请等待任务完成", 409)
hostname = body.hostname
if hostname is not None:
try:
hostname = validate_guest_hostname(hostname)
except ValueError as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc
public_key = body.ssh_public_key
if public_key is not None:
try:
public_key = _validate_ssh_public_key(public_key.strip())
except ValueError as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc
password = (body.root_password.get_secret_value()
if body.root_password is not None else None)
password_hash = None
if password is not None:
try:
password_hash = hash_root_password(password)
except ValueError as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc
service = new_service(profile)
current_vm = service.get_vm(canonical_id)
if (not current_vm.get("managed")
or current_vm.get("os_source") != "cloud_template"):
raise VMError("只有面板创建的公网云镜像虚拟机支持修改设置", 409)
result = service.change_cloud_vm_settings(
canonical_id,
hostname=hostname,
root_password_hash=password_hash,
ssh_public_key=public_key,
confirm_restart=body.confirm_restart,
)
# Keep the panel's encrypted reveal credential in sync only after
# the engine confirms the seed was replaced. A missing legacy
# record is left untouched rather than inventing an auth mode.
existing = get_credential(profile, canonical_id)
if password is not None or public_key is not None:
if existing is not None:
generation = existing["generation"]
auth_mode = existing["auth_mode"]
saved_password = existing.get("password")
else:
generation = (result.get("vm", {}).get("credential_generation")
if isinstance(result.get("vm"), dict) else None)
if not isinstance(generation, str) or not generation:
generation = f"panel-seed-{canonical_id}.iso"
auth_mode = "ssh_key" if public_key is not None and password is None else "password"
saved_password = None
if password is not None:
saved_password = password
auth_mode = "ssh_key_password" if public_key is not None or auth_mode in {
"ssh_key", "ssh_key_password"
} else "password"
elif public_key is not None and auth_mode == "password":
auth_mode = "ssh_key_password"
save_credential(profile, canonical_id, auth_mode, generation, saved_password)
LOG.info("vm settings changed: uuid=%s admin=%s", canonical_id, _session.username)
return result
@app.post("/api/v1/vms/{vm_id}/actions")
def action(vm_id: str, body: ActionBody, _session: Session = Depends(require_approved_write)):
with reinstall_lock, rescue_lock, snapshot_lock:
forbid_long_job_during_update()
profile = current_profile()
require_vm_access(_session, profile, vm_id)
try:
normalized_id = str(uuid.UUID(vm_id))
except ValueError:
normalized_id = vm_id
forbid_pending_snapshot(profile, normalized_id, "执行电源操作")
if (template_scope(profile), normalized_id) in pending_reinstalls:
raise VMError("虚拟机正在重装系统,暂时不能执行其他操作", 409)
service = new_service(profile)
if (host_scope(profile), normalized_id) in pending_rescues:
raise VMError("虚拟机正在切换救援模式,请等待任务完成", 409)
rescue_record = current_rescue_state(profile, normalized_id, service)
current_vm = service.get_vm(normalized_id)
staged = bool(current_vm.get("rescue_staged"))
if staged and rescue_record is not None:
if body.action not in {"poweroff", "start"}:
raise VMError(
"救援 ISO 已预挂载;请强制断电后开机进入救援,或取消救援配置", 409)
elif rescue_record is not None or current_vm.get("rescue_mode") or staged:
raise VMError("虚拟机处于救援模式或救援任务中,请先退出救援模式", 409)
vm = service.action(vm_id, body.action)
if staged and body.action == "start":
if not vm.get("rescue_mode") or vm.get("rescue_staged"):
raise VMError(
"虚拟机已发送开机指令,但未确认从救援 ISO 启动;请检查 VNC 控制台", 502)
rescue_record["status"] = "active"
save_rescue_state(profile, normalized_id, rescue_record)
LOG.info("vm action: uuid=%s action=%s admin=%s", vm.get("uuid"), body.action, _session.username)
return vm
@app.delete("/api/v1/vms/{vm_id}")
def delete_vm(
vm_id: str,
delete_disks: bool = False,
_session: Session = Depends(require_approved_write),
):
with reinstall_lock, rescue_lock, snapshot_lock:
forbid_long_job_during_update()
profile = current_profile()
require_vm_access(_session, profile, vm_id)
system_cleanup = (
internal_mode and actor().get("system_cleanup") is True
)
try:
normalized_id = str(uuid.UUID(vm_id))
except ValueError:
normalized_id = vm_id
if (host_scope(profile), normalized_id) in pending_snapshot_jobs:
raise VMError("虚拟机正在执行快照任务,暂时不能删除", 409)
if (template_scope(profile), normalized_id) in pending_reinstalls:
raise VMError("虚拟机正在重装系统,暂时不能删除", 409)
service = new_service(profile)
if (host_scope(profile), normalized_id) in pending_rescues:
raise VMError("虚拟机处于救援模式或救援任务中,请先退出救援模式", 409)
if not system_cleanup:
rescue_record = current_rescue_state(profile, normalized_id, service)
rescue_vm = service.get_vm(normalized_id)
rescue_protected = (
rescue_record is not None
or rescue_vm.get("rescue_mode")
or rescue_vm.get("rescue_staged")
)
else:
rescue_protected = False
if rescue_protected:
raise VMError("虚拟机处于救援模式或救援任务中,请先退出救援模式", 409)
registered_snapshots = list_snapshot_records(normalized_id)
if registered_snapshots and not delete_disks:
raise VMError("虚拟机仍有本地快照;删除磁盘时会一并清理快照", 409)
# Validate every registered recovery point before the VM delete,
# but keep all of them intact until that deletion commits. This
# prevents a later VM-delete failure from destroying the user's
# last rollback points.
if delete_disks and registered_snapshots:
for snapshot in registered_snapshots:
service.validate_snapshot_volume(
normalized_id,
snapshot["snapshot_key"],
snapshot["volume_name"],
)
result = service.delete_vm(vm_id, delete_disks)
deleted_snapshots = []
if delete_disks and registered_snapshots:
try:
for snapshot in registered_snapshots:
service.delete_snapshot_volume(
normalized_id,
snapshot["snapshot_key"],
snapshot["volume_name"],
allow_missing_vm=True,
)
delete_snapshot_record(
normalized_id, snapshot["snapshot_key"])
deleted_snapshots.append(snapshot["snapshot_key"])
except Exception as exc:
# The VM deletion already committed. Keep every
# unconfirmed catalog row so reconciliation can identify
# exactly which volume still needs attention.
raise VMDeleteSnapshotCleanupPending(
"虚拟机已删除,但本地快照清理结果不完整;必须核查原删除操作"
) from exc
if deleted_snapshots:
result["deleted_snapshots"] = deleted_snapshots
if system_cleanup:
try:
delete_rescue_state(profile, result["uuid"])
except Exception as exc:
LOG.exception("Could not remove rescue state for deleted VM %s", result["uuid"])
raise VMDeleteSnapshotCleanupPending(
"虚拟机已删除,但救援状态清理未完成;必须核查原删除操作"
) from exc
if database_mode:
try:
state.users.delete_vm_owner(host_scope(profile), result["uuid"])
except Exception as exc:
LOG.exception("Could not remove owner for deleted VM %s", result["uuid"])
raise VMDeleteSnapshotCleanupPending(
"虚拟机已删除,但归属记录清理未完成;必须核查原删除操作"
) from exc
try:
delete_credential(profile, result["uuid"])
except Exception as exc:
LOG.exception("Could not remove credential for deleted VM %s", result["uuid"])
raise VMDeleteSnapshotCleanupPending(
"虚拟机已删除,但登录凭据清理未完成;必须核查原删除操作"
) from exc
try:
release_vm_public_ip_claims(profile, result["uuid"])
except Exception as exc:
LOG.exception("Could not release public IP claim for deleted VM %s", result["uuid"])
# Do not let PHP free the assigned address while the engine
# still has a stale claim. The deletion outcome stays
# uncertain and requires explicit reconciliation.
raise PublicIpClaimCleanupPending(
"虚拟机已删除,但公网 IP 占用清理失败,必须人工核查",
503,
) from exc
try:
service.delete_firewall_filter(result["uuid"])
delete_firewall_record(profile, result["uuid"])
except Exception as exc:
LOG.exception(
"Could not remove firewall state for deleted VM %s",
result["uuid"],
)
raise VMDeleteFirewallCleanupPending(
"虚拟机已删除,但防火墙过滤器清理未完成;必须核查原删除操作"
) from exc
LOG.info(
"vm deleted: uuid=%s delete_disks=%s admin=%s",
result.get("uuid"), delete_disks, _session.username,
)
return result
@app.websocket("/api/v1/vms/{vm_id}/console")
async def console(websocket: WebSocket, vm_id: str):
host = websocket.headers.get("host")
if websocket.url.scheme != "wss":
await websocket.close(code=4403)
return
if not _origin_matches_host(websocket.headers.get("origin"), host):
await websocket.close(code=4403)
return
session = await asyncio.to_thread(sessions.get, websocket.cookies.get(COOKIE_NAME))
if session is None:
await websocket.close(code=4401)
return
try:
account = await asyncio.to_thread(account_for_session, session)
except HTTPException:
await websocket.close(code=4401)
return
if not account["admin"] and not account["approved"]:
await websocket.close(code=4403)
return
profile = manager.current()
if profile is None:
await websocket.close(code=4409)
return
try:
if profile.get("key_locked"):
await websocket.close(code=4423)
return
await asyncio.to_thread(require_vm_access, session, profile, vm_id)
port = await asyncio.to_thread(new_service(profile).console_port, vm_id)
except VMError:
await websocket.close(code=4404)
return
await websocket.accept()
async def watch_console_access() -> None:
while True:
await asyncio.sleep(CONSOLE_ACCESS_CHECK_SECONDS)
try:
refreshed = await asyncio.to_thread(sessions.get, session.token)
if refreshed is None or refreshed.username != session.username:
return
account = await asyncio.to_thread(account_for_session, refreshed)
if not account["admin"] and not account["approved"]:
return
current = manager.current()
if (current is None or current.get("key_locked")
or host_scope(current) != host_scope(profile)):
return
await asyncio.to_thread(require_vm_access, refreshed, current, vm_id)
except (HTTPException, VMError, DatabaseError, RemoteHostError,
KeyError, TypeError, ValueError):
return
relay_task = asyncio.create_task(relay_console(
websocket,
port=port,
ssh_target=profile["ssh_target"],
ssh_binary=settings.ssh_binary,
known_hosts_file=profile["ssh_known_hosts_file"],
ssh_config_file=profile["ssh_config_file"],
))
access_task = asyncio.create_task(watch_console_access())
try:
done, _pending = await asyncio.wait(
{relay_task, access_task}, return_when=asyncio.FIRST_COMPLETED)
if access_task in done:
relay_task.cancel()
await asyncio.gather(relay_task, return_exceptions=True)
with suppress(RuntimeError, OSError):
await websocket.close(code=4403)
return
await relay_task
except ConsoleRelayError as exc:
LOG.warning("VNC relay failed for vm %r: %s", vm_id, exc)
with suppress(RuntimeError, OSError):
await websocket.close(code=1011)
finally:
access_task.cancel()
await asyncio.gather(access_task, return_exceptions=True)
return app