You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/virsh_panel/app.py

2504 lines
117 KiB
Python

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

"""Browser setup, authenticated API, and VM console for one remote host."""
from __future__ import annotations
import asyncio
import hashlib
import hmac
import ipaddress
import json
import logging
import os
import subprocess
import tempfile
import threading
import time
import uuid
import xml.etree.ElementTree as ET
from concurrent.futures import ThreadPoolExecutor
from contextlib import asynccontextmanager, suppress
from pathlib import Path
from typing import Any, Callable, Literal
from urllib.parse import urlsplit
from fastapi import BackgroundTasks, Depends, FastAPI, HTTPException, Query, Request, WebSocket
from fastapi.responses import FileResponse, JSONResponse
from fastapi.staticfiles import StaticFiles
from pydantic import BaseModel, ConfigDict, Field, SecretStr
from .auth import DatabaseSessionStore, LoginLimiter, Session, SessionStore, verify_password
from .cloud_init import _validate_guest_auth, generate_root_password, hash_root_password
from .cloud_templates import (
TemplateError, TemplateStore, download_verified_image, image_version_for, template_id_for,
template_scope, template_volume_name, validate_template_name,
)
from .config import Settings
from .console import ConsoleRelayError, relay_console
from .database import DatabaseError, make_components
from .database_bootstrap import DatabaseBootstrap, DatabaseSetupError
from .libvirt_service import VMError, VMService
from .metrics import (MAX_QUERY_POINTS, RETENTION_SECONDS, RANGES, MetricsCollector,
host_scope, query_interval_seconds)
from .panel_state import PanelState, SetupError
from .remote_host import HostProfile, RemoteHostError, RemoteHostManager
from .rescue_templates import (
MAX_RESCUE_ISO_BYTES, RescueTemplateStore, rescue_scope,
rescue_volume_name, verify_bootable_iso,
)
from .ssh_keys import SSHKeyError
from .smtp_service import SMTPError, SMTPSettingsStore, send_verification_code
from .updater import BRANCH, REPOSITORY_URL, UpdateError, Updater
from .users import UserError, normalize_email
COOKIE_NAME = "virsh_panel_session"
CONSOLE_ACCESS_CHECK_SECONDS = 15
LOG = logging.getLogger(__name__)
STARTUP_LOG = logging.getLogger("uvicorn.error")
class SetupBody(BaseModel):
model_config = ConfigDict(extra="forbid")
token: str = Field(min_length=1, max_length=256)
username: str = Field(min_length=3, max_length=32)
password: str = Field(min_length=12, max_length=1024)
class DatabaseSetupBody(BaseModel):
model_config = ConfigDict(extra="forbid")
host: str = Field(min_length=1, max_length=253)
port: int = Field(ge=1, le=65535)
database: str = Field(min_length=1, max_length=64)
username: str = Field(min_length=1, max_length=128)
password: SecretStr
setup_token: str = Field(min_length=1, max_length=256)
class UpdateSettingsBody(BaseModel):
model_config = ConfigDict(extra="forbid")
repository_url: str = Field(default=REPOSITORY_URL, max_length=2048)
branch: str = Field(default=BRANCH, max_length=128)
access_token: SecretStr | None = None
class UpdateApplyBody(BaseModel):
model_config = ConfigDict(extra="forbid")
expected_current_commit: str = Field(pattern=r"^(?:[0-9a-f]{40}|[0-9a-f]{64})$")
expected_remote_commit: str = Field(pattern=r"^(?:[0-9a-f]{40}|[0-9a-f]{64})$")
class LoginBody(BaseModel):
model_config = ConfigDict(extra="forbid")
email: str | None = Field(default=None, min_length=3, max_length=254)
# Only the injected, database-free test harness may use the historical
# username login. Production explicitly rejects this field below.
username: str | None = Field(default=None, min_length=1, max_length=64)
password: str = Field(min_length=1, max_length=1024)
class RegistrationCodeBody(BaseModel):
model_config = ConfigDict(extra="forbid")
email: str = Field(min_length=3, max_length=254)
class RegistrationBody(RegistrationCodeBody):
username: str = Field(min_length=3, max_length=32)
password: str = Field(min_length=12, max_length=1024)
code: str = Field(pattern=r"^[0-9]{6}$")
class SMTPSettingsBody(BaseModel):
model_config = ConfigDict(extra="forbid")
host: str = Field(min_length=1, max_length=253)
port: int = Field(ge=1, le=65535)
tls_mode: Literal["ssl", "starttls", "none"]
username: str = Field(default="", max_length=256)
password: SecretStr | None = None
from_address: str = Field(min_length=3, max_length=254)
class UnlockBody(BaseModel):
model_config = ConfigDict(extra="forbid")
passphrase: str = Field(min_length=1, max_length=1024)
class HostScanBody(BaseModel):
model_config = ConfigDict(extra="forbid")
host: str = Field(min_length=1, max_length=253)
port: int = Field(ge=1, le=65535)
class HostBody(HostScanBody):
private_key: str = Field(default="", max_length=65536)
private_key_passphrase: str = Field(default="", max_length=1024)
host_key: str = Field(min_length=1, max_length=4096)
confirmed_fingerprint: str = Field(min_length=1, max_length=256)
storage_pool: str = Field(default="default", min_length=1, max_length=4096)
iso_pool: str = Field(default="iso", min_length=1, max_length=4096)
rescue_iso_pool: str = Field(default="", max_length=4096)
default_network: str = Field(default="default", min_length=1, max_length=128)
region: str = Field(default="", max_length=64,
pattern=r"^[^\x00-\x1f\x7f]*$")
public_pool_cidr: str = Field(default="", max_length=18)
public_gateway: str = Field(default="", max_length=15)
public_dns_servers: list[str] = Field(default_factory=list, max_length=4)
public_reserved_ips: list[str] = Field(default_factory=list, max_length=254)
public_vm_prefix: int | None = Field(default=None, ge=24, le=30)
def profile(self) -> HostProfile:
return HostProfile(
host=self.host,
port=self.port,
private_key=self.private_key,
host_key=self.host_key,
private_key_passphrase=self.private_key_passphrase,
storage_pool=self.storage_pool,
iso_pool=self.iso_pool,
rescue_iso_pool=self.rescue_iso_pool,
default_network=self.default_network,
region=self.region,
public_pool_cidr=self.public_pool_cidr,
public_gateway=self.public_gateway,
public_dns_servers=self.public_dns_servers,
public_reserved_ips=self.public_reserved_ips,
public_vm_prefix=self.public_vm_prefix,
)
class CreateISOBody(BaseModel):
model_config = ConfigDict(extra="forbid")
source: Literal["iso"] = "iso"
name: str = Field(min_length=1, max_length=63)
vcpu: int = Field(ge=1, le=64)
memory_mib: int = Field(ge=512, le=262144)
disk_gib: int = Field(ge=1, le=4096)
network: str = Field(min_length=1, max_length=128)
iso: str = Field(min_length=1, max_length=255)
class CreateCloudBody(BaseModel):
model_config = ConfigDict(extra="forbid")
source: Literal["cloud"]
name: str = Field(min_length=1, max_length=63)
vcpu: int = Field(ge=1, le=64)
memory_mib: int = Field(ge=512, le=262144)
disk_gib: int = Field(ge=1, le=4096)
network: str = Field(min_length=1, max_length=128)
template_id: str = Field(min_length=1, max_length=80)
public_ip_cidr: str = Field(default="", max_length=64)
gateway: str = Field(default="", max_length=64)
dns_servers: list[str] = Field(default_factory=list, max_length=4)
guest_ssh_public_key: str = Field(default="", max_length=16384)
guest_ssh_key_id: int | None = Field(default=None, ge=1)
root_password: SecretStr | None = None
class SSHKeyBody(BaseModel):
model_config = ConfigDict(extra="forbid")
name: str = Field(min_length=1, max_length=80)
public_key: str = Field(min_length=1, max_length=16384)
class ImportTemplateBody(BaseModel):
model_config = ConfigDict(extra="forbid")
url: str = Field(min_length=16, max_length=2048)
sha256: str = Field(min_length=64, max_length=64)
name: str = Field(min_length=1, max_length=64)
class RenameTemplateBody(BaseModel):
model_config = ConfigDict(extra="forbid")
name: str = Field(min_length=1, max_length=64)
class ReinstallBody(BaseModel):
model_config = ConfigDict(extra="forbid")
template_id: str = Field(min_length=1, max_length=80)
ssh_public_key: str = Field(default="", max_length=16384)
ssh_key_id: int | None = Field(default=None, ge=1)
root_password: SecretStr | None = None
confirmation_name: str = Field(min_length=1, max_length=63)
class EnterRescueBody(BaseModel):
model_config = ConfigDict(extra="forbid")
template_id: str = Field(pattern=r"^[0-9a-f]{32}$")
confirmation_name: str = Field(min_length=1, max_length=63)
class ExitRescueBody(BaseModel):
model_config = ConfigDict(extra="forbid")
confirmation_name: str = Field(min_length=1, max_length=63)
class ForceRescueBody(BaseModel):
model_config = ConfigDict(extra="forbid")
confirmation_name: str = Field(min_length=1, max_length=63)
ack_data_loss: Literal[True]
class ActionBody(BaseModel):
model_config = ConfigDict(extra="forbid")
action: Literal["start", "shutdown", "reboot", "poweroff", "reset"]
def _public_host(profile: dict | None) -> dict:
if profile is None:
return {"configured": False}
keys = (
"host", "port", "key_type", "fingerprint", "storage_pool",
"iso_pool", "default_network", "private_key_configured",
"private_key_encrypted", "key_locked", "private_key_passphrase_saved",
)
return {
"configured": True,
**{key: profile[key] for key in keys if key in profile},
"rescue_iso_pool": profile.get("rescue_iso_pool", ""),
"region": profile.get("region", ""),
"public_pool_cidr": profile.get("public_pool_cidr", ""),
"public_gateway": profile.get("public_gateway", ""),
"public_dns_servers": profile.get("public_dns_servers", []),
"public_reserved_ips": profile.get("public_reserved_ips", []),
"public_vm_prefix": profile.get("public_vm_prefix"),
}
def _peer(request: Request) -> str:
# Uvicorn has already processed forwarding headers from its trusted local
# reverse proxy. Never read a raw client-supplied X-Real-IP here, because
# that would let a login attacker reset the rate limit.
return request.client.host if request.client else "unknown"
def _origin_matches_host(origin: str | None, host: str | None) -> bool:
if not origin or not host:
return False
try:
parsed = urlsplit(origin)
except ValueError:
return False
if parsed.scheme != "https":
return False
return bool(
parsed.netloc == host
and parsed.hostname
and not parsed.username
and not parsed.password
and not parsed.path
and not parsed.query
and not parsed.fragment
)
def _plain_loopback_allowed(scope: dict, host: str | None) -> bool:
"""Permit local diagnostics only when both peer and requested host are local.
The request scheme is supplied by the ASGI server. Uvicorn's proxy header
handling may upgrade it to HTTPS/WSS only for its explicitly trusted proxy
address; this function never trusts an incoming forwarding header itself.
"""
client = scope.get("client")
if not client or not host:
return False
try:
peer_is_local = ipaddress.ip_address(client[0]).is_loopback
host_is_local = urlsplit("//" + host).hostname in {
"localhost", "127.0.0.1", "::1",
}
except ValueError:
return False
return peer_is_local and host_is_local
def create_app(
settings: Settings | None = None,
*,
panel_state: PanelState | None = None,
remote_manager: RemoteHostManager | None = None,
service_factory: Callable[[dict], VMService] | None = None,
metric_store: Any | None = None,
) -> FastAPI:
settings = settings or Settings.from_env()
settings.validate()
# Injected objects keep the existing isolated HTTP test harness working.
# Normal production startup never uses the old JSON-backed stores.
database_mode = panel_state is None and remote_manager is None
bootstrap = DatabaseBootstrap(settings.data_dir) if database_mode else None
data_store = None
updater = None
if database_mode:
configured_database = bootstrap.load_config()
if configured_database is None:
setup_token = bootstrap.ensure_token()
state = None
manager = None
templates = None
else:
state, manager, templates, data_store = make_components(
settings.data_dir, configured_database, bootstrap.master_key(),
settings.ssh_binary,
)
setup_token = (bootstrap.ensure_token()
if state.setup_required() else state.initialize_setup_token())
else:
state = panel_state or PanelState(settings.data_dir)
setup_token = state.initialize_setup_token()
manager = remote_manager or RemoteHostManager(
settings.data_dir, ssh_binary=settings.ssh_binary)
templates = TemplateStore(settings.data_dir)
if setup_token is not None:
# Uvicorn writes this logger to stderr, which the supplied systemd
# service captures in its journal. Never expose the token over HTTP.
STARTUP_LOG.warning("VIRSH_PANEL_SETUP_TOKEN=%s", setup_token)
if database_mode and (settings.data_dir / "generations").exists():
STARTUP_LOG.warning(
"Legacy data/generations detected; its old SSH key files are not migrated "
"or automatically removed. Verify and remove them separately."
)
import_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="cloud-image-import")
import_lock = threading.RLock()
import_jobs: dict[str, dict] = {}
pending_templates: set[tuple[str, str]] = set()
rescue_catalog = RescueTemplateStore(data_store)
rescue_import_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="rescue-iso-import")
rescue_import_lock = threading.RLock()
rescue_import_jobs: dict[str, dict] = {}
pending_rescue_imports: set[str] = set()
reinstall_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="vm-reinstall")
reinstall_lock = threading.RLock()
reinstall_jobs: dict[str, dict] = {}
pending_reinstalls: set[tuple[str, str]] = set()
rescue_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="vm-rescue")
rescue_lock = threading.RLock()
rescue_jobs: dict[str, dict] = {}
pending_rescues: set[tuple[str, str]] = set()
test_rescue_states: dict[str, str] = {}
# The production store encrypts each credential record with a fresh AES-GCM
# nonce. The in-memory fallback is only for injected, database-free tests.
test_credentials: dict[str, str] = {}
def ip_claim_key(profile: dict, address: str) -> str:
return f"{host_scope(profile)}.{address}"
def insert_public_ip_claim(profile: dict, address: str, vm_id: str) -> bool:
if data_store is None:
return True
claim = json.dumps({"vm_uuid": vm_id, "reserved_at": int(time.time())},
separators=(",", ":"))
return data_store.insert("ip_claim", ip_claim_key(profile, address), claim)
def allocate_public_ip(profile: dict, service: VMService,
vm_id: str) -> str:
"""Choose and reserve a public IP, including across panel processes.
Claims remain after a successful create and are released on panel VM
deletion. A claim from an uncertain create is retained rather than
risking reuse of an address whose guest may still exist.
"""
try:
network = ipaddress.IPv4Network(profile["public_pool_cidr"], strict=True)
gateway = ipaddress.IPv4Address(profile["public_gateway"])
guest_prefix = profile.get("public_vm_prefix")
if guest_prefix is None:
guest_prefix = network.prefixlen
if (not 24 <= network.prefixlen <= 30
or not network.network_address.is_global
or not network.broadcast_address.is_global
or gateway not in network
or gateway in (network.network_address, network.broadcast_address)
or isinstance(guest_prefix, bool)
or not isinstance(guest_prefix, int)
or not 24 <= guest_prefix <= network.prefixlen):
raise ValueError("Invalid host public IPv4 pool")
blocked = {network.network_address, network.broadcast_address, gateway}
for value in (*profile.get("public_reserved_ips", []),
*profile.get("public_dns_servers", [])):
address = ipaddress.ip_address(value)
if isinstance(address, ipaddress.IPv4Address) and address in network:
blocked.add(address)
try:
host_address = ipaddress.ip_address(profile["host"])
except ValueError:
host_address = None
if isinstance(host_address, ipaddress.IPv4Address) and host_address in network:
blocked.add(host_address)
except (KeyError, TypeError, ValueError) as exc:
raise VMError("宿主机公网地址池配置损坏,请管理员重新保存设置", 500) from exc
# Include all libvirt-visible panel IPs, regardless of VM ownership.
# VMService also checks duplicates under its own create lock.
for existing_vm in service.list_vms():
configured = existing_vm.get("public_ip_cidr")
if isinstance(configured, str):
try:
address = ipaddress.IPv4Address(configured.split("/", 1)[0])
except ValueError:
continue
if address in network:
blocked.add(address)
scope_prefix = f"{host_scope(profile)}."
if data_store is not None:
for key in data_store.list("ip_claim"):
if key.startswith(scope_prefix):
try:
address = ipaddress.IPv4Address(key[len(scope_prefix):])
except ValueError:
continue
if address in network:
blocked.add(address)
for address in network.hosts():
if address in blocked:
continue
if not insert_public_ip_claim(profile, str(address), vm_id):
continue
return f"{address}/{guest_prefix}"
raise VMError("公网地址池中没有可分配的 IP,请检查已用和保留地址", 409)
def claim_explicit_public_ip(profile: dict, public_ip_cidr: str,
vm_id: str) -> None:
if data_store is None:
return
try:
address = str(ipaddress.IPv4Interface(public_ip_cidr).ip)
except (TypeError, ValueError):
# The libvirt create path returns the detailed input error.
return
if not insert_public_ip_claim(profile, address, vm_id):
raise VMError("该公网 IP 已被面板中的虚拟机或创建任务占用", 409)
def release_public_ip_claim(profile: dict, public_ip_cidr: str,
vm_id: str) -> None:
if data_store is None:
return
try:
address = str(ipaddress.IPv4Interface(public_ip_cidr).ip)
key = ip_claim_key(profile, address)
except (TypeError, ValueError, KeyError):
return
raw = data_store.get("ip_claim", key)
if raw is None:
return
try:
claim = json.loads(raw)
except (TypeError, ValueError):
LOG.error("Could not parse public IP claim for %s", key)
return
if isinstance(claim, dict) and claim.get("vm_uuid") == vm_id:
data_store.delete("ip_claim", key)
def release_vm_public_ip_claims(profile: dict, vm_id: str) -> None:
if data_store is None:
return
prefix = f"{host_scope(profile)}."
for key, raw in data_store.list("ip_claim").items():
if not key.startswith(prefix):
continue
try:
claim = json.loads(raw)
except (TypeError, ValueError):
LOG.error("Could not parse public IP claim for %s", key)
continue
if isinstance(claim, dict) and claim.get("vm_uuid") == vm_id:
data_store.delete("ip_claim", key)
def credential_key(profile: dict, vm_id: str) -> str:
try:
canonical_id = str(uuid.UUID(vm_id))
return f"{host_scope(profile)}.{canonical_id}"
except (TypeError, ValueError, AttributeError, KeyError) as exc:
raise VMError("虚拟机或宿主机身份信息无效", 400) from exc
def save_credential(profile: dict, vm_id: str, auth_mode: str,
generation: str, password: str | None = None) -> None:
key = credential_key(profile, vm_id)
if not isinstance(generation, str) or not generation:
raise ValueError("虚拟机系统配置代号无效")
value = json.dumps({"auth_mode": auth_mode, "password": password,
"generation": generation},
ensure_ascii=False, separators=(",", ":"))
if data_store is not None:
data_store.set("vm_credential", key, value)
else:
test_credentials[key] = value
def delete_credential(profile: dict, vm_id: str) -> None:
key = credential_key(profile, vm_id)
if data_store is not None:
data_store.delete("vm_credential", key)
else:
test_credentials.pop(key, None)
def rescue_state_key(profile: dict, vm_id: str) -> str:
return credential_key(profile, vm_id)
def rescue_xml_matches(first: str, second: str) -> bool:
try:
return ET.canonicalize(first) == ET.canonicalize(second)
except (TypeError, ET.ParseError):
return False
def load_rescue_state(profile: dict, vm_id: str) -> dict | None:
key = rescue_state_key(profile, vm_id)
raw = (data_store.get("vm_rescue", key) if data_store is not None
else test_rescue_states.get(key))
if raw is None:
return None
try:
record = json.loads(raw)
except (TypeError, ValueError) as exc:
raise DatabaseError("虚拟机救援记录损坏") from exc
if (not isinstance(record, dict)
or not isinstance(record.get("original_xml"), str)
or not record["original_xml"].startswith("<domain")
or not isinstance(record.get("template_id"), str)
or record.get("volume_name") != rescue_volume_name(record["template_id"])
or record.get("status") not in {
"queued", "running", "staged", "active", "failed", "exiting",
}):
raise DatabaseError("虚拟机救援记录损坏")
return record
def save_rescue_state(profile: dict, vm_id: str, record: dict) -> None:
key = rescue_state_key(profile, vm_id)
value = json.dumps(record, ensure_ascii=False, separators=(",", ":"))
if data_store is not None:
data_store.set("vm_rescue", key, value)
else:
test_rescue_states[key] = value
def delete_rescue_state(profile: dict, vm_id: str) -> None:
key = rescue_state_key(profile, vm_id)
if data_store is not None:
data_store.delete("vm_rescue", key)
else:
test_rescue_states.pop(key, None)
def current_rescue_state(profile: dict, vm_id: str,
service: VMService | None = None) -> dict | None:
"""Clear a stale record only when the original inactive XML is intact.
An interrupted worker may have stopped before changing the VM. The
database job is marked failed on restart, but its recovery record must
not indefinitely block normal operations in that safe case.
Callers hold rescue_lock so this cannot race a live rescue worker.
"""
record = load_rescue_state(profile, vm_id)
if record is None or (host_scope(profile), vm_id) in pending_rescues:
return record
service = service or new_service(profile)
current_vm = service.get_vm(vm_id)
if current_vm.get("rescue_mode") or current_vm.get("rescue_staged"):
return record
try:
original_still_defined = rescue_xml_matches(
service.prepare_rescue_vm(vm_id, record["volume_name"]),
record["original_xml"],
)
except VMError:
return record
if original_still_defined:
delete_rescue_state(profile, vm_id)
return None
return record
def parse_credential(raw: str | None) -> dict | None:
if raw is None:
return None
try:
value = json.loads(raw)
except (TypeError, ValueError) as exc:
raise DatabaseError("虚拟机登录凭据记录损坏") from exc
if (not isinstance(value, dict)
or value.get("auth_mode") not in ("password", "ssh_key", "ssh_key_password")
or (value["auth_mode"] in ("password", "ssh_key_password")
and (not isinstance(value.get("password"), str)
or not value["password"]))
or not isinstance(value.get("generation"), str)
or not value["generation"]
or (value["auth_mode"] == "ssh_key" and value.get("password") is not None)):
raise DatabaseError("虚拟机登录凭据记录损坏")
return value
def get_credential(profile: dict, vm_id: str) -> dict | None:
key = credential_key(profile, vm_id)
raw = (data_store.get("vm_credential", key) if data_store is not None
else test_credentials.get(key))
return parse_credential(raw)
def decorate_vm(vm: dict, credential: dict | None, profile: dict) -> dict:
cloud = bool(vm.get("managed") and vm.get("os_source") == "cloud_template")
current_credential = (credential if cloud and credential
and credential["generation"] == vm.get("credential_generation")
else None)
return {
**{key: value for key, value in vm.items() if key != "credential_generation"},
"region": profile.get("region", ""),
"login_name": "root" if cloud else None,
"auth_mode": current_credential["auth_mode"] if current_credential else "unknown",
"password_available": bool(current_credential and
current_credential["auth_mode"] in (
"password", "ssh_key_password")),
}
def save_job(kind: str, job_id: str, job: dict) -> None:
if data_store is not None:
data_store.set(kind, job_id, json.dumps(
job, ensure_ascii=False, separators=(",", ":")))
def delete_job(kind: str, job_id: str) -> None:
if data_store is not None:
data_store.delete(kind, job_id)
def restore_jobs() -> None:
"""Expose interrupted jobs as failed, never pretend work resumed."""
if data_store is None:
return
for kind, destination in (("import_job", import_jobs),
("reinstall_job", reinstall_jobs),
("rescue_import_job", rescue_import_jobs),
("rescue_job", rescue_jobs)):
for job_id, raw in data_store.list(kind).items():
try:
job = json.loads(raw)
except (TypeError, ValueError) as exc:
raise DatabaseError("数据库任务记录损坏") from exc
if not isinstance(job, dict):
raise DatabaseError("数据库任务记录损坏")
if job.get("status") in {"queued", "running"}:
job["status"] = "failed"
job["phase"] = "面板重启,任务已中断"
job["error"] = "面板在任务执行期间重启;请检查宿主机实际状态,勿直接重复提交"
save_job(kind, job_id, job)
destination[job_id] = job
restore_jobs()
sessions = DatabaseSessionStore(data_store) if data_store is not None else SessionStore()
login_limiter = LoginLimiter()
registration_limiter = LoginLimiter()
code_send_limiter = LoginLimiter()
setup_limiter = LoginLimiter()
unlock_limiter = LoginLimiter()
def metric_profile():
if manager is None:
return None
background = getattr(manager, "current_for_background", None)
return background() if callable(background) else manager.current()
metric_collector = MetricsCollector(
lambda: data_store,
metric_profile,
lambda profile: new_service(profile),
) if database_mode else None
@asynccontextmanager
async def lifespan(_app: FastAPI):
if metric_collector is not None:
metric_collector.start()
try:
yield
finally:
if metric_collector is not None:
metric_collector.close()
reinstall_executor.shutdown(wait=True, cancel_futures=False)
import_executor.shutdown(wait=True, cancel_futures=False)
rescue_executor.shutdown(wait=True, cancel_futures=False)
rescue_import_executor.shutdown(wait=True, cancel_futures=False)
close_manager = getattr(manager, "close", None) if manager is not None else None
if callable(close_manager):
close_manager()
app = FastAPI(title="Virsh Panel API", docs_url=None, redoc_url=None, lifespan=lifespan)
app.mount("/static", StaticFiles(directory=str(settings.static_dir)), name="static")
app.mount(
"/vendor/novnc",
StaticFiles(directory=str(settings.static_dir / "vendor" / "novnc")),
name="novnc",
)
@app.middleware("http")
async def security_headers(request: Request, call_next):
if request.url.scheme != "https" and not (
request.url.path == "/api/v1/health"
and request.url.scheme == "http"
and _plain_loopback_allowed(request.scope, request.headers.get("host"))
):
return JSONResponse(status_code=400, content={"error": "请使用 HTTPS 访问面板"})
response = await call_next(request)
response.headers["X-Content-Type-Options"] = "nosniff"
response.headers["Referrer-Policy"] = "no-referrer"
response.headers["X-Frame-Options"] = "DENY"
response.headers["Content-Security-Policy"] = (
"default-src 'self'; script-src 'self'; style-src 'self'; "
"img-src 'self' data:; connect-src 'self' wss:; "
"object-src 'none'; frame-ancestors 'none'; base-uri 'self'"
)
if (request.url.path == "/" or request.url.path.startswith("/api/")
or request.url.path in ("/static/app.js", "/static/styles.css")):
response.headers["Cache-Control"] = "no-store"
return response
@app.exception_handler(VMError)
async def handle_vm_error(_request: Request, exc: VMError):
return JSONResponse(status_code=exc.status, content={"error": str(exc)})
@app.exception_handler(RemoteHostError)
async def handle_remote_error(_request: Request, exc: RemoteHostError):
content = {"error": str(exc)}
if exc.status == 423:
content["error_code"] = "key_locked"
return JSONResponse(status_code=exc.status, content=content)
@app.exception_handler(SetupError)
async def handle_setup_error(_request: Request, exc: SetupError):
return JSONResponse(status_code=exc.status, content={"error": str(exc)})
@app.exception_handler(DatabaseSetupError)
async def handle_database_setup_error(_request: Request, exc: DatabaseSetupError):
return JSONResponse(status_code=exc.status, content={"error": str(exc)})
@app.exception_handler(DatabaseError)
async def handle_database_error(_request: Request, exc: DatabaseError):
return JSONResponse(status_code=exc.status, content={"error": str(exc)})
@app.exception_handler(UpdateError)
async def handle_update_error(_request: Request, exc: UpdateError):
return JSONResponse(status_code=exc.status, content={"error": str(exc)})
@app.exception_handler(TemplateError)
async def handle_template_error(_request: Request, exc: TemplateError):
return JSONResponse(status_code=exc.status, content={"error": str(exc)})
@app.exception_handler(UserError)
async def handle_user_error(_request: Request, exc: UserError):
return JSONResponse(status_code=exc.status, content={"error": str(exc)})
@app.exception_handler(SMTPError)
async def handle_smtp_error(_request: Request, exc: SMTPError):
return JSONResponse(status_code=exc.status, content={"error": str(exc)})
@app.exception_handler(SSHKeyError)
async def handle_ssh_key_error(_request: Request, exc: SSHKeyError):
return JSONResponse(status_code=exc.status, content={"error": str(exc)})
def account_for_session(session: Session) -> dict:
if database_mode:
if state is None:
raise HTTPException(status_code=401, detail="请先登录")
user_id = getattr(session, "user_id", None)
if isinstance(user_id, bool) or not isinstance(user_id, int) or user_id < 1:
raise HTTPException(status_code=401, detail="账号不可用,请重新登录")
account = state.users.get_by_id(user_id)
if (account is None or account["username"] != session.username
or not account["email_verified"]):
raise HTTPException(status_code=401, detail="账号不可用,请重新登录")
return account
# The injected JSON state is retained solely for the isolated HTTP
# test harness; production authorization always uses the users table.
return {"id": 0, "username": session.username, "admin": True,
"approved": True, "email_verified": True}
def host_ready() -> bool:
if manager is None:
return False
profile = manager.current()
return profile is not None and not profile.get("key_locked", False)
def require_session(request: Request) -> Session:
session = sessions.get(request.cookies.get(COOKIE_NAME))
if session is None:
raise HTTPException(status_code=401, detail="请先登录")
account_for_session(session)
return session
def require_write(request: Request) -> Session:
session = require_session(request)
csrf = request.headers.get("X-CSRF-Token", "")
if not hmac.compare_digest(csrf, session.csrf_token):
raise HTTPException(status_code=403, detail="请求验证失败")
return session
def require_admin(request: Request) -> Session:
session = require_session(request)
if not account_for_session(session)["admin"]:
raise HTTPException(status_code=403, detail="仅管理员可以执行此操作")
return session
def require_admin_write(request: Request) -> Session:
session = require_write(request)
if not account_for_session(session)["admin"]:
raise HTTPException(status_code=403, detail="仅管理员可以执行此操作")
return session
def require_approved(request: Request) -> Session:
session = require_session(request)
account = account_for_session(session)
if not account["admin"] and not account["approved"]:
raise HTTPException(status_code=403, detail="账号等待管理员批准")
return session
def require_approved_write(request: Request) -> Session:
session = require_write(request)
account = account_for_session(session)
if not account["admin"] and not account["approved"]:
raise HTTPException(status_code=403, detail="账号等待管理员批准")
return session
def require_vm_access(session: Session, profile: dict, vm_id: str) -> str:
if not database_mode:
return vm_id
try:
canonical_id = str(uuid.UUID(vm_id))
except (TypeError, ValueError, AttributeError) as exc:
raise VMError("虚拟机 UUID 无效", 400) from exc
account = account_for_session(session)
if not account["admin"] and not state.users.owns_vm(
host_scope(profile), canonical_id, account["id"]):
raise VMError("虚拟机不存在", 404)
return canonical_id
def smtp_store() -> SMTPSettingsStore:
if data_store is None:
raise SMTPError("请先完成数据库配置", 409)
return SMTPSettingsStore(data_store)
def ssh_key_store():
if not database_mode or state is None:
raise SSHKeyError("请先完成数据库配置", 409)
return state.ssh_keys
def current_updater() -> Updater:
nonlocal updater
if data_store is None:
raise UpdateError("请先完成数据库配置", 409)
if updater is None:
updater = Updater(settings.static_dir.parent, data_store)
return updater
def forbid_long_job_during_update() -> None:
if data_store is None:
return
active_id = data_store.get_value("update.active_job")
if active_id:
job = current_updater().get_job(active_id)
if job is not None and job.get("status") in {"queued", "running"}:
raise HTTPException(status_code=409, detail="面板正在更新,暂时不能开始新任务")
def current_profile() -> dict:
profile = manager.current()
if profile is None:
raise HTTPException(status_code=409, detail="请先在面板中配置宿主机")
if profile.get("key_locked"):
raise RemoteHostError("宿主机 SSH 私钥已锁定,请先在面板解锁", 423)
return profile
def new_service(profile: dict) -> VMService:
if service_factory is not None:
return service_factory(profile)
return VMService(
profile["libvirt_uri"],
profile["storage_pool"],
profile["iso_pool"],
profile["default_network"],
rescue_iso_pool=profile.get("rescue_iso_pool", ""),
)
def verified_created_vm(profile: dict, result: dict, requested_uuid: str) -> dict:
if result.get("uuid") == requested_uuid:
return result
# Keep the preallocated owner/credential records if libvirt may have
# created the requested guest. Never delete a returned, mismatched UUID:
# it may refer to a different existing VM.
LOG.error("VM create returned a mismatched UUID: expected=%s", requested_uuid)
try:
verified = new_service(profile).get_vm(requested_uuid)
except VMError:
verified = None
if verified is not None and verified.get("uuid") == requested_uuid:
LOG.warning("Recovered VM result by querying the requested UUID: %s", requested_uuid)
return verified
raise VMError("虚拟机创建结果与预分配 UUID 不一致;请由管理员核查宿主机及归属记录", 502)
def current_service() -> VMService:
return new_service(current_profile())
def validate_staged_libvirt(
staged, body: HostBody, *, ensure_paths: bool = False,
) -> tuple[dict | None, str | None]:
service = new_service({
"libvirt_uri": staged.libvirt_uri,
"storage_pool": body.storage_pool,
"iso_pool": body.iso_pool,
"rescue_iso_pool": body.rescue_iso_pool,
"default_network": body.default_network,
})
# A successful libvirt API call is required before activation. Missing
# storage/network resources should not prevent connecting an otherwise
# healthy host; the UI can guide the operator to fix those separately.
service.list_vms()
if ensure_paths and (body.rescue_iso_pool or any(path.startswith("/") for path in (
body.storage_pool, body.iso_pool))):
service.ensure_storage_pools()
try:
return service.resources(), None
except VMError as exc:
return None, str(exc)
def session_response(username: str) -> JSONResponse:
if database_mode:
account = state.users.get_by_username(username)
if account is None:
raise HTTPException(status_code=401, detail="账号不可用,请重新登录")
session = sessions.issue(username, user_id=account["id"])
else:
session = sessions.issue(username)
account = account_for_session(session)
response = JSONResponse({
"authenticated": True,
"username": session.username,
"csrf_token": session.csrf_token,
"admin": account["admin"],
"approved": account["approved"],
"host_ready": host_ready(),
})
response.set_cookie(
COOKIE_NAME,
session.token,
max_age=8 * 60 * 60,
httponly=True,
secure=settings.secure_cookie,
samesite="strict",
path="/",
)
return response
@app.get("/", include_in_schema=False)
def index():
return FileResponse(settings.static_dir / "index.html")
@app.get("/api/v1/health")
def health():
if data_store is not None:
# A running HTTP listener alone is not a usable post-update panel.
# The updater treats this endpoint as readiness for the configured DB.
if database_mode:
state.users.has_users()
else:
data_store.get("admin", "singleton")
return {"status": "ok"}
@app.get("/api/v1/database/status")
def database_status():
if bootstrap is None:
return {"configured": True, "setup_required": False}
return {"configured": bootstrap.configured(),
"setup_required": not bootstrap.configured()}
@app.post("/api/v1/database/setup")
def setup_database(body: DatabaseSetupBody, request: Request):
nonlocal state, manager, templates, data_store, updater, sessions, rescue_catalog
if bootstrap is None:
raise DatabaseSetupError("数据库初始化仅供正常部署使用", 409)
peer = _peer(request)
if not setup_limiter.allowed(peer):
raise HTTPException(status_code=429, detail="初始化尝试过多,请稍后再试")
def connect(config: dict, master_key: bytes):
try:
return make_components(
settings.data_dir, config, master_key, settings.ssh_binary)
except Exception as exc:
# Connector errors may include database host details. Never
# reflect the exception, credentials or driver SQL in HTTP.
LOG.warning("database setup connection failed: %s", type(exc).__name__)
raise DatabaseSetupError(
"无法连接或初始化 MariaDB;请检查地址、账号、数据库和建表权限", 502,
) from exc
try:
components = bootstrap.configure({
"host": body.host,
"port": body.port,
"database": body.database,
"username": body.username,
"password": body.password.get_secret_value(),
}, body.setup_token, connect)
except DatabaseSetupError:
setup_limiter.failed(peer)
raise
state, manager, templates, data_store = components
rescue_catalog = RescueTemplateStore(data_store)
sessions = DatabaseSessionStore(data_store)
restore_jobs()
setup_limiter.succeeded(peer)
LOG.info("panel database configured")
return {"configured": True}
@app.get("/api/v1/setup/status")
def setup_status():
if state is None:
raise HTTPException(status_code=409, detail="请先配置数据库")
return {"setup_required": state.setup_required()}
@app.post("/api/v1/setup/admin")
def setup_admin(body: SetupBody, request: Request):
if state is None:
raise HTTPException(status_code=409, detail="请先配置数据库")
peer = _peer(request)
if not setup_limiter.allowed(peer):
raise HTTPException(status_code=429, detail="初始化尝试过多,请稍后再试")
try:
username = state.create_admin(body.token, body.username, body.password)
except SetupError:
setup_limiter.failed(peer)
raise
setup_limiter.succeeded(peer)
LOG.info("panel initial user created: username=%s", username)
return session_response(username)
@app.get("/api/v1/session")
def session_info(request: Request):
session = sessions.get(request.cookies.get(COOKIE_NAME))
if session is None:
return {"authenticated": False, "username": None, "csrf_token": None}
try:
account = account_for_session(session)
except HTTPException:
return {"authenticated": False, "username": None, "csrf_token": None}
return {
"authenticated": True,
"username": session.username,
"csrf_token": session.csrf_token,
"admin": account["admin"],
"approved": account["approved"],
"host_ready": host_ready(),
}
@app.post("/api/v1/login")
def login(body: LoginBody, request: Request):
if state is None:
raise HTTPException(status_code=409, detail="请先配置数据库")
peer = _peer(request)
if not login_limiter.allowed(peer):
raise HTTPException(status_code=429, detail="登录尝试过多,请稍后再试")
if database_mode:
if body.email is None or body.username is not None:
raise HTTPException(status_code=422, detail="请使用邮箱登录")
try:
email = normalize_email(body.email)
except UserError:
login_limiter.failed(peer)
raise HTTPException(status_code=401, detail="邮箱或密码错误") from None
if not state.users.has_users():
raise HTTPException(status_code=409, detail="请先完成初始化")
account = state.users.get_by_email(email)
encoded = account["password_hash"] if account else None
username = account["username"] if account else ""
identity_ok = account is not None
else:
legacy_admin = state.admin()
if legacy_admin is None:
raise HTTPException(status_code=409, detail="请先完成初始化")
username, encoded = legacy_admin
account = {"email_verified": True}
identity_ok = hmac.compare_digest(body.username or "", username)
password_ok = verify_password(body.password, encoded) if encoded else False
if not password_ok or not identity_ok or not account["email_verified"]:
login_limiter.failed(peer)
raise HTTPException(status_code=401, detail="邮箱或密码错误")
login_limiter.succeeded(peer)
return session_response(username)
def deliver_registration_code(config, email: str, code: str, users) -> None:
try:
send_verification_code(config, email, code)
except Exception as exc:
# Background delivery must not expose whether an email exists or
# relay SMTP server diagnostics to an unauthenticated client.
LOG.warning("Registration email delivery failed: %s", type(exc).__name__)
try:
users.revoke_registration_code(email, code)
except DatabaseError:
LOG.exception("Could not revoke undelivered registration code")
@app.post("/api/v1/register/code")
def send_registration_code(body: RegistrationCodeBody, request: Request,
background_tasks: BackgroundTasks):
if not database_mode or state is None or not state.users.has_users():
raise HTTPException(status_code=409, detail="请先完成面板初始化")
peer = _peer(request)
if not code_send_limiter.allowed(peer):
raise HTTPException(status_code=429, detail="验证码请求过多,请稍后再试")
email = normalize_email(body.email)
config = smtp_store().get()
if config is None:
raise SMTPError("管理员尚未配置 SMTP,暂不能注册", 503)
code = state.users.generate_code()
try:
should_send = state.users.issue_registration_code(email, code)
except UserError as exc:
if exc.status != 429:
raise
should_send = False
code_send_limiter.failed(peer)
if should_send:
# Sending after the HTTP response avoids exposing whether an
# address is registered via SMTP latency or transport failures.
background_tasks.add_task(
deliver_registration_code, config, email, code, state.users,
)
# Identical public response for existing addresses and cooldowns.
return {"sent": True}
@app.post("/api/v1/register", status_code=201)
def register(body: RegistrationBody, request: Request):
if not database_mode or state is None or not state.users.has_users():
raise HTTPException(status_code=409, detail="请先完成面板初始化")
peer = _peer(request)
if not registration_limiter.allowed(peer):
raise HTTPException(status_code=429, detail="注册尝试过多,请稍后再试")
try:
state.users.register_with_code(body.username, body.email,
body.password, body.code)
except UserError:
registration_limiter.failed(peer)
raise
registration_limiter.succeeded(peer)
return {"registered": True, "approved": False}
@app.post("/api/v1/logout")
def logout(request: Request, _session: Session = Depends(require_write)):
sessions.revoke(request.cookies.get(COOKIE_NAME))
response = JSONResponse({"ok": True})
response.delete_cookie(COOKIE_NAME, path="/")
return response
@app.get("/api/v1/ssh-keys")
def list_ssh_keys(_session: Session = Depends(require_session)):
account = account_for_session(_session)
return {"items": ssh_key_store().list_for_user(account["id"])}
@app.post("/api/v1/ssh-keys", status_code=201)
def create_ssh_key(body: SSHKeyBody,
_session: Session = Depends(require_write)):
account = account_for_session(_session)
return ssh_key_store().create_for_user(
account["id"], body.name, body.public_key,
)
@app.delete("/api/v1/ssh-keys/{key_id}")
def delete_ssh_key(key_id: int, _session: Session = Depends(require_write)):
if key_id < 1:
raise SSHKeyError("公钥不存在", 404)
account = account_for_session(_session)
if not ssh_key_store().delete_for_user(account["id"], key_id):
raise SSHKeyError("公钥不存在", 404)
return {"deleted": True}
@app.get("/api/v1/smtp/settings")
def get_smtp_settings(_session: Session = Depends(require_admin)):
return smtp_store().public_status()
@app.put("/api/v1/smtp/settings")
def save_smtp_settings(body: SMTPSettingsBody,
_session: Session = Depends(require_admin_write)):
settings_data = body.model_dump(exclude={"password"})
settings_data["password"] = (body.password.get_secret_value()
if body.password is not None else "")
config = smtp_store().save(settings_data)
LOG.info("SMTP settings changed: admin=%s", _session.username)
return config.public_status()
@app.get("/api/v1/users/pending")
def pending_users(_session: Session = Depends(require_admin)):
if not database_mode:
return {"items": []}
return {"items": state.users.list_pending_users()}
@app.post("/api/v1/users/{user_id}/approve")
def approve_user(user_id: int, _session: Session = Depends(require_admin_write)):
if not database_mode:
raise HTTPException(status_code=409, detail="数据库用户功能不可用")
account = state.users.approve_user(user_id)
if account is None:
raise HTTPException(status_code=404, detail="用户不存在")
LOG.info("user approved: id=%s admin=%s", user_id, _session.username)
return {"approved": True}
def public_update_settings() -> dict:
info = current_updater().settings()
return {
"repository_url": info["repository"],
"branch": info["branch"],
"token_configured": info["token_configured"],
"checkout_ready": info["checkout_ready"],
"configured": info["token_configured"] and info["checkout_ready"],
"current_version": info["current_commit"],
"active_job_id": info.get("active_job_id"),
"last_job_id": info.get("last_job_id"),
"checkout_error": info.get("checkout_error"),
}
@app.get("/api/v1/update/settings")
def update_settings(_session: Session = Depends(require_admin)):
return public_update_settings()
@app.put("/api/v1/update/settings")
def save_update_settings(body: UpdateSettingsBody,
_session: Session = Depends(require_admin_write)):
if body.repository_url != REPOSITORY_URL or body.branch != BRANCH:
raise UpdateError("更新仓库和分支固定为面板指定的 Gitea main 分支", 422)
service = current_updater()
if body.access_token is not None:
token = body.access_token.get_secret_value()
if token:
service.set_token(token)
if not service.settings()["token_configured"]:
raise UpdateError("请填写 Gitea 私有仓库访问令牌", 422)
LOG.info("panel update access token configured: admin=%s", _session.username)
return public_update_settings()
@app.post("/api/v1/update/check")
def check_update(_session: Session = Depends(require_admin_write)):
result = current_updater().check()
return {
"repository_url": result["repository"],
"branch": result["branch"],
"current_version": result["current_commit"],
"latest_version": result["remote_commit"],
"update_available": result["status"] == "update_available",
}
@app.post("/api/v1/update/apply", status_code=202)
def apply_update(body: UpdateApplyBody, _session: Session = Depends(require_admin_write)):
with import_lock, reinstall_lock, rescue_lock, rescue_import_lock:
if (pending_templates or pending_reinstalls or pending_rescues
or pending_rescue_imports):
raise UpdateError("镜像导入或虚拟机任务正在执行,请等待完成后再更新面板", 409)
try:
unit = subprocess.run(
["/usr/bin/systemctl", "is-active", "virsh-panel-update.service"],
capture_output=True, text=True, timeout=5, check=False,
)
except (OSError, subprocess.TimeoutExpired):
unit = None
if unit is not None and unit.stdout.strip() in {
"activating", "active", "deactivating", "reloading",
}:
raise UpdateError("独立更新服务仍在运行,请等待其完全结束后再提交", 409)
service = current_updater()
job = service.create_job(
expected_current_commit=body.expected_current_commit,
expected_remote_commit=body.expected_remote_commit,
)
try:
# The web service has a read-only code tree. A separate one-shot
# unit performs the Git update and restarts this service.
trigger = subprocess.run(
["/usr/bin/systemctl", "--no-block", "start", "virsh-panel-update.service"],
capture_output=True, text=True, timeout=10, check=False,
)
except (OSError, subprocess.TimeoutExpired):
trigger = None
if trigger is None or trigger.returncode != 0:
service.fail_job(job["job_id"],
"无法启动独立更新服务;请检查 virsh-panel-update.service 是否已安装")
raise UpdateError("无法启动独立更新服务;请检查 systemd 配置", 503)
LOG.info("panel update queued: job=%s admin=%s", job["job_id"], _session.username)
return {"job_id": job["job_id"], "status": "queued"}
@app.get("/api/v1/update/jobs/{job_id}")
def update_job(job_id: str, _session: Session = Depends(require_admin)):
job = current_updater().get_job(job_id)
if job is None:
raise UpdateError("更新任务不存在", 404)
return job
@app.get("/api/v1/host")
def host(_session: Session = Depends(require_admin)):
return _public_host(manager.current())
@app.post("/api/v1/host/scan")
def scan_host(body: HostScanBody, _session: Session = Depends(require_admin_write)):
return manager.scan(body.host, body.port)
@app.post("/api/v1/host/unlock")
def unlock_host(body: UnlockBody, request: Request, _session: Session = Depends(require_admin_write)):
peer = _peer(request)
if not unlock_limiter.allowed(peer):
raise HTTPException(status_code=429, detail="私钥口令尝试过多,请稍后再试")
try:
profile = manager.unlock_current(body.passphrase)
except RemoteHostError as exc:
if exc.status == 400:
unlock_limiter.failed(peer)
raise
unlock_limiter.succeeded(peer)
LOG.info("remote host SSH key unlocked: host=%s admin=%s", profile.get("host"), _session.username)
return _public_host(profile)
@app.post("/api/v1/host/test")
def test_host(body: HostBody, _session: Session = Depends(require_admin_write)):
staged = manager.stage(body.profile(), confirmed_fingerprint=body.confirmed_fingerprint)
try:
result = manager.test(staged)
resources, resource_warning = validate_staged_libvirt(staged, body)
return {
**result,
"resources": resources,
"warnings": [resource_warning] if resource_warning else [],
}
finally:
manager.abort(staged)
@app.put("/api/v1/host")
def save_host(body: HostBody, _session: Session = Depends(require_admin_write)):
# A running reinstall uses the saved SSH host identity and paths. Do
# not replace them while its libvirt operation is in flight.
with reinstall_lock, rescue_lock, rescue_import_lock:
forbid_long_job_during_update()
if (pending_reinstalls or pending_rescues or pending_rescue_imports
or (data_store is not None and data_store.list("vm_rescue"))):
raise VMError("请先完成救援或重装任务并退出救援模式,再修改宿主机配置", 409)
if any(
job.get("status") == "succeeded" and "generated_password" in job
for job in reinstall_jobs.values()):
raise VMError("请先完成重装任务并确认保存自动生成的密码,再修改宿主机配置", 409)
staged = manager.stage(body.profile(), confirmed_fingerprint=body.confirmed_fingerprint)
committed = False
try:
manager.test(staged)
_resources, resource_warning = validate_staged_libvirt(staged, body, ensure_paths=True)
profile = manager.commit(staged)
committed = True
finally:
if not committed:
manager.abort(staged)
LOG.info("remote host configured: host=%s port=%s admin=%s", body.host, body.port, _session.username)
return {
**_public_host(profile),
"warnings": [resource_warning] if resource_warning else [],
}
@app.get("/api/v1/resources")
def resources(_session: Session = Depends(require_approved)):
profile = current_profile()
return {
**new_service(profile).resources(),
"public_pool_cidr": profile.get("public_pool_cidr", ""),
"public_pool_enabled": bool(profile.get("public_pool_cidr")),
"public_vm_prefix": profile.get("public_vm_prefix"),
}
@app.get("/api/v1/vm-templates")
def list_templates(_session: Session = Depends(require_approved)):
profile = current_profile()
return {"templates": templates.list(template_scope(profile))}
def run_template_import(job_id: str, profile: dict, scope: str,
body: ImportTemplateBody, template_id: str) -> None:
image_path = None
volume_name = template_volume_name(template_id)
uploaded = False
try:
image_path, virtual_size = download_verified_image(
body.url, body.sha256.lower(), settings.data_dir)
result = new_service(profile).import_template(image_path, volume_name, virtual_size)
uploaded = True
record = {
"id": template_id,
"name": body.name,
"os_family": "AlmaLinux",
"os_version": image_version_for(body.url),
"sha256": body.sha256.lower(),
"volume_name": volume_name,
"virtual_size": virtual_size,
"image_bytes": result["image_bytes"],
}
templates.add(scope, record)
with import_lock:
import_jobs[job_id] = {"scope": scope, "status": "completed",
"template": record}
save_job("import_job", job_id, import_jobs[job_id])
except Exception as exc:
if isinstance(exc, (TemplateError, VMError)):
error = str(exc)
else:
LOG.exception("Cloud template import failed: job=%s", job_id)
error = "云镜像导入失败,请查看面板日志"
if uploaded:
error += f";宿主机模板卷 {volume_name} 可能需要人工检查"
with import_lock:
import_jobs[job_id] = {"scope": scope, "status": "failed", "error": error}
save_job("import_job", job_id, import_jobs[job_id])
finally:
if image_path is not None:
image_path.unlink(missing_ok=True)
with import_lock:
pending_templates.discard((scope, template_id))
@app.post("/api/v1/vm-templates/import", status_code=202)
def import_template(body: ImportTemplateBody, _session: Session = Depends(require_admin_write)):
profile = current_profile()
version = image_version_for(body.url)
name = validate_template_name(body.name)
body = body.model_copy(update={"name": name})
if not all(char in "0123456789abcdefABCDEF" for char in body.sha256):
raise TemplateError("SHA-256 校验值必须是十六进制")
scope = template_scope(profile)
template_id = template_id_for(scope, body.sha256.lower(), version)
with import_lock:
forbid_long_job_during_update()
existing_templates = templates.list(scope)
if any(row["id"] == template_id for row in existing_templates):
raise TemplateError("此云镜像已经导入", 409)
if any(row["name"].casefold() == name.casefold()
for row in existing_templates):
raise TemplateError("当前宿主机已有同名云镜像模板", 409)
if (scope, template_id) in pending_templates:
raise TemplateError("该云镜像正在导入", 409)
if pending_templates:
raise TemplateError("已有云镜像正在导入,请等待完成后再提交", 409)
job_id = str(uuid.uuid4())
pending_templates.add((scope, template_id))
import_jobs[job_id] = {"scope": scope, "status": "running"}
save_job("import_job", job_id, import_jobs[job_id])
try:
import_executor.submit(run_template_import, job_id, profile,
scope, body, template_id)
except RuntimeError as exc:
pending_templates.discard((scope, template_id))
import_jobs.pop(job_id, None)
delete_job("import_job", job_id)
raise TemplateError("面板正在停止,无法开始导入任务", 503) from exc
LOG.info("cloud template import queued: job=%s admin=%s", job_id, _session.username)
return {"job_id": job_id, "status": "running"}
@app.patch("/api/v1/vm-templates/{template_id}")
def rename_template(template_id: str, body: RenameTemplateBody,
_session: Session = Depends(require_admin_write)):
scope = template_scope(current_profile())
with import_lock:
if pending_templates:
raise TemplateError("云镜像正在导入,请稍后修改模板名称", 409)
return {"template": templates.rename(scope, template_id, body.name)}
@app.get("/api/v1/vm-templates/import-jobs/{job_id}")
def import_job(job_id: str, _session: Session = Depends(require_admin)):
scope = template_scope(current_profile())
with import_lock:
job = import_jobs.get(job_id)
if job is None or job["scope"] != scope:
raise TemplateError("导入任务不存在", 404)
return {key: value for key, value in job.items() if key != "scope"}
@app.get("/api/v1/rescue-templates")
def list_rescue_templates(_session: Session = Depends(require_approved)):
return {"templates": rescue_catalog.list(rescue_scope(current_profile()))}
def run_rescue_import(job_id: str, profile: dict, scope: str,
record: dict, image_path: Path) -> None:
uploaded = False
cataloged = False
try:
with rescue_import_lock:
rescue_import_jobs[job_id]["status"] = "running"
rescue_import_jobs[job_id]["phase"] = "正在传输 ISO 到宿主机"
save_job("rescue_import_job", job_id, rescue_import_jobs[job_id])
new_service(profile).import_rescue_iso(image_path, record["volume_name"])
uploaded = True
rescue_catalog.add(scope, record)
cataloged = True
with rescue_import_lock:
rescue_import_jobs[job_id] = {
"scope": scope, "status": "completed", "phase": "救援模版已导入",
"template": record,
}
save_job("rescue_import_job", job_id, rescue_import_jobs[job_id])
except Exception as exc:
if isinstance(exc, (TemplateError, VMError)):
error = str(exc)
else:
LOG.exception("Rescue ISO import failed: job=%s", job_id)
error = "救援 ISO 导入失败,请查看面板日志"
if cataloged:
try:
rescue_catalog.delete(scope, record["id"])
except Exception:
LOG.exception("Could not clean failed rescue catalog entry: job=%s", job_id)
error += ";数据库中可能留有救援模版记录,请管理员检查"
if uploaded:
try:
new_service(profile).delete_rescue_iso(record["volume_name"])
except Exception:
LOG.exception("Could not clean failed rescue ISO import: job=%s", job_id)
error += ";宿主机上可能留有救援 ISO 卷,请管理员检查"
with rescue_import_lock:
rescue_import_jobs[job_id] = {
"scope": scope, "status": "failed", "phase": "导入失败", "error": error,
}
try:
save_job("rescue_import_job", job_id, rescue_import_jobs[job_id])
except DatabaseError:
LOG.exception("Could not persist rescue import failure: job=%s", job_id)
finally:
image_path.unlink(missing_ok=True)
with rescue_import_lock:
pending_rescue_imports.discard(scope)
@app.post("/api/v1/rescue-templates", status_code=202)
async def upload_rescue_template(
request: Request,
name: str = Query(min_length=1, max_length=64),
_session: Session = Depends(require_admin_write),
):
name = validate_template_name(name)
profile = current_profile()
scope = rescue_scope(profile)
with rescue_import_lock:
forbid_long_job_during_update()
if scope in pending_rescue_imports:
raise TemplateError("已有救援 ISO 正在导入,请等待完成", 409)
if any(row["name"].casefold() == name.casefold()
for row in rescue_catalog.list(scope)):
raise TemplateError("当前宿主机已有同名救援模版", 409)
pending_rescue_imports.add(scope)
image_path: Path | None = None
handed_off = False
try:
declared_size = request.headers.get("content-length")
if declared_size is not None:
try:
if int(declared_size) > MAX_RESCUE_ISO_BYTES:
raise TemplateError("救援 ISO 超过 8 GiB 限制", 413)
except ValueError as exc:
raise TemplateError("上传长度无效", 400) from exc
fd, raw_path = tempfile.mkstemp(prefix=".rescue-upload-", suffix=".iso",
dir=settings.data_dir)
image_path = Path(raw_path)
os.fchmod(fd, 0o600)
checksum = hashlib.sha256()
size = 0
with os.fdopen(fd, "wb") as output:
async for chunk in request.stream():
size += len(chunk)
if size > MAX_RESCUE_ISO_BYTES:
raise TemplateError("救援 ISO 超过 8 GiB 限制", 413)
output.write(chunk)
checksum.update(chunk)
output.flush()
os.fsync(output.fileno())
verify_bootable_iso(image_path)
template_id = uuid.uuid4().hex
record = {
"id": template_id, "name": name,
"volume_name": rescue_volume_name(template_id),
"size_bytes": size, "sha256": checksum.hexdigest(),
}
job_id = str(uuid.uuid4())
with rescue_import_lock:
rescue_import_jobs[job_id] = {
"scope": scope, "status": "queued", "phase": "等待传输到宿主机",
}
save_job("rescue_import_job", job_id, rescue_import_jobs[job_id])
try:
rescue_import_executor.submit(
run_rescue_import, job_id, profile, scope, record, image_path,
)
except RuntimeError as exc:
rescue_import_jobs.pop(job_id, None)
delete_job("rescue_import_job", job_id)
raise TemplateError("面板正在停止,无法导入救援 ISO", 503) from exc
LOG.info("rescue ISO import queued: job=%s admin=%s", job_id, _session.username)
handed_off = True # The worker now owns and removes the temporary file.
return {"job_id": job_id, "status": "queued"}
finally:
if not handed_off:
if image_path is not None:
image_path.unlink(missing_ok=True)
with rescue_import_lock:
pending_rescue_imports.discard(scope)
@app.get("/api/v1/rescue-templates/import-jobs/{job_id}")
def rescue_import_job(job_id: str, _session: Session = Depends(require_admin)):
scope = rescue_scope(current_profile())
with rescue_import_lock:
job = rescue_import_jobs.get(job_id)
if job is None or job["scope"] != scope:
raise TemplateError("救援模版导入任务不存在", 404)
return {key: value for key, value in job.items() if key != "scope"}
@app.delete("/api/v1/rescue-templates/{template_id}")
def delete_rescue_template(template_id: str,
_session: Session = Depends(require_admin_write)):
profile = current_profile()
scope = rescue_scope(profile)
with rescue_lock, rescue_import_lock:
forbid_long_job_during_update()
if scope in pending_rescue_imports:
raise TemplateError("救援 ISO 正在导入,请稍后删除", 409)
record = rescue_catalog.get(scope, template_id)
states = (data_store.list("vm_rescue") if data_store is not None
else test_rescue_states.copy())
for raw in states.values():
try:
state_record = json.loads(raw)
except (TypeError, ValueError) as exc:
raise DatabaseError("虚拟机救援记录损坏") from exc
if not isinstance(state_record, dict):
raise DatabaseError("虚拟机救援记录损坏")
if state_record.get("volume_name") == record["volume_name"]:
raise TemplateError("此救援模版正被虚拟机使用,须先退出救援模式", 409)
try:
new_service(profile).delete_rescue_iso(record["volume_name"])
except VMError as exc:
if exc.status != 404:
raise
rescue_catalog.delete(scope, template_id)
LOG.info("rescue ISO deleted: id=%s admin=%s", template_id, _session.username)
return {"deleted": True}
@app.get("/api/v1/vms")
def list_vms(_session: Session = Depends(require_approved)):
profile = current_profile()
vms = new_service(profile).list_vms()
if database_mode:
account = account_for_session(_session)
if not account["admin"]:
owned_ids = state.users.list_owned_vm_ids(host_scope(profile), account["id"])
vms = [vm for vm in vms if vm["uuid"] in owned_ids]
records = (data_store.list("vm_credential") if data_store is not None
else test_credentials.copy())
return {"items": [decorate_vm(
vm, parse_credential(records.get(credential_key(profile, vm["uuid"]))), profile,
) for vm in vms]}
@app.get("/api/v1/vms/{vm_id}")
def get_vm(vm_id: str, _session: Session = Depends(require_approved)):
profile = current_profile()
require_vm_access(_session, profile, vm_id)
vm = new_service(profile).get_vm(vm_id)
return decorate_vm(vm, get_credential(profile, vm["uuid"]), profile)
@app.post("/api/v1/vms/{vm_id}/credential")
def reveal_vm_credential(vm_id: str, _session: Session = Depends(require_approved_write)):
profile = current_profile()
require_vm_access(_session, profile, vm_id)
vm = new_service(profile).get_vm(vm_id)
credential = get_credential(profile, vm["uuid"])
decorated = decorate_vm(vm, credential, profile)
LOG.info("vm credential viewed: uuid=%s admin=%s", vm["uuid"], _session.username)
return {
"login_name": decorated["login_name"],
"auth_mode": decorated["auth_mode"],
"password": credential["password"] if decorated["password_available"] else None,
}
@app.get("/api/v1/vms/{vm_id}/metrics")
def get_vm_metrics(
vm_id: str,
range_: Literal["1h", "24h", "7d", "30d"] = Query("1h", alias="range"),
_session: Session = Depends(require_approved),
):
try:
canonical_id = str(uuid.UUID(vm_id))
except (TypeError, ValueError, AttributeError) as exc:
raise VMError("虚拟机 UUID 无效", 400) from exc
profile = current_profile()
require_vm_access(_session, profile, canonical_id)
# Never reveal another host's retained samples for a reused UUID.
new_service(profile).get_vm(canonical_id)
backend = metric_store if metric_store is not None else data_store
if backend is None:
raise HTTPException(status_code=503, detail="指标数据库尚未就绪")
now = int(time.time())
window = RANGES[range_]
interval = query_interval_seconds(window)
since = max(now - window, now - RETENTION_SECONDS)
try:
scope = host_scope(profile)
except (KeyError, ValueError, TypeError) as exc:
raise HTTPException(status_code=409, detail="宿主机身份信息不完整") from exc
points = backend.query_metric_samples(
scope, canonical_id, since=since, until=now,
interval_seconds=interval, limit=MAX_QUERY_POINTS,
)
return {"vm_id": canonical_id, "range": range_,
"interval_seconds": interval, "points": points}
@app.post("/api/v1/vms", status_code=201)
def create_vm(body: CreateISOBody | CreateCloudBody,
_session: Session = Depends(require_approved_write)):
# Keep the update submission lock until libvirt has finished creating
# the VM. Otherwise an update can restart the web process mid-create.
with reinstall_lock:
forbid_long_job_during_update()
profile = current_profile()
account = account_for_session(_session)
owner_id = account["id"] if database_mode and not account["admin"] else None
if isinstance(body, CreateCloudBody):
public_key = body.guest_ssh_public_key.strip()
root_password = (body.root_password.get_secret_value()
if body.root_password is not None else "")
if body.guest_ssh_key_id is not None:
if public_key:
raise HTTPException(
status_code=422,
detail="已保存公钥和手动公钥只能选择一种",
)
saved_key = ssh_key_store().get_for_user(
account["id"], body.guest_ssh_key_id,
)
if saved_key is None:
raise SSHKeyError("公钥不存在", 404)
public_key = saved_key["public_key"]
# A cloud VM always has a recoverable initial root password,
# even when a saved or one-off SSH key was selected.
if not root_password:
root_password = generate_root_password()
password_hash = None
if root_password:
try:
password_hash = hash_root_password(root_password)
except ValueError as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc
auto_public_ip = bool(profile.get("public_pool_cidr"))
if auto_public_ip and (body.public_ip_cidr or body.gateway or body.dns_servers):
raise HTTPException(
status_code=422,
detail="宿主机已配置公网地址池,创建时不能手动指定 IP、网关或 DNS",
)
if not auto_public_ip and not (
body.public_ip_cidr and body.gateway and body.dns_servers
):
raise HTTPException(
status_code=422,
detail="宿主机尚未配置公网地址池,请手动填写 IP、网关和 DNS",
)
record = templates.get(template_scope(profile), body.template_id)
spec = body.model_dump(exclude={
"source", "template_id", "root_password", "guest_ssh_key_id",
})
spec["os_name"] = record.get("name")
spec["guest_ssh_public_key"] = public_key
if password_hash is not None:
spec["root_password_hash"] = password_hash
requested_uuid = str(uuid.uuid4())
spec["vm_uuid"] = requested_uuid
service = new_service(profile)
allocated_ip = ""
owner_assigned = False
try:
if auto_public_ip:
allocated_ip = allocate_public_ip(
profile, service, requested_uuid,
)
spec["public_ip_cidr"] = allocated_ip
spec["gateway"] = profile["public_gateway"]
spec["dns_servers"] = list(profile["public_dns_servers"])
else:
claim_explicit_public_ip(profile, body.public_ip_cidr, requested_uuid)
allocated_ip = body.public_ip_cidr
if owner_id is not None:
state.users.assign_vm(host_scope(profile), requested_uuid, owner_id)
owner_assigned = True
save_credential(profile, requested_uuid,
("ssh_key_password" if public_key and root_password
else "ssh_key" if public_key else "password"),
f"panel-seed-{requested_uuid}.iso",
root_password or None)
vm = service.create_cloud_vm(spec, record["volume_name"])
except Exception as exc:
# An incomplete libvirt rollback may have left the guest
# behind. Keep its credential in that exceptional case so
# the operator can still recover access if it exists.
rollback_incomplete = (isinstance(exc, VMError)
and "回滚不完整" in str(exc))
if not rollback_incomplete:
try:
delete_credential(profile, requested_uuid)
except DatabaseError:
LOG.exception("Could not remove unused VM credential for %s", requested_uuid)
if owner_assigned:
try:
state.users.delete_vm_owner(host_scope(profile), requested_uuid)
except DatabaseError:
LOG.exception("Could not remove unused VM owner for %s", requested_uuid)
if allocated_ip:
try:
release_public_ip_claim(profile, allocated_ip, requested_uuid)
except DatabaseError:
LOG.exception("Could not release unused public IP claim for %s", requested_uuid)
raise
vm = verified_created_vm(profile, vm, requested_uuid)
else:
spec = body.model_dump(exclude={"source"})
if database_mode:
requested_uuid = str(uuid.uuid4())
spec["vm_uuid"] = requested_uuid
if owner_id is not None:
state.users.assign_vm(host_scope(profile), requested_uuid, owner_id)
try:
vm = new_service(profile).create_vm(spec)
except Exception as exc:
rollback_incomplete = (isinstance(exc, VMError)
and "回滚不完整" in str(exc))
if owner_id is not None and not rollback_incomplete:
state.users.delete_vm_owner(host_scope(profile), requested_uuid)
raise
if database_mode:
vm = verified_created_vm(profile, vm, requested_uuid)
LOG.info("vm created: uuid=%s admin=%s", vm.get("uuid"), _session.username)
credential = get_credential(profile, vm["uuid"]) if isinstance(body, CreateCloudBody) else None
return decorate_vm(vm, credential, profile)
def run_reinstall(
job_id: str, profile: dict, scope: str, vm_id: str,
template_volume_name: str, template_name: str | None, ssh_public_key: str,
root_password_hash: str | None, generated_password: str | None,
credential_password: str | None,
) -> None:
vm_reinstalled = False
def report_progress(phase: str) -> None:
if not isinstance(phase, str):
return
with reinstall_lock:
job = reinstall_jobs.get(job_id)
if job is not None and job["status"] == "running":
job["phase"] = phase[:200]
save_job("reinstall_job", job_id, job)
try:
with reinstall_lock:
reinstall_jobs[job_id]["status"] = "running"
reinstall_jobs[job_id]["phase"] = "正在检查虚拟机"
save_job("reinstall_job", job_id, reinstall_jobs[job_id])
result = new_service(profile).reinstall_cloud_vm(
vm_id, template_volume_name,
os_name=template_name,
ssh_public_key=ssh_public_key,
root_password_hash=root_password_hash,
progress=report_progress,
)
vm_reinstalled = True
save_credential(profile, vm_id,
("ssh_key_password" if ssh_public_key and credential_password
else "ssh_key" if ssh_public_key else "password"),
result["credential_generation"],
credential_password)
with reinstall_lock:
job = reinstall_jobs[job_id]
job["status"] = "succeeded"
job["phase"] = "重装完成"
if isinstance(result, dict) and result.get("warning"):
job["warning"] = str(result["warning"])
if generated_password is not None:
# DatabaseStore encrypts this one-time password. Keep it
# until the operator explicitly acknowledges saving it.
job["generated_password"] = generated_password
save_job("reinstall_job", job_id, job)
LOG.info("vm reinstall succeeded: job=%s uuid=%s", job_id, vm_id)
except Exception as exc:
if isinstance(exc, (VMError, TemplateError, ValueError)):
error = str(exc)
else:
LOG.exception("VM reinstall failed: job=%s uuid=%s", job_id, vm_id)
error = "重装系统失败,请查看面板日志"
with reinstall_lock:
job = reinstall_jobs[job_id]
if vm_reinstalled:
job["status"] = "succeeded"
job["phase"] = "重装已完成,但凭据或任务记录保存失败"
job["warning"] = "新系统已启动,但面板无法确认初始密码已保存;请立即保存结果页显示的密码并检查数据库"
if generated_password is not None:
job["generated_password"] = generated_password
else:
job["status"] = "failed"
job["phase"] = "重装失败"
job["error"] = error
try:
save_job("reinstall_job", job_id, job)
except DatabaseError:
LOG.exception("Could not persist reinstall status: job=%s", job_id)
if not vm_reinstalled:
LOG.warning("vm reinstall failed: job=%s uuid=%s", job_id, vm_id)
finally:
with reinstall_lock:
pending_reinstalls.discard((scope, vm_id))
@app.post("/api/v1/vms/{vm_id}/reinstall", status_code=202)
def reinstall_vm(vm_id: str, body: ReinstallBody,
_session: Session = Depends(require_approved_write)):
ssh_public_key = body.ssh_public_key.strip()
supplied_password = (body.root_password.get_secret_value()
if body.root_password is not None else "")
if body.ssh_key_id is not None:
if ssh_public_key:
raise HTTPException(
status_code=422,
detail="已保存公钥和手动公钥只能选择一种",
)
account = account_for_session(_session)
saved_key = ssh_key_store().get_for_user(account["id"], body.ssh_key_id)
if saved_key is None:
raise SSHKeyError("公钥不存在", 404)
ssh_public_key = saved_key["public_key"]
generated_password = None
if not supplied_password and not ssh_public_key:
generated_password = generate_root_password()
supplied_password = generated_password
try:
if ssh_public_key:
ssh_public_key, _unused_hash = _validate_guest_auth(ssh_public_key, None)
root_password_hash = (hash_root_password(supplied_password)
if supplied_password else None)
except ValueError as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc
with reinstall_lock, rescue_lock:
forbid_long_job_during_update()
profile = current_profile()
require_vm_access(_session, profile, vm_id)
canonical_id = str(uuid.UUID(vm_id))
service = new_service(profile)
rescue_record = current_rescue_state(profile, canonical_id, service)
rescue_vm = service.get_vm(canonical_id)
if ((host_scope(profile), canonical_id) in pending_rescues
or rescue_record is not None
or rescue_vm.get("rescue_mode")
or rescue_vm.get("rescue_staged")):
raise VMError("虚拟机处于救援模式或救援任务中,请先退出救援模式", 409)
scope = template_scope(profile)
record = templates.get(scope, body.template_id)
vm = service.get_vm(vm_id)
if not vm.get("managed") or not vm.get("public_ip_cidr"):
raise VMError("只有面板创建的公网云镜像虚拟机支持自动重装", 409)
if body.confirmation_name != vm["name"]:
raise HTTPException(status_code=422, detail="确认名称与虚拟机名称不一致")
normalized_id = vm["uuid"]
key = (scope, normalized_id)
if key in pending_reinstalls:
raise VMError("该虚拟机正在重装系统", 409)
job_id = str(uuid.uuid4())
reinstall_jobs[job_id] = {
"owner": _session.username,
"scope": scope,
"status": "queued",
"phase": "等待重装任务开始",
"vm_id": normalized_id,
"template_id": record["id"],
}
save_job("reinstall_job", job_id, reinstall_jobs[job_id])
pending_reinstalls.add(key)
try:
reinstall_executor.submit(
run_reinstall, job_id, profile, scope, normalized_id,
record["volume_name"], record.get("name"), ssh_public_key, root_password_hash,
generated_password, supplied_password or None,
)
except RuntimeError as exc:
pending_reinstalls.discard(key)
reinstall_jobs.pop(job_id, None)
delete_job("reinstall_job", job_id)
raise VMError("面板正在停止,无法开始重装任务", 503) from exc
LOG.info("vm reinstall queued: job=%s uuid=%s admin=%s",
job_id, normalized_id, _session.username)
return {"job_id": job_id, "status": "queued"}
@app.get("/api/v1/reinstall-jobs/{job_id}")
def reinstall_job(job_id: str, _session: Session = Depends(require_approved_write)):
scope = template_scope(current_profile())
with reinstall_lock:
job = reinstall_jobs.get(job_id)
if (job is None or job["scope"] != scope
or job["owner"] != _session.username):
raise VMError("重装任务不存在", 404)
result = {key: value for key, value in job.items()
if key not in ("scope", "owner", "generated_password")}
if job["status"] == "succeeded" and "generated_password" in job:
result["generated_password"] = job["generated_password"]
return result
@app.post("/api/v1/reinstall-jobs/{job_id}/ack-password")
def acknowledge_reinstall_password(job_id: str,
_session: Session = Depends(require_approved_write)):
scope = template_scope(current_profile())
with reinstall_lock:
job = reinstall_jobs.get(job_id)
if (job is None or job["scope"] != scope
or job["owner"] != _session.username):
raise VMError("重装任务不存在", 404)
if job["status"] != "succeeded":
raise VMError("重装任务尚未成功,不能确认密码", 409)
job.pop("generated_password", None)
save_job("reinstall_job", job_id, job)
return {"acknowledged": True}
def run_rescue_job(job_id: str, profile: dict, vm_id: str,
operation: Literal["enter", "exit", "force"]) -> None:
vm_scope = host_scope(profile)
key = (vm_scope, vm_id)
completed = False
try:
with rescue_lock:
job = rescue_jobs[job_id]
job["status"] = "running"
job["phase"] = (
"正在预挂载救援 ISO 并等待正常关机" if operation == "enter"
else "正在强制断电并从救援 ISO 冷启动" if operation == "force"
else "正在正常关机并恢复原系统启动配置"
)
save_job("rescue_job", job_id, job)
state_record = load_rescue_state(profile, vm_id)
if state_record is None:
raise VMError("救援恢复记录丢失,已停止操作", 409)
state_record["status"] = "exiting" if operation == "exit" else "running"
save_rescue_state(profile, vm_id, state_record)
service = new_service(profile)
if operation == "enter":
result = service.enter_rescue_vm(vm_id, state_record["volume_name"],
state_record["original_xml"])
elif operation == "force":
result = service.force_start_staged_rescue_vm(
vm_id, state_record["original_xml"])
else:
result = service.exit_rescue_vm(vm_id, state_record["original_xml"])
completed = True
staged = (operation == "enter" and isinstance(result, dict)
and result.get("pending_manual_restart") is True)
with rescue_lock:
if operation in {"enter", "force"}:
state_record["status"] = "staged" if staged else "active"
save_rescue_state(profile, vm_id, state_record)
else:
delete_rescue_state(profile, vm_id)
job = rescue_jobs[job_id]
job["status"] = "completed"
job["phase"] = ("救援 ISO 已预挂载,等待手动强制断电后开机" if staged
else "救援系统已启动" if operation in {"enter", "force"}
else "已退出救援模式,原系统已启动")
if staged:
job["pending_manual_restart"] = True
if isinstance(result, dict) and result.get("warning"):
job["warning"] = str(result["warning"])
save_job("rescue_job", job_id, job)
LOG.info("vm rescue %s succeeded: job=%s uuid=%s", operation, job_id, vm_id)
except Exception as exc:
if isinstance(exc, (VMError, TemplateError)):
error = str(exc)
else:
LOG.exception("VM rescue %s failed: job=%s uuid=%s", operation, job_id, vm_id)
error = "紧急救援操作失败,请查看面板和宿主机日志"
with rescue_lock:
try:
state_record = load_rescue_state(profile, vm_id)
if state_record is not None:
# If the original inactive definition survived, the
# failed job cannot strand an otherwise normal VM in
# a database-only "rescue" state.
service = new_service(profile)
actual_vm = service.get_vm(vm_id)
if (not actual_vm.get("rescue_mode")
and rescue_xml_matches(
service.prepare_rescue_vm(
vm_id, state_record["volume_name"]),
state_record["original_xml"],
)):
delete_rescue_state(profile, vm_id)
else:
state_record["status"] = "failed"
save_rescue_state(profile, vm_id, state_record)
except (DatabaseError, VMError, TemplateError):
LOG.exception("Could not save rescue recovery state: job=%s", job_id)
job = rescue_jobs[job_id]
job["status"] = "failed"
job["phase"] = "救援操作失败"
if "正常关机超时" in error or "无法请求虚拟机正常关机" in error:
job["error"] = error
else:
job["error"] = error + ";请确认虚拟机实际运行状态,必要时联系管理员"
if completed:
job["error"] = "虚拟机操作已完成,但数据库状态保存失败;请管理员检查救援记录"
try:
save_job("rescue_job", job_id, job)
except DatabaseError:
LOG.exception("Could not persist rescue job failure: job=%s", job_id)
finally:
with rescue_lock:
pending_rescues.discard(key)
@app.get("/api/v1/vms/{vm_id}/rescue")
def get_vm_rescue(vm_id: str, _session: Session = Depends(require_approved)):
profile = current_profile()
canonical_id = require_vm_access(_session, profile, vm_id)
with rescue_lock:
service = new_service(profile)
vm = service.get_vm(canonical_id)
record = current_rescue_state(profile, vm["uuid"], service)
staged = bool(vm.get("rescue_staged"))
active = bool(vm.get("rescue_mode")) and not staged
if (record is not None
and (host_scope(profile), vm["uuid"]) not in pending_rescues):
if staged and record["status"] in {"queued", "running", "failed"}:
record["status"] = "staged"
save_rescue_state(profile, vm["uuid"], record)
elif active and record["status"] in {"queued", "running", "staged"}:
record["status"] = "active"
save_rescue_state(profile, vm["uuid"], record)
template_name = None
template_id = record["template_id"] if record is not None else None
if record is not None:
try:
template_name = rescue_catalog.get(
rescue_scope(profile), record["template_id"])["name"]
except TemplateError:
pass
return {
"active": active,
"staged": staged,
"power_state": vm.get("state", "unknown"),
"template_id": template_id,
"template_name": template_name,
"status": record["status"] if record is not None else None,
"job_id": record.get("job_id") if record is not None else None,
"can_exit": (active or staged) and record is not None,
}
@app.post("/api/v1/vms/{vm_id}/rescue", status_code=202)
def enter_vm_rescue(vm_id: str, body: EnterRescueBody,
_session: Session = Depends(require_approved_write)):
with reinstall_lock, rescue_lock:
forbid_long_job_during_update()
profile = current_profile()
canonical_id = require_vm_access(_session, profile, vm_id)
vm_scope = host_scope(profile)
key = (vm_scope, canonical_id)
if key in pending_rescues or (template_scope(profile), canonical_id) in pending_reinstalls:
raise VMError("该虚拟机已有救援或重装任务,暂时不能重复操作", 409)
service = new_service(profile)
vm = service.get_vm(canonical_id)
if body.confirmation_name != vm["name"]:
raise VMError("确认名称与虚拟机名称不一致", 422)
if vm.get("rescue_mode") or vm.get("rescue_staged"):
raise VMError("虚拟机已预挂载救援 ISO 或处于救援模式,请先退出", 409)
record = rescue_catalog.get(rescue_scope(profile), body.template_id)
original_xml = service.prepare_rescue_vm(canonical_id, record["volume_name"])
prior = load_rescue_state(profile, canonical_id)
if prior is not None:
if not rescue_xml_matches(prior["original_xml"], original_xml):
raise VMError("虚拟机仍有未完成的救援记录,请管理员检查后再操作", 409)
delete_rescue_state(profile, canonical_id)
job_id = str(uuid.uuid4())
state_record = {
"template_id": record["id"], "volume_name": record["volume_name"],
"original_xml": original_xml, "status": "queued", "job_id": job_id,
}
save_rescue_state(profile, canonical_id, state_record)
rescue_jobs[job_id] = {
"scope": vm_scope, "owner": _session.username,
"vm_id": canonical_id, "operation": "enter",
"status": "queued", "phase": "等待救援任务开始",
}
try:
save_job("rescue_job", job_id, rescue_jobs[job_id])
pending_rescues.add(key)
rescue_executor.submit(run_rescue_job, job_id, profile, canonical_id, "enter")
except Exception:
pending_rescues.discard(key)
rescue_jobs.pop(job_id, None)
delete_job("rescue_job", job_id)
delete_rescue_state(profile, canonical_id)
raise
LOG.info("vm rescue enter queued: job=%s uuid=%s user=%s",
job_id, canonical_id, _session.username)
return {"job_id": job_id, "status": "queued"}
@app.post("/api/v1/vms/{vm_id}/rescue/force-start", status_code=202)
def force_start_vm_rescue(vm_id: str, body: ForceRescueBody,
_session: Session = Depends(require_approved_write)):
"""Explicit, confirmed power cycle into an already staged rescue ISO."""
with reinstall_lock, rescue_lock:
forbid_long_job_during_update()
profile = current_profile()
canonical_id = require_vm_access(_session, profile, vm_id)
vm_scope = host_scope(profile)
key = (vm_scope, canonical_id)
if key in pending_rescues or (template_scope(profile), canonical_id) in pending_reinstalls:
raise VMError("该虚拟机已有救援或重装任务,请等待完成", 409)
record = load_rescue_state(profile, canonical_id)
if record is None:
raise VMError("缺少原始启动配置备份,不能安全强制重启进入救援", 409)
vm = new_service(profile).get_vm(canonical_id)
if body.confirmation_name != vm["name"]:
raise VMError("确认名称与虚拟机名称不一致", 422)
if not vm.get("rescue_staged"):
raise VMError("救援 ISO 尚未预挂载,或虚拟机已经进入救援模式", 409)
job_id = str(uuid.uuid4())
rescue_jobs[job_id] = {
"scope": vm_scope, "owner": _session.username,
"vm_id": canonical_id, "operation": "force",
"status": "queued", "phase": "等待手动确认的救援冷启动",
}
previous_status = record["status"]
previous_job_id = record.get("job_id")
try:
save_job("rescue_job", job_id, rescue_jobs[job_id])
record["status"] = "queued"
record["job_id"] = job_id
save_rescue_state(profile, canonical_id, record)
pending_rescues.add(key)
rescue_executor.submit(run_rescue_job, job_id, profile, canonical_id, "force")
except Exception:
pending_rescues.discard(key)
rescue_jobs.pop(job_id, None)
delete_job("rescue_job", job_id)
record["status"] = previous_status
record["job_id"] = previous_job_id
save_rescue_state(profile, canonical_id, record)
raise
LOG.warning("vm rescue force-start queued: job=%s uuid=%s user=%s",
job_id, canonical_id, _session.username)
return {"job_id": job_id, "status": "queued"}
@app.post("/api/v1/vms/{vm_id}/rescue/exit", status_code=202)
def exit_vm_rescue(vm_id: str, body: ExitRescueBody,
_session: Session = Depends(require_approved_write)):
with reinstall_lock, rescue_lock:
forbid_long_job_during_update()
profile = current_profile()
canonical_id = require_vm_access(_session, profile, vm_id)
vm_scope = host_scope(profile)
key = (vm_scope, canonical_id)
if key in pending_rescues or (template_scope(profile), canonical_id) in pending_reinstalls:
raise VMError("该虚拟机已有救援或重装任务,请等待完成", 409)
record = load_rescue_state(profile, canonical_id)
if record is None:
raise VMError("找不到原始启动配置,无法安全退出救援模式", 409)
vm = new_service(profile).get_vm(canonical_id)
if body.confirmation_name != vm["name"]:
raise VMError("确认名称与虚拟机名称不一致", 422)
if not (vm.get("rescue_mode") or vm.get("rescue_staged")):
raise VMError("虚拟机当前并非救援模式,请管理员核对配置", 409)
job_id = str(uuid.uuid4())
rescue_jobs[job_id] = {
"scope": vm_scope, "owner": _session.username,
"vm_id": canonical_id, "operation": "exit",
"status": "queued", "phase": "等待退出救援模式",
}
save_job("rescue_job", job_id, rescue_jobs[job_id])
record["status"] = "exiting"
record["job_id"] = job_id
save_rescue_state(profile, canonical_id, record)
pending_rescues.add(key)
try:
rescue_executor.submit(run_rescue_job, job_id, profile, canonical_id, "exit")
except RuntimeError as exc:
pending_rescues.discard(key)
record["status"] = "active"
save_rescue_state(profile, canonical_id, record)
rescue_jobs.pop(job_id, None)
delete_job("rescue_job", job_id)
raise VMError("面板正在停止,无法退出救援模式", 503) from exc
LOG.info("vm rescue exit queued: job=%s uuid=%s user=%s",
job_id, canonical_id, _session.username)
return {"job_id": job_id, "status": "queued"}
@app.get("/api/v1/rescue-jobs/{job_id}")
def get_rescue_job(job_id: str, _session: Session = Depends(require_approved)):
profile = current_profile()
vm_scope = host_scope(profile)
with rescue_lock:
job = rescue_jobs.get(job_id)
if (job is None or job["scope"] != vm_scope
or (job["owner"] != _session.username
and not account_for_session(_session)["admin"])):
raise VMError("救援任务不存在", 404)
require_vm_access(_session, profile, job["vm_id"])
return {key: value for key, value in job.items()
if key not in {"scope", "owner", "vm_id"}}
@app.post("/api/v1/vms/{vm_id}/actions")
def action(vm_id: str, body: ActionBody, _session: Session = Depends(require_approved_write)):
with reinstall_lock, rescue_lock:
forbid_long_job_during_update()
profile = current_profile()
require_vm_access(_session, profile, vm_id)
try:
normalized_id = str(uuid.UUID(vm_id))
except ValueError:
normalized_id = vm_id
if (template_scope(profile), normalized_id) in pending_reinstalls:
raise VMError("虚拟机正在重装系统,暂时不能执行其他操作", 409)
service = new_service(profile)
if (host_scope(profile), normalized_id) in pending_rescues:
raise VMError("虚拟机正在切换救援模式,请等待任务完成", 409)
rescue_record = current_rescue_state(profile, normalized_id, service)
current_vm = service.get_vm(normalized_id)
staged = bool(current_vm.get("rescue_staged"))
if staged and rescue_record is not None:
if body.action not in {"poweroff", "start"}:
raise VMError(
"救援 ISO 已预挂载;请强制断电后开机进入救援,或取消救援配置", 409)
elif rescue_record is not None or current_vm.get("rescue_mode") or staged:
raise VMError("虚拟机处于救援模式或救援任务中,请先退出救援模式", 409)
vm = service.action(vm_id, body.action)
if staged and body.action == "start":
if not vm.get("rescue_mode") or vm.get("rescue_staged"):
raise VMError(
"虚拟机已发送开机指令,但未确认从救援 ISO 启动;请检查 VNC 控制台", 502)
rescue_record["status"] = "active"
save_rescue_state(profile, normalized_id, rescue_record)
LOG.info("vm action: uuid=%s action=%s admin=%s", vm.get("uuid"), body.action, _session.username)
return vm
@app.delete("/api/v1/vms/{vm_id}")
def delete_vm(
vm_id: str,
delete_disks: bool = False,
_session: Session = Depends(require_approved_write),
):
with reinstall_lock, rescue_lock:
forbid_long_job_during_update()
profile = current_profile()
require_vm_access(_session, profile, vm_id)
try:
normalized_id = str(uuid.UUID(vm_id))
except ValueError:
normalized_id = vm_id
if (template_scope(profile), normalized_id) in pending_reinstalls:
raise VMError("虚拟机正在重装系统,暂时不能删除", 409)
service = new_service(profile)
rescue_record = current_rescue_state(profile, normalized_id, service)
rescue_vm = service.get_vm(normalized_id)
if ((host_scope(profile), normalized_id) in pending_rescues
or rescue_record is not None
or rescue_vm.get("rescue_mode")
or rescue_vm.get("rescue_staged")):
raise VMError("虚拟机处于救援模式或救援任务中,请先退出救援模式", 409)
result = service.delete_vm(vm_id, delete_disks)
if database_mode:
try:
state.users.delete_vm_owner(host_scope(profile), result["uuid"])
except DatabaseError:
LOG.exception("Could not remove owner for deleted VM %s", result["uuid"])
result = {**result, "warning": "虚拟机已删除,但数据库中的归属记录未能清理"}
try:
delete_credential(profile, result["uuid"])
except DatabaseError:
LOG.exception("Could not remove credential for deleted VM %s", result["uuid"])
result = {**result, "warning": "虚拟机已删除,但数据库中的登录凭据或归属记录未能完全清理;请检查数据库状态"}
try:
release_vm_public_ip_claims(profile, result["uuid"])
except DatabaseError:
LOG.exception("Could not release public IP claim for deleted VM %s", result["uuid"])
result = {**result, "warning": "虚拟机已删除,但公网 IP 占用记录未能清理;请检查数据库状态"}
LOG.info(
"vm deleted: uuid=%s delete_disks=%s admin=%s",
result.get("uuid"), delete_disks, _session.username,
)
return result
@app.websocket("/api/v1/vms/{vm_id}/console")
async def console(websocket: WebSocket, vm_id: str):
host = websocket.headers.get("host")
if websocket.url.scheme != "wss":
await websocket.close(code=4403)
return
if not _origin_matches_host(websocket.headers.get("origin"), host):
await websocket.close(code=4403)
return
session = await asyncio.to_thread(sessions.get, websocket.cookies.get(COOKIE_NAME))
if session is None:
await websocket.close(code=4401)
return
try:
account = await asyncio.to_thread(account_for_session, session)
except HTTPException:
await websocket.close(code=4401)
return
if not account["admin"] and not account["approved"]:
await websocket.close(code=4403)
return
profile = manager.current()
if profile is None:
await websocket.close(code=4409)
return
try:
if profile.get("key_locked"):
await websocket.close(code=4423)
return
await asyncio.to_thread(require_vm_access, session, profile, vm_id)
port = await asyncio.to_thread(new_service(profile).console_port, vm_id)
except VMError:
await websocket.close(code=4404)
return
await websocket.accept()
async def watch_console_access() -> None:
while True:
await asyncio.sleep(CONSOLE_ACCESS_CHECK_SECONDS)
try:
refreshed = await asyncio.to_thread(sessions.get, session.token)
if refreshed is None or refreshed.username != session.username:
return
account = await asyncio.to_thread(account_for_session, refreshed)
if not account["admin"] and not account["approved"]:
return
current = manager.current()
if (current is None or current.get("key_locked")
or host_scope(current) != host_scope(profile)):
return
await asyncio.to_thread(require_vm_access, refreshed, current, vm_id)
except (HTTPException, VMError, DatabaseError, RemoteHostError,
KeyError, TypeError, ValueError):
return
relay_task = asyncio.create_task(relay_console(
websocket,
port=port,
ssh_target=profile["ssh_target"],
ssh_binary=settings.ssh_binary,
known_hosts_file=profile["ssh_known_hosts_file"],
ssh_config_file=profile["ssh_config_file"],
))
access_task = asyncio.create_task(watch_console_access())
try:
done, _pending = await asyncio.wait(
{relay_task, access_task}, return_when=asyncio.FIRST_COMPLETED)
if access_task in done:
relay_task.cancel()
await asyncio.gather(relay_task, return_exceptions=True)
with suppress(RuntimeError, OSError):
await websocket.close(code=4403)
return
await relay_task
except ConsoleRelayError as exc:
LOG.warning("VNC relay failed for vm %r: %s", vm_id, exc)
with suppress(RuntimeError, OSError):
await websocket.close(code=1011)
finally:
access_task.cancel()
await asyncio.gather(access_task, return_exceptions=True)
return app