|
|
"""Browser setup, authenticated API, and VM console for one remote host."""
|
|
|
|
|
|
from __future__ import annotations
|
|
|
|
|
|
import asyncio
|
|
|
import hashlib
|
|
|
import hmac
|
|
|
import ipaddress
|
|
|
import json
|
|
|
import logging
|
|
|
import os
|
|
|
import re
|
|
|
import secrets
|
|
|
import subprocess
|
|
|
import tempfile
|
|
|
import threading
|
|
|
import time
|
|
|
import uuid
|
|
|
import xml.etree.ElementTree as ET
|
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
|
from contextlib import asynccontextmanager, suppress
|
|
|
from pathlib import Path
|
|
|
from typing import Any, Callable, Literal
|
|
|
from urllib.parse import urlsplit
|
|
|
|
|
|
from fastapi import BackgroundTasks, Depends, FastAPI, HTTPException, Query, Request, WebSocket
|
|
|
from fastapi.responses import FileResponse, JSONResponse
|
|
|
from fastapi.staticfiles import StaticFiles
|
|
|
from pydantic import BaseModel, ConfigDict, Field, SecretStr, model_validator
|
|
|
|
|
|
from .auth import DatabaseSessionStore, LoginLimiter, Session, SessionStore, verify_password
|
|
|
from .cloud_init import (
|
|
|
_validate_guest_auth, _validate_ssh_public_key, generate_root_password,
|
|
|
hash_root_password, validate_cloud_network, validate_guest_hostname,
|
|
|
)
|
|
|
from .cloud_templates import (
|
|
|
TemplateError, TemplateStore, download_verified_image, image_metadata_for,
|
|
|
template_id_for, template_scope, template_volume_name, validate_template_name,
|
|
|
)
|
|
|
from .config import Settings
|
|
|
from .console import ConsoleRelayError, relay_console
|
|
|
from .database import DatabaseError, make_components
|
|
|
from .database_bootstrap import DatabaseBootstrap, DatabaseSetupError
|
|
|
from .libvirt_service import (
|
|
|
FIREWALL_FILTER_COMPILER_VERSION,
|
|
|
REINSTALL_CLEANUP_PENDING_ERROR_CODE,
|
|
|
REINSTALL_RESULT_UNCERTAIN_ERROR_CODE,
|
|
|
SUPPORTED_FIREWALL_FILTER_COMPILER_VERSIONS,
|
|
|
SNAPSHOT_KEY_RE, VMError, VMService, VMSnapshotUncertainError,
|
|
|
build_firewall_filter_xml, firewall_policy_digest, snapshot_volume_name,
|
|
|
)
|
|
|
from .metrics import (MAX_QUERY_POINTS, RETENTION_SECONDS, RANGES, MetricsCollector,
|
|
|
host_scope, query_interval_seconds)
|
|
|
from .panel_state import PanelState, SetupError
|
|
|
from .platform_network_policy import (
|
|
|
default_platform_policy_snapshot, normalize_platform_policy_snapshot,
|
|
|
)
|
|
|
from .remote_host import (
|
|
|
MAX_PUBLIC_NETWORK_POOLS,
|
|
|
PUBLIC_POOL_KEY,
|
|
|
HostProfile,
|
|
|
RemoteHostError,
|
|
|
RemoteHostManager,
|
|
|
validate_public_network_pool,
|
|
|
validate_public_network_pools,
|
|
|
)
|
|
|
from .rescue_templates import (
|
|
|
MAX_RESCUE_ISO_BYTES, RescueTemplateStore, rescue_scope,
|
|
|
rescue_volume_name, verify_bootable_iso,
|
|
|
)
|
|
|
from .ssh_keys import SSHKeyError
|
|
|
from .smtp_service import SMTPError, SMTPSettingsStore, send_verification_code
|
|
|
from .updater import BRANCH, REPOSITORY_URL, UpdateError, Updater
|
|
|
from .user_isos import (
|
|
|
UserISOFiles, UserISOStore, download_user_iso, user_iso_scope,
|
|
|
user_iso_volume_name, validate_user_iso_name, validate_user_iso_url,
|
|
|
)
|
|
|
from .users import UserError, normalize_email
|
|
|
from .engine_context import EngineComponents, TRUSTED_ACTOR, actor, actor_account, actor_session
|
|
|
|
|
|
|
|
|
COOKIE_NAME = "virsh_panel_session"
|
|
|
CONSOLE_ACCESS_CHECK_SECONDS = 15
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
STARTUP_LOG = logging.getLogger("uvicorn.error")
|
|
|
SNAPSHOT_PUBLIC_ERROR_CODES = {
|
|
|
"snapshot_guest_agent_unavailable",
|
|
|
"live_snapshot_unsupported",
|
|
|
"snapshot_pool_unsupported",
|
|
|
"snapshot_result_uncertain",
|
|
|
}
|
|
|
REINSTALL_PUBLIC_ERROR_CODES = {
|
|
|
REINSTALL_CLEANUP_PENDING_ERROR_CODE,
|
|
|
REINSTALL_RESULT_UNCERTAIN_ERROR_CODE,
|
|
|
}
|
|
|
# These phases are emitted only after the replacement definition has been
|
|
|
# created or the old-volume cleanup boundary has been crossed. Keep this
|
|
|
# compatibility list for rows written by releases that did not persist the
|
|
|
# explicit ``reinstall_committed`` marker before a process restart.
|
|
|
REINSTALL_COMMITTED_PHASES = frozenset({
|
|
|
"cleaning_up", "cleanup_pending", "result_uncertain", "completed",
|
|
|
"重装已完成,但凭据或任务记录保存失败",
|
|
|
"重装完成,但旧系统盘清理待核对", "重装结果待核对",
|
|
|
})
|
|
|
|
|
|
|
|
|
def reinstall_phase_committed(value: Any) -> bool:
|
|
|
return isinstance(value, str) and value.strip() in REINSTALL_COMMITTED_PHASES
|
|
|
|
|
|
|
|
|
def reinstall_cleanup_pending(job: dict[str, Any]) -> bool:
|
|
|
phase = job.get("phase")
|
|
|
if isinstance(phase, str):
|
|
|
phase = phase.strip()
|
|
|
error_code = job.get("error_code")
|
|
|
return (
|
|
|
error_code == REINSTALL_CLEANUP_PENDING_ERROR_CODE
|
|
|
or job.get("cleanup_pending") is True
|
|
|
or phase in {"cleanup_pending", "重装完成,但旧系统盘清理待核对"}
|
|
|
or (isinstance(job.get("warning"), str)
|
|
|
and job["warning"].strip().startswith("旧存储卷"))
|
|
|
)
|
|
|
|
|
|
|
|
|
def reinstall_post_commit(job: dict[str, Any]) -> bool:
|
|
|
error_code = job.get("error_code")
|
|
|
return (
|
|
|
job.get("reinstall_committed") is True
|
|
|
or error_code in REINSTALL_PUBLIC_ERROR_CODES
|
|
|
or reinstall_phase_committed(job.get("phase"))
|
|
|
or reinstall_cleanup_pending(job)
|
|
|
or (isinstance(job.get("warning"), str)
|
|
|
and bool(job["warning"].strip()))
|
|
|
)
|
|
|
|
|
|
|
|
|
class PublicIpClaimCleanupPending(VMError):
|
|
|
"""The VM mutation completed but its engine IP claim did not."""
|
|
|
|
|
|
|
|
|
class VMDeleteSnapshotCleanupPending(VMError):
|
|
|
"""The VM and its primary disks are gone but snapshot GC is incomplete."""
|
|
|
|
|
|
def __init__(self, message: str) -> None:
|
|
|
super().__init__(
|
|
|
message, 503,
|
|
|
error_code="vm_delete_snapshot_cleanup_pending",
|
|
|
)
|
|
|
|
|
|
|
|
|
class VMDeleteFirewallCleanupPending(VMError):
|
|
|
"""The VM is gone but its deterministic nwfilter still needs cleanup."""
|
|
|
|
|
|
def __init__(self, message: str) -> None:
|
|
|
super().__init__(
|
|
|
message, 503,
|
|
|
error_code="vm_delete_firewall_cleanup_pending",
|
|
|
)
|
|
|
|
|
|
|
|
|
class SetupBody(BaseModel):
|
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
|
|
token: str = Field(min_length=1, max_length=256)
|
|
|
username: str = Field(min_length=3, max_length=32)
|
|
|
password: str = Field(min_length=12, max_length=1024)
|
|
|
|
|
|
|
|
|
class DatabaseSetupBody(BaseModel):
|
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
|
|
host: str = Field(min_length=1, max_length=253)
|
|
|
port: int = Field(ge=1, le=65535)
|
|
|
database: str = Field(min_length=1, max_length=64)
|
|
|
username: str = Field(min_length=1, max_length=128)
|
|
|
password: SecretStr
|
|
|
setup_token: str = Field(min_length=1, max_length=256)
|
|
|
|
|
|
|
|
|
class UpdateSettingsBody(BaseModel):
|
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
|
|
repository_url: str = Field(default=REPOSITORY_URL, max_length=2048)
|
|
|
branch: str = Field(default=BRANCH, max_length=128)
|
|
|
access_token: SecretStr | None = None
|
|
|
|
|
|
|
|
|
class UpdateApplyBody(BaseModel):
|
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
|
|
expected_current_commit: str = Field(pattern=r"^(?:[0-9a-f]{40}|[0-9a-f]{64})$")
|
|
|
expected_remote_commit: str = Field(pattern=r"^(?:[0-9a-f]{40}|[0-9a-f]{64})$")
|
|
|
|
|
|
|
|
|
class LoginBody(BaseModel):
|
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
|
|
email: str | None = Field(default=None, min_length=3, max_length=254)
|
|
|
# Only the injected, database-free test harness may use the historical
|
|
|
# username login. Production explicitly rejects this field below.
|
|
|
username: str | None = Field(default=None, min_length=1, max_length=64)
|
|
|
password: str = Field(min_length=1, max_length=1024)
|
|
|
|
|
|
|
|
|
class RegistrationCodeBody(BaseModel):
|
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
|
|
email: str = Field(min_length=3, max_length=254)
|
|
|
|
|
|
|
|
|
class RegistrationBody(RegistrationCodeBody):
|
|
|
username: str = Field(min_length=3, max_length=32)
|
|
|
password: str = Field(min_length=12, max_length=1024)
|
|
|
code: str = Field(pattern=r"^[0-9]{6}$")
|
|
|
|
|
|
|
|
|
class SMTPSettingsBody(BaseModel):
|
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
|
|
host: str = Field(min_length=1, max_length=253)
|
|
|
port: int = Field(ge=1, le=65535)
|
|
|
tls_mode: Literal["ssl", "starttls", "none"]
|
|
|
username: str = Field(default="", max_length=256)
|
|
|
password: SecretStr | None = None
|
|
|
from_address: str = Field(min_length=3, max_length=254)
|
|
|
|
|
|
|
|
|
class UnlockBody(BaseModel):
|
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
|
|
passphrase: str = Field(min_length=1, max_length=1024)
|
|
|
|
|
|
|
|
|
class HostScanBody(BaseModel):
|
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
|
|
host: str = Field(min_length=1, max_length=253)
|
|
|
port: int = Field(ge=1, le=65535)
|
|
|
|
|
|
|
|
|
class HostBody(HostScanBody):
|
|
|
private_key: str = Field(default="", max_length=65536)
|
|
|
private_key_passphrase: str = Field(default="", max_length=1024)
|
|
|
host_key: str = Field(min_length=1, max_length=4096)
|
|
|
confirmed_fingerprint: str = Field(min_length=1, max_length=256)
|
|
|
storage_pool: str = Field(default="default", min_length=1, max_length=4096)
|
|
|
iso_pool: str = Field(default="iso", min_length=1, max_length=4096)
|
|
|
rescue_iso_pool: str = Field(default="", max_length=4096)
|
|
|
default_network: str = Field(default="default", min_length=1, max_length=128)
|
|
|
region: str = Field(default="", max_length=64,
|
|
|
pattern=r"^[^\x00-\x1f\x7f]*$")
|
|
|
public_pool_cidr: str = Field(default="", max_length=18)
|
|
|
public_gateway: str = Field(default="", max_length=15)
|
|
|
public_dns_servers: list[str] = Field(default_factory=list, max_length=4)
|
|
|
public_reserved_ips: list[str] = Field(default_factory=list, max_length=254)
|
|
|
public_vm_prefix: int | None = Field(default=None, ge=24, le=30)
|
|
|
|
|
|
def profile(self) -> HostProfile:
|
|
|
return HostProfile(
|
|
|
host=self.host,
|
|
|
port=self.port,
|
|
|
private_key=self.private_key,
|
|
|
host_key=self.host_key,
|
|
|
private_key_passphrase=self.private_key_passphrase,
|
|
|
storage_pool=self.storage_pool,
|
|
|
iso_pool=self.iso_pool,
|
|
|
rescue_iso_pool=self.rescue_iso_pool,
|
|
|
default_network=self.default_network,
|
|
|
region=self.region,
|
|
|
public_pool_cidr=self.public_pool_cidr,
|
|
|
public_gateway=self.public_gateway,
|
|
|
public_dns_servers=self.public_dns_servers,
|
|
|
public_reserved_ips=self.public_reserved_ips,
|
|
|
public_vm_prefix=self.public_vm_prefix,
|
|
|
)
|
|
|
|
|
|
|
|
|
class PublicNetworkPoolBody(BaseModel):
|
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
|
|
revision: int = Field(ge=1, le=(1 << 63) - 1)
|
|
|
state: Literal["active", "draining", "disabled"] = "active"
|
|
|
network_selector: str = Field(min_length=1, max_length=128)
|
|
|
cidr: str = Field(min_length=7, max_length=18)
|
|
|
gateway: str = Field(min_length=7, max_length=15)
|
|
|
dns_servers: list[str] = Field(min_length=1, max_length=4)
|
|
|
reserved_ips: list[str] = Field(default_factory=list, max_length=254)
|
|
|
guest_prefix_length: int = Field(ge=24, le=30)
|
|
|
|
|
|
|
|
|
class CreateISOBody(BaseModel):
|
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
|
|
source: Literal["iso"] = "iso"
|
|
|
name: str = Field(min_length=1, max_length=63)
|
|
|
vcpu: int = Field(ge=1, le=64)
|
|
|
memory_mib: int = Field(ge=512, le=262144)
|
|
|
disk_gib: int = Field(ge=1, le=4096)
|
|
|
bandwidth_mbps: int | None = Field(default=None, ge=1, le=100000)
|
|
|
network: str = Field(min_length=1, max_length=128)
|
|
|
iso: str | None = Field(default=None, min_length=1, max_length=255)
|
|
|
user_iso_id: str | None = Field(default=None, pattern=r"^[0-9a-f]{32}$")
|
|
|
|
|
|
@model_validator(mode="after")
|
|
|
def exactly_one_iso_source(self):
|
|
|
if (self.iso is None) == (self.user_iso_id is None):
|
|
|
raise ValueError("iso 与 user_iso_id 必须且只能选择一个")
|
|
|
return self
|
|
|
|
|
|
|
|
|
class CreateCloudBody(BaseModel):
|
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
|
|
source: Literal["cloud"]
|
|
|
name: str = Field(min_length=1, max_length=63)
|
|
|
vcpu: int = Field(ge=1, le=64)
|
|
|
memory_mib: int = Field(ge=512, le=262144)
|
|
|
disk_gib: int = Field(ge=1, le=4096)
|
|
|
bandwidth_mbps: int | None = Field(default=None, ge=1, le=100000)
|
|
|
network: str = Field(min_length=1, max_length=128)
|
|
|
template_id: str = Field(min_length=1, max_length=80)
|
|
|
public_pool_key: str = Field(default="", max_length=32,
|
|
|
pattern=r"^(?:|[0-9a-f]{32})$")
|
|
|
public_ip_cidr: str = Field(default="", max_length=64)
|
|
|
gateway: str = Field(default="", max_length=64)
|
|
|
dns_servers: list[str] = Field(default_factory=list, max_length=4)
|
|
|
guest_ssh_public_key: str = Field(default="", max_length=16384)
|
|
|
guest_ssh_key_id: int | None = Field(default=None, ge=1)
|
|
|
root_password: SecretStr | None = None
|
|
|
platform_network_policy_snapshot: dict[str, Any] = Field(
|
|
|
default_factory=default_platform_policy_snapshot)
|
|
|
|
|
|
@model_validator(mode="after")
|
|
|
def validate_platform_network_policy(self):
|
|
|
try:
|
|
|
self.platform_network_policy_snapshot = (
|
|
|
normalize_platform_policy_snapshot(
|
|
|
self.platform_network_policy_snapshot))
|
|
|
except ValueError as exc:
|
|
|
raise ValueError("平台网络策略快照无效") from exc
|
|
|
return self
|
|
|
|
|
|
|
|
|
class ChangeNetworkBody(BaseModel):
|
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
|
|
public_ip_cidr: str = Field(min_length=7, max_length=64)
|
|
|
public_pool_key: str = Field(default="", max_length=32,
|
|
|
pattern=r"^(?:|[0-9a-f]{32})$")
|
|
|
confirm_restart: bool = False
|
|
|
|
|
|
|
|
|
class VMSettingsBody(BaseModel):
|
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
|
|
# Every field is optional so an administrator can change one setting at a
|
|
|
# time. The validator below rejects an empty patch; omitted credentials
|
|
|
# are preserved from the existing cloud-init seed.
|
|
|
hostname: str | None = Field(default=None, max_length=253)
|
|
|
root_password: SecretStr | None = Field(default=None, min_length=12, max_length=128)
|
|
|
ssh_public_key: str | None = Field(default=None, max_length=16384)
|
|
|
confirm_restart: bool = False
|
|
|
|
|
|
@model_validator(mode="after")
|
|
|
def require_setting(self):
|
|
|
if self.hostname is None and self.root_password is None and self.ssh_public_key is None:
|
|
|
raise ValueError("至少需要修改一项虚拟机设置")
|
|
|
return self
|
|
|
|
|
|
|
|
|
class SSHKeyBody(BaseModel):
|
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
|
|
name: str = Field(min_length=1, max_length=80)
|
|
|
public_key: str = Field(min_length=1, max_length=16384)
|
|
|
|
|
|
|
|
|
class ImportTemplateBody(BaseModel):
|
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
|
|
url: str = Field(min_length=16, max_length=2048)
|
|
|
sha256: str = Field(default="", max_length=64)
|
|
|
name: str = Field(min_length=1, max_length=64)
|
|
|
|
|
|
|
|
|
class ImportUserISOBody(BaseModel):
|
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
|
|
url: str = Field(min_length=1, max_length=2048)
|
|
|
sha256: str = Field(default="", max_length=64)
|
|
|
name: str = Field(min_length=1, max_length=64)
|
|
|
|
|
|
|
|
|
class RenameTemplateBody(BaseModel):
|
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
|
|
name: str = Field(min_length=1, max_length=64)
|
|
|
|
|
|
|
|
|
class DeleteTemplateBody(BaseModel):
|
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
|
|
created_at: int = Field(ge=1)
|
|
|
|
|
|
|
|
|
class ReinstallBody(BaseModel):
|
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
|
|
template_id: str = Field(min_length=1, max_length=80)
|
|
|
ssh_public_key: str = Field(default="", max_length=16384)
|
|
|
ssh_key_id: int | None = Field(default=None, ge=1)
|
|
|
root_password: SecretStr | None = None
|
|
|
confirmation_name: str = Field(min_length=1, max_length=63)
|
|
|
|
|
|
|
|
|
class EnterRescueBody(BaseModel):
|
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
|
|
template_id: str = Field(pattern=r"^[0-9a-f]{32}$")
|
|
|
confirmation_name: str = Field(min_length=1, max_length=63)
|
|
|
|
|
|
|
|
|
class ExitRescueBody(BaseModel):
|
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
|
|
confirmation_name: str = Field(min_length=1, max_length=63)
|
|
|
# PHP's billing ledger supplies the authoritative target. The original
|
|
|
# XML was saved before rescue mode and may predate a traffic-quota block.
|
|
|
network_suspended: bool = False
|
|
|
|
|
|
|
|
|
class ForceRescueBody(BaseModel):
|
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
|
|
confirmation_name: str = Field(min_length=1, max_length=63)
|
|
|
ack_data_loss: Literal[True]
|
|
|
|
|
|
|
|
|
class ActionBody(BaseModel):
|
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
|
|
action: Literal["start", "shutdown", "reboot", "poweroff", "reset"]
|
|
|
|
|
|
|
|
|
class CreateSnapshotBody(BaseModel):
|
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
|
|
snapshot_key: str = Field(pattern=r"^[0-9a-f]{32}$")
|
|
|
name: str = Field(min_length=1, max_length=64)
|
|
|
consistency_mode: Literal["filesystem", "crash"] = "filesystem"
|
|
|
|
|
|
@model_validator(mode="after")
|
|
|
def validate_name(self):
|
|
|
if (self.name != self.name.strip()
|
|
|
or any(ord(char) < 32 or ord(char) == 127 for char in self.name)):
|
|
|
raise ValueError("快照名称不能包含首尾空白或控制字符")
|
|
|
return self
|
|
|
|
|
|
|
|
|
class EmptySnapshotBody(BaseModel):
|
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
|
|
|
|
|
class ServiceResumeBody(BaseModel):
|
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
|
|
# Only restart a guest that the expiry policy itself stopped. A tenant
|
|
|
# that had already shut the VM down before expiry must remain shut down
|
|
|
# after renewal.
|
|
|
start_if_suspended: bool = False
|
|
|
|
|
|
|
|
|
class FirewallRuleBody(BaseModel):
|
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
|
|
protocol: Literal["tcp", "udp", "icmp"]
|
|
|
source_cidr: str = Field(min_length=1, max_length=18)
|
|
|
port_from: int | None = Field(default=None, ge=1, le=65535)
|
|
|
port_to: int | None = Field(default=None, ge=1, le=65535)
|
|
|
description: str = Field(default="", max_length=120)
|
|
|
|
|
|
@model_validator(mode="after")
|
|
|
def validate_firewall_rule(self):
|
|
|
try:
|
|
|
source = ipaddress.ip_network(self.source_cidr, strict=False)
|
|
|
except ValueError as exc:
|
|
|
raise ValueError("来源必须是 IPv4 CIDR") from exc
|
|
|
if source.version != 4:
|
|
|
raise ValueError("防火墙一期仅支持 IPv4")
|
|
|
if self.source_cidr != str(source):
|
|
|
raise ValueError("来源必须使用规范 IPv4 CIDR")
|
|
|
self.source_cidr = str(source)
|
|
|
if self.protocol in {"tcp", "udp"}:
|
|
|
if (self.port_from is None or self.port_to is None
|
|
|
or self.port_from > self.port_to):
|
|
|
raise ValueError("TCP/UDP 必须提供有效端口范围")
|
|
|
elif self.port_from is not None or self.port_to is not None:
|
|
|
raise ValueError("ICMP 规则不能包含端口")
|
|
|
if (self.description != self.description.strip()
|
|
|
or any(ord(char) < 32 or ord(char) == 127
|
|
|
for char in self.description)):
|
|
|
raise ValueError("规则说明不能包含首尾空白或控制字符")
|
|
|
return self
|
|
|
|
|
|
|
|
|
class FirewallPolicyBody(BaseModel):
|
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
|
|
enabled: bool
|
|
|
revision: int = Field(ge=1, le=9223372036854775807)
|
|
|
policy_digest: str = Field(pattern=r"^[0-9a-f]{64}$")
|
|
|
rules: list[FirewallRuleBody] = Field(default_factory=list, max_length=50)
|
|
|
|
|
|
|
|
|
def _public_host(profile: dict | None) -> dict:
|
|
|
if profile is None:
|
|
|
return {"configured": False}
|
|
|
keys = (
|
|
|
"host", "port", "key_type", "fingerprint", "storage_pool",
|
|
|
"iso_pool", "default_network", "private_key_configured",
|
|
|
"private_key_encrypted", "key_locked", "private_key_passphrase_saved",
|
|
|
)
|
|
|
return {
|
|
|
"configured": True,
|
|
|
**{key: profile[key] for key in keys if key in profile},
|
|
|
"rescue_iso_pool": profile.get("rescue_iso_pool", ""),
|
|
|
"region": profile.get("region", ""),
|
|
|
"public_pool_cidr": profile.get("public_pool_cidr", ""),
|
|
|
"public_gateway": profile.get("public_gateway", ""),
|
|
|
"public_dns_servers": profile.get("public_dns_servers", []),
|
|
|
"public_reserved_ips": profile.get("public_reserved_ips", []),
|
|
|
"public_vm_prefix": profile.get("public_vm_prefix"),
|
|
|
}
|
|
|
|
|
|
|
|
|
def _peer(request: Request) -> str:
|
|
|
# Uvicorn has already processed forwarding headers from its trusted local
|
|
|
# reverse proxy. Never read a raw client-supplied X-Real-IP here, because
|
|
|
# that would let a login attacker reset the rate limit.
|
|
|
return request.client.host if request.client else "unknown"
|
|
|
|
|
|
|
|
|
def _origin_matches_host(origin: str | None, host: str | None) -> bool:
|
|
|
if not origin or not host:
|
|
|
return False
|
|
|
try:
|
|
|
parsed = urlsplit(origin)
|
|
|
except ValueError:
|
|
|
return False
|
|
|
if parsed.scheme != "https":
|
|
|
return False
|
|
|
return bool(
|
|
|
parsed.netloc == host
|
|
|
and parsed.hostname
|
|
|
and not parsed.username
|
|
|
and not parsed.password
|
|
|
and not parsed.path
|
|
|
and not parsed.query
|
|
|
and not parsed.fragment
|
|
|
)
|
|
|
|
|
|
|
|
|
def _plain_loopback_allowed(scope: dict, host: str | None) -> bool:
|
|
|
"""Permit local diagnostics only when both peer and requested host are local.
|
|
|
|
|
|
The request scheme is supplied by the ASGI server. Uvicorn's proxy header
|
|
|
handling may upgrade it to HTTPS/WSS only for its explicitly trusted proxy
|
|
|
address; this function never trusts an incoming forwarding header itself.
|
|
|
"""
|
|
|
client = scope.get("client")
|
|
|
if not client or not host:
|
|
|
return False
|
|
|
try:
|
|
|
peer_is_local = ipaddress.ip_address(client[0]).is_loopback
|
|
|
host_is_local = urlsplit("//" + host).hostname in {
|
|
|
"localhost", "127.0.0.1", "::1",
|
|
|
}
|
|
|
except ValueError:
|
|
|
return False
|
|
|
return peer_is_local and host_is_local
|
|
|
|
|
|
|
|
|
def create_app(
|
|
|
settings: Settings | None = None,
|
|
|
*,
|
|
|
panel_state: PanelState | None = None,
|
|
|
remote_manager: RemoteHostManager | None = None,
|
|
|
service_factory: Callable[[dict], VMService] | None = None,
|
|
|
metric_store: Any | None = None,
|
|
|
engine_components: EngineComponents | None = None,
|
|
|
) -> FastAPI:
|
|
|
settings = settings or Settings.from_env()
|
|
|
settings.validate()
|
|
|
# Injected objects keep the existing isolated HTTP test harness working.
|
|
|
# Normal production startup never uses the old JSON-backed stores.
|
|
|
internal_mode = engine_components is not None
|
|
|
database_mode = internal_mode or (panel_state is None and remote_manager is None)
|
|
|
bootstrap = DatabaseBootstrap(settings.data_dir) if database_mode and not internal_mode else None
|
|
|
data_store = None
|
|
|
updater = None
|
|
|
if internal_mode:
|
|
|
state = engine_components.state
|
|
|
manager = engine_components.manager
|
|
|
templates = engine_components.templates
|
|
|
data_store = engine_components.store
|
|
|
metric_store = engine_components.metric_store
|
|
|
setup_token = None
|
|
|
elif database_mode:
|
|
|
configured_database = bootstrap.load_config()
|
|
|
if configured_database is None:
|
|
|
setup_token = bootstrap.ensure_token()
|
|
|
state = None
|
|
|
manager = None
|
|
|
templates = None
|
|
|
else:
|
|
|
state, manager, templates, data_store = make_components(
|
|
|
settings.data_dir, configured_database, bootstrap.master_key(),
|
|
|
settings.ssh_binary,
|
|
|
)
|
|
|
setup_token = (bootstrap.ensure_token()
|
|
|
if state.setup_required() else state.initialize_setup_token())
|
|
|
else:
|
|
|
state = panel_state or PanelState(settings.data_dir)
|
|
|
setup_token = state.initialize_setup_token()
|
|
|
manager = remote_manager or RemoteHostManager(
|
|
|
settings.data_dir, ssh_binary=settings.ssh_binary)
|
|
|
templates = TemplateStore(settings.data_dir)
|
|
|
if setup_token is not None:
|
|
|
# Uvicorn writes this logger to stderr, which the supplied systemd
|
|
|
# service captures in its journal. Never expose the token over HTTP.
|
|
|
STARTUP_LOG.warning("VIRSH_PANEL_SETUP_TOKEN=%s", setup_token)
|
|
|
if database_mode and (settings.data_dir / "generations").exists():
|
|
|
STARTUP_LOG.warning(
|
|
|
"Legacy data/generations detected; its old SSH key files are not migrated "
|
|
|
"or automatically removed. Verify and remove them separately."
|
|
|
)
|
|
|
shared_cloud_template_runtime = (
|
|
|
engine_components.cloud_templates
|
|
|
if internal_mode and engine_components.cloud_templates is not None
|
|
|
else None
|
|
|
)
|
|
|
import_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="cloud-image-import")
|
|
|
import_lock = threading.RLock()
|
|
|
import_jobs: dict[str, dict] = {}
|
|
|
pending_templates: set[tuple[str, str]] = set()
|
|
|
rescue_catalog = RescueTemplateStore(data_store)
|
|
|
rescue_import_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="rescue-iso-import")
|
|
|
rescue_import_lock = threading.RLock()
|
|
|
rescue_import_jobs: dict[str, dict] = {}
|
|
|
pending_rescue_imports: set[str] = set()
|
|
|
shared_user_iso_runtime = (
|
|
|
engine_components.user_isos
|
|
|
if internal_mode and engine_components.user_isos is not None
|
|
|
else None
|
|
|
)
|
|
|
if shared_user_iso_runtime is not None:
|
|
|
user_iso_catalog = shared_user_iso_runtime.catalog
|
|
|
user_iso_files = shared_user_iso_runtime.files
|
|
|
user_iso_import_executor = shared_user_iso_runtime.executor
|
|
|
user_iso_import_lock = shared_user_iso_runtime.lock
|
|
|
user_iso_import_jobs = shared_user_iso_runtime.jobs
|
|
|
pending_user_iso_imports = shared_user_iso_runtime.pending
|
|
|
else:
|
|
|
user_iso_catalog = UserISOStore(data_store)
|
|
|
user_iso_files = UserISOFiles(settings.data_dir)
|
|
|
user_iso_import_executor = ThreadPoolExecutor(
|
|
|
max_workers=2, thread_name_prefix="user-iso-import")
|
|
|
user_iso_import_lock = threading.RLock()
|
|
|
user_iso_import_jobs: dict[str, dict] = {}
|
|
|
pending_user_iso_imports: set[str] = set()
|
|
|
# One child app represents one host. Serialize marker recheck, cache
|
|
|
# transfer and marker commit so concurrent VM creates cannot both act on a
|
|
|
# stale "not ready" observation for the same remote volume.
|
|
|
user_iso_cache_lock = threading.RLock()
|
|
|
cloud_template_cache_lock = threading.RLock()
|
|
|
reinstall_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="vm-reinstall")
|
|
|
reinstall_lock = threading.RLock()
|
|
|
reinstall_jobs: dict[str, dict] = {}
|
|
|
pending_reinstalls: set[tuple[str, str]] = set()
|
|
|
rescue_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="vm-rescue")
|
|
|
rescue_lock = threading.RLock()
|
|
|
rescue_jobs: dict[str, dict] = {}
|
|
|
pending_rescues: set[tuple[str, str]] = set()
|
|
|
# A full local backup can consume all currently allocated guest blocks.
|
|
|
# Serialize snapshot I/O per host so two sparse targets cannot both pass a
|
|
|
# stale free-space observation and then fill the pool together. This is a
|
|
|
# dedicated executor: unrelated VM reads and operations remain concurrent.
|
|
|
snapshot_executor = ThreadPoolExecutor(
|
|
|
max_workers=1, thread_name_prefix="vm-snapshot")
|
|
|
snapshot_lock = threading.RLock()
|
|
|
snapshot_jobs: dict[str, dict] = {}
|
|
|
snapshot_futures: dict[str, Any] = {}
|
|
|
pending_snapshot_jobs: set[tuple[str, str]] = set()
|
|
|
snapshot_suspend_barriers: set[tuple[str, str]] = set()
|
|
|
test_snapshots: dict[str, str] = {}
|
|
|
test_rescue_states: dict[str, str] = {}
|
|
|
test_firewalls: dict[str, str] = {}
|
|
|
# The production store encrypts each credential record with a fresh AES-GCM
|
|
|
# nonce. The in-memory fallback is only for injected, database-free tests.
|
|
|
test_credentials: dict[str, str] = {}
|
|
|
public_pool_catalog_lock = threading.RLock()
|
|
|
public_pool_catalog_kind = "public_pool_catalog"
|
|
|
public_pool_catalog_key = "current"
|
|
|
|
|
|
def legacy_public_network_pool(profile: dict) -> dict | None:
|
|
|
if not profile.get("public_pool_cidr"):
|
|
|
return None
|
|
|
try:
|
|
|
return validate_public_network_pool({
|
|
|
"pool_key": "legacy-default",
|
|
|
"revision": 1,
|
|
|
"state": "active",
|
|
|
"network_selector": profile["default_network"],
|
|
|
"cidr": profile["public_pool_cidr"],
|
|
|
"gateway": profile["public_gateway"],
|
|
|
"dns_servers": profile.get("public_dns_servers", []),
|
|
|
"reserved_ips": profile.get("public_reserved_ips", []),
|
|
|
"guest_prefix_length": (
|
|
|
profile.get("public_vm_prefix")
|
|
|
if profile.get("public_vm_prefix") is not None
|
|
|
else ipaddress.IPv4Network(
|
|
|
profile["public_pool_cidr"], strict=True).prefixlen
|
|
|
),
|
|
|
}, profile["host"], allow_legacy_key=True)
|
|
|
except (KeyError, TypeError, ValueError, RemoteHostError) as exc:
|
|
|
raise VMError(
|
|
|
"宿主机旧公网地址池配置损坏,请管理员重新保存设置", 500,
|
|
|
) from exc
|
|
|
|
|
|
def read_native_public_network_pools(
|
|
|
profile: dict,
|
|
|
) -> tuple[str | None, list[dict]]:
|
|
|
if data_store is None:
|
|
|
return None, []
|
|
|
raw = data_store.get(public_pool_catalog_kind, public_pool_catalog_key)
|
|
|
if raw is None:
|
|
|
return None, []
|
|
|
try:
|
|
|
catalog = json.loads(raw)
|
|
|
if (not isinstance(catalog, dict) or set(catalog) != {"version", "pools"}
|
|
|
or catalog["version"] != 1
|
|
|
or not isinstance(catalog["pools"], list)):
|
|
|
raise ValueError("invalid public pool catalog")
|
|
|
pools = validate_public_network_pools(catalog["pools"], profile["host"])
|
|
|
except (KeyError, TypeError, ValueError, RemoteHostError) as exc:
|
|
|
raise VMError(
|
|
|
"宿主机公网地址池目录损坏,请管理员核查引擎记录", 500,
|
|
|
) from exc
|
|
|
return raw, pools
|
|
|
|
|
|
def public_network_pools(profile: dict) -> tuple[list[dict], bool]:
|
|
|
raw, pools = read_native_public_network_pools(profile)
|
|
|
if raw is not None:
|
|
|
return pools, False
|
|
|
legacy = legacy_public_network_pool(profile)
|
|
|
return ([] if legacy is None else [legacy]), legacy is not None
|
|
|
|
|
|
def public_pool_runtime_fields(pool: dict) -> dict:
|
|
|
return {
|
|
|
key: pool[key] for key in (
|
|
|
"network_selector", "cidr", "gateway", "dns_servers",
|
|
|
"reserved_ips", "guest_prefix_length",
|
|
|
)
|
|
|
}
|
|
|
|
|
|
def public_pool_for_key(
|
|
|
profile: dict, pool_key: str, *, active: bool = False,
|
|
|
) -> dict:
|
|
|
pools, _compatibility = public_network_pools(profile)
|
|
|
pool = next((item for item in pools if item["pool_key"] == pool_key), None)
|
|
|
if pool is None:
|
|
|
raise VMError("公网地址池不存在或尚未同步到宿主机", 409)
|
|
|
if active and pool["state"] != "active":
|
|
|
raise VMError("公网地址池没有开放新地址分配", 409)
|
|
|
return pool
|
|
|
|
|
|
def public_pool_for_ip(
|
|
|
profile: dict,
|
|
|
public_ip_cidr: str,
|
|
|
*,
|
|
|
active: bool = False,
|
|
|
) -> dict:
|
|
|
try:
|
|
|
requested = ipaddress.IPv4Interface(public_ip_cidr)
|
|
|
except (TypeError, ValueError) as exc:
|
|
|
raise VMError("预留公网 IP 格式无效", 422) from exc
|
|
|
matches = []
|
|
|
pools, _compatibility = public_network_pools(profile)
|
|
|
for pool in pools:
|
|
|
network = ipaddress.IPv4Network(pool["cidr"], strict=True)
|
|
|
if (requested.ip in network
|
|
|
and requested.network.prefixlen == pool["guest_prefix_length"]
|
|
|
and (not active or pool["state"] == "active")):
|
|
|
matches.append(pool)
|
|
|
if len(matches) != 1:
|
|
|
raise VMError("预留公网 IP 无法唯一匹配宿主机地址池", 422)
|
|
|
return matches[0]
|
|
|
|
|
|
def ip_claim_key(profile: dict, address: str) -> str:
|
|
|
return f"{host_scope(profile)}.{address}"
|
|
|
|
|
|
def insert_public_ip_claim(
|
|
|
profile: dict, address: str, vm_id: str, pool_key: str | None = None,
|
|
|
) -> bool:
|
|
|
if data_store is None:
|
|
|
return True
|
|
|
claim_value = {"vm_uuid": vm_id, "reserved_at": int(time.time())}
|
|
|
if pool_key is not None:
|
|
|
claim_value["pool_key"] = pool_key
|
|
|
claim = json.dumps(claim_value, separators=(",", ":"))
|
|
|
return data_store.insert("ip_claim", ip_claim_key(profile, address), claim)
|
|
|
|
|
|
def public_ip_claim_addresses(profile: dict) -> set[ipaddress.IPv4Address]:
|
|
|
claimed: set[ipaddress.IPv4Address] = set()
|
|
|
if data_store is None:
|
|
|
return claimed
|
|
|
scope_prefix = f"{host_scope(profile)}."
|
|
|
for key in data_store.list("ip_claim"):
|
|
|
if not key.startswith(scope_prefix):
|
|
|
continue
|
|
|
try:
|
|
|
claimed.add(ipaddress.IPv4Address(key[len(scope_prefix):]))
|
|
|
except ValueError:
|
|
|
continue
|
|
|
return claimed
|
|
|
|
|
|
def public_network_pool_state(
|
|
|
profile: dict,
|
|
|
pool: dict,
|
|
|
*,
|
|
|
claimed_addresses: set[ipaddress.IPv4Address] | None = None,
|
|
|
) -> dict:
|
|
|
"""Build address state from the panel-owned claim ledger only.
|
|
|
|
|
|
Dedicated hosts are configured out of band and the panel owns every
|
|
|
sellable IP. Do not enumerate libvirt domains on this hot path; the
|
|
|
atomic claim below is the engine-side concurrency guard.
|
|
|
"""
|
|
|
try:
|
|
|
network = ipaddress.IPv4Network(pool["cidr"], strict=True)
|
|
|
gateway = ipaddress.IPv4Address(pool["gateway"])
|
|
|
guest_prefix = pool["guest_prefix_length"]
|
|
|
dns = [ipaddress.IPv4Address(value) for value in pool["dns_servers"]]
|
|
|
reserved = [ipaddress.IPv4Address(value) for value in pool["reserved_ips"]]
|
|
|
statically_blocked = {
|
|
|
network.network_address, network.broadcast_address, gateway,
|
|
|
*dns, *reserved,
|
|
|
}
|
|
|
try:
|
|
|
host_address = ipaddress.IPv4Address(profile.get("host", ""))
|
|
|
except (TypeError, ValueError):
|
|
|
host_address = None
|
|
|
if host_address is not None and host_address in network:
|
|
|
statically_blocked.add(host_address)
|
|
|
except (KeyError, TypeError, ValueError) as exc:
|
|
|
raise VMError(
|
|
|
"宿主机公网地址池配置损坏,请管理员修复设置", 500,
|
|
|
) from exc
|
|
|
|
|
|
allocated: set[ipaddress.IPv4Address] = set()
|
|
|
if claimed_addresses is None:
|
|
|
claimed_addresses = public_ip_claim_addresses(profile)
|
|
|
allocated.update(address for address in claimed_addresses
|
|
|
if address in network)
|
|
|
|
|
|
usable = set(network.hosts())
|
|
|
reserved_in_pool = usable & statically_blocked
|
|
|
allocated_in_pool = (usable - reserved_in_pool) & allocated
|
|
|
available = sorted(usable - reserved_in_pool - allocated_in_pool)
|
|
|
return {
|
|
|
**pool,
|
|
|
"configuration_state": "ready",
|
|
|
"pool_cidr": str(network),
|
|
|
"guest_prefix": guest_prefix,
|
|
|
"usable_count": len(usable),
|
|
|
"reserved_count": len(reserved_in_pool),
|
|
|
"allocated_count": len(allocated_in_pool),
|
|
|
"available": available,
|
|
|
"_network": network,
|
|
|
"_usable": usable,
|
|
|
"_reserved": reserved_in_pool,
|
|
|
"_allocated": allocated_in_pool,
|
|
|
}
|
|
|
|
|
|
def allocate_public_ip(profile: dict, pool: dict, vm_id: str) -> str:
|
|
|
"""Choose and reserve a public IP, including across panel processes.
|
|
|
|
|
|
Claims remain after a successful create and are released on panel VM
|
|
|
deletion. A claim from an uncertain create is retained rather than
|
|
|
risking reuse of an address whose guest may still exist.
|
|
|
"""
|
|
|
if pool["state"] != "active":
|
|
|
raise VMError("公网地址池没有开放新地址分配", 409)
|
|
|
state = public_network_pool_state(profile, pool)
|
|
|
for address in state["available"]:
|
|
|
if not insert_public_ip_claim(
|
|
|
profile, str(address), vm_id, pool["pool_key"]):
|
|
|
continue
|
|
|
return f"{address}/{state['guest_prefix']}"
|
|
|
raise VMError("公网地址池中没有可分配的 IP,请检查已用和保留地址", 409)
|
|
|
|
|
|
def claim_scheduled_public_ip(
|
|
|
profile: dict,
|
|
|
pool: dict,
|
|
|
public_ip_cidr: str,
|
|
|
vm_id: str,
|
|
|
) -> str:
|
|
|
"""Atomically claim the exact pool address reserved by PHP.
|
|
|
|
|
|
PHP owns the commercial reservation, while this engine claim closes
|
|
|
the race between concurrent workers and libvirt domain creation. The
|
|
|
host profile remains authoritative for prefix, gateway and DNS.
|
|
|
"""
|
|
|
if pool["state"] != "active":
|
|
|
raise VMError("公网地址池没有开放新地址分配", 409)
|
|
|
state = public_network_pool_state(profile, pool)
|
|
|
try:
|
|
|
requested = ipaddress.IPv4Interface(public_ip_cidr)
|
|
|
except (TypeError, ValueError) as exc:
|
|
|
raise VMError("预留公网 IP 格式无效", 422) from exc
|
|
|
address = requested.ip
|
|
|
if (requested.network.prefixlen != state["guest_prefix"]
|
|
|
or address not in state["_network"]
|
|
|
or address not in state["_usable"]):
|
|
|
raise VMError("预留公网 IP 不属于宿主机可分配地址池", 422)
|
|
|
if address in state["_reserved"]:
|
|
|
raise VMError("预留公网 IP 是宿主机保留地址,不能分配", 409)
|
|
|
if address in state["_allocated"]:
|
|
|
raise VMError("预留公网 IP 已被虚拟机或其他任务占用", 409)
|
|
|
if not insert_public_ip_claim(
|
|
|
profile, str(address), vm_id, pool["pool_key"]):
|
|
|
raise VMError("预留公网 IP 已被其他创建任务占用", 409)
|
|
|
return f"{address}/{state['guest_prefix']}"
|
|
|
|
|
|
def claim_explicit_public_ip(profile: dict, public_ip_cidr: str,
|
|
|
vm_id: str) -> None:
|
|
|
if data_store is None:
|
|
|
return
|
|
|
try:
|
|
|
address = str(ipaddress.IPv4Interface(public_ip_cidr).ip)
|
|
|
except (TypeError, ValueError):
|
|
|
# The libvirt create path returns the detailed input error.
|
|
|
return
|
|
|
if not insert_public_ip_claim(profile, address, vm_id):
|
|
|
raise VMError("该公网 IP 已被面板中的虚拟机或创建任务占用", 409)
|
|
|
|
|
|
def release_public_ip_claim(profile: dict, public_ip_cidr: str,
|
|
|
vm_id: str) -> None:
|
|
|
if data_store is None:
|
|
|
return
|
|
|
try:
|
|
|
address = str(ipaddress.IPv4Interface(public_ip_cidr).ip)
|
|
|
key = ip_claim_key(profile, address)
|
|
|
except (TypeError, ValueError, KeyError):
|
|
|
return
|
|
|
raw = data_store.get("ip_claim", key)
|
|
|
if raw is None:
|
|
|
return
|
|
|
try:
|
|
|
claim = json.loads(raw)
|
|
|
except (TypeError, ValueError):
|
|
|
LOG.error("Could not parse public IP claim for %s", key)
|
|
|
return
|
|
|
if isinstance(claim, dict) and claim.get("vm_uuid") == vm_id:
|
|
|
data_store.delete("ip_claim", key)
|
|
|
|
|
|
def release_vm_public_ip_claims(profile: dict, vm_id: str) -> None:
|
|
|
if data_store is None:
|
|
|
return
|
|
|
prefix = f"{host_scope(profile)}."
|
|
|
for key, raw in data_store.list("ip_claim").items():
|
|
|
if not key.startswith(prefix):
|
|
|
continue
|
|
|
try:
|
|
|
claim = json.loads(raw)
|
|
|
except (TypeError, ValueError):
|
|
|
LOG.error("Could not parse public IP claim for %s", key)
|
|
|
continue
|
|
|
if isinstance(claim, dict) and claim.get("vm_uuid") == vm_id:
|
|
|
data_store.delete("ip_claim", key)
|
|
|
|
|
|
def compare_and_set_public_pool_catalog(
|
|
|
expected: str | None, value: str,
|
|
|
) -> bool:
|
|
|
if data_store is None:
|
|
|
raise VMError("公网地址池目录存储不可用", 409)
|
|
|
compare = getattr(data_store, "compare_and_set", None)
|
|
|
if callable(compare):
|
|
|
return bool(compare(
|
|
|
public_pool_catalog_kind, public_pool_catalog_key,
|
|
|
expected, value,
|
|
|
))
|
|
|
# Database-free tests and the retired single-process browser mode use
|
|
|
# a simple store. Keep the same compare-and-set behavior locally.
|
|
|
with public_pool_catalog_lock:
|
|
|
current = data_store.get(
|
|
|
public_pool_catalog_kind, public_pool_catalog_key)
|
|
|
if current != expected:
|
|
|
return False
|
|
|
data_store.set(public_pool_catalog_kind, public_pool_catalog_key, value)
|
|
|
return True
|
|
|
|
|
|
def upsert_public_network_pool(
|
|
|
profile: dict, pool_key: str, body: PublicNetworkPoolBody,
|
|
|
) -> tuple[dict, bool]:
|
|
|
if not PUBLIC_POOL_KEY.fullmatch(pool_key):
|
|
|
raise VMError("公网地址池标识必须是 32 位小写十六进制", 422)
|
|
|
try:
|
|
|
requested = validate_public_network_pool({
|
|
|
"pool_key": pool_key,
|
|
|
**body.model_dump(),
|
|
|
}, profile["host"])
|
|
|
except RemoteHostError as exc:
|
|
|
raise VMError(str(exc), exc.status) from exc
|
|
|
# The data model stores a selector now so a later multi-line release
|
|
|
# does not need another migration. Network-sensitive VM operations still
|
|
|
# instantiate the host's default VMService, so accepting another bridge
|
|
|
# today would create guests that cannot be safely reinstalled or edited.
|
|
|
if requested["network_selector"] != profile["default_network"]:
|
|
|
raise VMError(
|
|
|
"当前阶段公网地址池必须使用宿主机默认网络;多线路网桥尚未启用", 409,
|
|
|
)
|
|
|
|
|
|
for _attempt in range(8):
|
|
|
raw, native = read_native_public_network_pools(profile)
|
|
|
existing = next(
|
|
|
(item for item in native if item["pool_key"] == pool_key), None)
|
|
|
if existing is not None:
|
|
|
if requested["revision"] < existing["revision"]:
|
|
|
raise VMError("公网地址池版本已更新,请刷新后重试", 409)
|
|
|
if requested["revision"] == existing["revision"]:
|
|
|
if requested == existing:
|
|
|
return existing, True
|
|
|
raise VMError("相同公网地址池版本对应了不同配置", 409)
|
|
|
|
|
|
candidate = [item for item in native if item["pool_key"] != pool_key]
|
|
|
candidate.append(requested)
|
|
|
try:
|
|
|
candidate = validate_public_network_pools(candidate, profile["host"])
|
|
|
except RemoteHostError as exc:
|
|
|
raise VMError(str(exc), exc.status) from exc
|
|
|
|
|
|
claimed_addresses = public_ip_claim_addresses(profile)
|
|
|
if existing is not None and (
|
|
|
public_pool_runtime_fields(existing)
|
|
|
!= public_pool_runtime_fields(requested)):
|
|
|
state = public_network_pool_state(
|
|
|
profile, existing, claimed_addresses=claimed_addresses)
|
|
|
if state["allocated_count"]:
|
|
|
raise VMError(
|
|
|
"公网地址池仍有虚拟机或任务占用,不能修改网段、网关、DNS或保留地址",
|
|
|
409,
|
|
|
)
|
|
|
if raw is None:
|
|
|
legacy = legacy_public_network_pool(profile)
|
|
|
if (legacy is not None
|
|
|
and public_pool_runtime_fields(legacy)
|
|
|
!= public_pool_runtime_fields(requested)):
|
|
|
state = public_network_pool_state(
|
|
|
profile, legacy, claimed_addresses=claimed_addresses)
|
|
|
if state["allocated_count"]:
|
|
|
raise VMError(
|
|
|
"旧公网地址池仍有虚拟机或任务占用;首个新地址池必须沿用原网络配置",
|
|
|
409,
|
|
|
)
|
|
|
|
|
|
encoded = json.dumps(
|
|
|
{"version": 1, "pools": candidate},
|
|
|
ensure_ascii=False, sort_keys=True, separators=(",", ":"),
|
|
|
)
|
|
|
if compare_and_set_public_pool_catalog(raw, encoded):
|
|
|
return requested, False
|
|
|
raise VMError("公网地址池配置正在被其他请求修改,请刷新后重试", 409)
|
|
|
|
|
|
def credential_key(profile: dict, vm_id: str) -> str:
|
|
|
try:
|
|
|
canonical_id = str(uuid.UUID(vm_id))
|
|
|
return f"{host_scope(profile)}.{canonical_id}"
|
|
|
except (TypeError, ValueError, AttributeError, KeyError) as exc:
|
|
|
raise VMError("虚拟机或宿主机身份信息无效", 400) from exc
|
|
|
|
|
|
def save_credential(profile: dict, vm_id: str, auth_mode: str,
|
|
|
generation: str, password: str | None = None) -> None:
|
|
|
key = credential_key(profile, vm_id)
|
|
|
if not isinstance(generation, str) or not generation:
|
|
|
raise ValueError("虚拟机系统配置代号无效")
|
|
|
value = json.dumps({"auth_mode": auth_mode, "password": password,
|
|
|
"generation": generation},
|
|
|
ensure_ascii=False, separators=(",", ":"))
|
|
|
if data_store is not None:
|
|
|
data_store.set("vm_credential", key, value)
|
|
|
else:
|
|
|
test_credentials[key] = value
|
|
|
|
|
|
def delete_credential(profile: dict, vm_id: str) -> None:
|
|
|
key = credential_key(profile, vm_id)
|
|
|
if data_store is not None:
|
|
|
data_store.delete("vm_credential", key)
|
|
|
else:
|
|
|
test_credentials.pop(key, None)
|
|
|
|
|
|
def rescue_state_key(profile: dict, vm_id: str) -> str:
|
|
|
return credential_key(profile, vm_id)
|
|
|
|
|
|
def rescue_xml_matches(first: str, second: str) -> bool:
|
|
|
try:
|
|
|
return ET.canonicalize(first) == ET.canonicalize(second)
|
|
|
except (TypeError, ET.ParseError):
|
|
|
return False
|
|
|
|
|
|
def load_rescue_state(profile: dict, vm_id: str) -> dict | None:
|
|
|
key = rescue_state_key(profile, vm_id)
|
|
|
raw = (data_store.get("vm_rescue", key) if data_store is not None
|
|
|
else test_rescue_states.get(key))
|
|
|
if raw is None:
|
|
|
return None
|
|
|
try:
|
|
|
record = json.loads(raw)
|
|
|
except (TypeError, ValueError) as exc:
|
|
|
raise DatabaseError("虚拟机救援记录损坏") from exc
|
|
|
if (not isinstance(record, dict)
|
|
|
or not isinstance(record.get("original_xml"), str)
|
|
|
or not record["original_xml"].startswith("<domain")
|
|
|
or not isinstance(record.get("template_id"), str)
|
|
|
or record.get("volume_name") != rescue_volume_name(record["template_id"])
|
|
|
or record.get("status") not in {
|
|
|
"queued", "running", "staged", "active", "failed", "exiting",
|
|
|
}):
|
|
|
raise DatabaseError("虚拟机救援记录损坏")
|
|
|
return record
|
|
|
|
|
|
def save_rescue_state(profile: dict, vm_id: str, record: dict) -> None:
|
|
|
key = rescue_state_key(profile, vm_id)
|
|
|
value = json.dumps(record, ensure_ascii=False, separators=(",", ":"))
|
|
|
if data_store is not None:
|
|
|
data_store.set("vm_rescue", key, value)
|
|
|
else:
|
|
|
test_rescue_states[key] = value
|
|
|
|
|
|
def delete_rescue_state(profile: dict, vm_id: str) -> None:
|
|
|
key = rescue_state_key(profile, vm_id)
|
|
|
if data_store is not None:
|
|
|
data_store.delete("vm_rescue", key)
|
|
|
else:
|
|
|
test_rescue_states.pop(key, None)
|
|
|
|
|
|
def current_rescue_state(profile: dict, vm_id: str,
|
|
|
service: VMService | None = None) -> dict | None:
|
|
|
"""Clear a stale record only when the original inactive XML is intact.
|
|
|
|
|
|
An interrupted worker may have stopped before changing the VM. The
|
|
|
database job is marked failed on restart, but its recovery record must
|
|
|
not indefinitely block normal operations in that safe case.
|
|
|
Callers hold rescue_lock so this cannot race a live rescue worker.
|
|
|
"""
|
|
|
record = load_rescue_state(profile, vm_id)
|
|
|
if record is None or (host_scope(profile), vm_id) in pending_rescues:
|
|
|
return record
|
|
|
service = service or new_service(profile)
|
|
|
current_vm = service.get_vm(vm_id)
|
|
|
if current_vm.get("rescue_mode") or current_vm.get("rescue_staged"):
|
|
|
return record
|
|
|
try:
|
|
|
original_still_defined = rescue_xml_matches(
|
|
|
service.prepare_rescue_vm(vm_id, record["volume_name"]),
|
|
|
record["original_xml"],
|
|
|
)
|
|
|
except VMError:
|
|
|
return record
|
|
|
if original_still_defined:
|
|
|
delete_rescue_state(profile, vm_id)
|
|
|
return None
|
|
|
return record
|
|
|
|
|
|
def parse_credential(raw: str | None) -> dict | None:
|
|
|
if raw is None:
|
|
|
return None
|
|
|
try:
|
|
|
value = json.loads(raw)
|
|
|
except (TypeError, ValueError) as exc:
|
|
|
raise DatabaseError("虚拟机登录凭据记录损坏") from exc
|
|
|
if (not isinstance(value, dict)
|
|
|
or value.get("auth_mode") not in ("password", "ssh_key", "ssh_key_password")
|
|
|
or (value["auth_mode"] in ("password", "ssh_key_password")
|
|
|
and (not isinstance(value.get("password"), str)
|
|
|
or not value["password"]))
|
|
|
or not isinstance(value.get("generation"), str)
|
|
|
or not value["generation"]
|
|
|
or (value["auth_mode"] == "ssh_key" and value.get("password") is not None)):
|
|
|
raise DatabaseError("虚拟机登录凭据记录损坏")
|
|
|
return value
|
|
|
|
|
|
def get_credential(profile: dict, vm_id: str) -> dict | None:
|
|
|
key = credential_key(profile, vm_id)
|
|
|
raw = (data_store.get("vm_credential", key) if data_store is not None
|
|
|
else test_credentials.get(key))
|
|
|
return parse_credential(raw)
|
|
|
|
|
|
def firewall_record_key(profile: dict, vm_id: str) -> str:
|
|
|
return credential_key(profile, vm_id)
|
|
|
|
|
|
def load_firewall_record(profile: dict, vm_id: str) -> dict | None:
|
|
|
key = firewall_record_key(profile, vm_id)
|
|
|
raw = (data_store.get("vm_firewall", key) if data_store is not None
|
|
|
else test_firewalls.get(key))
|
|
|
if raw is None:
|
|
|
return None
|
|
|
try:
|
|
|
record = json.loads(raw)
|
|
|
policy = FirewallPolicyBody.model_validate({
|
|
|
"enabled": record["enabled"],
|
|
|
"revision": record["revision"],
|
|
|
"policy_digest": record["policy_digest"],
|
|
|
"rules": record["rules"],
|
|
|
})
|
|
|
except (KeyError, TypeError, ValueError) as exc:
|
|
|
raise DatabaseError("虚拟机防火墙记录损坏") from exc
|
|
|
record_version = record.get("version") if isinstance(record, dict) else None
|
|
|
if (not isinstance(record, dict) or record_version not in {1, 2}
|
|
|
or record.get("vm_id") != str(uuid.UUID(vm_id))
|
|
|
or not re.fullmatch(r"[0-9a-f]{64}",
|
|
|
record.get("policy_digest", ""))
|
|
|
or not re.fullmatch(r"[0-9a-f]{64}",
|
|
|
record.get("filter_digest", ""))
|
|
|
or isinstance(record.get("applied_at"), bool)
|
|
|
or not isinstance(record.get("applied_at"), int)):
|
|
|
raise DatabaseError("虚拟机防火墙记录损坏")
|
|
|
filter_version = record.get("filter_version", 1)
|
|
|
if (isinstance(filter_version, bool)
|
|
|
or not isinstance(filter_version, int)
|
|
|
or filter_version not in SUPPORTED_FIREWALL_FILTER_COMPILER_VERSIONS):
|
|
|
raise DatabaseError("虚拟机防火墙记录损坏")
|
|
|
try:
|
|
|
platform_policy = (
|
|
|
normalize_platform_policy_snapshot(
|
|
|
record["platform_policy_snapshot"])
|
|
|
if record_version == 2
|
|
|
else default_platform_policy_snapshot()
|
|
|
)
|
|
|
except (KeyError, TypeError, ValueError) as exc:
|
|
|
raise DatabaseError("虚拟机防火墙记录损坏") from exc
|
|
|
if (record_version == 2
|
|
|
and (not isinstance(record.get("platform_policy_digest"), str)
|
|
|
or not hmac.compare_digest(
|
|
|
record["platform_policy_digest"],
|
|
|
platform_policy["digest"],
|
|
|
))):
|
|
|
raise DatabaseError("虚拟机防火墙记录损坏")
|
|
|
if record_version == 1 and filter_version >= 4:
|
|
|
raise DatabaseError("虚拟机防火墙记录损坏")
|
|
|
normalized_policy = policy.model_dump()
|
|
|
if not hmac.compare_digest(
|
|
|
normalized_policy["policy_digest"],
|
|
|
firewall_policy_digest(
|
|
|
normalized_policy["enabled"],
|
|
|
normalized_policy["rules"],
|
|
|
)):
|
|
|
raise DatabaseError("虚拟机防火墙记录损坏")
|
|
|
expected_filter_digest = build_firewall_filter_xml(
|
|
|
vm_id,
|
|
|
normalized_policy["enabled"],
|
|
|
normalized_policy["rules"],
|
|
|
compiler_version=filter_version,
|
|
|
platform_policy_snapshot=platform_policy,
|
|
|
)[3]
|
|
|
if not hmac.compare_digest(
|
|
|
record["filter_digest"], expected_filter_digest):
|
|
|
raise DatabaseError("虚拟机防火墙记录损坏")
|
|
|
return {
|
|
|
**record,
|
|
|
**normalized_policy,
|
|
|
"version": record_version,
|
|
|
"filter_version": filter_version,
|
|
|
"platform_policy_snapshot": platform_policy,
|
|
|
"platform_policy_digest": platform_policy["digest"],
|
|
|
}
|
|
|
|
|
|
def save_firewall_record(profile: dict, vm_id: str, record: dict) -> None:
|
|
|
key = firewall_record_key(profile, vm_id)
|
|
|
encoded = json.dumps(
|
|
|
record, ensure_ascii=False, separators=(",", ":"))
|
|
|
if data_store is not None:
|
|
|
data_store.set("vm_firewall", key, encoded)
|
|
|
else:
|
|
|
test_firewalls[key] = encoded
|
|
|
|
|
|
def delete_firewall_record(profile: dict, vm_id: str) -> None:
|
|
|
key = firewall_record_key(profile, vm_id)
|
|
|
if data_store is not None:
|
|
|
data_store.delete("vm_firewall", key)
|
|
|
else:
|
|
|
test_firewalls.pop(key, None)
|
|
|
|
|
|
def is_default_platform_firewall(
|
|
|
vm_id: str, observed: dict,
|
|
|
) -> bool:
|
|
|
"""Recognize the exact record-free wrapper installed at VM creation."""
|
|
|
observed_digest = observed.get("filter_digest")
|
|
|
platform_policy = observed.get("platform_policy_snapshot")
|
|
|
expected_digest = build_firewall_filter_xml(
|
|
|
vm_id, False, [],
|
|
|
platform_policy_snapshot=platform_policy)[3]
|
|
|
return bool(
|
|
|
observed.get("filter_mode") == "managed"
|
|
|
and observed.get("sync_state") == "active"
|
|
|
and observed.get("coherent") is True
|
|
|
and isinstance(observed_digest, str)
|
|
|
and hmac.compare_digest(observed_digest, expected_digest)
|
|
|
)
|
|
|
|
|
|
def upgrade_firewall_filter_if_needed(
|
|
|
profile: dict, vm_id: str, service: VMService,
|
|
|
) -> bool:
|
|
|
"""Verify the host filter and recompile trusted legacy policy only."""
|
|
|
record = load_firewall_record(profile, vm_id)
|
|
|
observed = service.get_firewall_policy(vm_id)
|
|
|
observed_filter_digest = observed.get("filter_digest")
|
|
|
observed_mode = observed.get("filter_mode")
|
|
|
try:
|
|
|
observed_platform_policy = normalize_platform_policy_snapshot(
|
|
|
observed.get("platform_policy_snapshot")
|
|
|
or default_platform_policy_snapshot())
|
|
|
except ValueError as exc:
|
|
|
raise VMError(
|
|
|
"虚拟机平台网络策略证据无效", 409,
|
|
|
error_code="firewall_drift_detected",
|
|
|
) from exc
|
|
|
observed_active = (
|
|
|
observed.get("sync_state") == "active"
|
|
|
and observed.get("coherent") is True)
|
|
|
if record is None:
|
|
|
if (observed_active and observed_mode == "baseline"
|
|
|
and observed_filter_digest is None):
|
|
|
return False
|
|
|
if is_default_platform_firewall(vm_id, observed):
|
|
|
return False
|
|
|
raise VMError(
|
|
|
"防火墙实际状态与已保存版本不一致,请先核查漂移",
|
|
|
409,
|
|
|
error_code="firewall_drift_detected",
|
|
|
)
|
|
|
observation_active = (
|
|
|
observed_active
|
|
|
and observed_mode == "managed"
|
|
|
and isinstance(observed_filter_digest, str))
|
|
|
if not hmac.compare_digest(
|
|
|
record["platform_policy_digest"],
|
|
|
observed_platform_policy["digest"]):
|
|
|
raise VMError(
|
|
|
"虚拟机平台网络策略与引擎记录不一致",
|
|
|
409,
|
|
|
error_code="firewall_drift_detected",
|
|
|
)
|
|
|
if record["filter_version"] >= FIREWALL_FILTER_COMPILER_VERSION:
|
|
|
if (observation_active and hmac.compare_digest(
|
|
|
observed_filter_digest, record["filter_digest"])):
|
|
|
return False
|
|
|
raise VMError(
|
|
|
"防火墙实际状态与已保存版本不一致,请先核查漂移",
|
|
|
409,
|
|
|
error_code="firewall_drift_detected",
|
|
|
)
|
|
|
expected_filter_digest = build_firewall_filter_xml(
|
|
|
vm_id, record["enabled"], record["rules"],
|
|
|
platform_policy_snapshot=observed_platform_policy)[3]
|
|
|
if not observation_active:
|
|
|
raise VMError(
|
|
|
"防火墙实际状态与已保存版本不一致,请先核查漂移",
|
|
|
409,
|
|
|
error_code="firewall_drift_detected",
|
|
|
)
|
|
|
if hmac.compare_digest(
|
|
|
observed_filter_digest, expected_filter_digest):
|
|
|
upgraded = {
|
|
|
**record,
|
|
|
"version": 2,
|
|
|
"filter_version": FIREWALL_FILTER_COMPILER_VERSION,
|
|
|
"filter_digest": expected_filter_digest,
|
|
|
"platform_policy_snapshot": observed_platform_policy,
|
|
|
"platform_policy_digest": observed_platform_policy["digest"],
|
|
|
"applied_at": int(time.time()),
|
|
|
}
|
|
|
try:
|
|
|
save_firewall_record(profile, vm_id, upgraded)
|
|
|
except Exception as exc:
|
|
|
raise VMError(
|
|
|
"防火墙已升级但本地证据未确认,已停止修改网络",
|
|
|
503,
|
|
|
error_code="firewall_result_uncertain",
|
|
|
) from exc
|
|
|
return True
|
|
|
if not hmac.compare_digest(
|
|
|
observed_filter_digest, record["filter_digest"]):
|
|
|
raise VMError(
|
|
|
"防火墙实际状态与已保存版本不一致,请先核查漂移",
|
|
|
409,
|
|
|
error_code="firewall_drift_detected",
|
|
|
)
|
|
|
result = service.apply_firewall_policy(
|
|
|
vm_id, enabled=record["enabled"], rules=record["rules"])
|
|
|
result_policy_digest = result.get("policy_digest")
|
|
|
result_filter_digest = result.get("filter_digest")
|
|
|
result_platform_digest = result.get("platform_policy_digest")
|
|
|
if (not isinstance(result_policy_digest, str)
|
|
|
or not isinstance(result_filter_digest, str)
|
|
|
or not hmac.compare_digest(
|
|
|
result_policy_digest, record["policy_digest"])
|
|
|
or not hmac.compare_digest(
|
|
|
result_filter_digest, expected_filter_digest)
|
|
|
or not isinstance(result_platform_digest, str)
|
|
|
or not hmac.compare_digest(
|
|
|
result_platform_digest,
|
|
|
observed_platform_policy["digest"])):
|
|
|
raise VMError(
|
|
|
"防火墙编译器升级结果不确定,已停止修改网络",
|
|
|
503,
|
|
|
error_code="firewall_result_uncertain",
|
|
|
)
|
|
|
upgraded = {
|
|
|
**record,
|
|
|
"version": 2,
|
|
|
"filter_version": FIREWALL_FILTER_COMPILER_VERSION,
|
|
|
"filter_digest": expected_filter_digest,
|
|
|
"platform_policy_snapshot": observed_platform_policy,
|
|
|
"platform_policy_digest": observed_platform_policy["digest"],
|
|
|
"applied_at": int(time.time()),
|
|
|
}
|
|
|
try:
|
|
|
save_firewall_record(profile, vm_id, upgraded)
|
|
|
except Exception as exc:
|
|
|
raise VMError(
|
|
|
"防火墙已升级但本地证据未确认,已停止修改网络",
|
|
|
503,
|
|
|
error_code="firewall_result_uncertain",
|
|
|
) from exc
|
|
|
return True
|
|
|
|
|
|
def snapshot_record_key(vm_id: str, snapshot_key: str) -> str:
|
|
|
try:
|
|
|
canonical_id = str(uuid.UUID(vm_id))
|
|
|
except (TypeError, ValueError, AttributeError) as exc:
|
|
|
raise VMError("虚拟机 UUID 无效", 400) from exc
|
|
|
if not isinstance(snapshot_key, str) or not SNAPSHOT_KEY_RE.fullmatch(snapshot_key):
|
|
|
raise VMError("快照标识无效", 422)
|
|
|
return f"{canonical_id}:{snapshot_key}"
|
|
|
|
|
|
def parse_snapshot_record(key: str, raw: str) -> dict:
|
|
|
try:
|
|
|
record = json.loads(raw)
|
|
|
vm_id, snapshot_key = key.split(":", 1)
|
|
|
canonical_id = str(uuid.UUID(vm_id))
|
|
|
except (TypeError, ValueError, AttributeError) as exc:
|
|
|
raise DatabaseError("虚拟机快照记录损坏") from exc
|
|
|
allowed_states = {
|
|
|
"creating", "ready", "restoring", "deleting", "failed", "uncertain",
|
|
|
}
|
|
|
if (not isinstance(record, dict)
|
|
|
or record.get("vm_id") != canonical_id
|
|
|
or record.get("snapshot_key") != snapshot_key
|
|
|
or not SNAPSHOT_KEY_RE.fullmatch(snapshot_key)
|
|
|
or record.get("volume_name") != snapshot_volume_name(
|
|
|
canonical_id, snapshot_key)
|
|
|
or record.get("state") not in allowed_states
|
|
|
or record.get("consistency") not in {"filesystem", "crash"}
|
|
|
or not isinstance(record.get("name"), str)
|
|
|
or not 1 <= len(record["name"]) <= 64
|
|
|
or not isinstance(record.get("created_at"), int)
|
|
|
or record["created_at"] < 1):
|
|
|
raise DatabaseError("虚拟机快照记录损坏")
|
|
|
size_bytes = record.get("size_bytes")
|
|
|
if (size_bytes is not None
|
|
|
and (isinstance(size_bytes, bool) or not isinstance(size_bytes, int)
|
|
|
or size_bytes < 0)):
|
|
|
raise DatabaseError("虚拟机快照记录损坏")
|
|
|
return record
|
|
|
|
|
|
def list_snapshot_records(vm_id: str) -> list[dict]:
|
|
|
canonical_id = str(uuid.UUID(vm_id))
|
|
|
prefix = canonical_id + ":"
|
|
|
rows = (data_store.list("vm_snapshot") if data_store is not None
|
|
|
else test_snapshots.copy())
|
|
|
records = [
|
|
|
parse_snapshot_record(key, raw)
|
|
|
for key, raw in rows.items()
|
|
|
if key.startswith(prefix)
|
|
|
]
|
|
|
return sorted(records, key=lambda row: (row["created_at"], row["snapshot_key"]),
|
|
|
reverse=True)
|
|
|
|
|
|
def has_retained_snapshots() -> bool:
|
|
|
rows = (data_store.list("vm_snapshot") if data_store is not None
|
|
|
else test_snapshots.copy())
|
|
|
return any(
|
|
|
parse_snapshot_record(key, raw)["state"] in {
|
|
|
"ready", "creating", "restoring", "deleting", "uncertain",
|
|
|
}
|
|
|
for key, raw in rows.items()
|
|
|
)
|
|
|
|
|
|
def get_snapshot_record(vm_id: str, snapshot_key: str) -> dict:
|
|
|
key = snapshot_record_key(vm_id, snapshot_key)
|
|
|
raw = (data_store.get("vm_snapshot", key) if data_store is not None
|
|
|
else test_snapshots.get(key))
|
|
|
if raw is None:
|
|
|
raise VMError("快照不存在", 404)
|
|
|
return parse_snapshot_record(key, raw)
|
|
|
|
|
|
def save_snapshot_record(record: dict) -> None:
|
|
|
key = snapshot_record_key(record["vm_id"], record["snapshot_key"])
|
|
|
value = json.dumps(record, ensure_ascii=False, separators=(",", ":"))
|
|
|
if data_store is not None:
|
|
|
data_store.set("vm_snapshot", key, value)
|
|
|
else:
|
|
|
test_snapshots[key] = value
|
|
|
|
|
|
def insert_snapshot_record(record: dict) -> bool:
|
|
|
key = snapshot_record_key(record["vm_id"], record["snapshot_key"])
|
|
|
value = json.dumps(record, ensure_ascii=False, separators=(",", ":"))
|
|
|
if data_store is not None:
|
|
|
return bool(data_store.insert("vm_snapshot", key, value))
|
|
|
if key in test_snapshots:
|
|
|
return False
|
|
|
test_snapshots[key] = value
|
|
|
return True
|
|
|
|
|
|
def delete_snapshot_record(vm_id: str, snapshot_key: str) -> None:
|
|
|
key = snapshot_record_key(vm_id, snapshot_key)
|
|
|
if data_store is not None:
|
|
|
data_store.delete("vm_snapshot", key)
|
|
|
else:
|
|
|
test_snapshots.pop(key, None)
|
|
|
|
|
|
def public_snapshot_record(record: dict) -> dict:
|
|
|
return {
|
|
|
key: record[key]
|
|
|
for key in (
|
|
|
"snapshot_key", "name", "state", "consistency", "created_at",
|
|
|
"completed_at", "size_bytes", "last_restored_at",
|
|
|
)
|
|
|
if key in record
|
|
|
}
|
|
|
|
|
|
def forbid_pending_snapshot(
|
|
|
profile: dict, vm_id: str, label: str, *, allow_uncertain: bool = False,
|
|
|
ignore_suspend_barrier: bool = False,
|
|
|
) -> None:
|
|
|
key = (host_scope(profile), str(uuid.UUID(vm_id)))
|
|
|
with snapshot_lock:
|
|
|
if key in snapshot_suspend_barriers and not ignore_suspend_barrier:
|
|
|
raise VMError(
|
|
|
f"虚拟机正在执行到期暂停策略,暂时不能{label}", 409)
|
|
|
if key in pending_snapshot_jobs:
|
|
|
raise VMError(f"虚拟机正在执行快照任务,暂时不能{label}", 409)
|
|
|
blocked_states = {"creating", "restoring", "deleting"}
|
|
|
if not allow_uncertain:
|
|
|
blocked_states.add("uncertain")
|
|
|
if any(row["state"] in blocked_states
|
|
|
for row in list_snapshot_records(key[1])):
|
|
|
raise VMError(
|
|
|
"虚拟机存在进行中或结果不确定的快照任务,必须先完成安全核对",
|
|
|
409,
|
|
|
)
|
|
|
|
|
|
def forbid_snapshot_suspend_barrier(
|
|
|
profile: dict, vm_id: str, label: str,
|
|
|
) -> None:
|
|
|
key = (host_scope(profile), str(uuid.UUID(vm_id)))
|
|
|
with snapshot_lock:
|
|
|
if key in snapshot_suspend_barriers:
|
|
|
raise VMError(
|
|
|
f"虚拟机正在执行到期暂停策略,暂时不能{label}", 409)
|
|
|
|
|
|
def forbid_retained_snapshot(profile: dict, vm_id: str, label: str) -> None:
|
|
|
"""Prevent replacing vda while a local recovery point is retained.
|
|
|
|
|
|
Instance settings may safely rotate the current cloud-init seed: a
|
|
|
later restore intentionally applies that current seed to the restored
|
|
|
disk. Reinstall is different because it replaces the disk baseline
|
|
|
that every retained local snapshot belongs to.
|
|
|
"""
|
|
|
canonical_id = str(uuid.UUID(vm_id))
|
|
|
forbid_pending_snapshot(profile, canonical_id, label)
|
|
|
with snapshot_lock:
|
|
|
if any(row["state"] != "failed"
|
|
|
for row in list_snapshot_records(canonical_id)):
|
|
|
raise VMError(
|
|
|
f"虚拟机仍保留本地快照,请删除全部快照后再{label}",
|
|
|
409,
|
|
|
)
|
|
|
|
|
|
def rollback_cancelled_snapshot_catalog(job: dict) -> None:
|
|
|
"""Roll back a snapshot row after its Future was cancelled queued.
|
|
|
|
|
|
The durable ``cancelled_before_start`` marker proves that the executor
|
|
|
accepted ``Future.cancel()`` before this helper runs. Consequently no
|
|
|
libvirt mutation can have started, and the admission-state row can be
|
|
|
restored without inspecting or changing the host.
|
|
|
"""
|
|
|
vm_id = job.get("vm_id")
|
|
|
snapshot_key = job.get("snapshot_key")
|
|
|
action = job.get("action")
|
|
|
if (not isinstance(vm_id, str) or not isinstance(snapshot_key, str)
|
|
|
or action not in {"create", "restore", "delete"}):
|
|
|
raise DatabaseError("取消的快照任务身份无效")
|
|
|
try:
|
|
|
record = get_snapshot_record(vm_id, snapshot_key)
|
|
|
except VMError as exc:
|
|
|
if action == "create" and exc.status == 404:
|
|
|
return
|
|
|
raise DatabaseError("取消的快照登记记录不存在") from exc
|
|
|
expected_state = {
|
|
|
"create": "creating",
|
|
|
"restore": "restoring",
|
|
|
"delete": "deleting",
|
|
|
}[action]
|
|
|
if action == "create":
|
|
|
if record["state"] != expected_state:
|
|
|
raise DatabaseError("取消的快照创建登记状态已变化")
|
|
|
delete_snapshot_record(vm_id, snapshot_key)
|
|
|
try:
|
|
|
get_snapshot_record(vm_id, snapshot_key)
|
|
|
except VMError as exc:
|
|
|
if exc.status == 404:
|
|
|
return
|
|
|
raise
|
|
|
raise DatabaseError("取消的快照创建登记未能删除")
|
|
|
if record["state"] not in {expected_state, "ready"}:
|
|
|
raise DatabaseError("取消的快照登记状态已变化")
|
|
|
if record["state"] != "ready":
|
|
|
record["state"] = "ready"
|
|
|
save_snapshot_record(record)
|
|
|
if get_snapshot_record(vm_id, snapshot_key)["state"] != "ready":
|
|
|
raise DatabaseError("取消的快照登记未能恢复")
|
|
|
|
|
|
def finish_cancelled_snapshot_job(job_id: str, job: dict) -> dict:
|
|
|
"""Finish the second phase of a durable pre-start cancellation."""
|
|
|
if (job.get("cancelled_before_start") is not True
|
|
|
or job.get("cancel_reason") != "network_suspend"):
|
|
|
raise DatabaseError("快照取消凭据无效")
|
|
|
rollback_cancelled_snapshot_catalog(job)
|
|
|
terminal = dict(job)
|
|
|
terminal.update(
|
|
|
status="failed",
|
|
|
phase="排队快照已由到期网络暂停策略取消",
|
|
|
error="快照任务在开始前已取消,未调用虚拟化操作",
|
|
|
cancelled_at=int(time.time()),
|
|
|
finished_at=int(time.time()),
|
|
|
)
|
|
|
terminal.pop("result", None)
|
|
|
terminal.pop("error_code", None)
|
|
|
# The terminal job is persisted only after catalog rollback. If this
|
|
|
# write fails, the durable phase-one tombstone remains queued and both
|
|
|
# startup recovery and operation polling can retry this exact step.
|
|
|
save_job("snapshot_job", job_id, terminal)
|
|
|
snapshot_jobs[job_id] = terminal
|
|
|
pending_snapshot_jobs.discard((terminal["scope"], terminal["vm_id"]))
|
|
|
snapshot_futures.pop(job_id, None)
|
|
|
return dict(terminal)
|
|
|
|
|
|
def cancel_queued_snapshot_for_network_suspend(
|
|
|
profile: dict, vm_id: str,
|
|
|
) -> None:
|
|
|
"""Cancel exactly one queued same-VM job without touching libvirt."""
|
|
|
scope = host_scope(profile)
|
|
|
matches = [
|
|
|
(job_id, job)
|
|
|
for job_id, job in snapshot_jobs.items()
|
|
|
if (job.get("scope") == scope and job.get("vm_id") == vm_id
|
|
|
and job.get("status") in {"queued", "running"})
|
|
|
]
|
|
|
if len(matches) > 1:
|
|
|
raise VMError("虚拟机存在多个快照任务,拒绝自动取消", 409)
|
|
|
if matches:
|
|
|
job_id, job = matches[0]
|
|
|
if job.get("status") != "queued":
|
|
|
raise VMError("虚拟机快照任务已经开始,暂时不能暂停网络", 409)
|
|
|
if job.get("cancelled_before_start") is not True:
|
|
|
future = snapshot_futures.get(job_id)
|
|
|
if future is None or not future.cancel():
|
|
|
# A Future becomes RUNNING before its callable acquires
|
|
|
# snapshot_lock. Never infer safety from the still-queued
|
|
|
# job row when the executor has already claimed it.
|
|
|
raise VMError(
|
|
|
"虚拟机快照任务已经由执行器领取,暂时不能暂停网络", 409)
|
|
|
tombstone = dict(job)
|
|
|
tombstone.update(
|
|
|
cancelled_before_start=True,
|
|
|
cancel_reason="network_suspend",
|
|
|
cancel_requested_at=int(time.time()),
|
|
|
phase="正在取消尚未开始的排队快照",
|
|
|
)
|
|
|
# Future.cancel() is already definitive process-local proof.
|
|
|
# Publish the tombstone in memory first so a transient phase-
|
|
|
# one database failure remains retryable by operation polling
|
|
|
# without ever letting the cancelled callable reach libvirt.
|
|
|
snapshot_jobs[job_id] = tombstone
|
|
|
job = tombstone
|
|
|
# Persist proof of Future.cancel() before changing the catalog.
|
|
|
try:
|
|
|
save_job("snapshot_job", job_id, tombstone)
|
|
|
except Exception as exc:
|
|
|
LOG.exception(
|
|
|
"Could not persist queued snapshot cancellation: job=%s uuid=%s",
|
|
|
job_id, vm_id,
|
|
|
)
|
|
|
raise VMError(
|
|
|
"排队快照已停止,但取消凭据尚未持久化,暂时不能暂停网络",
|
|
|
503,
|
|
|
) from exc
|
|
|
try:
|
|
|
finish_cancelled_snapshot_job(job_id, job)
|
|
|
except Exception as exc:
|
|
|
LOG.exception(
|
|
|
"Could not finish queued snapshot cancellation: job=%s uuid=%s",
|
|
|
job_id, vm_id,
|
|
|
)
|
|
|
raise VMError(
|
|
|
"排队快照已停止,但登记回滚尚未完成,暂时不能暂停网络",
|
|
|
503,
|
|
|
) from exc
|
|
|
if (scope, vm_id) in pending_snapshot_jobs:
|
|
|
raise VMError("虚拟机快照任务状态尚未收敛,暂时不能暂停网络", 409)
|
|
|
|
|
|
def decorate_vm(vm: dict, credential: dict | None, profile: dict) -> dict:
|
|
|
cloud = bool(vm.get("managed") and vm.get("os_source") == "cloud_template")
|
|
|
current_credential = (credential if cloud and credential
|
|
|
and credential["generation"] == vm.get("credential_generation")
|
|
|
else None)
|
|
|
return {
|
|
|
**{key: value for key, value in vm.items() if key != "credential_generation"},
|
|
|
"region": profile.get("region", ""),
|
|
|
"login_name": "root" if cloud else None,
|
|
|
"auth_mode": current_credential["auth_mode"] if current_credential else "unknown",
|
|
|
"password_available": bool(current_credential and
|
|
|
current_credential["auth_mode"] in (
|
|
|
"password", "ssh_key_password")),
|
|
|
}
|
|
|
|
|
|
def save_job(kind: str, job_id: str, job: dict) -> None:
|
|
|
delegated = TRUSTED_ACTOR.get() if internal_mode else None
|
|
|
if delegated and delegated.get("operation_id"):
|
|
|
job.setdefault("operation_id", delegated["operation_id"])
|
|
|
if data_store is not None:
|
|
|
data_store.set(kind, job_id, json.dumps(
|
|
|
job, ensure_ascii=False, separators=(",", ":")))
|
|
|
|
|
|
def delete_job(kind: str, job_id: str) -> None:
|
|
|
if data_store is not None:
|
|
|
data_store.delete(kind, job_id)
|
|
|
|
|
|
def save_user_iso_job(job_id: str, job: dict) -> None:
|
|
|
if shared_user_iso_runtime is not None:
|
|
|
shared_user_iso_runtime.save_job(job_id, job)
|
|
|
else:
|
|
|
save_job("user_iso_import_job", job_id, job)
|
|
|
|
|
|
def delete_user_iso_job(job_id: str) -> None:
|
|
|
if shared_user_iso_runtime is not None:
|
|
|
shared_user_iso_runtime.delete_job(job_id)
|
|
|
else:
|
|
|
delete_job("user_iso_import_job", job_id)
|
|
|
|
|
|
def restore_jobs() -> None:
|
|
|
"""Restore interrupted jobs without pretending mutations resumed."""
|
|
|
if data_store is None:
|
|
|
return
|
|
|
pending_reinstalls.clear()
|
|
|
destinations = [
|
|
|
("reinstall_job", reinstall_jobs),
|
|
|
("rescue_import_job", rescue_import_jobs),
|
|
|
("rescue_job", rescue_jobs),
|
|
|
("snapshot_job", snapshot_jobs),
|
|
|
]
|
|
|
if shared_cloud_template_runtime is None:
|
|
|
destinations.append(("import_job", import_jobs))
|
|
|
if shared_user_iso_runtime is None:
|
|
|
destinations.append(("user_iso_import_job", user_iso_import_jobs))
|
|
|
for kind, destination in destinations:
|
|
|
for job_id, raw in data_store.list(kind).items():
|
|
|
try:
|
|
|
job = json.loads(raw)
|
|
|
except (TypeError, ValueError) as exc:
|
|
|
raise DatabaseError("数据库任务记录损坏") from exc
|
|
|
if not isinstance(job, dict):
|
|
|
raise DatabaseError("数据库任务记录损坏")
|
|
|
if (kind == "reinstall_job"
|
|
|
and job.get("status") in {
|
|
|
"queued", "running", "succeeded", "completed", "done",
|
|
|
"failed", "cancelled", "canceled", "uncertain",
|
|
|
}
|
|
|
and reinstall_post_commit(job)):
|
|
|
# Releases before the cleanup boundary could persist a
|
|
|
# warning/phase without the explicit marker. Upgrade any
|
|
|
# such durable shape on startup so a direct engine poll
|
|
|
# cannot present it as a completed or retryable reinstall.
|
|
|
cleanup_pending = reinstall_cleanup_pending(job)
|
|
|
job["status"] = "uncertain"
|
|
|
job["phase"] = (
|
|
|
"重装完成,但旧系统盘清理待核对"
|
|
|
if cleanup_pending else "重装结果待核对"
|
|
|
)
|
|
|
job["error_code"] = (
|
|
|
REINSTALL_CLEANUP_PENDING_ERROR_CODE
|
|
|
if cleanup_pending else REINSTALL_RESULT_UNCERTAIN_ERROR_CODE
|
|
|
)
|
|
|
job["error"] = (
|
|
|
"重装已完成,但旧系统盘清理待核对"
|
|
|
if cleanup_pending else "重装结果待核对"
|
|
|
)
|
|
|
job["reinstall_committed"] = True
|
|
|
if cleanup_pending:
|
|
|
job["cleanup_pending"] = True
|
|
|
job.pop("finished_at", None)
|
|
|
save_job(kind, job_id, job)
|
|
|
if (kind == "snapshot_job"
|
|
|
and job.get("status") == "queued"
|
|
|
and job.get("cancelled_before_start") is True):
|
|
|
# Phase one proves the previous process cancelled the
|
|
|
# Future before its callable began. Finish only the local
|
|
|
# catalog rollback; never reinterpret this as an
|
|
|
# interrupted libvirt mutation.
|
|
|
try:
|
|
|
job = finish_cancelled_snapshot_job(job_id, job)
|
|
|
except Exception:
|
|
|
LOG.exception(
|
|
|
"Could not finish durable queued snapshot cancellation: %s",
|
|
|
job_id,
|
|
|
)
|
|
|
destination[job_id] = job
|
|
|
continue
|
|
|
if (kind == "snapshot_job"
|
|
|
and job.get("action") == "create"
|
|
|
and job.get("consistency") == "filesystem"
|
|
|
and job.get("guest_freeze_state") == "attempted"):
|
|
|
try:
|
|
|
profile = manager.current()
|
|
|
if profile is None or profile.get("key_locked"):
|
|
|
raise VMError(
|
|
|
"宿主机未配置或密钥已锁定,无法恢复访客文件系统")
|
|
|
recovery_service = (
|
|
|
service_factory(profile)
|
|
|
if service_factory is not None
|
|
|
else VMService(
|
|
|
profile["libvirt_uri"],
|
|
|
profile["storage_pool"],
|
|
|
profile["iso_pool"],
|
|
|
profile["default_network"],
|
|
|
rescue_iso_pool=profile.get(
|
|
|
"rescue_iso_pool", ""),
|
|
|
)
|
|
|
)
|
|
|
recovery_service.thaw_guest_filesystems(job["vm_id"])
|
|
|
except Exception:
|
|
|
job["freeze_recovery"] = "failed"
|
|
|
LOG.exception(
|
|
|
"Could not recover interrupted guest freeze: "
|
|
|
"job=%s uuid=%s",
|
|
|
job_id, job.get("vm_id"),
|
|
|
)
|
|
|
else:
|
|
|
job["guest_freeze_state"] = "recovered"
|
|
|
job["freeze_recovery"] = "confirmed"
|
|
|
job["freeze_recovered_at"] = int(time.time())
|
|
|
save_job(kind, job_id, job)
|
|
|
if (kind == "reinstall_job"
|
|
|
and job.get("status") == "running"):
|
|
|
# The replacement domain was durably observed as started,
|
|
|
# or may have crossed that boundary before the process
|
|
|
# ended. Preserve an uncertain result in either case;
|
|
|
# a durable post-commit phase or marker authorizes PHP
|
|
|
# to copy the replacement credential/spec into its cache.
|
|
|
committed = reinstall_post_commit(job)
|
|
|
cleanup_pending = reinstall_cleanup_pending(job)
|
|
|
job["status"] = "uncertain"
|
|
|
job["phase"] = (
|
|
|
"重装完成,但旧系统盘清理待核对"
|
|
|
if cleanup_pending else "重装结果待核对"
|
|
|
)
|
|
|
# A stale terminal timestamp must not make the recovered
|
|
|
# operation look settled while its physical result is
|
|
|
# still awaiting inspection.
|
|
|
job.pop("finished_at", None)
|
|
|
if committed:
|
|
|
job["error_code"] = (
|
|
|
REINSTALL_CLEANUP_PENDING_ERROR_CODE
|
|
|
if cleanup_pending else REINSTALL_RESULT_UNCERTAIN_ERROR_CODE
|
|
|
)
|
|
|
job["error"] = (
|
|
|
"重装已完成,但旧系统盘清理待核对"
|
|
|
if cleanup_pending
|
|
|
else "重装结果待核对;面板重启后未能确认旧存储卷清理状态"
|
|
|
)
|
|
|
job["reinstall_committed"] = True
|
|
|
if cleanup_pending:
|
|
|
job["cleanup_pending"] = True
|
|
|
else:
|
|
|
for key in ("error_code", "cleanup_pending", "warning",
|
|
|
"generated_password", "reinstall_committed"):
|
|
|
job.pop(key, None)
|
|
|
job["error"] = "重装结果待核对;面板重启后无法确认新系统是否已启动"
|
|
|
save_job(kind, job_id, job)
|
|
|
elif job.get("status") in {"queued", "running"}:
|
|
|
if kind == "snapshot_job":
|
|
|
# A push backup or disk switch can continue or commit
|
|
|
# after the HTTP process disappears. Never convert that
|
|
|
# ambiguity into a retryable terminal failure.
|
|
|
job["status"] = "uncertain"
|
|
|
job["phase"] = "引擎重启,快照结果待核查"
|
|
|
job["error"] = "引擎在快照任务期间重启;必须核查存储卷和虚拟机定义,不能重复提交"
|
|
|
try:
|
|
|
record = get_snapshot_record(
|
|
|
job["vm_id"], job["snapshot_key"])
|
|
|
record["state"] = "uncertain"
|
|
|
save_snapshot_record(record)
|
|
|
except (KeyError, VMError, DatabaseError, ValueError):
|
|
|
LOG.exception(
|
|
|
"Could not mark interrupted snapshot record uncertain: %s",
|
|
|
job_id)
|
|
|
else:
|
|
|
job["status"] = "failed"
|
|
|
job["phase"] = "面板重启,任务已中断"
|
|
|
job["error"] = "面板在任务执行期间重启;请检查宿主机实际状态,勿直接重复提交"
|
|
|
for key in ("generated_password", "reinstall_committed",
|
|
|
"cleanup_pending", "error_code", "warning"):
|
|
|
job.pop(key, None)
|
|
|
save_job(kind, job_id, job)
|
|
|
elif kind == "snapshot_job" and job.get("status") == "uncertain":
|
|
|
# A previous process may have persisted the job outcome
|
|
|
# before it could mirror that state into the catalog.
|
|
|
# The durable operation identity exists, so preserve the
|
|
|
# row for evidence-only reconciliation rather than
|
|
|
# treating it as the pre-queue crash window below.
|
|
|
try:
|
|
|
record = get_snapshot_record(
|
|
|
job["vm_id"], job["snapshot_key"])
|
|
|
if record["state"] in {
|
|
|
"creating", "restoring", "deleting"}:
|
|
|
record["state"] = "uncertain"
|
|
|
save_snapshot_record(record)
|
|
|
except (KeyError, VMError, DatabaseError, ValueError):
|
|
|
LOG.exception(
|
|
|
"Could not align uncertain snapshot catalog row: %s",
|
|
|
job_id,
|
|
|
)
|
|
|
if (kind == "reinstall_job"
|
|
|
and job.get("status") in {"queued", "running", "uncertain"}
|
|
|
and isinstance(job.get("scope"), str)
|
|
|
and isinstance(job.get("vm_id"), str)):
|
|
|
# Rebuild the in-process conflict barrier after a panel
|
|
|
# restart. Other VM mutation endpoints consult this set
|
|
|
# before touching the same guest.
|
|
|
pending_reinstalls.add((job["scope"], job["vm_id"]))
|
|
|
destination[job_id] = job
|
|
|
|
|
|
# The registry row is reserved before the durable job row. A process
|
|
|
# loss in that narrow window otherwise leaves a permanent
|
|
|
# creating/restoring/deleting record with no operation to reconcile.
|
|
|
# No libvirt mutation is invoked until queue_snapshot_job has durably
|
|
|
# saved its job row. Therefore a transitional catalog row with no
|
|
|
# corresponding durable job proves the process died before the worker
|
|
|
# could cross a physical mutation boundary.
|
|
|
durable_snapshot_records = {
|
|
|
(job.get("vm_id"), job.get("snapshot_key"))
|
|
|
for job in snapshot_jobs.values()
|
|
|
}
|
|
|
for key, raw in data_store.list("vm_snapshot").items():
|
|
|
record = parse_snapshot_record(key, raw)
|
|
|
identity = (record["vm_id"], record["snapshot_key"])
|
|
|
if (record["state"] in {"creating", "restoring", "deleting"}
|
|
|
and identity not in durable_snapshot_records):
|
|
|
if record["state"] == "creating":
|
|
|
delete_snapshot_record(*identity)
|
|
|
else:
|
|
|
record["state"] = "ready"
|
|
|
record["interrupted_at"] = int(time.time())
|
|
|
save_snapshot_record(record)
|
|
|
LOG.warning(
|
|
|
"Rolled back snapshot registry row created before durable job: "
|
|
|
"uuid=%s snapshot=%s",
|
|
|
*identity,
|
|
|
)
|
|
|
|
|
|
restore_jobs()
|
|
|
sessions = DatabaseSessionStore(data_store) if data_store is not None else SessionStore()
|
|
|
login_limiter = LoginLimiter()
|
|
|
registration_limiter = LoginLimiter()
|
|
|
code_send_limiter = LoginLimiter()
|
|
|
setup_limiter = LoginLimiter()
|
|
|
unlock_limiter = LoginLimiter()
|
|
|
def metric_profile():
|
|
|
if manager is None:
|
|
|
return None
|
|
|
background = getattr(manager, "current_for_background", None)
|
|
|
return background() if callable(background) else manager.current()
|
|
|
|
|
|
metric_collector = MetricsCollector(
|
|
|
lambda: metric_store if metric_store is not None else data_store,
|
|
|
metric_profile,
|
|
|
lambda profile: new_service(profile),
|
|
|
owner_getter=lambda scope, vm_id: state.users.owner_of_vm(scope, vm_id),
|
|
|
) if database_mode else None
|
|
|
|
|
|
@asynccontextmanager
|
|
|
async def lifespan(_app: FastAPI):
|
|
|
if metric_collector is not None:
|
|
|
metric_collector.start()
|
|
|
try:
|
|
|
yield
|
|
|
finally:
|
|
|
if metric_collector is not None:
|
|
|
metric_collector.close()
|
|
|
reinstall_executor.shutdown(wait=True, cancel_futures=False)
|
|
|
snapshot_executor.shutdown(wait=True, cancel_futures=False)
|
|
|
import_executor.shutdown(wait=True, cancel_futures=False)
|
|
|
rescue_executor.shutdown(wait=True, cancel_futures=False)
|
|
|
rescue_import_executor.shutdown(wait=True, cancel_futures=False)
|
|
|
if shared_user_iso_runtime is None:
|
|
|
user_iso_import_executor.shutdown(wait=True, cancel_futures=False)
|
|
|
close_manager = getattr(manager, "close", None) if manager is not None else None
|
|
|
if callable(close_manager):
|
|
|
close_manager()
|
|
|
|
|
|
app = FastAPI(title="Virsh Panel API", docs_url=None, redoc_url=None, lifespan=lifespan)
|
|
|
|
|
|
app.mount("/static", StaticFiles(directory=str(settings.static_dir)), name="static")
|
|
|
app.mount(
|
|
|
"/vendor/novnc",
|
|
|
StaticFiles(directory=str(settings.static_dir / "vendor" / "novnc")),
|
|
|
name="novnc",
|
|
|
)
|
|
|
|
|
|
@app.middleware("http")
|
|
|
async def security_headers(request: Request, call_next):
|
|
|
if not internal_mode and request.url.scheme != "https" and not (
|
|
|
request.url.path == "/api/v1/health"
|
|
|
and request.url.scheme == "http"
|
|
|
and _plain_loopback_allowed(request.scope, request.headers.get("host"))
|
|
|
):
|
|
|
return JSONResponse(status_code=400, content={"error": "请使用 HTTPS 访问面板"})
|
|
|
response = await call_next(request)
|
|
|
response.headers["X-Content-Type-Options"] = "nosniff"
|
|
|
response.headers["Referrer-Policy"] = "no-referrer"
|
|
|
response.headers["X-Frame-Options"] = "DENY"
|
|
|
response.headers["Content-Security-Policy"] = (
|
|
|
"default-src 'self'; script-src 'self'; style-src 'self'; "
|
|
|
"img-src 'self' data:; connect-src 'self' wss:; "
|
|
|
"object-src 'none'; frame-ancestors 'none'; base-uri 'self'"
|
|
|
)
|
|
|
if (request.url.path == "/" or request.url.path.startswith("/api/")
|
|
|
or request.url.path in ("/static/app.js", "/static/styles.css")):
|
|
|
response.headers["Cache-Control"] = "no-store"
|
|
|
return response
|
|
|
|
|
|
@app.exception_handler(VMError)
|
|
|
async def handle_vm_error(_request: Request, exc: VMError):
|
|
|
content = {"error": str(exc)}
|
|
|
if exc.error_code:
|
|
|
content["error_code"] = exc.error_code
|
|
|
return JSONResponse(status_code=exc.status, content=content)
|
|
|
|
|
|
@app.exception_handler(PublicIpClaimCleanupPending)
|
|
|
async def handle_public_ip_claim_cleanup(
|
|
|
_request: Request, exc: PublicIpClaimCleanupPending,
|
|
|
):
|
|
|
return JSONResponse(status_code=exc.status, content={
|
|
|
"error": str(exc),
|
|
|
"error_code": "public_ip_claim_cleanup_pending",
|
|
|
})
|
|
|
|
|
|
@app.exception_handler(RemoteHostError)
|
|
|
async def handle_remote_error(_request: Request, exc: RemoteHostError):
|
|
|
content = {"error": str(exc), "error_code": exc.error_code}
|
|
|
if exc.status == 423:
|
|
|
content["error_code"] = "key_locked"
|
|
|
return JSONResponse(status_code=exc.status, content=content)
|
|
|
|
|
|
@app.exception_handler(SetupError)
|
|
|
async def handle_setup_error(_request: Request, exc: SetupError):
|
|
|
return JSONResponse(status_code=exc.status, content={"error": str(exc)})
|
|
|
|
|
|
@app.exception_handler(DatabaseSetupError)
|
|
|
async def handle_database_setup_error(_request: Request, exc: DatabaseSetupError):
|
|
|
return JSONResponse(status_code=exc.status, content={"error": str(exc)})
|
|
|
|
|
|
@app.exception_handler(DatabaseError)
|
|
|
async def handle_database_error(_request: Request, exc: DatabaseError):
|
|
|
return JSONResponse(status_code=exc.status, content={"error": str(exc)})
|
|
|
|
|
|
@app.exception_handler(UpdateError)
|
|
|
async def handle_update_error(_request: Request, exc: UpdateError):
|
|
|
return JSONResponse(status_code=exc.status, content={"error": str(exc)})
|
|
|
|
|
|
@app.exception_handler(TemplateError)
|
|
|
async def handle_template_error(_request: Request, exc: TemplateError):
|
|
|
return JSONResponse(status_code=exc.status, content={"error": str(exc)})
|
|
|
|
|
|
@app.exception_handler(UserError)
|
|
|
async def handle_user_error(_request: Request, exc: UserError):
|
|
|
return JSONResponse(status_code=exc.status, content={"error": str(exc)})
|
|
|
|
|
|
@app.exception_handler(SMTPError)
|
|
|
async def handle_smtp_error(_request: Request, exc: SMTPError):
|
|
|
return JSONResponse(status_code=exc.status, content={"error": str(exc)})
|
|
|
|
|
|
@app.exception_handler(SSHKeyError)
|
|
|
async def handle_ssh_key_error(_request: Request, exc: SSHKeyError):
|
|
|
return JSONResponse(status_code=exc.status, content={"error": str(exc)})
|
|
|
|
|
|
def account_for_session(session: Session) -> dict:
|
|
|
if internal_mode:
|
|
|
return actor_account(session)
|
|
|
if database_mode:
|
|
|
if state is None:
|
|
|
raise HTTPException(status_code=401, detail="请先登录")
|
|
|
user_id = getattr(session, "user_id", None)
|
|
|
if isinstance(user_id, bool) or not isinstance(user_id, int) or user_id < 1:
|
|
|
raise HTTPException(status_code=401, detail="账号不可用,请重新登录")
|
|
|
account = state.users.get_by_id(user_id)
|
|
|
if (account is None or account["username"] != session.username
|
|
|
or not account["email_verified"]):
|
|
|
raise HTTPException(status_code=401, detail="账号不可用,请重新登录")
|
|
|
return account
|
|
|
# The injected JSON state is retained solely for the isolated HTTP
|
|
|
# test harness; production authorization always uses the users table.
|
|
|
return {"id": 0, "username": session.username, "admin": True,
|
|
|
"approved": True, "email_verified": True}
|
|
|
|
|
|
def host_ready() -> bool:
|
|
|
if manager is None:
|
|
|
return False
|
|
|
profile = manager.current()
|
|
|
return profile is not None and not profile.get("key_locked", False)
|
|
|
|
|
|
def require_session(request: Request) -> Session:
|
|
|
if internal_mode:
|
|
|
return actor_session()
|
|
|
session = sessions.get(request.cookies.get(COOKIE_NAME))
|
|
|
if session is None:
|
|
|
raise HTTPException(status_code=401, detail="请先登录")
|
|
|
account_for_session(session)
|
|
|
return session
|
|
|
|
|
|
def require_write(request: Request) -> Session:
|
|
|
session = require_session(request)
|
|
|
if internal_mode:
|
|
|
if not actor()["can_manage"]:
|
|
|
raise HTTPException(status_code=403, detail="当前账户不能执行虚拟机操作")
|
|
|
return session
|
|
|
csrf = request.headers.get("X-CSRF-Token", "")
|
|
|
if not hmac.compare_digest(csrf, session.csrf_token):
|
|
|
raise HTTPException(status_code=403, detail="请求验证失败")
|
|
|
return session
|
|
|
|
|
|
def require_admin(request: Request) -> Session:
|
|
|
session = require_session(request)
|
|
|
if not account_for_session(session)["admin"]:
|
|
|
raise HTTPException(status_code=403, detail="仅管理员可以执行此操作")
|
|
|
return session
|
|
|
|
|
|
def require_admin_write(request: Request) -> Session:
|
|
|
session = require_write(request)
|
|
|
if internal_mode and not actor()["is_super_admin"]:
|
|
|
raise HTTPException(status_code=403, detail="只有超级管理员可以修改宿主机或镜像")
|
|
|
if not account_for_session(session)["admin"]:
|
|
|
raise HTTPException(status_code=403, detail="仅管理员可以执行此操作")
|
|
|
return session
|
|
|
|
|
|
def require_approved(request: Request) -> Session:
|
|
|
session = require_session(request)
|
|
|
account = account_for_session(session)
|
|
|
if not account["admin"] and not account["approved"]:
|
|
|
raise HTTPException(status_code=403, detail="账号等待管理员批准")
|
|
|
return session
|
|
|
|
|
|
def require_approved_write(request: Request) -> Session:
|
|
|
session = require_write(request)
|
|
|
account = account_for_session(session)
|
|
|
if not account["admin"] and not account["approved"]:
|
|
|
raise HTTPException(status_code=403, detail="账号等待管理员批准")
|
|
|
return session
|
|
|
|
|
|
def require_internal_service_write(request: Request) -> Session:
|
|
|
if not internal_mode:
|
|
|
raise HTTPException(status_code=404, detail="Not Found")
|
|
|
return require_approved_write(request)
|
|
|
|
|
|
def require_vm_access(session: Session, profile: dict, vm_id: str) -> str:
|
|
|
if not database_mode:
|
|
|
return vm_id
|
|
|
try:
|
|
|
canonical_id = str(uuid.UUID(vm_id))
|
|
|
except (TypeError, ValueError, AttributeError) as exc:
|
|
|
raise VMError("虚拟机 UUID 无效", 400) from exc
|
|
|
account = account_for_session(session)
|
|
|
if not account["admin"] and not state.users.owns_vm(
|
|
|
host_scope(profile), canonical_id, account["id"]):
|
|
|
raise VMError("虚拟机不存在", 404)
|
|
|
return canonical_id
|
|
|
|
|
|
def smtp_store() -> SMTPSettingsStore:
|
|
|
if data_store is None:
|
|
|
raise SMTPError("请先完成数据库配置", 409)
|
|
|
return SMTPSettingsStore(data_store)
|
|
|
|
|
|
def ssh_key_store():
|
|
|
if not database_mode or state is None:
|
|
|
raise SSHKeyError("请先完成数据库配置", 409)
|
|
|
return state.ssh_keys
|
|
|
|
|
|
def current_updater() -> Updater:
|
|
|
nonlocal updater
|
|
|
if data_store is None:
|
|
|
raise UpdateError("请先完成数据库配置", 409)
|
|
|
if updater is None:
|
|
|
updater = Updater(settings.static_dir.parent, data_store)
|
|
|
return updater
|
|
|
|
|
|
def forbid_long_job_during_update() -> None:
|
|
|
if data_store is None:
|
|
|
return
|
|
|
active_id = data_store.get_value("update.active_job")
|
|
|
if active_id:
|
|
|
job = current_updater().get_job(active_id)
|
|
|
if job is not None and job.get("status") in {"queued", "running"}:
|
|
|
raise HTTPException(status_code=409, detail="面板正在更新,暂时不能开始新任务")
|
|
|
|
|
|
def current_profile() -> dict:
|
|
|
profile = manager.current()
|
|
|
if profile is None:
|
|
|
raise HTTPException(status_code=409, detail="请先在面板中配置宿主机")
|
|
|
if profile.get("key_locked"):
|
|
|
raise RemoteHostError("宿主机 SSH 私钥已锁定,请先在面板解锁", 423)
|
|
|
return profile
|
|
|
|
|
|
def new_service(profile: dict) -> VMService:
|
|
|
if service_factory is not None:
|
|
|
return service_factory(profile)
|
|
|
return VMService(
|
|
|
profile["libvirt_uri"],
|
|
|
profile["storage_pool"],
|
|
|
profile["iso_pool"],
|
|
|
profile["default_network"],
|
|
|
rescue_iso_pool=profile.get("rescue_iso_pool", ""),
|
|
|
)
|
|
|
|
|
|
def verified_created_vm(profile: dict, result: dict, requested_uuid: str) -> dict:
|
|
|
if result.get("uuid") == requested_uuid:
|
|
|
return result
|
|
|
# Keep the preallocated owner/credential records if libvirt may have
|
|
|
# created the requested guest. Never delete a returned, mismatched UUID:
|
|
|
# it may refer to a different existing VM.
|
|
|
LOG.error("VM create returned a mismatched UUID: expected=%s", requested_uuid)
|
|
|
try:
|
|
|
verified = new_service(profile).get_vm(requested_uuid)
|
|
|
except VMError:
|
|
|
verified = None
|
|
|
if verified is not None and verified.get("uuid") == requested_uuid:
|
|
|
LOG.warning("Recovered VM result by querying the requested UUID: %s", requested_uuid)
|
|
|
return verified
|
|
|
raise VMError("虚拟机创建结果与预分配 UUID 不一致;请由管理员核查宿主机及归属记录", 502)
|
|
|
|
|
|
def current_service() -> VMService:
|
|
|
return new_service(current_profile())
|
|
|
|
|
|
def validate_staged_libvirt(
|
|
|
staged, body: HostBody, *, ensure_paths: bool = False,
|
|
|
) -> tuple[dict | None, str | None]:
|
|
|
service = new_service({
|
|
|
"libvirt_uri": staged.libvirt_uri,
|
|
|
"storage_pool": body.storage_pool,
|
|
|
"iso_pool": body.iso_pool,
|
|
|
"rescue_iso_pool": body.rescue_iso_pool,
|
|
|
"default_network": body.default_network,
|
|
|
})
|
|
|
# Validate only host-owned storage/network resources here. Host save
|
|
|
# and connection testing must never enumerate every guest domain;
|
|
|
# capacity and IP scheduling are panel-owned and VM operations use
|
|
|
# targeted UUID reads when they need a particular guest.
|
|
|
if ensure_paths and (body.rescue_iso_pool or any(path.startswith("/") for path in (
|
|
|
body.storage_pool, body.iso_pool))):
|
|
|
service.ensure_storage_pools()
|
|
|
try:
|
|
|
return service.resources(), None
|
|
|
except VMError as exc:
|
|
|
return None, str(exc)
|
|
|
|
|
|
def session_response(username: str) -> JSONResponse:
|
|
|
if database_mode:
|
|
|
account = state.users.get_by_username(username)
|
|
|
if account is None:
|
|
|
raise HTTPException(status_code=401, detail="账号不可用,请重新登录")
|
|
|
session = sessions.issue(username, user_id=account["id"])
|
|
|
else:
|
|
|
session = sessions.issue(username)
|
|
|
account = account_for_session(session)
|
|
|
response = JSONResponse({
|
|
|
"authenticated": True,
|
|
|
"username": session.username,
|
|
|
"csrf_token": session.csrf_token,
|
|
|
"admin": account["admin"],
|
|
|
"approved": account["approved"],
|
|
|
"host_ready": host_ready(),
|
|
|
})
|
|
|
response.set_cookie(
|
|
|
COOKIE_NAME,
|
|
|
session.token,
|
|
|
max_age=8 * 60 * 60,
|
|
|
httponly=True,
|
|
|
secure=settings.secure_cookie,
|
|
|
samesite="strict",
|
|
|
path="/",
|
|
|
)
|
|
|
return response
|
|
|
|
|
|
@app.get("/", include_in_schema=False)
|
|
|
def index():
|
|
|
return FileResponse(settings.static_dir / "index.html")
|
|
|
|
|
|
@app.get("/api/v1/health")
|
|
|
def health():
|
|
|
if data_store is not None:
|
|
|
# A running HTTP listener alone is not a usable post-update panel.
|
|
|
# The updater treats this endpoint as readiness for the configured DB.
|
|
|
if database_mode:
|
|
|
state.users.has_users()
|
|
|
else:
|
|
|
data_store.get("admin", "singleton")
|
|
|
return {"status": "ok"}
|
|
|
|
|
|
@app.get("/api/v1/database/status")
|
|
|
def database_status():
|
|
|
if bootstrap is None:
|
|
|
return {"configured": True, "setup_required": False}
|
|
|
return {"configured": bootstrap.configured(),
|
|
|
"setup_required": not bootstrap.configured()}
|
|
|
|
|
|
@app.post("/api/v1/database/setup")
|
|
|
def setup_database(body: DatabaseSetupBody, request: Request):
|
|
|
nonlocal state, manager, templates, data_store, updater, sessions, rescue_catalog
|
|
|
nonlocal user_iso_catalog
|
|
|
if bootstrap is None:
|
|
|
raise DatabaseSetupError("数据库初始化仅供正常部署使用", 409)
|
|
|
peer = _peer(request)
|
|
|
if not setup_limiter.allowed(peer):
|
|
|
raise HTTPException(status_code=429, detail="初始化尝试过多,请稍后再试")
|
|
|
|
|
|
def connect(config: dict, master_key: bytes):
|
|
|
try:
|
|
|
return make_components(
|
|
|
settings.data_dir, config, master_key, settings.ssh_binary)
|
|
|
except Exception as exc:
|
|
|
# Connector errors may include database host details. Never
|
|
|
# reflect the exception, credentials or driver SQL in HTTP.
|
|
|
LOG.warning("database setup connection failed: %s", type(exc).__name__)
|
|
|
raise DatabaseSetupError(
|
|
|
"无法连接或初始化 MariaDB;请检查地址、账号、数据库和建表权限", 502,
|
|
|
) from exc
|
|
|
|
|
|
try:
|
|
|
components = bootstrap.configure({
|
|
|
"host": body.host,
|
|
|
"port": body.port,
|
|
|
"database": body.database,
|
|
|
"username": body.username,
|
|
|
"password": body.password.get_secret_value(),
|
|
|
}, body.setup_token, connect)
|
|
|
except DatabaseSetupError:
|
|
|
setup_limiter.failed(peer)
|
|
|
raise
|
|
|
state, manager, templates, data_store = components
|
|
|
rescue_catalog = RescueTemplateStore(data_store)
|
|
|
user_iso_catalog = UserISOStore(data_store)
|
|
|
sessions = DatabaseSessionStore(data_store)
|
|
|
restore_jobs()
|
|
|
setup_limiter.succeeded(peer)
|
|
|
LOG.info("panel database configured")
|
|
|
return {"configured": True}
|
|
|
|
|
|
@app.get("/api/v1/setup/status")
|
|
|
def setup_status():
|
|
|
if state is None:
|
|
|
raise HTTPException(status_code=409, detail="请先配置数据库")
|
|
|
return {"setup_required": state.setup_required()}
|
|
|
|
|
|
@app.post("/api/v1/setup/admin")
|
|
|
def setup_admin(body: SetupBody, request: Request):
|
|
|
if state is None:
|
|
|
raise HTTPException(status_code=409, detail="请先配置数据库")
|
|
|
peer = _peer(request)
|
|
|
if not setup_limiter.allowed(peer):
|
|
|
raise HTTPException(status_code=429, detail="初始化尝试过多,请稍后再试")
|
|
|
try:
|
|
|
username = state.create_admin(body.token, body.username, body.password)
|
|
|
except SetupError:
|
|
|
setup_limiter.failed(peer)
|
|
|
raise
|
|
|
setup_limiter.succeeded(peer)
|
|
|
LOG.info("panel initial user created: username=%s", username)
|
|
|
return session_response(username)
|
|
|
|
|
|
@app.get("/api/v1/session")
|
|
|
def session_info(request: Request):
|
|
|
session = sessions.get(request.cookies.get(COOKIE_NAME))
|
|
|
if session is None:
|
|
|
return {"authenticated": False, "username": None, "csrf_token": None}
|
|
|
try:
|
|
|
account = account_for_session(session)
|
|
|
except HTTPException:
|
|
|
return {"authenticated": False, "username": None, "csrf_token": None}
|
|
|
return {
|
|
|
"authenticated": True,
|
|
|
"username": session.username,
|
|
|
"csrf_token": session.csrf_token,
|
|
|
"admin": account["admin"],
|
|
|
"approved": account["approved"],
|
|
|
"host_ready": host_ready(),
|
|
|
}
|
|
|
|
|
|
@app.post("/api/v1/login")
|
|
|
def login(body: LoginBody, request: Request):
|
|
|
if state is None:
|
|
|
raise HTTPException(status_code=409, detail="请先配置数据库")
|
|
|
peer = _peer(request)
|
|
|
if not login_limiter.allowed(peer):
|
|
|
raise HTTPException(status_code=429, detail="登录尝试过多,请稍后再试")
|
|
|
if database_mode:
|
|
|
if body.email is None or body.username is not None:
|
|
|
raise HTTPException(status_code=422, detail="请使用邮箱登录")
|
|
|
try:
|
|
|
email = normalize_email(body.email)
|
|
|
except UserError:
|
|
|
login_limiter.failed(peer)
|
|
|
raise HTTPException(status_code=401, detail="邮箱或密码错误") from None
|
|
|
if not state.users.has_users():
|
|
|
raise HTTPException(status_code=409, detail="请先完成初始化")
|
|
|
account = state.users.get_by_email(email)
|
|
|
encoded = account["password_hash"] if account else None
|
|
|
username = account["username"] if account else ""
|
|
|
identity_ok = account is not None
|
|
|
else:
|
|
|
legacy_admin = state.admin()
|
|
|
if legacy_admin is None:
|
|
|
raise HTTPException(status_code=409, detail="请先完成初始化")
|
|
|
username, encoded = legacy_admin
|
|
|
account = {"email_verified": True}
|
|
|
identity_ok = hmac.compare_digest(body.username or "", username)
|
|
|
password_ok = verify_password(body.password, encoded) if encoded else False
|
|
|
if not password_ok or not identity_ok or not account["email_verified"]:
|
|
|
login_limiter.failed(peer)
|
|
|
raise HTTPException(status_code=401, detail="邮箱或密码错误")
|
|
|
login_limiter.succeeded(peer)
|
|
|
return session_response(username)
|
|
|
|
|
|
def deliver_registration_code(config, email: str, code: str, users) -> None:
|
|
|
try:
|
|
|
send_verification_code(config, email, code)
|
|
|
except Exception as exc:
|
|
|
# Background delivery must not expose whether an email exists or
|
|
|
# relay SMTP server diagnostics to an unauthenticated client.
|
|
|
LOG.warning("Registration email delivery failed: %s", type(exc).__name__)
|
|
|
try:
|
|
|
users.revoke_registration_code(email, code)
|
|
|
except DatabaseError:
|
|
|
LOG.exception("Could not revoke undelivered registration code")
|
|
|
|
|
|
@app.post("/api/v1/register/code")
|
|
|
def send_registration_code(body: RegistrationCodeBody, request: Request,
|
|
|
background_tasks: BackgroundTasks):
|
|
|
if not database_mode or state is None or not state.users.has_users():
|
|
|
raise HTTPException(status_code=409, detail="请先完成面板初始化")
|
|
|
peer = _peer(request)
|
|
|
if not code_send_limiter.allowed(peer):
|
|
|
raise HTTPException(status_code=429, detail="验证码请求过多,请稍后再试")
|
|
|
email = normalize_email(body.email)
|
|
|
config = smtp_store().get()
|
|
|
if config is None:
|
|
|
raise SMTPError("管理员尚未配置 SMTP,暂不能注册", 503)
|
|
|
code = state.users.generate_code()
|
|
|
try:
|
|
|
should_send = state.users.issue_registration_code(email, code)
|
|
|
except UserError as exc:
|
|
|
if exc.status != 429:
|
|
|
raise
|
|
|
should_send = False
|
|
|
code_send_limiter.failed(peer)
|
|
|
if should_send:
|
|
|
# Sending after the HTTP response avoids exposing whether an
|
|
|
# address is registered via SMTP latency or transport failures.
|
|
|
background_tasks.add_task(
|
|
|
deliver_registration_code, config, email, code, state.users,
|
|
|
)
|
|
|
# Identical public response for existing addresses and cooldowns.
|
|
|
return {"sent": True}
|
|
|
|
|
|
@app.post("/api/v1/register", status_code=201)
|
|
|
def register(body: RegistrationBody, request: Request):
|
|
|
if not database_mode or state is None or not state.users.has_users():
|
|
|
raise HTTPException(status_code=409, detail="请先完成面板初始化")
|
|
|
peer = _peer(request)
|
|
|
if not registration_limiter.allowed(peer):
|
|
|
raise HTTPException(status_code=429, detail="注册尝试过多,请稍后再试")
|
|
|
try:
|
|
|
state.users.register_with_code(body.username, body.email,
|
|
|
body.password, body.code)
|
|
|
except UserError:
|
|
|
registration_limiter.failed(peer)
|
|
|
raise
|
|
|
registration_limiter.succeeded(peer)
|
|
|
return {"registered": True, "approved": False}
|
|
|
|
|
|
@app.post("/api/v1/logout")
|
|
|
def logout(request: Request, _session: Session = Depends(require_write)):
|
|
|
sessions.revoke(request.cookies.get(COOKIE_NAME))
|
|
|
response = JSONResponse({"ok": True})
|
|
|
response.delete_cookie(COOKIE_NAME, path="/")
|
|
|
return response
|
|
|
|
|
|
@app.get("/api/v1/ssh-keys")
|
|
|
def list_ssh_keys(_session: Session = Depends(require_session)):
|
|
|
account = account_for_session(_session)
|
|
|
return {"items": ssh_key_store().list_for_user(account["id"])}
|
|
|
|
|
|
@app.post("/api/v1/ssh-keys", status_code=201)
|
|
|
def create_ssh_key(body: SSHKeyBody,
|
|
|
_session: Session = Depends(require_write)):
|
|
|
account = account_for_session(_session)
|
|
|
return ssh_key_store().create_for_user(
|
|
|
account["id"], body.name, body.public_key,
|
|
|
)
|
|
|
|
|
|
@app.delete("/api/v1/ssh-keys/{key_id}")
|
|
|
def delete_ssh_key(key_id: int, _session: Session = Depends(require_write)):
|
|
|
if key_id < 1:
|
|
|
raise SSHKeyError("公钥不存在", 404)
|
|
|
account = account_for_session(_session)
|
|
|
if not ssh_key_store().delete_for_user(account["id"], key_id):
|
|
|
raise SSHKeyError("公钥不存在", 404)
|
|
|
return {"deleted": True}
|
|
|
|
|
|
@app.get("/api/v1/smtp/settings")
|
|
|
def get_smtp_settings(_session: Session = Depends(require_admin)):
|
|
|
return smtp_store().public_status()
|
|
|
|
|
|
@app.put("/api/v1/smtp/settings")
|
|
|
def save_smtp_settings(body: SMTPSettingsBody,
|
|
|
_session: Session = Depends(require_admin_write)):
|
|
|
settings_data = body.model_dump(exclude={"password"})
|
|
|
settings_data["password"] = (body.password.get_secret_value()
|
|
|
if body.password is not None else "")
|
|
|
config = smtp_store().save(settings_data)
|
|
|
LOG.info("SMTP settings changed: admin=%s", _session.username)
|
|
|
return config.public_status()
|
|
|
|
|
|
@app.get("/api/v1/users/pending")
|
|
|
def pending_users(_session: Session = Depends(require_admin)):
|
|
|
if not database_mode:
|
|
|
return {"items": []}
|
|
|
return {"items": state.users.list_pending_users()}
|
|
|
|
|
|
@app.post("/api/v1/users/{user_id}/approve")
|
|
|
def approve_user(user_id: int, _session: Session = Depends(require_admin_write)):
|
|
|
if not database_mode:
|
|
|
raise HTTPException(status_code=409, detail="数据库用户功能不可用")
|
|
|
account = state.users.approve_user(user_id)
|
|
|
if account is None:
|
|
|
raise HTTPException(status_code=404, detail="用户不存在")
|
|
|
LOG.info("user approved: id=%s admin=%s", user_id, _session.username)
|
|
|
return {"approved": True}
|
|
|
|
|
|
def public_update_settings() -> dict:
|
|
|
info = current_updater().settings()
|
|
|
return {
|
|
|
"repository_url": info["repository"],
|
|
|
"branch": info["branch"],
|
|
|
"token_configured": info["token_configured"],
|
|
|
"checkout_ready": info["checkout_ready"],
|
|
|
"configured": info["token_configured"] and info["checkout_ready"],
|
|
|
"current_version": info["current_commit"],
|
|
|
"active_job_id": info.get("active_job_id"),
|
|
|
"last_job_id": info.get("last_job_id"),
|
|
|
"checkout_error": info.get("checkout_error"),
|
|
|
}
|
|
|
|
|
|
@app.get("/api/v1/update/settings")
|
|
|
def update_settings(_session: Session = Depends(require_admin)):
|
|
|
return public_update_settings()
|
|
|
|
|
|
@app.put("/api/v1/update/settings")
|
|
|
def save_update_settings(body: UpdateSettingsBody,
|
|
|
_session: Session = Depends(require_admin_write)):
|
|
|
if body.repository_url != REPOSITORY_URL or body.branch != BRANCH:
|
|
|
raise UpdateError("更新仓库和分支固定为面板指定的 Gitea main 分支", 422)
|
|
|
service = current_updater()
|
|
|
if body.access_token is not None:
|
|
|
token = body.access_token.get_secret_value()
|
|
|
if token:
|
|
|
service.set_token(token)
|
|
|
if not service.settings()["token_configured"]:
|
|
|
raise UpdateError("请填写 Gitea 私有仓库访问令牌", 422)
|
|
|
LOG.info("panel update access token configured: admin=%s", _session.username)
|
|
|
return public_update_settings()
|
|
|
|
|
|
@app.post("/api/v1/update/check")
|
|
|
def check_update(_session: Session = Depends(require_admin_write)):
|
|
|
result = current_updater().check()
|
|
|
return {
|
|
|
"repository_url": result["repository"],
|
|
|
"branch": result["branch"],
|
|
|
"current_version": result["current_commit"],
|
|
|
"latest_version": result["remote_commit"],
|
|
|
"update_available": result["status"] == "update_available",
|
|
|
}
|
|
|
|
|
|
@app.post("/api/v1/update/apply", status_code=202)
|
|
|
def apply_update(body: UpdateApplyBody, _session: Session = Depends(require_admin_write)):
|
|
|
with (import_lock, reinstall_lock, rescue_lock, snapshot_lock,
|
|
|
rescue_import_lock):
|
|
|
if (pending_templates or pending_reinstalls or pending_rescues
|
|
|
or pending_snapshot_jobs or pending_rescue_imports):
|
|
|
raise UpdateError("镜像导入或虚拟机任务正在执行,请等待完成后再更新面板", 409)
|
|
|
try:
|
|
|
unit = subprocess.run(
|
|
|
["/usr/bin/systemctl", "is-active", "virsh-panel-update.service"],
|
|
|
capture_output=True, text=True, timeout=5, check=False,
|
|
|
)
|
|
|
except (OSError, subprocess.TimeoutExpired):
|
|
|
unit = None
|
|
|
if unit is not None and unit.stdout.strip() in {
|
|
|
"activating", "active", "deactivating", "reloading",
|
|
|
}:
|
|
|
raise UpdateError("独立更新服务仍在运行,请等待其完全结束后再提交", 409)
|
|
|
service = current_updater()
|
|
|
job = service.create_job(
|
|
|
expected_current_commit=body.expected_current_commit,
|
|
|
expected_remote_commit=body.expected_remote_commit,
|
|
|
)
|
|
|
try:
|
|
|
# The web service has a read-only code tree. A separate one-shot
|
|
|
# unit performs the Git update and restarts this service.
|
|
|
trigger = subprocess.run(
|
|
|
["/usr/bin/systemctl", "--no-block", "start", "virsh-panel-update.service"],
|
|
|
capture_output=True, text=True, timeout=10, check=False,
|
|
|
)
|
|
|
except (OSError, subprocess.TimeoutExpired):
|
|
|
trigger = None
|
|
|
if trigger is None or trigger.returncode != 0:
|
|
|
service.fail_job(job["job_id"],
|
|
|
"无法启动独立更新服务;请检查 virsh-panel-update.service 是否已安装")
|
|
|
raise UpdateError("无法启动独立更新服务;请检查 systemd 配置", 503)
|
|
|
LOG.info("panel update queued: job=%s admin=%s", job["job_id"], _session.username)
|
|
|
return {"job_id": job["job_id"], "status": "queued"}
|
|
|
|
|
|
@app.get("/api/v1/update/jobs/{job_id}")
|
|
|
def update_job(job_id: str, _session: Session = Depends(require_admin)):
|
|
|
job = current_updater().get_job(job_id)
|
|
|
if job is None:
|
|
|
raise UpdateError("更新任务不存在", 404)
|
|
|
return job
|
|
|
|
|
|
@app.get("/api/v1/host")
|
|
|
def host(_session: Session = Depends(require_admin)):
|
|
|
return _public_host(manager.current())
|
|
|
|
|
|
@app.get("/api/v1/host/network-pools")
|
|
|
def host_network_pools(_session: Session = Depends(require_admin)):
|
|
|
profile = current_profile()
|
|
|
pools, compatibility_mode = public_network_pools(profile)
|
|
|
return {
|
|
|
"items": pools,
|
|
|
"count": len(pools),
|
|
|
"maximum": MAX_PUBLIC_NETWORK_POOLS,
|
|
|
"compatibility_mode": compatibility_mode,
|
|
|
}
|
|
|
|
|
|
@app.put("/api/v1/host/network-pools/{pool_key}")
|
|
|
def save_host_network_pool(
|
|
|
pool_key: str,
|
|
|
body: PublicNetworkPoolBody,
|
|
|
_session: Session = Depends(require_admin_write),
|
|
|
):
|
|
|
# Host saves and pool mutations share this lock so a concurrent bridge
|
|
|
# change cannot commit between catalog validation and installation.
|
|
|
# Production runs one engine worker; the catalog CAS still protects
|
|
|
# independent pool writers and test/in-process concurrency.
|
|
|
with public_pool_catalog_lock:
|
|
|
profile = current_profile()
|
|
|
pool, idempotent = upsert_public_network_pool(profile, pool_key, body)
|
|
|
LOG.info(
|
|
|
"host public network pool saved: host=%s pool=%s revision=%s admin=%s",
|
|
|
profile.get("host"), pool_key, pool["revision"], _session.username,
|
|
|
)
|
|
|
return {
|
|
|
"pool": pool,
|
|
|
"applied": not idempotent,
|
|
|
"idempotent": idempotent,
|
|
|
"compatibility_mode": False,
|
|
|
}
|
|
|
|
|
|
@app.post("/api/v1/host/scan")
|
|
|
def scan_host(body: HostScanBody, _session: Session = Depends(require_admin_write)):
|
|
|
return manager.scan(body.host, body.port)
|
|
|
|
|
|
@app.post("/api/v1/host/unlock")
|
|
|
def unlock_host(body: UnlockBody, request: Request, _session: Session = Depends(require_admin_write)):
|
|
|
peer = _peer(request)
|
|
|
if not unlock_limiter.allowed(peer):
|
|
|
raise HTTPException(status_code=429, detail="私钥口令尝试过多,请稍后再试")
|
|
|
try:
|
|
|
profile = manager.unlock_current(body.passphrase)
|
|
|
except RemoteHostError as exc:
|
|
|
if exc.status == 400:
|
|
|
unlock_limiter.failed(peer)
|
|
|
raise
|
|
|
unlock_limiter.succeeded(peer)
|
|
|
LOG.info("remote host SSH key unlocked: host=%s admin=%s", profile.get("host"), _session.username)
|
|
|
return _public_host(profile)
|
|
|
|
|
|
@app.post("/api/v1/host/test")
|
|
|
def test_host(body: HostBody, _session: Session = Depends(require_admin_write)):
|
|
|
staged = manager.stage(body.profile(), confirmed_fingerprint=body.confirmed_fingerprint)
|
|
|
try:
|
|
|
result = manager.test(staged)
|
|
|
resources, resource_warning = validate_staged_libvirt(staged, body)
|
|
|
return {
|
|
|
**result,
|
|
|
"resources": resources,
|
|
|
"warnings": [resource_warning] if resource_warning else [],
|
|
|
}
|
|
|
finally:
|
|
|
manager.abort(staged)
|
|
|
|
|
|
@app.put("/api/v1/host")
|
|
|
def save_host(body: HostBody, _session: Session = Depends(require_admin_write)):
|
|
|
# A running reinstall uses the saved SSH host identity and paths. Do
|
|
|
# not replace them while its libvirt operation is in flight.
|
|
|
with (reinstall_lock, rescue_lock, snapshot_lock, rescue_import_lock,
|
|
|
public_pool_catalog_lock):
|
|
|
forbid_long_job_during_update()
|
|
|
if (pending_reinstalls or pending_rescues or pending_snapshot_jobs
|
|
|
or pending_rescue_imports
|
|
|
or any(job.get("status") in {"queued", "running", "uncertain"}
|
|
|
for job in reinstall_jobs.values())
|
|
|
or (data_store is not None and data_store.list("vm_rescue"))):
|
|
|
raise VMError("请先完成救援或重装任务并退出救援模式,再修改宿主机配置", 409)
|
|
|
if any(
|
|
|
job.get("status") in {"succeeded", "uncertain"}
|
|
|
and "generated_password" in job
|
|
|
for job in reinstall_jobs.values()):
|
|
|
raise VMError("请先完成重装任务并确认保存自动生成的密码,再修改宿主机配置", 409)
|
|
|
current = manager.current()
|
|
|
if current is not None:
|
|
|
remote_identity_changed = (
|
|
|
current.get("host") != body.host.lower()
|
|
|
or current.get("port") != body.port
|
|
|
or current.get("fingerprint") != body.confirmed_fingerprint
|
|
|
or current.get("storage_pool") != body.storage_pool
|
|
|
)
|
|
|
if remote_identity_changed and has_retained_snapshots():
|
|
|
raise VMError(
|
|
|
"宿主机仍保留本地快照;更换远端宿主身份或存储池前请先删除全部快照",
|
|
|
409,
|
|
|
)
|
|
|
_raw_catalog, native_pools = read_native_public_network_pools(current)
|
|
|
if any(
|
|
|
pool["network_selector"] != body.default_network
|
|
|
for pool in native_pools):
|
|
|
raise VMError(
|
|
|
"宿主机已有公网地址池;请先在公网地址池管理中处理网络接入,再修改默认网络",
|
|
|
409,
|
|
|
)
|
|
|
staged = manager.stage(body.profile(), confirmed_fingerprint=body.confirmed_fingerprint)
|
|
|
committed = False
|
|
|
try:
|
|
|
manager.test(staged)
|
|
|
_resources, resource_warning = validate_staged_libvirt(staged, body, ensure_paths=True)
|
|
|
profile = manager.commit(staged)
|
|
|
committed = True
|
|
|
finally:
|
|
|
if not committed:
|
|
|
manager.abort(staged)
|
|
|
LOG.info("remote host configured: host=%s port=%s admin=%s", body.host, body.port, _session.username)
|
|
|
return {
|
|
|
**_public_host(profile),
|
|
|
"warnings": [resource_warning] if resource_warning else [],
|
|
|
}
|
|
|
|
|
|
@app.get("/api/v1/resources")
|
|
|
def resources(_session: Session = Depends(require_approved)):
|
|
|
profile = current_profile()
|
|
|
values = new_service(profile).resources()
|
|
|
# Root-pool ISO filenames are retained only for legacy/admin clients.
|
|
|
# Ordinary users select owner-scoped opaque IDs from /user-isos.
|
|
|
if database_mode and not account_for_session(_session)["admin"]:
|
|
|
values["isos"] = []
|
|
|
pools, compatibility_mode = public_network_pools(profile)
|
|
|
primary = next(
|
|
|
(pool for pool in pools if pool["state"] == "active"), None)
|
|
|
return {
|
|
|
**values,
|
|
|
# Keep the original scalar fields during the PHP rollout. They
|
|
|
# describe only the deterministic primary active pool; new code
|
|
|
# uses host/network-pools for the configured catalog.
|
|
|
"public_pool_cidr": "" if primary is None else primary["cidr"],
|
|
|
"public_pool_enabled": primary is not None,
|
|
|
"public_vm_prefix": (
|
|
|
None if primary is None else primary["guest_prefix_length"]),
|
|
|
"public_network_pool_count": len(pools),
|
|
|
"public_network_pool_compatibility_mode": compatibility_mode,
|
|
|
}
|
|
|
|
|
|
def cloud_template_record(profile: dict, template_id: str) -> dict:
|
|
|
if shared_cloud_template_runtime is not None:
|
|
|
try:
|
|
|
return shared_cloud_template_runtime.get(template_id)
|
|
|
except TemplateError as exc:
|
|
|
if exc.status != 404:
|
|
|
raise
|
|
|
# Host-scoped template IDs were used before the catalog became
|
|
|
# global. Keep already-paid/queued operations executable on
|
|
|
# their original host while administrators re-import those
|
|
|
# images into the new global catalog. They cannot be promoted
|
|
|
# automatically because the old flow intentionally discarded
|
|
|
# the verified local source after uploading it to libvirt.
|
|
|
return templates.get(template_scope(profile), template_id)
|
|
|
return templates.get(template_scope(profile), template_id)
|
|
|
|
|
|
def cloud_template_replica_scope(profile: dict) -> str:
|
|
|
"""Bind a ready marker to the actual remote host configuration.
|
|
|
|
|
|
``template_scope`` deliberately keeps a stable logical host identity.
|
|
|
That is useful for legacy catalogs, but is too weak as proof that a
|
|
|
cached volume still belongs to the same physical hypervisor. A host
|
|
|
rebind or SSH host-key change must invalidate every ready marker.
|
|
|
"""
|
|
|
identity = [
|
|
|
"virt-cloud-template-replica-v2",
|
|
|
profile.get("host_id"),
|
|
|
profile.get("host"),
|
|
|
profile.get("port"),
|
|
|
profile.get("fingerprint"),
|
|
|
profile.get("storage_pool"),
|
|
|
]
|
|
|
return hashlib.sha256(json.dumps(
|
|
|
identity, ensure_ascii=False, separators=(",", ":"),
|
|
|
).encode("utf-8")).hexdigest()
|
|
|
|
|
|
def cloud_template_replica_volume_name(record: dict) -> str:
|
|
|
source_name = record["volume_name"]
|
|
|
if not isinstance(source_name, str) or not source_name.endswith(".qcow2"):
|
|
|
raise DatabaseError("云镜像源卷名无效")
|
|
|
return f"{source_name[:-6]}-{secrets.token_hex(8)}.qcow2"
|
|
|
|
|
|
def valid_cloud_template_replica_volume_name(
|
|
|
record: dict, volume_name: object) -> bool:
|
|
|
source_name = record.get("volume_name")
|
|
|
if (not isinstance(source_name, str)
|
|
|
or not source_name.endswith(".qcow2")
|
|
|
or not isinstance(volume_name, str)):
|
|
|
return False
|
|
|
prefix = re.escape(source_name[:-6])
|
|
|
return re.fullmatch(
|
|
|
rf"{prefix}-[0-9a-f]{{16}}\.qcow2", volume_name,
|
|
|
) is not None
|
|
|
|
|
|
def cloud_template_replica_payload(
|
|
|
profile: dict, record: dict,
|
|
|
status: Literal["syncing", "ready"],
|
|
|
replica_volume_name: str,
|
|
|
replica_identity: str | None = None) -> dict:
|
|
|
if not valid_cloud_template_replica_volume_name(
|
|
|
record, replica_volume_name):
|
|
|
raise DatabaseError("云镜像宿主机缓存卷名无效")
|
|
|
result = {
|
|
|
"status": status,
|
|
|
"replica_scope": cloud_template_replica_scope(profile),
|
|
|
"source_volume_name": record["volume_name"],
|
|
|
"replica_volume_name": replica_volume_name,
|
|
|
"sha256": record["sha256"],
|
|
|
"virtual_size": record["virtual_size"],
|
|
|
"image_bytes": record["image_bytes"],
|
|
|
}
|
|
|
if status == "ready":
|
|
|
if (not isinstance(replica_identity, str)
|
|
|
or re.fullmatch(r"[0-9a-f]{64}", replica_identity) is None):
|
|
|
raise DatabaseError("云镜像宿主机缓存身份无效")
|
|
|
result["replica_identity"] = replica_identity
|
|
|
return result
|
|
|
|
|
|
def cloud_template_replica_marker(
|
|
|
profile: dict, record: dict) -> dict | None:
|
|
|
if data_store is None:
|
|
|
return None
|
|
|
raw = data_store.get("cloud_template_replica", record["id"])
|
|
|
if raw is None:
|
|
|
return None
|
|
|
try:
|
|
|
value = json.loads(raw)
|
|
|
except (TypeError, ValueError) as exc:
|
|
|
raise DatabaseError("云镜像宿主机缓存记录损坏") from exc
|
|
|
if not isinstance(value, dict):
|
|
|
raise DatabaseError("云镜像宿主机缓存记录损坏")
|
|
|
status = value.get("status")
|
|
|
replica_volume_name = value.get("replica_volume_name")
|
|
|
if (status not in {"syncing", "ready"}
|
|
|
or not valid_cloud_template_replica_volume_name(
|
|
|
record, replica_volume_name)):
|
|
|
# Older/incomplete marker schemas are evidence only that a cache
|
|
|
# may exist, never proof that it is complete or belongs to this
|
|
|
# physical host generation.
|
|
|
return None
|
|
|
replica_identity = value.get("replica_identity")
|
|
|
if status == "ready" and (
|
|
|
not isinstance(replica_identity, str)
|
|
|
or re.fullmatch(r"[0-9a-f]{64}", replica_identity) is None):
|
|
|
return None
|
|
|
expected = cloud_template_replica_payload(
|
|
|
profile, record, status, replica_volume_name,
|
|
|
replica_identity if status == "ready" else None,
|
|
|
)
|
|
|
return value if value == expected else None
|
|
|
|
|
|
def save_cloud_template_replica(
|
|
|
profile: dict, record: dict,
|
|
|
status: Literal["syncing", "ready"],
|
|
|
replica_volume_name: str,
|
|
|
replica_identity: str | None = None) -> None:
|
|
|
if data_store is None:
|
|
|
return
|
|
|
data_store.set(
|
|
|
"cloud_template_replica", record["id"],
|
|
|
json.dumps(
|
|
|
cloud_template_replica_payload(
|
|
|
profile, record, status, replica_volume_name,
|
|
|
replica_identity),
|
|
|
separators=(",", ":"),
|
|
|
),
|
|
|
)
|
|
|
|
|
|
def ensure_cloud_template_replica(
|
|
|
profile: dict, service: VMService, template_id: str) -> dict:
|
|
|
record = cloud_template_record(profile, template_id)
|
|
|
if shared_cloud_template_runtime is None:
|
|
|
return record
|
|
|
with cloud_template_cache_lock:
|
|
|
# Re-read after entering the per-host critical section. Another
|
|
|
# create may have completed this same first-use transfer while the
|
|
|
# request was waiting for the lock.
|
|
|
try:
|
|
|
record = shared_cloud_template_runtime.get(template_id)
|
|
|
except TemplateError as exc:
|
|
|
if exc.status != 404:
|
|
|
raise
|
|
|
return templates.get(template_scope(profile), template_id)
|
|
|
marker = cloud_template_replica_marker(profile, record)
|
|
|
trusted_volume_name = None
|
|
|
trusted_identity = None
|
|
|
if marker is not None and marker["status"] == "ready":
|
|
|
trusted_volume_name = marker["replica_volume_name"]
|
|
|
trusted_identity = marker["replica_identity"]
|
|
|
# Persist a different, never-before-used fallback generation
|
|
|
# before asking libvirt whether the ready volume is still the
|
|
|
# exact volume proven by the marker. If it is not, the old
|
|
|
# name is never overwritten or silently reused.
|
|
|
desired_volume_name = cloud_template_replica_volume_name(record)
|
|
|
elif marker is not None:
|
|
|
# A crash after writing ``syncing`` may have left a partial
|
|
|
# upload. Retry that planned generation as untrusted; the
|
|
|
# service safely replaces it only when no VM references it.
|
|
|
desired_volume_name = marker["replica_volume_name"]
|
|
|
else:
|
|
|
desired_volume_name = cloud_template_replica_volume_name(record)
|
|
|
# Keep a proven ready marker until the global source has itself
|
|
|
# been verified. A damaged source must not downgrade evidence of
|
|
|
# an already complete host replica.
|
|
|
with shared_cloud_template_runtime.source(template_id) as (
|
|
|
record, source_path):
|
|
|
# Once source verification succeeds, always downgrade the
|
|
|
# durable marker before touching libvirt. Even a previously
|
|
|
# ready replica may have disappeared or need replacement; a
|
|
|
# power loss during that upload must never leave ready proof
|
|
|
# pointing at a partially streamed qcow2 volume.
|
|
|
save_cloud_template_replica(
|
|
|
profile, record, "syncing", desired_volume_name)
|
|
|
cached = service.cache_template(
|
|
|
source_path, desired_volume_name,
|
|
|
record["virtual_size"],
|
|
|
trusted_replica_volume_name=trusted_volume_name,
|
|
|
trusted_replica_identity=trusted_identity,
|
|
|
)
|
|
|
cached_volume_name = cached.get("volume_name")
|
|
|
accepted_names = {desired_volume_name}
|
|
|
if trusted_volume_name is not None:
|
|
|
accepted_names.add(trusted_volume_name)
|
|
|
if cached_volume_name not in accepted_names:
|
|
|
raise DatabaseError("云镜像宿主机缓存返回了无效卷")
|
|
|
save_cloud_template_replica(
|
|
|
profile, record, "ready", cached_volume_name,
|
|
|
cached.get("replica_identity"))
|
|
|
return {**record, "volume_name": cached_volume_name}
|
|
|
|
|
|
def user_iso_context(session: Session) -> tuple[int, str]:
|
|
|
owner_id = account_for_session(session)["id"]
|
|
|
return owner_id, user_iso_scope(owner_id)
|
|
|
|
|
|
def user_iso_replica_key(owner_id: int, iso_id: str) -> str:
|
|
|
return f"{owner_id}:{iso_id}"
|
|
|
|
|
|
def trusted_user_iso_replica(owner_id: int, record: dict) -> bool:
|
|
|
if data_store is None:
|
|
|
return False
|
|
|
raw = data_store.get(
|
|
|
"user_iso_replica", user_iso_replica_key(owner_id, record["id"]))
|
|
|
if raw is None:
|
|
|
return False
|
|
|
try:
|
|
|
replica = json.loads(raw)
|
|
|
except (TypeError, ValueError) as exc:
|
|
|
raise DatabaseError("用户 ISO 宿主机缓存记录损坏") from exc
|
|
|
if not isinstance(replica, dict):
|
|
|
raise DatabaseError("用户 ISO 宿主机缓存记录损坏")
|
|
|
return replica == {
|
|
|
"status": "ready",
|
|
|
"volume_name": record["volume_name"],
|
|
|
"size_bytes": record["size_bytes"],
|
|
|
"sha256": record["sha256"],
|
|
|
}
|
|
|
|
|
|
def save_user_iso_replica(owner_id: int, record: dict, *,
|
|
|
status: Literal["syncing", "ready"]) -> None:
|
|
|
if data_store is None:
|
|
|
return
|
|
|
data_store.set(
|
|
|
"user_iso_replica", user_iso_replica_key(owner_id, record["id"]),
|
|
|
json.dumps({
|
|
|
"status": status,
|
|
|
"volume_name": record["volume_name"],
|
|
|
"size_bytes": record["size_bytes"],
|
|
|
"sha256": record["sha256"],
|
|
|
}, separators=(",", ":")),
|
|
|
)
|
|
|
|
|
|
def prune_user_iso_replica_records(owner_id: int,
|
|
|
active_ids: set[str]) -> None:
|
|
|
if data_store is None:
|
|
|
return
|
|
|
prefix = f"{owner_id}:"
|
|
|
for key in data_store.list("user_iso_replica"):
|
|
|
if key.startswith(prefix) and key[len(prefix):] not in active_ids:
|
|
|
data_store.delete("user_iso_replica", key)
|
|
|
|
|
|
def public_user_iso(record: dict) -> dict:
|
|
|
return {key: value for key, value in record.items()
|
|
|
if key in {"id", "name", "size_bytes", "sha256", "created_at"}}
|
|
|
|
|
|
def public_user_iso_error(value: Any) -> str:
|
|
|
safe = {
|
|
|
"面板在任务执行期间重启;请检查宿主机实际状态,勿直接重复提交",
|
|
|
"ISO 下载地址无效", "ISO 下载地址无法解析",
|
|
|
"ISO 下载地址的文件名必须以 .iso 结尾",
|
|
|
"ISO 下载地址解析结果无效",
|
|
|
"ISO 下载地址不能指向内网、回环或保留地址",
|
|
|
"ISO 下载地址发生重定向,请使用最终 HTTPS 直链",
|
|
|
"无法连接 ISO 下载地址", "ISO 下载响应长度无效",
|
|
|
"ISO 下载响应不能使用内容压缩", "ISO 下载超时",
|
|
|
"ISO 下载内容不完整", "ISO 的 SHA-256 校验值不匹配",
|
|
|
"ISO 大小必须在 1 MiB 至可用配额之间",
|
|
|
"下载文件不是有效的 ISO 9660 镜像",
|
|
|
"ISO 未包含可识别的 El Torito 启动记录",
|
|
|
"ISO 超过用户目录的 10 GiB 总配额或剩余容量", "用户 ISO 可用配额不足",
|
|
|
"每个宿主机最多保存 5 个用户 ISO",
|
|
|
"用户 ISO 目录的 10 GiB 存储配额已用完",
|
|
|
"用户 ISO 目录总容量不能超过 10 GiB",
|
|
|
"宿主机用户 ISO 存储空间不足",
|
|
|
"用户 ISO 上传到宿主机失败",
|
|
|
"用户 ISO 导入失败,请查看引擎日志",
|
|
|
}
|
|
|
if isinstance(value, str) and value in safe:
|
|
|
return value
|
|
|
if isinstance(value, str) and re.fullmatch(
|
|
|
r"ISO 下载服务器返回 HTTP [1-5][0-9]{2}", value):
|
|
|
return value
|
|
|
return "ISO 下载或导入失败,请联系管理员查看日志"
|
|
|
|
|
|
# Async job records are persisted by the engine and may contain text from
|
|
|
# libvirt, QEMU, a remote host, or a future implementation. The browser
|
|
|
# endpoints below deliberately expose a small, operation-specific DTO;
|
|
|
# never return a job dictionary wholesale. Keep these sets local to the
|
|
|
# app factory so an engine instance cannot accidentally share mutable
|
|
|
# presentation state with another host.
|
|
|
public_job_statuses = frozenset({
|
|
|
"queued", "pending", "running", "succeeded", "completed", "done",
|
|
|
"failed", "uncertain", "cancelled", "canceled", "blocked", "unknown",
|
|
|
})
|
|
|
public_iso_phases = frozenset({
|
|
|
"等待下载", "正在安全下载 ISO", "正在保存到面板 ISO 目录",
|
|
|
"ISO 已保存,可用于所有宿主机", "导入失败", "引擎重启,任务已中断",
|
|
|
"面板在 ISO 下载期间重启;请重新提交导入",
|
|
|
})
|
|
|
public_reinstall_phases = frozenset({
|
|
|
"等待重装任务开始", "正在检查虚拟机", "重装完成", "重装失败",
|
|
|
"重装完成,但旧系统盘清理待核对", "重装结果待核对",
|
|
|
"重装已完成,但凭据或任务记录保存失败", "面板重启,任务已中断",
|
|
|
# These are stable progress identifiers consumed by the standalone
|
|
|
# engine UI; they are not arbitrary text from libvirt.
|
|
|
"shutting_down", "cloning", "starting", "cleaning_up",
|
|
|
"cleanup_pending", "result_uncertain", "completed",
|
|
|
})
|
|
|
public_rescue_phases = frozenset({
|
|
|
"等待救援任务开始", "等待手动确认的救援冷启动", "等待退出救援模式",
|
|
|
"正在预挂载救援 ISO 并等待正常关机", "正在强制断电并从救援 ISO 冷启动",
|
|
|
"正在正常关机并恢复原系统启动配置",
|
|
|
"救援 ISO 已预挂载,等待手动强制断电后开机", "救援系统已启动",
|
|
|
"已退出救援模式,原系统已启动", "救援操作失败",
|
|
|
"面板重启,任务已中断",
|
|
|
})
|
|
|
|
|
|
# Canonical messages retain useful, already-reviewed user guidance while
|
|
|
# removing any suffix supplied by a host-controlled exception. Prefixes
|
|
|
# are matched only to select a fixed replacement; the original value is
|
|
|
# never returned.
|
|
|
public_reinstall_errors = (
|
|
|
("重装失败且自动回滚不完整", "重装失败且自动回滚不完整;旧系统盘已保留,请检查虚拟机与新建存储卷"),
|
|
|
("重装失败,已恢复原虚拟机定义及旧系统盘", "重装失败,已恢复原虚拟机定义及旧系统盘"),
|
|
|
("正常关机超时,重装已停止", "正常关机超时,重装已停止;旧系统盘未改变"),
|
|
|
("重装系统失败", "重装系统失败,请查看面板日志"),
|
|
|
("虚拟机配置无效", "虚拟机配置无效,重装已停止"),
|
|
|
("虚拟机磁盘或网卡配置已变化", "虚拟机配置已变化,重装已停止"),
|
|
|
("虚拟机网桥或 MAC 已变化", "虚拟机网桥或 MAC 已变化,不能自动沿用原网络"),
|
|
|
("关机后虚拟机配置已变化", "关机后虚拟机配置已变化,重装已停止"),
|
|
|
("关机后存储卷已变化", "关机后存储卷已变化,重装已停止"),
|
|
|
("原配置盘的公网 IP 或 MAC 与虚拟机不一致", "原配置盘的公网 IP 或 MAC 与虚拟机不一致,已停止重装"),
|
|
|
("无法校验原 cloud-init 网络配置", "无法校验原 cloud-init 网络配置,已停止重装"),
|
|
|
("模板卷在宿主机上不存在", "模板卷在宿主机上不存在,请重新导入"),
|
|
|
("模板卷不是 qcow2 格式", "模板卷格式不受支持,已停止重装"),
|
|
|
("模板容量超过原系统盘", "所选模板容量超过原系统盘,无法保留原磁盘大小"),
|
|
|
("存储池空间不足", "存储池空间不足,无法创建新系统盘"),
|
|
|
("新系统盘或配置盘名称冲突", "新系统盘或配置盘名称冲突,已停止重装"),
|
|
|
("宿主机未能克隆新系统盘", "宿主机未能克隆新系统盘,重装已停止"),
|
|
|
("宿主机未能创建新配置盘", "宿主机未能创建新配置盘,重装已停止"),
|
|
|
("宿主机未能切换虚拟机定义", "宿主机未能切换虚拟机定义,重装已停止"),
|
|
|
("宿主机未能保存虚拟机网络配置", "宿主机未能保存新的虚拟机配置,重装已停止"),
|
|
|
("重装已完成,但旧系统盘清理待核对", "重装已完成,但旧系统盘清理待核对,请管理员检查宿主机"),
|
|
|
("重装结果待核对", "重装结果待核对,请检查虚拟机状态和存储卷"),
|
|
|
)
|
|
|
public_rescue_errors = (
|
|
|
("救援 ISO 预挂载失败且原虚拟机持久配置未能恢复", "救援 ISO 预挂载失败且原虚拟机持久配置未能恢复;没有强制断电,请检查宿主机"),
|
|
|
("救援 ISO 预挂载失败", "救援 ISO 预挂载失败,原虚拟机未受影响"),
|
|
|
("救援启动失败,已恢复原虚拟机配置及运行状态", "救援启动失败,已恢复原虚拟机配置及运行状态"),
|
|
|
("救援系统强制重启后未能启动", "救援系统强制重启后未能启动;救援 ISO 配置已保留,请检查控制台"),
|
|
|
("已尝试强制启动,但救援系统未进入运行状态", "已尝试强制启动,但救援系统未进入运行状态;救援 ISO 配置已保留,请检查控制台或重试开机"),
|
|
|
("退出救援失败且救援虚拟机未能完全恢复", "退出救援失败且救援虚拟机未能完全恢复;没有强制断电,请检查宿主机"),
|
|
|
("退出救援失败,已恢复救援启动配置及运行状态", "退出救援失败,已恢复救援启动配置及运行状态"),
|
|
|
("原系统已启动但仍显示救援模式", "原系统已启动但仍显示救援模式,请检查虚拟机配置"),
|
|
|
("关机后救援启动配置已变化", "关机后救援启动配置已变化,救援已停止;请检查虚拟机状态"),
|
|
|
("强制断电失败", "强制断电失败;救援配置已保留,请检查虚拟机状态"),
|
|
|
("救援 ISO 启动设备与预挂载记录不匹配", "救援 ISO 启动设备与预挂载记录不匹配,已拒绝强制重启"),
|
|
|
("救援恢复记录丢失", "救援恢复记录丢失,已停止操作"),
|
|
|
("救援操作失败", "紧急救援操作失败,请查看面板和宿主机日志"),
|
|
|
("正常关机超时", "正常关机超时,救援未完成,请核对救援状态后再继续"),
|
|
|
("无法请求虚拟机正常关机", "无法请求虚拟机正常关机,请核对救援状态"),
|
|
|
)
|
|
|
public_reinstall_warnings = (
|
|
|
("新系统可能已启动,但启动返回错误", "新系统可能已启动,但启动结果需要通过控制台确认;旧系统盘已保留"),
|
|
|
("新系统已启动,但面板未能读取完整虚拟机状态", "新系统已启动,但面板未能读取完整虚拟机状态,请检查宿主机"),
|
|
|
("新系统已启动,但面板无法确认初始密码已保存", "新系统已启动,但面板无法确认初始密码已保存;请立即保存结果页显示的密码"),
|
|
|
("旧存储卷", "重装已完成,但旧系统盘清理需要管理员检查"),
|
|
|
)
|
|
|
public_rescue_warnings = (
|
|
|
("救援配置已运行,但宿主机关机状态返回异常", "救援配置已运行,但宿主机关机状态需要通过 VNC 确认"),
|
|
|
("救援系统可能已启动,但宿主机启动返回错误", "救援系统可能已启动,请通过控制台确认"),
|
|
|
("救援系统可能已启动,但宿主机返回错误", "救援系统可能已启动,请通过控制台确认"),
|
|
|
("原系统可能已启动,但宿主机启动返回错误", "原系统可能已启动,请通过控制台确认"),
|
|
|
("救援 ISO 已配置为下次冷启动", "救援 ISO 已配置为下次冷启动,请手动强制断电后开机"),
|
|
|
("正常关机超时", "正常关机超时;救援 ISO 可能已预挂载,请核对救援状态"),
|
|
|
("正常关机", "正常关机未完成;请核对救援状态后再继续"),
|
|
|
)
|
|
|
|
|
|
def public_status(value: Any) -> str:
|
|
|
return value if isinstance(value, str) and value in public_job_statuses else "unknown"
|
|
|
|
|
|
def public_timestamp(value: Any) -> int | None:
|
|
|
if isinstance(value, bool):
|
|
|
return None
|
|
|
try:
|
|
|
result = int(value)
|
|
|
except (TypeError, ValueError, OverflowError):
|
|
|
return None
|
|
|
return result if result >= 0 else None
|
|
|
|
|
|
def public_identifier(value: Any, *, minimum: int = 1, maximum: int = 128) -> str | None:
|
|
|
if not isinstance(value, str) or not (minimum <= len(value) <= maximum):
|
|
|
return None
|
|
|
return value if re.fullmatch(r"[A-Za-z0-9_-]+", value) else None
|
|
|
|
|
|
def canonical_job_text(value: Any, mappings: tuple[tuple[str, str], ...], fallback: str) -> str | None:
|
|
|
if not isinstance(value, str) or not value.strip():
|
|
|
return None
|
|
|
text = value.strip()
|
|
|
for prefix, canonical in mappings:
|
|
|
if text.startswith(prefix):
|
|
|
return canonical
|
|
|
return fallback
|
|
|
|
|
|
def public_phase(value: Any, status: str, allowed: frozenset[str], *, kind: str) -> str:
|
|
|
if kind == "reinstall" and status == "uncertain":
|
|
|
if isinstance(value, str) and value in {
|
|
|
"重装完成,但旧系统盘清理待核对", "重装结果待核对"}:
|
|
|
return value
|
|
|
return "等待核对结果"
|
|
|
if isinstance(value, str) and value in allowed:
|
|
|
return value
|
|
|
if kind == "reinstall":
|
|
|
return {
|
|
|
"queued": "等待重装任务开始", "pending": "等待重装任务开始",
|
|
|
"running": "正在执行重装任务", "succeeded": "重装完成",
|
|
|
"completed": "重装完成", "done": "重装完成",
|
|
|
"failed": "重装失败", "uncertain": "等待核对结果",
|
|
|
}.get(status, "重装任务状态待核对")
|
|
|
if kind == "rescue":
|
|
|
return {
|
|
|
"queued": "等待救援任务开始", "pending": "等待救援任务开始",
|
|
|
"running": "正在执行救援任务", "succeeded": "救援操作已完成",
|
|
|
"completed": "救援操作已完成", "done": "救援操作已完成",
|
|
|
"failed": "救援操作失败", "uncertain": "等待核对结果",
|
|
|
}.get(status, "救援任务状态待核对")
|
|
|
return {
|
|
|
"queued": "等待下载", "pending": "等待下载", "running": "正在处理",
|
|
|
"succeeded": "已完成", "completed": "已完成", "done": "已完成",
|
|
|
"failed": "导入失败", "uncertain": "等待核对结果",
|
|
|
}.get(status, "任务状态待核对")
|
|
|
|
|
|
def public_warning(value: Any, mappings: tuple[tuple[str, str], ...], fallback: str) -> str | None:
|
|
|
return canonical_job_text(value, mappings, fallback)
|
|
|
|
|
|
def public_user_iso_job(job: dict) -> dict:
|
|
|
status = public_status(job.get("status"))
|
|
|
result: dict[str, Any] = {}
|
|
|
job_id = public_identifier(job.get("job_id"), minimum=8)
|
|
|
if job_id is not None:
|
|
|
result["job_id"] = job_id
|
|
|
result["status"] = status
|
|
|
result["phase"] = public_phase(
|
|
|
job.get("phase"), status, public_iso_phases, kind="iso")
|
|
|
for key in ("created_at", "started_at", "finished_at"):
|
|
|
timestamp = public_timestamp(job.get(key))
|
|
|
if timestamp is not None:
|
|
|
result[key] = timestamp
|
|
|
if isinstance(job.get("iso"), dict):
|
|
|
result["iso"] = public_user_iso(job["iso"])
|
|
|
if "error" in job:
|
|
|
result["error"] = public_user_iso_error(job.get("error"))
|
|
|
return result
|
|
|
|
|
|
def public_user_iso_listing(value: Any) -> dict:
|
|
|
"""Re-sanitize shared-runtime output at this app's HTTP boundary."""
|
|
|
if not isinstance(value, dict):
|
|
|
return {"isos": []}
|
|
|
result: dict[str, Any] = {
|
|
|
"isos": [public_user_iso(item) for item in value.get("isos", [])
|
|
|
if isinstance(item, dict)],
|
|
|
}
|
|
|
active = value.get("active_import_job")
|
|
|
if isinstance(active, dict):
|
|
|
result["active_import_job"] = public_user_iso_job(active)
|
|
|
return result
|
|
|
|
|
|
@app.get("/api/v1/user-isos")
|
|
|
def list_user_isos(_session: Session = Depends(require_approved)):
|
|
|
owner_id, scope = user_iso_context(_session)
|
|
|
if shared_user_iso_runtime is not None:
|
|
|
return public_user_iso_listing(shared_user_iso_runtime.list_for(owner_id))
|
|
|
result = {
|
|
|
"isos": [public_user_iso(row) for row in user_iso_catalog.list(scope)],
|
|
|
}
|
|
|
with user_iso_import_lock:
|
|
|
active = [job for job in user_iso_import_jobs.values()
|
|
|
if job.get("scope") == scope
|
|
|
and job.get("owner_user_id") == owner_id
|
|
|
and job.get("status") in {"queued", "running"}]
|
|
|
if active:
|
|
|
# Only one import per owner/host can be submitted. Sorting is
|
|
|
# defensive for an upgraded database that contains an older
|
|
|
# duplicate; never expose either owner's private scope field.
|
|
|
current = max(active, key=lambda job: (
|
|
|
int(job.get("created_at", 0)), str(job.get("job_id", ""))))
|
|
|
result["active_import_job"] = public_user_iso_job(current)
|
|
|
return result
|
|
|
|
|
|
@app.get("/api/v1/user-isos/import-jobs/{job_id}")
|
|
|
def user_iso_import_job(job_id: str,
|
|
|
_session: Session = Depends(require_approved)):
|
|
|
owner_id, scope = user_iso_context(_session)
|
|
|
if shared_user_iso_runtime is not None:
|
|
|
return public_user_iso_job(
|
|
|
shared_user_iso_runtime.get_job_for(owner_id, job_id))
|
|
|
with user_iso_import_lock:
|
|
|
job = user_iso_import_jobs.get(job_id)
|
|
|
if (job is None or job.get("scope") != scope
|
|
|
or job.get("owner_user_id") != owner_id):
|
|
|
raise TemplateError("ISO 导入任务不存在", 404)
|
|
|
return public_user_iso_job(job)
|
|
|
|
|
|
@app.get("/api/v1/user-isos/{iso_id}")
|
|
|
def get_user_iso(iso_id: str,
|
|
|
_session: Session = Depends(require_approved)):
|
|
|
owner_id, scope = user_iso_context(_session)
|
|
|
if shared_user_iso_runtime is not None:
|
|
|
return {"iso": public_user_iso(
|
|
|
shared_user_iso_runtime.get_for(owner_id, iso_id))}
|
|
|
return {"iso": public_user_iso(user_iso_catalog.get(scope, iso_id))}
|
|
|
|
|
|
def run_user_iso_import(job_id: str, owner_id: int,
|
|
|
scope: str, body: ImportUserISOBody) -> None:
|
|
|
image_path: Path | None = None
|
|
|
iso_id = ""
|
|
|
published = False
|
|
|
cataloged = False
|
|
|
try:
|
|
|
with user_iso_import_lock:
|
|
|
job = user_iso_import_jobs[job_id]
|
|
|
job.update(status="running", phase="正在安全下载 ISO",
|
|
|
started_at=int(time.time()))
|
|
|
save_user_iso_job(job_id, job)
|
|
|
remaining = user_iso_catalog.remaining_bytes(scope)
|
|
|
available = user_iso_files.available_bytes(owner_id)
|
|
|
maximum = min(remaining, available)
|
|
|
if maximum < 1024 * 1024:
|
|
|
raise TemplateError("面板用户 ISO 持久目录空间不足", 409)
|
|
|
image_path, size, digest = download_user_iso(
|
|
|
body.url, body.sha256.lower(),
|
|
|
user_iso_files.owner_directory(owner_id), max_bytes=maximum,
|
|
|
)
|
|
|
iso_id = uuid.uuid4().hex
|
|
|
record = {
|
|
|
"id": iso_id,
|
|
|
"name": body.name,
|
|
|
"volume_name": user_iso_volume_name(iso_id),
|
|
|
"size_bytes": size,
|
|
|
"sha256": digest,
|
|
|
"created_at": int(time.time()),
|
|
|
}
|
|
|
with user_iso_import_lock:
|
|
|
job = user_iso_import_jobs[job_id]
|
|
|
job["phase"] = "正在保存到面板 ISO 目录"
|
|
|
save_user_iso_job(job_id, job)
|
|
|
user_iso_files.publish(
|
|
|
image_path, owner_id, iso_id, expected_size=size)
|
|
|
image_path = None
|
|
|
published = True
|
|
|
user_iso_catalog.add(scope, record)
|
|
|
cataloged = True
|
|
|
user_iso_import_jobs[job_id] = {
|
|
|
"job_id": job_id, "scope": scope, "owner_user_id": owner_id,
|
|
|
"status": "completed", "phase": "ISO 已保存,可用于所有宿主机",
|
|
|
"created_at": user_iso_import_jobs[job_id]["created_at"],
|
|
|
"started_at": user_iso_import_jobs[job_id].get("started_at"),
|
|
|
"finished_at": int(time.time()), "iso": record,
|
|
|
}
|
|
|
save_user_iso_job(job_id, user_iso_import_jobs[job_id])
|
|
|
except Exception as exc:
|
|
|
if isinstance(exc, TemplateError):
|
|
|
error = str(exc)
|
|
|
else:
|
|
|
LOG.exception("User ISO import failed: job=%s owner=%s", job_id, owner_id)
|
|
|
error = "用户 ISO 导入失败,请查看引擎日志"
|
|
|
if cataloged:
|
|
|
try:
|
|
|
user_iso_catalog.delete(scope, iso_id)
|
|
|
except Exception:
|
|
|
LOG.exception("Could not clean failed user ISO catalog: job=%s", job_id)
|
|
|
if published and iso_id:
|
|
|
try:
|
|
|
user_iso_files.delete(owner_id, iso_id)
|
|
|
except Exception:
|
|
|
LOG.exception("Could not clean failed user ISO source: job=%s", job_id)
|
|
|
with user_iso_import_lock:
|
|
|
previous = user_iso_import_jobs.get(job_id, {})
|
|
|
user_iso_import_jobs[job_id] = {
|
|
|
"job_id": job_id, "scope": scope, "owner_user_id": owner_id,
|
|
|
"status": "failed", "phase": "导入失败", "error": error,
|
|
|
"created_at": previous.get("created_at", int(time.time())),
|
|
|
"started_at": previous.get("started_at"),
|
|
|
"finished_at": int(time.time()),
|
|
|
}
|
|
|
try:
|
|
|
save_user_iso_job(job_id, user_iso_import_jobs[job_id])
|
|
|
except DatabaseError:
|
|
|
LOG.exception("Could not persist user ISO import failure: job=%s", job_id)
|
|
|
finally:
|
|
|
if image_path is not None:
|
|
|
image_path.unlink(missing_ok=True)
|
|
|
with user_iso_import_lock:
|
|
|
pending_user_iso_imports.discard(scope)
|
|
|
|
|
|
@app.post("/api/v1/user-isos/import", status_code=202)
|
|
|
def import_user_iso(body: ImportUserISOBody,
|
|
|
_session: Session = Depends(require_approved_write)):
|
|
|
owner_id, scope = user_iso_context(_session)
|
|
|
if shared_user_iso_runtime is not None:
|
|
|
return shared_user_iso_runtime.queue_import(
|
|
|
owner_id, body.url, body.name, body.sha256)
|
|
|
validate_user_iso_url(body.url)
|
|
|
if body.sha256 and (len(body.sha256) != 64
|
|
|
or not all(char in "0123456789abcdefABCDEF"
|
|
|
for char in body.sha256)):
|
|
|
raise TemplateError(
|
|
|
"SHA-256 可留空;填写时必须为 64 位十六进制校验值", 422)
|
|
|
name = validate_user_iso_name(body.name)
|
|
|
body = body.model_copy(update={"name": name, "sha256": body.sha256.lower()})
|
|
|
with user_iso_import_lock:
|
|
|
forbid_long_job_during_update()
|
|
|
existing = user_iso_catalog.list(scope)
|
|
|
user_iso_catalog.remaining_bytes(scope)
|
|
|
if any(row["name"].casefold() == name.casefold() for row in existing):
|
|
|
raise TemplateError("当前用户已有同名 ISO", 409)
|
|
|
if scope in pending_user_iso_imports:
|
|
|
raise TemplateError("已有用户 ISO 正在导入,请等待完成", 409)
|
|
|
job_id = str(uuid.uuid4())
|
|
|
pending_user_iso_imports.add(scope)
|
|
|
user_iso_import_jobs[job_id] = {
|
|
|
"job_id": job_id, "scope": scope, "owner_user_id": owner_id,
|
|
|
"status": "queued", "phase": "等待下载", "created_at": int(time.time()),
|
|
|
}
|
|
|
try:
|
|
|
# Persist before handing work to the executor. Both steps
|
|
|
# share one rollback path so a database failure cannot leave
|
|
|
# this owner permanently stuck in the in-memory pending set.
|
|
|
save_user_iso_job(job_id, user_iso_import_jobs[job_id])
|
|
|
user_iso_import_executor.submit(
|
|
|
run_user_iso_import, job_id, owner_id, scope, body)
|
|
|
except Exception as exc:
|
|
|
pending_user_iso_imports.discard(scope)
|
|
|
user_iso_import_jobs.pop(job_id, None)
|
|
|
with suppress(DatabaseError):
|
|
|
delete_user_iso_job(job_id)
|
|
|
if isinstance(exc, RuntimeError):
|
|
|
raise TemplateError(
|
|
|
"引擎正在停止,无法开始 ISO 导入", 503) from exc
|
|
|
raise
|
|
|
LOG.info("user ISO import queued: job=%s owner=%s", job_id, owner_id)
|
|
|
return {"job_id": job_id, "status": "queued"}
|
|
|
|
|
|
@app.delete("/api/v1/user-isos/{iso_id}")
|
|
|
def delete_user_iso(iso_id: str,
|
|
|
_session: Session = Depends(require_approved_write)):
|
|
|
owner_id, scope = user_iso_context(_session)
|
|
|
if shared_user_iso_runtime is not None:
|
|
|
shared_user_iso_runtime.delete_for(owner_id, iso_id)
|
|
|
LOG.info("user ISO deleted: id=%s owner=%s", iso_id, owner_id)
|
|
|
return {"deleted": True}
|
|
|
with user_iso_import_lock:
|
|
|
forbid_long_job_during_update()
|
|
|
if scope in pending_user_iso_imports:
|
|
|
raise TemplateError("用户 ISO 正在导入,请稍后删除", 409)
|
|
|
user_iso_catalog.get(scope, iso_id)
|
|
|
user_iso_files.delete(owner_id, iso_id)
|
|
|
user_iso_catalog.delete(scope, iso_id)
|
|
|
LOG.info("user ISO deleted: id=%s owner=%s", iso_id, owner_id)
|
|
|
return {"deleted": True}
|
|
|
|
|
|
@app.get("/api/v1/vm-templates")
|
|
|
def list_templates(_session: Session = Depends(require_approved)):
|
|
|
if shared_cloud_template_runtime is not None:
|
|
|
return {"templates": shared_cloud_template_runtime.list()}
|
|
|
profile = current_profile()
|
|
|
return {"templates": templates.list(template_scope(profile))}
|
|
|
|
|
|
def run_template_import(job_id: str, profile: dict, scope: str,
|
|
|
body: ImportTemplateBody, pending_key: str) -> None:
|
|
|
image_path = None
|
|
|
volume_name = ""
|
|
|
uploaded = False
|
|
|
try:
|
|
|
image_path, virtual_size, actual_sha256 = download_verified_image(
|
|
|
body.url, body.sha256.lower(), settings.data_dir)
|
|
|
os_family, version = image_metadata_for(body.url)
|
|
|
template_id = template_id_for(
|
|
|
scope, actual_sha256, version, os_family
|
|
|
)
|
|
|
volume_name = template_volume_name(template_id)
|
|
|
with import_lock:
|
|
|
if any(row["id"] == template_id for row in templates.list(scope)):
|
|
|
raise TemplateError("此云镜像已经导入", 409)
|
|
|
result = new_service(profile).import_template(image_path, volume_name, virtual_size)
|
|
|
uploaded = True
|
|
|
record = {
|
|
|
"id": template_id,
|
|
|
"name": body.name,
|
|
|
"os_family": os_family,
|
|
|
"os_version": version,
|
|
|
"sha256": actual_sha256,
|
|
|
"volume_name": volume_name,
|
|
|
"virtual_size": virtual_size,
|
|
|
"image_bytes": result["image_bytes"],
|
|
|
}
|
|
|
templates.add(scope, record)
|
|
|
with import_lock:
|
|
|
import_jobs[job_id] = {"scope": scope, "status": "completed",
|
|
|
"template": record}
|
|
|
save_job("import_job", job_id, import_jobs[job_id])
|
|
|
except Exception as exc:
|
|
|
if isinstance(exc, (TemplateError, VMError)):
|
|
|
error = str(exc)
|
|
|
else:
|
|
|
LOG.exception("Cloud template import failed: job=%s", job_id)
|
|
|
error = "云镜像导入失败,请查看面板日志"
|
|
|
if uploaded and volume_name:
|
|
|
error += f";宿主机模板卷 {volume_name} 可能需要人工检查"
|
|
|
with import_lock:
|
|
|
import_jobs[job_id] = {"scope": scope, "status": "failed", "error": error}
|
|
|
save_job("import_job", job_id, import_jobs[job_id])
|
|
|
finally:
|
|
|
if image_path is not None:
|
|
|
image_path.unlink(missing_ok=True)
|
|
|
with import_lock:
|
|
|
pending_templates.discard((scope, pending_key))
|
|
|
|
|
|
@app.post("/api/v1/vm-templates/import", status_code=202)
|
|
|
def import_template(body: ImportTemplateBody, _session: Session = Depends(require_admin_write)):
|
|
|
if shared_cloud_template_runtime is not None:
|
|
|
forbid_long_job_during_update()
|
|
|
return shared_cloud_template_runtime.queue_import(
|
|
|
body.url, body.name, body.sha256)
|
|
|
profile = current_profile()
|
|
|
if body.sha256 and (len(body.sha256) != 64
|
|
|
or not all(char in "0123456789abcdefABCDEF"
|
|
|
for char in body.sha256)):
|
|
|
raise TemplateError(
|
|
|
"SHA-256 可留空;填写时必须为 64 位十六进制校验值", 422
|
|
|
)
|
|
|
os_family, version = image_metadata_for(body.url)
|
|
|
name = validate_template_name(body.name)
|
|
|
body = body.model_copy(update={"name": name})
|
|
|
scope = template_scope(profile)
|
|
|
expected_sha256 = body.sha256.lower()
|
|
|
pending_key = expected_sha256 or hashlib.sha256(body.url.encode("utf-8")).hexdigest()
|
|
|
with import_lock:
|
|
|
forbid_long_job_during_update()
|
|
|
existing_templates = templates.list(scope)
|
|
|
if expected_sha256 and any(
|
|
|
row["id"] == template_id_for(
|
|
|
scope, expected_sha256, version, os_family
|
|
|
)
|
|
|
for row in existing_templates):
|
|
|
raise TemplateError("此云镜像已经导入", 409)
|
|
|
if any(row["name"].casefold() == name.casefold()
|
|
|
for row in existing_templates):
|
|
|
raise TemplateError("当前宿主机已有同名云镜像模板", 409)
|
|
|
if (scope, pending_key) in pending_templates:
|
|
|
raise TemplateError("该云镜像正在导入", 409)
|
|
|
if pending_templates:
|
|
|
raise TemplateError("已有云镜像正在导入,请等待完成后再提交", 409)
|
|
|
job_id = str(uuid.uuid4())
|
|
|
pending_templates.add((scope, pending_key))
|
|
|
import_jobs[job_id] = {"scope": scope, "status": "running"}
|
|
|
save_job("import_job", job_id, import_jobs[job_id])
|
|
|
try:
|
|
|
import_executor.submit(run_template_import, job_id, profile,
|
|
|
scope, body, pending_key)
|
|
|
except RuntimeError as exc:
|
|
|
pending_templates.discard((scope, pending_key))
|
|
|
import_jobs.pop(job_id, None)
|
|
|
delete_job("import_job", job_id)
|
|
|
raise TemplateError("面板正在停止,无法开始导入任务", 503) from exc
|
|
|
LOG.info("cloud template import queued: job=%s admin=%s", job_id, _session.username)
|
|
|
return {"job_id": job_id, "status": "running"}
|
|
|
|
|
|
@app.patch("/api/v1/vm-templates/{template_id}")
|
|
|
def rename_template(template_id: str, body: RenameTemplateBody,
|
|
|
_session: Session = Depends(require_admin_write)):
|
|
|
if shared_cloud_template_runtime is not None:
|
|
|
forbid_long_job_during_update()
|
|
|
return {"template": shared_cloud_template_runtime.rename(
|
|
|
template_id, body.name)}
|
|
|
scope = template_scope(current_profile())
|
|
|
with import_lock:
|
|
|
if pending_templates:
|
|
|
raise TemplateError("云镜像正在导入,请稍后修改模板名称", 409)
|
|
|
return {"template": templates.rename(scope, template_id, body.name)}
|
|
|
|
|
|
@app.get("/api/v1/vm-templates/import-jobs/{job_id}")
|
|
|
def import_job(job_id: str, _session: Session = Depends(require_admin)):
|
|
|
if shared_cloud_template_runtime is not None:
|
|
|
return shared_cloud_template_runtime.get_job(job_id)
|
|
|
scope = template_scope(current_profile())
|
|
|
with import_lock:
|
|
|
job = import_jobs.get(job_id)
|
|
|
if job is None or job["scope"] != scope:
|
|
|
raise TemplateError("导入任务不存在", 404)
|
|
|
return {key: value for key, value in job.items() if key != "scope"}
|
|
|
|
|
|
@app.get("/api/v1/rescue-templates")
|
|
|
def list_rescue_templates(_session: Session = Depends(require_approved)):
|
|
|
return {"templates": rescue_catalog.list(rescue_scope(current_profile()))}
|
|
|
|
|
|
def run_rescue_import(job_id: str, profile: dict, scope: str,
|
|
|
record: dict, image_path: Path) -> None:
|
|
|
uploaded = False
|
|
|
cataloged = False
|
|
|
try:
|
|
|
with rescue_import_lock:
|
|
|
rescue_import_jobs[job_id]["status"] = "running"
|
|
|
rescue_import_jobs[job_id]["phase"] = "正在传输 ISO 到宿主机"
|
|
|
save_job("rescue_import_job", job_id, rescue_import_jobs[job_id])
|
|
|
new_service(profile).import_rescue_iso(image_path, record["volume_name"])
|
|
|
uploaded = True
|
|
|
rescue_catalog.add(scope, record)
|
|
|
cataloged = True
|
|
|
with rescue_import_lock:
|
|
|
rescue_import_jobs[job_id] = {
|
|
|
"scope": scope, "status": "completed", "phase": "救援模版已导入",
|
|
|
"template": record,
|
|
|
}
|
|
|
save_job("rescue_import_job", job_id, rescue_import_jobs[job_id])
|
|
|
except Exception as exc:
|
|
|
if isinstance(exc, (TemplateError, VMError)):
|
|
|
error = str(exc)
|
|
|
else:
|
|
|
LOG.exception("Rescue ISO import failed: job=%s", job_id)
|
|
|
error = "救援 ISO 导入失败,请查看面板日志"
|
|
|
if cataloged:
|
|
|
try:
|
|
|
rescue_catalog.delete(scope, record["id"])
|
|
|
except Exception:
|
|
|
LOG.exception("Could not clean failed rescue catalog entry: job=%s", job_id)
|
|
|
error += ";数据库中可能留有救援模版记录,请管理员检查"
|
|
|
if uploaded:
|
|
|
try:
|
|
|
new_service(profile).delete_rescue_iso(record["volume_name"])
|
|
|
except Exception:
|
|
|
LOG.exception("Could not clean failed rescue ISO import: job=%s", job_id)
|
|
|
error += ";宿主机上可能留有救援 ISO 卷,请管理员检查"
|
|
|
with rescue_import_lock:
|
|
|
rescue_import_jobs[job_id] = {
|
|
|
"scope": scope, "status": "failed", "phase": "导入失败", "error": error,
|
|
|
}
|
|
|
try:
|
|
|
save_job("rescue_import_job", job_id, rescue_import_jobs[job_id])
|
|
|
except DatabaseError:
|
|
|
LOG.exception("Could not persist rescue import failure: job=%s", job_id)
|
|
|
finally:
|
|
|
image_path.unlink(missing_ok=True)
|
|
|
with rescue_import_lock:
|
|
|
pending_rescue_imports.discard(scope)
|
|
|
|
|
|
@app.post("/api/v1/rescue-templates", status_code=202)
|
|
|
async def upload_rescue_template(
|
|
|
request: Request,
|
|
|
name: str = Query(min_length=1, max_length=64),
|
|
|
_session: Session = Depends(require_admin_write),
|
|
|
):
|
|
|
name = validate_template_name(name)
|
|
|
profile = current_profile()
|
|
|
scope = rescue_scope(profile)
|
|
|
with rescue_import_lock:
|
|
|
forbid_long_job_during_update()
|
|
|
if scope in pending_rescue_imports:
|
|
|
raise TemplateError("已有救援 ISO 正在导入,请等待完成", 409)
|
|
|
if any(row["name"].casefold() == name.casefold()
|
|
|
for row in rescue_catalog.list(scope)):
|
|
|
raise TemplateError("当前宿主机已有同名救援模版", 409)
|
|
|
pending_rescue_imports.add(scope)
|
|
|
image_path: Path | None = None
|
|
|
handed_off = False
|
|
|
try:
|
|
|
declared_size = request.headers.get("content-length")
|
|
|
if declared_size is not None:
|
|
|
try:
|
|
|
if int(declared_size) > MAX_RESCUE_ISO_BYTES:
|
|
|
raise TemplateError("救援 ISO 超过 8 GiB 限制", 413)
|
|
|
except ValueError as exc:
|
|
|
raise TemplateError("上传长度无效", 400) from exc
|
|
|
fd, raw_path = tempfile.mkstemp(prefix=".rescue-upload-", suffix=".iso",
|
|
|
dir=settings.data_dir)
|
|
|
image_path = Path(raw_path)
|
|
|
os.fchmod(fd, 0o600)
|
|
|
checksum = hashlib.sha256()
|
|
|
size = 0
|
|
|
with os.fdopen(fd, "wb") as output:
|
|
|
async for chunk in request.stream():
|
|
|
size += len(chunk)
|
|
|
if size > MAX_RESCUE_ISO_BYTES:
|
|
|
raise TemplateError("救援 ISO 超过 8 GiB 限制", 413)
|
|
|
output.write(chunk)
|
|
|
checksum.update(chunk)
|
|
|
output.flush()
|
|
|
os.fsync(output.fileno())
|
|
|
if internal_mode:
|
|
|
validation = request.scope.get("engine_upload_validation", {})
|
|
|
if not validation.get("complete") or not validation.get("valid"):
|
|
|
raise HTTPException(status_code=401, detail="救援 ISO 摘要与签名不匹配")
|
|
|
verify_bootable_iso(image_path)
|
|
|
template_id = uuid.uuid4().hex
|
|
|
record = {
|
|
|
"id": template_id, "name": name,
|
|
|
"volume_name": rescue_volume_name(template_id),
|
|
|
"size_bytes": size, "sha256": checksum.hexdigest(),
|
|
|
}
|
|
|
job_id = str(uuid.uuid4())
|
|
|
with rescue_import_lock:
|
|
|
rescue_import_jobs[job_id] = {
|
|
|
"scope": scope, "status": "queued", "phase": "等待传输到宿主机",
|
|
|
}
|
|
|
save_job("rescue_import_job", job_id, rescue_import_jobs[job_id])
|
|
|
try:
|
|
|
rescue_import_executor.submit(
|
|
|
run_rescue_import, job_id, profile, scope, record, image_path,
|
|
|
)
|
|
|
except RuntimeError as exc:
|
|
|
rescue_import_jobs.pop(job_id, None)
|
|
|
delete_job("rescue_import_job", job_id)
|
|
|
raise TemplateError("面板正在停止,无法导入救援 ISO", 503) from exc
|
|
|
LOG.info("rescue ISO import queued: job=%s admin=%s", job_id, _session.username)
|
|
|
handed_off = True # The worker now owns and removes the temporary file.
|
|
|
return {"job_id": job_id, "status": "queued"}
|
|
|
finally:
|
|
|
if not handed_off:
|
|
|
if image_path is not None:
|
|
|
image_path.unlink(missing_ok=True)
|
|
|
with rescue_import_lock:
|
|
|
pending_rescue_imports.discard(scope)
|
|
|
|
|
|
@app.get("/api/v1/rescue-templates/import-jobs/{job_id}")
|
|
|
def rescue_import_job(job_id: str, _session: Session = Depends(require_admin)):
|
|
|
scope = rescue_scope(current_profile())
|
|
|
with rescue_import_lock:
|
|
|
job = rescue_import_jobs.get(job_id)
|
|
|
if job is None or job["scope"] != scope:
|
|
|
raise TemplateError("救援模版导入任务不存在", 404)
|
|
|
return {key: value for key, value in job.items() if key != "scope"}
|
|
|
|
|
|
@app.delete("/api/v1/rescue-templates/{template_id}")
|
|
|
def delete_rescue_template(template_id: str,
|
|
|
_session: Session = Depends(require_admin_write)):
|
|
|
profile = current_profile()
|
|
|
scope = rescue_scope(profile)
|
|
|
with rescue_lock, rescue_import_lock:
|
|
|
forbid_long_job_during_update()
|
|
|
if scope in pending_rescue_imports:
|
|
|
raise TemplateError("救援 ISO 正在导入,请稍后删除", 409)
|
|
|
record = rescue_catalog.get(scope, template_id)
|
|
|
states = (data_store.list("vm_rescue") if data_store is not None
|
|
|
else test_rescue_states.copy())
|
|
|
for raw in states.values():
|
|
|
try:
|
|
|
state_record = json.loads(raw)
|
|
|
except (TypeError, ValueError) as exc:
|
|
|
raise DatabaseError("虚拟机救援记录损坏") from exc
|
|
|
if not isinstance(state_record, dict):
|
|
|
raise DatabaseError("虚拟机救援记录损坏")
|
|
|
if state_record.get("volume_name") == record["volume_name"]:
|
|
|
raise TemplateError("此救援模版正被虚拟机使用,须先退出救援模式", 409)
|
|
|
try:
|
|
|
new_service(profile).delete_rescue_iso(record["volume_name"])
|
|
|
except VMError as exc:
|
|
|
# Only the target volume's precise absence is idempotent.
|
|
|
# A missing/misconfigured pool can also be a 404, but must
|
|
|
# preserve the catalog entry so the administrator can retry.
|
|
|
if (exc.status != 404
|
|
|
or exc.error_code != "rescue_volume_not_found"):
|
|
|
raise
|
|
|
rescue_catalog.delete(scope, template_id)
|
|
|
LOG.info("rescue ISO deleted: id=%s admin=%s", template_id, _session.username)
|
|
|
return {"deleted": True}
|
|
|
|
|
|
@app.get("/api/v1/vms")
|
|
|
def list_vms(_session: Session = Depends(require_approved)):
|
|
|
profile = current_profile()
|
|
|
vms = new_service(profile).list_vms()
|
|
|
if database_mode:
|
|
|
account = account_for_session(_session)
|
|
|
if not account["admin"]:
|
|
|
owned_ids = state.users.list_owned_vm_ids(host_scope(profile), account["id"])
|
|
|
vms = [vm for vm in vms if vm["uuid"] in owned_ids]
|
|
|
records = (data_store.list("vm_credential") if data_store is not None
|
|
|
else test_credentials.copy())
|
|
|
return {"items": [decorate_vm(
|
|
|
vm, parse_credential(records.get(credential_key(profile, vm["uuid"]))), profile,
|
|
|
) for vm in vms]}
|
|
|
|
|
|
@app.get("/api/v1/vm-states")
|
|
|
def list_vm_states(
|
|
|
ids: str | None = Query(None, max_length=4096),
|
|
|
_session: Session = Depends(require_approved),
|
|
|
):
|
|
|
profile = current_profile()
|
|
|
if ids is None:
|
|
|
vms = new_service(profile).list_vm_states()
|
|
|
if database_mode:
|
|
|
account = account_for_session(_session)
|
|
|
if not account["admin"]:
|
|
|
owned_ids = state.users.list_owned_vm_ids(host_scope(profile), account["id"])
|
|
|
vms = [vm for vm in vms if vm["uuid"] in owned_ids]
|
|
|
return {"items": vms}
|
|
|
raw_ids = ids.split(",")
|
|
|
if not 1 <= len(raw_ids) <= 100 or any(not value for value in raw_ids):
|
|
|
raise VMError("虚拟机状态查询数量必须为 1 到 100", 400)
|
|
|
canonical_ids: list[str] = []
|
|
|
seen: set[str] = set()
|
|
|
for value in raw_ids:
|
|
|
try:
|
|
|
canonical = str(uuid.UUID(value))
|
|
|
except (TypeError, ValueError, AttributeError) as exc:
|
|
|
raise VMError("虚拟机 UUID 无效", 400) from exc
|
|
|
if canonical not in seen:
|
|
|
canonical_ids.append(canonical)
|
|
|
seen.add(canonical)
|
|
|
if database_mode:
|
|
|
account = account_for_session(_session)
|
|
|
if not account["admin"]:
|
|
|
owned_ids = state.users.list_owned_vm_ids(
|
|
|
host_scope(profile), account["id"]
|
|
|
)
|
|
|
if any(vm_id not in owned_ids for vm_id in canonical_ids):
|
|
|
raise VMError("虚拟机不存在", 404)
|
|
|
return {"items": new_service(profile).list_vm_states(canonical_ids)}
|
|
|
|
|
|
@app.get("/api/v1/vm-metadata")
|
|
|
def list_vm_metadata(
|
|
|
ids: str = Query(..., min_length=1, max_length=4096),
|
|
|
_session: Session = Depends(require_approved),
|
|
|
):
|
|
|
raw_ids = ids.split(",")
|
|
|
if not 1 <= len(raw_ids) <= 100 or any(not value for value in raw_ids):
|
|
|
raise VMError("虚拟机元数据查询数量必须为 1 到 100", 400)
|
|
|
canonical_ids: list[str] = []
|
|
|
seen: set[str] = set()
|
|
|
for value in raw_ids:
|
|
|
try:
|
|
|
canonical = str(uuid.UUID(value))
|
|
|
except (TypeError, ValueError, AttributeError) as exc:
|
|
|
raise VMError("虚拟机 UUID 无效", 400) from exc
|
|
|
if canonical not in seen:
|
|
|
canonical_ids.append(canonical)
|
|
|
seen.add(canonical)
|
|
|
|
|
|
profile = current_profile()
|
|
|
if database_mode:
|
|
|
account = account_for_session(_session)
|
|
|
if not account["admin"]:
|
|
|
owned_ids = state.users.list_owned_vm_ids(
|
|
|
host_scope(profile), account["id"]
|
|
|
)
|
|
|
if any(vm_id not in owned_ids for vm_id in canonical_ids):
|
|
|
# Reject the complete request rather than disclosing which
|
|
|
# UUID in a mixed-owner batch exists on this host.
|
|
|
raise VMError("虚拟机不存在", 404)
|
|
|
return {"items": new_service(profile).list_vm_metadata(canonical_ids)}
|
|
|
|
|
|
@app.get("/api/v1/vms/{vm_id}/state")
|
|
|
def get_vm_state(vm_id: str, _session: Session = Depends(require_approved)):
|
|
|
profile = current_profile()
|
|
|
require_vm_access(_session, profile, vm_id)
|
|
|
return new_service(profile).get_vm_state(vm_id)
|
|
|
|
|
|
@app.get("/api/v1/vms/{vm_id}")
|
|
|
def get_vm(vm_id: str, _session: Session = Depends(require_approved)):
|
|
|
profile = current_profile()
|
|
|
require_vm_access(_session, profile, vm_id)
|
|
|
vm = new_service(profile).get_vm(vm_id)
|
|
|
return decorate_vm(vm, get_credential(profile, vm["uuid"]), profile)
|
|
|
|
|
|
@app.get("/api/v1/vms/{vm_id}/firewall")
|
|
|
def get_vm_firewall(
|
|
|
vm_id: str,
|
|
|
_session: Session = Depends(require_approved),
|
|
|
):
|
|
|
profile = current_profile()
|
|
|
canonical_id = require_vm_access(_session, profile, vm_id)
|
|
|
observed = new_service(profile).get_firewall_policy(canonical_id)
|
|
|
record = load_firewall_record(profile, canonical_id)
|
|
|
record_free_default = (
|
|
|
record is None
|
|
|
and (
|
|
|
observed.get("filter_mode") == "baseline"
|
|
|
or is_default_platform_firewall(canonical_id, observed)
|
|
|
)
|
|
|
)
|
|
|
if record_free_default:
|
|
|
response = {
|
|
|
"enabled": False,
|
|
|
"revision": 0,
|
|
|
"policy_digest": firewall_policy_digest(False, []),
|
|
|
"filter_digest": observed.get("filter_digest"),
|
|
|
"rules": [],
|
|
|
"sync_state": observed.get("sync_state", "active"),
|
|
|
}
|
|
|
if observed.get("filter_mode") == "managed":
|
|
|
response["filter_version"] = FIREWALL_FILTER_COMPILER_VERSION
|
|
|
return response
|
|
|
applied = bool(
|
|
|
record is not None
|
|
|
and observed.get("filter_mode") == "managed"
|
|
|
and observed.get("sync_state") == "active"
|
|
|
and observed.get("filter_digest") == record["filter_digest"]
|
|
|
and observed.get("platform_policy_digest")
|
|
|
== record["platform_policy_digest"]
|
|
|
)
|
|
|
return {
|
|
|
"enabled": record["enabled"] if applied else observed.get("enabled"),
|
|
|
"revision": record["revision"] if applied else None,
|
|
|
"policy_digest": record["policy_digest"] if applied else None,
|
|
|
"filter_digest": observed.get("filter_digest"),
|
|
|
"rules": record["rules"] if applied else [],
|
|
|
"sync_state": "active" if applied else "drifted",
|
|
|
}
|
|
|
|
|
|
@app.put("/api/v1/vms/{vm_id}/firewall")
|
|
|
def put_vm_firewall(
|
|
|
vm_id: str,
|
|
|
body: FirewallPolicyBody,
|
|
|
_session: Session = Depends(require_internal_service_write),
|
|
|
):
|
|
|
"""Apply one complete typed IPv4 policy; raw nwfilter XML is forbidden."""
|
|
|
forbid_long_job_during_update()
|
|
|
profile = current_profile()
|
|
|
canonical_id = require_vm_access(_session, profile, vm_id)
|
|
|
vm_scope = host_scope(profile)
|
|
|
# These locks protect only the short conflict check. The remote
|
|
|
# libvirt call runs without any host-global lock; EngineApplication's
|
|
|
# per-VM target lock serializes conflicting mutations for this VM.
|
|
|
with reinstall_lock, rescue_lock:
|
|
|
if (template_scope(profile), canonical_id) in pending_reinstalls:
|
|
|
raise VMError("虚拟机正在重装系统,暂时不能修改防火墙", 409)
|
|
|
if ((vm_scope, canonical_id) in pending_rescues
|
|
|
or load_rescue_state(profile, canonical_id) is not None):
|
|
|
raise VMError("虚拟机处于救援模式或切换中,暂时不能修改防火墙", 409)
|
|
|
requested = body.model_dump()
|
|
|
requested_rules = requested["rules"]
|
|
|
requested_digest = firewall_policy_digest(
|
|
|
requested["enabled"], requested_rules)
|
|
|
service = new_service(profile)
|
|
|
observed = service.get_firewall_policy(canonical_id)
|
|
|
try:
|
|
|
platform_policy = normalize_platform_policy_snapshot(
|
|
|
observed.get("platform_policy_snapshot")
|
|
|
or default_platform_policy_snapshot())
|
|
|
except ValueError as exc:
|
|
|
raise VMError(
|
|
|
"虚拟机平台网络策略证据无效", 409,
|
|
|
error_code="firewall_drift_detected",
|
|
|
) from exc
|
|
|
current_filter_digest = build_firewall_filter_xml(
|
|
|
canonical_id, requested["enabled"], requested_rules,
|
|
|
platform_policy_snapshot=platform_policy)[3]
|
|
|
if not hmac.compare_digest(
|
|
|
requested["policy_digest"], requested_digest):
|
|
|
raise VMError(
|
|
|
"防火墙策略摘要与规范化规则不一致", 422,
|
|
|
error_code="firewall_rule_invalid",
|
|
|
)
|
|
|
previous = load_firewall_record(profile, canonical_id)
|
|
|
if previous is not None:
|
|
|
if not hmac.compare_digest(
|
|
|
previous["platform_policy_digest"],
|
|
|
platform_policy["digest"]):
|
|
|
raise VMError(
|
|
|
"虚拟机平台网络策略与引擎记录不一致",
|
|
|
409,
|
|
|
error_code="firewall_drift_detected",
|
|
|
)
|
|
|
if requested["revision"] < previous["revision"]:
|
|
|
raise VMError("防火墙策略版本已更新,请刷新后重试", 409)
|
|
|
if (requested["revision"] == previous["revision"]
|
|
|
and requested_digest != previous["policy_digest"]):
|
|
|
raise VMError("相同防火墙版本对应了不同规则", 409)
|
|
|
if requested["revision"] == previous["revision"]:
|
|
|
observed_filter_digest = observed.get("filter_digest")
|
|
|
if (observed.get("filter_mode") == "managed"
|
|
|
and observed.get("sync_state") == "active"
|
|
|
and isinstance(observed_filter_digest, str)
|
|
|
and hmac.compare_digest(
|
|
|
observed_filter_digest, current_filter_digest)):
|
|
|
if (previous["filter_version"]
|
|
|
!= FIREWALL_FILTER_COMPILER_VERSION
|
|
|
or not hmac.compare_digest(
|
|
|
previous["filter_digest"],
|
|
|
current_filter_digest,
|
|
|
)):
|
|
|
previous = {
|
|
|
**previous,
|
|
|
"version": 2,
|
|
|
"filter_version": FIREWALL_FILTER_COMPILER_VERSION,
|
|
|
"filter_digest": current_filter_digest,
|
|
|
"platform_policy_snapshot": platform_policy,
|
|
|
"platform_policy_digest": platform_policy["digest"],
|
|
|
"applied_at": int(time.time()),
|
|
|
}
|
|
|
save_firewall_record(profile, canonical_id, previous)
|
|
|
return {
|
|
|
"enabled": previous["enabled"],
|
|
|
"revision": previous["revision"],
|
|
|
"policy_digest": previous["policy_digest"],
|
|
|
"filter_digest": current_filter_digest,
|
|
|
"filter_version": FIREWALL_FILTER_COMPILER_VERSION,
|
|
|
"rules": previous["rules"],
|
|
|
"sync_state": "active",
|
|
|
}
|
|
|
legacy_compiler_match = (
|
|
|
previous["filter_version"]
|
|
|
< FIREWALL_FILTER_COMPILER_VERSION
|
|
|
and observed.get("filter_mode") == "managed"
|
|
|
and observed.get("sync_state") == "active"
|
|
|
and isinstance(observed_filter_digest, str)
|
|
|
and hmac.compare_digest(
|
|
|
observed_filter_digest, previous["filter_digest"])
|
|
|
)
|
|
|
if not legacy_compiler_match:
|
|
|
raise VMError(
|
|
|
"防火墙实际状态与已保存版本不一致,请先核查漂移",
|
|
|
409,
|
|
|
error_code="firewall_drift_detected",
|
|
|
)
|
|
|
|
|
|
result = service.apply_firewall_policy(
|
|
|
canonical_id,
|
|
|
enabled=requested["enabled"],
|
|
|
rules=requested_rules,
|
|
|
)
|
|
|
if result.get("policy_digest") != requested_digest:
|
|
|
raise VMError(
|
|
|
"防火墙策略摘要校验失败,结果待核对", 503,
|
|
|
error_code="firewall_result_uncertain",
|
|
|
)
|
|
|
if result.get("platform_policy_digest") != platform_policy["digest"]:
|
|
|
raise VMError(
|
|
|
"平台网络策略摘要校验失败,结果待核对", 503,
|
|
|
error_code="firewall_result_uncertain",
|
|
|
)
|
|
|
record = {
|
|
|
"version": 2,
|
|
|
"filter_version": FIREWALL_FILTER_COMPILER_VERSION,
|
|
|
"vm_id": canonical_id,
|
|
|
"enabled": requested["enabled"],
|
|
|
"revision": requested["revision"],
|
|
|
"rules": result["rules"],
|
|
|
"policy_digest": result["policy_digest"],
|
|
|
"filter_digest": result["filter_digest"],
|
|
|
"platform_policy_snapshot": platform_policy,
|
|
|
"platform_policy_digest": platform_policy["digest"],
|
|
|
"applied_at": int(time.time()),
|
|
|
}
|
|
|
try:
|
|
|
save_firewall_record(profile, canonical_id, record)
|
|
|
except Exception as exc:
|
|
|
LOG.exception(
|
|
|
"Firewall applied but local evidence could not be saved: uuid=%s",
|
|
|
canonical_id,
|
|
|
)
|
|
|
raise VMError(
|
|
|
"防火墙已下发但引擎记录未确认,请查询原操作", 503,
|
|
|
error_code="firewall_result_uncertain",
|
|
|
) from exc
|
|
|
LOG.info(
|
|
|
"vm firewall applied: uuid=%s revision=%s enabled=%s actor=%s",
|
|
|
canonical_id, body.revision, body.enabled, _session.username,
|
|
|
)
|
|
|
return {
|
|
|
"enabled": record["enabled"],
|
|
|
"revision": record["revision"],
|
|
|
"policy_digest": record["policy_digest"],
|
|
|
"filter_digest": record["filter_digest"],
|
|
|
"filter_version": record["filter_version"],
|
|
|
"rules": record["rules"],
|
|
|
"sync_state": "active",
|
|
|
}
|
|
|
|
|
|
def run_snapshot_job(job_id: str, profile: dict) -> None:
|
|
|
"""Run one host-local snapshot mutation outside the HTTP request."""
|
|
|
job = None
|
|
|
try:
|
|
|
with snapshot_lock:
|
|
|
job = snapshot_jobs[job_id]
|
|
|
if job.get("cancelled_before_start") is True:
|
|
|
try:
|
|
|
finish_cancelled_snapshot_job(job_id, job)
|
|
|
except Exception:
|
|
|
LOG.exception(
|
|
|
"Could not finish cancelled snapshot in runner: %s",
|
|
|
job_id,
|
|
|
)
|
|
|
snapshot_futures.pop(job_id, None)
|
|
|
return
|
|
|
if job.get("status") != "queued":
|
|
|
# A submitted callable may only cross the libvirt boundary
|
|
|
# after this queued -> running transition. Terminal or
|
|
|
# otherwise claimed jobs are never replayed here.
|
|
|
if job.get("status") in {
|
|
|
"failed", "succeeded", "completed", "uncertain"}:
|
|
|
pending_snapshot_jobs.discard(
|
|
|
(host_scope(profile), job.get("vm_id")))
|
|
|
snapshot_futures.pop(job_id, None)
|
|
|
return
|
|
|
job["status"] = "running"
|
|
|
job["phase"] = {
|
|
|
"create": "正在建立在线快照时间点",
|
|
|
"restore": "正在构建快照恢复系统盘",
|
|
|
"delete": "正在安全删除快照",
|
|
|
}[job["action"]]
|
|
|
job["started_at"] = int(time.time())
|
|
|
save_job("snapshot_job", job_id, job)
|
|
|
except Exception:
|
|
|
LOG.exception("Could not start snapshot job safely: %s", job_id)
|
|
|
rollback_complete = False
|
|
|
try:
|
|
|
if job is None:
|
|
|
raise KeyError(job_id)
|
|
|
record = get_snapshot_record(job["vm_id"], job["snapshot_key"])
|
|
|
if job["action"] == "create":
|
|
|
delete_snapshot_record(job["vm_id"], job["snapshot_key"])
|
|
|
else:
|
|
|
record["state"] = "ready"
|
|
|
save_snapshot_record(record)
|
|
|
rollback_complete = True
|
|
|
except Exception:
|
|
|
LOG.exception("Could not roll back unstarted snapshot job: %s", job_id)
|
|
|
with snapshot_lock:
|
|
|
current = snapshot_jobs.get(job_id)
|
|
|
if current is not None:
|
|
|
current.update(
|
|
|
status=("failed" if rollback_complete else "uncertain"),
|
|
|
phase=("快照任务未启动" if rollback_complete
|
|
|
else "快照登记回滚结果待核查"),
|
|
|
error=(
|
|
|
"快照任务状态无法持久化,尚未调用虚拟化操作"
|
|
|
if rollback_complete else
|
|
|
"快照任务尚未调用虚拟化操作,但登记状态回滚无法确认"
|
|
|
),
|
|
|
finished_at=int(time.time()),
|
|
|
)
|
|
|
try:
|
|
|
save_job("snapshot_job", job_id, current)
|
|
|
except DatabaseError:
|
|
|
LOG.exception("Could not persist unstarted snapshot failure: %s", job_id)
|
|
|
if isinstance(job, dict) and isinstance(job.get("vm_id"), str):
|
|
|
pending_snapshot_jobs.discard(
|
|
|
(host_scope(profile), job["vm_id"]))
|
|
|
snapshot_futures.pop(job_id, None)
|
|
|
return
|
|
|
|
|
|
def report_progress(phase: str) -> None:
|
|
|
labels = {
|
|
|
"freezing": "正在通过 QEMU Guest Agent 冻结访客文件系统",
|
|
|
"thawed": "访客文件系统已确认解冻",
|
|
|
"copying": "正在复制在线快照数据",
|
|
|
"verifying": "正在验证独立快照卷",
|
|
|
"cloning": "正在构建快照恢复系统盘",
|
|
|
"shutting_down": "正在正常关闭虚拟机以切换系统盘",
|
|
|
"switching": "正在切换快照恢复系统盘",
|
|
|
"starting": "正在从快照恢复系统盘启动",
|
|
|
}
|
|
|
label = labels.get(phase)
|
|
|
if label is None:
|
|
|
return
|
|
|
with snapshot_lock:
|
|
|
current = snapshot_jobs.get(job_id)
|
|
|
if current is not None and current.get("status") == "running":
|
|
|
current["phase"] = label
|
|
|
if phase == "freezing":
|
|
|
current["guest_freeze_state"] = "attempted"
|
|
|
elif phase == "thawed":
|
|
|
current["guest_freeze_state"] = "thawed"
|
|
|
save_job("snapshot_job", job_id, current)
|
|
|
|
|
|
action = job["action"]
|
|
|
vm_id = job["vm_id"]
|
|
|
snapshot_key = job["snapshot_key"]
|
|
|
uncertain = False
|
|
|
physical_committed = False
|
|
|
committed_record = None
|
|
|
try:
|
|
|
service = new_service(profile)
|
|
|
if action == "create":
|
|
|
result = service.create_live_snapshot(
|
|
|
vm_id, snapshot_key,
|
|
|
consistency=job["consistency"], progress=report_progress)
|
|
|
physical_committed = True
|
|
|
verified_capacity = result.get("size_bytes")
|
|
|
verified_identity = result.get("volume_identity")
|
|
|
if (isinstance(verified_capacity, bool)
|
|
|
or not isinstance(verified_capacity, int)
|
|
|
or verified_capacity <= 0
|
|
|
or not isinstance(verified_identity, str)
|
|
|
or re.fullmatch(r"[0-9a-f]{64}", verified_identity) is None):
|
|
|
raise VMSnapshotUncertainError(
|
|
|
"快照复制完成,但卷验证标记无效")
|
|
|
with snapshot_lock:
|
|
|
current = snapshot_jobs[job_id]
|
|
|
current["backup_verified_capacity"] = verified_capacity
|
|
|
current["backup_verified_identity"] = verified_identity
|
|
|
save_job("snapshot_job", job_id, current)
|
|
|
try:
|
|
|
service.release_live_snapshot_completed_evidence(vm_id)
|
|
|
except Exception:
|
|
|
# The durable marker is now authoritative. Failure to
|
|
|
# prune retained libvirt stats must not erase success.
|
|
|
LOG.exception(
|
|
|
"Could not release retained snapshot evidence: job=%s",
|
|
|
job_id,
|
|
|
)
|
|
|
record = get_snapshot_record(vm_id, snapshot_key)
|
|
|
committed_record = record
|
|
|
if result.get("volume_name") != record["volume_name"]:
|
|
|
raise VMSnapshotUncertainError(
|
|
|
"快照复制完成,但存储卷身份与登记记录不一致")
|
|
|
now = int(time.time())
|
|
|
record.update(
|
|
|
state="ready",
|
|
|
completed_at=now,
|
|
|
size_bytes=int(result["size_bytes"]),
|
|
|
)
|
|
|
save_snapshot_record(record)
|
|
|
public_result = {
|
|
|
"snapshot_key": snapshot_key,
|
|
|
"created": True,
|
|
|
"consistency": record["consistency"],
|
|
|
"created_at": record["created_at"],
|
|
|
"size_bytes": record["size_bytes"],
|
|
|
"power_state": result.get("power_state", "running"),
|
|
|
}
|
|
|
elif action == "restore":
|
|
|
record = get_snapshot_record(vm_id, snapshot_key)
|
|
|
committed_record = record
|
|
|
plan = service.snapshot_restore_plan(
|
|
|
vm_id, job["operation_id"])
|
|
|
with snapshot_lock:
|
|
|
current = snapshot_jobs[job_id]
|
|
|
current["restore_old_volume"] = plan["old_volume_name"]
|
|
|
current["restore_replacement_volume"] = (
|
|
|
plan["replacement_volume_name"])
|
|
|
current["restore_was_running"] = plan["was_running"]
|
|
|
save_job("snapshot_job", job_id, current)
|
|
|
result = service.restore_live_snapshot(
|
|
|
vm_id, snapshot_key, record["volume_name"],
|
|
|
job["operation_id"],
|
|
|
expected_old_volume=plan["old_volume_name"],
|
|
|
progress=report_progress)
|
|
|
physical_committed = True
|
|
|
now = int(time.time())
|
|
|
record.update(state="ready", last_restored_at=now)
|
|
|
save_snapshot_record(record)
|
|
|
public_result = {
|
|
|
"snapshot_key": snapshot_key,
|
|
|
"restored": True,
|
|
|
"consistency": record["consistency"],
|
|
|
"power_state": result.get("power_state", "stopped"),
|
|
|
}
|
|
|
if isinstance(result.get("warning"), str):
|
|
|
public_result["warning"] = result["warning"]
|
|
|
else:
|
|
|
record = get_snapshot_record(vm_id, snapshot_key)
|
|
|
committed_record = record
|
|
|
service.delete_snapshot_volume(
|
|
|
vm_id, snapshot_key, record["volume_name"])
|
|
|
physical_committed = True
|
|
|
# Physical deletion commits before the encrypted registry row
|
|
|
# is removed. A registry failure therefore remains uncertain,
|
|
|
# never a retryable failed DELETE.
|
|
|
delete_snapshot_record(vm_id, snapshot_key)
|
|
|
public_result = {
|
|
|
"snapshot_key": snapshot_key,
|
|
|
"deleted": True,
|
|
|
"consistency": record["consistency"],
|
|
|
}
|
|
|
with snapshot_lock:
|
|
|
current = snapshot_jobs[job_id]
|
|
|
current.update(
|
|
|
status="succeeded",
|
|
|
phase={
|
|
|
"create": "在线快照创建完成",
|
|
|
"restore": "快照恢复完成",
|
|
|
"delete": "快照删除完成",
|
|
|
}[action],
|
|
|
result=public_result,
|
|
|
finished_at=int(time.time()),
|
|
|
)
|
|
|
save_job("snapshot_job", job_id, current)
|
|
|
LOG.info(
|
|
|
"vm snapshot %s succeeded: job=%s uuid=%s snapshot=%s",
|
|
|
action, job_id, vm_id, snapshot_key)
|
|
|
except Exception as exc:
|
|
|
uncertain = (physical_committed
|
|
|
or isinstance(exc, VMSnapshotUncertainError))
|
|
|
if not isinstance(exc, (VMError, DatabaseError, ValueError)):
|
|
|
uncertain = True
|
|
|
LOG.exception(
|
|
|
"VM snapshot %s failed unexpectedly: job=%s uuid=%s",
|
|
|
action, job_id, vm_id)
|
|
|
error = (str(exc) if isinstance(exc, (VMError, DatabaseError, ValueError))
|
|
|
else "快照任务出现未确认错误,请管理员核查引擎日志和宿主机存储")
|
|
|
error_code = (exc.error_code if isinstance(exc, VMError)
|
|
|
and exc.error_code in SNAPSHOT_PUBLIC_ERROR_CODES
|
|
|
else None)
|
|
|
try:
|
|
|
record = (committed_record if committed_record is not None
|
|
|
else get_snapshot_record(vm_id, snapshot_key))
|
|
|
if uncertain:
|
|
|
record["state"] = "uncertain"
|
|
|
save_snapshot_record(record)
|
|
|
elif action == "create":
|
|
|
delete_snapshot_record(vm_id, snapshot_key)
|
|
|
else:
|
|
|
record["state"] = "ready"
|
|
|
save_snapshot_record(record)
|
|
|
except Exception:
|
|
|
uncertain = True
|
|
|
LOG.exception("Could not reconcile snapshot record after job failure: %s", job_id)
|
|
|
with snapshot_lock:
|
|
|
current = snapshot_jobs[job_id]
|
|
|
current.update(
|
|
|
status="uncertain" if uncertain else "failed",
|
|
|
phase="快照结果待核查" if uncertain else "快照任务失败",
|
|
|
error=("快照结果不确定,不能重复提交;请管理员核查宿主机存储和虚拟机定义"
|
|
|
if uncertain else error),
|
|
|
finished_at=int(time.time()),
|
|
|
)
|
|
|
if error_code is not None:
|
|
|
current["error_code"] = error_code
|
|
|
try:
|
|
|
save_job("snapshot_job", job_id, current)
|
|
|
except DatabaseError:
|
|
|
LOG.exception("Could not persist snapshot job failure: %s", job_id)
|
|
|
finally:
|
|
|
with snapshot_lock:
|
|
|
pending_snapshot_jobs.discard((host_scope(profile), vm_id))
|
|
|
snapshot_futures.pop(job_id, None)
|
|
|
|
|
|
def reconcile_snapshot_job(job_id: str) -> dict:
|
|
|
"""Safely converge one uncertain job from read-only host evidence."""
|
|
|
with snapshot_lock:
|
|
|
current = snapshot_jobs.get(job_id)
|
|
|
if current is None and data_store is not None:
|
|
|
raw = data_store.get("snapshot_job", job_id)
|
|
|
if raw is not None:
|
|
|
try:
|
|
|
persisted = json.loads(raw)
|
|
|
except (TypeError, ValueError):
|
|
|
persisted = None
|
|
|
if isinstance(persisted, dict):
|
|
|
with snapshot_lock:
|
|
|
current = snapshot_jobs.setdefault(job_id, persisted)
|
|
|
with snapshot_lock:
|
|
|
current = snapshot_jobs.get(job_id)
|
|
|
if current is None:
|
|
|
raise VMError("快照任务不存在", 404)
|
|
|
if current.get("status") != "uncertain":
|
|
|
return dict(current)
|
|
|
job = dict(current)
|
|
|
profile = manager.current()
|
|
|
if profile is None or profile.get("key_locked"):
|
|
|
return job
|
|
|
try:
|
|
|
record = None
|
|
|
try:
|
|
|
record = get_snapshot_record(
|
|
|
job["vm_id"], job["snapshot_key"])
|
|
|
except VMError as exc:
|
|
|
if job.get("action") != "delete" or exc.status != 404:
|
|
|
raise
|
|
|
service = new_service(profile)
|
|
|
if (job.get("action") == "create"
|
|
|
and job.get("consistency") == "filesystem"
|
|
|
and job.get("guest_freeze_state") == "attempted"):
|
|
|
service.thaw_guest_filesystems(job["vm_id"])
|
|
|
with snapshot_lock:
|
|
|
current = snapshot_jobs.get(job_id)
|
|
|
if current is None or current.get("status") != "uncertain":
|
|
|
return dict(current or job)
|
|
|
current["guest_freeze_state"] = "recovered"
|
|
|
current["freeze_recovery"] = "confirmed"
|
|
|
current["freeze_recovered_at"] = int(time.time())
|
|
|
save_job("snapshot_job", job_id, current)
|
|
|
job = dict(current)
|
|
|
if job["action"] == "create":
|
|
|
if record is None:
|
|
|
return job
|
|
|
evidence = service.reconcile_live_snapshot_create(
|
|
|
job["vm_id"], job["snapshot_key"],
|
|
|
record["volume_name"],
|
|
|
verified_capacity=job.get("backup_verified_capacity"),
|
|
|
verified_identity=job.get("backup_verified_identity"),
|
|
|
)
|
|
|
elif job["action"] == "delete":
|
|
|
snapshot_volume = (
|
|
|
record["volume_name"] if record is not None
|
|
|
else snapshot_volume_name(
|
|
|
job["vm_id"], job["snapshot_key"])
|
|
|
)
|
|
|
evidence = service.reconcile_live_snapshot_delete(
|
|
|
job["vm_id"], job["snapshot_key"],
|
|
|
snapshot_volume)
|
|
|
else:
|
|
|
if record is None:
|
|
|
return job
|
|
|
old_volume = job.get("restore_old_volume")
|
|
|
was_running = job.get("restore_was_running")
|
|
|
replacement_volume = job.get("restore_replacement_volume")
|
|
|
if (old_volume is None and was_running is None
|
|
|
and replacement_volume is None):
|
|
|
# The runner persists all three plan fields atomically
|
|
|
# before calling libvirt. Their complete absence proves
|
|
|
# restore never crossed its physical mutation boundary.
|
|
|
evidence = {
|
|
|
"outcome": "failed",
|
|
|
"snapshot_key": job["snapshot_key"],
|
|
|
}
|
|
|
elif (not isinstance(old_volume, str)
|
|
|
or not isinstance(was_running, bool)
|
|
|
or not isinstance(replacement_volume, str)):
|
|
|
return job
|
|
|
else:
|
|
|
evidence = service.reconcile_live_snapshot_restore(
|
|
|
job["vm_id"], job["snapshot_key"],
|
|
|
record["volume_name"], job["operation_id"], old_volume,
|
|
|
was_running)
|
|
|
except (VMError, DatabaseError, KeyError, ValueError):
|
|
|
# An unavailable host, damaged evidence, or missing private
|
|
|
# restore tombstone is not permission to guess a terminal result.
|
|
|
return job
|
|
|
outcome = evidence.get("outcome")
|
|
|
if outcome not in {"succeeded", "failed"}:
|
|
|
return job
|
|
|
now = int(time.time())
|
|
|
if outcome == "succeeded":
|
|
|
if job["action"] == "create":
|
|
|
record.update(
|
|
|
state="ready",
|
|
|
completed_at=now,
|
|
|
size_bytes=int(evidence["size_bytes"]),
|
|
|
)
|
|
|
save_snapshot_record(record)
|
|
|
public_result = {
|
|
|
"snapshot_key": job["snapshot_key"],
|
|
|
"created": True,
|
|
|
"consistency": record["consistency"],
|
|
|
"created_at": record["created_at"],
|
|
|
"size_bytes": record["size_bytes"],
|
|
|
"power_state": evidence.get("power_state", "stopped"),
|
|
|
}
|
|
|
elif job["action"] == "restore":
|
|
|
record.update(state="ready", last_restored_at=now)
|
|
|
save_snapshot_record(record)
|
|
|
public_result = {
|
|
|
"snapshot_key": job["snapshot_key"],
|
|
|
"restored": True,
|
|
|
"consistency": record["consistency"],
|
|
|
"power_state": evidence.get("power_state", "stopped"),
|
|
|
}
|
|
|
else:
|
|
|
if record is not None:
|
|
|
delete_snapshot_record(job["vm_id"], job["snapshot_key"])
|
|
|
public_result = {
|
|
|
"snapshot_key": job["snapshot_key"],
|
|
|
"deleted": True,
|
|
|
"consistency": job["consistency"],
|
|
|
}
|
|
|
else:
|
|
|
if record is None:
|
|
|
return job
|
|
|
if job["action"] == "create":
|
|
|
# Reconciliation proved that no usable recovery point was
|
|
|
# created. Preserve the durable job audit, but remove the
|
|
|
# hidden catalog row so it cannot accumulate outside the
|
|
|
# two-snapshot UI and later block VM expiry deletion.
|
|
|
delete_snapshot_record(job["vm_id"], job["snapshot_key"])
|
|
|
else:
|
|
|
record["state"] = "ready"
|
|
|
save_snapshot_record(record)
|
|
|
public_result = None
|
|
|
with snapshot_lock:
|
|
|
current = snapshot_jobs.get(job_id)
|
|
|
if current is None or current.get("status") != "uncertain":
|
|
|
return dict(current or job)
|
|
|
current.update(
|
|
|
status=outcome,
|
|
|
phase=("快照自动核对完成" if outcome == "succeeded"
|
|
|
else "快照自动核对确认未生效"),
|
|
|
finished_at=now,
|
|
|
)
|
|
|
if public_result is not None:
|
|
|
current["result"] = public_result
|
|
|
current.pop("error", None)
|
|
|
current.pop("error_code", None)
|
|
|
else:
|
|
|
current.pop("result", None)
|
|
|
current["error"] = "快照操作经宿主机证据确认未生效"
|
|
|
current.pop("error_code", None)
|
|
|
save_job("snapshot_job", job_id, current)
|
|
|
saved = dict(current)
|
|
|
if job["action"] == "create":
|
|
|
try:
|
|
|
service.release_live_snapshot_completed_evidence(job["vm_id"])
|
|
|
except Exception:
|
|
|
LOG.exception(
|
|
|
"Could not release reconciled snapshot evidence: job=%s",
|
|
|
job_id,
|
|
|
)
|
|
|
return saved
|
|
|
|
|
|
app.state.reconcile_snapshot_job = reconcile_snapshot_job
|
|
|
|
|
|
def snapshot_job_state(job_id: str) -> dict | None:
|
|
|
with snapshot_lock:
|
|
|
current = snapshot_jobs.get(job_id)
|
|
|
if (current is not None
|
|
|
and current.get("status") == "queued"
|
|
|
and current.get("cancelled_before_start") is True):
|
|
|
try:
|
|
|
current = finish_cancelled_snapshot_job(job_id, current)
|
|
|
except Exception:
|
|
|
LOG.exception(
|
|
|
"Could not finish cancelled snapshot during poll: %s",
|
|
|
job_id,
|
|
|
)
|
|
|
return dict(current) if current is not None else None
|
|
|
|
|
|
app.state.snapshot_job_state = snapshot_job_state
|
|
|
|
|
|
def queue_snapshot_job(
|
|
|
*, profile: dict, owner: str, vm_id: str, snapshot_key: str,
|
|
|
action: Literal["create", "restore", "delete"], consistency: str,
|
|
|
) -> dict:
|
|
|
job_id = str(uuid.uuid4())
|
|
|
delegated = TRUSTED_ACTOR.get() if internal_mode else None
|
|
|
operation_id = (
|
|
|
delegated.get("operation_id")
|
|
|
if isinstance(delegated, dict) and delegated.get("operation_id")
|
|
|
else job_id
|
|
|
)
|
|
|
job = {
|
|
|
"job_id": job_id,
|
|
|
"operation_id": operation_id,
|
|
|
"scope": host_scope(profile),
|
|
|
"owner": owner,
|
|
|
"vm_id": vm_id,
|
|
|
"snapshot_key": snapshot_key,
|
|
|
"action": action,
|
|
|
"consistency": consistency,
|
|
|
"status": "queued",
|
|
|
"phase": "等待快照任务开始",
|
|
|
"created_at": int(time.time()),
|
|
|
}
|
|
|
# Do not publish an in-memory queued job until its durable identity
|
|
|
# exists. Otherwise an initial database failure leaves a ghost with no
|
|
|
# Future that can permanently block fail-safe network isolation.
|
|
|
save_job("snapshot_job", job_id, job)
|
|
|
snapshot_jobs[job_id] = job
|
|
|
pending_snapshot_jobs.add((job["scope"], vm_id))
|
|
|
try:
|
|
|
future = snapshot_executor.submit(
|
|
|
run_snapshot_job, job_id, profile)
|
|
|
snapshot_futures[job_id] = future
|
|
|
except RuntimeError as exc:
|
|
|
pending_snapshot_jobs.discard((job["scope"], vm_id))
|
|
|
terminal = {
|
|
|
**job,
|
|
|
"status": "failed",
|
|
|
"phase": "快照任务未进入执行器",
|
|
|
"error": "快照任务在开始前失败,未调用虚拟化操作",
|
|
|
"failed_before_start": True,
|
|
|
"finished_at": int(time.time()),
|
|
|
}
|
|
|
snapshot_jobs[job_id] = terminal
|
|
|
try:
|
|
|
save_job("snapshot_job", job_id, terminal)
|
|
|
except DatabaseError:
|
|
|
# Keep the in-memory terminal proof. Operation polling compares
|
|
|
# it with the durable queued row and retries persistence; the
|
|
|
# original mutation is never submitted or replayed.
|
|
|
LOG.exception(
|
|
|
"Could not persist pre-start snapshot submit failure: %s",
|
|
|
job_id,
|
|
|
)
|
|
|
raise VMError("引擎正在停止,无法开始快照任务", 503) from exc
|
|
|
return {"job_id": job_id, "status": "queued"}
|
|
|
|
|
|
@app.get("/api/v1/vms/{vm_id}/snapshots")
|
|
|
def list_vm_snapshots(
|
|
|
vm_id: str, _session: Session = Depends(require_approved),
|
|
|
):
|
|
|
profile = current_profile()
|
|
|
canonical_id = require_vm_access(_session, profile, vm_id)
|
|
|
return {"items": [
|
|
|
public_snapshot_record(record)
|
|
|
for record in list_snapshot_records(canonical_id)
|
|
|
], "limit": 2}
|
|
|
|
|
|
@app.get("/api/v1/vms/{vm_id}/snapshots/{snapshot_key}")
|
|
|
def get_vm_snapshot(
|
|
|
vm_id: str, snapshot_key: str,
|
|
|
_session: Session = Depends(require_approved),
|
|
|
):
|
|
|
profile = current_profile()
|
|
|
canonical_id = require_vm_access(_session, profile, vm_id)
|
|
|
return public_snapshot_record(
|
|
|
get_snapshot_record(canonical_id, snapshot_key))
|
|
|
|
|
|
@app.post("/api/v1/vms/{vm_id}/snapshots", status_code=202)
|
|
|
def create_vm_snapshot(
|
|
|
vm_id: str, body: CreateSnapshotBody,
|
|
|
_session: Session = Depends(require_approved_write),
|
|
|
):
|
|
|
preflight_profile = current_profile()
|
|
|
preflight_id = require_vm_access(
|
|
|
_session, preflight_profile, vm_id)
|
|
|
forbid_snapshot_suspend_barrier(
|
|
|
preflight_profile, preflight_id, "创建快照")
|
|
|
with reinstall_lock, rescue_lock, snapshot_lock:
|
|
|
forbid_long_job_during_update()
|
|
|
profile = current_profile()
|
|
|
canonical_id = require_vm_access(_session, profile, vm_id)
|
|
|
scope = host_scope(profile)
|
|
|
forbid_pending_snapshot(profile, canonical_id, "创建快照")
|
|
|
if ((template_scope(profile), canonical_id) in pending_reinstalls
|
|
|
or (scope, canonical_id) in pending_rescues):
|
|
|
raise VMError("虚拟机正在执行重装或救援任务,不能创建快照", 409)
|
|
|
service = new_service(profile)
|
|
|
vm = service.get_vm(canonical_id)
|
|
|
if (not vm.get("managed")
|
|
|
or vm.get("os_source") != "cloud_template"):
|
|
|
raise VMError("只有面板创建的云镜像虚拟机支持本地快照", 409)
|
|
|
if (vm.get("rescue_mode") or vm.get("rescue_staged")):
|
|
|
raise VMError("虚拟机处于救援模式,不能创建快照", 409)
|
|
|
records = list_snapshot_records(canonical_id)
|
|
|
if len([row for row in records if row["state"] != "failed"]) >= 2:
|
|
|
raise VMError("每台虚拟机最多保留两个本地快照", 409)
|
|
|
record = {
|
|
|
"snapshot_key": body.snapshot_key,
|
|
|
"vm_id": canonical_id,
|
|
|
"name": body.name,
|
|
|
"state": "creating",
|
|
|
"consistency": body.consistency_mode,
|
|
|
"volume_name": snapshot_volume_name(
|
|
|
canonical_id, body.snapshot_key),
|
|
|
"created_at": int(time.time()),
|
|
|
}
|
|
|
if not insert_snapshot_record(record):
|
|
|
raise VMError("快照标识已存在,不能重复创建", 409)
|
|
|
try:
|
|
|
queued = queue_snapshot_job(
|
|
|
profile=profile, owner=_session.username,
|
|
|
vm_id=canonical_id, snapshot_key=body.snapshot_key,
|
|
|
action="create", consistency=body.consistency_mode)
|
|
|
except Exception:
|
|
|
delete_snapshot_record(canonical_id, body.snapshot_key)
|
|
|
raise
|
|
|
return queued
|
|
|
|
|
|
@app.post(
|
|
|
"/api/v1/vms/{vm_id}/snapshots/{snapshot_key}/restore",
|
|
|
status_code=202,
|
|
|
)
|
|
|
def restore_vm_snapshot(
|
|
|
vm_id: str, snapshot_key: str, _body: EmptySnapshotBody,
|
|
|
_session: Session = Depends(require_approved_write),
|
|
|
):
|
|
|
preflight_profile = current_profile()
|
|
|
preflight_id = require_vm_access(
|
|
|
_session, preflight_profile, vm_id)
|
|
|
forbid_snapshot_suspend_barrier(
|
|
|
preflight_profile, preflight_id, "恢复快照")
|
|
|
with reinstall_lock, rescue_lock, snapshot_lock:
|
|
|
forbid_long_job_during_update()
|
|
|
profile = current_profile()
|
|
|
canonical_id = require_vm_access(_session, profile, vm_id)
|
|
|
scope = host_scope(profile)
|
|
|
forbid_pending_snapshot(profile, canonical_id, "恢复快照")
|
|
|
if ((template_scope(profile), canonical_id) in pending_reinstalls
|
|
|
or (scope, canonical_id) in pending_rescues):
|
|
|
raise VMError("虚拟机已有快照、重装或救援任务,请等待完成", 409)
|
|
|
record = get_snapshot_record(canonical_id, snapshot_key)
|
|
|
if record["state"] != "ready":
|
|
|
raise VMError("快照当前不可恢复,请刷新状态后重试", 409)
|
|
|
vm = new_service(profile).get_vm(canonical_id)
|
|
|
if vm.get("rescue_mode") or vm.get("rescue_staged"):
|
|
|
raise VMError("虚拟机处于救援模式,不能恢复快照", 409)
|
|
|
record["state"] = "restoring"
|
|
|
save_snapshot_record(record)
|
|
|
try:
|
|
|
queued = queue_snapshot_job(
|
|
|
profile=profile, owner=_session.username,
|
|
|
vm_id=canonical_id, snapshot_key=snapshot_key,
|
|
|
action="restore", consistency=record["consistency"])
|
|
|
except Exception:
|
|
|
record["state"] = "ready"
|
|
|
save_snapshot_record(record)
|
|
|
raise
|
|
|
return queued
|
|
|
|
|
|
@app.delete(
|
|
|
"/api/v1/vms/{vm_id}/snapshots/{snapshot_key}", status_code=202,
|
|
|
)
|
|
|
def delete_vm_snapshot(
|
|
|
vm_id: str, snapshot_key: str, _body: EmptySnapshotBody,
|
|
|
_session: Session = Depends(require_approved_write),
|
|
|
):
|
|
|
preflight_profile = current_profile()
|
|
|
preflight_id = require_vm_access(
|
|
|
_session, preflight_profile, vm_id)
|
|
|
forbid_snapshot_suspend_barrier(
|
|
|
preflight_profile, preflight_id, "删除快照")
|
|
|
with reinstall_lock, rescue_lock, snapshot_lock:
|
|
|
forbid_long_job_during_update()
|
|
|
profile = current_profile()
|
|
|
canonical_id = require_vm_access(_session, profile, vm_id)
|
|
|
scope = host_scope(profile)
|
|
|
forbid_pending_snapshot(profile, canonical_id, "删除快照")
|
|
|
if ((template_scope(profile), canonical_id) in pending_reinstalls
|
|
|
or (scope, canonical_id) in pending_rescues):
|
|
|
raise VMError("虚拟机已有快照、重装或救援任务,请等待完成", 409)
|
|
|
record = get_snapshot_record(canonical_id, snapshot_key)
|
|
|
if record["state"] != "ready":
|
|
|
raise VMError("快照当前不可删除,请刷新状态后重试", 409)
|
|
|
record["state"] = "deleting"
|
|
|
save_snapshot_record(record)
|
|
|
try:
|
|
|
queued = queue_snapshot_job(
|
|
|
profile=profile, owner=_session.username,
|
|
|
vm_id=canonical_id, snapshot_key=snapshot_key,
|
|
|
action="delete", consistency=record["consistency"])
|
|
|
except Exception:
|
|
|
record["state"] = "ready"
|
|
|
save_snapshot_record(record)
|
|
|
raise
|
|
|
return queued
|
|
|
|
|
|
@app.post("/api/v1/vms/{vm_id}/credential")
|
|
|
def reveal_vm_credential(vm_id: str, _session: Session = Depends(require_approved_write)):
|
|
|
profile = current_profile()
|
|
|
require_vm_access(_session, profile, vm_id)
|
|
|
vm = new_service(profile).get_vm(vm_id)
|
|
|
credential = get_credential(profile, vm["uuid"])
|
|
|
decorated = decorate_vm(vm, credential, profile)
|
|
|
LOG.info("vm credential viewed: uuid=%s admin=%s", vm["uuid"], _session.username)
|
|
|
return {
|
|
|
"login_name": decorated["login_name"],
|
|
|
"auth_mode": decorated["auth_mode"],
|
|
|
"password": credential["password"] if decorated["password_available"] else None,
|
|
|
}
|
|
|
|
|
|
@app.get("/api/v1/vms/{vm_id}/metrics")
|
|
|
def get_vm_metrics(
|
|
|
vm_id: str,
|
|
|
range_: Literal["1h", "24h", "7d", "30d"] = Query("1h", alias="range"),
|
|
|
_session: Session = Depends(require_approved),
|
|
|
):
|
|
|
try:
|
|
|
canonical_id = str(uuid.UUID(vm_id))
|
|
|
except (TypeError, ValueError, AttributeError) as exc:
|
|
|
raise VMError("虚拟机 UUID 无效", 400) from exc
|
|
|
profile = current_profile()
|
|
|
require_vm_access(_session, profile, canonical_id)
|
|
|
# Never reveal another host's retained samples for a reused UUID.
|
|
|
new_service(profile).get_vm(canonical_id)
|
|
|
backend = metric_store if metric_store is not None else data_store
|
|
|
if backend is None:
|
|
|
raise HTTPException(status_code=503, detail="指标数据库尚未就绪")
|
|
|
now = int(time.time())
|
|
|
window = RANGES[range_]
|
|
|
interval = query_interval_seconds(window)
|
|
|
since = max(now - window, now - RETENTION_SECONDS)
|
|
|
try:
|
|
|
scope = host_scope(profile)
|
|
|
except (KeyError, ValueError, TypeError) as exc:
|
|
|
raise HTTPException(status_code=409, detail="宿主机身份信息不完整") from exc
|
|
|
points = backend.query_metric_samples(
|
|
|
scope, canonical_id, since=since, until=now,
|
|
|
interval_seconds=interval, limit=MAX_QUERY_POINTS,
|
|
|
)
|
|
|
return {"vm_id": canonical_id, "range": range_,
|
|
|
"interval_seconds": interval, "points": points}
|
|
|
|
|
|
@app.get("/api/v1/vms/{vm_id}/traffic/daily")
|
|
|
def get_vm_daily_traffic(
|
|
|
vm_id: str,
|
|
|
days: int = Query(30, ge=1, le=30),
|
|
|
_session: Session = Depends(require_approved),
|
|
|
):
|
|
|
try:
|
|
|
canonical_id = str(uuid.UUID(vm_id))
|
|
|
except (TypeError, ValueError, AttributeError) as exc:
|
|
|
raise VMError("虚拟机 UUID 无效", 400) from exc
|
|
|
profile = current_profile()
|
|
|
require_vm_access(_session, profile, canonical_id)
|
|
|
# Match the metrics endpoint's live inventory check before consulting
|
|
|
# retained rows. The owner and host filters below isolate old tenants.
|
|
|
new_service(profile).get_vm(canonical_id)
|
|
|
backend = metric_store if metric_store is not None else data_store
|
|
|
if backend is None:
|
|
|
raise HTTPException(status_code=503, detail="指标数据库尚未就绪")
|
|
|
try:
|
|
|
scope = host_scope(profile)
|
|
|
except (KeyError, ValueError, TypeError) as exc:
|
|
|
raise HTTPException(status_code=409, detail="宿主机身份信息不完整") from exc
|
|
|
owner_id = (state.users.owner_of_vm(scope, canonical_id)
|
|
|
if database_mode else None)
|
|
|
configured_host_id = profile.get("host_id")
|
|
|
metric_host_id = configured_host_id if isinstance(configured_host_id, str) else None
|
|
|
now = int(time.time())
|
|
|
day_seconds = 24 * 60 * 60
|
|
|
today_start = now - now % day_seconds
|
|
|
since = max(0, today_start - (days - 1) * day_seconds)
|
|
|
rows = backend.query_daily_traffic(
|
|
|
scope, canonical_id, host_id=metric_host_id, user_id=owner_id,
|
|
|
since=since, until=now, limit=days,
|
|
|
)
|
|
|
by_day = {int(row["day_start"]): row for row in rows}
|
|
|
points = []
|
|
|
for day_start in range(since, today_start + 1, day_seconds):
|
|
|
row = by_day.get(day_start)
|
|
|
points.append({
|
|
|
"date": time.strftime("%Y-%m-%d", time.gmtime(day_start)),
|
|
|
"day_start": day_start,
|
|
|
"download_bytes": row.get("download_bytes") if row else None,
|
|
|
"upload_bytes": row.get("upload_bytes") if row else None,
|
|
|
"total_bytes": row.get("total_bytes") if row else None,
|
|
|
})
|
|
|
return {"vm_id": canonical_id, "days": days, "timezone": "UTC",
|
|
|
"from": since, "until": now, "points": points}
|
|
|
|
|
|
@app.post("/api/v1/vms", status_code=201)
|
|
|
def create_vm(body: CreateISOBody | CreateCloudBody,
|
|
|
_session: Session = Depends(require_approved_write)):
|
|
|
# Keep the update submission lock until libvirt has finished creating
|
|
|
# the VM. Otherwise an update can restart the web process mid-create.
|
|
|
with reinstall_lock:
|
|
|
forbid_long_job_during_update()
|
|
|
profile = current_profile()
|
|
|
account = account_for_session(_session)
|
|
|
owner_id = account["id"] if database_mode and (internal_mode or not account["admin"]) else None
|
|
|
if isinstance(body, CreateCloudBody):
|
|
|
public_key = body.guest_ssh_public_key.strip()
|
|
|
root_password = (body.root_password.get_secret_value()
|
|
|
if body.root_password is not None else "")
|
|
|
if body.guest_ssh_key_id is not None:
|
|
|
if public_key:
|
|
|
raise HTTPException(
|
|
|
status_code=422,
|
|
|
detail="已保存公钥和手动公钥只能选择一种",
|
|
|
)
|
|
|
saved_key = ssh_key_store().get_for_user(
|
|
|
account["id"], body.guest_ssh_key_id,
|
|
|
)
|
|
|
if saved_key is None:
|
|
|
raise SSHKeyError("公钥不存在", 404)
|
|
|
public_key = saved_key["public_key"]
|
|
|
# A cloud VM always has a recoverable initial root password,
|
|
|
# even when a saved or one-off SSH key was selected.
|
|
|
if not root_password:
|
|
|
root_password = generate_root_password()
|
|
|
password_hash = None
|
|
|
if root_password:
|
|
|
try:
|
|
|
password_hash = hash_root_password(root_password)
|
|
|
except ValueError as exc:
|
|
|
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
|
|
configured_public_pools, _compatibility_mode = public_network_pools(
|
|
|
profile)
|
|
|
auto_public_ip = bool(configured_public_pools)
|
|
|
scheduled_public_ip = bool(body.public_ip_cidr)
|
|
|
selected_public_pool = None
|
|
|
if auto_public_ip:
|
|
|
if body.gateway or body.dns_servers:
|
|
|
raise HTTPException(
|
|
|
status_code=422,
|
|
|
detail="宿主机已配置公网地址池,网关和 DNS 必须使用宿主机配置",
|
|
|
)
|
|
|
if scheduled_public_ip and not internal_mode:
|
|
|
raise HTTPException(
|
|
|
status_code=422,
|
|
|
detail="宿主机已配置公网地址池,创建时不能手动指定 IP",
|
|
|
)
|
|
|
if scheduled_public_ip and not actor().get("can_provision"):
|
|
|
raise HTTPException(
|
|
|
status_code=403,
|
|
|
detail="只有已确认订单的内部调度任务可以指定预留公网 IP",
|
|
|
)
|
|
|
if body.public_pool_key:
|
|
|
selected_public_pool = public_pool_for_key(
|
|
|
profile, body.public_pool_key, active=True)
|
|
|
elif scheduled_public_ip:
|
|
|
# Old schedulers did not send a pool key. Infer it
|
|
|
# only when the exact reserved address has one
|
|
|
# unambiguous active-pool owner.
|
|
|
selected_public_pool = public_pool_for_ip(
|
|
|
profile, body.public_ip_cidr, active=True)
|
|
|
else:
|
|
|
# Preserve the legacy automatic-allocation request
|
|
|
# during the migration window. Pools are normalized
|
|
|
# by key, so every process makes the same choice.
|
|
|
selected_public_pool = next(
|
|
|
(pool for pool in configured_public_pools
|
|
|
if pool["state"] == "active"), None)
|
|
|
if selected_public_pool is None:
|
|
|
raise HTTPException(
|
|
|
status_code=409,
|
|
|
detail="宿主机没有开放新地址分配的公网地址池",
|
|
|
)
|
|
|
if body.network != selected_public_pool["network_selector"]:
|
|
|
raise HTTPException(
|
|
|
status_code=422,
|
|
|
detail="虚拟机网络与所选公网地址池不一致",
|
|
|
)
|
|
|
elif body.public_pool_key:
|
|
|
raise HTTPException(
|
|
|
status_code=422,
|
|
|
detail="宿主机尚未配置可用的公网地址池",
|
|
|
)
|
|
|
if not auto_public_ip and not (
|
|
|
body.public_ip_cidr and body.gateway and body.dns_servers
|
|
|
):
|
|
|
raise HTTPException(
|
|
|
status_code=422,
|
|
|
detail="宿主机尚未配置公网地址池,请手动填写 IP、网关和 DNS",
|
|
|
)
|
|
|
service = new_service(profile)
|
|
|
# The catalog is engine-global. The selected host receives a
|
|
|
# verified replica only on first use, before IP/credential/VM
|
|
|
# state is mutated.
|
|
|
record = ensure_cloud_template_replica(
|
|
|
profile, service, body.template_id)
|
|
|
spec = body.model_dump(exclude_none=True, exclude={
|
|
|
"source", "template_id", "root_password", "guest_ssh_key_id",
|
|
|
"public_pool_key",
|
|
|
})
|
|
|
spec["os_name"] = record.get("name")
|
|
|
spec["guest_ssh_public_key"] = public_key
|
|
|
if password_hash is not None:
|
|
|
spec["root_password_hash"] = password_hash
|
|
|
requested_uuid = actor()["vm_uuid"] if internal_mode else str(uuid.uuid4())
|
|
|
spec["vm_uuid"] = requested_uuid
|
|
|
allocated_ip = ""
|
|
|
owner_assigned = False
|
|
|
try:
|
|
|
if auto_public_ip:
|
|
|
assert selected_public_pool is not None
|
|
|
if scheduled_public_ip:
|
|
|
allocated_ip = claim_scheduled_public_ip(
|
|
|
profile, selected_public_pool,
|
|
|
body.public_ip_cidr,
|
|
|
requested_uuid,
|
|
|
)
|
|
|
else:
|
|
|
allocated_ip = allocate_public_ip(
|
|
|
profile, selected_public_pool,
|
|
|
requested_uuid,
|
|
|
)
|
|
|
spec["public_ip_cidr"] = allocated_ip
|
|
|
spec["public_pool_key"] = selected_public_pool["pool_key"]
|
|
|
spec["gateway"] = selected_public_pool["gateway"]
|
|
|
spec["dns_servers"] = list(
|
|
|
selected_public_pool["dns_servers"])
|
|
|
else:
|
|
|
claim_explicit_public_ip(profile, body.public_ip_cidr, requested_uuid)
|
|
|
allocated_ip = body.public_ip_cidr
|
|
|
if owner_id is not None:
|
|
|
state.users.assign_vm(host_scope(profile), requested_uuid, owner_id)
|
|
|
owner_assigned = True
|
|
|
save_credential(profile, requested_uuid,
|
|
|
("ssh_key_password" if public_key and root_password
|
|
|
else "ssh_key" if public_key else "password"),
|
|
|
f"panel-seed-{requested_uuid}.iso",
|
|
|
root_password or None)
|
|
|
vm = service.create_cloud_vm(spec, record["volume_name"])
|
|
|
except Exception as exc:
|
|
|
# An incomplete libvirt rollback may have left the guest
|
|
|
# behind. Keep its credential in that exceptional case so
|
|
|
# the operator can still recover access if it exists.
|
|
|
rollback_incomplete = (internal_mode or (isinstance(exc, VMError)
|
|
|
and "回滚不完整" in str(exc)))
|
|
|
if not rollback_incomplete:
|
|
|
try:
|
|
|
delete_credential(profile, requested_uuid)
|
|
|
except DatabaseError:
|
|
|
LOG.exception("Could not remove unused VM credential for %s", requested_uuid)
|
|
|
if owner_assigned:
|
|
|
try:
|
|
|
state.users.delete_vm_owner(host_scope(profile), requested_uuid)
|
|
|
except DatabaseError:
|
|
|
LOG.exception("Could not remove unused VM owner for %s", requested_uuid)
|
|
|
if allocated_ip:
|
|
|
try:
|
|
|
release_public_ip_claim(profile, allocated_ip, requested_uuid)
|
|
|
except DatabaseError:
|
|
|
LOG.exception("Could not release unused public IP claim for %s", requested_uuid)
|
|
|
raise
|
|
|
vm = verified_created_vm(profile, vm, requested_uuid)
|
|
|
else:
|
|
|
user_iso_record = None
|
|
|
if body.user_iso_id is not None:
|
|
|
user_scope = user_iso_scope(account["id"])
|
|
|
user_iso_record = user_iso_catalog.get(
|
|
|
user_scope, body.user_iso_id)
|
|
|
spec = body.model_dump(exclude_none=True, exclude={
|
|
|
"source", "iso", "user_iso_id",
|
|
|
})
|
|
|
else:
|
|
|
spec = body.model_dump(exclude_none=True, exclude={
|
|
|
"source", "user_iso_id",
|
|
|
})
|
|
|
if database_mode:
|
|
|
requested_uuid = actor()["vm_uuid"] if internal_mode else str(uuid.uuid4())
|
|
|
spec["vm_uuid"] = requested_uuid
|
|
|
if owner_id is not None:
|
|
|
state.users.assign_vm(host_scope(profile), requested_uuid, owner_id)
|
|
|
try:
|
|
|
service = new_service(profile)
|
|
|
if user_iso_record is not None:
|
|
|
with user_iso_cache_lock:
|
|
|
# Re-read both catalog and host marker only after
|
|
|
# entering the per-host cache critical section.
|
|
|
# Two first-use VM creates therefore cannot both
|
|
|
# act on the same stale marker=false observation.
|
|
|
user_iso_record = user_iso_catalog.get(
|
|
|
user_iso_scope(account["id"]),
|
|
|
user_iso_record["id"],
|
|
|
)
|
|
|
active_user_iso_records = user_iso_catalog.list(
|
|
|
user_iso_scope(account["id"]))
|
|
|
active_user_iso_volumes = {
|
|
|
row["volume_name"]
|
|
|
for row in active_user_iso_records
|
|
|
}
|
|
|
replica_ready = trusted_user_iso_replica(
|
|
|
account["id"], user_iso_record)
|
|
|
# A ready marker proves a prior transfer completed.
|
|
|
# Keep that durable proof while validating/reusing
|
|
|
# the cache; downgrading it before a fallible source
|
|
|
# check would strand an already referenced volume
|
|
|
# after an engine restart.
|
|
|
if not replica_ready:
|
|
|
save_user_iso_replica(
|
|
|
account["id"], user_iso_record,
|
|
|
status="syncing")
|
|
|
if shared_user_iso_runtime is not None:
|
|
|
with shared_user_iso_runtime.source(
|
|
|
account["id"], user_iso_record["id"]) as (
|
|
|
user_iso_record, source_path):
|
|
|
service.cache_user_iso(
|
|
|
source_path, account["id"],
|
|
|
user_iso_record["volume_name"],
|
|
|
active_user_iso_volumes,
|
|
|
trusted_existing=replica_ready)
|
|
|
save_user_iso_replica(
|
|
|
account["id"], user_iso_record,
|
|
|
status="ready")
|
|
|
prune_user_iso_replica_records(
|
|
|
account["id"],
|
|
|
{row["id"] for row in active_user_iso_records},
|
|
|
)
|
|
|
vm = service.create_user_iso_vm(
|
|
|
spec, account["id"],
|
|
|
user_iso_record["volume_name"],
|
|
|
user_iso_record["id"],
|
|
|
user_iso_record["name"],
|
|
|
)
|
|
|
else:
|
|
|
source_path = user_iso_files.path(
|
|
|
account["id"], user_iso_record["id"],
|
|
|
expected_size=user_iso_record["size_bytes"])
|
|
|
service.cache_user_iso(
|
|
|
source_path, account["id"],
|
|
|
user_iso_record["volume_name"],
|
|
|
active_user_iso_volumes,
|
|
|
trusted_existing=replica_ready)
|
|
|
save_user_iso_replica(
|
|
|
account["id"], user_iso_record,
|
|
|
status="ready")
|
|
|
prune_user_iso_replica_records(
|
|
|
account["id"],
|
|
|
{row["id"] for row in active_user_iso_records},
|
|
|
)
|
|
|
vm = service.create_user_iso_vm(
|
|
|
spec, account["id"],
|
|
|
user_iso_record["volume_name"],
|
|
|
user_iso_record["id"],
|
|
|
user_iso_record["name"],
|
|
|
)
|
|
|
else:
|
|
|
vm = service.create_vm(spec)
|
|
|
except Exception as exc:
|
|
|
rollback_incomplete = (internal_mode or (isinstance(exc, VMError)
|
|
|
and "回滚不完整" in str(exc)))
|
|
|
if owner_id is not None and not rollback_incomplete:
|
|
|
state.users.delete_vm_owner(host_scope(profile), requested_uuid)
|
|
|
raise
|
|
|
if database_mode:
|
|
|
vm = verified_created_vm(profile, vm, requested_uuid)
|
|
|
LOG.info("vm created: uuid=%s admin=%s", vm.get("uuid"), _session.username)
|
|
|
credential = get_credential(profile, vm["uuid"]) if isinstance(body, CreateCloudBody) else None
|
|
|
return decorate_vm(vm, credential, profile)
|
|
|
|
|
|
def run_reinstall(
|
|
|
job_id: str, profile: dict, scope: str, vm_id: str,
|
|
|
template_volume_name: str, template_name: str | None, ssh_public_key: str,
|
|
|
root_password_hash: str | None, generated_password: str | None,
|
|
|
credential_password: str | None,
|
|
|
) -> None:
|
|
|
vm_reinstalled = False
|
|
|
def report_progress(phase: str) -> None:
|
|
|
if not isinstance(phase, str):
|
|
|
return
|
|
|
with reinstall_lock:
|
|
|
job = reinstall_jobs.get(job_id)
|
|
|
if job is not None and job["status"] == "running":
|
|
|
job["phase"] = phase[:200]
|
|
|
# ``cleaning_up`` is emitted only after the replacement
|
|
|
# domain has started. Persist that boundary before
|
|
|
# deleting either old volume, so a process restart or a
|
|
|
# later callback failure cannot turn a committed guest
|
|
|
# into a retryable/failed reinstall.
|
|
|
if phase in {"cleaning_up", "cleanup_pending", "result_uncertain", "completed"}:
|
|
|
job["reinstall_committed"] = True
|
|
|
try:
|
|
|
save_job("reinstall_job", job_id, job)
|
|
|
except Exception:
|
|
|
# Progress persistence is advisory. The mutation must
|
|
|
# continue; the final result or the committed marker
|
|
|
# is persisted by the worker when the callback works.
|
|
|
LOG.exception("Could not persist reinstall progress: job=%s", job_id)
|
|
|
|
|
|
try:
|
|
|
with reinstall_lock:
|
|
|
reinstall_jobs[job_id]["status"] = "running"
|
|
|
reinstall_jobs[job_id]["phase"] = "正在检查虚拟机"
|
|
|
save_job("reinstall_job", job_id, reinstall_jobs[job_id])
|
|
|
result = new_service(profile).reinstall_cloud_vm(
|
|
|
vm_id, template_volume_name,
|
|
|
os_name=template_name,
|
|
|
ssh_public_key=ssh_public_key,
|
|
|
root_password_hash=root_password_hash,
|
|
|
progress=report_progress,
|
|
|
)
|
|
|
if not isinstance(result, dict):
|
|
|
raise VMError("重装结果无效,无法确认虚拟机状态", 502)
|
|
|
credential_generation = result.get("credential_generation")
|
|
|
if not isinstance(credential_generation, str) or not credential_generation:
|
|
|
raise VMError("重装结果缺少新的系统配置代号,无法保存凭据", 502)
|
|
|
result_status = result.get("status")
|
|
|
if result_status is not None and not isinstance(result_status, str):
|
|
|
raise VMError("重装结果状态无效,无法确认虚拟机状态", 502)
|
|
|
if isinstance(result_status, str):
|
|
|
result_status = result_status.strip().lower()
|
|
|
warning = result.get("warning")
|
|
|
error_code = result.get("error_code")
|
|
|
if not isinstance(error_code, str):
|
|
|
error_code = None
|
|
|
result_committed = result.get("reinstall_committed") is True
|
|
|
result_cleanup_pending = result.get("cleanup_pending") is True
|
|
|
result_has_warning = isinstance(warning, str) and bool(warning.strip())
|
|
|
result_has_error_code = isinstance(error_code, str) and bool(error_code.strip())
|
|
|
# A service implementation must not return an explicit failed
|
|
|
# result after crossing the replacement boundary without marking
|
|
|
# it. Only an explicit ``reinstall_committed=false`` is proof of
|
|
|
# a pre-commit failure; every other returned non-success result
|
|
|
# stays uncertain so it cannot authorize a destructive replay.
|
|
|
if (result_status == "failed"
|
|
|
and result.get("reinstall_committed") is False):
|
|
|
raise VMError("重装结果报告失败,未确认新系统已启动", 502)
|
|
|
# The service returns only after the replacement definition has
|
|
|
# started. Mark that commit before touching the credential store,
|
|
|
# so a persistence error cannot be mistaken for a pre-commit
|
|
|
# rollback and leave the panel pointing at the old password.
|
|
|
vm_reinstalled = True
|
|
|
save_credential(profile, vm_id,
|
|
|
("ssh_key_password" if ssh_public_key and credential_password
|
|
|
else "ssh_key" if ssh_public_key else "password"),
|
|
|
credential_generation,
|
|
|
credential_password)
|
|
|
success_statuses = {None, "succeeded", "completed", "done"}
|
|
|
partial = (
|
|
|
result_status not in success_statuses
|
|
|
or result_has_warning
|
|
|
or result_cleanup_pending
|
|
|
or result_has_error_code
|
|
|
or (
|
|
|
result_committed and result_status not in success_statuses
|
|
|
)
|
|
|
)
|
|
|
with reinstall_lock:
|
|
|
job = reinstall_jobs[job_id]
|
|
|
if partial:
|
|
|
if error_code not in REINSTALL_PUBLIC_ERROR_CODES:
|
|
|
error_code = (
|
|
|
REINSTALL_CLEANUP_PENDING_ERROR_CODE
|
|
|
if result.get("cleanup_pending") is True
|
|
|
or (isinstance(warning, str)
|
|
|
and warning.strip().startswith("旧存储卷"))
|
|
|
else REINSTALL_RESULT_UNCERTAIN_ERROR_CODE
|
|
|
)
|
|
|
job["status"] = "uncertain"
|
|
|
job["phase"] = (
|
|
|
"重装完成,但旧系统盘清理待核对"
|
|
|
if error_code == REINSTALL_CLEANUP_PENDING_ERROR_CODE
|
|
|
else "重装结果待核对"
|
|
|
)
|
|
|
job.pop("finished_at", None)
|
|
|
job["error_code"] = error_code
|
|
|
job["error"] = (
|
|
|
"重装已完成,但旧系统盘清理待核对"
|
|
|
if error_code == REINSTALL_CLEANUP_PENDING_ERROR_CODE
|
|
|
else "重装结果待核对"
|
|
|
)
|
|
|
job["reinstall_committed"] = True
|
|
|
if result.get("cleanup_pending") is True:
|
|
|
job["cleanup_pending"] = True
|
|
|
vm = result.get("vm")
|
|
|
if isinstance(vm, dict):
|
|
|
# Keep only the bounded observation needed by the
|
|
|
# PHP runtime cache; never persist disk source paths.
|
|
|
job["vm"] = {
|
|
|
key: vm[key] for key in ("uuid", "name", "state")
|
|
|
if key in vm
|
|
|
}
|
|
|
else:
|
|
|
job["status"] = "succeeded"
|
|
|
job["phase"] = "重装完成"
|
|
|
# A progress marker is needed only while the final result
|
|
|
# is unresolved. Do not let it make a normal completion
|
|
|
# look like a partial commit to the PHP worker.
|
|
|
for key in ("reinstall_committed", "cleanup_pending",
|
|
|
"error_code", "error", "warning"):
|
|
|
job.pop(key, None)
|
|
|
if isinstance(warning, str) and warning:
|
|
|
job["warning"] = warning
|
|
|
if generated_password is not None:
|
|
|
# DatabaseStore encrypts this one-time password. Keep it
|
|
|
# until the operator explicitly acknowledges saving it.
|
|
|
job["generated_password"] = generated_password
|
|
|
save_job("reinstall_job", job_id, job)
|
|
|
LOG.info("vm reinstall %s: job=%s uuid=%s",
|
|
|
"awaits cleanup reconciliation" if partial else "succeeded",
|
|
|
job_id, vm_id)
|
|
|
except Exception as exc:
|
|
|
if isinstance(exc, (VMError, TemplateError, ValueError)):
|
|
|
error = str(exc)
|
|
|
else:
|
|
|
LOG.exception("VM reinstall failed: job=%s uuid=%s", job_id, vm_id)
|
|
|
error = "重装系统失败,请查看面板日志"
|
|
|
with reinstall_lock:
|
|
|
job = reinstall_jobs[job_id]
|
|
|
committed = vm_reinstalled or job.get("reinstall_committed") is True
|
|
|
if committed:
|
|
|
# The replacement is live even when credential/job
|
|
|
# persistence failed. Keep the operation uncertain so it
|
|
|
# cannot be presented as success or replayed blindly.
|
|
|
job["status"] = "uncertain"
|
|
|
job["phase"] = "重装结果待核对"
|
|
|
job.pop("finished_at", None)
|
|
|
job["error_code"] = REINSTALL_RESULT_UNCERTAIN_ERROR_CODE
|
|
|
job["error"] = "重装结果待核对;面板未能确认新凭据已保存"
|
|
|
job["reinstall_committed"] = True
|
|
|
job["warning"] = "新系统已启动,但面板无法确认初始密码已保存;请立即保存结果页显示的密码并检查数据库"
|
|
|
if generated_password is not None:
|
|
|
job["generated_password"] = generated_password
|
|
|
else:
|
|
|
job["status"] = "failed"
|
|
|
job["phase"] = "重装失败"
|
|
|
job["error"] = error
|
|
|
job.pop("generated_password", None)
|
|
|
try:
|
|
|
save_job("reinstall_job", job_id, job)
|
|
|
except Exception:
|
|
|
LOG.exception("Could not persist reinstall status: job=%s", job_id)
|
|
|
# A failed terminal write leaves the durable row at its
|
|
|
# previous phase. Keep the in-memory guard fail-closed
|
|
|
# and make one best-effort retry with an explicit
|
|
|
# uncertain state; a later process restart will normalize
|
|
|
# the old running row the same way.
|
|
|
job["status"] = "uncertain"
|
|
|
job["phase"] = "重装结果待核对"
|
|
|
job.pop("finished_at", None)
|
|
|
if committed:
|
|
|
job["error_code"] = REINSTALL_RESULT_UNCERTAIN_ERROR_CODE
|
|
|
job["error"] = "重装结果待核对;面板未能保存任务状态"
|
|
|
else:
|
|
|
job.pop("error_code", None)
|
|
|
job["error"] = "重装结果待核对;面板未能保存任务状态"
|
|
|
job.pop("generated_password", None)
|
|
|
try:
|
|
|
save_job("reinstall_job", job_id, job)
|
|
|
except Exception:
|
|
|
LOG.exception("Could not persist uncertain reinstall status: job=%s", job_id)
|
|
|
if not committed:
|
|
|
LOG.warning("vm reinstall failed: job=%s uuid=%s", job_id, vm_id)
|
|
|
finally:
|
|
|
with reinstall_lock:
|
|
|
job = reinstall_jobs.get(job_id)
|
|
|
if job is None or job.get("status") not in {
|
|
|
"queued", "running", "uncertain"}:
|
|
|
pending_reinstalls.discard((scope, vm_id))
|
|
|
else:
|
|
|
pending_reinstalls.add((scope, vm_id))
|
|
|
|
|
|
@app.post("/api/v1/vms/{vm_id}/reinstall", status_code=202)
|
|
|
def reinstall_vm(vm_id: str, body: ReinstallBody,
|
|
|
_session: Session = Depends(require_approved_write)):
|
|
|
ssh_public_key = body.ssh_public_key.strip()
|
|
|
supplied_password = (body.root_password.get_secret_value()
|
|
|
if body.root_password is not None else "")
|
|
|
if body.ssh_key_id is not None:
|
|
|
if ssh_public_key:
|
|
|
raise HTTPException(
|
|
|
status_code=422,
|
|
|
detail="已保存公钥和手动公钥只能选择一种",
|
|
|
)
|
|
|
account = account_for_session(_session)
|
|
|
saved_key = ssh_key_store().get_for_user(account["id"], body.ssh_key_id)
|
|
|
if saved_key is None:
|
|
|
raise SSHKeyError("公钥不存在", 404)
|
|
|
ssh_public_key = saved_key["public_key"]
|
|
|
generated_password = None
|
|
|
if not supplied_password and not ssh_public_key:
|
|
|
generated_password = generate_root_password()
|
|
|
supplied_password = generated_password
|
|
|
try:
|
|
|
if ssh_public_key:
|
|
|
ssh_public_key, _unused_hash = _validate_guest_auth(ssh_public_key, None)
|
|
|
root_password_hash = (hash_root_password(supplied_password)
|
|
|
if supplied_password else None)
|
|
|
except ValueError as exc:
|
|
|
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
|
|
|
|
|
with reinstall_lock, rescue_lock, snapshot_lock:
|
|
|
forbid_long_job_during_update()
|
|
|
profile = current_profile()
|
|
|
require_vm_access(_session, profile, vm_id)
|
|
|
canonical_id = str(uuid.UUID(vm_id))
|
|
|
forbid_retained_snapshot(profile, canonical_id, "重装系统")
|
|
|
service = new_service(profile)
|
|
|
rescue_record = current_rescue_state(profile, canonical_id, service)
|
|
|
rescue_vm = service.get_vm(canonical_id)
|
|
|
if ((host_scope(profile), canonical_id) in pending_rescues
|
|
|
or rescue_record is not None
|
|
|
or rescue_vm.get("rescue_mode")
|
|
|
or rescue_vm.get("rescue_staged")):
|
|
|
raise VMError("虚拟机处于救援模式或救援任务中,请先退出救援模式", 409)
|
|
|
scope = template_scope(profile)
|
|
|
vm = service.get_vm(vm_id)
|
|
|
if not vm.get("managed") or not vm.get("public_ip_cidr"):
|
|
|
raise VMError("只有面板创建的公网云镜像虚拟机支持自动重装", 409)
|
|
|
if body.confirmation_name != vm["name"]:
|
|
|
raise HTTPException(status_code=422, detail="确认名称与虚拟机名称不一致")
|
|
|
normalized_id = vm["uuid"]
|
|
|
key = (scope, normalized_id)
|
|
|
if key in pending_reinstalls:
|
|
|
raise VMError("该虚拟机正在重装系统", 409)
|
|
|
if any(
|
|
|
job.get("scope") == scope
|
|
|
and job.get("vm_id") == normalized_id
|
|
|
and job.get("status") in {"queued", "running", "uncertain"}
|
|
|
for job in reinstall_jobs.values()):
|
|
|
raise VMError("上一次重装结果待核对,请先由管理员核查虚拟机状态", 409)
|
|
|
# Finish and verify the target-host replica before scheduling any
|
|
|
# task that can shut down the guest or replace its system disk.
|
|
|
record = ensure_cloud_template_replica(
|
|
|
profile, service, body.template_id)
|
|
|
job_id = str(uuid.uuid4())
|
|
|
reinstall_jobs[job_id] = {
|
|
|
"owner": _session.username,
|
|
|
"scope": scope,
|
|
|
"status": "queued",
|
|
|
"phase": "等待重装任务开始",
|
|
|
"vm_id": normalized_id,
|
|
|
"template_id": record["id"],
|
|
|
}
|
|
|
if generated_password is not None:
|
|
|
# The engine record is encrypted at rest by DatabaseStore and
|
|
|
# is the only recovery copy if the process exits after the
|
|
|
# replacement guest starts but before the final result save.
|
|
|
reinstall_jobs[job_id]["generated_password"] = generated_password
|
|
|
save_job("reinstall_job", job_id, reinstall_jobs[job_id])
|
|
|
pending_reinstalls.add(key)
|
|
|
try:
|
|
|
reinstall_executor.submit(
|
|
|
run_reinstall, job_id, profile, scope, normalized_id,
|
|
|
record["volume_name"], record.get("name"), ssh_public_key, root_password_hash,
|
|
|
generated_password, supplied_password or None,
|
|
|
)
|
|
|
except RuntimeError as exc:
|
|
|
pending_reinstalls.discard(key)
|
|
|
reinstall_jobs.pop(job_id, None)
|
|
|
delete_job("reinstall_job", job_id)
|
|
|
raise VMError("面板正在停止,无法开始重装任务", 503) from exc
|
|
|
LOG.info("vm reinstall queued: job=%s uuid=%s admin=%s",
|
|
|
job_id, normalized_id, _session.username)
|
|
|
return {"job_id": job_id, "status": "queued"}
|
|
|
|
|
|
@app.get("/api/v1/reinstall-jobs/{job_id}")
|
|
|
def reinstall_job(job_id: str, _session: Session = Depends(require_approved_write)):
|
|
|
scope = template_scope(current_profile())
|
|
|
with reinstall_lock:
|
|
|
job = reinstall_jobs.get(job_id)
|
|
|
if (job is None or job["scope"] != scope
|
|
|
or job["owner"] != _session.username):
|
|
|
raise VMError("重装任务不存在", 404)
|
|
|
status = public_status(job.get("status"))
|
|
|
result: dict[str, Any] = {"status": status}
|
|
|
for key in ("job_id", "vm_id", "template_id", "operation_id"):
|
|
|
identifier = public_identifier(job.get(key), minimum=1)
|
|
|
if identifier is not None:
|
|
|
result[key] = identifier
|
|
|
if (status == "uncertain"
|
|
|
and job.get("error_code") in REINSTALL_PUBLIC_ERROR_CODES):
|
|
|
result["error_code"] = job["error_code"]
|
|
|
result["phase"] = public_phase(
|
|
|
job.get("phase"), status, public_reinstall_phases, kind="reinstall")
|
|
|
for key in ("created_at", "started_at", "finished_at"):
|
|
|
timestamp = public_timestamp(job.get(key))
|
|
|
if timestamp is not None:
|
|
|
result[key] = timestamp
|
|
|
if "warning" in job:
|
|
|
warning = public_warning(
|
|
|
job.get("warning"), public_reinstall_warnings,
|
|
|
"重装任务已完成,但仍有需要管理员检查的警告")
|
|
|
if warning is not None:
|
|
|
result["warning"] = warning
|
|
|
if "error" in job:
|
|
|
error = canonical_job_text(
|
|
|
job.get("error"), public_reinstall_errors,
|
|
|
"重装系统失败,请查看面板日志")
|
|
|
if error is not None:
|
|
|
result["error"] = error
|
|
|
if status in {"succeeded", "uncertain"} and isinstance(job.get("generated_password"), str):
|
|
|
password = job["generated_password"]
|
|
|
if 1 <= len(password) <= 128 and re.fullmatch(r"[^\x00-\x1f\x7f]+", password):
|
|
|
result["generated_password"] = password
|
|
|
return result
|
|
|
|
|
|
@app.post("/api/v1/reinstall-jobs/{job_id}/ack-password")
|
|
|
def acknowledge_reinstall_password(job_id: str,
|
|
|
_session: Session = Depends(require_approved_write)):
|
|
|
scope = template_scope(current_profile())
|
|
|
with reinstall_lock:
|
|
|
job = reinstall_jobs.get(job_id)
|
|
|
if (job is None or job["scope"] != scope
|
|
|
or job["owner"] != _session.username):
|
|
|
raise VMError("重装任务不存在", 404)
|
|
|
if job["status"] not in {"succeeded", "uncertain"}:
|
|
|
raise VMError("重装任务尚未完成,不能确认密码", 409)
|
|
|
job.pop("generated_password", None)
|
|
|
save_job("reinstall_job", job_id, job)
|
|
|
return {"acknowledged": True}
|
|
|
|
|
|
def run_rescue_job(job_id: str, profile: dict, vm_id: str,
|
|
|
operation: Literal["enter", "exit", "force"],
|
|
|
network_suspended: bool = False) -> None:
|
|
|
vm_scope = host_scope(profile)
|
|
|
key = (vm_scope, vm_id)
|
|
|
completed = False
|
|
|
try:
|
|
|
with rescue_lock:
|
|
|
job = rescue_jobs[job_id]
|
|
|
job["status"] = "running"
|
|
|
job["phase"] = (
|
|
|
"正在预挂载救援 ISO 并等待正常关机" if operation == "enter"
|
|
|
else "正在强制断电并从救援 ISO 冷启动" if operation == "force"
|
|
|
else "正在正常关机并恢复原系统启动配置"
|
|
|
)
|
|
|
save_job("rescue_job", job_id, job)
|
|
|
state_record = load_rescue_state(profile, vm_id)
|
|
|
if state_record is None:
|
|
|
raise VMError("救援恢复记录丢失,已停止操作", 409)
|
|
|
state_record["status"] = "exiting" if operation == "exit" else "running"
|
|
|
save_rescue_state(profile, vm_id, state_record)
|
|
|
service = new_service(profile)
|
|
|
if operation == "enter":
|
|
|
result = service.enter_rescue_vm(vm_id, state_record["volume_name"],
|
|
|
state_record["original_xml"])
|
|
|
elif operation == "force":
|
|
|
result = service.force_start_staged_rescue_vm(
|
|
|
vm_id, state_record["original_xml"])
|
|
|
else:
|
|
|
result = service.exit_rescue_vm(
|
|
|
vm_id, state_record["original_xml"],
|
|
|
network_suspended=network_suspended)
|
|
|
completed = True
|
|
|
staged = (operation == "enter" and isinstance(result, dict)
|
|
|
and result.get("pending_manual_restart") is True)
|
|
|
with rescue_lock:
|
|
|
if operation in {"enter", "force"}:
|
|
|
state_record["status"] = "staged" if staged else "active"
|
|
|
save_rescue_state(profile, vm_id, state_record)
|
|
|
else:
|
|
|
delete_rescue_state(profile, vm_id)
|
|
|
job = rescue_jobs[job_id]
|
|
|
job["status"] = "completed"
|
|
|
job["phase"] = ("救援 ISO 已预挂载,等待手动强制断电后开机" if staged
|
|
|
else "救援系统已启动" if operation in {"enter", "force"}
|
|
|
else "已退出救援模式,原系统已启动")
|
|
|
if staged:
|
|
|
job["pending_manual_restart"] = True
|
|
|
if isinstance(result, dict) and result.get("warning"):
|
|
|
job["warning"] = str(result["warning"])
|
|
|
save_job("rescue_job", job_id, job)
|
|
|
LOG.info("vm rescue %s succeeded: job=%s uuid=%s", operation, job_id, vm_id)
|
|
|
except Exception as exc:
|
|
|
if isinstance(exc, (VMError, TemplateError)):
|
|
|
error = str(exc)
|
|
|
else:
|
|
|
LOG.exception("VM rescue %s failed: job=%s uuid=%s", operation, job_id, vm_id)
|
|
|
error = "紧急救援操作失败,请查看面板和宿主机日志"
|
|
|
with rescue_lock:
|
|
|
try:
|
|
|
state_record = load_rescue_state(profile, vm_id)
|
|
|
if state_record is not None:
|
|
|
# If the original inactive definition survived, the
|
|
|
# failed job cannot strand an otherwise normal VM in
|
|
|
# a database-only "rescue" state.
|
|
|
service = new_service(profile)
|
|
|
actual_vm = service.get_vm(vm_id)
|
|
|
if (not actual_vm.get("rescue_mode")
|
|
|
and rescue_xml_matches(
|
|
|
service.prepare_rescue_vm(
|
|
|
vm_id, state_record["volume_name"]),
|
|
|
state_record["original_xml"],
|
|
|
)):
|
|
|
delete_rescue_state(profile, vm_id)
|
|
|
else:
|
|
|
state_record["status"] = "failed"
|
|
|
save_rescue_state(profile, vm_id, state_record)
|
|
|
except (DatabaseError, VMError, TemplateError):
|
|
|
LOG.exception("Could not save rescue recovery state: job=%s", job_id)
|
|
|
job = rescue_jobs[job_id]
|
|
|
job["status"] = "failed"
|
|
|
job["phase"] = "救援操作失败"
|
|
|
if "正常关机超时" in error or "无法请求虚拟机正常关机" in error:
|
|
|
job["error"] = error
|
|
|
else:
|
|
|
job["error"] = error + ";请确认虚拟机实际运行状态,必要时联系管理员"
|
|
|
if completed:
|
|
|
job["error"] = "虚拟机操作已完成,但数据库状态保存失败;请管理员检查救援记录"
|
|
|
try:
|
|
|
save_job("rescue_job", job_id, job)
|
|
|
except DatabaseError:
|
|
|
LOG.exception("Could not persist rescue job failure: job=%s", job_id)
|
|
|
finally:
|
|
|
with rescue_lock:
|
|
|
pending_rescues.discard(key)
|
|
|
|
|
|
@app.get("/api/v1/vms/{vm_id}/rescue")
|
|
|
def get_vm_rescue(vm_id: str, _session: Session = Depends(require_approved)):
|
|
|
profile = current_profile()
|
|
|
canonical_id = require_vm_access(_session, profile, vm_id)
|
|
|
with rescue_lock:
|
|
|
service = new_service(profile)
|
|
|
vm = service.get_vm(canonical_id)
|
|
|
record = current_rescue_state(profile, vm["uuid"], service)
|
|
|
staged = bool(vm.get("rescue_staged"))
|
|
|
active = bool(vm.get("rescue_mode")) and not staged
|
|
|
if (record is not None
|
|
|
and (host_scope(profile), vm["uuid"]) not in pending_rescues):
|
|
|
if staged and record["status"] in {"queued", "running", "failed"}:
|
|
|
record["status"] = "staged"
|
|
|
save_rescue_state(profile, vm["uuid"], record)
|
|
|
elif active and record["status"] in {"queued", "running", "staged"}:
|
|
|
record["status"] = "active"
|
|
|
save_rescue_state(profile, vm["uuid"], record)
|
|
|
template_name = None
|
|
|
template_id = record["template_id"] if record is not None else None
|
|
|
if record is not None:
|
|
|
try:
|
|
|
template_name = rescue_catalog.get(
|
|
|
rescue_scope(profile), record["template_id"])["name"]
|
|
|
except TemplateError:
|
|
|
pass
|
|
|
return {
|
|
|
"active": active,
|
|
|
"staged": staged,
|
|
|
"power_state": vm.get("state", "unknown"),
|
|
|
"template_id": template_id,
|
|
|
"template_name": template_name,
|
|
|
"status": record["status"] if record is not None else None,
|
|
|
"job_id": record.get("job_id") if record is not None else None,
|
|
|
"can_exit": (active or staged) and record is not None,
|
|
|
}
|
|
|
|
|
|
@app.post("/api/v1/vms/{vm_id}/rescue", status_code=202)
|
|
|
def enter_vm_rescue(vm_id: str, body: EnterRescueBody,
|
|
|
_session: Session = Depends(require_approved_write)):
|
|
|
with reinstall_lock, rescue_lock, snapshot_lock:
|
|
|
forbid_long_job_during_update()
|
|
|
profile = current_profile()
|
|
|
canonical_id = require_vm_access(_session, profile, vm_id)
|
|
|
forbid_pending_snapshot(profile, canonical_id, "进入救援模式")
|
|
|
vm_scope = host_scope(profile)
|
|
|
key = (vm_scope, canonical_id)
|
|
|
if key in pending_rescues or (template_scope(profile), canonical_id) in pending_reinstalls:
|
|
|
raise VMError("该虚拟机已有救援或重装任务,暂时不能重复操作", 409)
|
|
|
service = new_service(profile)
|
|
|
vm = service.get_vm(canonical_id)
|
|
|
if body.confirmation_name != vm["name"]:
|
|
|
raise VMError("确认名称与虚拟机名称不一致", 422)
|
|
|
if vm.get("rescue_mode") or vm.get("rescue_staged"):
|
|
|
raise VMError("虚拟机已预挂载救援 ISO 或处于救援模式,请先退出", 409)
|
|
|
record = rescue_catalog.get(rescue_scope(profile), body.template_id)
|
|
|
original_xml = service.prepare_rescue_vm(canonical_id, record["volume_name"])
|
|
|
prior = load_rescue_state(profile, canonical_id)
|
|
|
if prior is not None:
|
|
|
if not rescue_xml_matches(prior["original_xml"], original_xml):
|
|
|
raise VMError("虚拟机仍有未完成的救援记录,请管理员检查后再操作", 409)
|
|
|
delete_rescue_state(profile, canonical_id)
|
|
|
job_id = str(uuid.uuid4())
|
|
|
state_record = {
|
|
|
"template_id": record["id"], "volume_name": record["volume_name"],
|
|
|
"original_xml": original_xml, "status": "queued", "job_id": job_id,
|
|
|
}
|
|
|
save_rescue_state(profile, canonical_id, state_record)
|
|
|
rescue_jobs[job_id] = {
|
|
|
"scope": vm_scope, "owner": _session.username,
|
|
|
"vm_id": canonical_id, "operation": "enter",
|
|
|
"status": "queued", "phase": "等待救援任务开始",
|
|
|
}
|
|
|
try:
|
|
|
save_job("rescue_job", job_id, rescue_jobs[job_id])
|
|
|
pending_rescues.add(key)
|
|
|
rescue_executor.submit(run_rescue_job, job_id, profile, canonical_id, "enter")
|
|
|
except Exception:
|
|
|
pending_rescues.discard(key)
|
|
|
rescue_jobs.pop(job_id, None)
|
|
|
delete_job("rescue_job", job_id)
|
|
|
delete_rescue_state(profile, canonical_id)
|
|
|
raise
|
|
|
LOG.info("vm rescue enter queued: job=%s uuid=%s user=%s",
|
|
|
job_id, canonical_id, _session.username)
|
|
|
return {"job_id": job_id, "status": "queued"}
|
|
|
|
|
|
@app.post("/api/v1/vms/{vm_id}/rescue/force-start", status_code=202)
|
|
|
def force_start_vm_rescue(vm_id: str, body: ForceRescueBody,
|
|
|
_session: Session = Depends(require_approved_write)):
|
|
|
"""Explicit, confirmed power cycle into an already staged rescue ISO."""
|
|
|
with reinstall_lock, rescue_lock, snapshot_lock:
|
|
|
forbid_long_job_during_update()
|
|
|
profile = current_profile()
|
|
|
canonical_id = require_vm_access(_session, profile, vm_id)
|
|
|
forbid_pending_snapshot(profile, canonical_id, "强制启动救援系统")
|
|
|
vm_scope = host_scope(profile)
|
|
|
key = (vm_scope, canonical_id)
|
|
|
if key in pending_rescues or (template_scope(profile), canonical_id) in pending_reinstalls:
|
|
|
raise VMError("该虚拟机已有救援或重装任务,请等待完成", 409)
|
|
|
record = load_rescue_state(profile, canonical_id)
|
|
|
if record is None:
|
|
|
raise VMError("缺少原始启动配置备份,不能安全强制重启进入救援", 409)
|
|
|
vm = new_service(profile).get_vm(canonical_id)
|
|
|
if body.confirmation_name != vm["name"]:
|
|
|
raise VMError("确认名称与虚拟机名称不一致", 422)
|
|
|
if not vm.get("rescue_staged"):
|
|
|
raise VMError("救援 ISO 尚未预挂载,或虚拟机已经进入救援模式", 409)
|
|
|
job_id = str(uuid.uuid4())
|
|
|
rescue_jobs[job_id] = {
|
|
|
"scope": vm_scope, "owner": _session.username,
|
|
|
"vm_id": canonical_id, "operation": "force",
|
|
|
"status": "queued", "phase": "等待手动确认的救援冷启动",
|
|
|
}
|
|
|
previous_status = record["status"]
|
|
|
previous_job_id = record.get("job_id")
|
|
|
try:
|
|
|
save_job("rescue_job", job_id, rescue_jobs[job_id])
|
|
|
record["status"] = "queued"
|
|
|
record["job_id"] = job_id
|
|
|
save_rescue_state(profile, canonical_id, record)
|
|
|
pending_rescues.add(key)
|
|
|
rescue_executor.submit(run_rescue_job, job_id, profile, canonical_id, "force")
|
|
|
except Exception:
|
|
|
pending_rescues.discard(key)
|
|
|
rescue_jobs.pop(job_id, None)
|
|
|
delete_job("rescue_job", job_id)
|
|
|
record["status"] = previous_status
|
|
|
record["job_id"] = previous_job_id
|
|
|
save_rescue_state(profile, canonical_id, record)
|
|
|
raise
|
|
|
LOG.warning("vm rescue force-start queued: job=%s uuid=%s user=%s",
|
|
|
job_id, canonical_id, _session.username)
|
|
|
return {"job_id": job_id, "status": "queued"}
|
|
|
|
|
|
@app.post("/api/v1/vms/{vm_id}/rescue/exit", status_code=202)
|
|
|
def exit_vm_rescue(vm_id: str, body: ExitRescueBody,
|
|
|
_session: Session = Depends(require_approved_write)):
|
|
|
with reinstall_lock, rescue_lock, snapshot_lock:
|
|
|
forbid_long_job_during_update()
|
|
|
profile = current_profile()
|
|
|
canonical_id = require_vm_access(_session, profile, vm_id)
|
|
|
forbid_pending_snapshot(profile, canonical_id, "退出救援模式")
|
|
|
vm_scope = host_scope(profile)
|
|
|
key = (vm_scope, canonical_id)
|
|
|
if key in pending_rescues or (template_scope(profile), canonical_id) in pending_reinstalls:
|
|
|
raise VMError("该虚拟机已有救援或重装任务,请等待完成", 409)
|
|
|
record = load_rescue_state(profile, canonical_id)
|
|
|
if record is None:
|
|
|
raise VMError("找不到原始启动配置,无法安全退出救援模式", 409)
|
|
|
vm = new_service(profile).get_vm(canonical_id)
|
|
|
if body.confirmation_name != vm["name"]:
|
|
|
raise VMError("确认名称与虚拟机名称不一致", 422)
|
|
|
if not (vm.get("rescue_mode") or vm.get("rescue_staged")):
|
|
|
raise VMError("虚拟机当前并非救援模式,请管理员核对配置", 409)
|
|
|
job_id = str(uuid.uuid4())
|
|
|
rescue_jobs[job_id] = {
|
|
|
"scope": vm_scope, "owner": _session.username,
|
|
|
"vm_id": canonical_id, "operation": "exit",
|
|
|
"status": "queued", "phase": "等待退出救援模式",
|
|
|
}
|
|
|
save_job("rescue_job", job_id, rescue_jobs[job_id])
|
|
|
record["status"] = "exiting"
|
|
|
record["job_id"] = job_id
|
|
|
save_rescue_state(profile, canonical_id, record)
|
|
|
pending_rescues.add(key)
|
|
|
try:
|
|
|
rescue_executor.submit(
|
|
|
run_rescue_job, job_id, profile, canonical_id, "exit",
|
|
|
body.network_suspended)
|
|
|
except RuntimeError as exc:
|
|
|
pending_rescues.discard(key)
|
|
|
record["status"] = "active"
|
|
|
save_rescue_state(profile, canonical_id, record)
|
|
|
rescue_jobs.pop(job_id, None)
|
|
|
delete_job("rescue_job", job_id)
|
|
|
raise VMError("面板正在停止,无法退出救援模式", 503) from exc
|
|
|
LOG.info("vm rescue exit queued: job=%s uuid=%s user=%s",
|
|
|
job_id, canonical_id, _session.username)
|
|
|
return {"job_id": job_id, "status": "queued"}
|
|
|
|
|
|
@app.get("/api/v1/rescue-jobs/{job_id}")
|
|
|
def get_rescue_job(job_id: str, _session: Session = Depends(require_approved)):
|
|
|
profile = current_profile()
|
|
|
vm_scope = host_scope(profile)
|
|
|
with rescue_lock:
|
|
|
job = rescue_jobs.get(job_id)
|
|
|
if (job is None or job["scope"] != vm_scope
|
|
|
or (job["owner"] != _session.username
|
|
|
and not account_for_session(_session)["admin"])):
|
|
|
raise VMError("救援任务不存在", 404)
|
|
|
require_vm_access(_session, profile, job["vm_id"])
|
|
|
status = public_status(job.get("status"))
|
|
|
result: dict[str, Any] = {"status": status}
|
|
|
for key in ("job_id", "operation", "operation_id"):
|
|
|
identifier = public_identifier(job.get(key), minimum=1)
|
|
|
if key == "operation":
|
|
|
identifier = job.get(key) if job.get(key) in {"enter", "force", "exit"} else None
|
|
|
if identifier is not None:
|
|
|
result[key] = identifier
|
|
|
result["phase"] = public_phase(
|
|
|
job.get("phase"), status, public_rescue_phases, kind="rescue")
|
|
|
for key in ("created_at", "started_at", "finished_at"):
|
|
|
timestamp = public_timestamp(job.get(key))
|
|
|
if timestamp is not None:
|
|
|
result[key] = timestamp
|
|
|
if isinstance(job.get("pending_manual_restart"), bool):
|
|
|
result["pending_manual_restart"] = job["pending_manual_restart"]
|
|
|
if "warning" in job:
|
|
|
warning = public_warning(
|
|
|
job.get("warning"), public_rescue_warnings,
|
|
|
"救援任务已完成,但仍有需要管理员检查的警告")
|
|
|
if warning is not None:
|
|
|
result["warning"] = warning
|
|
|
if "error" in job:
|
|
|
error = canonical_job_text(
|
|
|
job.get("error"), public_rescue_errors,
|
|
|
"紧急救援操作失败,请查看面板和宿主机日志")
|
|
|
if error is not None:
|
|
|
result["error"] = error
|
|
|
return result
|
|
|
|
|
|
def apply_private_vm_target(vm_id: str, session: Session,
|
|
|
method: str, label: str, **options) -> dict:
|
|
|
with reinstall_lock, rescue_lock:
|
|
|
forbid_long_job_during_update()
|
|
|
profile = current_profile()
|
|
|
canonical_id = require_vm_access(session, profile, vm_id)
|
|
|
# Network isolation is a fail-safe policy target and cannot remain
|
|
|
# blocked forever by durable snapshot uncertainty. Service stop is
|
|
|
# different: an uncertain restore may later reconcile by restoring
|
|
|
# its recorded running state, so bypassing it here could power a
|
|
|
# purged guest back on without a durable policy tombstone.
|
|
|
fail_safe_suspend = method == "network_suspend"
|
|
|
barrier_key = (host_scope(profile), canonical_id)
|
|
|
barrier_installed = False
|
|
|
with snapshot_lock:
|
|
|
if fail_safe_suspend:
|
|
|
if barrier_key in snapshot_suspend_barriers:
|
|
|
raise VMError(
|
|
|
"虚拟机正在执行到期暂停策略,暂时不能重复暂停网络",
|
|
|
409,
|
|
|
)
|
|
|
snapshot_suspend_barriers.add(barrier_key)
|
|
|
barrier_installed = True
|
|
|
try:
|
|
|
# Only a Future that the executor confirms was still
|
|
|
# pending may be cancelled. Its two-phase tombstone and
|
|
|
# catalog rollback complete inside this same lock.
|
|
|
cancel_queued_snapshot_for_network_suspend(
|
|
|
profile, canonical_id)
|
|
|
forbid_pending_snapshot(
|
|
|
profile, canonical_id, label,
|
|
|
allow_uncertain=True,
|
|
|
ignore_suspend_barrier=True,
|
|
|
)
|
|
|
except Exception:
|
|
|
snapshot_suspend_barriers.discard(barrier_key)
|
|
|
barrier_installed = False
|
|
|
raise
|
|
|
else:
|
|
|
forbid_pending_snapshot(profile, canonical_id, label)
|
|
|
try:
|
|
|
if (template_scope(profile), canonical_id) in pending_reinstalls:
|
|
|
raise VMError(f"虚拟机正在重装系统,暂时不能执行{label}", 409)
|
|
|
if (host_scope(profile), canonical_id) in pending_rescues:
|
|
|
raise VMError(f"虚拟机正在切换救援模式,暂时不能执行{label}", 409)
|
|
|
vm = getattr(new_service(profile), method)(
|
|
|
canonical_id, **options)
|
|
|
finally:
|
|
|
if barrier_installed:
|
|
|
with snapshot_lock:
|
|
|
snapshot_suspend_barriers.discard(barrier_key)
|
|
|
LOG.info("vm target state applied: uuid=%s target=%s actor=%s",
|
|
|
vm.get("uuid"), method, session.username)
|
|
|
return vm
|
|
|
|
|
|
@app.post("/api/v1/vms/{vm_id}/service-suspend")
|
|
|
def service_suspend_vm(
|
|
|
vm_id: str,
|
|
|
_session: Session = Depends(require_internal_service_write),
|
|
|
):
|
|
|
return apply_private_vm_target(
|
|
|
vm_id, _session, "service_suspend", "服务暂停")
|
|
|
|
|
|
@app.post("/api/v1/vms/{vm_id}/service-resume")
|
|
|
def service_resume_vm(
|
|
|
vm_id: str,
|
|
|
body: ServiceResumeBody,
|
|
|
_session: Session = Depends(require_internal_service_write),
|
|
|
):
|
|
|
return apply_private_vm_target(
|
|
|
vm_id, _session, "service_resume", "服务恢复",
|
|
|
start_if_suspended=body.start_if_suspended)
|
|
|
|
|
|
@app.post("/api/v1/vms/{vm_id}/network-suspend")
|
|
|
def network_suspend_vm(
|
|
|
vm_id: str,
|
|
|
_session: Session = Depends(require_internal_service_write),
|
|
|
):
|
|
|
return apply_private_vm_target(
|
|
|
vm_id, _session, "network_suspend", "网络暂停")
|
|
|
|
|
|
@app.post("/api/v1/vms/{vm_id}/network-resume")
|
|
|
def network_resume_vm(
|
|
|
vm_id: str,
|
|
|
_session: Session = Depends(require_internal_service_write),
|
|
|
):
|
|
|
return apply_private_vm_target(
|
|
|
vm_id, _session, "network_resume", "网络恢复")
|
|
|
|
|
|
@app.post("/api/v1/vms/{vm_id}/network")
|
|
|
def change_network_vm(
|
|
|
vm_id: str,
|
|
|
body: ChangeNetworkBody,
|
|
|
request: Request,
|
|
|
_session: Session = Depends(require_internal_service_write),
|
|
|
):
|
|
|
"""Change one managed cloud VM's public IPv4 on the same host.
|
|
|
|
|
|
The engine validates the host pool and holds the new claim before
|
|
|
touching libvirt. A 5xx/transport failure deliberately keeps both IP
|
|
|
claims for reconciliation; only deterministic validation failures
|
|
|
release the new reservation.
|
|
|
"""
|
|
|
with reinstall_lock, rescue_lock, snapshot_lock:
|
|
|
forbid_long_job_during_update()
|
|
|
profile = current_profile()
|
|
|
try:
|
|
|
normalized_id = str(uuid.UUID(vm_id))
|
|
|
except (TypeError, ValueError, AttributeError) as exc:
|
|
|
raise VMError("虚拟机 UUID 无效", 400) from exc
|
|
|
require_vm_access(_session, profile, normalized_id)
|
|
|
forbid_pending_snapshot(profile, normalized_id, "修改网络")
|
|
|
if (template_scope(profile), normalized_id) in pending_reinstalls:
|
|
|
raise VMError("虚拟机正在重装系统,暂时不能修改网络", 409)
|
|
|
if (host_scope(profile), normalized_id) in pending_rescues:
|
|
|
raise VMError("虚拟机正在切换救援模式,请先修改完成后再试", 409)
|
|
|
service = new_service(profile)
|
|
|
vm = service.get_vm(normalized_id)
|
|
|
if (not vm.get("managed") or vm.get("os_source") != "cloud_template"
|
|
|
or not vm.get("public_ip_cidr")):
|
|
|
raise VMError("只有面板创建的公网云镜像虚拟机支持更换 IP", 409)
|
|
|
if not body.confirm_restart:
|
|
|
raise VMError("更换公网 IP 前必须确认会短暂重启虚拟机", 422)
|
|
|
|
|
|
old_ip = str(vm["public_ip_cidr"])
|
|
|
try:
|
|
|
old_pool_key = vm.get("public_pool_key")
|
|
|
# Guests created before pool metadata existed, and guests
|
|
|
# created while the synthetic legacy pool was active, are
|
|
|
# adopted by exact CIDR membership after the native catalog
|
|
|
# is installed.
|
|
|
if (isinstance(old_pool_key, str) and old_pool_key
|
|
|
and old_pool_key != "legacy-default"):
|
|
|
current_pool = public_pool_for_key(
|
|
|
profile, old_pool_key, active=False)
|
|
|
inferred_current_pool = public_pool_for_ip(
|
|
|
profile, old_ip, active=False)
|
|
|
if inferred_current_pool["pool_key"] != current_pool["pool_key"]:
|
|
|
raise VMError(
|
|
|
"虚拟机公网 IP 与记录的地址池不一致,已停止网络修改",
|
|
|
409,
|
|
|
)
|
|
|
else:
|
|
|
current_pool = public_pool_for_ip(
|
|
|
profile, old_ip, active=False)
|
|
|
target_pool = (
|
|
|
public_pool_for_key(profile, body.public_pool_key, active=True)
|
|
|
if body.public_pool_key else
|
|
|
public_pool_for_ip(profile, body.public_ip_cidr, active=True)
|
|
|
)
|
|
|
except VMError:
|
|
|
raise
|
|
|
except (TypeError, ValueError) as exc:
|
|
|
raise VMError("虚拟机当前公网 IP 配置无效,已停止网络修改", 409) from exc
|
|
|
if (target_pool["network_selector"]
|
|
|
!= current_pool["network_selector"]):
|
|
|
raise VMError(
|
|
|
"不允许通过换 IP 接口迁移到不同的物理网络接入",
|
|
|
409,
|
|
|
)
|
|
|
try:
|
|
|
validated = validate_cloud_network(
|
|
|
body.public_ip_cidr,
|
|
|
target_pool["gateway"],
|
|
|
target_pool["dns_servers"],
|
|
|
require_auth=False,
|
|
|
)
|
|
|
except (TypeError, ValueError) as exc:
|
|
|
raise VMError("目标公网 IP 配置无效", 422) from exc
|
|
|
if validated["public_ip_cidr"] == old_ip:
|
|
|
return {
|
|
|
"vm": vm,
|
|
|
"public_pool_key": target_pool["pool_key"],
|
|
|
"public_ip_cidr": old_ip,
|
|
|
"gateway": validated["gateway"],
|
|
|
"dns_servers": list(validated["dns_servers"]),
|
|
|
"network_changed": False,
|
|
|
}
|
|
|
upgrade_firewall_filter_if_needed(
|
|
|
profile, normalized_id, service)
|
|
|
mark_network_mutation_started = request.scope.get(
|
|
|
"virt_cloud_mark_network_mutation_started")
|
|
|
if callable(mark_network_mutation_started):
|
|
|
mark_network_mutation_started()
|
|
|
claim_scheduled_public_ip(
|
|
|
profile, target_pool,
|
|
|
validated["public_ip_cidr"], normalized_id,
|
|
|
)
|
|
|
claim_reserved = True
|
|
|
try:
|
|
|
network = {
|
|
|
**validated,
|
|
|
"public_pool_key": target_pool["pool_key"],
|
|
|
"source_public_pool_key": current_pool["pool_key"],
|
|
|
}
|
|
|
result = service.change_cloud_vm_network(
|
|
|
normalized_id, network,
|
|
|
confirm_restart=body.confirm_restart,
|
|
|
)
|
|
|
except Exception as exc:
|
|
|
# A successful libvirt request can lose its response. Never
|
|
|
# release a new claim on a 5xx/unknown result; doing so could
|
|
|
# allocate the same address to a second guest.
|
|
|
if (claim_reserved and isinstance(exc, VMError)
|
|
|
and exc.status < 500):
|
|
|
release_public_ip_claim(
|
|
|
profile, validated["public_ip_cidr"], normalized_id)
|
|
|
raise
|
|
|
try:
|
|
|
release_public_ip_claim(profile, old_ip, normalized_id)
|
|
|
except DatabaseError as exc:
|
|
|
LOG.exception("Could not release old public IP claim for %s", normalized_id)
|
|
|
# The guest may already use the target address, but PHP must
|
|
|
# keep both ledger rows unavailable until the engine claim
|
|
|
# swap is complete. A 5xx makes the durable operation
|
|
|
# uncertain instead of silently terminal-successful.
|
|
|
raise PublicIpClaimCleanupPending(
|
|
|
"新公网 IP 已生效,但旧地址占用清理失败,必须人工核查",
|
|
|
503,
|
|
|
) from exc
|
|
|
return result
|
|
|
|
|
|
@app.post("/api/v1/vms/{vm_id}/settings")
|
|
|
def change_vm_settings(
|
|
|
vm_id: str,
|
|
|
body: VMSettingsBody,
|
|
|
_session: Session = Depends(require_approved_write),
|
|
|
):
|
|
|
"""Change guest hostname/root credentials from the authenticated panel boundary.
|
|
|
|
|
|
The public-key value is validated directly from this request. This
|
|
|
endpoint never accepts an SSH-key ID and never reads the tenant key
|
|
|
catalog. Password hashing and credential persistence happen inside
|
|
|
the authenticated engine so plaintext is not written to operation
|
|
|
records or logs.
|
|
|
"""
|
|
|
with reinstall_lock, rescue_lock, snapshot_lock:
|
|
|
forbid_long_job_during_update()
|
|
|
profile = current_profile()
|
|
|
canonical_id = require_vm_access(_session, profile, vm_id)
|
|
|
forbid_pending_snapshot(profile, canonical_id, "修改设置")
|
|
|
if (template_scope(profile), canonical_id) in pending_reinstalls:
|
|
|
raise VMError("虚拟机正在重装系统,暂时不能修改设置", 409)
|
|
|
if (host_scope(profile), canonical_id) in pending_rescues:
|
|
|
raise VMError("虚拟机正在切换救援模式,请等待任务完成", 409)
|
|
|
|
|
|
hostname = body.hostname
|
|
|
if hostname is not None:
|
|
|
try:
|
|
|
hostname = validate_guest_hostname(hostname)
|
|
|
except ValueError as exc:
|
|
|
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
|
|
public_key = body.ssh_public_key
|
|
|
if public_key is not None:
|
|
|
try:
|
|
|
public_key = _validate_ssh_public_key(public_key.strip())
|
|
|
except ValueError as exc:
|
|
|
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
|
|
|
|
|
password = (body.root_password.get_secret_value()
|
|
|
if body.root_password is not None else None)
|
|
|
password_hash = None
|
|
|
if password is not None:
|
|
|
try:
|
|
|
password_hash = hash_root_password(password)
|
|
|
except ValueError as exc:
|
|
|
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
|
|
|
|
|
service = new_service(profile)
|
|
|
current_vm = service.get_vm(canonical_id)
|
|
|
if (not current_vm.get("managed")
|
|
|
or current_vm.get("os_source") != "cloud_template"):
|
|
|
raise VMError("只有面板创建的公网云镜像虚拟机支持修改设置", 409)
|
|
|
result = service.change_cloud_vm_settings(
|
|
|
canonical_id,
|
|
|
hostname=hostname,
|
|
|
root_password_hash=password_hash,
|
|
|
ssh_public_key=public_key,
|
|
|
confirm_restart=body.confirm_restart,
|
|
|
)
|
|
|
|
|
|
# Keep the panel's encrypted reveal credential in sync only after
|
|
|
# the engine confirms the seed was replaced. A missing legacy
|
|
|
# record is left untouched rather than inventing an auth mode.
|
|
|
existing = get_credential(profile, canonical_id)
|
|
|
if password is not None or public_key is not None:
|
|
|
if existing is not None:
|
|
|
generation = existing["generation"]
|
|
|
auth_mode = existing["auth_mode"]
|
|
|
saved_password = existing.get("password")
|
|
|
else:
|
|
|
generation = (result.get("vm", {}).get("credential_generation")
|
|
|
if isinstance(result.get("vm"), dict) else None)
|
|
|
if not isinstance(generation, str) or not generation:
|
|
|
generation = f"panel-seed-{canonical_id}.iso"
|
|
|
auth_mode = "ssh_key" if public_key is not None and password is None else "password"
|
|
|
saved_password = None
|
|
|
if password is not None:
|
|
|
saved_password = password
|
|
|
auth_mode = "ssh_key_password" if public_key is not None or auth_mode in {
|
|
|
"ssh_key", "ssh_key_password"
|
|
|
} else "password"
|
|
|
elif public_key is not None and auth_mode == "password":
|
|
|
auth_mode = "ssh_key_password"
|
|
|
save_credential(profile, canonical_id, auth_mode, generation, saved_password)
|
|
|
|
|
|
LOG.info("vm settings changed: uuid=%s admin=%s", canonical_id, _session.username)
|
|
|
return result
|
|
|
|
|
|
@app.post("/api/v1/vms/{vm_id}/actions")
|
|
|
def action(vm_id: str, body: ActionBody, _session: Session = Depends(require_approved_write)):
|
|
|
with reinstall_lock, rescue_lock, snapshot_lock:
|
|
|
forbid_long_job_during_update()
|
|
|
profile = current_profile()
|
|
|
require_vm_access(_session, profile, vm_id)
|
|
|
try:
|
|
|
normalized_id = str(uuid.UUID(vm_id))
|
|
|
except ValueError:
|
|
|
normalized_id = vm_id
|
|
|
forbid_pending_snapshot(profile, normalized_id, "执行电源操作")
|
|
|
if (template_scope(profile), normalized_id) in pending_reinstalls:
|
|
|
raise VMError("虚拟机正在重装系统,暂时不能执行其他操作", 409)
|
|
|
service = new_service(profile)
|
|
|
if (host_scope(profile), normalized_id) in pending_rescues:
|
|
|
raise VMError("虚拟机正在切换救援模式,请等待任务完成", 409)
|
|
|
rescue_record = current_rescue_state(profile, normalized_id, service)
|
|
|
current_vm = service.get_vm(normalized_id)
|
|
|
staged = bool(current_vm.get("rescue_staged"))
|
|
|
if staged and rescue_record is not None:
|
|
|
if body.action not in {"poweroff", "start"}:
|
|
|
raise VMError(
|
|
|
"救援 ISO 已预挂载;请强制断电后开机进入救援,或取消救援配置", 409)
|
|
|
elif rescue_record is not None or current_vm.get("rescue_mode") or staged:
|
|
|
raise VMError("虚拟机处于救援模式或救援任务中,请先退出救援模式", 409)
|
|
|
vm = service.action(vm_id, body.action)
|
|
|
if staged and body.action == "start":
|
|
|
if not vm.get("rescue_mode") or vm.get("rescue_staged"):
|
|
|
raise VMError(
|
|
|
"虚拟机已发送开机指令,但未确认从救援 ISO 启动;请检查 VNC 控制台", 502)
|
|
|
rescue_record["status"] = "active"
|
|
|
save_rescue_state(profile, normalized_id, rescue_record)
|
|
|
LOG.info("vm action: uuid=%s action=%s admin=%s", vm.get("uuid"), body.action, _session.username)
|
|
|
return vm
|
|
|
|
|
|
@app.delete("/api/v1/vms/{vm_id}")
|
|
|
def delete_vm(
|
|
|
vm_id: str,
|
|
|
delete_disks: bool = False,
|
|
|
_session: Session = Depends(require_approved_write),
|
|
|
):
|
|
|
with reinstall_lock, rescue_lock, snapshot_lock:
|
|
|
forbid_long_job_during_update()
|
|
|
profile = current_profile()
|
|
|
require_vm_access(_session, profile, vm_id)
|
|
|
system_cleanup = (
|
|
|
internal_mode and actor().get("system_cleanup") is True
|
|
|
)
|
|
|
try:
|
|
|
normalized_id = str(uuid.UUID(vm_id))
|
|
|
except ValueError:
|
|
|
normalized_id = vm_id
|
|
|
if (host_scope(profile), normalized_id) in pending_snapshot_jobs:
|
|
|
raise VMError("虚拟机正在执行快照任务,暂时不能删除", 409)
|
|
|
if (template_scope(profile), normalized_id) in pending_reinstalls:
|
|
|
raise VMError("虚拟机正在重装系统,暂时不能删除", 409)
|
|
|
service = new_service(profile)
|
|
|
if (host_scope(profile), normalized_id) in pending_rescues:
|
|
|
raise VMError("虚拟机处于救援模式或救援任务中,请先退出救援模式", 409)
|
|
|
if not system_cleanup:
|
|
|
rescue_record = current_rescue_state(profile, normalized_id, service)
|
|
|
rescue_vm = service.get_vm(normalized_id)
|
|
|
rescue_protected = (
|
|
|
rescue_record is not None
|
|
|
or rescue_vm.get("rescue_mode")
|
|
|
or rescue_vm.get("rescue_staged")
|
|
|
)
|
|
|
else:
|
|
|
rescue_protected = False
|
|
|
if rescue_protected:
|
|
|
raise VMError("虚拟机处于救援模式或救援任务中,请先退出救援模式", 409)
|
|
|
registered_snapshots = list_snapshot_records(normalized_id)
|
|
|
if registered_snapshots and not delete_disks:
|
|
|
raise VMError("虚拟机仍有本地快照;删除磁盘时会一并清理快照", 409)
|
|
|
# Validate every registered recovery point before the VM delete,
|
|
|
# but keep all of them intact until that deletion commits. This
|
|
|
# prevents a later VM-delete failure from destroying the user's
|
|
|
# last rollback points.
|
|
|
if delete_disks and registered_snapshots:
|
|
|
for snapshot in registered_snapshots:
|
|
|
service.validate_snapshot_volume(
|
|
|
normalized_id,
|
|
|
snapshot["snapshot_key"],
|
|
|
snapshot["volume_name"],
|
|
|
)
|
|
|
result = service.delete_vm(vm_id, delete_disks)
|
|
|
deleted_snapshots = []
|
|
|
if delete_disks and registered_snapshots:
|
|
|
try:
|
|
|
for snapshot in registered_snapshots:
|
|
|
service.delete_snapshot_volume(
|
|
|
normalized_id,
|
|
|
snapshot["snapshot_key"],
|
|
|
snapshot["volume_name"],
|
|
|
allow_missing_vm=True,
|
|
|
)
|
|
|
delete_snapshot_record(
|
|
|
normalized_id, snapshot["snapshot_key"])
|
|
|
deleted_snapshots.append(snapshot["snapshot_key"])
|
|
|
except Exception as exc:
|
|
|
# The VM deletion already committed. Keep every
|
|
|
# unconfirmed catalog row so reconciliation can identify
|
|
|
# exactly which volume still needs attention.
|
|
|
raise VMDeleteSnapshotCleanupPending(
|
|
|
"虚拟机已删除,但本地快照清理结果不完整;必须核查原删除操作"
|
|
|
) from exc
|
|
|
if deleted_snapshots:
|
|
|
result["deleted_snapshots"] = deleted_snapshots
|
|
|
if system_cleanup:
|
|
|
try:
|
|
|
delete_rescue_state(profile, result["uuid"])
|
|
|
except Exception as exc:
|
|
|
LOG.exception("Could not remove rescue state for deleted VM %s", result["uuid"])
|
|
|
raise VMDeleteSnapshotCleanupPending(
|
|
|
"虚拟机已删除,但救援状态清理未完成;必须核查原删除操作"
|
|
|
) from exc
|
|
|
if database_mode:
|
|
|
try:
|
|
|
state.users.delete_vm_owner(host_scope(profile), result["uuid"])
|
|
|
except Exception as exc:
|
|
|
LOG.exception("Could not remove owner for deleted VM %s", result["uuid"])
|
|
|
raise VMDeleteSnapshotCleanupPending(
|
|
|
"虚拟机已删除,但归属记录清理未完成;必须核查原删除操作"
|
|
|
) from exc
|
|
|
try:
|
|
|
delete_credential(profile, result["uuid"])
|
|
|
except Exception as exc:
|
|
|
LOG.exception("Could not remove credential for deleted VM %s", result["uuid"])
|
|
|
raise VMDeleteSnapshotCleanupPending(
|
|
|
"虚拟机已删除,但登录凭据清理未完成;必须核查原删除操作"
|
|
|
) from exc
|
|
|
try:
|
|
|
release_vm_public_ip_claims(profile, result["uuid"])
|
|
|
except Exception as exc:
|
|
|
LOG.exception("Could not release public IP claim for deleted VM %s", result["uuid"])
|
|
|
# Do not let PHP free the assigned address while the engine
|
|
|
# still has a stale claim. The deletion outcome stays
|
|
|
# uncertain and requires explicit reconciliation.
|
|
|
raise PublicIpClaimCleanupPending(
|
|
|
"虚拟机已删除,但公网 IP 占用清理失败,必须人工核查",
|
|
|
503,
|
|
|
) from exc
|
|
|
try:
|
|
|
service.delete_firewall_filter(result["uuid"])
|
|
|
delete_firewall_record(profile, result["uuid"])
|
|
|
except Exception as exc:
|
|
|
LOG.exception(
|
|
|
"Could not remove firewall state for deleted VM %s",
|
|
|
result["uuid"],
|
|
|
)
|
|
|
raise VMDeleteFirewallCleanupPending(
|
|
|
"虚拟机已删除,但防火墙过滤器清理未完成;必须核查原删除操作"
|
|
|
) from exc
|
|
|
LOG.info(
|
|
|
"vm deleted: uuid=%s delete_disks=%s admin=%s",
|
|
|
result.get("uuid"), delete_disks, _session.username,
|
|
|
)
|
|
|
return result
|
|
|
|
|
|
@app.websocket("/api/v1/vms/{vm_id}/console")
|
|
|
async def console(websocket: WebSocket, vm_id: str):
|
|
|
host = websocket.headers.get("host")
|
|
|
if websocket.url.scheme != "wss":
|
|
|
await websocket.close(code=4403)
|
|
|
return
|
|
|
if not _origin_matches_host(websocket.headers.get("origin"), host):
|
|
|
await websocket.close(code=4403)
|
|
|
return
|
|
|
session = await asyncio.to_thread(sessions.get, websocket.cookies.get(COOKIE_NAME))
|
|
|
if session is None:
|
|
|
await websocket.close(code=4401)
|
|
|
return
|
|
|
try:
|
|
|
account = await asyncio.to_thread(account_for_session, session)
|
|
|
except HTTPException:
|
|
|
await websocket.close(code=4401)
|
|
|
return
|
|
|
if not account["admin"] and not account["approved"]:
|
|
|
await websocket.close(code=4403)
|
|
|
return
|
|
|
profile = manager.current()
|
|
|
if profile is None:
|
|
|
await websocket.close(code=4409)
|
|
|
return
|
|
|
try:
|
|
|
if profile.get("key_locked"):
|
|
|
await websocket.close(code=4423)
|
|
|
return
|
|
|
await asyncio.to_thread(require_vm_access, session, profile, vm_id)
|
|
|
port = await asyncio.to_thread(new_service(profile).console_port, vm_id)
|
|
|
except VMError:
|
|
|
await websocket.close(code=4404)
|
|
|
return
|
|
|
await websocket.accept()
|
|
|
async def watch_console_access() -> None:
|
|
|
while True:
|
|
|
await asyncio.sleep(CONSOLE_ACCESS_CHECK_SECONDS)
|
|
|
try:
|
|
|
refreshed = await asyncio.to_thread(sessions.get, session.token)
|
|
|
if refreshed is None or refreshed.username != session.username:
|
|
|
return
|
|
|
account = await asyncio.to_thread(account_for_session, refreshed)
|
|
|
if not account["admin"] and not account["approved"]:
|
|
|
return
|
|
|
current = manager.current()
|
|
|
if (current is None or current.get("key_locked")
|
|
|
or host_scope(current) != host_scope(profile)):
|
|
|
return
|
|
|
await asyncio.to_thread(require_vm_access, refreshed, current, vm_id)
|
|
|
except (HTTPException, VMError, DatabaseError, RemoteHostError,
|
|
|
KeyError, TypeError, ValueError):
|
|
|
return
|
|
|
|
|
|
relay_task = asyncio.create_task(relay_console(
|
|
|
websocket,
|
|
|
port=port,
|
|
|
ssh_target=profile["ssh_target"],
|
|
|
ssh_binary=settings.ssh_binary,
|
|
|
known_hosts_file=profile["ssh_known_hosts_file"],
|
|
|
ssh_config_file=profile["ssh_config_file"],
|
|
|
))
|
|
|
access_task = asyncio.create_task(watch_console_access())
|
|
|
try:
|
|
|
done, _pending = await asyncio.wait(
|
|
|
{relay_task, access_task}, return_when=asyncio.FIRST_COMPLETED)
|
|
|
if access_task in done:
|
|
|
relay_task.cancel()
|
|
|
await asyncio.gather(relay_task, return_exceptions=True)
|
|
|
with suppress(RuntimeError, OSError):
|
|
|
await websocket.close(code=4403)
|
|
|
return
|
|
|
await relay_task
|
|
|
except ConsoleRelayError as exc:
|
|
|
LOG.warning("VNC relay failed for vm %r: %s", vm_id, exc)
|
|
|
with suppress(RuntimeError, OSError):
|
|
|
await websocket.close(code=1011)
|
|
|
finally:
|
|
|
access_task.cancel()
|
|
|
await asyncio.gather(access_task, return_exceptions=True)
|
|
|
|
|
|
return app
|