You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/engine/virsh_panel/cloud_template_runtime.py

1117 lines
49 KiB
Python

"""Engine-global cloud template catalog and persistent source images.
The legacy cloud-template flow downloads an image inside one host child app,
uploads it immediately to that host, and then deletes the local download. A
multi-host control plane needs a different ownership boundary: the engine owns
one immutable source image and each hypervisor owns only a disposable replica.
This module deliberately contains no libvirt code. It provides the global
catalog, import jobs and verified ``source()`` leases that a host-specific
cache layer can consume later.
"""
from __future__ import annotations
import hashlib
import json
import logging
import os
import re
import shutil
import stat
import threading
import time
import uuid
from concurrent.futures import ThreadPoolExecutor
from contextlib import contextmanager
from pathlib import Path
from typing import Any, Iterator
from .cloud_templates import (
GIB,
MAX_IMAGE_BYTES,
MAX_VIRTUAL_BYTES,
SHA256_RE,
TemplateError,
download_verified_image,
image_metadata_for,
template_id_for,
template_volume_name,
validate_template_metadata,
validate_template_name,
)
LOG = logging.getLogger(__name__)
# Keep the existing, already validated template/volume identifier grammar, but
# replace its old host scope with one fixed engine-global namespace. The full
# digest remains in the catalog and every source/replica marker must compare it;
# the shortened digest in the public ID is never treated as integrity proof.
GLOBAL_TEMPLATE_SCOPE = hashlib.sha256(
b"virt-cloud/global-cloud-template/v1"
).hexdigest()
CATALOG_KIND = "cloud_template"
IMPORT_JOB_KIND = "cloud_template_import_job"
CACHE_TOMBSTONE_KIND = "cloud_template_tombstone"
GENERATION_KIND = "cloud_template_generation"
SOURCE_DIRECTORY_NAME = "cloud-templates"
PERSISTENT_FREE_RESERVE_BYTES = 256 * 1024**2
MIN_DOWNLOAD_BYTES = 1024 * 1024
SOURCE_FILE_RE = re.compile(
r"panel-template-(?:alma(?:8|9|10)|debian(?:11|12|13)|"
r"ubuntu(?:2004|2204|2404|2504|2510|2604)|"
r"centosstream(?:9|10))-"
r"[0-9a-f]{8}-[0-9a-f]{16}\.qcow2\Z"
)
TEMPORARY_FILE_RE = re.compile(r"\.cloud-image-[A-Za-z0-9_-]+\.qcow2\Z")
def cloud_template_id_for(digest: str, version: str,
os_family: str = "AlmaLinux") -> str:
"""Return a stable ID that is independent of every target hypervisor."""
if not isinstance(digest, str) or not SHA256_RE.fullmatch(digest):
raise TemplateError("云镜像 SHA-256 格式无效")
return template_id_for(
GLOBAL_TEMPLATE_SCOPE, digest.lower(), version, os_family
)
class GlobalCloudTemplateStore:
"""Strict global catalog backed by its own encrypted record kind."""
def __init__(self, store: Any | None) -> None:
self.store = store
self._memory: dict[str, str] = {}
self._lock = threading.RLock()
@staticmethod
def _validate(record: Any) -> dict[str, Any]:
if not isinstance(record, dict):
raise TemplateError("全局云镜像目录记录损坏", 500)
try:
template_id = record["id"]
name = record["name"]
os_family = record["os_family"]
os_version = record["os_version"]
digest = record["sha256"]
volume_name = record["volume_name"]
virtual_size = record["virtual_size"]
image_bytes = record["image_bytes"]
created_at = record["created_at"]
except KeyError as exc:
raise TemplateError("全局云镜像目录记录损坏", 500) from exc
try:
normalized_name = validate_template_name(name)
except TemplateError as exc:
raise TemplateError("全局云镜像目录记录损坏", 500) from exc
try:
expected_id = cloud_template_id_for(
digest, os_version, os_family
)
expected_volume_name = template_volume_name(template_id)
except TemplateError as exc:
raise TemplateError("全局云镜像目录记录损坏", 500) from exc
valid = (
isinstance(template_id, str)
and isinstance(name, str)
and normalized_name == name
and os_family in {"AlmaLinux", "Debian", "Ubuntu", "CentOS Stream"}
and isinstance(os_version, str)
and isinstance(digest, str)
and SHA256_RE.fullmatch(digest) is not None
and digest == digest.lower()
and template_id == expected_id
and volume_name == expected_volume_name
and not isinstance(virtual_size, bool)
and isinstance(virtual_size, int)
and GIB <= virtual_size <= MAX_VIRTUAL_BYTES
and not isinstance(image_bytes, bool)
and isinstance(image_bytes, int)
and 0 < image_bytes <= MAX_IMAGE_BYTES
and not isinstance(created_at, bool)
and isinstance(created_at, int)
and created_at > 0
)
if not valid:
raise TemplateError("全局云镜像目录记录损坏", 500)
# Reject unrecognised fields instead of accidentally persisting a URL,
# local path, host identity or another future secret in public records.
if set(record) != {
"id", "name", "os_family", "os_version", "sha256",
"volume_name", "virtual_size", "image_bytes", "created_at",
}:
raise TemplateError("全局云镜像目录记录损坏", 500)
return record
def _source(self) -> dict[str, str]:
return self.store.list(CATALOG_KIND) if self.store is not None else self._memory.copy()
def _list(self) -> list[dict[str, Any]]:
records = []
for key, payload in self._source().items():
try:
record = json.loads(payload)
except (TypeError, ValueError) as exc:
raise TemplateError("全局云镜像目录记录损坏", 500) from exc
record = self._validate(record)
if key != record["id"]:
raise TemplateError("全局云镜像目录记录损坏", 500)
records.append(record)
return sorted(
records,
key=lambda item: (item["created_at"], item["id"]),
reverse=True,
)
def list(self) -> list[dict[str, Any]]:
with self._lock:
return self._list()
def get(self, template_id: str) -> dict[str, Any]:
# This validates the opaque identifier before consulting storage.
template_volume_name(template_id)
with self._lock:
for record in self._list():
if record["id"] == template_id:
return record
raise TemplateError("云镜像模板不存在", 404)
def add(self, record: dict[str, Any]) -> dict[str, Any]:
record = self._validate(record.copy())
encoded = json.dumps(record, ensure_ascii=False, separators=(",", ":"))
with self._lock:
current = self._list()
if any(row["id"] == record["id"] for row in current):
raise TemplateError("此云镜像已经导入", 409)
if any(row["name"].casefold() == record["name"].casefold()
for row in current):
raise TemplateError("已有同名云镜像模板", 409)
if self.store is not None:
if not self.store.insert(CATALOG_KIND, record["id"], encoded):
raise TemplateError("此云镜像已经导入", 409)
else:
self._memory[record["id"]] = encoded
return record
def rename(self, template_id: str, name: str) -> dict[str, Any]:
name = validate_template_name(name)
with self._lock:
current = self._list()
target = next((row for row in current if row["id"] == template_id), None)
if target is None:
raise TemplateError("云镜像模板不存在", 404)
if any(row["id"] != template_id
and row["name"].casefold() == name.casefold()
for row in current):
raise TemplateError("已有同名云镜像模板", 409)
updated = {**target, "name": name}
encoded = json.dumps(updated, ensure_ascii=False, separators=(",", ":"))
if self.store is not None:
self.store.set(CATALOG_KIND, template_id, encoded)
else:
self._memory[template_id] = encoded
return updated
def delete(self, template_id: str) -> None:
# Validate the opaque identifier before touching persistent storage.
template_volume_name(template_id)
with self._lock:
self.get(template_id)
if self.store is not None:
self.store.delete(CATALOG_KIND, template_id)
else:
self._memory.pop(template_id, None)
class GlobalCloudTemplateFiles:
"""Private engine-owned source files, never paths supplied by a caller."""
def __init__(self, data_dir: Path) -> None:
data_dir = Path(data_dir)
if not data_dir.is_absolute() or data_dir == Path("/"):
raise ValueError("cloud template data directory must be absolute and dedicated")
self.data_dir = data_dir
self.root = data_dir / SOURCE_DIRECTORY_NAME
@staticmethod
def _ensure_directory(path: Path) -> None:
try:
info = path.lstat()
except FileNotFoundError:
path.mkdir(mode=0o700)
info = path.lstat()
if (not stat.S_ISDIR(info.st_mode) or stat.S_ISLNK(info.st_mode)
or info.st_uid != os.geteuid()):
raise TemplateError("云镜像持久目录不安全,请联系管理员", 500)
if stat.S_IMODE(info.st_mode) != 0o700:
os.chmod(path, 0o700)
def directory(self) -> Path:
self.data_dir.mkdir(mode=0o700, parents=True, exist_ok=True)
self._ensure_directory(self.data_dir)
self._ensure_directory(self.root)
return self.root
def available_bytes(self) -> int:
"""Return writable bytes while preserving space for panel operation."""
free = int(shutil.disk_usage(self.directory()).free)
return max(0, free - PERSISTENT_FREE_RESERVE_BYTES)
def path(self, template_id: str, *, expected_size: int | None = None) -> Path:
path = self.directory() / template_volume_name(template_id)
try:
info = path.lstat()
except FileNotFoundError as exc:
raise TemplateError("云镜像源文件不存在,请重新导入", 409) from exc
if (not stat.S_ISREG(info.st_mode) or stat.S_ISLNK(info.st_mode)
or info.st_uid != os.geteuid()
or stat.S_IMODE(info.st_mode) != 0o600
or (expected_size is not None and info.st_size != expected_size)):
raise TemplateError("云镜像源文件已损坏,请联系管理员", 500)
return path
def publish(self, temporary: Path, template_id: str, *, expected_size: int) -> Path:
temporary = Path(temporary)
directory = self.directory()
try:
info = temporary.lstat()
except FileNotFoundError as exc:
raise TemplateError("云镜像下载临时文件丢失", 500) from exc
if (not stat.S_ISREG(info.st_mode) or stat.S_ISLNK(info.st_mode)
or info.st_uid != os.geteuid()
or stat.S_IMODE(info.st_mode) != 0o600
or temporary.parent != directory
or info.st_size != expected_size
or expected_size <= 0 or expected_size > MAX_IMAGE_BYTES):
raise TemplateError("云镜像下载临时文件无效", 500)
destination = directory / template_volume_name(template_id)
try:
# download_verified_image creates the temporary in this directory.
# A hard link publishes atomically and cannot replace an existing
# immutable source with attacker-controlled bytes.
os.link(temporary, destination)
os.chmod(destination, 0o600)
temporary.unlink()
descriptor = os.open(
destination.parent,
os.O_RDONLY | getattr(os, "O_DIRECTORY", 0),
)
try:
os.fsync(descriptor)
finally:
os.close(descriptor)
except FileExistsError as exc:
raise TemplateError("此云镜像已经导入", 409) from exc
except OSError as exc:
destination.unlink(missing_ok=True)
raise TemplateError("云镜像无法保存到持久目录", 507) from exc
return destination
def delete(self, template_id: str) -> None:
path = self.directory() / template_volume_name(template_id)
try:
info = path.lstat()
except FileNotFoundError:
return
if not stat.S_ISREG(info.st_mode) or stat.S_ISLNK(info.st_mode):
raise TemplateError("云镜像源文件无法安全删除", 500)
try:
path.unlink()
except OSError as exc:
raise TemplateError("云镜像源文件删除失败,请联系管理员", 500) from exc
def reconcile(self, catalog: GlobalCloudTemplateStore) -> None:
"""Remove only provable interrupted downloads and uncataloged sources."""
try:
self.root.lstat()
except FileNotFoundError:
return
self._ensure_directory(self.root)
active = {row["volume_name"] for row in catalog.list()}
for candidate in self.root.iterdir():
try:
info = candidate.lstat()
except OSError:
continue
if (not stat.S_ISREG(info.st_mode) or stat.S_ISLNK(info.st_mode)
or info.st_uid != os.geteuid()):
continue
interrupted = TEMPORARY_FILE_RE.fullmatch(candidate.name) is not None
orphan = (SOURCE_FILE_RE.fullmatch(candidate.name) is not None
and candidate.name not in active)
if interrupted or orphan:
candidate.unlink(missing_ok=True)
class CloudTemplateRuntime:
"""One engine-process runtime shared by all host child applications."""
def __init__(self, store: Any | None, data_dir: Path) -> None:
self.store = store
self.catalog = GlobalCloudTemplateStore(store)
self.files = GlobalCloudTemplateFiles(data_dir)
self.executor = ThreadPoolExecutor(
max_workers=1, thread_name_prefix="global-cloud-template-import"
)
self.lock = threading.RLock()
self.jobs: dict[str, dict[str, Any]] = {}
self.pending: set[str] = set()
self.upload_reservations: dict[str, dict[str, str]] = {}
self.in_use: dict[str, int] = {}
self.verified_sources: dict[str, tuple[int, int, int, int, str]] = {}
self.memory_generations: dict[str, int] = {}
self._restored = False
@staticmethod
def public_record(record: dict[str, Any]) -> dict[str, Any]:
return {key: value for key, value in record.items() if key in {
"id", "name", "os_family", "os_version", "sha256",
"virtual_size", "image_bytes", "created_at",
}}
@staticmethod
def public_error(value: Any) -> str:
safe = {
"面板在云镜像导入期间重启;请重新提交导入",
"云镜像地址无效",
"仅支持 AlmaLinux 官方仓库的 9/10 x86_64 GenericCloud qcow2 镜像",
"仅支持 AlmaLinux、Debian、Ubuntu 和 CentOS Stream 官方 x86_64 云镜像直链",
"官方镜像地址发生重定向,请使用最终的官方 HTTPS 直链",
"官方云镜像地址无法解析",
"官方云镜像地址解析结果无效",
"官方云镜像地址解析到非公网地址",
"无法连接官方云镜像下载地址",
"官方云镜像重定向目标不安全",
"官方云镜像重定向次数过多",
"官方云镜像下载服务器返回异常状态",
"官方云镜像下载响应格式无效",
"云镜像下载内容不完整",
"SHA-256 可留空;填写时必须为 64 位十六进制校验值",
"下载官方云镜像失败,请检查面板服务器网络",
"云镜像下载超过 60 分钟,请检查面板服务器网络",
"云镜像超过 4 GiB 限制",
"云镜像 SHA-256 不匹配,已丢弃下载内容",
"下载文件不是有效的 qcow2 镜像",
"云镜像 qcow2 版本或簇大小不受支持",
"云镜像 qcow2 头部不完整",
"云镜像包含不受支持或外部依赖的 qcow2 特性",
"云镜像含外部依赖、加密或容量超出支持范围",
"此云镜像已经导入",
"已有同名云镜像模板",
"云镜像无法保存到持久目录",
"面板云镜像持久目录空间不足",
"云镜像缓存清理记录损坏,请联系管理员",
"云镜像版本记录损坏,请联系管理员",
"旧云镜像缓存仍在清理,请稍后重新导入",
"云镜像导入失败,请查看引擎日志",
}
if isinstance(value, str) and value in safe:
return value
return "云镜像下载或保存失败,请联系管理员查看日志"
@classmethod
def public_job(cls, job: dict[str, Any]) -> dict[str, Any]:
allowed = {
"job_id", "status", "phase", "created_at", "started_at",
"finished_at", "template", "error",
}
result = {key: value for key, value in job.items() if key in allowed}
if isinstance(result.get("template"), dict):
result["template"] = cls.public_record(result["template"])
if "error" in result:
result["error"] = cls.public_error(result["error"])
return result
def list(self) -> list[dict[str, Any]]:
"""Return the public, host-independent template list."""
with self.lock:
return [self.public_record(row) for row in self.catalog.list()]
def status(self) -> dict[str, Any]:
"""Return the public list plus the current import, when one exists."""
result: dict[str, Any] = {"templates": self.list()}
with self.lock:
active = [job for job in self.jobs.values()
if job.get("status") in {"queued", "running"}]
if active:
current = max(active, key=lambda job: (
int(job.get("created_at", 0)), str(job.get("job_id", ""))))
result["active_import_job"] = self.public_job(current)
return result
def get(self, template_id: str) -> dict[str, Any]:
return self.catalog.get(template_id)
def get_job(self, job_id: str) -> dict[str, Any]:
with self.lock:
job = self.jobs.get(job_id)
if job is None:
raise TemplateError("云镜像导入任务不存在", 404)
return self.public_job(job)
def rename(self, template_id: str, name: str) -> dict[str, Any]:
with self.lock:
if self.pending:
raise TemplateError("云镜像正在导入,请稍后修改模板名称", 409)
return self.public_record(self.catalog.rename(template_id, name))
def delete(self, template_id: str) -> None:
with self.lock:
if self.pending:
raise TemplateError("云镜像正在导入,请稍后删除模板", 409)
if self.in_use.get(template_id, 0):
raise TemplateError("云镜像正在同步到宿主机,请稍后删除", 409)
self.catalog.get(template_id)
# Delete bytes first. If the catalog write then fails, a retry can
# finish the deletion without leaving unaccounted source files.
self.files.delete(template_id)
self.catalog.delete(template_id)
self.verified_sources.pop(template_id, None)
def _remaining_download_bytes(self) -> int:
maximum = min(MAX_IMAGE_BYTES, self.files.available_bytes())
if maximum < MIN_DOWNLOAD_BYTES:
raise TemplateError("面板云镜像持久目录空间不足", 507)
return maximum
def _remember_generation(self, template_id: str, created_at: int) -> None:
if isinstance(created_at, bool) or not isinstance(created_at, int) \
or created_at < 1:
raise TemplateError("全局云镜像目录记录损坏", 500)
if self.store is None:
self.memory_generations[template_id] = max(
created_at, self.memory_generations.get(template_id, 0))
return
raw = self.store.get(GENERATION_KIND, template_id)
try:
previous = 0 if raw is None else int(raw)
except (TypeError, ValueError) as exc:
raise TemplateError("云镜像版本记录损坏,请联系管理员", 500) from exc
if previous < 0 or (raw is not None and str(previous) != raw):
raise TemplateError("云镜像版本记录损坏,请联系管理员", 500)
if created_at > previous:
self.store.set(GENERATION_KIND, template_id, str(created_at))
def _next_generation(self, template_id: str, floor: int = 0) -> int:
if self.store is None:
previous = self.memory_generations.get(template_id, 0)
else:
raw = self.store.get(GENERATION_KIND, template_id)
try:
previous = 0 if raw is None else int(raw)
except (TypeError, ValueError) as exc:
raise TemplateError(
"云镜像版本记录损坏,请联系管理员", 500
) from exc
if previous < 0 or (raw is not None and str(previous) != raw):
raise TemplateError("云镜像版本记录损坏,请联系管理员", 500)
generation = max(int(time.time()), previous + 1, floor + 1)
if self.store is None:
self.memory_generations[template_id] = generation
else:
self.store.set(GENERATION_KIND, template_id, str(generation))
return generation
@staticmethod
def _decode_cache_tombstone(template_id: str, raw: str) -> dict[str, Any]:
try:
tombstone = json.loads(raw)
deletion_id = tombstone["deletion_id"]
template_created_at = tombstone["template_created_at"]
pending_hosts = tombstone["pending_hosts"]
except (KeyError, TypeError, ValueError) as exc:
raise TemplateError(
"云镜像缓存清理记录损坏,请联系管理员", 500
) from exc
if (not isinstance(tombstone, dict)
or not isinstance(deletion_id, str)
or re.fullmatch(r"[0-9a-f]{32}", deletion_id) is None
or isinstance(template_created_at, bool)
or not isinstance(template_created_at, int)
or template_created_at < 1
or tombstone.get("template_id") != template_id
or tombstone.get("source_volume_name")
!= template_volume_name(template_id)
or not isinstance(tombstone.get("source_deleted"), bool)
or not isinstance(pending_hosts, list)
or any(not isinstance(host_id, str) or not host_id
for host_id in pending_hosts)
or len(set(pending_hosts)) != len(pending_hosts)
or isinstance(tombstone.get("created_at"), bool)
or not isinstance(tombstone.get("created_at"), int)
or tombstone["created_at"] < 1):
raise TemplateError(
"云镜像缓存清理记录损坏,请联系管理员", 500
)
return tombstone
def _reconcile_cache_tombstone(
self, template_id: str, tombstone: dict[str, Any]
) -> dict[str, Any] | None:
"""Resolve the crash window around source/catalog removal.
A provisional tombstone is written before unlinking the immutable
source. If the engine stops before the final flag is persisted, the
catalog and source together prove whether deletion started: a valid
pair cancels the provisional marker, while either missing half means
the requested deletion must be completed before host garbage
collection can continue.
"""
if self.store is None or tombstone["source_deleted"]:
return tombstone
try:
record = self.catalog.get(template_id)
except TemplateError as exc:
if exc.status != 404:
raise
record = None
if record is not None:
if (record["created_at"] != tombstone["template_created_at"]
or record["volume_name"]
!= tombstone["source_volume_name"]):
raise TemplateError(
"云镜像缓存清理记录损坏,请联系管理员", 500
)
try:
self.files.path(
template_id, expected_size=record["image_bytes"])
except TemplateError:
# The immutable source is already absent or damaged after a
# committed deletion attempt. Finish retiring the catalog.
self.files.delete(template_id)
self.catalog.delete(template_id)
self.verified_sources.pop(template_id, None)
else:
# Both durable halves still exist: deletion never committed.
self.store.delete(CACHE_TOMBSTONE_KIND, template_id)
return None
else:
# The catalog is gone, so ensure no source bytes survived an
# interrupted or externally retried database operation.
self.files.delete(template_id)
self.verified_sources.pop(template_id, None)
tombstone["source_deleted"] = True
self.store.set(
CACHE_TOMBSTONE_KIND,
template_id,
json.dumps(tombstone, separators=(",", ":")),
)
return tombstone
def _prepare_reimport(self, template_id: str) -> int | None:
"""Retire a completed deletion marker before reusing a stable ID.
Template IDs are content-derived, so importing identical bytes after
a deletion intentionally produces the same identifier. Reuse is safe
only after every old host replica has been reclaimed; otherwise a
delayed cleanup task could delete a newly-created replica.
"""
if self.store is None:
return None
raw = self.store.get(CACHE_TOMBSTONE_KIND, template_id)
if raw is None:
return None
tombstone = self._decode_cache_tombstone(template_id, raw)
tombstone = self._reconcile_cache_tombstone(
template_id, tombstone)
if tombstone is None:
return None
template_created_at = tombstone["template_created_at"]
pending_hosts = tombstone["pending_hosts"]
if tombstone["source_deleted"] is not True:
raise TemplateError(
"云镜像缓存清理记录损坏,请联系管理员", 500
)
if pending_hosts:
raise TemplateError("旧云镜像缓存仍在清理,请稍后重新导入", 409)
self.store.delete(CACHE_TOMBSTONE_KIND, template_id)
return template_created_at
def _reconcile_cache_tombstones(self) -> None:
"""Advance provisional delete records after a crash or lost DB ack."""
if self.store is None:
return
for template_id, raw in self.store.list(
CACHE_TOMBSTONE_KIND).items():
tombstone = self._decode_cache_tombstone(template_id, raw)
self._reconcile_cache_tombstone(template_id, tombstone)
def reserve_upload(self, name: str, os_family: str,
os_version: str) -> dict[str, Any]:
"""Reserve the single import slot before receiving a browser upload."""
name = validate_template_name(name)
os_family, os_version = validate_template_metadata(
os_family, os_version)
reservation_id = "upload-" + uuid.uuid4().hex
with self.lock:
existing = self.catalog.list()
directory = self.files.directory()
maximum = self._remaining_download_bytes()
if any(row["name"].casefold() == name.casefold()
for row in existing):
raise TemplateError("已有同名云镜像模板", 409)
if self.pending:
raise TemplateError(
"已有云镜像正在导入,请等待完成后再提交", 409
)
self.pending.add(reservation_id)
self.upload_reservations[reservation_id] = {
"name": name,
"os_family": os_family,
"os_version": os_version,
}
return {
"reservation_id": reservation_id,
"directory": directory,
"max_bytes": maximum,
}
def cancel_upload(self, reservation_id: str) -> None:
"""Release a pre-body reservation after validation or transport failure."""
with self.lock:
self.upload_reservations.pop(reservation_id, None)
self.pending.discard(reservation_id)
def queue_upload(self, reservation_id: str, image_path: Path,
digest: str, virtual_size: int) -> dict[str, str]:
"""Queue publication of an already streamed and validated qcow2 file."""
image_path = Path(image_path)
if not isinstance(digest, str) or SHA256_RE.fullmatch(digest) is None:
raise TemplateError("云镜像 SHA-256 格式无效", 422)
digest = digest.lower()
if (isinstance(virtual_size, bool) or not isinstance(virtual_size, int)
or not GIB <= virtual_size <= MAX_VIRTUAL_BYTES):
raise TemplateError("云镜像系统盘容量超出支持范围", 422)
try:
info = image_path.lstat()
except FileNotFoundError as exc:
raise TemplateError("上传的云镜像临时文件丢失", 500) from exc
if (not stat.S_ISREG(info.st_mode) or stat.S_ISLNK(info.st_mode)
or info.st_uid != os.geteuid()
or stat.S_IMODE(info.st_mode) != 0o600
or image_path.parent != self.files.directory()
or info.st_size <= 0 or info.st_size > MAX_IMAGE_BYTES):
raise TemplateError("上传的云镜像临时文件无效", 500)
with self.lock:
metadata = self.upload_reservations.get(reservation_id)
if metadata is None or reservation_id not in self.pending:
raise TemplateError("云镜像上传预约已失效,请重新上传", 409)
name = metadata["name"]
os_family = metadata["os_family"]
os_version = metadata["os_version"]
template_id = cloud_template_id_for(
digest, os_version, os_family)
existing = self.catalog.list()
if any(row["id"] == template_id for row in existing):
raise TemplateError("此云镜像已经导入", 409)
if any(row["name"].casefold() == name.casefold()
for row in existing):
raise TemplateError("已有同名云镜像模板", 409)
job_id = str(uuid.uuid4())
job = {
"job_id": job_id,
"status": "queued",
"phase": "等待保存上传的云镜像",
"created_at": int(time.time()),
}
try:
self.save_job(job_id, job)
self.executor.submit(
self._run_uploaded_import,
job_id,
image_path,
name,
os_family,
os_version,
digest,
virtual_size,
reservation_id,
)
except Exception as exc:
self.delete_job(job_id)
self.upload_reservations.pop(reservation_id, None)
self.pending.discard(reservation_id)
if isinstance(exc, RuntimeError):
raise TemplateError(
"引擎正在停止,无法开始云镜像导入", 503
) from exc
raise
return {"job_id": job_id, "status": "queued"}
def queue_import(self, url: str, name: str,
expected_sha256: str = "") -> dict[str, str]:
os_family, version = image_metadata_for(url)
name = validate_template_name(name)
if (not isinstance(expected_sha256, str)
or (expected_sha256 and not SHA256_RE.fullmatch(expected_sha256))):
raise TemplateError(
"SHA-256 可留空;填写时必须为 64 位十六进制校验值", 422
)
expected_sha256 = expected_sha256.lower()
pending_key = expected_sha256 or hashlib.sha256(url.encode("utf-8")).hexdigest()
with self.lock:
existing = self.catalog.list()
self._remaining_download_bytes()
if expected_sha256 and any(
row["id"] == cloud_template_id_for(
expected_sha256, version, os_family
)
for row in existing):
raise TemplateError("此云镜像已经导入", 409)
if any(row["name"].casefold() == name.casefold() for row in existing):
raise TemplateError("已有同名云镜像模板", 409)
if pending_key in self.pending:
raise TemplateError("该云镜像正在导入", 409)
if self.pending:
raise TemplateError(
"已有云镜像正在导入,请等待完成后再提交", 409
)
job_id = str(uuid.uuid4())
job = {
"job_id": job_id,
"status": "queued",
"phase": "等待下载",
"created_at": int(time.time()),
}
self.pending.add(pending_key)
try:
self.save_job(job_id, job)
self.executor.submit(
self._run_import,
job_id,
url,
name,
expected_sha256,
pending_key,
)
except Exception as exc:
self.pending.discard(pending_key)
self.delete_job(job_id)
if isinstance(exc, RuntimeError):
raise TemplateError(
"引擎正在停止,无法开始云镜像导入", 503
) from exc
raise
return {"job_id": job_id, "status": "queued"}
def _run_import(self, job_id: str, url: str, name: str,
expected_sha256: str, pending_key: str) -> None:
def acquire() -> tuple[Path, int, str, str, str]:
with self.lock:
directory = self.files.directory()
maximum = self._remaining_download_bytes()
temporary, virtual_size, actual_sha256 = download_verified_image(
url, expected_sha256, directory, max_bytes=maximum
)
os_family, version = image_metadata_for(url)
return (
temporary, virtual_size, actual_sha256, os_family, version
)
self._run_import_job(
job_id,
name,
pending_key,
"正在下载并校验云镜像",
acquire,
)
def _run_uploaded_import(
self, job_id: str, image_path: Path, name: str,
os_family: str, version: str, digest: str, virtual_size: int,
reservation_id: str) -> None:
def acquire() -> tuple[Path, int, str, str, str]:
return image_path, virtual_size, digest, os_family, version
self._run_import_job(
job_id,
name,
reservation_id,
"正在校验并保存上传的云镜像",
acquire,
)
def _run_import_job(
self, job_id: str, name: str, pending_key: str,
running_phase: str, acquire) -> None:
"""Run the common catalog/publish transaction for every import source."""
temporary: Path | None = None
published = False
cataloged = False
commit_uncertain = False
template_id = ""
record: dict[str, Any] | None = None
try:
with self.lock:
job = self.jobs[job_id]
job.update(
status="running",
phase=running_phase,
started_at=int(time.time()),
)
self.save_job(job_id, job)
(temporary, virtual_size, actual_sha256,
os_family, version) = acquire()
template_id = cloud_template_id_for(
actual_sha256, version, os_family
)
size = temporary.stat().st_size
record = {
"id": template_id,
"name": name,
"os_family": os_family,
"os_version": version,
"sha256": actual_sha256,
"volume_name": template_volume_name(template_id),
"virtual_size": virtual_size,
"image_bytes": size,
"created_at": int(time.time()),
}
with self.lock:
existing = self.catalog.list()
if any(row["id"] == template_id for row in existing):
raise TemplateError("此云镜像已经导入", 409)
if any(row["name"].casefold() == name.casefold()
for row in existing):
raise TemplateError("已有同名云镜像模板", 409)
previous_created_at = self._prepare_reimport(template_id)
# created_at doubles as the public deletion generation. A
# durable counter prevents reuse even when an earlier import
# reserved a generation and failed before catalog commit.
record["created_at"] = self._next_generation(
template_id, previous_created_at or 0)
job = self.jobs[job_id]
job["phase"] = "正在保存云镜像源文件"
# Persist the deterministic candidate before publishing. If
# the process stops after the catalog commit but before the
# terminal job write, startup can prove that this interrupted
# job owns the committed catalog/source pair.
job["candidate_template_id"] = template_id
self.save_job(job_id, job)
source = self.files.publish(
temporary, template_id, expected_size=size
)
temporary = None
published = True
self._mark_verified(template_id, source, actual_sha256)
self.catalog.add(record)
cataloged = True
self._save_completed_job(job_id, job, record)
except Exception as exc:
# A database insert can commit and still raise (for example when
# the connection drops before its acknowledgement reaches this
# process). Once the source was published, never delete it until
# the catalog has been read back. An exact record is proof of a
# completed import; an unavailable read is ambiguous and must
# preserve the source for startup reconciliation.
committed: dict[str, Any] | None = None
if published and template_id and not cataloged:
try:
existing = self.catalog.get(template_id)
except TemplateError as read_exc:
if read_exc.status != 404:
commit_uncertain = True
except Exception:
commit_uncertain = True
LOG.exception(
"Could not resolve cloud template catalog commit: job=%s",
job_id,
)
else:
cataloged = True
if record is not None and existing == record:
committed = existing
if committed is not None:
with self.lock:
previous = self.jobs.get(job_id, {})
self._save_completed_job(job_id, previous, committed)
return
if isinstance(exc, TemplateError):
error = str(exc)
else:
LOG.exception("Global cloud template import failed: job=%s", job_id)
error = "云镜像导入失败,请查看引擎日志"
with self.lock:
if cataloged or commit_uncertain:
# A catalog write is the commit point. Do not remove a
# committed (or ambiguously committed) template merely
# because its acknowledgement failed. Startup can prove
# the candidate from the persisted running job.
pass
elif published and template_id:
try:
self.files.delete(template_id)
except Exception:
LOG.exception(
"Could not remove failed cloud template source: job=%s",
job_id,
)
previous = self.jobs.get(job_id, {})
failed = {
"job_id": job_id,
"status": "failed",
"phase": "导入失败",
"error": error,
"created_at": previous.get("created_at", int(time.time())),
"started_at": previous.get("started_at"),
"finished_at": int(time.time()),
}
if template_id:
failed["candidate_template_id"] = template_id
if commit_uncertain:
failed["commit_uncertain"] = True
try:
self.save_job(job_id, failed)
except Exception:
LOG.exception(
"Could not persist cloud template import failure: job=%s",
job_id,
)
finally:
if temporary is not None:
temporary.unlink(missing_ok=True)
with self.lock:
self.upload_reservations.pop(pending_key, None)
self.pending.discard(pending_key)
def _save_completed_job(self, job_id: str, previous: dict[str, Any],
record: dict[str, Any]) -> None:
"""Publish an in-memory success even if its final DB write fails.
The catalog record is the durable commit point. Reporting a failed
import after that point invites a duplicate retry, so the running
process must expose success while startup reconstructs the same result
from the candidate ID if the terminal job write was lost.
"""
completed = {
"job_id": job_id,
"status": "completed",
"phase": "云镜像已保存,可用于所有宿主机",
"created_at": previous.get("created_at", int(time.time())),
"started_at": previous.get("started_at"),
"finished_at": int(time.time()),
"template": record,
}
try:
self.save_job(job_id, completed)
except Exception:
# save_job updates self.jobs before touching persistent storage.
# Keep that completed state and let restore_jobs repair a stale
# persisted running record on the next process start.
self.jobs[job_id] = completed
LOG.exception(
"Could not persist completed cloud template import: job=%s",
job_id,
)
def _mark_verified(self, template_id: str, path: Path, digest: str) -> None:
info = path.stat()
self.verified_sources[template_id] = (
info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns, digest
)
def _verify_source(self, template_id: str, path: Path, digest: str) -> None:
info = path.stat()
identity = (info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns)
cached = self.verified_sources.get(template_id)
if cached is not None and cached[:4] == identity and cached[4] == digest:
return
checksum = hashlib.sha256()
with path.open("rb") as source:
while chunk := source.read(1024 * 1024):
checksum.update(chunk)
if checksum.hexdigest() != digest:
raise TemplateError("云镜像源文件的 SHA-256 校验值不匹配", 500)
self._mark_verified(template_id, path, digest)
@contextmanager
def source(self, template_id: str) -> Iterator[tuple[dict[str, Any], Path]]:
with self.lock:
record = self.catalog.get(template_id)
path = self.files.path(template_id, expected_size=record["image_bytes"])
self._verify_source(template_id, path, record["sha256"])
self.in_use[template_id] = self.in_use.get(template_id, 0) + 1
try:
yield record, path
finally:
with self.lock:
count = self.in_use.get(template_id, 0)
if count <= 1:
self.in_use.pop(template_id, None)
else:
self.in_use[template_id] = count - 1
def save_job(self, job_id: str, job: dict[str, Any]) -> None:
self.jobs[job_id] = job
if self.store is not None:
self.store.set(
IMPORT_JOB_KIND,
job_id,
json.dumps(job, ensure_ascii=False, separators=(",", ":")),
)
def delete_job(self, job_id: str) -> None:
self.jobs.pop(job_id, None)
if self.store is not None:
self.store.delete(IMPORT_JOB_KIND, job_id)
def restore_jobs(self) -> None:
"""Reconcile local files and fail imports interrupted by a restart."""
with self.lock:
if self._restored:
return
self.files.reconcile(self.catalog)
self._reconcile_cache_tombstones()
for record in self.catalog.list():
self._remember_generation(
record["id"], record["created_at"])
source = self.store.list(IMPORT_JOB_KIND) if self.store is not None else {}
for job_id, raw in source.items():
try:
job = json.loads(raw)
except (TypeError, ValueError) as exc:
raise TemplateError("云镜像导入任务记录损坏", 500) from exc
if not isinstance(job, dict) or job.get("job_id") != job_id:
raise TemplateError("云镜像导入任务记录损坏", 500)
interrupted = job.get("status") in {"queued", "running"}
uncertain = job.get("commit_uncertain") is True
if interrupted or uncertain:
candidate = job.get("candidate_template_id")
committed: dict[str, Any] | None = None
if isinstance(candidate, str):
try:
committed = self.catalog.get(candidate)
path = self.files.path(
candidate,
expected_size=committed["image_bytes"],
)
self._verify_source(
candidate, path, committed["sha256"]
)
except Exception:
committed = None
if committed is not None:
self._save_completed_job(job_id, job, committed)
continue
job.update(
status="failed",
phase="引擎重启,任务已中断",
error="面板在云镜像导入期间重启;请重新提交导入",
finished_at=int(time.time()),
)
self.save_job(job_id, job)
self.jobs[job_id] = job
self._restored = True
def has_active_jobs(self) -> bool:
with self.lock:
return bool(self.pending) or any(
job.get("status") in {"queued", "running"}
for job in self.jobs.values()
)
def close(self) -> None:
self.executor.shutdown(wait=True, cancel_futures=False)