You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1117 lines
49 KiB
Python
1117 lines
49 KiB
Python
"""Engine-global cloud template catalog and persistent source images.
|
|
|
|
The legacy cloud-template flow downloads an image inside one host child app,
|
|
uploads it immediately to that host, and then deletes the local download. A
|
|
multi-host control plane needs a different ownership boundary: the engine owns
|
|
one immutable source image and each hypervisor owns only a disposable replica.
|
|
|
|
This module deliberately contains no libvirt code. It provides the global
|
|
catalog, import jobs and verified ``source()`` leases that a host-specific
|
|
cache layer can consume later.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import logging
|
|
import os
|
|
import re
|
|
import shutil
|
|
import stat
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from contextlib import contextmanager
|
|
from pathlib import Path
|
|
from typing import Any, Iterator
|
|
|
|
from .cloud_templates import (
|
|
GIB,
|
|
MAX_IMAGE_BYTES,
|
|
MAX_VIRTUAL_BYTES,
|
|
SHA256_RE,
|
|
TemplateError,
|
|
download_verified_image,
|
|
image_metadata_for,
|
|
template_id_for,
|
|
template_volume_name,
|
|
validate_template_metadata,
|
|
validate_template_name,
|
|
)
|
|
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
# Keep the existing, already validated template/volume identifier grammar, but
|
|
# replace its old host scope with one fixed engine-global namespace. The full
|
|
# digest remains in the catalog and every source/replica marker must compare it;
|
|
# the shortened digest in the public ID is never treated as integrity proof.
|
|
GLOBAL_TEMPLATE_SCOPE = hashlib.sha256(
|
|
b"virt-cloud/global-cloud-template/v1"
|
|
).hexdigest()
|
|
|
|
CATALOG_KIND = "cloud_template"
|
|
IMPORT_JOB_KIND = "cloud_template_import_job"
|
|
CACHE_TOMBSTONE_KIND = "cloud_template_tombstone"
|
|
GENERATION_KIND = "cloud_template_generation"
|
|
SOURCE_DIRECTORY_NAME = "cloud-templates"
|
|
PERSISTENT_FREE_RESERVE_BYTES = 256 * 1024**2
|
|
MIN_DOWNLOAD_BYTES = 1024 * 1024
|
|
SOURCE_FILE_RE = re.compile(
|
|
r"panel-template-(?:alma(?:8|9|10)|debian(?:11|12|13)|"
|
|
r"ubuntu(?:2004|2204|2404|2504|2510|2604)|"
|
|
r"centosstream(?:9|10))-"
|
|
r"[0-9a-f]{8}-[0-9a-f]{16}\.qcow2\Z"
|
|
)
|
|
TEMPORARY_FILE_RE = re.compile(r"\.cloud-image-[A-Za-z0-9_-]+\.qcow2\Z")
|
|
|
|
|
|
def cloud_template_id_for(digest: str, version: str,
|
|
os_family: str = "AlmaLinux") -> str:
|
|
"""Return a stable ID that is independent of every target hypervisor."""
|
|
if not isinstance(digest, str) or not SHA256_RE.fullmatch(digest):
|
|
raise TemplateError("云镜像 SHA-256 格式无效")
|
|
return template_id_for(
|
|
GLOBAL_TEMPLATE_SCOPE, digest.lower(), version, os_family
|
|
)
|
|
|
|
|
|
class GlobalCloudTemplateStore:
|
|
"""Strict global catalog backed by its own encrypted record kind."""
|
|
|
|
def __init__(self, store: Any | None) -> None:
|
|
self.store = store
|
|
self._memory: dict[str, str] = {}
|
|
self._lock = threading.RLock()
|
|
|
|
@staticmethod
|
|
def _validate(record: Any) -> dict[str, Any]:
|
|
if not isinstance(record, dict):
|
|
raise TemplateError("全局云镜像目录记录损坏", 500)
|
|
try:
|
|
template_id = record["id"]
|
|
name = record["name"]
|
|
os_family = record["os_family"]
|
|
os_version = record["os_version"]
|
|
digest = record["sha256"]
|
|
volume_name = record["volume_name"]
|
|
virtual_size = record["virtual_size"]
|
|
image_bytes = record["image_bytes"]
|
|
created_at = record["created_at"]
|
|
except KeyError as exc:
|
|
raise TemplateError("全局云镜像目录记录损坏", 500) from exc
|
|
try:
|
|
normalized_name = validate_template_name(name)
|
|
except TemplateError as exc:
|
|
raise TemplateError("全局云镜像目录记录损坏", 500) from exc
|
|
try:
|
|
expected_id = cloud_template_id_for(
|
|
digest, os_version, os_family
|
|
)
|
|
expected_volume_name = template_volume_name(template_id)
|
|
except TemplateError as exc:
|
|
raise TemplateError("全局云镜像目录记录损坏", 500) from exc
|
|
valid = (
|
|
isinstance(template_id, str)
|
|
and isinstance(name, str)
|
|
and normalized_name == name
|
|
and os_family in {"AlmaLinux", "Debian", "Ubuntu", "CentOS Stream"}
|
|
and isinstance(os_version, str)
|
|
and isinstance(digest, str)
|
|
and SHA256_RE.fullmatch(digest) is not None
|
|
and digest == digest.lower()
|
|
and template_id == expected_id
|
|
and volume_name == expected_volume_name
|
|
and not isinstance(virtual_size, bool)
|
|
and isinstance(virtual_size, int)
|
|
and GIB <= virtual_size <= MAX_VIRTUAL_BYTES
|
|
and not isinstance(image_bytes, bool)
|
|
and isinstance(image_bytes, int)
|
|
and 0 < image_bytes <= MAX_IMAGE_BYTES
|
|
and not isinstance(created_at, bool)
|
|
and isinstance(created_at, int)
|
|
and created_at > 0
|
|
)
|
|
if not valid:
|
|
raise TemplateError("全局云镜像目录记录损坏", 500)
|
|
# Reject unrecognised fields instead of accidentally persisting a URL,
|
|
# local path, host identity or another future secret in public records.
|
|
if set(record) != {
|
|
"id", "name", "os_family", "os_version", "sha256",
|
|
"volume_name", "virtual_size", "image_bytes", "created_at",
|
|
}:
|
|
raise TemplateError("全局云镜像目录记录损坏", 500)
|
|
return record
|
|
|
|
def _source(self) -> dict[str, str]:
|
|
return self.store.list(CATALOG_KIND) if self.store is not None else self._memory.copy()
|
|
|
|
def _list(self) -> list[dict[str, Any]]:
|
|
records = []
|
|
for key, payload in self._source().items():
|
|
try:
|
|
record = json.loads(payload)
|
|
except (TypeError, ValueError) as exc:
|
|
raise TemplateError("全局云镜像目录记录损坏", 500) from exc
|
|
record = self._validate(record)
|
|
if key != record["id"]:
|
|
raise TemplateError("全局云镜像目录记录损坏", 500)
|
|
records.append(record)
|
|
return sorted(
|
|
records,
|
|
key=lambda item: (item["created_at"], item["id"]),
|
|
reverse=True,
|
|
)
|
|
|
|
def list(self) -> list[dict[str, Any]]:
|
|
with self._lock:
|
|
return self._list()
|
|
|
|
def get(self, template_id: str) -> dict[str, Any]:
|
|
# This validates the opaque identifier before consulting storage.
|
|
template_volume_name(template_id)
|
|
with self._lock:
|
|
for record in self._list():
|
|
if record["id"] == template_id:
|
|
return record
|
|
raise TemplateError("云镜像模板不存在", 404)
|
|
|
|
def add(self, record: dict[str, Any]) -> dict[str, Any]:
|
|
record = self._validate(record.copy())
|
|
encoded = json.dumps(record, ensure_ascii=False, separators=(",", ":"))
|
|
with self._lock:
|
|
current = self._list()
|
|
if any(row["id"] == record["id"] for row in current):
|
|
raise TemplateError("此云镜像已经导入", 409)
|
|
if any(row["name"].casefold() == record["name"].casefold()
|
|
for row in current):
|
|
raise TemplateError("已有同名云镜像模板", 409)
|
|
if self.store is not None:
|
|
if not self.store.insert(CATALOG_KIND, record["id"], encoded):
|
|
raise TemplateError("此云镜像已经导入", 409)
|
|
else:
|
|
self._memory[record["id"]] = encoded
|
|
return record
|
|
|
|
def rename(self, template_id: str, name: str) -> dict[str, Any]:
|
|
name = validate_template_name(name)
|
|
with self._lock:
|
|
current = self._list()
|
|
target = next((row for row in current if row["id"] == template_id), None)
|
|
if target is None:
|
|
raise TemplateError("云镜像模板不存在", 404)
|
|
if any(row["id"] != template_id
|
|
and row["name"].casefold() == name.casefold()
|
|
for row in current):
|
|
raise TemplateError("已有同名云镜像模板", 409)
|
|
updated = {**target, "name": name}
|
|
encoded = json.dumps(updated, ensure_ascii=False, separators=(",", ":"))
|
|
if self.store is not None:
|
|
self.store.set(CATALOG_KIND, template_id, encoded)
|
|
else:
|
|
self._memory[template_id] = encoded
|
|
return updated
|
|
|
|
def delete(self, template_id: str) -> None:
|
|
# Validate the opaque identifier before touching persistent storage.
|
|
template_volume_name(template_id)
|
|
with self._lock:
|
|
self.get(template_id)
|
|
if self.store is not None:
|
|
self.store.delete(CATALOG_KIND, template_id)
|
|
else:
|
|
self._memory.pop(template_id, None)
|
|
|
|
|
|
class GlobalCloudTemplateFiles:
|
|
"""Private engine-owned source files, never paths supplied by a caller."""
|
|
|
|
def __init__(self, data_dir: Path) -> None:
|
|
data_dir = Path(data_dir)
|
|
if not data_dir.is_absolute() or data_dir == Path("/"):
|
|
raise ValueError("cloud template data directory must be absolute and dedicated")
|
|
self.data_dir = data_dir
|
|
self.root = data_dir / SOURCE_DIRECTORY_NAME
|
|
|
|
@staticmethod
|
|
def _ensure_directory(path: Path) -> None:
|
|
try:
|
|
info = path.lstat()
|
|
except FileNotFoundError:
|
|
path.mkdir(mode=0o700)
|
|
info = path.lstat()
|
|
if (not stat.S_ISDIR(info.st_mode) or stat.S_ISLNK(info.st_mode)
|
|
or info.st_uid != os.geteuid()):
|
|
raise TemplateError("云镜像持久目录不安全,请联系管理员", 500)
|
|
if stat.S_IMODE(info.st_mode) != 0o700:
|
|
os.chmod(path, 0o700)
|
|
|
|
def directory(self) -> Path:
|
|
self.data_dir.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
self._ensure_directory(self.data_dir)
|
|
self._ensure_directory(self.root)
|
|
return self.root
|
|
|
|
def available_bytes(self) -> int:
|
|
"""Return writable bytes while preserving space for panel operation."""
|
|
free = int(shutil.disk_usage(self.directory()).free)
|
|
return max(0, free - PERSISTENT_FREE_RESERVE_BYTES)
|
|
|
|
def path(self, template_id: str, *, expected_size: int | None = None) -> Path:
|
|
path = self.directory() / template_volume_name(template_id)
|
|
try:
|
|
info = path.lstat()
|
|
except FileNotFoundError as exc:
|
|
raise TemplateError("云镜像源文件不存在,请重新导入", 409) from exc
|
|
if (not stat.S_ISREG(info.st_mode) or stat.S_ISLNK(info.st_mode)
|
|
or info.st_uid != os.geteuid()
|
|
or stat.S_IMODE(info.st_mode) != 0o600
|
|
or (expected_size is not None and info.st_size != expected_size)):
|
|
raise TemplateError("云镜像源文件已损坏,请联系管理员", 500)
|
|
return path
|
|
|
|
def publish(self, temporary: Path, template_id: str, *, expected_size: int) -> Path:
|
|
temporary = Path(temporary)
|
|
directory = self.directory()
|
|
try:
|
|
info = temporary.lstat()
|
|
except FileNotFoundError as exc:
|
|
raise TemplateError("云镜像下载临时文件丢失", 500) from exc
|
|
if (not stat.S_ISREG(info.st_mode) or stat.S_ISLNK(info.st_mode)
|
|
or info.st_uid != os.geteuid()
|
|
or stat.S_IMODE(info.st_mode) != 0o600
|
|
or temporary.parent != directory
|
|
or info.st_size != expected_size
|
|
or expected_size <= 0 or expected_size > MAX_IMAGE_BYTES):
|
|
raise TemplateError("云镜像下载临时文件无效", 500)
|
|
destination = directory / template_volume_name(template_id)
|
|
try:
|
|
# download_verified_image creates the temporary in this directory.
|
|
# A hard link publishes atomically and cannot replace an existing
|
|
# immutable source with attacker-controlled bytes.
|
|
os.link(temporary, destination)
|
|
os.chmod(destination, 0o600)
|
|
temporary.unlink()
|
|
descriptor = os.open(
|
|
destination.parent,
|
|
os.O_RDONLY | getattr(os, "O_DIRECTORY", 0),
|
|
)
|
|
try:
|
|
os.fsync(descriptor)
|
|
finally:
|
|
os.close(descriptor)
|
|
except FileExistsError as exc:
|
|
raise TemplateError("此云镜像已经导入", 409) from exc
|
|
except OSError as exc:
|
|
destination.unlink(missing_ok=True)
|
|
raise TemplateError("云镜像无法保存到持久目录", 507) from exc
|
|
return destination
|
|
|
|
def delete(self, template_id: str) -> None:
|
|
path = self.directory() / template_volume_name(template_id)
|
|
try:
|
|
info = path.lstat()
|
|
except FileNotFoundError:
|
|
return
|
|
if not stat.S_ISREG(info.st_mode) or stat.S_ISLNK(info.st_mode):
|
|
raise TemplateError("云镜像源文件无法安全删除", 500)
|
|
try:
|
|
path.unlink()
|
|
except OSError as exc:
|
|
raise TemplateError("云镜像源文件删除失败,请联系管理员", 500) from exc
|
|
|
|
def reconcile(self, catalog: GlobalCloudTemplateStore) -> None:
|
|
"""Remove only provable interrupted downloads and uncataloged sources."""
|
|
try:
|
|
self.root.lstat()
|
|
except FileNotFoundError:
|
|
return
|
|
self._ensure_directory(self.root)
|
|
active = {row["volume_name"] for row in catalog.list()}
|
|
for candidate in self.root.iterdir():
|
|
try:
|
|
info = candidate.lstat()
|
|
except OSError:
|
|
continue
|
|
if (not stat.S_ISREG(info.st_mode) or stat.S_ISLNK(info.st_mode)
|
|
or info.st_uid != os.geteuid()):
|
|
continue
|
|
interrupted = TEMPORARY_FILE_RE.fullmatch(candidate.name) is not None
|
|
orphan = (SOURCE_FILE_RE.fullmatch(candidate.name) is not None
|
|
and candidate.name not in active)
|
|
if interrupted or orphan:
|
|
candidate.unlink(missing_ok=True)
|
|
|
|
|
|
class CloudTemplateRuntime:
|
|
"""One engine-process runtime shared by all host child applications."""
|
|
|
|
def __init__(self, store: Any | None, data_dir: Path) -> None:
|
|
self.store = store
|
|
self.catalog = GlobalCloudTemplateStore(store)
|
|
self.files = GlobalCloudTemplateFiles(data_dir)
|
|
self.executor = ThreadPoolExecutor(
|
|
max_workers=1, thread_name_prefix="global-cloud-template-import"
|
|
)
|
|
self.lock = threading.RLock()
|
|
self.jobs: dict[str, dict[str, Any]] = {}
|
|
self.pending: set[str] = set()
|
|
self.upload_reservations: dict[str, dict[str, str]] = {}
|
|
self.in_use: dict[str, int] = {}
|
|
self.verified_sources: dict[str, tuple[int, int, int, int, str]] = {}
|
|
self.memory_generations: dict[str, int] = {}
|
|
self._restored = False
|
|
|
|
@staticmethod
|
|
def public_record(record: dict[str, Any]) -> dict[str, Any]:
|
|
return {key: value for key, value in record.items() if key in {
|
|
"id", "name", "os_family", "os_version", "sha256",
|
|
"virtual_size", "image_bytes", "created_at",
|
|
}}
|
|
|
|
@staticmethod
|
|
def public_error(value: Any) -> str:
|
|
safe = {
|
|
"面板在云镜像导入期间重启;请重新提交导入",
|
|
"云镜像地址无效",
|
|
"仅支持 AlmaLinux 官方仓库的 9/10 x86_64 GenericCloud qcow2 镜像",
|
|
"仅支持 AlmaLinux、Debian、Ubuntu 和 CentOS Stream 官方 x86_64 云镜像直链",
|
|
"官方镜像地址发生重定向,请使用最终的官方 HTTPS 直链",
|
|
"官方云镜像地址无法解析",
|
|
"官方云镜像地址解析结果无效",
|
|
"官方云镜像地址解析到非公网地址",
|
|
"无法连接官方云镜像下载地址",
|
|
"官方云镜像重定向目标不安全",
|
|
"官方云镜像重定向次数过多",
|
|
"官方云镜像下载服务器返回异常状态",
|
|
"官方云镜像下载响应格式无效",
|
|
"云镜像下载内容不完整",
|
|
"SHA-256 可留空;填写时必须为 64 位十六进制校验值",
|
|
"下载官方云镜像失败,请检查面板服务器网络",
|
|
"云镜像下载超过 60 分钟,请检查面板服务器网络",
|
|
"云镜像超过 4 GiB 限制",
|
|
"云镜像 SHA-256 不匹配,已丢弃下载内容",
|
|
"下载文件不是有效的 qcow2 镜像",
|
|
"云镜像 qcow2 版本或簇大小不受支持",
|
|
"云镜像 qcow2 头部不完整",
|
|
"云镜像包含不受支持或外部依赖的 qcow2 特性",
|
|
"云镜像含外部依赖、加密或容量超出支持范围",
|
|
"此云镜像已经导入",
|
|
"已有同名云镜像模板",
|
|
"云镜像无法保存到持久目录",
|
|
"面板云镜像持久目录空间不足",
|
|
"云镜像缓存清理记录损坏,请联系管理员",
|
|
"云镜像版本记录损坏,请联系管理员",
|
|
"旧云镜像缓存仍在清理,请稍后重新导入",
|
|
"云镜像导入失败,请查看引擎日志",
|
|
}
|
|
if isinstance(value, str) and value in safe:
|
|
return value
|
|
return "云镜像下载或保存失败,请联系管理员查看日志"
|
|
|
|
@classmethod
|
|
def public_job(cls, job: dict[str, Any]) -> dict[str, Any]:
|
|
allowed = {
|
|
"job_id", "status", "phase", "created_at", "started_at",
|
|
"finished_at", "template", "error",
|
|
}
|
|
result = {key: value for key, value in job.items() if key in allowed}
|
|
if isinstance(result.get("template"), dict):
|
|
result["template"] = cls.public_record(result["template"])
|
|
if "error" in result:
|
|
result["error"] = cls.public_error(result["error"])
|
|
return result
|
|
|
|
def list(self) -> list[dict[str, Any]]:
|
|
"""Return the public, host-independent template list."""
|
|
with self.lock:
|
|
return [self.public_record(row) for row in self.catalog.list()]
|
|
|
|
def status(self) -> dict[str, Any]:
|
|
"""Return the public list plus the current import, when one exists."""
|
|
result: dict[str, Any] = {"templates": self.list()}
|
|
with self.lock:
|
|
active = [job for job in self.jobs.values()
|
|
if job.get("status") in {"queued", "running"}]
|
|
if active:
|
|
current = max(active, key=lambda job: (
|
|
int(job.get("created_at", 0)), str(job.get("job_id", ""))))
|
|
result["active_import_job"] = self.public_job(current)
|
|
return result
|
|
|
|
def get(self, template_id: str) -> dict[str, Any]:
|
|
return self.catalog.get(template_id)
|
|
|
|
def get_job(self, job_id: str) -> dict[str, Any]:
|
|
with self.lock:
|
|
job = self.jobs.get(job_id)
|
|
if job is None:
|
|
raise TemplateError("云镜像导入任务不存在", 404)
|
|
return self.public_job(job)
|
|
|
|
def rename(self, template_id: str, name: str) -> dict[str, Any]:
|
|
with self.lock:
|
|
if self.pending:
|
|
raise TemplateError("云镜像正在导入,请稍后修改模板名称", 409)
|
|
return self.public_record(self.catalog.rename(template_id, name))
|
|
|
|
def delete(self, template_id: str) -> None:
|
|
with self.lock:
|
|
if self.pending:
|
|
raise TemplateError("云镜像正在导入,请稍后删除模板", 409)
|
|
if self.in_use.get(template_id, 0):
|
|
raise TemplateError("云镜像正在同步到宿主机,请稍后删除", 409)
|
|
self.catalog.get(template_id)
|
|
# Delete bytes first. If the catalog write then fails, a retry can
|
|
# finish the deletion without leaving unaccounted source files.
|
|
self.files.delete(template_id)
|
|
self.catalog.delete(template_id)
|
|
self.verified_sources.pop(template_id, None)
|
|
|
|
def _remaining_download_bytes(self) -> int:
|
|
maximum = min(MAX_IMAGE_BYTES, self.files.available_bytes())
|
|
if maximum < MIN_DOWNLOAD_BYTES:
|
|
raise TemplateError("面板云镜像持久目录空间不足", 507)
|
|
return maximum
|
|
|
|
def _remember_generation(self, template_id: str, created_at: int) -> None:
|
|
if isinstance(created_at, bool) or not isinstance(created_at, int) \
|
|
or created_at < 1:
|
|
raise TemplateError("全局云镜像目录记录损坏", 500)
|
|
if self.store is None:
|
|
self.memory_generations[template_id] = max(
|
|
created_at, self.memory_generations.get(template_id, 0))
|
|
return
|
|
raw = self.store.get(GENERATION_KIND, template_id)
|
|
try:
|
|
previous = 0 if raw is None else int(raw)
|
|
except (TypeError, ValueError) as exc:
|
|
raise TemplateError("云镜像版本记录损坏,请联系管理员", 500) from exc
|
|
if previous < 0 or (raw is not None and str(previous) != raw):
|
|
raise TemplateError("云镜像版本记录损坏,请联系管理员", 500)
|
|
if created_at > previous:
|
|
self.store.set(GENERATION_KIND, template_id, str(created_at))
|
|
|
|
def _next_generation(self, template_id: str, floor: int = 0) -> int:
|
|
if self.store is None:
|
|
previous = self.memory_generations.get(template_id, 0)
|
|
else:
|
|
raw = self.store.get(GENERATION_KIND, template_id)
|
|
try:
|
|
previous = 0 if raw is None else int(raw)
|
|
except (TypeError, ValueError) as exc:
|
|
raise TemplateError(
|
|
"云镜像版本记录损坏,请联系管理员", 500
|
|
) from exc
|
|
if previous < 0 or (raw is not None and str(previous) != raw):
|
|
raise TemplateError("云镜像版本记录损坏,请联系管理员", 500)
|
|
generation = max(int(time.time()), previous + 1, floor + 1)
|
|
if self.store is None:
|
|
self.memory_generations[template_id] = generation
|
|
else:
|
|
self.store.set(GENERATION_KIND, template_id, str(generation))
|
|
return generation
|
|
|
|
@staticmethod
|
|
def _decode_cache_tombstone(template_id: str, raw: str) -> dict[str, Any]:
|
|
try:
|
|
tombstone = json.loads(raw)
|
|
deletion_id = tombstone["deletion_id"]
|
|
template_created_at = tombstone["template_created_at"]
|
|
pending_hosts = tombstone["pending_hosts"]
|
|
except (KeyError, TypeError, ValueError) as exc:
|
|
raise TemplateError(
|
|
"云镜像缓存清理记录损坏,请联系管理员", 500
|
|
) from exc
|
|
if (not isinstance(tombstone, dict)
|
|
or not isinstance(deletion_id, str)
|
|
or re.fullmatch(r"[0-9a-f]{32}", deletion_id) is None
|
|
or isinstance(template_created_at, bool)
|
|
or not isinstance(template_created_at, int)
|
|
or template_created_at < 1
|
|
or tombstone.get("template_id") != template_id
|
|
or tombstone.get("source_volume_name")
|
|
!= template_volume_name(template_id)
|
|
or not isinstance(tombstone.get("source_deleted"), bool)
|
|
or not isinstance(pending_hosts, list)
|
|
or any(not isinstance(host_id, str) or not host_id
|
|
for host_id in pending_hosts)
|
|
or len(set(pending_hosts)) != len(pending_hosts)
|
|
or isinstance(tombstone.get("created_at"), bool)
|
|
or not isinstance(tombstone.get("created_at"), int)
|
|
or tombstone["created_at"] < 1):
|
|
raise TemplateError(
|
|
"云镜像缓存清理记录损坏,请联系管理员", 500
|
|
)
|
|
return tombstone
|
|
|
|
def _reconcile_cache_tombstone(
|
|
self, template_id: str, tombstone: dict[str, Any]
|
|
) -> dict[str, Any] | None:
|
|
"""Resolve the crash window around source/catalog removal.
|
|
|
|
A provisional tombstone is written before unlinking the immutable
|
|
source. If the engine stops before the final flag is persisted, the
|
|
catalog and source together prove whether deletion started: a valid
|
|
pair cancels the provisional marker, while either missing half means
|
|
the requested deletion must be completed before host garbage
|
|
collection can continue.
|
|
"""
|
|
if self.store is None or tombstone["source_deleted"]:
|
|
return tombstone
|
|
try:
|
|
record = self.catalog.get(template_id)
|
|
except TemplateError as exc:
|
|
if exc.status != 404:
|
|
raise
|
|
record = None
|
|
if record is not None:
|
|
if (record["created_at"] != tombstone["template_created_at"]
|
|
or record["volume_name"]
|
|
!= tombstone["source_volume_name"]):
|
|
raise TemplateError(
|
|
"云镜像缓存清理记录损坏,请联系管理员", 500
|
|
)
|
|
try:
|
|
self.files.path(
|
|
template_id, expected_size=record["image_bytes"])
|
|
except TemplateError:
|
|
# The immutable source is already absent or damaged after a
|
|
# committed deletion attempt. Finish retiring the catalog.
|
|
self.files.delete(template_id)
|
|
self.catalog.delete(template_id)
|
|
self.verified_sources.pop(template_id, None)
|
|
else:
|
|
# Both durable halves still exist: deletion never committed.
|
|
self.store.delete(CACHE_TOMBSTONE_KIND, template_id)
|
|
return None
|
|
else:
|
|
# The catalog is gone, so ensure no source bytes survived an
|
|
# interrupted or externally retried database operation.
|
|
self.files.delete(template_id)
|
|
self.verified_sources.pop(template_id, None)
|
|
tombstone["source_deleted"] = True
|
|
self.store.set(
|
|
CACHE_TOMBSTONE_KIND,
|
|
template_id,
|
|
json.dumps(tombstone, separators=(",", ":")),
|
|
)
|
|
return tombstone
|
|
|
|
def _prepare_reimport(self, template_id: str) -> int | None:
|
|
"""Retire a completed deletion marker before reusing a stable ID.
|
|
|
|
Template IDs are content-derived, so importing identical bytes after
|
|
a deletion intentionally produces the same identifier. Reuse is safe
|
|
only after every old host replica has been reclaimed; otherwise a
|
|
delayed cleanup task could delete a newly-created replica.
|
|
"""
|
|
if self.store is None:
|
|
return None
|
|
raw = self.store.get(CACHE_TOMBSTONE_KIND, template_id)
|
|
if raw is None:
|
|
return None
|
|
tombstone = self._decode_cache_tombstone(template_id, raw)
|
|
tombstone = self._reconcile_cache_tombstone(
|
|
template_id, tombstone)
|
|
if tombstone is None:
|
|
return None
|
|
template_created_at = tombstone["template_created_at"]
|
|
pending_hosts = tombstone["pending_hosts"]
|
|
if tombstone["source_deleted"] is not True:
|
|
raise TemplateError(
|
|
"云镜像缓存清理记录损坏,请联系管理员", 500
|
|
)
|
|
if pending_hosts:
|
|
raise TemplateError("旧云镜像缓存仍在清理,请稍后重新导入", 409)
|
|
self.store.delete(CACHE_TOMBSTONE_KIND, template_id)
|
|
return template_created_at
|
|
|
|
def _reconcile_cache_tombstones(self) -> None:
|
|
"""Advance provisional delete records after a crash or lost DB ack."""
|
|
if self.store is None:
|
|
return
|
|
for template_id, raw in self.store.list(
|
|
CACHE_TOMBSTONE_KIND).items():
|
|
tombstone = self._decode_cache_tombstone(template_id, raw)
|
|
self._reconcile_cache_tombstone(template_id, tombstone)
|
|
|
|
def reserve_upload(self, name: str, os_family: str,
|
|
os_version: str) -> dict[str, Any]:
|
|
"""Reserve the single import slot before receiving a browser upload."""
|
|
name = validate_template_name(name)
|
|
os_family, os_version = validate_template_metadata(
|
|
os_family, os_version)
|
|
reservation_id = "upload-" + uuid.uuid4().hex
|
|
with self.lock:
|
|
existing = self.catalog.list()
|
|
directory = self.files.directory()
|
|
maximum = self._remaining_download_bytes()
|
|
if any(row["name"].casefold() == name.casefold()
|
|
for row in existing):
|
|
raise TemplateError("已有同名云镜像模板", 409)
|
|
if self.pending:
|
|
raise TemplateError(
|
|
"已有云镜像正在导入,请等待完成后再提交", 409
|
|
)
|
|
self.pending.add(reservation_id)
|
|
self.upload_reservations[reservation_id] = {
|
|
"name": name,
|
|
"os_family": os_family,
|
|
"os_version": os_version,
|
|
}
|
|
return {
|
|
"reservation_id": reservation_id,
|
|
"directory": directory,
|
|
"max_bytes": maximum,
|
|
}
|
|
|
|
def cancel_upload(self, reservation_id: str) -> None:
|
|
"""Release a pre-body reservation after validation or transport failure."""
|
|
with self.lock:
|
|
self.upload_reservations.pop(reservation_id, None)
|
|
self.pending.discard(reservation_id)
|
|
|
|
def queue_upload(self, reservation_id: str, image_path: Path,
|
|
digest: str, virtual_size: int) -> dict[str, str]:
|
|
"""Queue publication of an already streamed and validated qcow2 file."""
|
|
image_path = Path(image_path)
|
|
if not isinstance(digest, str) or SHA256_RE.fullmatch(digest) is None:
|
|
raise TemplateError("云镜像 SHA-256 格式无效", 422)
|
|
digest = digest.lower()
|
|
if (isinstance(virtual_size, bool) or not isinstance(virtual_size, int)
|
|
or not GIB <= virtual_size <= MAX_VIRTUAL_BYTES):
|
|
raise TemplateError("云镜像系统盘容量超出支持范围", 422)
|
|
try:
|
|
info = image_path.lstat()
|
|
except FileNotFoundError as exc:
|
|
raise TemplateError("上传的云镜像临时文件丢失", 500) from exc
|
|
if (not stat.S_ISREG(info.st_mode) or stat.S_ISLNK(info.st_mode)
|
|
or info.st_uid != os.geteuid()
|
|
or stat.S_IMODE(info.st_mode) != 0o600
|
|
or image_path.parent != self.files.directory()
|
|
or info.st_size <= 0 or info.st_size > MAX_IMAGE_BYTES):
|
|
raise TemplateError("上传的云镜像临时文件无效", 500)
|
|
with self.lock:
|
|
metadata = self.upload_reservations.get(reservation_id)
|
|
if metadata is None or reservation_id not in self.pending:
|
|
raise TemplateError("云镜像上传预约已失效,请重新上传", 409)
|
|
name = metadata["name"]
|
|
os_family = metadata["os_family"]
|
|
os_version = metadata["os_version"]
|
|
template_id = cloud_template_id_for(
|
|
digest, os_version, os_family)
|
|
existing = self.catalog.list()
|
|
if any(row["id"] == template_id for row in existing):
|
|
raise TemplateError("此云镜像已经导入", 409)
|
|
if any(row["name"].casefold() == name.casefold()
|
|
for row in existing):
|
|
raise TemplateError("已有同名云镜像模板", 409)
|
|
job_id = str(uuid.uuid4())
|
|
job = {
|
|
"job_id": job_id,
|
|
"status": "queued",
|
|
"phase": "等待保存上传的云镜像",
|
|
"created_at": int(time.time()),
|
|
}
|
|
try:
|
|
self.save_job(job_id, job)
|
|
self.executor.submit(
|
|
self._run_uploaded_import,
|
|
job_id,
|
|
image_path,
|
|
name,
|
|
os_family,
|
|
os_version,
|
|
digest,
|
|
virtual_size,
|
|
reservation_id,
|
|
)
|
|
except Exception as exc:
|
|
self.delete_job(job_id)
|
|
self.upload_reservations.pop(reservation_id, None)
|
|
self.pending.discard(reservation_id)
|
|
if isinstance(exc, RuntimeError):
|
|
raise TemplateError(
|
|
"引擎正在停止,无法开始云镜像导入", 503
|
|
) from exc
|
|
raise
|
|
return {"job_id": job_id, "status": "queued"}
|
|
|
|
def queue_import(self, url: str, name: str,
|
|
expected_sha256: str = "") -> dict[str, str]:
|
|
os_family, version = image_metadata_for(url)
|
|
name = validate_template_name(name)
|
|
if (not isinstance(expected_sha256, str)
|
|
or (expected_sha256 and not SHA256_RE.fullmatch(expected_sha256))):
|
|
raise TemplateError(
|
|
"SHA-256 可留空;填写时必须为 64 位十六进制校验值", 422
|
|
)
|
|
expected_sha256 = expected_sha256.lower()
|
|
pending_key = expected_sha256 or hashlib.sha256(url.encode("utf-8")).hexdigest()
|
|
with self.lock:
|
|
existing = self.catalog.list()
|
|
self._remaining_download_bytes()
|
|
if expected_sha256 and any(
|
|
row["id"] == cloud_template_id_for(
|
|
expected_sha256, version, os_family
|
|
)
|
|
for row in existing):
|
|
raise TemplateError("此云镜像已经导入", 409)
|
|
if any(row["name"].casefold() == name.casefold() for row in existing):
|
|
raise TemplateError("已有同名云镜像模板", 409)
|
|
if pending_key in self.pending:
|
|
raise TemplateError("该云镜像正在导入", 409)
|
|
if self.pending:
|
|
raise TemplateError(
|
|
"已有云镜像正在导入,请等待完成后再提交", 409
|
|
)
|
|
job_id = str(uuid.uuid4())
|
|
job = {
|
|
"job_id": job_id,
|
|
"status": "queued",
|
|
"phase": "等待下载",
|
|
"created_at": int(time.time()),
|
|
}
|
|
self.pending.add(pending_key)
|
|
try:
|
|
self.save_job(job_id, job)
|
|
self.executor.submit(
|
|
self._run_import,
|
|
job_id,
|
|
url,
|
|
name,
|
|
expected_sha256,
|
|
pending_key,
|
|
)
|
|
except Exception as exc:
|
|
self.pending.discard(pending_key)
|
|
self.delete_job(job_id)
|
|
if isinstance(exc, RuntimeError):
|
|
raise TemplateError(
|
|
"引擎正在停止,无法开始云镜像导入", 503
|
|
) from exc
|
|
raise
|
|
return {"job_id": job_id, "status": "queued"}
|
|
|
|
def _run_import(self, job_id: str, url: str, name: str,
|
|
expected_sha256: str, pending_key: str) -> None:
|
|
def acquire() -> tuple[Path, int, str, str, str]:
|
|
with self.lock:
|
|
directory = self.files.directory()
|
|
maximum = self._remaining_download_bytes()
|
|
temporary, virtual_size, actual_sha256 = download_verified_image(
|
|
url, expected_sha256, directory, max_bytes=maximum
|
|
)
|
|
os_family, version = image_metadata_for(url)
|
|
return (
|
|
temporary, virtual_size, actual_sha256, os_family, version
|
|
)
|
|
|
|
self._run_import_job(
|
|
job_id,
|
|
name,
|
|
pending_key,
|
|
"正在下载并校验云镜像",
|
|
acquire,
|
|
)
|
|
|
|
def _run_uploaded_import(
|
|
self, job_id: str, image_path: Path, name: str,
|
|
os_family: str, version: str, digest: str, virtual_size: int,
|
|
reservation_id: str) -> None:
|
|
def acquire() -> tuple[Path, int, str, str, str]:
|
|
return image_path, virtual_size, digest, os_family, version
|
|
|
|
self._run_import_job(
|
|
job_id,
|
|
name,
|
|
reservation_id,
|
|
"正在校验并保存上传的云镜像",
|
|
acquire,
|
|
)
|
|
|
|
def _run_import_job(
|
|
self, job_id: str, name: str, pending_key: str,
|
|
running_phase: str, acquire) -> None:
|
|
"""Run the common catalog/publish transaction for every import source."""
|
|
temporary: Path | None = None
|
|
published = False
|
|
cataloged = False
|
|
commit_uncertain = False
|
|
template_id = ""
|
|
record: dict[str, Any] | None = None
|
|
try:
|
|
with self.lock:
|
|
job = self.jobs[job_id]
|
|
job.update(
|
|
status="running",
|
|
phase=running_phase,
|
|
started_at=int(time.time()),
|
|
)
|
|
self.save_job(job_id, job)
|
|
(temporary, virtual_size, actual_sha256,
|
|
os_family, version) = acquire()
|
|
template_id = cloud_template_id_for(
|
|
actual_sha256, version, os_family
|
|
)
|
|
size = temporary.stat().st_size
|
|
record = {
|
|
"id": template_id,
|
|
"name": name,
|
|
"os_family": os_family,
|
|
"os_version": version,
|
|
"sha256": actual_sha256,
|
|
"volume_name": template_volume_name(template_id),
|
|
"virtual_size": virtual_size,
|
|
"image_bytes": size,
|
|
"created_at": int(time.time()),
|
|
}
|
|
with self.lock:
|
|
existing = self.catalog.list()
|
|
if any(row["id"] == template_id for row in existing):
|
|
raise TemplateError("此云镜像已经导入", 409)
|
|
if any(row["name"].casefold() == name.casefold()
|
|
for row in existing):
|
|
raise TemplateError("已有同名云镜像模板", 409)
|
|
previous_created_at = self._prepare_reimport(template_id)
|
|
# created_at doubles as the public deletion generation. A
|
|
# durable counter prevents reuse even when an earlier import
|
|
# reserved a generation and failed before catalog commit.
|
|
record["created_at"] = self._next_generation(
|
|
template_id, previous_created_at or 0)
|
|
job = self.jobs[job_id]
|
|
job["phase"] = "正在保存云镜像源文件"
|
|
# Persist the deterministic candidate before publishing. If
|
|
# the process stops after the catalog commit but before the
|
|
# terminal job write, startup can prove that this interrupted
|
|
# job owns the committed catalog/source pair.
|
|
job["candidate_template_id"] = template_id
|
|
self.save_job(job_id, job)
|
|
source = self.files.publish(
|
|
temporary, template_id, expected_size=size
|
|
)
|
|
temporary = None
|
|
published = True
|
|
self._mark_verified(template_id, source, actual_sha256)
|
|
self.catalog.add(record)
|
|
cataloged = True
|
|
self._save_completed_job(job_id, job, record)
|
|
except Exception as exc:
|
|
# A database insert can commit and still raise (for example when
|
|
# the connection drops before its acknowledgement reaches this
|
|
# process). Once the source was published, never delete it until
|
|
# the catalog has been read back. An exact record is proof of a
|
|
# completed import; an unavailable read is ambiguous and must
|
|
# preserve the source for startup reconciliation.
|
|
committed: dict[str, Any] | None = None
|
|
if published and template_id and not cataloged:
|
|
try:
|
|
existing = self.catalog.get(template_id)
|
|
except TemplateError as read_exc:
|
|
if read_exc.status != 404:
|
|
commit_uncertain = True
|
|
except Exception:
|
|
commit_uncertain = True
|
|
LOG.exception(
|
|
"Could not resolve cloud template catalog commit: job=%s",
|
|
job_id,
|
|
)
|
|
else:
|
|
cataloged = True
|
|
if record is not None and existing == record:
|
|
committed = existing
|
|
if committed is not None:
|
|
with self.lock:
|
|
previous = self.jobs.get(job_id, {})
|
|
self._save_completed_job(job_id, previous, committed)
|
|
return
|
|
if isinstance(exc, TemplateError):
|
|
error = str(exc)
|
|
else:
|
|
LOG.exception("Global cloud template import failed: job=%s", job_id)
|
|
error = "云镜像导入失败,请查看引擎日志"
|
|
with self.lock:
|
|
if cataloged or commit_uncertain:
|
|
# A catalog write is the commit point. Do not remove a
|
|
# committed (or ambiguously committed) template merely
|
|
# because its acknowledgement failed. Startup can prove
|
|
# the candidate from the persisted running job.
|
|
pass
|
|
elif published and template_id:
|
|
try:
|
|
self.files.delete(template_id)
|
|
except Exception:
|
|
LOG.exception(
|
|
"Could not remove failed cloud template source: job=%s",
|
|
job_id,
|
|
)
|
|
previous = self.jobs.get(job_id, {})
|
|
failed = {
|
|
"job_id": job_id,
|
|
"status": "failed",
|
|
"phase": "导入失败",
|
|
"error": error,
|
|
"created_at": previous.get("created_at", int(time.time())),
|
|
"started_at": previous.get("started_at"),
|
|
"finished_at": int(time.time()),
|
|
}
|
|
if template_id:
|
|
failed["candidate_template_id"] = template_id
|
|
if commit_uncertain:
|
|
failed["commit_uncertain"] = True
|
|
try:
|
|
self.save_job(job_id, failed)
|
|
except Exception:
|
|
LOG.exception(
|
|
"Could not persist cloud template import failure: job=%s",
|
|
job_id,
|
|
)
|
|
finally:
|
|
if temporary is not None:
|
|
temporary.unlink(missing_ok=True)
|
|
with self.lock:
|
|
self.upload_reservations.pop(pending_key, None)
|
|
self.pending.discard(pending_key)
|
|
|
|
def _save_completed_job(self, job_id: str, previous: dict[str, Any],
|
|
record: dict[str, Any]) -> None:
|
|
"""Publish an in-memory success even if its final DB write fails.
|
|
|
|
The catalog record is the durable commit point. Reporting a failed
|
|
import after that point invites a duplicate retry, so the running
|
|
process must expose success while startup reconstructs the same result
|
|
from the candidate ID if the terminal job write was lost.
|
|
"""
|
|
completed = {
|
|
"job_id": job_id,
|
|
"status": "completed",
|
|
"phase": "云镜像已保存,可用于所有宿主机",
|
|
"created_at": previous.get("created_at", int(time.time())),
|
|
"started_at": previous.get("started_at"),
|
|
"finished_at": int(time.time()),
|
|
"template": record,
|
|
}
|
|
try:
|
|
self.save_job(job_id, completed)
|
|
except Exception:
|
|
# save_job updates self.jobs before touching persistent storage.
|
|
# Keep that completed state and let restore_jobs repair a stale
|
|
# persisted running record on the next process start.
|
|
self.jobs[job_id] = completed
|
|
LOG.exception(
|
|
"Could not persist completed cloud template import: job=%s",
|
|
job_id,
|
|
)
|
|
|
|
def _mark_verified(self, template_id: str, path: Path, digest: str) -> None:
|
|
info = path.stat()
|
|
self.verified_sources[template_id] = (
|
|
info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns, digest
|
|
)
|
|
|
|
def _verify_source(self, template_id: str, path: Path, digest: str) -> None:
|
|
info = path.stat()
|
|
identity = (info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns)
|
|
cached = self.verified_sources.get(template_id)
|
|
if cached is not None and cached[:4] == identity and cached[4] == digest:
|
|
return
|
|
checksum = hashlib.sha256()
|
|
with path.open("rb") as source:
|
|
while chunk := source.read(1024 * 1024):
|
|
checksum.update(chunk)
|
|
if checksum.hexdigest() != digest:
|
|
raise TemplateError("云镜像源文件的 SHA-256 校验值不匹配", 500)
|
|
self._mark_verified(template_id, path, digest)
|
|
|
|
@contextmanager
|
|
def source(self, template_id: str) -> Iterator[tuple[dict[str, Any], Path]]:
|
|
with self.lock:
|
|
record = self.catalog.get(template_id)
|
|
path = self.files.path(template_id, expected_size=record["image_bytes"])
|
|
self._verify_source(template_id, path, record["sha256"])
|
|
self.in_use[template_id] = self.in_use.get(template_id, 0) + 1
|
|
try:
|
|
yield record, path
|
|
finally:
|
|
with self.lock:
|
|
count = self.in_use.get(template_id, 0)
|
|
if count <= 1:
|
|
self.in_use.pop(template_id, None)
|
|
else:
|
|
self.in_use[template_id] = count - 1
|
|
|
|
def save_job(self, job_id: str, job: dict[str, Any]) -> None:
|
|
self.jobs[job_id] = job
|
|
if self.store is not None:
|
|
self.store.set(
|
|
IMPORT_JOB_KIND,
|
|
job_id,
|
|
json.dumps(job, ensure_ascii=False, separators=(",", ":")),
|
|
)
|
|
|
|
def delete_job(self, job_id: str) -> None:
|
|
self.jobs.pop(job_id, None)
|
|
if self.store is not None:
|
|
self.store.delete(IMPORT_JOB_KIND, job_id)
|
|
|
|
def restore_jobs(self) -> None:
|
|
"""Reconcile local files and fail imports interrupted by a restart."""
|
|
with self.lock:
|
|
if self._restored:
|
|
return
|
|
self.files.reconcile(self.catalog)
|
|
self._reconcile_cache_tombstones()
|
|
for record in self.catalog.list():
|
|
self._remember_generation(
|
|
record["id"], record["created_at"])
|
|
source = self.store.list(IMPORT_JOB_KIND) if self.store is not None else {}
|
|
for job_id, raw in source.items():
|
|
try:
|
|
job = json.loads(raw)
|
|
except (TypeError, ValueError) as exc:
|
|
raise TemplateError("云镜像导入任务记录损坏", 500) from exc
|
|
if not isinstance(job, dict) or job.get("job_id") != job_id:
|
|
raise TemplateError("云镜像导入任务记录损坏", 500)
|
|
interrupted = job.get("status") in {"queued", "running"}
|
|
uncertain = job.get("commit_uncertain") is True
|
|
if interrupted or uncertain:
|
|
candidate = job.get("candidate_template_id")
|
|
committed: dict[str, Any] | None = None
|
|
if isinstance(candidate, str):
|
|
try:
|
|
committed = self.catalog.get(candidate)
|
|
path = self.files.path(
|
|
candidate,
|
|
expected_size=committed["image_bytes"],
|
|
)
|
|
self._verify_source(
|
|
candidate, path, committed["sha256"]
|
|
)
|
|
except Exception:
|
|
committed = None
|
|
if committed is not None:
|
|
self._save_completed_job(job_id, job, committed)
|
|
continue
|
|
job.update(
|
|
status="failed",
|
|
phase="引擎重启,任务已中断",
|
|
error="面板在云镜像导入期间重启;请重新提交导入",
|
|
finished_at=int(time.time()),
|
|
)
|
|
self.save_job(job_id, job)
|
|
self.jobs[job_id] = job
|
|
self._restored = True
|
|
|
|
def has_active_jobs(self) -> bool:
|
|
with self.lock:
|
|
return bool(self.pending) or any(
|
|
job.get("status") in {"queued", "running"}
|
|
for job in self.jobs.values()
|
|
)
|
|
|
|
def close(self) -> None:
|
|
self.executor.shutdown(wait=True, cancel_futures=False)
|