You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
865 lines
39 KiB
Python
865 lines
39 KiB
Python
"""Owner-scoped catalogs and hardened remote downloads for user ISO images."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import http.client
|
|
import ipaddress
|
|
import json
|
|
import logging
|
|
import os
|
|
import re
|
|
import shutil
|
|
import socket
|
|
import ssl
|
|
import stat
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from contextlib import contextmanager
|
|
from pathlib import Path
|
|
from typing import Any
|
|
from urllib.parse import SplitResult, urlsplit
|
|
|
|
from .cloud_templates import TemplateError
|
|
|
|
|
|
USER_ISO_ID_RE = re.compile(r"[0-9a-f]{32}\Z")
|
|
USER_ISO_VOLUME_RE = re.compile(r"panel-user-iso-[0-9a-f]{32}\.iso\Z")
|
|
HOST_ID_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]{0,63}\Z")
|
|
SHA256_RE = re.compile(r"[0-9a-fA-F]{64}\Z")
|
|
MAX_USER_ISO_BYTES = 10 * 1024**3
|
|
MAX_USER_ISOS = 5
|
|
MAX_USER_ISO_TOTAL_BYTES = 10 * 1024**3
|
|
MAX_DOWNLOAD_SECONDS = 3600
|
|
PERSISTENT_FREE_RESERVE_BYTES = 256 * 1024**2
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
|
|
def _positive_user_id(user_id: Any) -> int:
|
|
if isinstance(user_id, bool) or not isinstance(user_id, int) or user_id < 1:
|
|
raise TemplateError("用户身份无效", 403)
|
|
return user_id
|
|
|
|
|
|
def user_iso_scope(user_id: int) -> str:
|
|
"""Return the engine-global catalog scope for one PHP owner."""
|
|
user_id = _positive_user_id(user_id)
|
|
identity = ["virt-cloud-user-iso", user_id]
|
|
raw = json.dumps(identity, ensure_ascii=False, separators=(",", ":"))
|
|
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
|
|
|
|
|
|
def user_iso_volume_name(iso_id: str) -> str:
|
|
if not isinstance(iso_id, str) or not USER_ISO_ID_RE.fullmatch(iso_id):
|
|
raise TemplateError("ISO 标识无效")
|
|
return f"panel-user-iso-{iso_id}.iso"
|
|
|
|
|
|
def validate_user_iso_name(name: Any) -> str:
|
|
if not isinstance(name, str) or not 1 <= len(name.strip()) <= 64:
|
|
raise TemplateError("ISO 名称应为 1 至 64 个字符")
|
|
if any(ord(char) < 32 or ord(char) == 127 for char in name):
|
|
raise TemplateError("ISO 名称不能包含控制字符")
|
|
return name.strip()
|
|
|
|
|
|
def validate_user_iso_url(url: Any) -> SplitResult:
|
|
"""Validate syntax before DNS; the downloader pins a separately checked IP."""
|
|
if not isinstance(url, str) or not 1 <= len(url) <= 2048:
|
|
raise TemplateError("ISO 下载地址无效")
|
|
try:
|
|
parsed = urlsplit(url)
|
|
port = parsed.port
|
|
hostname = parsed.hostname
|
|
except (TypeError, ValueError) as exc:
|
|
raise TemplateError("ISO 下载地址无效") from exc
|
|
if (parsed.scheme != "https" or not hostname or port not in (None, 443)
|
|
or parsed.username or parsed.password or parsed.fragment):
|
|
raise TemplateError("ISO 仅支持无账号、无跳转的 HTTPS 443 直链")
|
|
if not parsed.path.lower().endswith(".iso"):
|
|
raise TemplateError("ISO 下载地址的文件名必须以 .iso 结尾", 422)
|
|
try:
|
|
url.encode("ascii")
|
|
hostname.encode("idna")
|
|
except UnicodeError as exc:
|
|
raise TemplateError("ISO 下载地址必须使用 ASCII/百分号编码") from exc
|
|
if any(ord(char) < 33 or ord(char) == 127 for char in url):
|
|
raise TemplateError("ISO 下载地址包含无效字符")
|
|
return parsed
|
|
|
|
|
|
def _resolve_global_addresses(hostname: str) -> list[str]:
|
|
try:
|
|
rows = socket.getaddrinfo(
|
|
hostname, 443, family=socket.AF_UNSPEC,
|
|
type=socket.SOCK_STREAM, proto=socket.IPPROTO_TCP,
|
|
)
|
|
except socket.gaierror as exc:
|
|
raise TemplateError("ISO 下载地址无法解析", 422) from exc
|
|
addresses: list[str] = []
|
|
for _family, _socktype, _proto, _canonname, sockaddr in rows:
|
|
try:
|
|
address = str(ipaddress.ip_address(sockaddr[0]))
|
|
except (IndexError, TypeError, ValueError) as exc:
|
|
raise TemplateError("ISO 下载地址解析结果无效", 422) from exc
|
|
# Reject the whole hostname when even one answer is unsafe. Otherwise
|
|
# a mixed public/private answer can become an SSRF failover primitive.
|
|
if not ipaddress.ip_address(address).is_global:
|
|
raise TemplateError("ISO 下载地址不能指向内网、回环或保留地址", 422)
|
|
if address not in addresses:
|
|
addresses.append(address)
|
|
if not addresses:
|
|
raise TemplateError("ISO 下载地址无法解析", 422)
|
|
return addresses
|
|
|
|
|
|
class _PinnedHTTPSConnection(http.client.HTTPSConnection):
|
|
"""TLS-connect to a validated address while verifying the original host."""
|
|
|
|
def __init__(self, hostname: str, address: str, *, timeout: float) -> None:
|
|
super().__init__(hostname, 443, timeout=timeout,
|
|
context=ssl.create_default_context())
|
|
self._pinned_address = address
|
|
|
|
def connect(self) -> None:
|
|
# There is deliberately no proxy/tunnel path here. Connecting to the
|
|
# already validated numeric address prevents a second DNS lookup and
|
|
# therefore closes the usual DNS-rebinding gap.
|
|
raw = socket.create_connection(
|
|
(self._pinned_address, 443), self.timeout, self.source_address,
|
|
)
|
|
try:
|
|
self.sock = self._context.wrap_socket(raw, server_hostname=self.host)
|
|
except Exception:
|
|
raw.close()
|
|
raise
|
|
|
|
|
|
def _open_pinned_response(parsed: SplitResult, addresses: list[str]):
|
|
hostname = parsed.hostname
|
|
assert hostname is not None
|
|
target = parsed.path or "/"
|
|
if parsed.query:
|
|
target += "?" + parsed.query
|
|
host_header = f"[{hostname}]" if ":" in hostname else hostname
|
|
last_error: Exception | None = None
|
|
for address in addresses:
|
|
connection = _PinnedHTTPSConnection(hostname, address, timeout=30)
|
|
try:
|
|
connection.request("GET", target, headers={
|
|
"Host": host_header,
|
|
"User-Agent": "virt-cloud-user-iso/1",
|
|
"Accept": "application/octet-stream,*/*;q=0.1",
|
|
"Accept-Encoding": "identity",
|
|
"Connection": "close",
|
|
})
|
|
return connection, connection.getresponse()
|
|
except (OSError, ssl.SSLError, http.client.HTTPException) as exc:
|
|
last_error = exc
|
|
connection.close()
|
|
raise TemplateError("无法连接 ISO 下载地址", 502) from last_error
|
|
|
|
|
|
def _declared_size(response: http.client.HTTPResponse) -> int | None:
|
|
values = response.headers.get_all("Content-Length", [])
|
|
if not values:
|
|
return None
|
|
if len(values) != 1 or not re.fullmatch(r"[0-9]+", values[0].strip()):
|
|
raise TemplateError("ISO 下载响应长度无效", 422)
|
|
return int(values[0])
|
|
|
|
|
|
def verify_user_iso(path: Path, *, max_bytes: int = MAX_USER_ISO_BYTES) -> None:
|
|
"""Require a bounded ISO9660 image with an El Torito boot catalog."""
|
|
size = path.stat().st_size
|
|
if not 1024 * 1024 <= size <= min(MAX_USER_ISO_BYTES, max_bytes):
|
|
raise TemplateError("ISO 大小必须在 1 MiB 至可用配额之间", 413)
|
|
bootable = False
|
|
with path.open("rb") as image:
|
|
for sector in range(16, 80):
|
|
image.seek(sector * 2048)
|
|
descriptor = image.read(2048)
|
|
if len(descriptor) != 2048 or descriptor[1:6] != b"CD001":
|
|
raise TemplateError("下载文件不是有效的 ISO 9660 镜像", 422)
|
|
if (descriptor[0] == 0
|
|
and descriptor[7:30].rstrip(b" ") == b"EL TORITO SPECIFICATION"
|
|
and int.from_bytes(descriptor[71:75], "little") > 0):
|
|
bootable = True
|
|
if descriptor[0] == 255:
|
|
break
|
|
if not bootable:
|
|
raise TemplateError("ISO 未包含可识别的 El Torito 启动记录", 422)
|
|
|
|
|
|
def download_user_iso(
|
|
url: str, expected_sha256: str, directory: Path, *,
|
|
max_bytes: int = MAX_USER_ISO_BYTES,
|
|
) -> tuple[Path, int, str]:
|
|
"""Download without proxies/redirects and pin TLS to validated public IPs."""
|
|
parsed = validate_user_iso_url(url)
|
|
if not isinstance(expected_sha256, str) or (
|
|
expected_sha256 and not SHA256_RE.fullmatch(expected_sha256)):
|
|
raise TemplateError("SHA-256 可留空;填写时必须为 64 位十六进制校验值")
|
|
if (isinstance(max_bytes, bool) or not isinstance(max_bytes, int)
|
|
or not 1024 * 1024 <= max_bytes <= MAX_USER_ISO_BYTES):
|
|
raise TemplateError("用户 ISO 可用配额不足", 409)
|
|
hostname = parsed.hostname
|
|
assert hostname is not None
|
|
addresses = _resolve_global_addresses(hostname)
|
|
fd, raw_path = tempfile.mkstemp(prefix=".user-iso-", suffix=".iso", dir=directory)
|
|
path = Path(raw_path)
|
|
connection = None
|
|
response = None
|
|
try:
|
|
os.fchmod(fd, 0o600)
|
|
connection, response = _open_pinned_response(parsed, addresses)
|
|
if 300 <= response.status < 400:
|
|
raise TemplateError("ISO 下载地址发生重定向,请使用最终 HTTPS 直链", 422)
|
|
if response.status != 200:
|
|
raise TemplateError(f"ISO 下载服务器返回 HTTP {response.status}", 422)
|
|
encoding = response.getheader("Content-Encoding", "identity").strip().lower()
|
|
if encoding not in {"", "identity"}:
|
|
raise TemplateError("ISO 下载响应不能使用内容压缩", 422)
|
|
declared = _declared_size(response)
|
|
if declared is not None and declared > max_bytes:
|
|
raise TemplateError("ISO 超过用户全局目录的 10 GiB 配额或剩余容量", 413)
|
|
checksum = hashlib.sha256()
|
|
size = 0
|
|
deadline = time.monotonic() + MAX_DOWNLOAD_SECONDS
|
|
with os.fdopen(fd, "wb") as output:
|
|
fd = -1
|
|
while True:
|
|
if time.monotonic() > deadline:
|
|
raise TemplateError("ISO 下载超时", 504)
|
|
chunk = response.read(1024 * 1024)
|
|
if not chunk:
|
|
break
|
|
size += len(chunk)
|
|
if size > max_bytes:
|
|
raise TemplateError("ISO 超过用户全局目录的 10 GiB 配额或剩余容量", 413)
|
|
output.write(chunk)
|
|
checksum.update(chunk)
|
|
output.flush()
|
|
os.fsync(output.fileno())
|
|
if declared is not None and size != declared:
|
|
raise TemplateError("ISO 下载内容不完整", 502)
|
|
digest = checksum.hexdigest()
|
|
if expected_sha256 and digest.lower() != expected_sha256.lower():
|
|
raise TemplateError("ISO 的 SHA-256 校验值不匹配", 422)
|
|
verify_user_iso(path, max_bytes=max_bytes)
|
|
return path, size, digest
|
|
except Exception:
|
|
path.unlink(missing_ok=True)
|
|
raise
|
|
finally:
|
|
if fd >= 0:
|
|
os.close(fd)
|
|
if response is not None:
|
|
response.close()
|
|
if connection is not None:
|
|
connection.close()
|
|
|
|
|
|
class UserISOFiles:
|
|
"""Private engine-local source files, separate from every host cache."""
|
|
|
|
def __init__(self, data_dir: Path) -> None:
|
|
data_dir = Path(data_dir)
|
|
if not data_dir.is_absolute() or data_dir == Path("/"):
|
|
raise ValueError("user ISO data directory must be absolute and dedicated")
|
|
self.data_dir = data_dir
|
|
self.root = data_dir / "user-isos"
|
|
|
|
@staticmethod
|
|
def _ensure_directory(path: Path) -> None:
|
|
try:
|
|
info = path.lstat()
|
|
except FileNotFoundError:
|
|
path.mkdir(mode=0o700)
|
|
info = path.lstat()
|
|
if (not stat.S_ISDIR(info.st_mode) or stat.S_ISLNK(info.st_mode)
|
|
or info.st_uid != os.geteuid()):
|
|
raise TemplateError("用户 ISO 持久目录不安全,请联系管理员", 500)
|
|
if stat.S_IMODE(info.st_mode) != 0o700:
|
|
os.chmod(path, 0o700)
|
|
|
|
def owner_directory(self, user_id: int) -> Path:
|
|
user_id = _positive_user_id(user_id)
|
|
self.data_dir.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
self._ensure_directory(self.data_dir)
|
|
self._ensure_directory(self.root)
|
|
owner = self.root / str(user_id)
|
|
self._ensure_directory(owner)
|
|
return owner
|
|
|
|
def available_bytes(self, user_id: int) -> int:
|
|
free = int(shutil.disk_usage(self.owner_directory(user_id)).free)
|
|
return max(0, free - PERSISTENT_FREE_RESERVE_BYTES)
|
|
|
|
def path(self, user_id: int, iso_id: str, *, expected_size: int | None = None) -> Path:
|
|
path = self.owner_directory(user_id) / user_iso_volume_name(iso_id)
|
|
try:
|
|
info = path.lstat()
|
|
except FileNotFoundError as exc:
|
|
raise TemplateError("ISO 源文件不存在,请删除后重新导入", 409) from exc
|
|
if (not stat.S_ISREG(info.st_mode) or stat.S_ISLNK(info.st_mode)
|
|
or info.st_uid != os.geteuid() or info.st_mode & 0o077
|
|
or (expected_size is not None and info.st_size != expected_size)):
|
|
raise TemplateError("ISO 源文件已损坏,请联系管理员", 500)
|
|
return path
|
|
|
|
def publish(self, temporary: Path, user_id: int, iso_id: str,
|
|
*, expected_size: int) -> Path:
|
|
temporary = Path(temporary)
|
|
try:
|
|
info = temporary.lstat()
|
|
except FileNotFoundError as exc:
|
|
raise TemplateError("ISO 下载临时文件丢失", 500) from exc
|
|
if (not stat.S_ISREG(info.st_mode) or stat.S_ISLNK(info.st_mode)
|
|
or info.st_uid != os.geteuid() or info.st_size != expected_size):
|
|
raise TemplateError("ISO 下载临时文件无效", 500)
|
|
destination = self.owner_directory(user_id) / user_iso_volume_name(iso_id)
|
|
try:
|
|
# The download is created in this same private directory. Linking
|
|
# publishes it atomically and refuses to replace an existing ID.
|
|
os.link(temporary, destination)
|
|
os.chmod(destination, 0o600)
|
|
temporary.unlink()
|
|
descriptor = os.open(destination.parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0))
|
|
try:
|
|
os.fsync(descriptor)
|
|
finally:
|
|
os.close(descriptor)
|
|
except FileExistsError as exc:
|
|
raise TemplateError("此 ISO 已存在", 409) from exc
|
|
except OSError as exc:
|
|
destination.unlink(missing_ok=True)
|
|
raise TemplateError("ISO 无法保存到面板持久目录", 507) from exc
|
|
return destination
|
|
|
|
def delete(self, user_id: int, iso_id: str) -> None:
|
|
path = self.owner_directory(user_id) / user_iso_volume_name(iso_id)
|
|
try:
|
|
info = path.lstat()
|
|
except FileNotFoundError:
|
|
return
|
|
if not stat.S_ISREG(info.st_mode) or stat.S_ISLNK(info.st_mode):
|
|
raise TemplateError("ISO 源文件无法安全删除", 500)
|
|
try:
|
|
path.unlink()
|
|
except OSError as exc:
|
|
raise TemplateError("ISO 源文件删除失败,请联系管理员", 500) from exc
|
|
|
|
def reconcile(self, catalog: UserISOStore) -> None:
|
|
"""Remove only provable interrupted-download and uncataloged files."""
|
|
try:
|
|
self.root.lstat()
|
|
except FileNotFoundError:
|
|
return
|
|
self._ensure_directory(self.root)
|
|
temporary_name = re.compile(r"\.user-iso-[A-Za-z0-9_-]{6,64}\.iso\Z")
|
|
for owner in self.root.iterdir():
|
|
try:
|
|
owner_id = int(owner.name)
|
|
info = owner.lstat()
|
|
except (OSError, ValueError):
|
|
continue
|
|
if (owner_id < 1 or str(owner_id) != owner.name
|
|
or not stat.S_ISDIR(info.st_mode)
|
|
or stat.S_ISLNK(info.st_mode)
|
|
or info.st_uid != os.geteuid()
|
|
or stat.S_IMODE(info.st_mode) != 0o700):
|
|
continue
|
|
active = {
|
|
row["volume_name"]
|
|
for row in catalog.list(user_iso_scope(owner_id))
|
|
}
|
|
for candidate in owner.iterdir():
|
|
try:
|
|
candidate_info = candidate.lstat()
|
|
except OSError:
|
|
continue
|
|
if (not stat.S_ISREG(candidate_info.st_mode)
|
|
or stat.S_ISLNK(candidate_info.st_mode)
|
|
or candidate_info.st_uid != os.geteuid()):
|
|
continue
|
|
interrupted = temporary_name.fullmatch(candidate.name) is not None
|
|
orphan = (USER_ISO_VOLUME_RE.fullmatch(candidate.name) is not None
|
|
and candidate.name not in active)
|
|
if interrupted or orphan:
|
|
candidate.unlink(missing_ok=True)
|
|
|
|
|
|
class UserISOStore:
|
|
"""Engine-global catalog whose API always requires an owner scope."""
|
|
|
|
def __init__(self, store: Any | None) -> None:
|
|
self.store = store
|
|
self._memory: dict[str, str] = {}
|
|
self._lock = threading.RLock()
|
|
|
|
@staticmethod
|
|
def _validate(record: Any) -> dict[str, Any]:
|
|
if (not isinstance(record, dict)
|
|
or not USER_ISO_ID_RE.fullmatch(str(record.get("id", "")))
|
|
or record.get("volume_name") != user_iso_volume_name(record["id"])
|
|
or not isinstance(record.get("name"), str)
|
|
or validate_user_iso_name(record["name"]) != record["name"]
|
|
or isinstance(record.get("size_bytes"), bool)
|
|
or not isinstance(record.get("size_bytes"), int)
|
|
or not 1024 * 1024 <= record["size_bytes"] <= MAX_USER_ISO_BYTES
|
|
or not SHA256_RE.fullmatch(str(record.get("sha256", "")))
|
|
or isinstance(record.get("created_at"), bool)
|
|
or not isinstance(record.get("created_at"), int)
|
|
or record["created_at"] < 1):
|
|
raise TemplateError("用户 ISO 数据库记录损坏", 500)
|
|
return record
|
|
|
|
def _list(self, scope: str) -> list[dict[str, Any]]:
|
|
source = (self.store.list("user_iso") if self.store is not None
|
|
else self._memory.copy())
|
|
records = []
|
|
for key, payload in source.items():
|
|
if not key.startswith(f"{scope}:"):
|
|
continue
|
|
try:
|
|
record = json.loads(payload)
|
|
except (TypeError, ValueError) as exc:
|
|
raise TemplateError("用户 ISO 数据库记录损坏", 500) from exc
|
|
records.append(self._validate(record))
|
|
return sorted(records, key=lambda item: (item["created_at"], item["id"]), reverse=True)
|
|
|
|
def list(self, scope: str) -> list[dict[str, Any]]:
|
|
with self._lock:
|
|
return self._list(scope)
|
|
|
|
def get(self, scope: str, iso_id: str) -> dict[str, Any]:
|
|
user_iso_volume_name(iso_id)
|
|
with self._lock:
|
|
for record in self._list(scope):
|
|
if record["id"] == iso_id:
|
|
return record
|
|
raise TemplateError("ISO 不存在或不属于当前用户", 404)
|
|
|
|
def remaining_bytes(self, scope: str) -> int:
|
|
with self._lock:
|
|
records = self._list(scope)
|
|
if len(records) >= MAX_USER_ISOS:
|
|
raise TemplateError("每个用户最多保存 5 个 ISO", 409)
|
|
remaining = MAX_USER_ISO_TOTAL_BYTES - sum(row["size_bytes"] for row in records)
|
|
if remaining < 1024 * 1024:
|
|
raise TemplateError("用户 ISO 全局目录的 10 GiB 存储配额已用完", 409)
|
|
return min(MAX_USER_ISO_BYTES, remaining)
|
|
|
|
def add(self, scope: str, record: dict[str, Any]) -> dict[str, Any]:
|
|
record = self._validate(record.copy())
|
|
key = f"{scope}:{record['id']}"
|
|
value = json.dumps(record, ensure_ascii=False, separators=(",", ":"))
|
|
with self._lock:
|
|
records = self._list(scope)
|
|
if len(records) >= MAX_USER_ISOS:
|
|
raise TemplateError("每个用户最多保存 5 个 ISO", 409)
|
|
if sum(row["size_bytes"] for row in records) + record["size_bytes"] > MAX_USER_ISO_TOTAL_BYTES:
|
|
raise TemplateError("用户 ISO 全局目录总容量不能超过 10 GiB", 409)
|
|
if any(row["id"] == record["id"] for row in records):
|
|
raise TemplateError("此 ISO 已存在", 409)
|
|
if any(row["name"].casefold() == record["name"].casefold() for row in records):
|
|
raise TemplateError("当前用户已有同名 ISO", 409)
|
|
if self.store is not None:
|
|
if not self.store.insert("user_iso", key, value):
|
|
raise TemplateError("此 ISO 已存在", 409)
|
|
else:
|
|
self._memory[key] = value
|
|
return record
|
|
|
|
def delete(self, scope: str, iso_id: str) -> None:
|
|
self.get(scope, iso_id)
|
|
key = f"{scope}:{iso_id}"
|
|
with self._lock:
|
|
if self.store is not None:
|
|
self.store.delete("user_iso", key)
|
|
else:
|
|
self._memory.pop(key, None)
|
|
|
|
|
|
class UserISORuntime:
|
|
"""One engine-process runtime shared by every per-host child app."""
|
|
|
|
def __init__(self, store: Any | None, data_dir: Path) -> None:
|
|
self.store = store
|
|
self.catalog = UserISOStore(store)
|
|
self.files = UserISOFiles(data_dir)
|
|
# Serializing global downloads makes the filesystem-free-space reserve
|
|
# meaningful across different owners: every job rechecks free space
|
|
# after the previous job has published its bytes.
|
|
self.executor = ThreadPoolExecutor(
|
|
max_workers=1, thread_name_prefix="user-iso-import")
|
|
self.lock = threading.RLock()
|
|
self.jobs: dict[str, dict[str, Any]] = {}
|
|
self.pending: set[str] = set()
|
|
self.in_use: dict[tuple[int, str], int] = {}
|
|
self.verified_sources: dict[tuple[int, str], tuple[int, int, int, int, str]] = {}
|
|
self._restored = False
|
|
|
|
def _mark_verified(self, user_id: int, iso_id: str, path: Path,
|
|
digest: str) -> None:
|
|
info = path.stat()
|
|
self.verified_sources[(user_id, iso_id)] = (
|
|
info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns, digest)
|
|
|
|
def _verify_source(self, user_id: int, iso_id: str, path: Path,
|
|
digest: str) -> None:
|
|
info = path.stat()
|
|
identity = (info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns)
|
|
cached = self.verified_sources.get((user_id, iso_id))
|
|
if cached is not None and cached[:4] == identity and cached[4] == digest:
|
|
return
|
|
checksum = hashlib.sha256()
|
|
with path.open("rb") as source:
|
|
while chunk := source.read(1024 * 1024):
|
|
checksum.update(chunk)
|
|
if checksum.hexdigest() != digest:
|
|
raise TemplateError("ISO 源文件的 SHA-256 校验值不匹配", 500)
|
|
self._mark_verified(user_id, iso_id, path, digest)
|
|
|
|
@staticmethod
|
|
def public_record(record: dict[str, Any]) -> dict[str, Any]:
|
|
return {key: value for key, value in record.items()
|
|
if key in {"id", "name", "size_bytes", "sha256", "created_at"}}
|
|
|
|
@staticmethod
|
|
def public_error(value: Any) -> str:
|
|
safe = {
|
|
"面板在 ISO 下载期间重启;请重新提交导入",
|
|
"ISO 下载地址无效", "ISO 下载地址无法解析",
|
|
"ISO 下载地址的文件名必须以 .iso 结尾",
|
|
"ISO 下载地址解析结果无效",
|
|
"ISO 下载地址不能指向内网、回环或保留地址",
|
|
"ISO 下载地址发生重定向,请使用最终 HTTPS 直链",
|
|
"无法连接 ISO 下载地址", "ISO 下载响应长度无效",
|
|
"ISO 下载响应不能使用内容压缩", "ISO 下载超时",
|
|
"ISO 下载内容不完整", "ISO 的 SHA-256 校验值不匹配",
|
|
"ISO 大小必须在 1 MiB 至可用配额之间",
|
|
"下载文件不是有效的 ISO 9660 镜像",
|
|
"ISO 未包含可识别的 El Torito 启动记录",
|
|
"ISO 超过用户全局目录的 10 GiB 配额或剩余容量",
|
|
"用户 ISO 可用配额不足",
|
|
"每个用户最多保存 5 个 ISO",
|
|
"用户 ISO 全局目录的 10 GiB 存储配额已用完",
|
|
"用户 ISO 全局目录总容量不能超过 10 GiB",
|
|
"面板用户 ISO 持久目录空间不足",
|
|
"ISO 无法保存到面板持久目录",
|
|
"用户 ISO 导入失败,请查看引擎日志",
|
|
}
|
|
if isinstance(value, str) and value in safe:
|
|
return value
|
|
if isinstance(value, str) and re.fullmatch(
|
|
r"ISO 下载服务器返回 HTTP [1-5][0-9]{2}", value):
|
|
return value
|
|
return "ISO 下载或保存失败,请联系管理员查看日志"
|
|
|
|
@classmethod
|
|
def public_job(cls, job: dict[str, Any]) -> dict[str, Any]:
|
|
allowed = {"job_id", "status", "phase", "created_at", "started_at",
|
|
"finished_at", "iso", "error"}
|
|
result = {key: value for key, value in job.items() if key in allowed}
|
|
if isinstance(result.get("iso"), dict):
|
|
result["iso"] = cls.public_record(result["iso"])
|
|
if "error" in result:
|
|
result["error"] = cls.public_error(result["error"])
|
|
return result
|
|
|
|
def list_for(self, user_id: int) -> dict[str, Any]:
|
|
user_id = _positive_user_id(user_id)
|
|
scope = user_iso_scope(user_id)
|
|
result = {
|
|
"isos": [self.public_record(row) for row in self.catalog.list(scope)],
|
|
}
|
|
with self.lock:
|
|
active = [job for job in self.jobs.values()
|
|
if job.get("scope") == scope
|
|
and job.get("owner_user_id") == user_id
|
|
and job.get("status") in {"queued", "running"}]
|
|
if active:
|
|
current = max(active, key=lambda job: (
|
|
int(job.get("created_at", 0)), str(job.get("job_id", ""))))
|
|
result["active_import_job"] = self.public_job(current)
|
|
return result
|
|
|
|
def get_for(self, user_id: int, iso_id: str) -> dict[str, Any]:
|
|
user_id = _positive_user_id(user_id)
|
|
return self.catalog.get(user_iso_scope(user_id), iso_id)
|
|
|
|
def get_job_for(self, user_id: int, job_id: str) -> dict[str, Any]:
|
|
user_id = _positive_user_id(user_id)
|
|
scope = user_iso_scope(user_id)
|
|
with self.lock:
|
|
job = self.jobs.get(job_id)
|
|
if (job is None or job.get("scope") != scope
|
|
or job.get("owner_user_id") != user_id):
|
|
raise TemplateError("ISO 导入任务不存在", 404)
|
|
return self.public_job(job)
|
|
|
|
def _remaining_download_bytes(self, user_id: int, scope: str) -> int:
|
|
remaining = self.catalog.remaining_bytes(scope)
|
|
available = self.files.available_bytes(user_id)
|
|
maximum = min(remaining, available)
|
|
if maximum < 1024 * 1024:
|
|
raise TemplateError("面板用户 ISO 持久目录空间不足", 409)
|
|
return maximum
|
|
|
|
def queue_import(self, user_id: int, url: str, name: str,
|
|
expected_sha256: str) -> dict[str, str]:
|
|
user_id = _positive_user_id(user_id)
|
|
validate_user_iso_url(url)
|
|
name = validate_user_iso_name(name)
|
|
if (not isinstance(expected_sha256, str)
|
|
or (expected_sha256 and not SHA256_RE.fullmatch(expected_sha256))):
|
|
raise TemplateError(
|
|
"SHA-256 可留空;填写时必须为 64 位十六进制校验值", 422)
|
|
expected_sha256 = expected_sha256.lower()
|
|
scope = user_iso_scope(user_id)
|
|
with self.lock:
|
|
existing = self.catalog.list(scope)
|
|
self._remaining_download_bytes(user_id, scope)
|
|
if any(row["name"].casefold() == name.casefold() for row in existing):
|
|
raise TemplateError("当前用户已有同名 ISO", 409)
|
|
if scope in self.pending:
|
|
raise TemplateError("已有用户 ISO 正在导入,请等待完成", 409)
|
|
job_id = str(uuid.uuid4())
|
|
self.pending.add(scope)
|
|
job = {
|
|
"job_id": job_id, "scope": scope, "owner_user_id": user_id,
|
|
"status": "queued", "phase": "等待下载", "created_at": int(time.time()),
|
|
}
|
|
try:
|
|
self.save_job(job_id, job)
|
|
self.executor.submit(
|
|
self._run_import, job_id, user_id, scope, url, name,
|
|
expected_sha256)
|
|
except Exception as exc:
|
|
self.pending.discard(scope)
|
|
self.delete_job(job_id)
|
|
if isinstance(exc, RuntimeError):
|
|
raise TemplateError("引擎正在停止,无法开始 ISO 导入", 503) from exc
|
|
raise
|
|
return {"job_id": job_id, "status": "queued"}
|
|
|
|
def _run_import(self, job_id: str, user_id: int, scope: str, url: str,
|
|
name: str, expected_sha256: str) -> None:
|
|
temporary: Path | None = None
|
|
published = False
|
|
cataloged = False
|
|
iso_id = ""
|
|
try:
|
|
with self.lock:
|
|
job = self.jobs[job_id]
|
|
job.update(status="running", phase="正在安全下载 ISO",
|
|
started_at=int(time.time()))
|
|
self.save_job(job_id, job)
|
|
maximum = self._remaining_download_bytes(user_id, scope)
|
|
directory = self.files.owner_directory(user_id)
|
|
temporary, size, digest = download_user_iso(
|
|
url, expected_sha256, directory, max_bytes=maximum)
|
|
iso_id = uuid.uuid4().hex
|
|
record = {
|
|
"id": iso_id, "name": name,
|
|
"volume_name": user_iso_volume_name(iso_id),
|
|
"size_bytes": size, "sha256": digest,
|
|
"created_at": int(time.time()),
|
|
}
|
|
with self.lock:
|
|
job = self.jobs[job_id]
|
|
job["phase"] = "正在保存到面板 ISO 目录"
|
|
self.save_job(job_id, job)
|
|
self.files.publish(temporary, user_id, iso_id, expected_size=size)
|
|
temporary = None
|
|
published = True
|
|
source_path = self.files.path(
|
|
user_id, iso_id, expected_size=size)
|
|
self._mark_verified(user_id, iso_id, source_path, digest)
|
|
self.catalog.add(scope, record)
|
|
cataloged = True
|
|
completed = {
|
|
"job_id": job_id, "scope": scope, "owner_user_id": user_id,
|
|
"status": "completed", "phase": "ISO 已保存,可用于所有宿主机",
|
|
"created_at": job["created_at"],
|
|
"started_at": job.get("started_at"),
|
|
"finished_at": int(time.time()), "iso": record,
|
|
}
|
|
self.save_job(job_id, completed)
|
|
except Exception as exc:
|
|
if isinstance(exc, TemplateError):
|
|
error = str(exc)
|
|
else:
|
|
LOG.exception("User ISO import failed: job=%s owner=%s", job_id, user_id)
|
|
error = "用户 ISO 导入失败,请查看引擎日志"
|
|
with self.lock:
|
|
if cataloged:
|
|
try:
|
|
self.catalog.delete(scope, iso_id)
|
|
except Exception:
|
|
LOG.exception("Could not clean failed user ISO catalog: job=%s", job_id)
|
|
if published:
|
|
try:
|
|
self.files.delete(user_id, iso_id)
|
|
except Exception:
|
|
LOG.exception("Could not clean failed user ISO source: job=%s", job_id)
|
|
previous = self.jobs.get(job_id, {})
|
|
failed = {
|
|
"job_id": job_id, "scope": scope, "owner_user_id": user_id,
|
|
"status": "failed", "phase": "导入失败", "error": error,
|
|
"created_at": previous.get("created_at", int(time.time())),
|
|
"started_at": previous.get("started_at"),
|
|
"finished_at": int(time.time()),
|
|
}
|
|
try:
|
|
self.save_job(job_id, failed)
|
|
except Exception:
|
|
LOG.exception("Could not persist user ISO import failure: job=%s", job_id)
|
|
finally:
|
|
if temporary is not None:
|
|
temporary.unlink(missing_ok=True)
|
|
with self.lock:
|
|
self.pending.discard(scope)
|
|
|
|
@contextmanager
|
|
def source(self, user_id: int, iso_id: str):
|
|
user_id = _positive_user_id(user_id)
|
|
key = (user_id, iso_id)
|
|
with self.lock:
|
|
record = self.get_for(user_id, iso_id)
|
|
path = self.files.path(
|
|
user_id, iso_id, expected_size=record["size_bytes"])
|
|
self._verify_source(
|
|
user_id, iso_id, path, record["sha256"])
|
|
self.in_use[key] = self.in_use.get(key, 0) + 1
|
|
try:
|
|
yield record, path
|
|
finally:
|
|
with self.lock:
|
|
count = self.in_use.get(key, 0)
|
|
if count <= 1:
|
|
self.in_use.pop(key, None)
|
|
else:
|
|
self.in_use[key] = count - 1
|
|
|
|
def delete_for(self, user_id: int, iso_id: str) -> None:
|
|
user_id = _positive_user_id(user_id)
|
|
scope = user_iso_scope(user_id)
|
|
key = (user_id, iso_id)
|
|
with self.lock:
|
|
if scope in self.pending:
|
|
raise TemplateError("用户 ISO 正在导入,请稍后删除", 409)
|
|
if self.in_use.get(key, 0):
|
|
raise TemplateError("ISO 正在同步到宿主机,请稍后删除", 409)
|
|
self.catalog.get(scope, iso_id)
|
|
# Delete bytes first. If the database write then fails, the
|
|
# catalog continues counting the quota and a retry can finish the
|
|
# deletion; users cannot fill the disk with unaccounted orphans.
|
|
self.files.delete(user_id, iso_id)
|
|
self.catalog.delete(scope, iso_id)
|
|
self.verified_sources.pop(key, None)
|
|
|
|
def save_job(self, job_id: str, job: dict[str, Any]) -> None:
|
|
self.jobs[job_id] = job
|
|
if self.store is not None:
|
|
self.store.set("user_iso_import_job", job_id, json.dumps(
|
|
job, ensure_ascii=False, separators=(",", ":")))
|
|
|
|
def delete_job(self, job_id: str) -> None:
|
|
self.jobs.pop(job_id, None)
|
|
if self.store is not None:
|
|
self.store.delete("user_iso_import_job", job_id)
|
|
|
|
def restore_jobs(self) -> None:
|
|
"""Mark work interrupted by an engine restart as failed exactly once."""
|
|
with self.lock:
|
|
if self._restored:
|
|
return
|
|
self.files.reconcile(self.catalog)
|
|
if self.store is not None:
|
|
for key, raw in self.store.list("user_iso_tombstone").items():
|
|
try:
|
|
tombstone = json.loads(raw)
|
|
owner_id = _positive_user_id(tombstone["user_id"])
|
|
iso_id = tombstone["iso_id"]
|
|
volume_name = user_iso_volume_name(iso_id)
|
|
pending_hosts = tombstone["pending_hosts"]
|
|
source_deleted = tombstone["source_deleted"]
|
|
except (KeyError, TypeError, ValueError) as exc:
|
|
raise TemplateError(
|
|
"用户 ISO 删除回收记录损坏", 500) from exc
|
|
if (key != f"{owner_id}:{iso_id}"
|
|
or tombstone.get("volume_name") != volume_name
|
|
or not isinstance(pending_hosts, list)
|
|
or any(not isinstance(host_id, str)
|
|
or not HOST_ID_RE.fullmatch(host_id)
|
|
for host_id in pending_hosts)
|
|
or len(set(pending_hosts)) != len(pending_hosts)
|
|
or not isinstance(source_deleted, bool)
|
|
or isinstance(tombstone.get("created_at"), bool)
|
|
or not isinstance(tombstone.get("created_at"), int)
|
|
or tombstone["created_at"] < 1):
|
|
raise TemplateError("用户 ISO 删除回收记录损坏", 500)
|
|
try:
|
|
catalog_scope = user_iso_scope(owner_id)
|
|
record = self.catalog.get(catalog_scope, iso_id)
|
|
except TemplateError as exc:
|
|
if exc.status != 404:
|
|
raise
|
|
exists = False
|
|
else:
|
|
try:
|
|
self.files.path(
|
|
owner_id, iso_id,
|
|
expected_size=record["size_bytes"])
|
|
except TemplateError:
|
|
# Crash window: source unlink committed but catalog
|
|
# deletion did not. Complete the user's deletion;
|
|
# retaining a visible record with no bytes would
|
|
# make every future deployment fail.
|
|
self.catalog.delete(catalog_scope, iso_id)
|
|
exists = False
|
|
else:
|
|
exists = True
|
|
if exists:
|
|
# The source/catalog deletion did not commit, so the
|
|
# user still owns the ISO and remote replicas stay.
|
|
self.store.delete("user_iso_tombstone", key)
|
|
elif not tombstone.get("source_deleted"):
|
|
tombstone["source_deleted"] = True
|
|
self.store.set("user_iso_tombstone", key, json.dumps(
|
|
tombstone, ensure_ascii=False, separators=(",", ":")))
|
|
source = self.store.list("user_iso_import_job") if self.store is not None else {}
|
|
for job_id, raw in source.items():
|
|
try:
|
|
job = json.loads(raw)
|
|
except (TypeError, ValueError) as exc:
|
|
raise TemplateError("用户 ISO 任务记录损坏", 500) from exc
|
|
if not isinstance(job, dict):
|
|
raise TemplateError("用户 ISO 任务记录损坏", 500)
|
|
if job.get("status") in {"queued", "running"}:
|
|
job.update(
|
|
status="failed",
|
|
phase="引擎重启,任务已中断",
|
|
error="面板在 ISO 下载期间重启;请重新提交导入",
|
|
finished_at=int(time.time()),
|
|
)
|
|
self.save_job(job_id, job)
|
|
self.jobs[job_id] = job
|
|
self._restored = True
|
|
|
|
def close(self) -> None:
|
|
self.executor.shutdown(wait=True, cancel_futures=False)
|
|
|
|
def has_active_jobs(self) -> bool:
|
|
with self.lock:
|
|
return bool(self.pending) or any(
|
|
job.get("status") in {"queued", "running"}
|
|
for job in self.jobs.values()
|
|
)
|