diff --git a/README.md b/README.md index 0d924f55..c017f7c8 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ PHP 8.5 网站与私有 Python/libvirt 引擎集成的虚拟机销售面板。 - 用户购买套餐开通 VM,余额扣款、库存扣减、确认创建失败退款;可手动续费,续费从已购期限末尾顺延。 - 套餐绑定宿主机,提供 CPU、内存、磁盘、网络带宽、账期天数和每期流量额度,用户不能随意修改套餐资源。 - 下载 RX 与上传 TX 按字节累加计费;当期额度用尽只断开 VM 网卡,不关机,下一已付费账期开始自动归零当期用量并恢复网络;无后续账期的最终到期才停机。 -- 管理员管理多宿主机、云镜像和独立救援 ISO;用户选择已有模板及自己的 SSH 公钥。 +- 管理员管理多宿主机、全局云镜像目录和独立救援 ISO;用户切换地区时云镜像选择保持不变,首次部署由引擎按需同步到目标宿主机并缓存复用。 - 用户仅操作自己的 VM:控制台、开关机、软/硬重启、重装、救援、删除、初始 root 密码查看/复制。 - CPU、内存、磁盘 I/O、网络带宽图表;后台持续采集,写入日志库,最长保留 30 天。 - 全部用户页及登录页使用原虚拟机面板风格;管理员后台保留 venix 样式。 diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index d264074e..651f0390 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -163,7 +163,7 @@ location = /engine-console { ## 7. 页面配置与验收 1. 管理员添加宿主机:现有 SSH/root/私钥口令、核验指纹,设置区域、网桥、目录、地址池/网关/DNS/网卡前缀,测试连接。 -2. 导入云镜像与救援模板;无已有模板不能开通云镜像 VM。云镜像 SHA-256 可留空;留空时仍会在下载后计算并保存实际摘要,填写时则会严格比对。预存其他系统图标不代表已实现其云模板导入;其他系统可用合适 ISO 安装,云镜像适配后续扩展。 +2. 导入云镜像与救援模板;无已有模板不能开通云镜像 VM。云镜像是面板全局目录,验证后的源文件持久保存在 `runtime/engine/cloud-templates/`;首次在某宿主机创建或重装时才同步一份缓存,之后复用,所以切换地区不会更换或刷新镜像列表。面板磁盘需为源镜像及 256 MiB 安全余量预留空间,各宿主机的镜像存储池也需容纳本地缓存。云镜像 SHA-256 可留空;留空时仍会在下载后计算并保存实际摘要,填写时则会严格比对。预存其他系统图标不代表已实现其云模板导入;其他系统可用合适 ISO 安装,云镜像适配后续扩展。 3. 套餐设置规格、每期价格、账期天数、每期流量额度、库存和已启用宿主机。用户提交部署后先生成 30 分钟有效的开通账单,支付成功才创建实例;实例到期前 7 天再生成续费账单。两种账单均可用余额支付,也可由启用的易支付、Paytaro 或 USDT 渠道生成支付订单。在线支付回调会自动结清对应账单;失效账单的晚到款只保留在账户余额中,不会创建实例或恢复已进入删除流程的实例。 4. 测试用户注册/充值/生成开通账单/账单支付,确认未支付时没有引擎任务,支付后只创建一个实例;再验证提醒邮件、到期仅断网、宽限期内续费恢复网络,以及到期满 3 天后永久删除实例和系统盘;同时验收任务、VM 控制台、SSH、重装、救援、手动删除与图表。 5. 日志库 `engine_vm_metric_samples` 应在所有用户退出后继续新增,`engine_vm_traffic_totals` 应持续累加 RX/TX 字节;稳定 MAC 用于跨 QEMU/tap 重启衔接计数,首批数据需等待采样间隔。 diff --git a/engine/tests/test_app.py b/engine/tests/test_app.py index 7c209eec..b853afa9 100644 --- a/engine/tests/test_app.py +++ b/engine/tests/test_app.py @@ -2,6 +2,7 @@ from __future__ import annotations +import hashlib import re import stat import sys @@ -224,6 +225,22 @@ class FakeVMService: return {"volume_name": volume_name, "virtual_size": virtual_size, "image_bytes": image_path.stat().st_size} + def cache_template(self, image_path, volume_name, virtual_size, *, + trusted_replica_volume_name=None, + trusted_replica_identity=None): + self.calls.append(("cache_template", volume_name, virtual_size, + image_path.stat().st_size, + trusted_replica_volume_name, + trusted_replica_identity)) + cached_volume_name = trusted_replica_volume_name or volume_name + replica_identity = trusted_replica_identity or hashlib.sha256( + f"{cached_volume_name}:{virtual_size}".encode("ascii") + ).hexdigest() + return {"volume_name": cached_volume_name, "virtual_size": virtual_size, + "image_bytes": image_path.stat().st_size, + "already_cached": trusted_replica_volume_name is not None, + "replica_identity": replica_identity} + def import_rescue_iso(self, image_path, volume_name): self.calls.append(("import_rescue_iso", volume_name, image_path.stat().st_size)) return {"volume_name": volume_name, "image_bytes": image_path.stat().st_size} diff --git a/engine/tests/test_cloud_template_runtime.py b/engine/tests/test_cloud_template_runtime.py new file mode 100644 index 00000000..2559a9cd --- /dev/null +++ b/engine/tests/test_cloud_template_runtime.py @@ -0,0 +1,515 @@ +"""Global cloud-template catalog, source file and import job tests.""" + +from __future__ import annotations + +import hashlib +import json +import os +import stat +import threading +import time +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +import pytest + +from virsh_panel.cloud_template_runtime import ( + CATALOG_KIND, + IMPORT_JOB_KIND, + CloudTemplateRuntime, + GlobalCloudTemplateFiles, + GlobalCloudTemplateStore, + PERSISTENT_FREE_RESERVE_BYTES, + cloud_template_id_for, +) +from virsh_panel.cloud_templates import GIB, TemplateError, template_volume_name + + +ALMA_10_URL = ( + "https://repo.almalinux.org/almalinux/10/cloud/x86_64/images/" + "AlmaLinux-10-GenericCloud-latest.x86_64.qcow2" +) + + +class MemoryStore: + def __init__(self) -> None: + self.values: dict[tuple[str, str], str] = {} + self.lock = threading.RLock() + + def list(self, kind: str) -> dict[str, str]: + with self.lock: + return { + key: value for (record_kind, key), value in self.values.items() + if record_kind == kind + } + + def get(self, kind: str, key: str) -> str | None: + with self.lock: + return self.values.get((kind, key)) + + def set(self, kind: str, key: str, value: str) -> None: + with self.lock: + self.values[(kind, key)] = value + + def insert(self, kind: str, key: str, value: str) -> bool: + with self.lock: + identity = (kind, key) + if identity in self.values: + return False + self.values[identity] = value + return True + + def delete(self, kind: str, key: str) -> None: + with self.lock: + self.values.pop((kind, key), None) + + +class CommitThenRaiseCatalogStore(MemoryStore): + """Simulate a committed INSERT whose acknowledgement is lost.""" + + def insert(self, kind: str, key: str, value: str) -> bool: + inserted = super().insert(kind, key, value) + if kind == CATALOG_KIND and inserted: + raise RuntimeError("catalog insert acknowledgement lost") + return inserted + + +class AmbiguousCatalogCommitStore(CommitThenRaiseCatalogStore): + """Also fail the first catalog read after the ambiguous INSERT.""" + + def __init__(self) -> None: + super().__init__() + self.fail_next_catalog_read = False + + def list(self, kind: str) -> dict[str, str]: + if kind == CATALOG_KIND and self.fail_next_catalog_read: + self.fail_next_catalog_read = False + raise RuntimeError("catalog read unavailable") + return super().list(kind) + + def insert(self, kind: str, key: str, value: str) -> bool: + try: + return super().insert(kind, key, value) + except RuntimeError: + self.fail_next_catalog_read = True + raise + + +class LoseFirstCompletedJobWriteStore(MemoryStore): + """Drop one terminal job update while retaining its running record.""" + + def __init__(self) -> None: + super().__init__() + self.lost_completed_write = False + + def set(self, kind: str, key: str, value: str) -> None: + if kind == IMPORT_JOB_KIND: + job = json.loads(value) + if (job.get("status") == "completed" + and not self.lost_completed_write): + self.lost_completed_write = True + raise RuntimeError("completed job write acknowledgement lost") + super().set(kind, key, value) + + +def wait_for_job(runtime: CloudTemplateRuntime, job_id: str) -> dict: + for _ in range(200): + job = runtime.get_job(job_id) + if job["status"] in {"completed", "failed"}: + return job + time.sleep(0.01) + raise AssertionError("cloud template import did not finish") + + +def fake_download_bytes(payload: bytes): + digest = hashlib.sha256(payload).hexdigest() + + def download(_url: str, expected_sha256: str, directory: Path, **_kwargs): + assert expected_sha256 in {"", digest} + source = Path(directory) / ".cloud-image-test.qcow2" + source.write_bytes(payload) + os.chmod(source, 0o600) + return source, 2 * GIB, digest + + return digest, download + + +def record_for(payload: bytes, *, name: str = "AlmaLinux 10") -> dict: + digest = hashlib.sha256(payload).hexdigest() + template_id = cloud_template_id_for(digest, "10") + return { + "id": template_id, + "name": name, + "os_family": "AlmaLinux", + "os_version": "10", + "sha256": digest, + "volume_name": template_volume_name(template_id), + "virtual_size": 2 * GIB, + "image_bytes": len(payload), + "created_at": 1, + } + + +def test_stable_global_id_and_catalog_use_an_independent_record_kind(): + digest = "a" * 64 + first = cloud_template_id_for(digest, "10") + second = cloud_template_id_for(digest.upper(), "10") + assert first == second + assert first.startswith("alma10-") + assert template_volume_name(first).endswith(".qcow2") + + store = MemoryStore() + catalog = GlobalCloudTemplateStore(store) + record = record_for(b"catalog-image") + catalog.add(record) + + assert catalog.get(record["id"]) == record + assert list(store.list(CATALOG_KIND)) == [record["id"]] + assert store.list("template") == {} + + +def test_import_publishes_private_source_and_restores_global_catalog(tmp_path): + store = MemoryStore() + runtime = CloudTemplateRuntime(store, tmp_path) + payload = b"verified-global-cloud-image" + digest, downloader = fake_download_bytes(payload) + try: + with patch( + "virsh_panel.cloud_template_runtime.download_verified_image", + side_effect=downloader, + ): + queued = runtime.queue_import( + ALMA_10_URL, "AlmaLinux 10", digest + ) + job = wait_for_job(runtime, queued["job_id"]) + + assert job["status"] == "completed", job + template_id = job["template"]["id"] + assert template_id == cloud_template_id_for(digest, "10") + assert "volume_name" not in job["template"] + listed = runtime.list() + assert listed == [job["template"]] + assert "active_import_job" not in runtime.status() + + root = tmp_path / "cloud-templates" + source = root / template_volume_name(template_id) + assert source.read_bytes() == payload + assert stat.S_IMODE(root.stat().st_mode) == 0o700 + assert stat.S_IMODE(source.stat().st_mode) == 0o600 + assert store.get(IMPORT_JOB_KIND, queued["job_id"]) is not None + + # A new runtime reads the same host-independent catalog and verifies + # the persisted source from disk instead of trusting process memory. + restored = CloudTemplateRuntime(store, tmp_path) + try: + restored.restore_jobs() + assert restored.list() == listed + with restored.source(template_id) as (record, path): + assert record["sha256"] == digest + assert path == source + assert restored.in_use[template_id] == 1 + assert template_id not in restored.in_use + finally: + restored.close() + finally: + runtime.close() + + +def test_available_bytes_preserves_panel_disk_reserve(tmp_path): + files = GlobalCloudTemplateFiles(tmp_path) + with patch( + "virsh_panel.cloud_template_runtime.shutil.disk_usage", + return_value=SimpleNamespace( + free=PERSISTENT_FREE_RESERVE_BYTES + 3 * 1024 * 1024 + ), + ): + assert files.available_bytes() == 3 * 1024 * 1024 + + with patch( + "virsh_panel.cloud_template_runtime.shutil.disk_usage", + return_value=SimpleNamespace(free=PERSISTENT_FREE_RESERVE_BYTES - 1), + ): + assert files.available_bytes() == 0 + + +def test_import_rejects_before_queue_when_only_reserved_space_remains(tmp_path): + runtime = CloudTemplateRuntime(MemoryStore(), tmp_path) + try: + with patch.object( + runtime.files, "available_bytes", return_value=1024 * 1024 - 1 + ): + with pytest.raises(TemplateError, match="空间不足") as insufficient: + runtime.queue_import(ALMA_10_URL, "No Disk Space", "") + assert insufficient.value.status == 507 + assert runtime.jobs == {} + assert runtime.pending == set() + finally: + runtime.close() + + +def test_import_passes_only_space_above_reserve_to_downloader(tmp_path): + runtime = CloudTemplateRuntime(MemoryStore(), tmp_path) + payload = b"bounded-global-cloud-image" + digest = hashlib.sha256(payload).hexdigest() + limits: list[int] = [] + + def bounded_download(_url: str, _expected: str, directory: Path, *, + max_bytes: int): + limits.append(max_bytes) + source = Path(directory) / ".cloud-image-bounded.qcow2" + source.write_bytes(payload) + os.chmod(source, 0o600) + return source, 2 * GIB, digest + + try: + with patch.object( + runtime.files, "available_bytes", + side_effect=[3 * 1024 * 1024, 2 * 1024 * 1024], + ), patch( + "virsh_panel.cloud_template_runtime.download_verified_image", + side_effect=bounded_download, + ): + queued = runtime.queue_import(ALMA_10_URL, "Bounded", "") + job = wait_for_job(runtime, queued["job_id"]) + assert job["status"] == "completed", job + assert limits == [2 * 1024 * 1024] + finally: + runtime.close() + + +def test_duplicate_digest_and_casefold_name_are_rejected_globally(): + store = MemoryStore() + catalog = GlobalCloudTemplateStore(store) + first = record_for(b"first-image", name="AlmaLinux Gold") + catalog.add(first) + + with pytest.raises(TemplateError, match="已经导入") as duplicate_digest: + catalog.add(first) + assert duplicate_digest.value.status == 409 + + second = record_for(b"second-image", name="almalinux gold") + with pytest.raises(TemplateError, match="同名") as duplicate_name: + catalog.add(second) + assert duplicate_name.value.status == 409 + + +def test_rename_changes_only_display_name_and_keeps_source_identity(tmp_path): + store = MemoryStore() + catalog = GlobalCloudTemplateStore(store) + record = record_for(b"rename-image") + catalog.add(record) + updated = catalog.rename(record["id"], "AlmaLinux 10 Production") + + assert updated["name"] == "AlmaLinux 10 Production" + for key in ( + "id", "sha256", "volume_name", "virtual_size", "image_bytes", + "os_family", "os_version", "created_at", + ): + assert updated[key] == record[key] + + files = GlobalCloudTemplateFiles(tmp_path) + temporary = files.directory() / ".cloud-image-rename.qcow2" + temporary.write_bytes(b"rename-image") + os.chmod(temporary, 0o600) + source = files.publish( + temporary, record["id"], expected_size=len(b"rename-image") + ) + assert source.name == record["volume_name"] + + +def test_source_rejects_permission_damage_and_content_tampering(tmp_path): + store = MemoryStore() + runtime = CloudTemplateRuntime(store, tmp_path) + payload = b"source-integrity" + record = record_for(payload) + runtime.catalog.add(record) + temporary = runtime.files.directory() / ".cloud-image-integrity.qcow2" + temporary.write_bytes(payload) + os.chmod(temporary, 0o600) + source = runtime.files.publish( + temporary, record["id"], expected_size=len(payload) + ) + try: + os.chmod(source, 0o640) + with pytest.raises(TemplateError, match="已损坏"): + with runtime.source(record["id"]): + pass + + os.chmod(source, 0o600) + source.write_bytes(b"x" * len(payload)) + with pytest.raises(TemplateError, match="SHA-256"): + with runtime.source(record["id"]): + pass + finally: + runtime.close() + + +def test_concurrent_import_is_serialized_and_second_request_is_rejected(tmp_path): + runtime = CloudTemplateRuntime(MemoryStore(), tmp_path) + started = threading.Event() + release = threading.Event() + payload = b"blocking-image" + digest = hashlib.sha256(payload).hexdigest() + + def blocking_download(_url: str, _expected: str, directory: Path, + **_kwargs): + started.set() + assert release.wait(timeout=5) + source = Path(directory) / ".cloud-image-blocking.qcow2" + source.write_bytes(payload) + os.chmod(source, 0o600) + return source, 2 * GIB, digest + + try: + with patch( + "virsh_panel.cloud_template_runtime.download_verified_image", + side_effect=blocking_download, + ): + first = runtime.queue_import(ALMA_10_URL, "First", "") + assert started.wait(timeout=2) + assert runtime.has_active_jobs() is True + with pytest.raises(TemplateError, match="正在导入") as busy: + runtime.queue_import(ALMA_10_URL, "Second", "") + assert busy.value.status == 409 + release.set() + assert wait_for_job(runtime, first["job_id"])["status"] == "completed" + assert runtime.has_active_jobs() is False + finally: + release.set() + runtime.close() + + +def test_restore_marks_interrupted_job_failed_and_removes_partial_files(tmp_path): + store = MemoryStore() + job_id = "interrupted-cloud-template" + store.set(IMPORT_JOB_KIND, job_id, json.dumps({ + "job_id": job_id, + "status": "running", + "phase": "正在下载并校验云镜像", + "created_at": 1, + })) + files = GlobalCloudTemplateFiles(tmp_path) + partial = files.directory() / ".cloud-image-abandoned.qcow2" + partial.write_bytes(b"partial") + os.chmod(partial, 0o600) + + runtime = CloudTemplateRuntime(store, tmp_path) + try: + runtime.restore_jobs() + job = runtime.get_job(job_id) + assert job["status"] == "failed" + assert "重启" in job["phase"] + assert "重启" in job["error"] + assert not partial.exists() + assert runtime.has_active_jobs() is False + finally: + runtime.close() + + +def test_import_failure_redacts_internal_paths_and_releases_pending(tmp_path): + runtime = CloudTemplateRuntime(MemoryStore(), tmp_path) + try: + with patch( + "virsh_panel.cloud_template_runtime.download_verified_image", + side_effect=TemplateError( + "私有目录 /home/virt-cloud/data/cloud-templates 损坏", 500 + ), + ): + queued = runtime.queue_import(ALMA_10_URL, "Broken", "") + failed = wait_for_job(runtime, queued["job_id"]) + assert failed["status"] == "failed" + assert failed["error"] == "云镜像下载或保存失败,请联系管理员查看日志" + assert "/home/" not in json.dumps(failed, ensure_ascii=False) + assert runtime.pending == set() + finally: + runtime.close() + + +def test_catalog_insert_commit_with_lost_ack_is_completed_not_rolled_back( + tmp_path): + store = CommitThenRaiseCatalogStore() + runtime = CloudTemplateRuntime(store, tmp_path) + payload = b"catalog-commit-ack-lost" + digest, downloader = fake_download_bytes(payload) + try: + with patch( + "virsh_panel.cloud_template_runtime.download_verified_image", + side_effect=downloader, + ): + queued = runtime.queue_import(ALMA_10_URL, "Committed", digest) + job = wait_for_job(runtime, queued["job_id"]) + + assert job["status"] == "completed" + template_id = cloud_template_id_for(digest, "10") + assert runtime.get(template_id)["sha256"] == digest + source = tmp_path / "cloud-templates" / template_volume_name(template_id) + assert source.read_bytes() == payload + finally: + runtime.close() + + +def test_ambiguous_catalog_commit_preserves_source_for_startup_recovery( + tmp_path): + store = AmbiguousCatalogCommitStore() + runtime = CloudTemplateRuntime(store, tmp_path) + payload = b"ambiguous-catalog-commit" + digest, downloader = fake_download_bytes(payload) + try: + with patch( + "virsh_panel.cloud_template_runtime.download_verified_image", + side_effect=downloader, + ): + queued = runtime.queue_import(ALMA_10_URL, "Ambiguous", digest) + failed = wait_for_job(runtime, queued["job_id"]) + + assert failed["status"] == "failed" + template_id = cloud_template_id_for(digest, "10") + source = tmp_path / "cloud-templates" / template_volume_name(template_id) + assert source.read_bytes() == payload + finally: + runtime.close() + + restored = CloudTemplateRuntime(store, tmp_path) + try: + restored.restore_jobs() + recovered = restored.get_job(queued["job_id"]) + assert recovered["status"] == "completed" + assert recovered["template"]["id"] == template_id + assert source.read_bytes() == payload + finally: + restored.close() + + +def test_lost_completed_job_write_is_success_in_memory_and_repaired_on_restart( + tmp_path): + store = LoseFirstCompletedJobWriteStore() + runtime = CloudTemplateRuntime(store, tmp_path) + payload = b"lost-completed-job-write" + digest, downloader = fake_download_bytes(payload) + try: + with patch( + "virsh_panel.cloud_template_runtime.download_verified_image", + side_effect=downloader, + ): + queued = runtime.queue_import(ALMA_10_URL, "Recoverable", digest) + completed = wait_for_job(runtime, queued["job_id"]) + + assert completed["status"] == "completed" + persisted = json.loads(store.get(IMPORT_JOB_KIND, queued["job_id"])) + assert persisted["status"] == "running" + assert persisted["candidate_template_id"] == cloud_template_id_for( + digest, "10" + ) + finally: + runtime.close() + + restored = CloudTemplateRuntime(store, tmp_path) + try: + restored.restore_jobs() + repaired = restored.get_job(queued["job_id"]) + assert repaired["status"] == "completed" + persisted = json.loads(store.get(IMPORT_JOB_KIND, queued["job_id"])) + assert persisted["status"] == "completed" + finally: + restored.close() diff --git a/engine/tests/test_cloud_templates.py b/engine/tests/test_cloud_templates.py index 28e74403..9f60cfea 100644 --- a/engine/tests/test_cloud_templates.py +++ b/engine/tests/test_cloud_templates.py @@ -123,6 +123,20 @@ def test_failed_hash_never_leaves_unverified_template_file(tmp_path): assert list(tmp_path.iterdir()) == [] +def test_download_limit_preserves_persistent_disk_reserve(tmp_path): + body = qcow2_fixture() + digest = hashlib.sha256(body).hexdigest() + opener = SimpleNamespace(open=lambda *_args, **_kwargs: io.BytesIO(body)) + with patch("virsh_panel.cloud_templates.urllib.request.build_opener", + return_value=opener): + with pytest.raises(TemplateError, match="持久目录空间不足") as error: + download_verified_image( + OFFICIAL, digest, tmp_path, max_bytes=len(body) - 1 + ) + assert error.value.status == 507 + assert list(tmp_path.iterdir()) == [] + + def test_redirect_to_nonofficial_target_is_rejected_before_following(): handler = _RejectRedirects() with pytest.raises(TemplateError, match="重定向"): diff --git a/engine/tests/test_engine_boundary.py b/engine/tests/test_engine_boundary.py index 17107e81..d1d59063 100644 --- a/engine/tests/test_engine_boundary.py +++ b/engine/tests/test_engine_boundary.py @@ -15,9 +15,11 @@ from unittest.mock import MagicMock, patch import pytest from fastapi.testclient import TestClient -from test_app import CREATE_BODY, FakeRemoteManager, FakeVMService, HOST_BODY, VM_ID +from test_app import (CLOUD_CREATE_BODY, CREATE_BODY, OFFICIAL_CLOUD_URL, + FakeRemoteManager, FakeVMService, HOST_BODY, VM_ID) from virsh_panel.app import create_app -from virsh_panel.cloud_templates import TemplateError +from virsh_panel.cloud_templates import (TemplateError, template_id_for, + template_scope, template_volume_name) from virsh_panel.config import Settings from virsh_panel.database import DatabaseError, DatabaseTemplateStore from virsh_panel.engine_context import (EngineComponents, ExternalOwnership, HostStore, @@ -25,6 +27,7 @@ from virsh_panel.engine_context import (EngineComponents, ExternalOwnership, Hos from virsh_panel.engine_security import (EngineAuthError, authenticate, encode_actor, signed_headers) from virsh_panel.internal import EngineApplication, HostRegistry +from virsh_panel.libvirt_service import VMError from virsh_panel.metrics import host_scope from virsh_panel.user_isos import user_iso_scope @@ -180,7 +183,7 @@ def boundary(tmp_path): logs = MemoryStore() config = {"service_secret": base64.b64encode(SECRET).decode(), "data_dir": str(tmp_path)} services = {} - def factory(host_id, user_isos): + def factory(host_id, user_isos, cloud_templates): (tmp_path / host_id).mkdir(mode=0o700, exist_ok=True) scoped = HostStore(store, host_id) manager = FakeRemoteManager() @@ -192,7 +195,8 @@ def boundary(tmp_path): service = Service() services[host_id] = service components = EngineComponents(SimpleNamespace(users=ExternalOwnership(scoped)), manager, - DatabaseTemplateStore(scoped), scoped, logs, user_isos) + DatabaseTemplateStore(scoped), scoped, logs, user_isos, + cloud_templates) app = create_app(Settings(data_dir=tmp_path / host_id), engine_components=components, service_factory=lambda profile: service) return components, app @@ -237,6 +241,223 @@ def import_user_iso_for_test(client, tmp_path, name, *, digest="e" * 64): return job["iso"]["id"] +def test_global_cloud_template_is_visible_and_cached_once_per_host( + boundary, tmp_path): + client, engine, store, _logs, services = boundary + payload = b"verified-global-cloud-template" + digest = hashlib.sha256(payload).hexdigest() + + def fake_download(_url, _sha256, directory, **_kwargs): + source = directory / ".cloud-image-boundary.qcow2" + source.write_bytes(payload) + source.chmod(0o600) + return source, 2 * 1024**3, digest + + denied = call(client, "POST", "/internal/v1/vm-templates/import", + USER, {"url": OFFICIAL_CLOUD_URL, + "name": "Not Allowed", "sha256": ""}) + assert denied.status_code == 403 + with patch( + "virsh_panel.cloud_template_runtime.download_verified_image", + side_effect=fake_download): + queued = call(client, "POST", "/internal/v1/vm-templates/import", + SUPER, {"url": OFFICIAL_CLOUD_URL, + "name": "AlmaLinux Global", "sha256": ""}) + assert queued.status_code == 202, queued.text + job_id = queued.json()["job_id"] + for _ in range(100): + job = call( + client, "GET", + f"/internal/v1/vm-templates/import-jobs/{job_id}", SUPER, + ).json() + if job["status"] in {"completed", "failed"}: + break + time.sleep(0.01) + assert job["status"] == "completed", job + template_id = job["template"]["id"] + assert services == {} + assert call(client, "GET", "/internal/v1/vm-templates", USER).json() == { + "templates": [job["template"]], + } + + provisioning = {**USER, "can_provision": True} + body = {**CLOUD_CREATE_BODY, "template_id": template_id} + hk = call( + client, "POST", "/internal/v1/hosts/hk/vms", + {**provisioning, "operation_id": "global-template-hk-0001"}, body, + ) + assert hk.status_code == 201, hk.text + jp = call( + client, "POST", "/internal/v1/hosts/jp/vms", + {**provisioning, "operation_id": "global-template-jp-0001"}, + {**body, "name": "cloud-vm-jp", "public_ip_cidr": "8.8.8.19/24"}, + ) + assert jp.status_code == 201, jp.text + first_markers = { + host_id: json.loads(HostStore(store, host_id).get( + "cloud_template_replica", template_id)) + for host_id in ("hk", "jp") + } + # The public catalog intentionally omits the private backing filename. + source_volume_name = template_volume_name(template_id) + for host_id, marker in first_markers.items(): + assert marker["status"] == "ready" + assert marker["sha256"] == digest + assert len(marker["replica_identity"]) == 64 + assert marker["replica_volume_name"] != source_volume_name + assert marker["replica_volume_name"].startswith( + source_volume_name.removesuffix(".qcow2") + "-") + created = [row for row in services[host_id].calls + if row[0] == "create_cloud_vm"] + assert created[-1][2] == marker["replica_volume_name"] + + observed_markers = [] + original_hk_cache = services["hk"].cache_template + + def inspect_hk_marker(*args, **kwargs): + marker = json.loads(HostStore(store, "hk").get( + "cloud_template_replica", template_id)) + observed_markers.append(marker) + return original_hk_cache(*args, **kwargs) + + services["hk"].cache_template = inspect_hk_marker + hk_again = call( + client, "POST", "/internal/v1/hosts/hk/vms", + {**provisioning, "operation_id": "global-template-hk-0002"}, + {**body, "name": "cloud-vm-hk-2", "public_ip_cidr": "8.8.8.20/24"}, + ) + assert hk_again.status_code == 201, hk_again.text + services["hk"].cache_template = original_hk_cache + assert [marker["status"] for marker in observed_markers] == ["syncing"] + fallback_volume_name = observed_markers[0]["replica_volume_name"] + assert fallback_volume_name != first_markers["hk"]["replica_volume_name"] + + hk_cache = [row for row in services["hk"].calls + if row[0] == "cache_template"] + jp_cache = [row for row in services["jp"].calls + if row[0] == "cache_template"] + assert hk_cache[0][4:] == (None, None) + assert hk_cache[1][1] == fallback_volume_name + assert hk_cache[1][4] == first_markers["hk"]["replica_volume_name"] + assert hk_cache[1][5] == first_markers["hk"]["replica_identity"] + assert jp_cache[0][4:] == (None, None) + for host_id in ("hk", "jp"): + marker = json.loads(HostStore(store, host_id).get( + "cloud_template_replica", template_id)) + assert marker["status"] == "ready" + assert marker["sha256"] == digest + assert len(marker["replica_identity"]) == 64 + # A successful trusted reuse keeps the proven prior generation even + # though the durable syncing marker contained a fresh fallback name. + assert marker["replica_volume_name"] == ( + first_markers[host_id]["replica_volume_name"]) + + # Rebinding the logical host record to a different SSH host identity must + # invalidate its ready proof even when the pool name is unchanged. + engine.registry.components["hk"].manager.profile["fingerprint"] = ( + "SHA256:replacement-host") + rebound = call( + client, "POST", "/internal/v1/hosts/hk/vms", + {**provisioning, "operation_id": "global-template-hk-0003"}, + {**body, "name": "cloud-vm-hk-3", "public_ip_cidr": "8.8.8.21/24"}, + ) + assert rebound.status_code == 201, rebound.text + hk_cache = [row for row in services["hk"].calls + if row[0] == "cache_template"] + assert hk_cache[-1][4:] == (None, None) + rebound_marker = json.loads(HostStore(store, "hk").get( + "cloud_template_replica", template_id)) + assert rebound_marker["status"] == "ready" + assert rebound_marker["replica_volume_name"] == hk_cache[-1][1] + assert (rebound_marker["replica_volume_name"] + != first_markers["hk"]["replica_volume_name"]) + + # A crash/failure after the syncing marker was stored must retry that + # exact planned generation. Allocating yet another name would leak the + # possibly partial remote volume and make recovery non-deterministic. + failed_calls = [] + + def fail_hk_cache(image_path, volume_name, virtual_size, **kwargs): + failed_calls.append((volume_name, kwargs, json.loads( + HostStore(store, "hk").get( + "cloud_template_replica", template_id)))) + raise VMError("云镜像缓存同步中断", 502) + + services["hk"].cache_template = fail_hk_cache + interrupted = call( + client, "POST", "/internal/v1/hosts/hk/vms", + {**provisioning, "operation_id": "global-template-hk-0004"}, + {**body, "name": "cloud-vm-hk-4", "public_ip_cidr": "8.8.8.22/24"}, + ) + assert interrupted.status_code == 502, interrupted.text + assert len(failed_calls) == 1 + planned_volume_name, failed_kwargs, failed_marker = failed_calls[0] + assert failed_marker["status"] == "syncing" + assert failed_marker["replica_volume_name"] == planned_volume_name + assert failed_kwargs["trusted_replica_volume_name"] == ( + rebound_marker["replica_volume_name"]) + assert failed_kwargs["trusted_replica_identity"] == ( + rebound_marker["replica_identity"]) + + services["hk"].cache_template = original_hk_cache + retried = call( + client, "POST", "/internal/v1/hosts/hk/vms", + {**provisioning, "operation_id": "global-template-hk-0005"}, + {**body, "name": "cloud-vm-hk-5", "public_ip_cidr": "8.8.8.23/24"}, + ) + assert retried.status_code == 201, retried.text + hk_cache = [row for row in services["hk"].calls + if row[0] == "cache_template"] + assert hk_cache[-1][1] == planned_volume_name + assert hk_cache[-1][4:] == (None, None) + recovered_marker = json.loads(HostStore(store, "hk").get( + "cloud_template_replica", template_id)) + assert recovered_marker["status"] == "ready" + assert recovered_marker["replica_volume_name"] == planned_volume_name + assert [row for row in services["hk"].calls + if row[0] == "create_cloud_vm"][-1][2] == planned_volume_name + assert (tmp_path / "cloud-templates" + / engine.registry.cloud_templates.get(template_id)["volume_name"]).is_file() + + +def test_legacy_host_template_id_remains_usable_for_an_already_queued_order( + boundary): + client, engine, store, _logs, services = boundary + # Initialize the original target host, then seed the same host-scoped + # catalog record an order created before the global-catalog upgrade holds. + assert call(client, "GET", "/internal/v1/hosts/legacy/vms").status_code == 200 + profile = engine.registry.components["legacy"].manager.profile + scope = template_scope(profile) + digest = "d" * 64 + template_id = template_id_for(scope, digest, "10") + DatabaseTemplateStore(HostStore(store, "legacy")).add(scope, { + "id": template_id, + "name": "Legacy AlmaLinux", + "os_family": "AlmaLinux", + "os_version": "10", + "sha256": digest, + "volume_name": template_volume_name(template_id), + "virtual_size": 2 * 1024**3, + "image_bytes": 1024, + "created_at": 1, + }) + + response = call( + client, "POST", "/internal/v1/hosts/legacy/vms", + {**USER, "can_provision": True, + "operation_id": "legacy-template-order-0001"}, + {**CLOUD_CREATE_BODY, "template_id": template_id, + "name": "legacy-paid-order", "public_ip_cidr": "8.8.8.22/24"}, + ) + + assert response.status_code == 201, response.text + assert not any(row[0] == "cache_template" + for row in services["legacy"].calls) + create = next(row for row in services["legacy"].calls + if row[0] == "create_cloud_vm") + assert create[2] == template_volume_name(template_id) + + def test_user_iso_routes_are_owner_scoped_and_create_uses_opaque_id(boundary, tmp_path): client, _engine, _store, _logs, services = boundary download = tmp_path / "download.iso" diff --git a/engine/tests/test_libvirt_service.py b/engine/tests/test_libvirt_service.py index 5aa0c959..c88a520c 100644 --- a/engine/tests/test_libvirt_service.py +++ b/engine/tests/test_libvirt_service.py @@ -17,6 +17,12 @@ from unittest.mock import patch SERVICE_PATH = pathlib.Path(__file__).resolve().parents[1] / "virsh_panel" / "libvirt_service.py" TEMPLATE_NAME = "panel-template-alma10-12345678-0123456789abcdef.qcow2" +TEMPLATE_REPLICA_ONE = ( + "panel-template-alma10-12345678-0123456789abcdef-1111111111111111.qcow2" +) +TEMPLATE_REPLICA_TWO = ( + "panel-template-alma10-12345678-0123456789abcdef-2222222222222222.qcow2" +) RESCUE_NAME = "panel-rescue-0123456789abcdef0123456789abcdef.iso" USER_ISO_ID = "abcdef0123456789abcdef0123456789" USER_ISO_NAME = f"panel-user-iso-{USER_ISO_ID}.iso" @@ -42,6 +48,8 @@ class FakeLibvirtError(Exception): class FakeVolume: + _timestamp_counter = 1_700_000_000 + def __init__(self, pool, name, *, capacity=0, fmt="qcow2"): self.pool = pool self._name = name @@ -49,6 +57,14 @@ class FakeVolume: self.format = fmt self.contents = bytearray() self.cloned_from = None + # libvirt exposes these output-only filesystem timestamps in volume + # XML. Every new FakeVolume instance represents a new generation even + # when a deleted volume is recreated with the same name and size. + FakeVolume._timestamp_counter += 1 + generation = str(FakeVolume._timestamp_counter) + self.btime = generation + self.ctime = generation + self.mtime = generation def name(self): return self._name @@ -66,6 +82,11 @@ class FakeVolume: ET.SubElement(root, "capacity", unit="bytes").text = str(self.capacity) target = ET.SubElement(root, "target") ET.SubElement(target, "format", type=self.format) + timestamps = ET.SubElement(target, "timestamps") + ET.SubElement(timestamps, "atime").text = "1800000000" + ET.SubElement(timestamps, "mtime").text = self.mtime + ET.SubElement(timestamps, "ctime").text = self.ctime + ET.SubElement(timestamps, "btime").text = self.btime return ET.tostring(root, encoding="unicode") def resize(self, capacity, flags): @@ -1614,6 +1635,20 @@ class VMServiceTests(unittest.TestCase): self.assertEqual(self.backend.pools["images"].volumes["cloud-vm.qcow2"].cloned_from, alma9_volume) + def test_cloud_vm_accepts_generated_template_replica_volume(self): + service = self.cloud_service() + self.backend.pools["images"].volumes[TEMPLATE_REPLICA_ONE] = FakeVolume( + self.backend.pools["images"], TEMPLATE_REPLICA_ONE, + capacity=2 * 1024**3, + ) + with patch.object(self.service_module, "build_seed_iso", return_value=b"SEED", + create=True): + service.create_cloud_vm(self.cloud_spec(), TEMPLATE_REPLICA_ONE) + self.assertEqual( + self.backend.pools["images"].volumes["cloud-vm.qcow2"].cloned_from, + TEMPLATE_REPLICA_ONE, + ) + def test_cloud_vm_clones_template_and_uses_nonboot_seed_iso(self): service = self.cloud_service() created = self.create_cloud_vm(service, self.cloud_spec(os_name="AlmaLinux 10")) @@ -2100,6 +2135,167 @@ class VMServiceTests(unittest.TestCase): self.assertNotIn(TEMPLATE_NAME, self.backend.pools["images"].volumes) self.assertEqual(self.backend.opened, self.backend.closed) + def test_template_cache_reuses_only_a_trusted_complete_replica(self): + with tempfile.TemporaryDirectory() as directory: + image = pathlib.Path(directory) / "global-template.qcow2" + payload = b"QFI\xfb" + b"global cloud image fixture" + image.write_bytes(payload) + first = self.service.cache_template( + image, TEMPLATE_REPLICA_ONE, 2 * 1024**3, + ) + pool = self.backend.pools["images"] + original = pool.volumes[TEMPLATE_REPLICA_ONE] + + trusted = self.service.cache_template( + image, TEMPLATE_REPLICA_TWO, 2 * 1024**3, + trusted_replica_volume_name=TEMPLATE_REPLICA_ONE, + trusted_replica_identity=first["replica_identity"], + ) + self.assertTrue(trusted["already_cached"]) + self.assertEqual(trusted["volume_name"], TEMPLATE_REPLICA_ONE) + self.assertIs(pool.volumes[TEMPLATE_REPLICA_ONE], original) + self.assertNotIn(TEMPLATE_REPLICA_TWO, pool.volumes) + + replaced = self.service.cache_template( + image, TEMPLATE_REPLICA_TWO, 2 * 1024**3, + ) + + self.assertFalse(first["already_cached"]) + self.assertFalse(replaced["already_cached"]) + self.assertEqual(replaced["volume_name"], TEMPLATE_REPLICA_TWO) + self.assertIs(pool.volumes[TEMPLATE_REPLICA_ONE], original) + self.assertEqual(bytes(pool.volumes[TEMPLATE_REPLICA_TWO].contents), payload) + self.assertNotIn(TEMPLATE_REPLICA_ONE, self.backend.deleted_volumes) + + def test_template_cache_uses_fresh_generation_when_pool_identity_changes(self): + with tempfile.TemporaryDirectory() as directory: + image = pathlib.Path(directory) / "global-template.qcow2" + payload = b"QFI\xfb" + b"global cloud image fixture" + image.write_bytes(payload) + first = self.service.cache_template( + image, TEMPLATE_REPLICA_ONE, 2 * 1024**3) + pool = self.backend.pools["images"] + original = pool.volumes[TEMPLATE_REPLICA_ONE] + pool.path = "/srv/redefined-images" + self.backend.existing_dirs.add(pool.path) + + replaced = self.service.cache_template( + image, TEMPLATE_REPLICA_TWO, 2 * 1024**3, + trusted_replica_volume_name=TEMPLATE_REPLICA_ONE, + trusted_replica_identity=first["replica_identity"], + ) + + self.assertFalse(replaced["already_cached"]) + self.assertEqual(replaced["volume_name"], TEMPLATE_REPLICA_TWO) + self.assertNotEqual( + replaced["replica_identity"], first["replica_identity"]) + self.assertIs(pool.volumes[TEMPLATE_REPLICA_ONE], original) + self.assertIn(TEMPLATE_REPLICA_TWO, pool.volumes) + self.assertNotIn(TEMPLATE_REPLICA_ONE, self.backend.deleted_volumes) + + def test_template_cache_rejects_a_recreated_same_name_volume(self): + with tempfile.TemporaryDirectory() as directory: + image = pathlib.Path(directory) / "global-template.qcow2" + payload = b"QFI\xfb" + b"global cloud image fixture" + image.write_bytes(payload) + first = self.service.cache_template( + image, TEMPLATE_REPLICA_ONE, 2 * 1024**3) + pool = self.backend.pools["images"] + original = pool.volumes[TEMPLATE_REPLICA_ONE] + + # Recreate the same visible libvirt identity and allocation with + # different bytes. Name/path/capacity/allocation alone must not + # make this new volume generation trusted. + recreated = FakeVolume( + pool, TEMPLATE_REPLICA_ONE, + capacity=original.capacity, fmt=original.format, + ) + recreated.contents[:] = b"x" * len(payload) + pool.volumes[TEMPLATE_REPLICA_ONE] = recreated + + replaced = self.service.cache_template( + image, TEMPLATE_REPLICA_TWO, 2 * 1024**3, + trusted_replica_volume_name=TEMPLATE_REPLICA_ONE, + trusted_replica_identity=first["replica_identity"], + ) + + self.assertFalse(replaced["already_cached"]) + self.assertEqual(replaced["volume_name"], TEMPLATE_REPLICA_TWO) + self.assertIs(pool.volumes[TEMPLATE_REPLICA_ONE], recreated) + self.assertEqual(len(recreated.contents), len(payload)) + self.assertNotEqual(bytes(recreated.contents), payload) + self.assertEqual(bytes(pool.volumes[TEMPLATE_REPLICA_TWO].contents), payload) + self.assertNotIn(TEMPLATE_REPLICA_ONE, self.backend.deleted_volumes) + + def test_template_cache_reuploads_an_interrupted_desired_generation(self): + with tempfile.TemporaryDirectory() as directory: + image = pathlib.Path(directory) / "global-template.qcow2" + payload = b"QFI\xfb" + b"global cloud image fixture" + image.write_bytes(payload) + self.service.cache_template( + image, TEMPLATE_REPLICA_ONE, 2 * 1024**3, + ) + pool = self.backend.pools["images"] + interrupted = pool.volumes[TEMPLATE_REPLICA_ONE] + interrupted.contents[:] = b"partial" + + recovered = self.service.cache_template( + image, TEMPLATE_REPLICA_ONE, 2 * 1024**3, + ) + + self.assertFalse(recovered["already_cached"]) + self.assertIsNot(pool.volumes[TEMPLATE_REPLICA_ONE], interrupted) + self.assertEqual(bytes(pool.volumes[TEMPLATE_REPLICA_ONE].contents), payload) + self.assertIn(TEMPLATE_REPLICA_ONE, self.backend.deleted_volumes) + + def test_template_cache_refuses_to_overwrite_its_trusted_generation(self): + with tempfile.TemporaryDirectory() as directory: + image = pathlib.Path(directory) / "global-template.qcow2" + image.write_bytes(b"QFI\xfbglobal cloud image fixture") + first = self.service.cache_template( + image, TEMPLATE_REPLICA_ONE, 2 * 1024**3, + ) + original = self.backend.pools["images"].volumes[TEMPLATE_REPLICA_ONE] + + with self.assertRaises(self.error_type) as raised: + self.service.cache_template( + image, TEMPLATE_REPLICA_ONE, 2 * 1024**3, + trusted_replica_volume_name=TEMPLATE_REPLICA_ONE, + trusted_replica_identity=first["replica_identity"], + ) + + self.assertEqual(raised.exception.status, 400) + self.assertIs( + self.backend.pools["images"].volumes[TEMPLATE_REPLICA_ONE], original) + self.assertNotIn(TEMPLATE_REPLICA_ONE, self.backend.deleted_volumes) + + def test_template_cache_does_not_replace_an_untrusted_referenced_volume(self): + with tempfile.TemporaryDirectory() as directory: + image = pathlib.Path(directory) / "global-template.qcow2" + image.write_bytes(b"QFI\xfbglobal cloud image fixture") + self.service.cache_template( + image, TEMPLATE_REPLICA_ONE, 2 * 1024**3, + ) + # A disk source referencing the desired replica generation is + # enough to make replacement unsafe, even if that definition was + # created outside the panel. + domain = FakeDomain(self.backend, f""" + external-template-user + 00000000-0000-4000-8000-000000000099 + + + + """, running=True) + self.backend.domains.append(domain) + + with self.assertRaises(self.error_type) as raised: + self.service.cache_template( + image, TEMPLATE_REPLICA_ONE, 2 * 1024**3, + ) + + self.assertEqual(raised.exception.status, 409) + self.assertIn(TEMPLATE_REPLICA_ONE, self.backend.pools["images"].volumes) + def rescue_pool(self): pool = self.backend.pools.get("rescue") if pool is None: diff --git a/engine/virsh_panel/app.py b/engine/virsh_panel/app.py index 5d948af6..97a21f3c 100644 --- a/engine/virsh_panel/app.py +++ b/engine/virsh_panel/app.py @@ -10,6 +10,7 @@ import json import logging import os import re +import secrets import subprocess import tempfile import threading @@ -425,6 +426,11 @@ def create_app( "Legacy data/generations detected; its old SSH key files are not migrated " "or automatically removed. Verify and remove them separately." ) + shared_cloud_template_runtime = ( + engine_components.cloud_templates + if internal_mode and engine_components.cloud_templates is not None + else None + ) import_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="cloud-image-import") import_lock = threading.RLock() import_jobs: dict[str, dict] = {} @@ -458,6 +464,7 @@ def create_app( # transfer and marker commit so concurrent VM creates cannot both act on a # stale "not ready" observation for the same remote volume. user_iso_cache_lock = threading.RLock() + cloud_template_cache_lock = threading.RLock() reinstall_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="vm-reinstall") reinstall_lock = threading.RLock() reinstall_jobs: dict[str, dict] = {} @@ -765,11 +772,12 @@ def create_app( if data_store is None: return destinations = [ - ("import_job", import_jobs), ("reinstall_job", reinstall_jobs), ("rescue_import_job", rescue_import_jobs), ("rescue_job", rescue_jobs), ] + if shared_cloud_template_runtime is None: + destinations.append(("import_job", import_jobs)) if shared_user_iso_runtime is None: destinations.append(("user_iso_import_job", user_iso_import_jobs)) for kind, destination in destinations: @@ -1587,6 +1595,196 @@ def create_app( "public_vm_prefix": profile.get("public_vm_prefix"), } + def cloud_template_record(profile: dict, template_id: str) -> dict: + if shared_cloud_template_runtime is not None: + try: + return shared_cloud_template_runtime.get(template_id) + except TemplateError as exc: + if exc.status != 404: + raise + # Host-scoped template IDs were used before the catalog became + # global. Keep already-paid/queued operations executable on + # their original host while administrators re-import those + # images into the new global catalog. They cannot be promoted + # automatically because the old flow intentionally discarded + # the verified local source after uploading it to libvirt. + return templates.get(template_scope(profile), template_id) + return templates.get(template_scope(profile), template_id) + + def cloud_template_replica_scope(profile: dict) -> str: + """Bind a ready marker to the actual remote host configuration. + + ``template_scope`` deliberately keeps a stable logical host identity. + That is useful for legacy catalogs, but is too weak as proof that a + cached volume still belongs to the same physical hypervisor. A host + rebind or SSH host-key change must invalidate every ready marker. + """ + identity = [ + "virt-cloud-template-replica-v2", + profile.get("host_id"), + profile.get("host"), + profile.get("port"), + profile.get("fingerprint"), + profile.get("storage_pool"), + ] + return hashlib.sha256(json.dumps( + identity, ensure_ascii=False, separators=(",", ":"), + ).encode("utf-8")).hexdigest() + + def cloud_template_replica_volume_name(record: dict) -> str: + source_name = record["volume_name"] + if not isinstance(source_name, str) or not source_name.endswith(".qcow2"): + raise DatabaseError("云镜像源卷名无效") + return f"{source_name[:-6]}-{secrets.token_hex(8)}.qcow2" + + def valid_cloud_template_replica_volume_name( + record: dict, volume_name: object) -> bool: + source_name = record.get("volume_name") + if (not isinstance(source_name, str) + or not source_name.endswith(".qcow2") + or not isinstance(volume_name, str)): + return False + prefix = re.escape(source_name[:-6]) + return re.fullmatch( + rf"{prefix}-[0-9a-f]{{16}}\.qcow2", volume_name, + ) is not None + + def cloud_template_replica_payload( + profile: dict, record: dict, + status: Literal["syncing", "ready"], + replica_volume_name: str, + replica_identity: str | None = None) -> dict: + if not valid_cloud_template_replica_volume_name( + record, replica_volume_name): + raise DatabaseError("云镜像宿主机缓存卷名无效") + result = { + "status": status, + "replica_scope": cloud_template_replica_scope(profile), + "source_volume_name": record["volume_name"], + "replica_volume_name": replica_volume_name, + "sha256": record["sha256"], + "virtual_size": record["virtual_size"], + "image_bytes": record["image_bytes"], + } + if status == "ready": + if (not isinstance(replica_identity, str) + or re.fullmatch(r"[0-9a-f]{64}", replica_identity) is None): + raise DatabaseError("云镜像宿主机缓存身份无效") + result["replica_identity"] = replica_identity + return result + + def cloud_template_replica_marker( + profile: dict, record: dict) -> dict | None: + if data_store is None: + return None + raw = data_store.get("cloud_template_replica", record["id"]) + if raw is None: + return None + try: + value = json.loads(raw) + except (TypeError, ValueError) as exc: + raise DatabaseError("云镜像宿主机缓存记录损坏") from exc + if not isinstance(value, dict): + raise DatabaseError("云镜像宿主机缓存记录损坏") + status = value.get("status") + replica_volume_name = value.get("replica_volume_name") + if (status not in {"syncing", "ready"} + or not valid_cloud_template_replica_volume_name( + record, replica_volume_name)): + # Older/incomplete marker schemas are evidence only that a cache + # may exist, never proof that it is complete or belongs to this + # physical host generation. + return None + replica_identity = value.get("replica_identity") + if status == "ready" and ( + not isinstance(replica_identity, str) + or re.fullmatch(r"[0-9a-f]{64}", replica_identity) is None): + return None + expected = cloud_template_replica_payload( + profile, record, status, replica_volume_name, + replica_identity if status == "ready" else None, + ) + return value if value == expected else None + + def save_cloud_template_replica( + profile: dict, record: dict, + status: Literal["syncing", "ready"], + replica_volume_name: str, + replica_identity: str | None = None) -> None: + if data_store is None: + return + data_store.set( + "cloud_template_replica", record["id"], + json.dumps( + cloud_template_replica_payload( + profile, record, status, replica_volume_name, + replica_identity), + separators=(",", ":"), + ), + ) + + def ensure_cloud_template_replica( + profile: dict, service: VMService, template_id: str) -> dict: + record = cloud_template_record(profile, template_id) + if shared_cloud_template_runtime is None: + return record + with cloud_template_cache_lock: + # Re-read after entering the per-host critical section. Another + # create may have completed this same first-use transfer while the + # request was waiting for the lock. + try: + record = shared_cloud_template_runtime.get(template_id) + except TemplateError as exc: + if exc.status != 404: + raise + return templates.get(template_scope(profile), template_id) + marker = cloud_template_replica_marker(profile, record) + trusted_volume_name = None + trusted_identity = None + if marker is not None and marker["status"] == "ready": + trusted_volume_name = marker["replica_volume_name"] + trusted_identity = marker["replica_identity"] + # Persist a different, never-before-used fallback generation + # before asking libvirt whether the ready volume is still the + # exact volume proven by the marker. If it is not, the old + # name is never overwritten or silently reused. + desired_volume_name = cloud_template_replica_volume_name(record) + elif marker is not None: + # A crash after writing ``syncing`` may have left a partial + # upload. Retry that planned generation as untrusted; the + # service safely replaces it only when no VM references it. + desired_volume_name = marker["replica_volume_name"] + else: + desired_volume_name = cloud_template_replica_volume_name(record) + # Keep a proven ready marker until the global source has itself + # been verified. A damaged source must not downgrade evidence of + # an already complete host replica. + with shared_cloud_template_runtime.source(template_id) as ( + record, source_path): + # Once source verification succeeds, always downgrade the + # durable marker before touching libvirt. Even a previously + # ready replica may have disappeared or need replacement; a + # power loss during that upload must never leave ready proof + # pointing at a partially streamed qcow2 volume. + save_cloud_template_replica( + profile, record, "syncing", desired_volume_name) + cached = service.cache_template( + source_path, desired_volume_name, + record["virtual_size"], + trusted_replica_volume_name=trusted_volume_name, + trusted_replica_identity=trusted_identity, + ) + cached_volume_name = cached.get("volume_name") + accepted_names = {desired_volume_name} + if trusted_volume_name is not None: + accepted_names.add(trusted_volume_name) + if cached_volume_name not in accepted_names: + raise DatabaseError("云镜像宿主机缓存返回了无效卷") + save_cloud_template_replica( + profile, record, "ready", cached_volume_name, + cached.get("replica_identity")) + return {**record, "volume_name": cached_volume_name} + def user_iso_context(session: Session) -> tuple[int, str]: owner_id = account_for_session(session)["id"] return owner_id, user_iso_scope(owner_id) @@ -1875,6 +2073,8 @@ def create_app( @app.get("/api/v1/vm-templates") def list_templates(_session: Session = Depends(require_approved)): + if shared_cloud_template_runtime is not None: + return {"templates": shared_cloud_template_runtime.list()} profile = current_profile() return {"templates": templates.list(template_scope(profile))} @@ -1928,6 +2128,10 @@ def create_app( @app.post("/api/v1/vm-templates/import", status_code=202) def import_template(body: ImportTemplateBody, _session: Session = Depends(require_admin_write)): + if shared_cloud_template_runtime is not None: + forbid_long_job_during_update() + return shared_cloud_template_runtime.queue_import( + body.url, body.name, body.sha256) profile = current_profile() if body.sha256 and (len(body.sha256) != 64 or not all(char in "0123456789abcdefABCDEF" @@ -1973,6 +2177,10 @@ def create_app( @app.patch("/api/v1/vm-templates/{template_id}") def rename_template(template_id: str, body: RenameTemplateBody, _session: Session = Depends(require_admin_write)): + if shared_cloud_template_runtime is not None: + forbid_long_job_during_update() + return {"template": shared_cloud_template_runtime.rename( + template_id, body.name)} scope = template_scope(current_profile()) with import_lock: if pending_templates: @@ -1981,6 +2189,8 @@ def create_app( @app.get("/api/v1/vm-templates/import-jobs/{job_id}") def import_job(job_id: str, _session: Session = Depends(require_admin)): + if shared_cloud_template_runtime is not None: + return shared_cloud_template_runtime.get_job(job_id) scope = template_scope(current_profile()) with import_lock: job = import_jobs.get(job_id) @@ -2372,7 +2582,12 @@ def create_app( status_code=422, detail="宿主机尚未配置公网地址池,请手动填写 IP、网关和 DNS", ) - record = templates.get(template_scope(profile), body.template_id) + service = new_service(profile) + # The catalog is engine-global. The selected host receives a + # verified replica only on first use, before IP/credential/VM + # state is mutated. + record = ensure_cloud_template_replica( + profile, service, body.template_id) spec = body.model_dump(exclude_none=True, exclude={ "source", "template_id", "root_password", "guest_ssh_key_id", }) @@ -2382,7 +2597,6 @@ def create_app( spec["root_password_hash"] = password_hash requested_uuid = actor()["vm_uuid"] if internal_mode else str(uuid.uuid4()) spec["vm_uuid"] = requested_uuid - service = new_service(profile) allocated_ip = "" owner_assigned = False try: @@ -2650,7 +2864,6 @@ def create_app( or rescue_vm.get("rescue_staged")): raise VMError("虚拟机处于救援模式或救援任务中,请先退出救援模式", 409) scope = template_scope(profile) - record = templates.get(scope, body.template_id) vm = service.get_vm(vm_id) if not vm.get("managed") or not vm.get("public_ip_cidr"): raise VMError("只有面板创建的公网云镜像虚拟机支持自动重装", 409) @@ -2660,6 +2873,10 @@ def create_app( key = (scope, normalized_id) if key in pending_reinstalls: raise VMError("该虚拟机正在重装系统", 409) + # Finish and verify the target-host replica before scheduling any + # task that can shut down the guest or replace its system disk. + record = ensure_cloud_template_replica( + profile, service, body.template_id) job_id = str(uuid.uuid4()) reinstall_jobs[job_id] = { "owner": _session.username, diff --git a/engine/virsh_panel/cloud_template_runtime.py b/engine/virsh_panel/cloud_template_runtime.py new file mode 100644 index 00000000..e85b54f3 --- /dev/null +++ b/engine/virsh_panel/cloud_template_runtime.py @@ -0,0 +1,743 @@ +"""Engine-global cloud template catalog and persistent source images. + +The legacy cloud-template flow downloads an image inside one host child app, +uploads it immediately to that host, and then deletes the local download. A +multi-host control plane needs a different ownership boundary: the engine owns +one immutable source image and each hypervisor owns only a disposable replica. + +This module deliberately contains no libvirt code. It provides the global +catalog, import jobs and verified ``source()`` leases that a host-specific +cache layer can consume later. +""" + +from __future__ import annotations + +import hashlib +import json +import logging +import os +import re +import shutil +import stat +import threading +import time +import uuid +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager +from pathlib import Path +from typing import Any, Iterator + +from .cloud_templates import ( + GIB, + MAX_IMAGE_BYTES, + MAX_VIRTUAL_BYTES, + SHA256_RE, + TemplateError, + download_verified_image, + image_version_for, + template_id_for, + template_volume_name, + validate_template_name, +) + + +LOG = logging.getLogger(__name__) + +# Keep the existing, already validated template/volume identifier grammar, but +# replace its old host scope with one fixed engine-global namespace. The full +# digest remains in the catalog and every source/replica marker must compare it; +# the shortened digest in the public ID is never treated as integrity proof. +GLOBAL_TEMPLATE_SCOPE = hashlib.sha256( + b"virt-cloud/global-cloud-template/v1" +).hexdigest() + +CATALOG_KIND = "cloud_template" +IMPORT_JOB_KIND = "cloud_template_import_job" +SOURCE_DIRECTORY_NAME = "cloud-templates" +PERSISTENT_FREE_RESERVE_BYTES = 256 * 1024**2 +MIN_DOWNLOAD_BYTES = 1024 * 1024 +SOURCE_FILE_RE = re.compile( + r"panel-template-alma(?:9|10)-[0-9a-f]{8}-[0-9a-f]{16}\.qcow2\Z" +) +TEMPORARY_FILE_RE = re.compile(r"\.cloud-image-[A-Za-z0-9_-]+\.qcow2\Z") + + +def cloud_template_id_for(digest: str, version: str) -> str: + """Return a stable ID that is independent of every target hypervisor.""" + if not isinstance(digest, str) or not SHA256_RE.fullmatch(digest): + raise TemplateError("云镜像 SHA-256 格式无效") + return template_id_for(GLOBAL_TEMPLATE_SCOPE, digest.lower(), version) + + +class GlobalCloudTemplateStore: + """Strict global catalog backed by its own encrypted record kind.""" + + def __init__(self, store: Any | None) -> None: + self.store = store + self._memory: dict[str, str] = {} + self._lock = threading.RLock() + + @staticmethod + def _validate(record: Any) -> dict[str, Any]: + if not isinstance(record, dict): + raise TemplateError("全局云镜像目录记录损坏", 500) + try: + template_id = record["id"] + name = record["name"] + os_family = record["os_family"] + os_version = record["os_version"] + digest = record["sha256"] + volume_name = record["volume_name"] + virtual_size = record["virtual_size"] + image_bytes = record["image_bytes"] + created_at = record["created_at"] + except KeyError as exc: + raise TemplateError("全局云镜像目录记录损坏", 500) from exc + try: + normalized_name = validate_template_name(name) + except TemplateError as exc: + raise TemplateError("全局云镜像目录记录损坏", 500) from exc + valid = ( + isinstance(template_id, str) + and isinstance(name, str) + and normalized_name == name + and os_family == "AlmaLinux" + and os_version in {"9", "10"} + and isinstance(digest, str) + and SHA256_RE.fullmatch(digest) is not None + and digest == digest.lower() + and template_id == cloud_template_id_for(digest, os_version) + and volume_name == template_volume_name(template_id) + and not isinstance(virtual_size, bool) + and isinstance(virtual_size, int) + and GIB <= virtual_size <= MAX_VIRTUAL_BYTES + and not isinstance(image_bytes, bool) + and isinstance(image_bytes, int) + and 0 < image_bytes <= MAX_IMAGE_BYTES + and not isinstance(created_at, bool) + and isinstance(created_at, int) + and created_at > 0 + ) + if not valid: + raise TemplateError("全局云镜像目录记录损坏", 500) + # Reject unrecognised fields instead of accidentally persisting a URL, + # local path, host identity or another future secret in public records. + if set(record) != { + "id", "name", "os_family", "os_version", "sha256", + "volume_name", "virtual_size", "image_bytes", "created_at", + }: + raise TemplateError("全局云镜像目录记录损坏", 500) + return record + + def _source(self) -> dict[str, str]: + return self.store.list(CATALOG_KIND) if self.store is not None else self._memory.copy() + + def _list(self) -> list[dict[str, Any]]: + records = [] + for key, payload in self._source().items(): + try: + record = json.loads(payload) + except (TypeError, ValueError) as exc: + raise TemplateError("全局云镜像目录记录损坏", 500) from exc + record = self._validate(record) + if key != record["id"]: + raise TemplateError("全局云镜像目录记录损坏", 500) + records.append(record) + return sorted( + records, + key=lambda item: (item["created_at"], item["id"]), + reverse=True, + ) + + def list(self) -> list[dict[str, Any]]: + with self._lock: + return self._list() + + def get(self, template_id: str) -> dict[str, Any]: + # This validates the opaque identifier before consulting storage. + template_volume_name(template_id) + with self._lock: + for record in self._list(): + if record["id"] == template_id: + return record + raise TemplateError("云镜像模板不存在", 404) + + def add(self, record: dict[str, Any]) -> dict[str, Any]: + record = self._validate(record.copy()) + encoded = json.dumps(record, ensure_ascii=False, separators=(",", ":")) + with self._lock: + current = self._list() + if any(row["id"] == record["id"] for row in current): + raise TemplateError("此云镜像已经导入", 409) + if any(row["name"].casefold() == record["name"].casefold() + for row in current): + raise TemplateError("已有同名云镜像模板", 409) + if self.store is not None: + if not self.store.insert(CATALOG_KIND, record["id"], encoded): + raise TemplateError("此云镜像已经导入", 409) + else: + self._memory[record["id"]] = encoded + return record + + def rename(self, template_id: str, name: str) -> dict[str, Any]: + name = validate_template_name(name) + with self._lock: + current = self._list() + target = next((row for row in current if row["id"] == template_id), None) + if target is None: + raise TemplateError("云镜像模板不存在", 404) + if any(row["id"] != template_id + and row["name"].casefold() == name.casefold() + for row in current): + raise TemplateError("已有同名云镜像模板", 409) + updated = {**target, "name": name} + encoded = json.dumps(updated, ensure_ascii=False, separators=(",", ":")) + if self.store is not None: + self.store.set(CATALOG_KIND, template_id, encoded) + else: + self._memory[template_id] = encoded + return updated + + +class GlobalCloudTemplateFiles: + """Private engine-owned source files, never paths supplied by a caller.""" + + def __init__(self, data_dir: Path) -> None: + data_dir = Path(data_dir) + if not data_dir.is_absolute() or data_dir == Path("/"): + raise ValueError("cloud template data directory must be absolute and dedicated") + self.data_dir = data_dir + self.root = data_dir / SOURCE_DIRECTORY_NAME + + @staticmethod + def _ensure_directory(path: Path) -> None: + try: + info = path.lstat() + except FileNotFoundError: + path.mkdir(mode=0o700) + info = path.lstat() + if (not stat.S_ISDIR(info.st_mode) or stat.S_ISLNK(info.st_mode) + or info.st_uid != os.geteuid()): + raise TemplateError("云镜像持久目录不安全,请联系管理员", 500) + if stat.S_IMODE(info.st_mode) != 0o700: + os.chmod(path, 0o700) + + def directory(self) -> Path: + self.data_dir.mkdir(mode=0o700, parents=True, exist_ok=True) + self._ensure_directory(self.data_dir) + self._ensure_directory(self.root) + return self.root + + def available_bytes(self) -> int: + """Return writable bytes while preserving space for panel operation.""" + free = int(shutil.disk_usage(self.directory()).free) + return max(0, free - PERSISTENT_FREE_RESERVE_BYTES) + + def path(self, template_id: str, *, expected_size: int | None = None) -> Path: + path = self.directory() / template_volume_name(template_id) + try: + info = path.lstat() + except FileNotFoundError as exc: + raise TemplateError("云镜像源文件不存在,请重新导入", 409) from exc + if (not stat.S_ISREG(info.st_mode) or stat.S_ISLNK(info.st_mode) + or info.st_uid != os.geteuid() + or stat.S_IMODE(info.st_mode) != 0o600 + or (expected_size is not None and info.st_size != expected_size)): + raise TemplateError("云镜像源文件已损坏,请联系管理员", 500) + return path + + def publish(self, temporary: Path, template_id: str, *, expected_size: int) -> Path: + temporary = Path(temporary) + directory = self.directory() + try: + info = temporary.lstat() + except FileNotFoundError as exc: + raise TemplateError("云镜像下载临时文件丢失", 500) from exc + if (not stat.S_ISREG(info.st_mode) or stat.S_ISLNK(info.st_mode) + or info.st_uid != os.geteuid() + or stat.S_IMODE(info.st_mode) != 0o600 + or temporary.parent != directory + or info.st_size != expected_size + or expected_size <= 0 or expected_size > MAX_IMAGE_BYTES): + raise TemplateError("云镜像下载临时文件无效", 500) + destination = directory / template_volume_name(template_id) + try: + # download_verified_image creates the temporary in this directory. + # A hard link publishes atomically and cannot replace an existing + # immutable source with attacker-controlled bytes. + os.link(temporary, destination) + os.chmod(destination, 0o600) + temporary.unlink() + descriptor = os.open( + destination.parent, + os.O_RDONLY | getattr(os, "O_DIRECTORY", 0), + ) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + except FileExistsError as exc: + raise TemplateError("此云镜像已经导入", 409) from exc + except OSError as exc: + destination.unlink(missing_ok=True) + raise TemplateError("云镜像无法保存到持久目录", 507) from exc + return destination + + def delete(self, template_id: str) -> None: + path = self.directory() / template_volume_name(template_id) + try: + info = path.lstat() + except FileNotFoundError: + return + if not stat.S_ISREG(info.st_mode) or stat.S_ISLNK(info.st_mode): + raise TemplateError("云镜像源文件无法安全删除", 500) + path.unlink() + + def reconcile(self, catalog: GlobalCloudTemplateStore) -> None: + """Remove only provable interrupted downloads and uncataloged sources.""" + try: + self.root.lstat() + except FileNotFoundError: + return + self._ensure_directory(self.root) + active = {row["volume_name"] for row in catalog.list()} + for candidate in self.root.iterdir(): + try: + info = candidate.lstat() + except OSError: + continue + if (not stat.S_ISREG(info.st_mode) or stat.S_ISLNK(info.st_mode) + or info.st_uid != os.geteuid()): + continue + interrupted = TEMPORARY_FILE_RE.fullmatch(candidate.name) is not None + orphan = (SOURCE_FILE_RE.fullmatch(candidate.name) is not None + and candidate.name not in active) + if interrupted or orphan: + candidate.unlink(missing_ok=True) + + +class CloudTemplateRuntime: + """One engine-process runtime shared by all host child applications.""" + + def __init__(self, store: Any | None, data_dir: Path) -> None: + self.store = store + self.catalog = GlobalCloudTemplateStore(store) + self.files = GlobalCloudTemplateFiles(data_dir) + self.executor = ThreadPoolExecutor( + max_workers=1, thread_name_prefix="global-cloud-template-import" + ) + self.lock = threading.RLock() + self.jobs: dict[str, dict[str, Any]] = {} + self.pending: set[str] = set() + self.in_use: dict[str, int] = {} + self.verified_sources: dict[str, tuple[int, int, int, int, str]] = {} + self._restored = False + + @staticmethod + def public_record(record: dict[str, Any]) -> dict[str, Any]: + return {key: value for key, value in record.items() if key in { + "id", "name", "os_family", "os_version", "sha256", + "virtual_size", "image_bytes", "created_at", + }} + + @staticmethod + def public_error(value: Any) -> str: + safe = { + "面板在云镜像导入期间重启;请重新提交导入", + "云镜像地址无效", + "仅支持 AlmaLinux 官方仓库的 9/10 x86_64 GenericCloud qcow2 镜像", + "官方镜像地址发生重定向,请使用最终的官方 HTTPS 直链", + "SHA-256 可留空;填写时必须为 64 位十六进制校验值", + "下载官方云镜像失败,请检查面板服务器网络", + "云镜像下载超过 60 分钟,请检查面板服务器网络", + "云镜像超过 4 GiB 限制", + "云镜像 SHA-256 不匹配,已丢弃下载内容", + "下载文件不是有效的 qcow2 镜像", + "云镜像 qcow2 版本或簇大小不受支持", + "云镜像 qcow2 头部不完整", + "云镜像包含不受支持或外部依赖的 qcow2 特性", + "云镜像含外部依赖、加密或容量超出支持范围", + "此云镜像已经导入", + "已有同名云镜像模板", + "云镜像无法保存到持久目录", + "面板云镜像持久目录空间不足", + "云镜像导入失败,请查看引擎日志", + } + if isinstance(value, str) and value in safe: + return value + return "云镜像下载或保存失败,请联系管理员查看日志" + + @classmethod + def public_job(cls, job: dict[str, Any]) -> dict[str, Any]: + allowed = { + "job_id", "status", "phase", "created_at", "started_at", + "finished_at", "template", "error", + } + result = {key: value for key, value in job.items() if key in allowed} + if isinstance(result.get("template"), dict): + result["template"] = cls.public_record(result["template"]) + if "error" in result: + result["error"] = cls.public_error(result["error"]) + return result + + def list(self) -> list[dict[str, Any]]: + """Return the public, host-independent template list.""" + return [self.public_record(row) for row in self.catalog.list()] + + def status(self) -> dict[str, Any]: + """Return the public list plus the current import, when one exists.""" + result: dict[str, Any] = {"templates": self.list()} + with self.lock: + active = [job for job in self.jobs.values() + if job.get("status") in {"queued", "running"}] + if active: + current = max(active, key=lambda job: ( + int(job.get("created_at", 0)), str(job.get("job_id", "")))) + result["active_import_job"] = self.public_job(current) + return result + + def get(self, template_id: str) -> dict[str, Any]: + return self.catalog.get(template_id) + + def get_job(self, job_id: str) -> dict[str, Any]: + with self.lock: + job = self.jobs.get(job_id) + if job is None: + raise TemplateError("云镜像导入任务不存在", 404) + return self.public_job(job) + + def rename(self, template_id: str, name: str) -> dict[str, Any]: + with self.lock: + if self.pending: + raise TemplateError("云镜像正在导入,请稍后修改模板名称", 409) + return self.public_record(self.catalog.rename(template_id, name)) + + def _remaining_download_bytes(self) -> int: + maximum = min(MAX_IMAGE_BYTES, self.files.available_bytes()) + if maximum < MIN_DOWNLOAD_BYTES: + raise TemplateError("面板云镜像持久目录空间不足", 507) + return maximum + + def queue_import(self, url: str, name: str, + expected_sha256: str = "") -> dict[str, str]: + version = image_version_for(url) + name = validate_template_name(name) + if (not isinstance(expected_sha256, str) + or (expected_sha256 and not SHA256_RE.fullmatch(expected_sha256))): + raise TemplateError( + "SHA-256 可留空;填写时必须为 64 位十六进制校验值", 422 + ) + expected_sha256 = expected_sha256.lower() + pending_key = expected_sha256 or hashlib.sha256(url.encode("utf-8")).hexdigest() + with self.lock: + existing = self.catalog.list() + self._remaining_download_bytes() + if expected_sha256 and any( + row["id"] == cloud_template_id_for(expected_sha256, version) + for row in existing): + raise TemplateError("此云镜像已经导入", 409) + if any(row["name"].casefold() == name.casefold() for row in existing): + raise TemplateError("已有同名云镜像模板", 409) + if pending_key in self.pending: + raise TemplateError("该云镜像正在导入", 409) + if self.pending: + raise TemplateError( + "已有云镜像正在导入,请等待完成后再提交", 409 + ) + job_id = str(uuid.uuid4()) + job = { + "job_id": job_id, + "status": "queued", + "phase": "等待下载", + "created_at": int(time.time()), + } + self.pending.add(pending_key) + try: + self.save_job(job_id, job) + self.executor.submit( + self._run_import, + job_id, + url, + name, + expected_sha256, + pending_key, + ) + except Exception as exc: + self.pending.discard(pending_key) + self.delete_job(job_id) + if isinstance(exc, RuntimeError): + raise TemplateError( + "引擎正在停止,无法开始云镜像导入", 503 + ) from exc + raise + return {"job_id": job_id, "status": "queued"} + + def _run_import(self, job_id: str, url: str, name: str, + expected_sha256: str, pending_key: str) -> None: + temporary: Path | None = None + published = False + cataloged = False + commit_uncertain = False + template_id = "" + record: dict[str, Any] | None = None + try: + with self.lock: + job = self.jobs[job_id] + job.update( + status="running", + phase="正在下载并校验云镜像", + started_at=int(time.time()), + ) + self.save_job(job_id, job) + directory = self.files.directory() + maximum = self._remaining_download_bytes() + temporary, virtual_size, actual_sha256 = download_verified_image( + url, expected_sha256, directory, max_bytes=maximum + ) + version = image_version_for(url) + template_id = cloud_template_id_for(actual_sha256, version) + size = temporary.stat().st_size + record = { + "id": template_id, + "name": name, + "os_family": "AlmaLinux", + "os_version": version, + "sha256": actual_sha256, + "volume_name": template_volume_name(template_id), + "virtual_size": virtual_size, + "image_bytes": size, + "created_at": int(time.time()), + } + with self.lock: + if any(row["id"] == template_id for row in self.catalog.list()): + raise TemplateError("此云镜像已经导入", 409) + if any(row["name"].casefold() == name.casefold() + for row in self.catalog.list()): + raise TemplateError("已有同名云镜像模板", 409) + job = self.jobs[job_id] + job["phase"] = "正在保存云镜像源文件" + # Persist the deterministic candidate before publishing. If + # the process stops after the catalog commit but before the + # terminal job write, startup can prove that this interrupted + # job owns the committed catalog/source pair. + job["candidate_template_id"] = template_id + self.save_job(job_id, job) + source = self.files.publish( + temporary, template_id, expected_size=size + ) + temporary = None + published = True + self._mark_verified(template_id, source, actual_sha256) + self.catalog.add(record) + cataloged = True + self._save_completed_job(job_id, job, record) + except Exception as exc: + # A database insert can commit and still raise (for example when + # the connection drops before its acknowledgement reaches this + # process). Once the source was published, never delete it until + # the catalog has been read back. An exact record is proof of a + # completed import; an unavailable read is ambiguous and must + # preserve the source for startup reconciliation. + committed: dict[str, Any] | None = None + if published and template_id and not cataloged: + try: + existing = self.catalog.get(template_id) + except TemplateError as read_exc: + if read_exc.status != 404: + commit_uncertain = True + except Exception: + commit_uncertain = True + LOG.exception( + "Could not resolve cloud template catalog commit: job=%s", + job_id, + ) + else: + cataloged = True + if record is not None and existing == record: + committed = existing + if committed is not None: + with self.lock: + previous = self.jobs.get(job_id, {}) + self._save_completed_job(job_id, previous, committed) + return + if isinstance(exc, TemplateError): + error = str(exc) + else: + LOG.exception("Global cloud template import failed: job=%s", job_id) + error = "云镜像导入失败,请查看引擎日志" + with self.lock: + if cataloged or commit_uncertain: + # A catalog write is the commit point. Do not remove a + # committed (or ambiguously committed) template merely + # because its acknowledgement failed. Startup can prove + # the candidate from the persisted running job. + pass + elif published and template_id: + try: + self.files.delete(template_id) + except Exception: + LOG.exception( + "Could not remove failed cloud template source: job=%s", + job_id, + ) + previous = self.jobs.get(job_id, {}) + failed = { + "job_id": job_id, + "status": "failed", + "phase": "导入失败", + "error": error, + "created_at": previous.get("created_at", int(time.time())), + "started_at": previous.get("started_at"), + "finished_at": int(time.time()), + } + if template_id: + failed["candidate_template_id"] = template_id + if commit_uncertain: + failed["commit_uncertain"] = True + try: + self.save_job(job_id, failed) + except Exception: + LOG.exception( + "Could not persist cloud template import failure: job=%s", + job_id, + ) + finally: + if temporary is not None: + temporary.unlink(missing_ok=True) + with self.lock: + self.pending.discard(pending_key) + + def _save_completed_job(self, job_id: str, previous: dict[str, Any], + record: dict[str, Any]) -> None: + """Publish an in-memory success even if its final DB write fails. + + The catalog record is the durable commit point. Reporting a failed + import after that point invites a duplicate retry, so the running + process must expose success while startup reconstructs the same result + from the candidate ID if the terminal job write was lost. + """ + completed = { + "job_id": job_id, + "status": "completed", + "phase": "云镜像已保存,可用于所有宿主机", + "created_at": previous.get("created_at", int(time.time())), + "started_at": previous.get("started_at"), + "finished_at": int(time.time()), + "template": record, + } + try: + self.save_job(job_id, completed) + except Exception: + # save_job updates self.jobs before touching persistent storage. + # Keep that completed state and let restore_jobs repair a stale + # persisted running record on the next process start. + self.jobs[job_id] = completed + LOG.exception( + "Could not persist completed cloud template import: job=%s", + job_id, + ) + + def _mark_verified(self, template_id: str, path: Path, digest: str) -> None: + info = path.stat() + self.verified_sources[template_id] = ( + info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns, digest + ) + + def _verify_source(self, template_id: str, path: Path, digest: str) -> None: + info = path.stat() + identity = (info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns) + cached = self.verified_sources.get(template_id) + if cached is not None and cached[:4] == identity and cached[4] == digest: + return + checksum = hashlib.sha256() + with path.open("rb") as source: + while chunk := source.read(1024 * 1024): + checksum.update(chunk) + if checksum.hexdigest() != digest: + raise TemplateError("云镜像源文件的 SHA-256 校验值不匹配", 500) + self._mark_verified(template_id, path, digest) + + @contextmanager + def source(self, template_id: str) -> Iterator[tuple[dict[str, Any], Path]]: + with self.lock: + record = self.catalog.get(template_id) + path = self.files.path(template_id, expected_size=record["image_bytes"]) + self._verify_source(template_id, path, record["sha256"]) + self.in_use[template_id] = self.in_use.get(template_id, 0) + 1 + try: + yield record, path + finally: + with self.lock: + count = self.in_use.get(template_id, 0) + if count <= 1: + self.in_use.pop(template_id, None) + else: + self.in_use[template_id] = count - 1 + + def save_job(self, job_id: str, job: dict[str, Any]) -> None: + self.jobs[job_id] = job + if self.store is not None: + self.store.set( + IMPORT_JOB_KIND, + job_id, + json.dumps(job, ensure_ascii=False, separators=(",", ":")), + ) + + def delete_job(self, job_id: str) -> None: + self.jobs.pop(job_id, None) + if self.store is not None: + self.store.delete(IMPORT_JOB_KIND, job_id) + + def restore_jobs(self) -> None: + """Reconcile local files and fail imports interrupted by a restart.""" + with self.lock: + if self._restored: + return + self.files.reconcile(self.catalog) + source = self.store.list(IMPORT_JOB_KIND) if self.store is not None else {} + for job_id, raw in source.items(): + try: + job = json.loads(raw) + except (TypeError, ValueError) as exc: + raise TemplateError("云镜像导入任务记录损坏", 500) from exc + if not isinstance(job, dict) or job.get("job_id") != job_id: + raise TemplateError("云镜像导入任务记录损坏", 500) + interrupted = job.get("status") in {"queued", "running"} + uncertain = job.get("commit_uncertain") is True + if interrupted or uncertain: + candidate = job.get("candidate_template_id") + committed: dict[str, Any] | None = None + if isinstance(candidate, str): + try: + committed = self.catalog.get(candidate) + path = self.files.path( + candidate, + expected_size=committed["image_bytes"], + ) + self._verify_source( + candidate, path, committed["sha256"] + ) + except Exception: + committed = None + if committed is not None: + self._save_completed_job(job_id, job, committed) + continue + job.update( + status="failed", + phase="引擎重启,任务已中断", + error="面板在云镜像导入期间重启;请重新提交导入", + finished_at=int(time.time()), + ) + self.save_job(job_id, job) + self.jobs[job_id] = job + self._restored = True + + def has_active_jobs(self) -> bool: + with self.lock: + return bool(self.pending) or any( + job.get("status") in {"queued", "running"} + for job in self.jobs.values() + ) + + def close(self) -> None: + self.executor.shutdown(wait=True, cancel_futures=False) diff --git a/engine/virsh_panel/cloud_templates.py b/engine/virsh_panel/cloud_templates.py index 5db595f4..a3655750 100644 --- a/engine/virsh_panel/cloud_templates.py +++ b/engine/virsh_panel/cloud_templates.py @@ -2,6 +2,7 @@ from __future__ import annotations +import errno import fcntl import hashlib import json @@ -103,7 +104,8 @@ def template_volume_name(template_id: str) -> str: def download_verified_image( - url: str, expected_sha256: str, directory: Path + url: str, expected_sha256: str, directory: Path, *, + max_bytes: int = MAX_IMAGE_BYTES, ) -> tuple[Path, int, str]: """Download privately and return path, qcow2 virtual size and actual digest.""" validate_image_url(url) @@ -111,6 +113,9 @@ def download_verified_image( raise TemplateError("SHA-256 校验值格式无效") if expected_sha256 and not SHA256_RE.fullmatch(expected_sha256): raise TemplateError("SHA-256 可留空;填写时必须为 64 位十六进制校验值") + if (isinstance(max_bytes, bool) or not isinstance(max_bytes, int) + or max_bytes < 1 or max_bytes > MAX_IMAGE_BYTES): + raise TemplateError("面板云镜像持久目录空间不足", 507) request = urllib.request.Request(url, headers={"User-Agent": "virsh-panel/1"}) fd, raw_path = tempfile.mkstemp(prefix=".cloud-image-", suffix=".qcow2", dir=directory) image_path = Path(raw_path) @@ -130,13 +135,19 @@ def download_verified_image( if not chunk: break size += len(chunk) - if size > MAX_IMAGE_BYTES: + if size > max_bytes: + if max_bytes < MAX_IMAGE_BYTES: + raise TemplateError( + "面板云镜像持久目录空间不足", 507) raise TemplateError("云镜像超过 4 GiB 限制", 413) output.write(chunk) checksum.update(chunk) output.flush() os.fsync(output.fileno()) except (OSError, urllib.error.URLError) as exc: + if getattr(exc, "errno", None) in {errno.ENOSPC, errno.EDQUOT}: + raise TemplateError( + "面板云镜像持久目录空间不足", 507) from exc raise TemplateError("下载官方云镜像失败,请检查面板服务器网络", 502) from exc actual_sha256 = checksum.hexdigest() if expected_sha256 and actual_sha256 != expected_sha256.lower(): diff --git a/engine/virsh_panel/engine_context.py b/engine/virsh_panel/engine_context.py index c9c9de66..d726fafa 100644 --- a/engine/virsh_panel/engine_context.py +++ b/engine/virsh_panel/engine_context.py @@ -216,13 +216,15 @@ class EngineComponents: store: Any metric_store: Any user_isos: Any | None = None + cloud_templates: Any | None = None @classmethod - def for_host(cls, store, log_store, host_id, data_dir, user_isos=None): + def for_host(cls, store, log_store, host_id, data_dir, user_isos=None, + cloud_templates=None): scoped = HostStore(store, host_id) state = SimpleNamespace(users=ExternalOwnership(scoped)) manager = StableHostManager(scoped, data_dir, host_id=host_id) return cls( state, manager, DatabaseTemplateStore(scoped), scoped, log_store, - user_isos, + user_isos, cloud_templates, ) diff --git a/engine/virsh_panel/internal.py b/engine/virsh_panel/internal.py index a05b54a5..7a11e539 100644 --- a/engine/virsh_panel/internal.py +++ b/engine/virsh_panel/internal.py @@ -27,6 +27,7 @@ from urllib.request import (HTTPRedirectHandler, ProxyHandler, Request as URLReq from .cloud_templates import TemplateError +from .cloud_template_runtime import CloudTemplateRuntime from .engine_context import (EngineComponents, HostStore, LogStore, MainStore, TRUSTED_ACTOR, actor_session) from .engine_security import (EngineAuthError, authenticate, decode_secret, @@ -96,7 +97,8 @@ def master_key(path: Path) -> bytes: class HostRegistry: - def __init__(self, config, store, log_store, *, factory=None, user_isos=None): + def __init__(self, config, store, log_store, *, factory=None, user_isos=None, + cloud_templates=None): self.config = config self.store = store self.log_store = log_store @@ -104,6 +106,9 @@ class HostRegistry: self.user_isos = user_isos or UserISORuntime( store, Path(config["data_dir"])) self._owns_user_isos = user_isos is None + self.cloud_templates = cloud_templates or CloudTemplateRuntime( + store, Path(config["data_dir"])) + self._owns_cloud_templates = cloud_templates is None self.components: dict[str, EngineComponents] = {} self.apps = {} # Constructed by synchronous factories before an event loop may exist. @@ -115,6 +120,7 @@ class HostRegistry: if self.lock is None: self.lock = asyncio.Lock() await asyncio.to_thread(self.user_isos.restore_jobs) + await asyncio.to_thread(self.cloud_templates.restore_jobs) await self.lifespans.__aenter__() for host_id in await asyncio.to_thread(self.store.list, "engine_host"): await self.get(host_id) @@ -123,6 +129,8 @@ class HostRegistry: await self.lifespans.aclose() if self._owns_user_isos: await asyncio.to_thread(self.user_isos.close) + if self._owns_cloud_templates: + await asyncio.to_thread(self.cloud_templates.close) async def get(self, host_id: str): if not HOST_ID.fullmatch(host_id): @@ -137,14 +145,22 @@ class HostRegistry: directory = Path(self.config["data_dir"]) / "hosts" / host_id if self.factory is not None: try: - inspect.signature(self.factory).bind(host_id, self.user_isos) + inspect.signature(self.factory).bind( + host_id, self.user_isos, self.cloud_templates) except TypeError: - components, app = self.factory(host_id) + try: + inspect.signature(self.factory).bind(host_id, self.user_isos) + except TypeError: + components, app = self.factory(host_id) + else: + components, app = self.factory(host_id, self.user_isos) else: - components, app = self.factory(host_id, self.user_isos) + components, app = self.factory( + host_id, self.user_isos, self.cloud_templates) else: components = await asyncio.to_thread(EngineComponents.for_host, - self.store, self.log_store, host_id, directory, self.user_isos) + self.store, self.log_store, host_id, directory, + self.user_isos, self.cloud_templates) settings = Settings(data_dir=directory) app = await asyncio.to_thread(create_app, settings, engine_components=components) self.components[host_id] = components @@ -369,6 +385,15 @@ class EngineApplication: await asyncio.to_thread(self.store.get, "engine", "readiness") await json_response(send, 200, {"status": "ok"}) return + cloud_template_match = re.fullmatch( + r"/internal/v1/(vm-templates(?:/import|/[A-Za-z0-9_-]+|" + r"/import-jobs/[A-Za-z0-9_-]+)?)", path) + if cloud_template_match is not None: + resource = cloud_template_match[1] + self._authorize_route(scope["method"], resource, delegated) + await self._cloud_template_dispatch( + scope["method"], resource, body, send) + return user_iso_match = re.fullmatch( r"/internal/v1/(user-isos(?:/import|/[0-9a-f]{32}|" r"/import-jobs/[A-Za-z0-9_-]+)?)", path) @@ -503,6 +528,49 @@ class EngineApplication: finally: self.active_operations.discard(operation["operation_id"]) + async def _cloud_template_dispatch( + self, method: str, resource: str, body: bytes | None, send) -> None: + runtime = self.registry.cloud_templates + try: + if method == "GET" and resource == "vm-templates": + records = await asyncio.to_thread(runtime.list) + await json_response(send, 200, {"templates": records}) + return + if method == "POST" and resource == "vm-templates/import": + from .app import ImportTemplateBody + from pydantic import ValidationError + try: + value = ImportTemplateBody.model_validate_json(body or b"") + except ValidationError as exc: + raise TemplateError("云镜像导入参数无效", 422) from exc + result = await asyncio.to_thread( + runtime.queue_import, value.url, value.name, value.sha256) + await json_response(send, 202, result) + return + job_match = re.fullmatch( + r"vm-templates/import-jobs/([A-Za-z0-9_-]+)", resource) + if method == "GET" and job_match is not None: + result = await asyncio.to_thread( + runtime.get_job, job_match[1]) + await json_response(send, 200, result) + return + template_match = re.fullmatch( + r"vm-templates/([A-Za-z0-9_-]+)", resource) + if method == "PATCH" and template_match is not None: + from .app import RenameTemplateBody + from pydantic import ValidationError + try: + value = RenameTemplateBody.model_validate_json(body or b"") + except ValidationError as exc: + raise TemplateError("云镜像名称无效", 422) from exc + record = await asyncio.to_thread( + runtime.rename, template_match[1], value.name) + await json_response(send, 200, {"template": record}) + return + raise EngineAuthError("内部资源不存在", 404) + except TemplateError as exc: + await json_response(send, exc.status, {"error": str(exc)}) + async def _user_iso_dispatch(self, method: str, resource: str, body: bytes | None, delegated: dict, send) -> None: runtime = self.registry.user_isos @@ -733,11 +801,20 @@ class EngineApplication: return {key: value for key, value in job.items() if key in allowed} def _long_running_host_job(self) -> bool: - if self.maintenance_tasks or self.registry.user_isos.has_active_jobs(): + cloud_templates = getattr(self.registry, "cloud_templates", None) + if (self.maintenance_tasks + or self.registry.user_isos.has_active_jobs() + or (cloud_templates is not None + and cloud_templates.has_active_jobs())): return True for components in self.registry.components.values(): - for kind in ("import_job", "rescue_import_job", - "reinstall_job", "rescue_job"): + kinds = ["rescue_import_job", "reinstall_job", "rescue_job"] + # A child app without the shared runtime can still run the legacy + # host-scoped importer. Production children use the global runtime; + # stale pre-upgrade import_job rows must not block updates forever. + if components.cloud_templates is None: + kinds.append("import_job") + for kind in kinds: for raw in components.store.list(kind).values(): try: status = json.loads(raw).get("status") diff --git a/engine/virsh_panel/libvirt_service.py b/engine/virsh_panel/libvirt_service.py index 56d0e4ca..adef985d 100644 --- a/engine/virsh_panel/libvirt_service.py +++ b/engine/virsh_panel/libvirt_service.py @@ -36,7 +36,10 @@ GIB = 1024**3 PANEL_NS = "urn:virsh-panel:1" NAME_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,62}\Z") BRIDGE_NAME_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,14}\Z") -TEMPLATE_VOLUME_RE = re.compile(r"panel-template-alma(?:9|10)-[0-9a-f]{8}-[0-9a-f]{16}\.qcow2\Z") +TEMPLATE_VOLUME_RE = re.compile( + r"panel-template-alma(?:9|10)-[0-9a-f]{8}-[0-9a-f]{16}" + r"(?:-[0-9a-f]{16})?\.qcow2\Z" +) RESCUE_VOLUME_RE = re.compile(r"panel-rescue-[0-9a-f]{32}\.iso\Z") _CREATE_LOCK = threading.RLock() REINSTALL_SHUTDOWN_TIMEOUT = 120 @@ -983,7 +986,8 @@ class VMService: def import_template(self, image_path: Path, volume_name: str, virtual_size: int) -> dict[str, Any]: """Place a verified qcow2 image in the configured host disk pool.""" - if not TEMPLATE_VOLUME_RE.fullmatch(volume_name): + if (not isinstance(volume_name, str) + or not TEMPLATE_VOLUME_RE.fullmatch(volume_name)): raise VMError("模板卷名称无效") if not isinstance(virtual_size, int) or not GIB <= virtual_size <= 4096 * GIB: raise VMError("模板虚拟容量无效") @@ -1029,6 +1033,180 @@ class VMService: LOG.warning("Template upload failed for %s: %s", volume_name, exc) raise VMError("模板镜像上传到宿主机失败", 502) from exc + @staticmethod + def _template_replica_identity(pool, volume) -> str: + """Fingerprint the concrete pool/volume backing a ready marker.""" + try: + pool_xml = ET.fromstring(pool.XMLDesc(0)) + pool_type = pool_xml.get("type", "") + pool_target = pool_xml.findtext("./target/path", "") + pool_uuid_method = getattr(pool, "UUIDString", None) + pool_uuid = pool_uuid_method() if callable(pool_uuid_method) else "" + volume_key_method = getattr(volume, "key", None) + volume_key = volume_key_method() if callable(volume_key_method) else "" + volume_path = volume.path() + volume_info = volume.info() + capacity = int(volume_info[1]) + allocation = int(volume_info[2]) + volume_xml = ET.fromstring(volume.XMLDesc(0)) + timestamps = volume_xml.find("./target/timestamps") + mtime = (timestamps.findtext("mtime", "").strip() + if timestamps is not None else "") + ctime = (timestamps.findtext("ctime", "").strip() + if timestamps is not None else "") + btime = (timestamps.findtext("btime", "").strip() + if timestamps is not None else "") + timestamp_re = re.compile(r"[0-9]+(?:\.[0-9]{1,9})?\Z") + if (timestamp_re.fullmatch(mtime) is None + or (timestamp_re.fullmatch(ctime) is None + and timestamp_re.fullmatch(btime) is None) + or (ctime and timestamp_re.fullmatch(ctime) is None) + or (btime and timestamp_re.fullmatch(btime) is None)): + raise ValueError("volume generation timestamps unavailable") + except (AttributeError, IndexError, TypeError, ValueError, + ET.ParseError, libvirt.libvirtError) as exc: + raise VMError("无法核对宿主机模板缓存身份", 502) from exc + fields = tuple(str(value) for value in ( + "virt-cloud-template-replica-v2", pool.name(), pool_uuid, + pool_type, pool_target, volume.name(), volume_key, volume_path, + str(capacity), str(allocation), btime, ctime, mtime, + )) + return hashlib.sha256("\0".join(fields).encode("utf-8")).hexdigest() + + def cache_template(self, image_path: Path, volume_name: str, + virtual_size: int, *, + trusted_replica_volume_name: str | None = None, + trusted_replica_identity: str | None = None) -> dict[str, Any]: + """Idempotently cache one engine-owned cloud image on this host. + + ``volume_name`` is the fresh generation reserved for an upload. A + prior generation is reusable only when both its name and its durable + identity are supplied and the concrete libvirt volume still matches. + If that proof fails, leave the prior generation untouched and upload + into ``volume_name``. A same-named desired volume has no ready proof + and may be an interrupted stream, so replace it only after proving no + domain references it. + """ + if (not isinstance(volume_name, str) + or not TEMPLATE_VOLUME_RE.fullmatch(volume_name)): + raise VMError("模板卷名称无效") + if not isinstance(virtual_size, int) or not GIB <= virtual_size <= 4096 * GIB: + raise VMError("模板虚拟容量无效") + image_path = Path(image_path) + if image_path.is_symlink() or not image_path.is_file(): + raise VMError("云镜像源文件无效") + image_size = image_path.stat().st_size + if image_size <= 0: + raise VMError("云镜像源文件无效") + if ((trusted_replica_volume_name is None) + != (trusted_replica_identity is None)): + raise VMError("模板缓存身份不完整") + if (trusted_replica_volume_name is not None + and (not isinstance(trusted_replica_volume_name, str) + or not TEMPLATE_VOLUME_RE.fullmatch( + trusted_replica_volume_name))): + raise VMError("模板缓存卷名称无效") + if (trusted_replica_identity is not None + and (not isinstance(trusted_replica_identity, str) + or re.fullmatch( + r"[0-9a-f]{64}", trusted_replica_identity) is None)): + raise VMError("模板缓存身份无效") + if trusted_replica_volume_name == volume_name: + raise VMError("模板缓存的新旧卷名称不能相同") + + with _CREATE_LOCK, self._connect() as conn: + pool = self._active_pool(conn, self.storage_pool) + self._refresh_directory_pool(pool) + if (trusted_replica_volume_name is not None + and trusted_replica_volume_name in pool.listVolumes()): + trusted = pool.storageVolLookupByName(trusted_replica_volume_name) + trusted_identity = None + try: + trusted_xml = ET.fromstring(trusted.XMLDesc(0)) + trusted_format = trusted_xml.find("./target/format") + trusted_capacity = int(trusted.info()[1]) + if (trusted_format is not None + and trusted_format.get("type") == "qcow2" + and trusted_capacity == virtual_size): + trusted_identity = self._template_replica_identity( + pool, trusted) + except (ET.ParseError, TypeError, ValueError, VMError, + libvirt.libvirtError): + LOG.warning( + "Could not prove trusted template replica %s; " + "uploading a fresh generation", + trusted_replica_volume_name, + ) + if (trusted_identity is not None + and secrets.compare_digest( + trusted_replica_identity, trusted_identity)): + return { + "volume_name": trusted_replica_volume_name, + "virtual_size": virtual_size, + "image_bytes": image_size, + "already_cached": True, + "replica_identity": trusted_identity, + } + + if volume_name in pool.listVolumes(): + existing = pool.storageVolLookupByName(volume_name) + try: + existing_path = existing.path() + except (AttributeError, libvirt.libvirtError) as exc: + raise VMError("无法核对宿主机模板卷路径", 502) from exc + if self._other_domain_uses_volume( + conn, "", pool.name(), volume_name, existing_path): + raise VMError( + "宿主机模板缓存未能确认完整且正被虚拟机引用", 409) + existing.delete(0) + self._refresh_directory_pool(pool) + + if image_size > pool.info()[3]: + raise VMError("宿主机存储池空间不足,无法缓存模板", 409) + volume = pool.createXML(self._volume_xml_bytes( + volume_name, virtual_size, "qcow2"), 0) + if volume is None: + raise VMError("宿主机未能创建模板缓存卷", 502) + try: + def read_chunks() -> Iterator[bytes]: + with image_path.open("rb") as image: + while chunk := image.read(1024 * 1024): + yield chunk + + self._upload_volume(conn, volume, read_chunks(), image_size) + self._refresh_directory_pool(pool) + uploaded = pool.storageVolLookupByName(volume_name) + uploaded_xml = ET.fromstring(uploaded.XMLDesc(0)) + uploaded_format = uploaded_xml.find("./target/format") + if (uploaded_format is None + or uploaded_format.get("type") != "qcow2" + or int(uploaded.info()[1]) != virtual_size): + raise VMError("宿主机模板缓存的格式或容量与源镜像不一致", 502) + replica_identity = self._template_replica_identity(pool, uploaded) + return { + "volume_name": volume_name, + "virtual_size": virtual_size, + "image_bytes": image_size, + "already_cached": False, + "replica_identity": replica_identity, + } + except Exception as exc: + try: + volume.delete(0) + except Exception: + LOG.exception( + "Template cache upload failed and volume %s remains", + volume_name, + ) + raise VMError( + f"模板缓存失败且宿主机卷 {volume_name} 未能清理,请人工检查", + 502, + ) from exc + if isinstance(exc, VMError): + raise + LOG.warning("Template cache upload failed for %s: %s", volume_name, exc) + raise VMError("云镜像缓存到宿主机失败", 502) from exc + @staticmethod def _check_user_iso_volume_name(volume_name: str) -> None: if not isinstance(volume_name, str) or not USER_ISO_VOLUME_RE.fullmatch(volume_name): diff --git a/web/app/Controllers/CloudAdminController.php b/web/app/Controllers/CloudAdminController.php index f6422f1c..b8198651 100644 --- a/web/app/Controllers/CloudAdminController.php +++ b/web/app/Controllers/CloudAdminController.php @@ -164,26 +164,44 @@ final class CloudAdminController $method = $request->getMethod(); $actor = $this->actor($request, $method !== 'GET'); $body = in_array($method, ['POST', 'PATCH', 'PUT'], true) ? CloudInput::object($request) : null; + $kind = $args['kind']; + if ($kind === 'cloud') { + if ($body !== null) { + unset($body['host_id']); + if ($method === 'POST' && isset($body['sha256']) + && is_string($body['sha256']) && trim($body['sha256']) === '') { + unset($body['sha256']); + } + } + $path = '/internal/v1/vm-templates'; + if (isset($args['id'])) { + $path .= '/' . rawurlencode($args['id']); + } elseif ($method === 'POST') { + $path .= '/import'; + } + if (isset($args['job_id'])) { + $path .= '/import-jobs/' . rawurlencode($args['job_id']); + } + return $response->withJson( + (new CloudEngineClient())->request($method, $path, $actor, $body) + ); + } + + // Rescue ISOs remain deliberately scoped to one host and its separate + // rescue directory. $query = $request->getQueryParams(); - $hostId = filter_var($query['host_id'] ?? $body['host_id'] ?? null, FILTER_VALIDATE_INT); + $hostId = filter_var($query['host_id'] ?? $body['host_id'] ?? null, + FILTER_VALIDATE_INT); if (!$hostId) { throw new CloudError('请选择宿主机'); } - $kind = $args['kind']; $this->host($hostId); if ($body !== null) { unset($body['host_id']); - if ($kind === 'cloud' && $method === 'POST' - && isset($body['sha256']) && is_string($body['sha256']) - && trim($body['sha256']) === '') { - unset($body['sha256']); - } } - $resource = $kind === 'cloud' ? 'vm-templates' : 'rescue-templates'; + $resource = 'rescue-templates'; if (isset($args['id'])) { $resource .= '/' . rawurlencode($args['id']); - } elseif ($kind === 'cloud' && $method === 'POST') { - $resource .= '/import'; } if (isset($args['job_id'])) { $resource .= '/import-jobs/' . rawurlencode($args['job_id']); diff --git a/web/app/Controllers/CloudApiController.php b/web/app/Controllers/CloudApiController.php index af86a531..0e544014 100644 --- a/web/app/Controllers/CloudApiController.php +++ b/web/app/Controllers/CloudApiController.php @@ -877,32 +877,43 @@ final class CloudApiController { $actor = $this->actor($request); $query = $request->getQueryParams(); + $engine = new CloudEngineClient(); + // Cloud images are a panel-wide catalog. Host-specific replication is + // owned by the private engine and must not make the browser reload or + // replace the catalog whenever a customer changes region or plan. + $catalog = $engine->request('GET', '/internal/v1/vm-templates', $actor); + $templatePayload = [ + 'templates' => is_array($catalog['templates'] ?? null) + ? $catalog['templates'] : [], + ]; + foreach (['catalog_version', 'replicas', 'replica_states'] as $field) { + if (isset($catalog[$field]) + && (is_array($catalog[$field]) || is_string($catalog[$field]))) { + $templatePayload[$field] = $catalog[$field]; + } + } if (isset($query['vm_id'])) { $hostId = (int) CloudAccess::instance($query['vm_id'], $actor)->host_id; - } else { + } elseif (isset($query['plan_id'])) { + // Compatibility for a browser opened before global catalogs were + // deployed. New clients request /resources once without a plan. $plan = Shop::where('id', $query['plan_id'] ?? 0)->where('status', 1)->first(); $hostId = (int) ($plan?->host_id ?? 0); if (!CloudHost::where('id', $hostId)->where('enabled', 1)->exists()) { throw new CloudError('套餐宿主机不可用', 404); } + } else { + return $this->json($response, $templatePayload); } - $engine = new CloudEngineClient(); if (!isset($query['vm_id'])) { - // Deployment needs only the cloud template catalog. Avoid the - // expensive resources, rescue ISO and personal ISO calls here. - $templates = $engine->host($hostId, 'GET', 'vm-templates', $actor); - return $this->json($response, [ - 'host_id' => $hostId, - 'templates' => $templates['templates'] ?? [], - ]); + return $this->json($response, $templatePayload + ['host_id' => $hostId]); } $value = $engine->host($hostId, 'GET', 'resources', $actor); - $templates = $engine->host($hostId, 'GET', 'vm-templates', $actor); $rescue = $engine->host($hostId, 'GET', 'rescue-templates', $actor); // The configured root ISO pool can contain administrator or legacy // files and is never exposed through this user API. unset($value['isos'], $value['iso_images']); - $value['templates'] = $templates['templates'] ?? []; + $value = array_merge($value, $templatePayload); $value['rescue_templates'] = $rescue['templates'] ?? []; $value['host_id'] = $hostId; return $this->json($response, $value); diff --git a/web/public/cloud/admin.js b/web/public/cloud/admin.js index 545136bb..f2efcc8c 100644 --- a/web/public/cloud/admin.js +++ b/web/public/cloud/admin.js @@ -89,33 +89,45 @@ async function saveHost(event) { finally { delete body.private_key; delete body.private_key_passphrase; byId('host-private-key').value = ''; byId('host-passphrase').value = ''; state.busy = false; byId('host-save').disabled = !canManage; } } function catalogPath() { return root.dataset.cloudAdmin === 'rescue' ? '/api/v1/admin/rescue-templates' : '/api/v1/admin/cloud-templates'; } +function isRescueCatalog() { return root.dataset.cloudAdmin === 'rescue'; } async function loadCatalog() { - const hostId = byId('catalog-host').value; state.selectedHost = hostId; const epoch = ++state.epoch; const list = byId('catalog-list'); list.replaceChildren(); if (!hostId) return list.append(element('p', 'text-muted', '请先添加宿主机。')); - notice('正在读取所选宿主机的模版…'); - try { const entries = collection(await api(`${catalogPath()}?host_id=${encodeURIComponent(hostId)}`), 'templates'); if (epoch !== state.epoch) return; + const rescue = isRescueCatalog(); + const hostId = rescue ? byId('catalog-host')?.value || '' : ''; + state.selectedHost = hostId; + const epoch = ++state.epoch; + const list = byId('catalog-list'); + list.replaceChildren(); + if (rescue && !hostId) return list.append(element('p', 'text-muted', '请先添加宿主机。')); + notice(rescue ? '正在读取所选宿主机的模版…' : '正在读取全局云镜像目录…'); + const url = rescue ? `${catalogPath()}?host_id=${encodeURIComponent(hostId)}` : catalogPath(); + try { const entries = collection(await api(url), 'templates'); if (epoch !== state.epoch) return; if (!entries.length) list.append(element('p', 'text-muted', '暂无模版。')); - for (const entry of entries) { const row = element('form', 'cloud-admin-row'); const name = element('input', 'form-control'); name.value = entry.name || ''; name.maxLength = 64; name.required = true; name.setAttribute('aria-label', '模版名称'); name.disabled = !canManage; const save = element('button', 'btn btn-outline-primary', '保存名称'); save.type = 'submit'; save.disabled = !canManage; const remove = element('button', 'btn btn-outline-danger', '删除'); remove.type = 'button'; remove.disabled = !canManage; row.append(name, save, remove, element('small', 'cloud-admin-meta', `${entry.os_family || '镜像'} ${entry.os_version || ''} · ${entry.size_bytes ? (entry.size_bytes / 1024 ** 3).toFixed(2) + ' GiB' : entry.id}`)); - if (root.dataset.cloudAdmin === 'rescue') { name.readOnly = true; save.hidden = true; save.disabled = true; } - row.addEventListener('submit', async event => { event.preventDefault(); if (root.dataset.cloudAdmin === 'rescue') return; save.disabled = true; try { await api(`${catalogPath()}/${encodeURIComponent(entry.id)}?host_id=${encodeURIComponent(hostId)}`, { method: 'PATCH', body: { host_id: Number(hostId), name: name.value.trim() } }); notice('名称已保存。'); } catch (error) { notice(error.message, true); } finally { save.disabled = !canManage; } }); - remove.addEventListener('click', async () => { if (!confirm(`删除所选宿主机上的模版“${entry.name}”?正在使用的模版可能不能删除。`)) return; remove.disabled = true; try { await api(`${catalogPath()}/${encodeURIComponent(entry.id)}?host_id=${encodeURIComponent(hostId)}`, { method: 'DELETE' }); await loadCatalog(); } catch (error) { notice(error.message, true); remove.disabled = !canManage; } }); list.append(row); - } notice(`已加载 ${entries.length} 个模版。`); + for (const entry of entries) { const row = element('form', 'cloud-admin-row'); const name = element('input', 'form-control'); name.value = entry.name || ''; name.maxLength = 64; name.required = true; name.setAttribute('aria-label', '模版名称'); name.disabled = !canManage; const save = element('button', 'btn btn-outline-primary', '保存名称'); save.type = 'submit'; save.disabled = !canManage; const remove = element('button', 'btn btn-outline-danger', '删除'); remove.type = 'button'; remove.disabled = !canManage; const imageBytes = Number(entry.image_bytes || entry.size_bytes || 0); + row.append(name, save); + if (rescue) row.append(remove); + row.append(element('small', 'cloud-admin-meta', `${entry.os_family || '镜像'} ${entry.os_version || ''} · ${imageBytes > 0 ? (imageBytes / 1024 ** 3).toFixed(2) + ' GiB' : entry.id}`)); + if (rescue) { name.readOnly = true; save.hidden = true; save.disabled = true; } + row.addEventListener('submit', async event => { event.preventDefault(); if (rescue) return; save.disabled = true; try { await api(`${catalogPath()}/${encodeURIComponent(entry.id)}`, { method: 'PATCH', body: { name: name.value.trim() } }); notice('名称已保存。'); } catch (error) { notice(error.message, true); } finally { save.disabled = !canManage; } }); + remove.addEventListener('click', async () => { if (!rescue || !confirm(`删除所选宿主机上的模版“${entry.name}”?正在使用的模版可能不能删除。`)) return; remove.disabled = true; try { await api(`${catalogPath()}/${encodeURIComponent(entry.id)}?host_id=${encodeURIComponent(hostId)}`, { method: 'DELETE' }); await loadCatalog(); } catch (error) { notice(error.message, true); remove.disabled = !canManage; } }); list.append(row); + } notice(rescue ? `已加载 ${entries.length} 个模版。` : `已加载 ${entries.length} 个全局云镜像。`); } catch (error) { notice(error.message, true); } } async function importProgress(result) { if (!result?.job_id) return; - const url = result.status_url || `${catalogPath()}/import-jobs/${encodeURIComponent(result.job_id)}?host_id=${encodeURIComponent(state.selectedHost)}`; - if (!url.startsWith('/api/v1/')) throw new Error('导入状态地址无效。'); + const suffix = isRescueCatalog() ? `?host_id=${encodeURIComponent(state.selectedHost)}` : ''; + const fallback = `${catalogPath()}/import-jobs/${encodeURIComponent(result.job_id)}${suffix}`; + const url = String(result.status_url || '').startsWith('/api/v1/') ? result.status_url : fallback; for (let i = 0; i < 1800; i++) { const job = await api(url); notice(job.phase || `导入状态:${job.status}`); if (['succeeded', 'completed', 'done'].includes(job.status)) return; if (['failed', 'uncertain', 'cancelled'].includes(job.status)) throw new Error(job.error || '导入未完成,请核查宿主机状态后再操作。'); await new Promise(resolve => setTimeout(resolve, 2000)); } throw new Error('导入仍在进行,请稍后刷新查看。不要重复提交。'); } async function importTemplate(event) { - event.preventDefault(); if (state.busy) return; const hostId = byId('catalog-host').value; if (!hostId) return notice('请选择宿主机。', true); + event.preventDefault(); if (state.busy) return; const sha256 = byId('template-sha256').value.trim().toLowerCase(); if (sha256 && !/^[a-f0-9]{64}$/.test(sha256)) return notice('SHA-256 可留空;如果填写,请输入完整的 64 位十六进制校验值。', true); - const body = { host_id: Number(hostId), name: byId('template-name').value.trim(), url: byId('template-url').value.trim() }; + const body = { name: byId('template-name').value.trim(), url: byId('template-url').value.trim() }; if (sha256) body.sha256 = sha256; - const button = byId('template-import-button'); state.busy = true; button.disabled = true; byId('catalog-host').disabled = true; + const button = byId('template-import-button'); state.busy = true; button.disabled = true; try { const result = await api('/api/v1/admin/cloud-templates/import', { method: 'POST', body }); await importProgress(result); await loadCatalog(); notice('镜像导入已完成。'); } - catch (error) { notice(error.message, true); } finally { state.busy = false; button.disabled = !canManage; byId('catalog-host').disabled = false; } + catch (error) { notice(error.message, true); } finally { state.busy = false; button.disabled = !canManage; } } async function uploadRescue(event) { event.preventDefault(); if (state.busy) return; const hostId = byId('catalog-host').value; const file = byId('rescue-file').files?.[0]; if (!hostId || !file) return notice('请选择宿主机和 ISO。', true); if (!file.name.toLowerCase().endsWith('.iso') || file.size > 8 * 1024 ** 3 || file.size < 32768) return notice('请选择小于 8 GiB 的有效 ISO 文件。', true); @@ -150,7 +162,8 @@ async function initialize() { if (!root) return; try { const session = await api('/api/v1/session'); state.csrf = session.csrf_token || state.csrf; if (!session.admin) throw new Error('需要管理员权限。'); if (!state.csrf) throw new Error('会话令牌缺失,请重新登录。'); if (root.dataset.cloudAdmin === 'hosts') { await Promise.all([loadHosts(), loadUserIsoSettings()]); renderHosts(); fillHost(null); byId('host-new').addEventListener('click', () => fillHost(null)); byId('host-scan').addEventListener('click', scanHost); byId('host-form').addEventListener('submit', saveHost); byId('user-iso-settings-form').addEventListener('submit', saveUserIsoSettings); for (const id of ['host-address', 'host-port']) byId(id).addEventListener('input', resetScan); if (!canManage) { byId('host-form').querySelectorAll('input,textarea,button').forEach(input => { input.disabled = true; }); byId('user-iso-settings-form').querySelectorAll('input,button').forEach(input => { input.disabled = true; }); } notice(canManage ? '请选择宿主机编辑,或添加新的宿主机。' : '当前管理员仅可查看配置。'); } - else if (['templates', 'rescue'].includes(root.dataset.cloudAdmin)) { await loadHosts(); const select = byId('catalog-host'); select.replaceChildren(); for (const host of state.hosts) { const option = element('option', '', `${host.name} · ${host.region || '地区未设置'}`); option.value = String(host.id); select.append(option); } select.addEventListener('change', loadCatalog); if (byId('template-import')) byId('template-import').addEventListener('submit', importTemplate); if (byId('rescue-upload')) byId('rescue-upload').addEventListener('submit', uploadRescue); await loadCatalog(); } + else if (root.dataset.cloudAdmin === 'templates') { byId('template-import')?.addEventListener('submit', importTemplate); await loadCatalog(); } + else if (root.dataset.cloudAdmin === 'rescue') { await loadHosts(); const select = byId('catalog-host'); select.replaceChildren(); for (const host of state.hosts) { const option = element('option', '', `${host.name} · ${host.region || '地区未设置'}`); option.value = String(host.id); select.append(option); } select.addEventListener('change', loadCatalog); byId('rescue-upload')?.addEventListener('submit', uploadRescue); await loadCatalog(); } else { byId('admin-records-refresh').addEventListener('click', records); await records(); } } catch (error) { notice(error.message, true); } } diff --git a/web/public/cloud/runtime.js b/web/public/cloud/runtime.js index 4f7f520f..5be561c9 100644 --- a/web/public/cloud/runtime.js +++ b/web/public/cloud/runtime.js @@ -1,7 +1,7 @@ // PHP owns authentication, authorisation, ownership and purchasing. No engine // address, token, host credential or unrestricted create API exists in this UI. const byId = (id) => document.getElementById(id); -const state = { csrfToken: '', userId: '', username: '', admin: false, vms: [], plans: [], sshKeys: [], templates: [], resources: null, selectedRegionHostId: '', invoices: [], invoicePage: 1, invoicePagination: { page: 1, per_page: 10, total: 0, total_pages: 1, has_previous: false, has_next: false, from: 0, to: 0 }, invoiceLoading: false, invoiceLoadPromise: null, invoicePaymentMethods: [], invoiceBalance: '0.00', selectedInvoice: null, selectedInvoicePaymentMethod: '', invoiceJobProgress: new Map(), invoicePollTimers: new Map(), view: 'list', selectedVmId: null, detailTab: 'operations', metricsRange: '24h', metricsPayload: null, metricsRequest: null, metricsPollTimer: null, metricsResizeTimer: null, trafficDailyPayload: null, trafficDailyVmId: '', trafficDailyRequest: null, trafficDailyResizeTimer: null, trafficDailyLoadedAt: 0, credentialPassword: '', credentialVisible: false, credentialTimer: null, rfb: null, consoleToken: 0, loadingVMs: false, selectedKeys: { create: '', reinstall: '' }, formEpoch: 0, resourceController: null, mutationBusy: false, provisionIdempotency: '', provisionFingerprint: '', rescueActive: false, rescueStaged: false }; +const state = { csrfToken: '', userId: '', username: '', admin: false, vms: [], plans: [], sshKeys: [], templates: [], resources: null, selectedRegionHostId: '', invoices: [], invoicePage: 1, invoicePagination: { page: 1, per_page: 10, total: 0, total_pages: 1, has_previous: false, has_next: false, from: 0, to: 0 }, invoiceLoading: false, invoiceLoadPromise: null, invoicePaymentMethods: [], invoiceBalance: '0.00', selectedInvoice: null, selectedInvoicePaymentMethod: '', invoiceJobProgress: new Map(), invoicePollTimers: new Map(), view: 'list', selectedVmId: null, detailTab: 'operations', metricsRange: '24h', metricsPayload: null, metricsRequest: null, metricsPollTimer: null, metricsResizeTimer: null, trafficDailyPayload: null, trafficDailyVmId: '', trafficDailyRequest: null, trafficDailyResizeTimer: null, trafficDailyLoadedAt: 0, credentialPassword: '', credentialVisible: false, credentialTimer: null, rfb: null, consoleToken: 0, loadingVMs: false, selectedKeys: { create: '', reinstall: '' }, resourceController: null, mutationBusy: false, provisionIdempotency: '', provisionFingerprint: '', rescueActive: false, rescueStaged: false }; // Keep the historical storage key so an interrupted deployment made just // before an online update can still reconcile to the same opening invoice. const PROVISION_INVOICE_STORAGE_PREFIX = 'virt-cloud:pending-order:'; @@ -361,7 +361,7 @@ function updateCreateAvailability() { const plan = state.plans.find(item => String(item.id) === byId('vm-plan')?.value); const validHost = plan && String(plan.host_id) === state.selectedRegionHostId; const validMedia = Boolean(byId('vm-template')?.value); - if (byId('create-submit')) byId('create-submit').disabled = state.mutationBusy || !validHost || !state.resources || !validMedia; + if (byId('create-submit')) byId('create-submit').disabled = state.mutationBusy || !validHost || !validMedia; } function strongPassword(value) { return !value || (value.length >= 12 && new TextEncoder().encode(value).length <= 256 && /[A-Z]/.test(value) && /[a-z]/.test(value) && /[0-9]/.test(value) && /[^A-Za-z0-9\s]/.test(value) && !/[\r\n\0]/.test(value)); } function formatUsd(value) { @@ -461,13 +461,12 @@ async function selectRegion(hostId) { if (!saleRegions().some(region => region.hostId === normalized)) return; if (state.selectedRegionHostId === normalized && byId('vm-plan').value) return; state.selectedRegionHostId = normalized; - state.formEpoch += 1; byId('vm-plan').value = ''; renderRegionPicker(); renderPlanPicker(); setDeploymentStepEnabled('vm-plan-step', true); - resetInstallChoices(); showError('create-error', ''); + updateCreateAvailability(); } async function selectPlan(planId) { if (state.mutationBusy) return; @@ -476,30 +475,29 @@ async function selectPlan(planId) { byId('vm-plan').value = String(plan.id); renderPlanPicker(); showError('create-error', ''); - await loadPlanResources(); + updateCreateAvailability(); } async function openCreate() { state.view = 'create'; state.selectedVmId = null; showError('create-error', ''); - state.formEpoch += 1; state.selectedRegionHostId = ''; byId('vm-region-host').value = ''; byId('vm-plan').value = ''; + state.selectedRegionHostId = ''; byId('vm-region-host').value = ''; byId('vm-plan').value = ''; setDeploymentStepEnabled('vm-plan-step', false); resetInstallChoices(); - try { - const [plans] = await Promise.all([api('/api/v1/plans'), loadKeys()]); - state.plans = items(plans); - renderRegionPicker(); - } catch (error) { showError('create-error', error.message); } -} -async function loadPlanResources() { - const planId = byId('vm-plan').value; - const plan = state.plans.find(item => String(item.id) === planId && String(item.host_id) === state.selectedRegionHostId); - const epoch = ++state.formEpoch; - resetInstallChoices(); - state.resourceController = new AbortController(); - if (!plan) return; + const results = await Promise.allSettled([ + api('/api/v1/plans').then(plans => { state.plans = items(plans); renderRegionPicker(); }), + loadKeys(), + loadGlobalTemplates(), + ]); + const failure = results.find(result => result.status === 'rejected'); + if (failure) showError('create-error', failure.reason?.message || '部署信息加载失败,请刷新页面重试。'); +} +async function loadGlobalTemplates() { + state.resourceController?.abort(); + const controller = new AbortController(); + state.resourceController = controller; renderTemplateCatalogStatus('vm-template', '正在加载系统镜像…', true); setDeploymentStepEnabled('vm-source-step', true); try { - const resources = await api(`/api/v1/resources?plan_id=${encodeURIComponent(planId)}`, { signal: state.resourceController.signal }); - if (epoch !== state.formEpoch || byId('vm-plan').value !== planId || String(plan.host_id) !== state.selectedRegionHostId) return; + const resources = await api('/api/v1/resources', { signal: controller.signal }); + if (controller !== state.resourceController) return; state.resources = resources; state.templates = items(resources.templates || resources.cloud_templates); renderTemplateCatalog('vm-template'); @@ -510,10 +508,15 @@ async function loadPlanResources() { setDeploymentStepEnabled('vm-config-step', true); updateCreateAvailability(); } catch (error) { - if (epoch === state.formEpoch && error.name !== 'AbortError') { - renderTemplateCatalogStatus('vm-template', '系统镜像加载失败,请重新选择实例规格。'); + if (controller === state.resourceController && error.name !== 'AbortError') { + renderTemplateCatalogStatus('vm-template', '系统镜像加载失败,请刷新页面后重试。'); showError('create-error', error.message); } + throw error; + } finally { + if (controller === state.resourceController) { + state.resourceController = null; + } } } function provisionInvoiceStorageKey() { return `${PROVISION_INVOICE_STORAGE_PREFIX}${state.userId || 'unknown'}`; } @@ -573,7 +576,7 @@ async function buy(event) { const password = byId('vm-root-password').value; if (!strongPassword(password)) return showError('create-error', '密码需至少 12 个字符,包含大写、小写、数字和符号。'); if (!byId('vm-template').value) return showError('create-error', '请选择系统镜像版本。'); - if (!byId('vm-plan').value || !state.resources) return showError('create-error', '请先选择可用套餐。'); + if (!byId('vm-plan').value) return showError('create-error', '请先选择可用套餐。'); const selectedPlan = state.plans.find(plan => String(plan.id) === byId('vm-plan').value); if (!selectedPlan || String(selectedPlan.host_id) !== state.selectedRegionHostId) return showError('create-error', '地区与实例规格不匹配,请重新选择。'); const purchaseSummary = selectedPlan ? `${formatUsd(selectedPlan.price)},${planBillingLabel(selectedPlan)}` : '所选套餐'; @@ -1989,7 +1992,7 @@ export { CLOUD_PATHS, VM_DETAIL_TABS, api, bindPageCleanup, buy, byId, clearCredential, clearVmMetrics, clearVmTrafficDaily, closeConsole, copyText, credential, drawVmTrafficDaily, hydrateVmDetailBootstrap, initializeSession, invoicePagePath, - loadInvoiceDetail, loadInvoices, loadKeys, loadPlanResources, loadVmMetrics, loadVmTrafficDaily, + loadGlobalTemplates, loadInvoiceDetail, loadInvoices, loadKeys, loadVmMetrics, loadVmTrafficDaily, loadVm, loadVmStatus, loadVms, navigate, openCreate, openReinstall, openRescue, reinstall, renderVMs, renderVmDetail, renderVmMetrics, rescueAction, selectVmByApiId, selectedVm, scheduleInvoicePoll, setVmDetailTab, showError, state, stopInvoicePoll, stopVmMetrics, diff --git a/web/resources/views/venix/admin/cloud/templates.tpl b/web/resources/views/venix/admin/cloud/templates.tpl index c9dbedf4..972fc2a1 100644 --- a/web/resources/views/venix/admin/cloud/templates.tpl +++ b/web/resources/views/venix/admin/cloud/templates.tpl @@ -1,5 +1,5 @@ {assign var=admin_title value='云镜像模版'}{assign var=admin_active value='cloud_templates'}{include file='admin/static_header.tpl'} -云镜像模版模版按宿主机隔离,普通用户只能选择已导入的镜像。仅导入可信、支持 cloud-init 的 qcow2 镜像。正在加载…宿主机 +云镜像模版这里维护全局云镜像目录,普通用户在所有可售地区看到同一份镜像列表。宿主机副本由引擎按需准备;仅导入可信、支持 cloud-init 的 qcow2 镜像。升级前按宿主机导入的旧镜像不会自动进入全局目录,请重新导入一次。正在加载… 导入云镜像模版名称可信 HTTPS 镜像地址SHA-256(可选)留空时面板会计算并保存实际 SHA-256,不与外部校验值比对;填写后则会在导入前核对。下载并导入已保存的模版 {include file='admin/static_footer.tpl'} diff --git a/web/resources/views/venix/layout_assets.tpl b/web/resources/views/venix/layout_assets.tpl index 655d48cc..7cfbbe18 100644 --- a/web/resources/views/venix/layout_assets.tpl +++ b/web/resources/views/venix/layout_assets.tpl @@ -1 +1 @@ -{assign var=venix_asset_version value='2026092807' scope='global'} +{assign var=venix_asset_version value='2026092809' scope='global'} diff --git a/web/tests/CloudUserISOContractTest.php b/web/tests/CloudUserISOContractTest.php index 469f5cd8..247167f0 100644 --- a/web/tests/CloudUserISOContractTest.php +++ b/web/tests/CloudUserISOContractTest.php @@ -76,10 +76,10 @@ $assert(str_contains($orders, "if (\$source !== 'cloud')") $assert(str_contains($controller, "\$available->pluck('host_id')") && str_contains($controller, "CloudHost::whereIn('id', \$hostIds)") - && str_contains($controller, 'Deployment needs only the cloud template catalog') - && str_contains($controller, "\$engine->host(\$hostId, 'GET', 'vm-templates', \$actor)") - && str_contains($controller, "'templates' => \$templates['templates'] ?? []"), - 'Plan listing must preload hosts and deployment resources must return a lightweight cloud-template catalog.'); + && str_contains($controller, "'/internal/v1/vm-templates'") + && str_contains($controller, "return \$this->json(\$response, \$templatePayload);") + && !str_contains($controller, "\$engine->host(\$hostId, 'GET', 'vm-templates', \$actor)"), + 'Plan listing must preload hosts and deployment resources must return the global cloud-template catalog.'); $assert(str_contains($isoClient, 'epoch !== pageState.loadEpoch') && str_contains($isoClient, "error.name !== 'AbortError'") diff --git a/web/tests/VirtualizationAssetsTest.php b/web/tests/VirtualizationAssetsTest.php index b3301517..ee290154 100644 --- a/web/tests/VirtualizationAssetsTest.php +++ b/web/tests/VirtualizationAssetsTest.php @@ -54,6 +54,7 @@ $routes = file_get_contents($root . '/config/routes.php'); $cloudPageController = file_get_contents($root . '/app/Controllers/CloudPageController.php'); $shopFormScript = file_get_contents($views . '/admin/shop/form_js.tpl'); $templateAdmin = file_get_contents($views . '/admin/cloud/templates.tpl'); +$rescueAdmin = file_get_contents($views . '/admin/cloud/rescue.tpl'); $adminScript = file_get_contents($public . '/admin.js'); $cloudAdminController = file_get_contents($root . '/app/Controllers/CloudAdminController.php'); $assert(str_contains($cloudShell, "file='user/panel_sidebar.tpl'") @@ -141,7 +142,22 @@ $resetEnd = $resetStart === false ? false : strpos($runtimeDeployScript, 'function renderRegionPicker()', $resetStart); $resetInstall = $resetStart === false || $resetEnd === false ? '' : substr($runtimeDeployScript, $resetStart, $resetEnd - $resetStart); -$resourcesStart = strpos($runtimeDeployScript, 'async function loadPlanResources()'); +$selectRegionStart = strpos($runtimeDeployScript, 'async function selectRegion('); +$selectRegionEnd = $selectRegionStart === false ? false + : strpos($runtimeDeployScript, 'async function selectPlan(', $selectRegionStart); +$selectRegion = $selectRegionStart === false || $selectRegionEnd === false ? '' + : substr($runtimeDeployScript, $selectRegionStart, $selectRegionEnd - $selectRegionStart); +$selectPlanStart = strpos($runtimeDeployScript, 'async function selectPlan('); +$selectPlanEnd = $selectPlanStart === false ? false + : strpos($runtimeDeployScript, 'async function openCreate()', $selectPlanStart); +$selectPlan = $selectPlanStart === false || $selectPlanEnd === false ? '' + : substr($runtimeDeployScript, $selectPlanStart, $selectPlanEnd - $selectPlanStart); +$openCreateStart = strpos($runtimeDeployScript, 'async function openCreate()'); +$openCreateEnd = $openCreateStart === false ? false + : strpos($runtimeDeployScript, 'async function loadGlobalTemplates()', $openCreateStart); +$openCreate = $openCreateStart === false || $openCreateEnd === false ? '' + : substr($runtimeDeployScript, $openCreateStart, $openCreateEnd - $openCreateStart); +$resourcesStart = strpos($runtimeDeployScript, 'async function loadGlobalTemplates()'); $resourcesEnd = $resourcesStart === false ? false : strpos($runtimeDeployScript, 'function provisionInvoiceStorageKey()', $resourcesStart); $resourceLoader = $resourcesStart === false || $resourcesEnd === false ? '' @@ -154,7 +170,7 @@ $templateStatus = $statusStart === false || $statusEnd === false ? '' $loadingPosition = strpos($resourceLoader, "renderTemplateCatalogStatus('vm-template', '正在加载系统镜像…', true)"); $requestPosition = strpos($resourceLoader, - 'const resources = await api(`/api/v1/resources?plan_id='); + "const resources = await api('/api/v1/resources'"); $templatesPosition = strpos($resourceLoader, 'state.templates = items(resources.templates || resources.cloud_templates)'); $catalogPosition = strpos($resourceLoader, "renderTemplateCatalog('vm-template')"); @@ -167,14 +183,40 @@ $assert($resetInstall !== '' && $loadingPosition < $requestPosition && $requestPosition < $templatesPosition && $templatesPosition < $catalogPosition + && !str_contains($resourceLoader, 'resetInstallChoices()') && str_contains($templateStatus, 'container.setAttribute("aria-busy", String(busy))') && str_contains($templateStatus, '"template-catalog-status"') && !str_contains($templateStatus, 'template-catalog-empty') - && str_contains($resourceLoader, '系统镜像加载失败,请重新选择实例规格。') - && str_contains($resourceLoader, - "epoch === state.formEpoch && error.name !== 'AbortError'") + && str_contains($resourceLoader, '系统镜像加载失败,请刷新页面后重试。') + && str_contains($resourceLoader, 'controller === state.resourceController') + && str_contains($resourceLoader, "error.name !== 'AbortError'") && str_contains($runtimeDeployScript, '暂无可用云镜像,请联系管理员导入模板。'), - 'Plan changes must distinguish the cloud-image loading, failure and true-empty states.'); + 'Global cloud-image loading must distinguish loading, failure and true-empty states.'); +$assert(str_contains($openCreate, 'Promise.allSettled([') + && str_contains($openCreate, "api('/api/v1/plans')") + && str_contains($openCreate, 'loadKeys()') + && str_contains($openCreate, 'loadGlobalTemplates()') + && substr_count($runtimeDeployScript, "api('/api/v1/resources'") === 1, + 'Deployment must load plans, keys and the global image directory in parallel exactly once.'); +$assert($selectRegion !== '' && $selectPlan !== '' + && !str_contains($selectRegion, 'resetInstallChoices()') + && !str_contains($selectRegion, 'renderTemplateCatalog(') + && !str_contains($selectRegion, 'loadGlobalTemplates(') + && !str_contains($selectRegion, "byId('vm-template').value") + && !str_contains($selectPlan, 'resetInstallChoices()') + && !str_contains($selectPlan, 'renderTemplateCatalog(') + && !str_contains($selectPlan, 'loadGlobalTemplates(') + && !str_contains($selectPlan, "byId('vm-template').value"), + 'Changing regions or plans must preserve the global image catalog and selected image.'); +$availabilityStart = strpos($runtimeDeployScript, 'function updateCreateAvailability()'); +$availabilityEnd = $availabilityStart === false ? false + : strpos($runtimeDeployScript, 'function strongPassword(', $availabilityStart); +$availability = $availabilityStart === false || $availabilityEnd === false ? '' + : substr($runtimeDeployScript, $availabilityStart, $availabilityEnd - $availabilityStart); +$assert(str_contains($availability, 'const validHost = plan') + && str_contains($availability, "const validMedia = Boolean(byId('vm-template')?.value)") + && !str_contains($availability, 'state.resources'), + 'Deployment submission must require a matching region/plan and selected global image.'); $assert(str_contains($baseStyles, '.deployment-step') && str_contains($baseStyles, '.region-option') && str_contains($baseStyles, '.plan-option') @@ -184,7 +226,30 @@ $assert(!str_contains($cloudTemplate, '购买并开通') && !str_contains($share $assert(str_contains($shopFormScript, 'Array.isArray(data.hosts)'), 'Plan editor must read the hosts collection returned by the admin API'); $assert(preg_match('/]*)>/', $templateAdmin, $shaInput) === 1 && !str_contains($shaInput[1], 'required'), 'Cloud image SHA-256 must be optional'); -$assert(str_contains($adminScript, 'if (sha256) body.sha256 = sha256;'), 'Blank cloud image SHA-256 must be omitted from the request'); +$importStart = strpos($adminScript, 'async function importTemplate('); +$importEnd = $importStart === false ? false : strpos($adminScript, 'async function uploadRescue(', $importStart); +$importScript = $importStart === false || $importEnd === false ? '' + : substr($adminScript, $importStart, $importEnd - $importStart); +$assert(!str_contains($templateAdmin, 'catalog-host') + && str_contains($templateAdmin, '全局云镜像目录') + && str_contains($rescueAdmin, 'id="catalog-host"') + && str_contains($adminScript, "root.dataset.cloudAdmin === 'templates'") + && str_contains($adminScript, "root.dataset.cloudAdmin === 'rescue'"), + 'Cloud images must be global while rescue templates remain host-scoped.'); +$assert(str_contains($importScript, 'if (sha256) body.sha256 = sha256;') + && !str_contains($importScript, 'host_id') + && !str_contains($importScript, 'catalog-host'), + 'Global cloud image imports must omit blank SHA-256 and all host selectors.'); +$globalAdminStart = strpos($cloudAdminController, "if (\$kind === 'cloud')"); +$globalAdminEnd = $globalAdminStart === false ? false + : strpos($cloudAdminController, '// Rescue ISOs remain', $globalAdminStart); +$globalAdmin = $globalAdminStart === false || $globalAdminEnd === false ? '' + : substr($cloudAdminController, $globalAdminStart, $globalAdminEnd - $globalAdminStart); +$assert(str_contains($globalAdmin, "'/internal/v1/vm-templates'") + && str_contains($globalAdmin, "unset(\$body['host_id'])") + && !str_contains($globalAdmin, 'filter_var(') + && !str_contains($globalAdmin, '->host('), + 'The PHP admin proxy must use the global engine catalog without a host dependency.'); $assert(str_contains($adminScript, 'function unixTime(value)') && str_contains($adminScript, 'orderStatus(record.status)'), 'Order diagnostics must format timestamps and localize task status');
模版按宿主机隔离,普通用户只能选择已导入的镜像。仅导入可信、支持 cloud-init 的 qcow2 镜像。
正在加载…
这里维护全局云镜像目录,普通用户在所有可售地区看到同一份镜像列表。宿主机副本由引擎按需准备;仅导入可信、支持 cloud-init 的 qcow2 镜像。升级前按宿主机导入的旧镜像不会自动进入全局目录,请重新导入一次。
留空时面板会计算并保存实际 SHA-256,不与外部校验值比对;填写后则会在导入前核对。