From b2654590c15fb47ea600ae6de3dbf4f34c301685 Mon Sep 17 00:00:00 2001 From: admin Date: Tue, 29 Sep 2026 06:00:10 +0800 Subject: [PATCH] feat: add browser cloud image uploads --- README.md | 6 +- docs/DEPLOYMENT.md | 8 +- engine/tests/test_cloud_templates.py | 50 +- engine/tests/test_engine_boundary.py | 456 ++++++++++++++++++ engine/virsh_panel/cloud_template_runtime.py | 154 +++++- engine/virsh_panel/cloud_templates.py | 166 +++++-- engine/virsh_panel/internal.py | 278 ++++++++++- web/app/Controllers/CloudAdminController.php | 113 +++++ web/app/Models/CloudTemplateCatalogLock.php | 1 + web/app/Services/Cloud/CloudEngineClient.php | 26 +- .../Cloud/CloudTemplateDeletionService.php | 142 +++++- web/config/routes_cloud.php | 1 + web/public/cloud/admin.css | 1 + web/public/cloud/admin.js | 76 ++- .../views/venix/admin/cloud/templates.tpl | 45 +- web/sql/cloud.sql | 11 +- web/tests/CloudFinancialWorkflowTest.php | 69 ++- web/tests/CloudTemplateErrorContractTest.php | 7 + web/tests/VirtualizationAssetsTest.php | 35 ++ 19 files changed, 1553 insertions(+), 92 deletions(-) diff --git a/README.md b/README.md index f4881fac..2e4e9a4c 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ PHP 8.5 网站与私有 Python/libvirt 引擎集成的虚拟机销售面板。 - 用户购买套餐开通 VM,余额扣款、库存扣减、确认创建失败退款;可手动续费,续费从已购期限末尾顺延。 - 套餐绑定宿主机,提供 CPU、内存、磁盘、网络带宽、账期天数和每期流量额度,用户不能随意修改套餐资源。 - 下载 RX 与上传 TX 按字节累加计费;当期额度用尽只断开 VM 网卡,不关机,下一已付费账期开始自动归零当期用量并恢复网络;无后续账期的最终到期才停机。 -- 管理员管理多宿主机、全局云镜像目录和独立救援 ISO;云镜像支持官方 AlmaLinux 8/9/10、Debian 11/12/13、CentOS Stream 9/10 和 Ubuntu 20.04/22.04/24.04/25.04/25.10/26.04。用户切换地区时云镜像选择保持不变,首次部署由引擎按需同步到目标宿主机并缓存复用。 +- 管理员管理多宿主机、全局云镜像目录和独立救援 ISO;云镜像既可从受信任的官方 HTTPS 地址下载,也可由管理员在浏览器上传不超过 4 GiB 的独立 QCOW2。支持 AlmaLinux 8/9/10、Debian 11/12/13、CentOS Stream 9/10 和 Ubuntu 20.04/22.04/24.04/25.04/25.10/26.04。用户切换地区时云镜像选择保持不变,首次部署由引擎按需同步到目标宿主机并缓存复用。 - 用户仅操作自己的 VM:控制台、开关机、软/硬重启、重装、救援、删除、初始 root 密码查看/复制。 - CPU、内存、磁盘 I/O、网络带宽图表;后台持续采集,写入日志库,最长保留 30 天。 - 全部用户页及登录页使用原虚拟机面板风格;管理员后台保留 venix 样式。 @@ -28,7 +28,7 @@ virt-cloud/ 网站文档根目录必须是 `web/public`。引擎仅监听 `127.0.0.1:8000`;不能把整个网站反代到引擎,只有 `/engine-console` 的 WebSocket 转发给引擎。 -云镜像导入只接受官方镜像站的 HTTPS 文件地址,不接受下载介绍页;引擎只跟随由官方入口返回且通过公网地址安全校验的跳转链。AlmaLinux、Debian 和 CentOS Stream 使用 `.qcow2`;Ubuntu 官方云镜像文件名为 `.img`,但下载内容必须是 qcow2。支持的文件地址形式例如: +云镜像的远程下载入口只接受官方镜像站的 HTTPS 文件地址,不接受下载介绍页;引擎只跟随由官方入口返回且通过公网地址安全校验的跳转链。AlmaLinux、Debian 和 CentOS Stream 使用 `.qcow2`;Ubuntu 官方云镜像文件名为 `.img`,但下载内容必须是 qcow2。支持的文件地址形式例如: ```text https://repo.almalinux.org/almalinux/8/cloud/x86_64/images/AlmaLinux-8-GenericCloud-latest.x86_64.qcow2 @@ -37,6 +37,8 @@ https://cloud.centos.org/centos/9-stream/x86_64/images/CentOS-Stream-GenericClou https://cloud-images.ubuntu.com/releases/noble/release/ubuntu-24.04-server-cloudimg-amd64.img ``` +自制或修复后的镜像使用管理员页面“从本机上传 QCOW2”。上传时必须选择与镜像内容一致的系统及版本;引擎会计算 SHA-256,并拒绝 raw/ISO、加密镜像、外部 data file、backing file、无法通过 QCOW2 结构检查的文件及超过 4 GiB 的文件。浏览器上传还需要按部署说明配置 PHP、Web 服务器和反代的请求体上限与超时。 + 全新安装见 [部署说明](docs/DEPLOYMENT.md),职责边界见 [集成架构](docs/INTEGRATION_PLAN.md)。配置使用 `web/config/config.php`,兼容旧文件名 `.config.php`;两者存在时前者优先。本版不导入原 Python 面板数据。 ## 验证范围 diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 582269d5..6a631864 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -7,12 +7,12 @@ - PHP 8.5 CLI/FPM:PDO MySQL、sodium、mbstring、curl、XML/DOM、OpenSSL、session 等扩展,Composer 2。 - MariaDB:业务库、日志库,初始化账号需要建表和执行幂等升级 DDL 的权限;PHP/PDO 中驱动名仍为 `mysql`。当前升级脚本使用 MariaDB 的 `ADD COLUMN IF NOT EXISTS`,不承诺兼容 Oracle MySQL。 - Redis 或兼容的 Valkey:沿用 venix 会话、验证码与限流。 -- Python 3.12 或更新的项目支持版本、venv/pip、libvirt 客户端及开发头文件、C 编译工具、pkg-config、OpenSSH 客户端。 +- Python 3.12 或更新的项目支持版本、venv/pip、libvirt 客户端及开发头文件、C 编译工具、pkg-config、OpenSSH 客户端、`/usr/bin/qemu-img`(用于镜像完整性检查)。 - Nginx/Apache、HTTPS、PHP-FPM,网站配置由你自行管理。 `libvirt-python` 是面板依赖,不代表面板服务器需要运行 KVM 宿主服务;远端宿主另需现有 KVM/QEMU、libvirt、网桥、UEFI 固件和存储目录。 -AlmaLinux 编译 Python libvirt 绑定常用系统依赖:`dnf install gcc pkgconf-pkg-config python3-devel libvirt-devel openssh-clients`。Python 可执行文件与开发头文件应使用相同版本。 +AlmaLinux 编译 Python libvirt 绑定常用系统依赖:`dnf install gcc pkgconf-pkg-config python3-devel libvirt-devel openssh-clients qemu-img`。Python 可执行文件与开发头文件应使用相同版本。 ## 2. 拉取与安装程序依赖 @@ -158,12 +158,12 @@ location = /engine-console { } ``` -不要公开 `/internal/v1/*` 或开放 8000 防火墙。PHP `/api/internal/cloud/session-introspection` 建议在 Web 层限定本机访问,仍有内部签名认证。救援上传需要合适的 FPM/反代超时和请求体上限,最大 ISO 8 GiB,入口只对管理员开放。 +不要公开 `/internal/v1/*` 或开放 8000 防火墙。PHP `/api/internal/cloud/session-introspection` 建议在 Web 层限定本机访问,仍有内部签名认证。管理员浏览器上传最大 4 GiB 云镜像或最大 8 GiB 救援 ISO 时,需要同步配置 PHP、FPM、Web 服务器及反代的请求体上限和至少 900 秒的读写/执行超时;入口只对管理员开放。代码会流式写入 0600 私有临时文件,不会把完整镜像载入 PHP 内存,但临时目录和 `runtime/engine/` 所在磁盘必须同时预留足够空间。 ## 7. 页面配置与验收 1. 管理员添加宿主机:现有 SSH/root/私钥口令、核验指纹,设置区域、网桥、目录、地址池/网关/DNS/网卡前缀,测试连接。 -2. 导入云镜像与救援模板;无已有模板不能开通云镜像 VM。云镜像是面板全局目录,验证后的源文件持久保存在 `runtime/engine/cloud-templates/`;首次在某宿主机创建或重装时才同步一份缓存,之后复用,所以切换地区不会更换或刷新镜像列表。支持 AlmaLinux 8/9/10、Debian 11/12/13、CentOS Stream 9/10、Ubuntu 20.04/22.04/24.04/25.04/25.10/26.04 官方 x86_64 云镜像,必须填写官方镜像站的 HTTPS 文件地址,不能是下载介绍页;引擎只跟随由官方入口返回且通过公网地址安全校验的跳转链。AlmaLinux、Debian 和 CentOS Stream 文件名必须为 `.qcow2`;Ubuntu 官方 `.img` 可使用,但实际内容必须是 qcow2,ISO 和 raw 镜像均不接受。例如 AlmaLinux 8 可使用 `https://repo.almalinux.org/almalinux/8/cloud/x86_64/images/AlmaLinux-8-GenericCloud-latest.x86_64.qcow2`,Debian 12 可使用 `https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-generic-amd64.qcow2`,CentOS Stream 9 可使用 `https://cloud.centos.org/centos/9-stream/x86_64/images/CentOS-Stream-GenericCloud-9-latest.x86_64.qcow2`,Ubuntu 24.04 可使用 `https://cloud-images.ubuntu.com/releases/noble/release/ubuntu-24.04-server-cloudimg-amd64.img`。面板磁盘需为源镜像及 256 MiB 安全余量预留空间,各宿主机的镜像存储池也需容纳本地缓存。云镜像 SHA-256 可留空;留空时仍会在下载后计算并保存实际摘要,填写时则会严格比对。 +2. 导入云镜像与救援模板;无已有模板不能开通云镜像 VM。云镜像是面板全局目录,验证后的源文件持久保存在 `runtime/engine/cloud-templates/`;首次在某宿主机创建或重装时才同步一份缓存,之后复用,所以切换地区不会更换或刷新镜像列表。支持 AlmaLinux 8/9/10、Debian 11/12/13、CentOS Stream 9/10、Ubuntu 20.04/22.04/24.04/25.04/25.10/26.04 x86_64 云镜像。官方镜像可填写官方镜像站的 HTTPS 文件地址,引擎只跟随由官方入口返回且通过公网地址安全校验的跳转链;例如 AlmaLinux 8 可使用 `https://repo.almalinux.org/almalinux/8/cloud/x86_64/images/AlmaLinux-8-GenericCloud-latest.x86_64.qcow2`,Debian 12 可使用 `https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-generic-amd64.qcow2`,CentOS Stream 9 可使用 `https://cloud.centos.org/centos/9-stream/x86_64/images/CentOS-Stream-GenericCloud-9-latest.x86_64.qcow2`,Ubuntu 24.04 可使用 `https://cloud-images.ubuntu.com/releases/noble/release/ubuntu-24.04-server-cloudimg-amd64.img`。自制或修复镜像使用管理员页面的本地 QCOW2 上传入口,并明确选择与内容一致的系统和版本;文件必须是 1 MiB 至 4 GiB 的独立、未加密 QCOW2,不能包含 backing file 或外部 data file,ISO 和 raw 镜像均不接受。面板磁盘需为上传临时文件、全局源镜像及 256 MiB 安全余量预留空间,各宿主机的镜像存储池也需容纳本地缓存。远程导入的 SHA-256 可留空;本地上传始终由面板计算、签名并保存实际摘要。 3. 套餐设置规格、每期价格、账期天数、每期流量额度、库存和已启用宿主机。用户提交部署后先生成 30 分钟有效的开通账单,支付成功才创建实例;实例到期前 7 天再生成续费账单。两种账单均可用余额支付,也可由启用的易支付、Paytaro 或 USDT 渠道生成支付订单。在线支付回调会自动结清对应账单;失效账单的晚到款只保留在账户余额中,不会创建实例或恢复已进入删除流程的实例。 4. 测试用户注册验证码、密码重置、充值、生成开通账单和账单支付,确认未支付时没有引擎任务,支付后只创建一个实例,并收到一次开通成功邮件;再验证到期前 7 天只生成一张续费账单并发送提醒、到期仅断网、宽限期内续费恢复网络,以及到期满 3 天后永久删除实例和系统盘;同时验收任务、VM 控制台、SSH、重装、救援、手动删除与图表。 5. 日志库 `engine_vm_metric_samples` 应在所有用户退出后继续新增,`engine_vm_traffic_totals` 应持续累加 RX/TX 字节;稳定 MAC 用于跨 QEMU/tap 重启衔接计数,首批数据需等待采样间隔。 diff --git a/engine/tests/test_cloud_templates.py b/engine/tests/test_cloud_templates.py index f39d5a77..e5927520 100644 --- a/engine/tests/test_cloud_templates.py +++ b/engine/tests/test_cloud_templates.py @@ -27,6 +27,7 @@ from virsh_panel.cloud_templates import ( template_scope, template_volume_name, validate_image_url, + verify_qcow2_image, ) @@ -193,7 +194,9 @@ def test_verified_image_is_private_and_has_expected_virtual_size(tmp_path): body = qcow2_fixture() digest = hashlib.sha256(body).hexdigest() with patch("virsh_panel.cloud_templates._open_image_response", - return_value=fake_image_response(body)): + return_value=fake_image_response(body)), patch( + "virsh_panel.cloud_templates._verify_qcow2_with_qemu" + ) as checked: downloaded, virtual_size, actual_sha256 = download_verified_image( OFFICIAL, digest, tmp_path ) @@ -202,6 +205,7 @@ def test_verified_image_is_private_and_has_expected_virtual_size(tmp_path): assert virtual_size == 2 * 1024**3 assert actual_sha256 == digest assert stat.S_IMODE(downloaded.stat().st_mode) == 0o600 + checked.assert_called_once_with(downloaded, 2 * 1024**3) finally: downloaded.unlink() @@ -210,7 +214,9 @@ def test_image_without_expected_hash_records_computed_digest(tmp_path): body = qcow2_fixture() digest = hashlib.sha256(body).hexdigest() with patch("virsh_panel.cloud_templates._open_image_response", - return_value=fake_image_response(body)): + return_value=fake_image_response(body)), patch( + "virsh_panel.cloud_templates._verify_qcow2_with_qemu" + ): downloaded, virtual_size, actual_sha256 = download_verified_image( OFFICIAL, "", tmp_path ) @@ -271,7 +277,9 @@ def test_debian_official_redirect_downloads_same_object_from_public_mirror( ), patch( "virsh_panel.cloud_templates._open_pinned_response", side_effect=[(connections[0], first), (connections[1], final)], - ) as opened: + ) as opened, patch( + "virsh_panel.cloud_templates._verify_qcow2_with_qemu" + ): downloaded, virtual_size, digest = download_verified_image( DEBIAN_12, hashlib.sha256(body).hexdigest(), tmp_path ) @@ -400,6 +408,42 @@ def test_external_data_file_feature_is_rejected(tmp_path): assert list(tmp_path.iterdir()) == [] +def test_qcow2_full_integrity_check_rejects_corrupt_image(tmp_path): + image = tmp_path / "corrupt.qcow2" + image.write_bytes(qcow2_fixture()) + info = [{"format": "qcow2", "virtual-size": 2 * 1024**3}] + with patch( + "virsh_panel.cloud_templates._qemu_img_json", + side_effect=[info, TemplateError("云镜像完整性检查失败", 422)], + ) as inspect: + with pytest.raises(TemplateError, match="完整性检查"): + verify_qcow2_image(image) + assert inspect.call_count == 2 + + +@pytest.mark.parametrize( + "info", + [ + [{"format": "raw", "virtual-size": 2 * 1024**3}], + [{"format": "qcow2", "virtual-size": 2 * 1024**3, + "backing-filename": "/private/base.qcow2"}], + [{"format": "qcow2", "virtual-size": 2 * 1024**3, + "format-specific": {"data": {"data-file": "/private/data"}}}], + [{"format": "qcow2", "virtual-size": 2 * 1024**3}, + {"format": "qcow2", "virtual-size": 2 * 1024**3}], + ], +) +def test_qcow2_full_integrity_check_rejects_dependencies_and_wrong_format( + tmp_path, info): + image = tmp_path / "unsafe.qcow2" + image.write_bytes(qcow2_fixture()) + with patch("virsh_panel.cloud_templates._qemu_img_json", + return_value=info) as inspect: + with pytest.raises(TemplateError, match="外部依赖"): + verify_qcow2_image(image) + inspect.assert_called_once() + + def test_template_catalog_is_private_and_scoped_to_host_and_disk_pool(tmp_path): data_dir = tmp_path / "data" data_dir.mkdir(mode=0o700) diff --git a/engine/tests/test_engine_boundary.py b/engine/tests/test_engine_boundary.py index 372943eb..20e7699c 100644 --- a/engine/tests/test_engine_boundary.py +++ b/engine/tests/test_engine_boundary.py @@ -5,12 +5,15 @@ import base64 import asyncio import hashlib import json +import os +import struct import threading import time from concurrent.futures import ThreadPoolExecutor from contextlib import contextmanager from types import SimpleNamespace from unittest.mock import MagicMock, patch +from urllib.parse import urlencode import pytest from fastapi.testclient import TestClient @@ -2144,6 +2147,459 @@ def test_streamed_iso_digest_must_match_before_import_is_queued(boundary): assert engine.registry.components["hk"].store.list("rescue_import_job") == {} +def cloud_upload_fixture(size=1024 * 1024 + 257): + header = bytearray(104) + header[:4] = b"QFI\xfb" + header[4:8] = struct.pack(">I", 3) + header[20:24] = struct.pack(">I", 16) + header[24:32] = struct.pack(">Q", 20 * 1024**3) + header[100:104] = struct.pack(">I", 104) + return bytes(header) + b"\0" * (size - len(header)) + + +def wait_for_cloud_upload_job(client, job_id): + path = f"/internal/v1/vm-templates/import-jobs/{job_id}" + for _ in range(200): + response = call(client, "GET", path, SUPER) + assert response.status_code == 200, response.text + job = response.json() + if job["status"] in {"completed", "failed"}: + return job + time.sleep(0.01) + raise AssertionError("uploaded cloud template import did not finish") + + +def cloud_engine_scope(method, target, body, actor, *, headers=None): + path, separator, query = target.partition("?") + request_headers = signed_headers(SECRET, method, target, body, actor) + request_headers.update(headers or {}) + return { + "type": "http", + "asgi": {"version": "3.0"}, + "http_version": "1.1", + "method": method, + "scheme": "http", + "path": path, + "raw_path": path.encode("ascii"), + "query_string": query.encode("ascii") if separator else b"", + "headers": [ + (name.lower().encode("ascii"), value.encode("latin-1")) + for name, value in request_headers.items() + ], + "client": ("testclient", 50000), + "server": ("127.0.0.1", 80), + } + + +async def cloud_engine_request(engine, method, target, body, actor, *, + headers=None, receive=None): + scope = cloud_engine_scope( + method, target, body, actor, headers=headers) + delivered = False + + async def receive_body(): + nonlocal delivered + if delivered: + return {"type": "http.disconnect"} + delivered = True + return {"type": "http.request", "body": body, "more_body": False} + + messages = [] + + async def send(message): + messages.append(message) + + await engine(scope, receive or receive_body, send) + status = next(message["status"] for message in messages + if message["type"] == "http.response.start") + raw = b"".join(message.get("body", b"") for message in messages + if message["type"] == "http.response.body") + return status, json.loads(raw) + + +def test_global_cloud_template_upload_streams_past_json_limit(boundary): + client, engine, *_ = boundary + query = urlencode({ + "name": "Debian 12 fixed", + "os_family": "Debian", + "os_version": "12", + }) + path = "/internal/v1/vm-templates/upload?" + query + payload = cloud_upload_fixture() + with patch("virsh_panel.cloud_templates._verify_qcow2_with_qemu"): + response = call(client, "POST", path, SUPER, raw=payload, headers={ + "X-Cloud-Content-SHA256": hashlib.sha256(payload).hexdigest(), + "Content-Type": "application/octet-stream", + }) + assert response.status_code == 202, response.text + job = wait_for_cloud_upload_job(client, response.json()["job_id"]) + assert job["status"] == "completed", job + assert job["template"]["name"] == "Debian 12 fixed" + assert job["template"]["os_family"] == "Debian" + assert job["template"]["os_version"] == "12" + assert job["template"]["virtual_size"] == 20 * 1024**3 + with engine.registry.cloud_templates.source( + job["template"]["id"]) as (_record, source): + assert source.stat().st_size == len(payload) + assert hashlib.sha256(source.read_bytes()).digest() == hashlib.sha256( + payload).digest() + + +def test_global_cloud_template_upload_digest_must_match_before_queue(boundary): + client, engine, *_ = boundary + path = ("/internal/v1/vm-templates/upload?" + "name=Debian+12&os_family=Debian&os_version=12") + signed_body = cloud_upload_fixture() + wrong_body = signed_body[:-1] + b"x" + signed = signed_headers(SECRET, "POST", path, signed_body, SUPER) + signed["X-Cloud-Content-SHA256"] = hashlib.sha256( + signed_body).hexdigest() + signed["Content-Type"] = "application/octet-stream" + response = client.post(path, content=wrong_body, headers=signed) + assert response.status_code == 401, response.text + assert engine.registry.cloud_templates.jobs == {} + assert engine.registry.cloud_templates.pending == set() + root = engine.registry.cloud_templates.files.directory() + assert not list(root.glob(".cloud-image-upload-*.qcow2")) + + +@pytest.mark.parametrize( + ("family", "version"), + [ + ("debian", "12"), + ("Debian", "14"), + ("CentOS", "10"), + ("Ubuntu", "24"), + ], +) +def test_global_cloud_template_upload_rejects_noncanonical_metadata( + boundary, family, version): + client, engine, *_ = boundary + path = "/internal/v1/vm-templates/upload?" + urlencode({ + "name": "Invalid metadata", + "os_family": family, + "os_version": version, + }) + payload = cloud_upload_fixture() + response = call(client, "POST", path, SUPER, raw=payload, headers={ + "X-Cloud-Content-SHA256": hashlib.sha256(payload).hexdigest(), + "Content-Type": "application/octet-stream", + }) + assert response.status_code == 422, response.text + assert engine.registry.cloud_templates.jobs == {} + + +def test_global_cloud_template_upload_requires_super_admin_and_octet_stream( + boundary): + client, engine, *_ = boundary + path = ("/internal/v1/vm-templates/upload?" + "name=Debian+12&os_family=Debian&os_version=12") + payload = cloud_upload_fixture() + digest = hashlib.sha256(payload).hexdigest() + forbidden = call(client, "POST", path, ADMIN, raw=payload, headers={ + "X-Cloud-Content-SHA256": digest, + "Content-Type": "application/octet-stream", + }) + assert forbidden.status_code == 403, forbidden.text + wrong_type = call(client, "POST", path, SUPER, raw=payload, headers={ + "X-Cloud-Content-SHA256": digest, + "Content-Type": "multipart/form-data", + }) + assert wrong_type.status_code == 415, wrong_type.text + assert engine.registry.cloud_templates.jobs == {} + + +def test_global_cloud_template_upload_enforces_minimum_size(boundary): + client, engine, *_ = boundary + path = ("/internal/v1/vm-templates/upload?" + "name=Too+Small&os_family=Debian&os_version=12") + payload = cloud_upload_fixture(104) + response = call(client, "POST", path, SUPER, raw=payload, headers={ + "X-Cloud-Content-SHA256": hashlib.sha256(payload).hexdigest(), + "Content-Type": "application/octet-stream", + }) + assert response.status_code == 422, response.text + assert engine.registry.cloud_templates.jobs == {} + assert engine.registry.cloud_templates.pending == set() + + +def test_global_cloud_template_upload_timeout_releases_reservation( + boundary, monkeypatch): + _client, engine, *_ = boundary + query = b"name=Timeout&os_family=Debian&os_version=12" + scope = { + "query_string": query, + "headers": [ + (b"content-type", b"application/octet-stream"), + (b"content-length", str(1024 * 1024).encode()), + ], + "engine_upload_validation": {"complete": False, "valid": False}, + "engine_upload_digest": "0" * 64, + } + + async def stalled_receive(): + await asyncio.sleep(1) + return {"type": "http.request", "body": b"", "more_body": False} + + async def send(_message): + raise AssertionError("timeout must not start a response") + + monkeypatch.setattr("virsh_panel.internal.CLOUD_UPLOAD_IDLE_SECONDS", 0.01) + with pytest.raises(TemplateError, match="上传超时") as timeout: + asyncio.run(engine._upload_cloud_template(scope, stalled_receive, send)) + assert timeout.value.status == 408 + assert engine.registry.cloud_templates.pending == set() + assert engine.registry.cloud_templates.upload_reservations == {} + root = engine.registry.cloud_templates.files.directory() + assert not list(root.glob(".cloud-image-upload-*.qcow2")) + + +def test_stalled_cloud_upload_does_not_block_other_engine_writes( + boundary, monkeypatch): + _client, engine, *_ = boundary + payload = cloud_upload_fixture() + upload_path = "/internal/v1/vm-templates/upload?" + urlencode({ + "name": "Stalled upload", + "os_family": "Debian", + "os_version": "12", + }) + upload_started = asyncio.Event() + release_upload = asyncio.Event() + + class FastRemoteHostManager: + def __init__(self, *_args, **_kwargs): + pass + + @staticmethod + def scan(host, port): + return {"host": host, "port": port, "reachable": True} + + @staticmethod + def close(): + pass + + monkeypatch.setattr( + "virsh_panel.remote_host.RemoteHostManager", FastRemoteHostManager) + + async def stalled_receive(): + upload_started.set() + await release_upload.wait() + return {"type": "http.disconnect"} + + async def scenario(): + upload = asyncio.create_task(cloud_engine_request( + engine, + "POST", + upload_path, + payload, + SUPER, + headers={ + "X-Cloud-Content-SHA256": hashlib.sha256(payload).hexdigest(), + "Content-Type": "application/octet-stream", + "Content-Length": str(len(payload)), + }, + receive=stalled_receive, + )) + await asyncio.wait_for(upload_started.wait(), timeout=1) + assert engine.registry.cloud_templates.pending + + scan_body = json.dumps({ + "host": "host.example.test", "port": 22, + }, separators=(",", ":")).encode() + status, result = await asyncio.wait_for(cloud_engine_request( + engine, + "POST", + "/internal/v1/hosts/scan", + scan_body, + SUPER, + headers={"Content-Type": "application/json"}, + ), timeout=0.5) + assert status == 200 + assert result["host"] == "host.example.test" + + release_upload.set() + upload_status, _result = await asyncio.wait_for(upload, timeout=1) + assert upload_status == 400 + + asyncio.run(scenario()) + assert engine.registry.cloud_templates.pending == set() + assert engine.registry.cloud_templates.upload_reservations == {} + + +def test_cloud_upload_fsync_does_not_block_event_loop_or_engine_writes( + boundary, monkeypatch): + _client, engine, *_ = boundary + payload = cloud_upload_fixture() + upload_path = "/internal/v1/vm-templates/upload?" + urlencode({ + "name": "Slow fsync upload", + "os_family": "Debian", + "os_version": "12", + }) + fsync_started = threading.Event() + release_fsync = threading.Event() + original_fsync = os.fsync + + def slow_fsync(descriptor): + fsync_started.set() + assert release_fsync.wait(timeout=2), "test did not release fsync" + return original_fsync(descriptor) + + class FastRemoteHostManager: + def __init__(self, *_args, **_kwargs): + pass + + @staticmethod + def scan(host, port): + return {"host": host, "port": port, "reachable": True} + + @staticmethod + def close(): + pass + + monkeypatch.setattr("virsh_panel.internal.os.fsync", slow_fsync) + monkeypatch.setattr( + "virsh_panel.remote_host.RemoteHostManager", FastRemoteHostManager) + monkeypatch.setattr( + "virsh_panel.cloud_templates._verify_qcow2_with_qemu", + lambda *_args: None) + + async def scenario(): + safety_release = threading.Timer(1.5, release_fsync.set) + safety_release.start() + started_at = time.monotonic() + try: + upload = asyncio.create_task(cloud_engine_request( + engine, + "POST", + upload_path, + payload, + SUPER, + headers={ + "X-Cloud-Content-SHA256": hashlib.sha256(payload).hexdigest(), + "Content-Type": "application/octet-stream", + "Content-Length": str(len(payload)), + }, + )) + while not fsync_started.is_set(): + await asyncio.sleep(0.001) + + scan_body = json.dumps({ + "host": "host.example.test", "port": 22, + }, separators=(",", ":")).encode() + status, _result = await asyncio.wait_for(cloud_engine_request( + engine, + "POST", + "/internal/v1/hosts/scan", + scan_body, + SUPER, + headers={"Content-Type": "application/json"}, + ), timeout=0.5) + assert status == 200 + assert time.monotonic() - started_at < 0.5 + assert not release_fsync.is_set() + release_fsync.set() + upload_status, _result = await asyncio.wait_for(upload, timeout=2) + assert upload_status == 202 + finally: + release_fsync.set() + safety_release.cancel() + + asyncio.run(scenario()) + + +def test_cloud_upload_rechecks_update_state_before_queue( + boundary, monkeypatch): + _client, engine, *_ = boundary + updater = FakeUpdater() + engine.updater = updater + payload = cloud_upload_fixture() + upload_path = "/internal/v1/vm-templates/upload?" + urlencode({ + "name": "Update race upload", + "os_family": "Debian", + "os_version": "12", + }) + upload_started = asyncio.Event() + release_upload = asyncio.Event() + delivered = False + monkeypatch.setattr( + "virsh_panel.cloud_templates._verify_qcow2_with_qemu", + lambda *_args: None) + + async def delayed_receive(): + nonlocal delivered + if delivered: + return {"type": "http.disconnect"} + delivered = True + upload_started.set() + await release_upload.wait() + return {"type": "http.request", "body": payload, "more_body": False} + + async def scenario(): + upload = asyncio.create_task(cloud_engine_request( + engine, + "POST", + upload_path, + payload, + SUPER, + headers={ + "X-Cloud-Content-SHA256": hashlib.sha256(payload).hexdigest(), + "Content-Type": "application/octet-stream", + "Content-Length": str(len(payload)), + }, + receive=delayed_receive, + )) + await asyncio.wait_for(upload_started.wait(), timeout=1) + assert engine.registry.cloud_templates.pending + updater.active = { + "job_id": "update-job-1234567890", "status": "running", + } + release_upload.set() + status, result = await asyncio.wait_for(upload, timeout=2) + assert status == 409 + assert result["error"] == "面板正在更新,暂时不能执行虚拟化写操作" + + try: + asyncio.run(scenario()) + finally: + updater.active = None + assert engine.registry.cloud_templates.pending == set() + assert engine.registry.cloud_templates.upload_reservations == {} + assert engine.registry.cloud_templates.jobs == {} + + +def test_upload_same_os_version_keeps_existing_template(boundary): + client, engine, *_ = boundary + + def upload(name, payload): + path = "/internal/v1/vm-templates/upload?" + urlencode({ + "name": name, + "os_family": "Debian", + "os_version": "12", + }) + response = call(client, "POST", path, SUPER, raw=payload, headers={ + "X-Cloud-Content-SHA256": hashlib.sha256(payload).hexdigest(), + "Content-Type": "application/octet-stream", + }) + assert response.status_code == 202, response.text + job = wait_for_cloud_upload_job(client, response.json()["job_id"]) + assert job["status"] == "completed", job + return job["template"] + + official_payload = cloud_upload_fixture()[:-1] + b"o" + fixed_payload = cloud_upload_fixture()[:-1] + b"f" + with patch("virsh_panel.cloud_templates._verify_qcow2_with_qemu"): + official = upload("Debian 12 official", official_payload) + fixed = upload("Debian 12 fixed", fixed_payload) + + assert official["id"] != fixed["id"] + templates = call( + client, "GET", "/internal/v1/vm-templates", SUPER).json()["templates"] + assert {row["id"] for row in templates} == {official["id"], fixed["id"]} + with engine.registry.cloud_templates.source( + official["id"]) as (_record, source): + assert hashlib.sha256(source.read_bytes()).hexdigest() == official["sha256"] + + def test_main_and_log_schema_do_not_create_python_accounts_or_cross_database_tables(): connection = MagicMock() cursor = connection.cursor.return_value.__enter__.return_value diff --git a/engine/virsh_panel/cloud_template_runtime.py b/engine/virsh_panel/cloud_template_runtime.py index 8d8e4745..0fe8d1ed 100644 --- a/engine/virsh_panel/cloud_template_runtime.py +++ b/engine/virsh_panel/cloud_template_runtime.py @@ -37,6 +37,7 @@ from .cloud_templates import ( image_metadata_for, template_id_for, template_volume_name, + validate_template_metadata, validate_template_name, ) @@ -357,6 +358,7 @@ class CloudTemplateRuntime: self.lock = threading.RLock() self.jobs: dict[str, dict[str, Any]] = {} self.pending: set[str] = set() + self.upload_reservations: dict[str, dict[str, str]] = {} self.in_use: dict[str, int] = {} self.verified_sources: dict[str, tuple[int, int, int, int, str]] = {} self.memory_generations: dict[str, int] = {} @@ -637,6 +639,108 @@ class CloudTemplateRuntime: tombstone = self._decode_cache_tombstone(template_id, raw) self._reconcile_cache_tombstone(template_id, tombstone) + def reserve_upload(self, name: str, os_family: str, + os_version: str) -> dict[str, Any]: + """Reserve the single import slot before receiving a browser upload.""" + name = validate_template_name(name) + os_family, os_version = validate_template_metadata( + os_family, os_version) + reservation_id = "upload-" + uuid.uuid4().hex + with self.lock: + existing = self.catalog.list() + directory = self.files.directory() + maximum = self._remaining_download_bytes() + if any(row["name"].casefold() == name.casefold() + for row in existing): + raise TemplateError("已有同名云镜像模板", 409) + if self.pending: + raise TemplateError( + "已有云镜像正在导入,请等待完成后再提交", 409 + ) + self.pending.add(reservation_id) + self.upload_reservations[reservation_id] = { + "name": name, + "os_family": os_family, + "os_version": os_version, + } + return { + "reservation_id": reservation_id, + "directory": directory, + "max_bytes": maximum, + } + + def cancel_upload(self, reservation_id: str) -> None: + """Release a pre-body reservation after validation or transport failure.""" + with self.lock: + self.upload_reservations.pop(reservation_id, None) + self.pending.discard(reservation_id) + + def queue_upload(self, reservation_id: str, image_path: Path, + digest: str, virtual_size: int) -> dict[str, str]: + """Queue publication of an already streamed and validated qcow2 file.""" + image_path = Path(image_path) + if not isinstance(digest, str) or SHA256_RE.fullmatch(digest) is None: + raise TemplateError("云镜像 SHA-256 格式无效", 422) + digest = digest.lower() + if (isinstance(virtual_size, bool) or not isinstance(virtual_size, int) + or not GIB <= virtual_size <= MAX_VIRTUAL_BYTES): + raise TemplateError("云镜像系统盘容量超出支持范围", 422) + try: + info = image_path.lstat() + except FileNotFoundError as exc: + raise TemplateError("上传的云镜像临时文件丢失", 500) from exc + if (not stat.S_ISREG(info.st_mode) or stat.S_ISLNK(info.st_mode) + or info.st_uid != os.geteuid() + or stat.S_IMODE(info.st_mode) != 0o600 + or image_path.parent != self.files.directory() + or info.st_size <= 0 or info.st_size > MAX_IMAGE_BYTES): + raise TemplateError("上传的云镜像临时文件无效", 500) + with self.lock: + metadata = self.upload_reservations.get(reservation_id) + if metadata is None or reservation_id not in self.pending: + raise TemplateError("云镜像上传预约已失效,请重新上传", 409) + name = metadata["name"] + os_family = metadata["os_family"] + os_version = metadata["os_version"] + template_id = cloud_template_id_for( + digest, os_version, os_family) + existing = self.catalog.list() + if any(row["id"] == template_id for row in existing): + raise TemplateError("此云镜像已经导入", 409) + if any(row["name"].casefold() == name.casefold() + for row in existing): + raise TemplateError("已有同名云镜像模板", 409) + job_id = str(uuid.uuid4()) + job = { + "job_id": job_id, + "status": "queued", + "phase": "等待保存上传的云镜像", + "created_at": int(time.time()), + } + try: + self.save_job(job_id, job) + self.executor.submit( + self._run_uploaded_import, + job_id, + image_path, + name, + os_family, + os_version, + digest, + virtual_size, + reservation_id, + ) + except Exception as exc: + self.delete_job(job_id) + self.upload_reservations.pop(reservation_id, None) + self.pending.discard(reservation_id) + if isinstance(exc, RuntimeError): + raise TemplateError( + "引擎正在停止,无法开始云镜像导入", 503 + ) from exc + raise + return {"job_id": job_id, "status": "queued"} + def queue_import(self, url: str, name: str, expected_sha256: str = "") -> dict[str, str]: os_family, version = image_metadata_for(url) @@ -695,6 +799,45 @@ class CloudTemplateRuntime: def _run_import(self, job_id: str, url: str, name: str, expected_sha256: str, pending_key: str) -> None: + def acquire() -> tuple[Path, int, str, str, str]: + with self.lock: + directory = self.files.directory() + maximum = self._remaining_download_bytes() + temporary, virtual_size, actual_sha256 = download_verified_image( + url, expected_sha256, directory, max_bytes=maximum + ) + os_family, version = image_metadata_for(url) + return ( + temporary, virtual_size, actual_sha256, os_family, version + ) + + self._run_import_job( + job_id, + name, + pending_key, + "正在下载并校验云镜像", + acquire, + ) + + def _run_uploaded_import( + self, job_id: str, image_path: Path, name: str, + os_family: str, version: str, digest: str, virtual_size: int, + reservation_id: str) -> None: + def acquire() -> tuple[Path, int, str, str, str]: + return image_path, virtual_size, digest, os_family, version + + self._run_import_job( + job_id, + name, + reservation_id, + "正在校验并保存上传的云镜像", + acquire, + ) + + def _run_import_job( + self, job_id: str, name: str, pending_key: str, + running_phase: str, acquire) -> None: + """Run the common catalog/publish transaction for every import source.""" temporary: Path | None = None published = False cataloged = False @@ -706,16 +849,12 @@ class CloudTemplateRuntime: job = self.jobs[job_id] job.update( status="running", - phase="正在下载并校验云镜像", + phase=running_phase, started_at=int(time.time()), ) self.save_job(job_id, job) - directory = self.files.directory() - maximum = self._remaining_download_bytes() - temporary, virtual_size, actual_sha256 = download_verified_image( - url, expected_sha256, directory, max_bytes=maximum - ) - os_family, version = image_metadata_for(url) + (temporary, virtual_size, actual_sha256, + os_family, version) = acquire() template_id = cloud_template_id_for( actual_sha256, version, os_family ) @@ -835,6 +974,7 @@ class CloudTemplateRuntime: if temporary is not None: temporary.unlink(missing_ok=True) with self.lock: + self.upload_reservations.pop(pending_key, None) self.pending.discard(pending_key) def _save_completed_job(self, job_id: str, previous: dict[str, Any], diff --git a/engine/virsh_panel/cloud_templates.py b/engine/virsh_panel/cloud_templates.py index 52c88112..a06b46ce 100644 --- a/engine/virsh_panel/cloud_templates.py +++ b/engine/virsh_panel/cloud_templates.py @@ -14,6 +14,7 @@ import socket import ssl import stat import struct +import subprocess import tempfile import threading import time @@ -27,6 +28,9 @@ MAX_IMAGE_BYTES = 4 * 1024**3 MAX_VIRTUAL_BYTES = 4096 * 1024**3 MAX_DOWNLOAD_SECONDS = 3600 MAX_IMAGE_REDIRECTS = 3 +QEMU_IMG = "/usr/bin/qemu-img" +QEMU_IMG_TIMEOUT_SECONDS = 300 +QEMU_IMG_OUTPUT_LIMIT = 1024 * 1024 GIB = 1024**3 ALMALINUX_IMAGE_PATH = re.compile( r"/almalinux/(?P8|9|10)/cloud/x86_64/images/" @@ -65,6 +69,12 @@ UBUNTU_RELEASES = { "questing": "25.10", "resolute": "26.04", } +SUPPORTED_TEMPLATE_VERSIONS = { + "AlmaLinux": frozenset({"8", "9", "10"}), + "Debian": frozenset({"11", "12", "13"}), + "Ubuntu": frozenset(UBUNTU_RELEASES.values()), + "CentOS Stream": frozenset({"9", "10"}), +} TEMPLATE_ID_RE = re.compile( r"(?:alma(?:8|9|10)|debian(?:11|12|13)|" r"ubuntu(?:2004|2204|2404|2504|2510|2604)|" @@ -342,16 +352,31 @@ def validate_template_name(name: str) -> str: return name.strip() +def validate_template_metadata(os_family: str, version: str) -> tuple[str, str]: + """Validate administrator-supplied metadata for an uploaded image. + + Remote imports derive these values from an allowlisted official URL. A + browser upload has no trustworthy filename or URL, so it must provide the + exact canonical values already supported by template identifiers. + """ + if (not isinstance(os_family, str) or not isinstance(version, str) + or version not in SUPPORTED_TEMPLATE_VERSIONS.get(os_family, ())): + raise TemplateError("云镜像系统或版本不受支持", 422) + return os_family, version + + def _template_id_prefix(os_family: str, version: str) -> str: - if not isinstance(os_family, str) or not isinstance(version, str): - raise TemplateError("云镜像系统版本无效") - if os_family == "AlmaLinux" and version in {"8", "9", "10"}: + try: + validate_template_metadata(os_family, version) + except TemplateError as exc: + raise TemplateError("云镜像系统版本无效") from exc + if os_family == "AlmaLinux": return f"alma{version}" - if os_family == "Debian" and version in {"11", "12", "13"}: + if os_family == "Debian": return f"debian{version}" - if os_family == "Ubuntu" and version in set(UBUNTU_RELEASES.values()): + if os_family == "Ubuntu": return f"ubuntu{version.replace('.', '')}" - if os_family == "CentOS Stream" and version in {"9", "10"}: + if os_family == "CentOS Stream": return f"centosstream{version}" raise TemplateError("云镜像系统版本无效") @@ -373,6 +398,106 @@ def template_volume_name(template_id: str) -> str: return f"panel-template-{template_id}.qcow2" +def _qemu_img_json(arguments: list[str]) -> object: + """Run the trusted local image inspector with bounded time and output.""" + with tempfile.TemporaryFile() as stdout, tempfile.TemporaryFile() as stderr: + try: + result = subprocess.run( + [QEMU_IMG, *arguments], + cwd="/", + env={"PATH": "/usr/bin:/bin", "LANG": "C", "LC_ALL": "C"}, + stdin=subprocess.DEVNULL, + stdout=stdout, + stderr=stderr, + timeout=QEMU_IMG_TIMEOUT_SECONDS, + check=False, + close_fds=True, + ) + except FileNotFoundError as exc: + raise TemplateError( + "面板未安装 qemu-img,无法验证云镜像", 503 + ) from exc + except (OSError, subprocess.TimeoutExpired) as exc: + raise TemplateError("云镜像完整性检查失败", 422) from exc + if (stdout.tell() > QEMU_IMG_OUTPUT_LIMIT + or stderr.tell() > QEMU_IMG_OUTPUT_LIMIT): + raise TemplateError("云镜像完整性检查失败", 422) + if result.returncode != 0: + raise TemplateError("云镜像完整性检查失败", 422) + stdout.seek(0) + try: + return json.loads(stdout.read(QEMU_IMG_OUTPUT_LIMIT + 1)) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise TemplateError("云镜像完整性检查失败", 422) from exc + + +def _contains_external_qcow2_dependency(value: object) -> bool: + """Reject dependency/encryption fields wherever qemu-img reports them.""" + if isinstance(value, dict): + for key, nested in value.items(): + normalized = str(key).lower().replace("_", "-") + if normalized in { + "backing-filename", "full-backing-filename", "data-file", + "full-data-file", "encrypted", "encrypt", "encryption", + } and (nested is not None and nested is not False + and nested != "" and nested != 0): + return True + if _contains_external_qcow2_dependency(nested): + return True + elif isinstance(value, list): + return any(_contains_external_qcow2_dependency(item) for item in value) + return False + + +def _verify_qcow2_with_qemu(image_path: Path, virtual_size: int) -> None: + path = str(Path(image_path)) + info = _qemu_img_json(["info", "--output=json", "--backing-chain", path]) + chain = info if isinstance(info, list) else [info] + if (len(chain) != 1 or not isinstance(chain[0], dict) + or chain[0].get("format") != "qcow2" + or chain[0].get("virtual-size") != virtual_size + or _contains_external_qcow2_dependency(chain[0])): + raise TemplateError( + "云镜像含外部依赖、加密或容量超出支持范围", 422 + ) + checked = _qemu_img_json(["check", "--output=json", "-f", "qcow2", path]) + if (not isinstance(checked, dict) + or any(isinstance(checked.get(key), int) and checked[key] > 0 + for key in ("corruptions", "leaks", "check-errors"))): + raise TemplateError("云镜像完整性检查失败", 422) + + +def verify_qcow2_image(image_path: Path) -> int: + """Return capacity after a header and full qemu-img integrity check.""" + with Path(image_path).open("rb") as image: + header = image.read(104) + if len(header) < 72 or header[:4] != b"QFI\xfb": + raise TemplateError("上传文件不是有效的 qcow2 镜像", 422) + qcow_version = struct.unpack(">I", header[4:8])[0] + backing_offset = struct.unpack(">Q", header[8:16])[0] + cluster_bits = struct.unpack(">I", header[20:24])[0] + virtual_size = struct.unpack(">Q", header[24:32])[0] + crypt_method = struct.unpack(">I", header[32:36])[0] + if qcow_version not in (2, 3) or not 9 <= cluster_bits <= 21: + raise TemplateError("云镜像 qcow2 版本或簇大小不受支持", 422) + if qcow_version == 3: + if len(header) < 104: + raise TemplateError("云镜像 qcow2 头部不完整", 422) + incompatible = struct.unpack(">Q", header[72:80])[0] + header_length = struct.unpack(">I", header[100:104])[0] + # The external data-file, dirty and corrupt flags all make this + # unsuitable as a standalone template. Reject unknown features. + if (incompatible & ~0x18 or header_length < 104 + or header_length > 1 << cluster_bits or header_length % 8): + raise TemplateError( + "云镜像包含不受支持或外部依赖的 qcow2 特性", 422) + if (backing_offset or crypt_method + or not GIB <= virtual_size <= MAX_VIRTUAL_BYTES): + raise TemplateError("云镜像含外部依赖、加密或容量超出支持范围", 422) + _verify_qcow2_with_qemu(Path(image_path), virtual_size) + return virtual_size + + def download_verified_image( url: str, expected_sha256: str, directory: Path, *, max_bytes: int = MAX_IMAGE_BYTES, @@ -439,29 +564,12 @@ def download_verified_image( actual_sha256 = checksum.hexdigest() if expected_sha256 and actual_sha256 != expected_sha256.lower(): raise TemplateError("云镜像 SHA-256 不匹配,已丢弃下载内容", 422) - with image_path.open("rb") as image: - header = image.read(104) - if len(header) < 72 or header[:4] != b"QFI\xfb": - raise TemplateError("下载文件不是有效的 qcow2 镜像", 422) - version = struct.unpack(">I", header[4:8])[0] - backing_offset = struct.unpack(">Q", header[8:16])[0] - cluster_bits = struct.unpack(">I", header[20:24])[0] - virtual_size = struct.unpack(">Q", header[24:32])[0] - crypt_method = struct.unpack(">I", header[32:36])[0] - if version not in (2, 3) or not 9 <= cluster_bits <= 21: - raise TemplateError("云镜像 qcow2 版本或簇大小不受支持", 422) - if version == 3: - if len(header) < 104: - raise TemplateError("云镜像 qcow2 头部不完整", 422) - incompatible = struct.unpack(">Q", header[72:80])[0] - header_length = struct.unpack(">I", header[100:104])[0] - # The external data-file, dirty and corrupt flags all make this - # unsuitable as a standalone template. Reject unknown features. - if (incompatible & ~0x18 or header_length < 104 - or header_length > 1 << cluster_bits or header_length % 8): - raise TemplateError("云镜像包含不受支持或外部依赖的 qcow2 特性", 422) - if backing_offset or crypt_method or not GIB <= virtual_size <= MAX_VIRTUAL_BYTES: - raise TemplateError("云镜像含外部依赖、加密或容量超出支持范围", 422) + try: + virtual_size = verify_qcow2_image(image_path) + except TemplateError as exc: + if str(exc) == "上传文件不是有效的 qcow2 镜像": + raise TemplateError("下载文件不是有效的 qcow2 镜像", 422) from exc + raise return image_path, virtual_size, actual_sha256 except BaseException: image_path.unlink(missing_ok=True) diff --git a/engine/virsh_panel/internal.py b/engine/virsh_panel/internal.py index fbf8b39f..daf7feff 100644 --- a/engine/virsh_panel/internal.py +++ b/engine/virsh_panel/internal.py @@ -7,6 +7,7 @@ PHP cookie, and rechecks the PHP session while the console remains connected. from __future__ import annotations import asyncio +import errno import hashlib import hmac import inspect @@ -17,17 +18,31 @@ import re import secrets import stat import subprocess +import tempfile +import threading import time import uuid from contextlib import AsyncExitStack, suppress from pathlib import Path -from urllib.parse import urlsplit +from urllib.parse import parse_qsl, urlsplit from urllib.request import (HTTPRedirectHandler, ProxyHandler, Request as URLRequest, build_opener) -from .cloud_templates import TemplateError, template_volume_name -from .cloud_template_runtime import CACHE_TOMBSTONE_KIND, CloudTemplateRuntime +from .cloud_templates import ( + MAX_DOWNLOAD_SECONDS, + MAX_IMAGE_BYTES, + TemplateError, + template_volume_name, + validate_template_metadata, + validate_template_name, + verify_qcow2_image, +) +from .cloud_template_runtime import ( + CACHE_TOMBSTONE_KIND, + MIN_DOWNLOAD_BYTES, + CloudTemplateRuntime, +) from .engine_context import (EngineComponents, HostStore, LogStore, MainStore, TRUSTED_ACTOR, actor_session) from .engine_security import (EngineAuthError, authenticate, decode_secret, @@ -41,6 +56,8 @@ HOST_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]{0,63}\Z") OP_ID = re.compile(r"[A-Za-z0-9_-]{8,80}\Z") VM_ID = r"[0-9a-fA-F-]{36}" JSON_LIMIT = 1024 * 1024 +CLOUD_UPLOAD_IDLE_SECONDS = 60 +CLOUD_UPLOAD_WRITE_BYTES = 1024 * 1024 READ_PATHS = ( r"host", r"resources", r"vm-templates", r"rescue-templates", r"user-isos", r"vms", r"vm-states", r"vm-metadata", @@ -49,7 +66,8 @@ READ_PATHS = ( r"(?:reinstall-jobs|rescue-jobs|vm-templates/import-jobs|rescue-templates/import-jobs|user-isos/import-jobs)/[A-Za-z0-9_-]+", ) WRITE_PATHS = { - "POST": (r"host/(?:scan|unlock|test|discover-vms)", r"vm-templates/import", r"rescue-templates", + "POST": (r"host/(?:scan|unlock|test|discover-vms)", + r"vm-templates/(?:import|upload)", r"rescue-templates", r"user-isos/import", r"vms", rf"vms/{VM_ID}/(?:actions|credential|reinstall|rescue|console-ticket)", rf"vms/{VM_ID}/(?:service|network)-(?:suspend|resume)", @@ -62,6 +80,63 @@ WRITE_PATHS = { } +class _CloudUploadWriter: + """Own a temporary upload descriptor used only from worker threads.""" + + def __init__(self, directory: Path) -> None: + descriptor, raw_path = tempfile.mkstemp( + prefix=".cloud-image-upload-", + suffix=".qcow2", + dir=directory, + ) + try: + os.fchmod(descriptor, 0o600) + except BaseException: + os.close(descriptor) + Path(raw_path).unlink(missing_ok=True) + raise + self.path = Path(raw_path) + self._descriptor = descriptor + self._lock = threading.Lock() + + def write(self, chunk: memoryview) -> None: + """Write the complete bounded chunk before accepting another one.""" + with self._lock: + if self._descriptor < 0: + raise OSError(errno.EBADF, "cloud upload is closed") + remaining = chunk + while remaining: + written = os.write(self._descriptor, remaining) + if written <= 0: + raise OSError(errno.EIO, "cloud upload write made no progress") + remaining = remaining[written:] + + def sync_and_close(self) -> None: + """Durably flush and close the descriptor before image validation.""" + with self._lock: + if self._descriptor < 0: + return + descriptor = self._descriptor + try: + os.fsync(descriptor) + finally: + try: + os.close(descriptor) + finally: + self._descriptor = -1 + + def close(self) -> None: + """Close after an interrupted request, waiting for an in-flight write.""" + with self._lock: + if self._descriptor < 0: + return + descriptor = self._descriptor + try: + os.close(descriptor) + finally: + self._descriptor = -1 + + class NoRedirects(HTTPRedirectHandler): def redirect_request(self, *args, **kwargs): raise ValueError("Session introspection redirects are not allowed") @@ -261,16 +336,28 @@ class EngineApplication: raise EngineAuthError("内部引擎仅接受本机服务请求", 403) headers = request_headers(scope) target = raw_target(scope) - upload = scope["method"] == "POST" and bool(re.fullmatch( - r"/internal/v1/hosts/[A-Za-z0-9_-]+/rescue-templates", scope["path"])) + upload_resource = None + if scope["method"] == "POST": + if scope["path"] == "/internal/v1/vm-templates/upload": + upload_resource = "vm-templates/upload" + elif re.fullmatch( + r"/internal/v1/hosts/[A-Za-z0-9_-]+/rescue-templates", + scope["path"]): + upload_resource = "rescue-templates" + upload = upload_resource is not None if upload: digest = headers.get("x-cloud-content-sha256", "") delegated = await asyncio.to_thread(authenticate, headers, self.secret, scope["method"], target, digest, self.store) - self._authorize_route(scope["method"], "rescue-templates", delegated) + self._authorize_route( + scope["method"], upload_resource, delegated) checksum = hashlib.sha256() validation = {"complete": False, "valid": False} - scope = {**scope, "engine_upload_validation": validation} + scope = { + **scope, + "engine_upload_validation": validation, + "engine_upload_digest": digest, + } async def verified_receive(): message = await receive() if message["type"] == "http.request": @@ -349,6 +436,12 @@ class EngineApplication: and re.fullmatch(r"/internal/v1/operations/[A-Za-z0-9_-]{8,80}", path) is not None and scope.get("query_string", b"") != b"reconcile=false" ) + # A browser upload can spend minutes waiting for request-body chunks. + # Its handler acquires the update coordination lock only to reserve and + # finally queue the import, so unrelated VM writes remain available. + if method == "POST" and path == "/internal/v1/vm-templates/upload": + await self._dispatch_regular(scope, receive, send, body, delegated) + return if method != "GET" or operation_reconcile: async with self._update_lock(): if await asyncio.to_thread(self.updater.active_job): @@ -387,13 +480,14 @@ class EngineApplication: await json_response(send, 200, {"status": "ok"}) return cloud_template_match = re.fullmatch( - r"/internal/v1/(vm-templates(?:/import|/[A-Za-z0-9_-]+|" + r"/internal/v1/(vm-templates(?:/(?:import|upload)|/[A-Za-z0-9_-]+|" r"/import-jobs/[A-Za-z0-9_-]+)?)", path) if cloud_template_match is not None: resource = cloud_template_match[1] self._authorize_route(scope["method"], resource, delegated) await self._cloud_template_dispatch( - scope["method"], resource, body, send) + scope["method"], resource, body, send, + scope=scope, receive=receive) return user_iso_match = re.fullmatch( r"/internal/v1/(user-isos(?:/import|/[0-9a-f]{32}|" @@ -531,8 +625,165 @@ class EngineApplication: finally: self.active_operations.discard(operation["operation_id"]) + @staticmethod + def _cloud_upload_metadata(scope) -> tuple[str, str, str]: + query = scope.get("query_string", b"") + if (not isinstance(query, bytes) or len(query) > 2048 + or re.search(rb"%(?![0-9A-Fa-f]{2})", query) is not None): + raise TemplateError("云镜像上传参数无效", 422) + try: + pairs = parse_qsl( + query.decode("ascii"), + keep_blank_values=True, + strict_parsing=True, + max_num_fields=3, + errors="strict", + ) + except (UnicodeDecodeError, ValueError) as exc: + raise TemplateError("云镜像上传参数无效", 422) from exc + if (len(pairs) != 3 + or {key for key, _value in pairs} + != {"name", "os_family", "os_version"}): + raise TemplateError("云镜像上传参数无效", 422) + values = dict(pairs) + name = validate_template_name(values["name"]) + os_family, os_version = validate_template_metadata( + values["os_family"], values["os_version"]) + return name, os_family, os_version + + @staticmethod + def _single_upload_header(scope, name: bytes) -> str | None: + values = [value.decode("latin-1") + for key, value in scope.get("headers", []) + if key.lower() == name] + if len(values) > 1: + raise TemplateError("云镜像上传请求头重复", 400) + return values[0] if values else None + + async def _upload_cloud_template(self, scope, receive, send) -> None: + runtime = self.registry.cloud_templates + name, os_family, os_version = self._cloud_upload_metadata(scope) + content_type = self._single_upload_header(scope, b"content-type") + if content_type is None or content_type.lower() != "application/octet-stream": + raise TemplateError( + "云镜像上传必须使用 application/octet-stream", 415) + content_encoding = self._single_upload_header(scope, b"content-encoding") + if content_encoding is not None and content_encoding.lower() != "identity": + raise TemplateError("云镜像上传内容不能压缩", 415) + + declared_raw = self._single_upload_header(scope, b"content-length") + if declared_raw is not None and re.fullmatch(r"[0-9]+", declared_raw) is None: + raise TemplateError("云镜像上传长度无效", 400) + declared = int(declared_raw) if declared_raw is not None else None + if declared is not None and declared < MIN_DOWNLOAD_BYTES: + raise TemplateError("云镜像文件不能小于 1 MiB", 422) + if declared is not None and declared > MAX_IMAGE_BYTES: + raise TemplateError("云镜像超过 4 GiB 限制", 413) + + async with self._update_lock(): + if await asyncio.to_thread(self.updater.active_job): + raise EngineAuthError( + "面板正在更新,暂时不能执行虚拟化写操作", 409) + reservation = await asyncio.to_thread( + runtime.reserve_upload, name, os_family, os_version) + reservation_id = reservation["reservation_id"] + maximum = reservation["max_bytes"] + if declared is not None and declared > maximum: + await asyncio.to_thread(runtime.cancel_upload, reservation_id) + raise TemplateError("面板云镜像持久目录空间不足", 507) + + image_path: Path | None = None + handed_off = False + writer: _CloudUploadWriter | None = None + try: + writer = await asyncio.to_thread( + _CloudUploadWriter, reservation["directory"]) + image_path = writer.path + checksum = hashlib.sha256() + size = 0 + deadline = time.monotonic() + MAX_DOWNLOAD_SECONDS + try: + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TemplateError("云镜像上传超时", 408) + try: + message = await asyncio.wait_for( + receive(), + timeout=min(CLOUD_UPLOAD_IDLE_SECONDS, remaining), + ) + except TimeoutError as exc: + raise TemplateError("云镜像上传超时", 408) from exc + if message["type"] == "http.disconnect": + raise TemplateError("云镜像上传已中断", 400) + if message["type"] != "http.request": + raise TemplateError("云镜像上传请求无效", 400) + chunk = message.get("body", b"") + size += len(chunk) + if declared is not None and size > declared: + raise TemplateError("云镜像上传内容超过声明长度", 400) + if size > maximum: + if maximum < MAX_IMAGE_BYTES: + raise TemplateError( + "面板云镜像持久目录空间不足", 507) + raise TemplateError("云镜像超过 4 GiB 限制", 413) + view = memoryview(chunk) + for offset in range(0, len(view), CLOUD_UPLOAD_WRITE_BYTES): + if time.monotonic() >= deadline: + raise TemplateError("云镜像上传超时", 408) + part = view[offset:offset + CLOUD_UPLOAD_WRITE_BYTES] + await asyncio.to_thread(writer.write, part) + checksum.update(part) + if not message.get("more_body", False): + break + await asyncio.to_thread(writer.sync_and_close) + except OSError as exc: + if exc.errno in {errno.ENOSPC, errno.EDQUOT}: + raise TemplateError( + "面板云镜像持久目录空间不足", 507) from exc + raise + if declared is not None and size != declared: + raise TemplateError("云镜像上传内容不完整", 400) + if size < MIN_DOWNLOAD_BYTES: + raise TemplateError("云镜像文件不能小于 1 MiB", 422) + validation = scope.get("engine_upload_validation", {}) + expected_digest = scope.get("engine_upload_digest", "") + digest = checksum.hexdigest() + if (not validation.get("complete") + or not validation.get("valid") + or not hmac.compare_digest(digest, expected_digest)): + raise EngineAuthError("云镜像摘要与签名不匹配", 401) + virtual_size = await asyncio.to_thread( + verify_qcow2_image, image_path) + # Recheck the update state at the commit boundary. An upload + # admitted before a service transition must not enqueue work + # against code that is being replaced. + async with self._update_lock(): + if await asyncio.to_thread(self.updater.active_job): + raise EngineAuthError( + "面板正在更新,暂时不能执行虚拟化写操作", 409) + result = await asyncio.to_thread( + runtime.queue_upload, + reservation_id, + image_path, + digest, + virtual_size, + ) + handed_off = True + await json_response(send, 202, result) + finally: + if not handed_off: + if writer is not None: + with suppress(OSError): + await asyncio.to_thread(writer.close) + if image_path is not None: + with suppress(OSError): + await asyncio.to_thread(image_path.unlink, missing_ok=True) + await asyncio.to_thread(runtime.cancel_upload, reservation_id) + async def _cloud_template_dispatch( - self, method: str, resource: str, body: bytes | None, send) -> None: + self, method: str, resource: str, body: bytes | None, send, + *, scope=None, receive=None) -> None: runtime = self.registry.cloud_templates try: if method == "GET" and resource == "vm-templates": @@ -557,6 +808,11 @@ class EngineApplication: runtime.queue_import, value.url, value.name, value.sha256) await json_response(send, 202, result) return + if method == "POST" and resource == "vm-templates/upload": + if scope is None or receive is None: + raise EngineAuthError("云镜像上传请求无效", 400) + await self._upload_cloud_template(scope, receive, send) + return job_match = re.fullmatch( r"vm-templates/import-jobs/([A-Za-z0-9_-]+)", resource) if method == "GET" and job_match is not None: diff --git a/web/app/Controllers/CloudAdminController.php b/web/app/Controllers/CloudAdminController.php index 2738f1c7..bf17717e 100644 --- a/web/app/Controllers/CloudAdminController.php +++ b/web/app/Controllers/CloudAdminController.php @@ -14,6 +14,14 @@ use App\Services\Cloud\CloudUserIsoFeature; final class CloudAdminController { private const HOST_COUNTRY_CODES = ['CN', 'JP', 'US', 'SG']; + private const CLOUD_IMAGE_UPLOAD_MAX_BYTES = 4 * 1024 * 1024 * 1024; + private const CLOUD_IMAGE_UPLOAD_MIN_BYTES = 1024 * 1024; + private const CLOUD_IMAGE_VERSIONS = [ + 'AlmaLinux' => ['8', '9', '10'], + 'Debian' => ['11', '12', '13'], + 'Ubuntu' => ['20.04', '22.04', '24.04', '25.04', '25.10', '26.04'], + 'CentOS Stream' => ['9', '10'], + ]; private function actor($request, bool $write = false): array { @@ -349,4 +357,109 @@ final class CloudAdminController unlink($temporary); } } + + public function cloudTemplateUpload($request, $response) + { + $actor = $this->actor($request, true); + $query = $request->getQueryParams(); + $name = $query['name'] ?? ''; + $family = $query['os_family'] ?? ''; + $version = $query['os_version'] ?? ''; + $filename = $query['filename'] ?? ''; + if (!is_string($name) || preg_match('//u', $name) !== 1 + || trim($name) === '' || mb_strlen(trim($name)) > 64 + || preg_match('/[\x00-\x1F\x7F]/u', $name) === 1) { + throw new CloudError('模板名称应为 1 至 64 个字符'); + } + if (!is_string($family) || !is_string($version) + || !isset(self::CLOUD_IMAGE_VERSIONS[$family]) + || !in_array($version, self::CLOUD_IMAGE_VERSIONS[$family], true)) { + throw new CloudError('云镜像系统版本无效'); + } + if (!is_string($filename) || strlen($filename) > 255 + || str_contains($filename, '/') || str_contains($filename, '\\') + || preg_match('/[\x00-\x1F\x7F]/', $filename) === 1 + || !str_ends_with(strtolower($filename), '.qcow2')) { + throw new CloudError('请选择 .qcow2 云镜像文件'); + } + if (strtolower(trim(explode(';', $request->getHeaderLine('Content-Type'))[0])) + !== 'application/octet-stream') { + throw new CloudError('请直接上传 QCOW2 文件内容'); + } + + $declaredSize = null; + $contentLength = trim($request->getHeaderLine('Content-Length')); + if ($contentLength !== '') { + if (preg_match('/^[0-9]+$/D', $contentLength) !== 1) { + throw new CloudError('云镜像上传长度无效'); + } + $declaredSize = (int) $contentLength; + if ($declaredSize < self::CLOUD_IMAGE_UPLOAD_MIN_BYTES) { + throw new CloudError('云镜像文件不能小于 1 MiB'); + } + if ($declaredSize > self::CLOUD_IMAGE_UPLOAD_MAX_BYTES) { + throw new CloudError('云镜像超过 4 GiB 限制', 413); + } + } + + $temporary = tempnam(sys_get_temp_dir(), 'cloud-template-'); + if ($temporary === false || !chmod($temporary, 0600)) { + if (is_string($temporary)) { + @unlink($temporary); + } + throw new CloudError('无法准备私有上传文件', 503); + } + $file = fopen($temporary, 'wb'); + try { + if ($file === false) { + throw new CloudError('无法准备私有上传文件', 503); + } + $bytes = 0; + $stream = $request->getBody(); + while (!$stream->eof()) { + $chunk = $stream->read(1048576); + if ($chunk === '' && !$stream->eof()) { + throw new CloudError('云镜像上传中断'); + } + $bytes += strlen($chunk); + if ($bytes > self::CLOUD_IMAGE_UPLOAD_MAX_BYTES) { + throw new CloudError('云镜像超过 4 GiB 限制', 413); + } + $length = strlen($chunk); + for ($offset = 0; $offset < $length;) { + $written = fwrite($file, substr($chunk, $offset)); + if ($written === false || $written === 0) { + throw new CloudError('无法保存上传文件', 503); + } + $offset += $written; + } + } + if ($bytes < self::CLOUD_IMAGE_UPLOAD_MIN_BYTES) { + throw new CloudError('云镜像文件不能小于 1 MiB'); + } + if ($declaredSize !== null && $bytes !== $declaredSize) { + throw new CloudError('云镜像上传内容不完整'); + } + if (!fflush($file) || (function_exists('fsync') && !fsync($file))) { + throw new CloudError('无法保存上传文件', 503); + } + fclose($file); + $file = null; + + $path = '/internal/v1/vm-templates/upload?name=' . rawurlencode(trim($name)) + . '&os_family=' . rawurlencode($family) + . '&os_version=' . rawurlencode($version); + $engine = new CloudEngineClient(); + $result = CloudTemplateDeletionService::queueImport( + static fn (): array => $engine->upload($path, $actor, $temporary), + 'upload' + ); + return $response->withJson($result, 202); + } finally { + if (is_resource($file)) { + fclose($file); + } + @unlink($temporary); + } + } } diff --git a/web/app/Models/CloudTemplateCatalogLock.php b/web/app/Models/CloudTemplateCatalogLock.php index e8c6638f..6704a13b 100644 --- a/web/app/Models/CloudTemplateCatalogLock.php +++ b/web/app/Models/CloudTemplateCatalogLock.php @@ -7,4 +7,5 @@ final class CloudTemplateCatalogLock extends Model protected $table = 'cloud_template_catalog_locks'; protected $primaryKey = 'id'; public $incrementing = false; + protected $hidden = ['lease_token']; } diff --git a/web/app/Services/Cloud/CloudEngineClient.php b/web/app/Services/Cloud/CloudEngineClient.php index eacdd715..086cbe4d 100644 --- a/web/app/Services/Cloud/CloudEngineClient.php +++ b/web/app/Services/Cloud/CloudEngineClient.php @@ -110,16 +110,22 @@ final class CloudEngineClient if ($path === null) { return null; } + $parts = explode('?', $path, 2); + $route = $parts[0]; + if (count($parts) === 2 + && $route !== '/internal/v1/vm-templates/upload') { + return null; + } $update = preg_match( - '#^/internal/v1/update/(?:settings|check|apply|jobs/[A-Za-z0-9_-]{16,64})$#D', $path + '#^/internal/v1/update/(?:settings|check|apply|jobs/[A-Za-z0-9_-]{16,64})$#D', $route ) === 1; $userIso = preg_match( '#^/internal/v1/user-isos(?:/import|/[0-9a-f]{32}|/import-jobs/[A-Za-z0-9_-]{8,80})?$#D', - $path + $route ) === 1; $vmTemplate = preg_match( - '#^/internal/v1/vm-templates(?:/import|/[A-Za-z0-9_-]{8,80}|/import-jobs/[A-Za-z0-9_-]{8,80})?$#D', - $path + '#^/internal/v1/vm-templates(?:/(?:import|upload)|/[A-Za-z0-9_-]{8,80}|/import-jobs/[A-Za-z0-9_-]{8,80})?$#D', + $route ) === 1; if (!$update && !$userIso && !$vmTemplate) { return null; @@ -161,6 +167,18 @@ final class CloudEngineClient '官方云镜像下载响应格式无效', '模板名称应为 1 至 64 个字符', '模板名称不能包含控制字符', '云镜像系统版本无效', '模板标识无效', '云镜像模板范围无效', + '云镜像上传参数无效', '云镜像系统或版本不受支持', + '云镜像上传请求头重复', + '云镜像上传必须使用 application/octet-stream', + '云镜像上传内容不能压缩', '云镜像上传长度无效', + '云镜像上传内容超过声明长度', '云镜像上传内容不完整', + '云镜像上传已中断', + '云镜像上传请求无效', '云镜像上传超时', + '云镜像文件不能小于 1 MiB', + '云镜像摘要与签名不匹配', + '上传文件不是有效的 qcow2 镜像', + '面板未安装 qemu-img,无法验证云镜像', + '云镜像完整性检查失败', '云镜像 SHA-256 格式无效', 'SHA-256 可留空;填写时必须为 64 位十六进制校验值', '云镜像导入任务不存在', '云镜像模板不存在', diff --git a/web/app/Services/Cloud/CloudTemplateDeletionService.php b/web/app/Services/Cloud/CloudTemplateDeletionService.php index 105da486..58452dd6 100644 --- a/web/app/Services/Cloud/CloudTemplateDeletionService.php +++ b/web/app/Services/Cloud/CloudTemplateDeletionService.php @@ -9,6 +9,12 @@ use App\Models\CloudTemplateRetirement; /** Serialize template use, import and retirement without trusting stale UI data. */ final class CloudTemplateDeletionService { + // The private engine allows at most one hour for an upload. The extra hour + // prevents a live transfer from losing its gate while still guaranteeing + // that a PHP crash cannot block catalog changes forever. + private const IMPORT_LEASE_SECONDS = 7200; + private const IMPORT_LEASE_KINDS = ['remote', 'upload']; + private const ACTIVE_STATUSES = [ 'awaiting_payment', 'queued', 'running', 'uncertain', ]; @@ -34,25 +40,26 @@ final class CloudTemplateDeletionService } } - /** - * Hold one durable catalog row until the engine has accepted the import. - * The engine queues downloads asynchronously, so this transaction never - * spans the large image transfer itself. - */ - public static function queueImport(callable $queue): array + /** Reserve the catalog, release the transaction, then contact the engine. */ + public static function queueImport( + callable $queue, string $leaseKind = 'remote' + ): array { - return (new CloudTemplateCatalogLock())->getConnection()->transaction( - static function () use ($queue): array { - self::lockCatalog(); - self::assertImportAllowed(); - $result = $queue(); - if (!is_array($result)) { - throw new CloudError('云镜像导入响应无效', 502); - } - return $result; - }, - 3 - ); + $token = self::reserveImport($leaseKind); + try { + $result = $queue(); + if (!is_array($result)) { + throw new CloudError('云镜像导入响应无效', 502); + } + } catch (\Throwable $error) { + self::clearImportLeaseSafely($token); + throw $error; + } + // The engine has durably accepted the download/upload job. A database + // outage here must not turn that success into an ambiguous client + // retry; the token-bound lease will also expire automatically. + self::clearImportLeaseSafely($token); + return $result; } /** Fetch, reconcile and optionally publish one full catalog atomically. */ @@ -88,7 +95,8 @@ final class CloudTemplateDeletionService } (new CloudTemplateRetirement())->getConnection()->transaction( function () use ($templateId, $templateCreatedAt): void { - self::lockCatalog(); + $catalogLock = self::lockCatalog(); + self::assertNoActiveImportLeaseLocked($catalogLock); $now = time(); CloudTemplateRetirement::query()->insertOrIgnore([ 'template_id' => $templateId, @@ -122,14 +130,16 @@ final class CloudTemplateDeletionService ); } - /** Called under the catalog lock by queueImport(). */ + /** Check both durable deletion and import gates under the catalog lock. */ public static function assertImportAllowed(): void { - if (CloudTemplateRetirement::where('state', 'deleting')->exists()) { - throw new CloudError( - '有云镜像删除尚未完成,请刷新目录确认后再导入', 409 - ); - } + (new CloudTemplateCatalogLock())->getConnection()->transaction( + static function (): void { + $catalogLock = self::lockCatalog(); + self::assertImportAllowedLocked($catalogLock); + }, + 3 + ); } /** Re-enable one stable ID only when a newer engine generation exists. */ @@ -317,7 +327,86 @@ final class CloudTemplateDeletionService ); } - private static function lockCatalog(): void + private static function reserveImport(string $leaseKind): string + { + if (!in_array($leaseKind, self::IMPORT_LEASE_KINDS, true)) { + throw new \InvalidArgumentException('云镜像导入预约类型无效'); + } + $token = bin2hex(random_bytes(16)); + return (new CloudTemplateCatalogLock())->getConnection()->transaction( + static function () use ($leaseKind, $token): string { + $catalogLock = self::lockCatalog(); + self::assertImportAllowedLocked($catalogLock); + $now = time(); + $catalogLock->lease_token = $token; + $catalogLock->lease_kind = $leaseKind; + $catalogLock->lease_until = $now + self::IMPORT_LEASE_SECONDS; + $catalogLock->updated_at = $now; + $catalogLock->save(); + return $token; + }, + 3 + ); + } + + private static function clearImportLeaseSafely(string $token): void + { + try { + (new CloudTemplateCatalogLock())->getConnection()->transaction( + static function () use ($token): void { + $catalogLock = self::lockCatalog(); + if (hash_equals( + (string) ($catalogLock->lease_token ?? ''), $token + )) { + $catalogLock->lease_token = null; + $catalogLock->lease_kind = null; + $catalogLock->lease_until = 0; + $catalogLock->updated_at = time(); + $catalogLock->save(); + } + }, + 3 + ); + } catch (\Throwable $error) { + error_log('Cloud template import lease cleanup failed: ' + . get_class($error)); + } + } + + private static function assertImportAllowedLocked( + CloudTemplateCatalogLock $catalogLock + ): void { + if (CloudTemplateRetirement::where('state', 'deleting')->exists()) { + throw new CloudError( + '有云镜像删除尚未完成,请刷新目录确认后再导入', 409 + ); + } + self::assertNoActiveImportLeaseLocked($catalogLock); + } + + private static function assertNoActiveImportLeaseLocked( + CloudTemplateCatalogLock $catalogLock + ): void { + $token = (string) ($catalogLock->lease_token ?? ''); + $leaseUntil = max(0, (int) ($catalogLock->lease_until ?? 0)); + if ($token !== '' && $leaseUntil > time()) { + throw new CloudError( + '已有云镜像导入正在进行,请等待完成后重试', 409 + ); + } + if ($token !== '' || $leaseUntil !== 0 + || ($catalogLock->lease_kind ?? null) !== null) { + // A crashed PHP worker leaves a durable gate. Once its generous + // deadline passes, the next serialized operation repairs it. + $catalogLock->lease_token = null; + $catalogLock->lease_kind = null; + $catalogLock->lease_until = 0; + $catalogLock->updated_at = time(); + $catalogLock->save(); + } + } + + private static function lockCatalog(): CloudTemplateCatalogLock { CloudTemplateCatalogLock::query()->insertOrIgnore([ 'id' => 1, 'updated_at' => 0, @@ -327,6 +416,7 @@ final class CloudTemplateDeletionService if ($lock === null) { throw new CloudError('无法锁定云镜像目录', 503); } + return $lock; } private static function assertNoReferences(string $templateId): void diff --git a/web/config/routes_cloud.php b/web/config/routes_cloud.php index 25935990..2a548fbe 100644 --- a/web/config/routes_cloud.php +++ b/web/config/routes_cloud.php @@ -58,6 +58,7 @@ $app->group('/api/v1', function () { $this->map($kind === 'cloud' ? ['PATCH', 'DELETE'] : ['DELETE'], '/admin/' . $name . '/{id}', 'App\Controllers\CloudAdminController:templates')->setArgument('kind', $kind); } $this->post('/admin/cloud-templates/import', 'App\Controllers\CloudAdminController:templates')->setArgument('kind', 'cloud'); + $this->post('/admin/cloud-templates/upload', 'App\Controllers\CloudAdminController:cloudTemplateUpload'); $this->post('/admin/rescue-templates/upload', 'App\Controllers\CloudAdminController:rescueUpload'); $this->map(['GET', 'PUT'], '/admin/update/settings', 'App\Controllers\AdminUpdateController:settings'); $this->post('/admin/update/check', 'App\Controllers\AdminUpdateController:check'); diff --git a/web/public/cloud/admin.css b/web/public/cloud/admin.css index b77ca8f6..118d6d76 100644 --- a/web/public/cloud/admin.css +++ b/web/public/cloud/admin.css @@ -1,6 +1,7 @@ /* Scoped to cloud admin page content. Never load client styles in venix. */ .cloud-admin { margin: 24px 0; } .cloud-admin .cloud-admin-grid { display: grid; grid-template-columns: minmax(240px, 1fr) minmax(320px, 1.5fr); gap: 24px; } +.cloud-admin .cloud-admin-imports { display: grid; gap: 24px; align-content: start; } .cloud-admin .cloud-admin-form-grid { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 16px; } .cloud-admin .cloud-admin-full { grid-column: 1 / -1; } .cloud-admin .cloud-admin-host { display: block; width: 100%; text-align: left; margin: 8px 0; } diff --git a/web/public/cloud/admin.js b/web/public/cloud/admin.js index cee50930..d398dbc1 100644 --- a/web/public/cloud/admin.js +++ b/web/public/cloud/admin.js @@ -14,6 +14,37 @@ async function api(path, options = {}) { if (response.status === 401) { location.assign('/auth/login'); throw new Error('登录已过期。'); } if (!response.ok || payload?.ret === 0) throw new Error(errorMessage(payload, response.status)); return payload; } +function uploadApi(path, file, onProgress) { + if (!canManage) return Promise.reject(new Error('当前管理员仅可查看,不能修改配置。')); + if (location.protocol !== 'https:' && !['localhost', '127.0.0.1', '[::1]'].includes(location.hostname)) return Promise.reject(new Error('请通过 HTTPS 上传云镜像。')); + return new Promise((resolve, reject) => { + const request = new XMLHttpRequest(); + request.open('POST', path, true); + request.withCredentials = true; + request.setRequestHeader('Accept', 'application/json'); + request.setRequestHeader('X-CSRF-Token', state.csrf); + request.setRequestHeader('Content-Type', 'application/octet-stream'); + request.upload.addEventListener('progress', event => { + if (event.lengthComputable && typeof onProgress === 'function') onProgress(event.loaded, event.total); + }); + request.addEventListener('load', () => { + let payload = null; + try { payload = request.responseText ? JSON.parse(request.responseText) : null; } + catch { + reject(new Error(request.status === 413 + ? '上传被 Web 服务器拒绝,请提高 PHP 与反代的请求体上限。' + : '服务返回非 JSON 响应,请检查 PHP 和反代配置。')); + return; + } + if (request.status === 401) { location.assign('/auth/login'); reject(new Error('登录已过期。')); return; } + if (request.status < 200 || request.status >= 300 || payload?.ret === 0) { reject(new Error(errorMessage(payload, request.status))); return; } + resolve(payload); + }); + request.addEventListener('error', () => reject(new Error('云镜像上传失败,请检查网络或反代限制。'))); + request.addEventListener('abort', () => reject(new Error('云镜像上传已取消。'))); + request.send(file); + }); +} function formList(value) { return String(value || '').split(/[\s,]+/).filter(Boolean); } function profile(host) { return { ...host, ...(host.profile || host.config || {}) }; } function orderStatus(value) { @@ -106,8 +137,9 @@ async function loadCatalog() { row.append(name, save, remove); row.append(element('small', 'cloud-admin-meta', `${entry.os_family || '镜像'} ${entry.os_version || ''} · ${imageBytes > 0 ? (imageBytes / 1024 ** 3).toFixed(2) + ' GiB' : entry.id}`)); if (rescue) { name.readOnly = true; save.hidden = true; save.disabled = true; } - row.addEventListener('submit', async event => { event.preventDefault(); if (rescue) return; save.disabled = true; try { await api(`${catalogPath()}/${encodeURIComponent(entry.id)}`, { method: 'PATCH', body: { name: name.value.trim() } }); notice('名称已保存。'); } catch (error) { notice(error.message, true); } finally { save.disabled = !canManage; } }); + row.addEventListener('submit', async event => { event.preventDefault(); if (rescue) return; if (state.busy) return notice('请等待当前导入任务完成。', true); save.disabled = true; try { await api(`${catalogPath()}/${encodeURIComponent(entry.id)}`, { method: 'PATCH', body: { name: name.value.trim() } }); notice('名称已保存。'); } catch (error) { notice(error.message, true); } finally { save.disabled = !canManage; } }); remove.addEventListener('click', async () => { + if (state.busy) return notice('请等待当前导入任务完成。', true); const message = rescue ? `删除所选宿主机上的模版“${entry.name}”?正在使用的模版可能不能删除。` : `删除全局云镜像模版“${entry.name}”?删除后不能再用于新建或重装;已创建实例不受影响。`; @@ -149,6 +181,46 @@ async function importTemplate(event) { try { const result = await api('/api/v1/admin/cloud-templates/import', { method: 'POST', body }); await importProgress(result); await loadCatalog(); notice('镜像导入已完成。'); } catch (error) { notice(error.message, true); } finally { state.busy = false; button.disabled = !canManage; } } +async function uploadCloudTemplate(event) { + event.preventDefault(); + if (state.busy) return; + const name = byId('template-upload-name').value.trim(); + const selection = byId('template-upload-system').value; + const separator = selection.indexOf('|'); + const osFamily = separator > 0 ? selection.slice(0, separator) : ''; + const osVersion = separator > 0 ? selection.slice(separator + 1) : ''; + const file = byId('template-upload-file').files?.[0]; + if (!name || name.length > 64) return notice('请填写 1 至 64 个字符的模版名称。', true); + if (!osFamily || !osVersion) return notice('请选择镜像实际对应的系统和版本。', true); + if (!file || !file.name.toLowerCase().endsWith('.qcow2')) return notice('请选择扩展名为 .qcow2 的云镜像文件。', true); + if (file.size < 1024 ** 2 || file.size > 4 * 1024 ** 3) return notice('QCOW2 文件必须在 1 MiB 至 4 GiB 之间。', true); + const button = byId('template-upload-button'); + const system = byId('template-upload-system'); + const input = byId('template-upload-file'); + state.busy = true; + button.disabled = true; + system.disabled = true; + input.disabled = true; + notice('正在上传云镜像,请勿关闭页面…'); + const query = new URLSearchParams({ name, os_family: osFamily, os_version: osVersion, filename: file.name }); + try { + const result = await uploadApi(`/api/v1/admin/cloud-templates/upload?${query}`, file, (loaded, total) => { + const percent = total > 0 ? Math.min(100, loaded * 100 / total) : 0; + notice(percent >= 100 ? '文件已上传至面板,正在校验并保存云镜像…' : `正在上传云镜像 ${percent.toFixed(1)}%,请勿关闭页面…`); + }); + await importProgress(result); + input.value = ''; + await loadCatalog(); + notice('本地云镜像上传并导入完成。'); + } catch (error) { + notice(error.message, true); + } finally { + state.busy = false; + button.disabled = !canManage; + system.disabled = !canManage; + input.disabled = !canManage; + } +} async function uploadRescue(event) { event.preventDefault(); if (state.busy) return; const hostId = byId('catalog-host').value; const file = byId('rescue-file').files?.[0]; if (!hostId || !file) return notice('请选择宿主机和 ISO。', true); if (!file.name.toLowerCase().endsWith('.iso') || file.size > 8 * 1024 ** 3 || file.size < 32768) return notice('请选择小于 8 GiB 的有效 ISO 文件。', true); const button = byId('rescue-upload-button'); state.busy = true; button.disabled = true; byId('catalog-host').disabled = true; notice('正在上传救援 ISO,请勿关闭页面…'); @@ -182,7 +254,7 @@ async function initialize() { if (!root) return; try { const session = await api('/api/v1/session'); state.csrf = session.csrf_token || state.csrf; if (!session.admin) throw new Error('需要管理员权限。'); if (!state.csrf) throw new Error('会话令牌缺失,请重新登录。'); if (root.dataset.cloudAdmin === 'hosts') { await Promise.all([loadHosts(), loadUserIsoSettings()]); renderHosts(); fillHost(null); byId('host-new').addEventListener('click', () => fillHost(null)); byId('host-scan').addEventListener('click', scanHost); byId('host-form').addEventListener('submit', saveHost); byId('user-iso-settings-form').addEventListener('submit', saveUserIsoSettings); for (const id of ['host-address', 'host-port']) byId(id).addEventListener('input', resetScan); if (!canManage) { byId('host-form').querySelectorAll('input,textarea,button').forEach(input => { input.disabled = true; }); byId('user-iso-settings-form').querySelectorAll('input,button').forEach(input => { input.disabled = true; }); } notice(canManage ? '请选择宿主机编辑,或添加新的宿主机。' : '当前管理员仅可查看配置。'); } - else if (root.dataset.cloudAdmin === 'templates') { byId('template-import')?.addEventListener('submit', importTemplate); await loadCatalog(); } + else if (root.dataset.cloudAdmin === 'templates') { byId('template-import')?.addEventListener('submit', importTemplate); byId('template-upload')?.addEventListener('submit', uploadCloudTemplate); await loadCatalog(); } else if (root.dataset.cloudAdmin === 'rescue') { await loadHosts(); const select = byId('catalog-host'); select.replaceChildren(); for (const host of state.hosts) { const option = element('option', '', `${host.name} · ${host.region || '地区未设置'}`); option.value = String(host.id); select.append(option); } select.addEventListener('change', loadCatalog); byId('rescue-upload')?.addEventListener('submit', uploadRescue); await loadCatalog(); } else { byId('admin-records-refresh').addEventListener('click', records); await records(); } } catch (error) { notice(error.message, true); } diff --git a/web/resources/views/venix/admin/cloud/templates.tpl b/web/resources/views/venix/admin/cloud/templates.tpl index 9929d0dd..0fb1eb9d 100644 --- a/web/resources/views/venix/admin/cloud/templates.tpl +++ b/web/resources/views/venix/admin/cloud/templates.tpl @@ -1,5 +1,46 @@ {assign var=admin_title value='云镜像模版'}{assign var=admin_active value='cloud_templates'}{include file='admin/static_header.tpl'} -

云镜像模版

这里维护全局云镜像目录,普通用户在所有可售地区看到同一份镜像列表。宿主机副本由引擎按需准备。删除会同时退役全局源镜像并异步清理宿主机缓存;已创建实例使用独立克隆磁盘,不受影响。当前支持 AlmaLinux 8/9/10、Debian 11/12/13、CentOS Stream 9/10,以及 Ubuntu 20.04/22.04/24.04/25.04/25.10/26.04 官方 x86_64 cloud-init 云镜像。升级前按宿主机导入的旧镜像不会自动进入全局目录,请重新导入一次。

正在加载…

-
导入云镜像

请粘贴受支持发行版官方镜像站的 HTTPS 文件地址,不能是下载介绍页;引擎只会跟随由官方入口返回且通过公网地址安全校验的跳转链。文件名应为 .qcow2;Ubuntu 官方 .img 也可使用,但其实际内容必须是 qcow2。

留空时面板会计算并保存实际 SHA-256,不与外部校验值比对;填写后则会在导入前核对。

已保存的模版
+
+

云镜像模版

+

这里维护全局云镜像目录,普通用户在所有可售地区看到同一份镜像列表。宿主机副本由引擎按需准备。删除会同时退役全局源镜像并异步清理宿主机缓存;已创建实例使用独立克隆磁盘,不受影响。当前支持 AlmaLinux 8/9/10、Debian 11/12/13、CentOS Stream 9/10,以及 Ubuntu 20.04/22.04/24.04/25.04/25.10/26.04 x86_64 cloud-init 云镜像。升级前按宿主机导入的旧镜像不会自动进入全局目录,请重新导入一次。

+

正在加载…

+
+
+
+
+
从官方地址下载
+ + + + +

请粘贴受支持发行版官方镜像站的 HTTPS 文件地址,不能是下载介绍页;引擎只会跟随由官方入口返回且通过公网地址安全校验的跳转链。文件名应为 .qcow2;Ubuntu 官方 .img 也可使用,但其实际内容必须是 qcow2。

+ + +

留空时面板会计算并保存实际 SHA-256,不与外部校验值比对;填写后则会在导入前核对。

+ +
+
+
+
+
从本机上传 QCOW2
+ + + + + + +

只接受独立、未加密且不含 backing file 的 QCOW2,面板会自动计算并保存 SHA-256。系统及版本用于用户部署页分类,请务必与镜像内容一致。上传期间请勿关闭页面;大小还受 PHP、Web 服务器及反代配置限制。

+ +
+
+
+
已保存的模版
+
+
+ {include file='admin/static_footer.tpl'} diff --git a/web/sql/cloud.sql b/web/sql/cloud.sql index 14d8aeb1..7335cc37 100644 --- a/web/sql/cloud.sql +++ b/web/sql/cloud.sql @@ -21,8 +21,17 @@ ALTER TABLE cloud_template_retirements ADD COLUMN IF NOT EXISTS template_created_at BIGINT UNSIGNED NOT NULL DEFAULT 0 AFTER state; CREATE TABLE IF NOT EXISTS cloud_template_catalog_locks ( id TINYINT UNSIGNED NOT NULL PRIMARY KEY, - updated_at BIGINT UNSIGNED NOT NULL + updated_at BIGINT UNSIGNED NOT NULL, + lease_token CHAR(32) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL, + lease_kind VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL, + lease_until BIGINT UNSIGNED NOT NULL DEFAULT 0 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +ALTER TABLE cloud_template_catalog_locks ADD COLUMN IF NOT EXISTS + lease_token CHAR(32) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL AFTER updated_at; +ALTER TABLE cloud_template_catalog_locks ADD COLUMN IF NOT EXISTS + lease_kind VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL AFTER lease_token; +ALTER TABLE cloud_template_catalog_locks ADD COLUMN IF NOT EXISTS + lease_until BIGINT UNSIGNED NOT NULL DEFAULT 0 AFTER lease_kind; INSERT IGNORE INTO cloud_template_catalog_locks (id, updated_at) VALUES (1, 0); ALTER TABLE cloud_hosts ADD COLUMN IF NOT EXISTS country_code CHAR(2) NOT NULL DEFAULT '' AFTER region; diff --git a/web/tests/CloudFinancialWorkflowTest.php b/web/tests/CloudFinancialWorkflowTest.php index dde22238..28245266 100644 --- a/web/tests/CloudFinancialWorkflowTest.php +++ b/web/tests/CloudFinancialWorkflowTest.php @@ -56,7 +56,7 @@ namespace { $schema->create('shop', function (Blueprint $t) { $t->increments('id'); $t->integer('host_id'); $t->string('name'); $t->string('price'); foreach (['cpu_cores', 'memory_mib', 'bandwidth_mbps', 'disk_gib', 'validity_days', 'traffic_quota_bytes', 'status', 'stock', 'sort'] as $v) { $t->integer($v)->default(0); } }); $schema->create('cloud_hosts', function (Blueprint $t) { $t->increments('id'); $t->string('name'); $t->string('region')->default(''); $t->integer('enabled'); $t->string('default_network'); }); $schema->create('cloud_template_retirements', function (Blueprint $t) { $t->string('template_id')->primary(); $t->string('state')->default('deleting'); $t->integer('template_created_at')->default(0); $t->integer('created_at'); $t->integer('updated_at'); }); - $schema->create('cloud_template_catalog_locks', function (Blueprint $t) { $t->integer('id')->primary(); $t->integer('updated_at'); }); + $schema->create('cloud_template_catalog_locks', function (Blueprint $t) { $t->integer('id')->primary(); $t->integer('updated_at'); $t->string('lease_token')->nullable(); $t->string('lease_kind')->nullable(); $t->integer('lease_until')->default(0); }); $schema->create('cloud_orders', function (Blueprint $t) { $t->increments('id'); foreach (['user_id', 'plan_id', 'host_id'] as $v) { $t->integer($v); } $t->string('idempotency_key'); $t->string('request_hash'); $t->string('amount'); $t->string('status'); $t->text('spec_json'); $t->text('payload_cipher')->nullable(); $t->integer('instance_id')->nullable(); $t->integer('refunded_at')->nullable(); $t->integer('stock_reserved_at')->nullable(); $t->integer('stock_released_at')->nullable(); $t->integer('created_at'); $t->integer('updated_at'); $t->unique(['user_id', 'idempotency_key']); }); $schema->create('cloud_jobs', function (Blueprint $t) { $t->increments('id'); $t->string('operation_id')->unique(); foreach (['order_id', 'instance_id'] as $v) { $t->integer($v)->nullable(); } foreach (['user_id', 'host_id'] as $v) { $t->integer($v); } $t->string('kind'); $t->string('status'); $t->text('payload_cipher'); $t->integer('attempts')->default(0); $t->integer('dispatched_at')->nullable(); $t->string('lease_token')->nullable(); $t->integer('lease_until')->default(0); $t->integer('available_at'); $t->string('error_code')->nullable(); $t->text('result_json')->nullable(); $t->integer('created_at'); $t->integer('updated_at'); $t->unique('order_id'); }); $schema->create('cloud_instances', function (Blueprint $t) { $t->increments('id'); foreach (['host_id', 'user_id'] as $v) { $t->integer($v); } foreach (['order_id', 'plan_id', 'current_period_id', 'expires_at', 'suspended_at', 'network_suspended_at'] as $v) { $t->integer($v)->nullable(); } foreach (['renewal_days', 'renewal_traffic_bytes', 'traffic_quota_bytes', 'traffic_download_bytes', 'traffic_upload_bytes', 'lifetime_download_bytes', 'lifetime_upload_bytes', 'traffic_synced_at'] as $v) { $t->integer($v)->default(0); } $t->string('renewal_price')->nullable(); $t->string('uuid'); $t->string('name'); $t->string('status'); $t->string('service_status')->default('exempt'); $t->string('network_status')->default('active'); $t->text('spec_json'); $t->text('credential_cipher')->nullable(); $t->integer('created_at'); $t->integer('updated_at'); $t->unique('order_id'); $t->unique(['host_id', 'uuid']); }); @@ -828,6 +828,73 @@ namespace { } CloudTemplateDeletionService::complete($retiredTemplate, 1); CloudTemplateDeletionService::assertImportAllowed(); + $leaseObserved = false; + $importResult = CloudTemplateDeletionService::queueImport( + function () use (&$leaseObserved, $retiredTemplate, $assert): array { + $lease = \App\Models\CloudTemplateCatalogLock::find(1); + $leaseObserved = $lease !== null + && preg_match('/^[a-f0-9]{32}$/D', (string) $lease->lease_token) === 1 + && (string) $lease->lease_kind === 'upload' + && (int) $lease->lease_until > time() + && $lease->getConnection()->transactionLevel() === 0; + try { + CloudTemplateDeletionService::queueImport( + static fn (): array => ['queued' => true] + ); + throw new RuntimeException( + 'A concurrent cloud-template import bypassed the lease.' + ); + } catch (CloudError $error) { + $assert($error->httpStatus === 409, + 'A concurrent import lease returned the wrong status.'); + } + try { + CloudTemplateDeletionService::begin($retiredTemplate, 1); + throw new RuntimeException( + 'Template deletion bypassed an active import lease.' + ); + } catch (CloudError $error) { + $assert($error->httpStatus === 409, + 'An import-gated deletion returned the wrong status.'); + } + return ['queued' => true]; + }, + 'upload' + ); + $clearedLease = \App\Models\CloudTemplateCatalogLock::find(1); + $assert($leaseObserved && ($importResult['queued'] ?? false) === true + && $clearedLease !== null && $clearedLease->lease_token === null + && $clearedLease->lease_kind === null + && (int) $clearedLease->lease_until === 0, + 'Cloud-template import did not use and clear its durable lease.'); + try { + CloudTemplateDeletionService::queueImport( + static function (): array { + throw new CloudError('synthetic import failure', 502); + }, + 'upload' + ); + throw new RuntimeException('A synthetic import failure was swallowed.'); + } catch (CloudError $error) { + $failedLease = \App\Models\CloudTemplateCatalogLock::find(1); + $assert($error->getMessage() === 'synthetic import failure' + && $failedLease !== null && $failedLease->lease_token === null + && (int) $failedLease->lease_until === 0, + 'A failed import did not release its durable lease.'); + } + \App\Models\CloudTemplateCatalogLock::where('id', 1)->update([ + 'lease_token' => str_repeat('a', 32), + 'lease_kind' => 'upload', + 'lease_until' => time() - 1, + ]); + CloudTemplateDeletionService::queueImport( + static fn (): array => ['queued' => true] + ); + $recoveredLease = \App\Models\CloudTemplateCatalogLock::find(1); + $assert($recoveredLease !== null && $recoveredLease->lease_token === null + && $recoveredLease->lease_kind === null + && (int) $recoveredLease->lease_until === 0, + 'An expired import lease was not recovered after a PHP crash.'); CloudTemplateDeletionService::begin($retiredTemplate, 1); $assert(CloudTemplateRetirement::find($retiredTemplate)->state === 'deleting', 'A retry did not reopen the durable engine deletion request.'); diff --git a/web/tests/CloudTemplateErrorContractTest.php b/web/tests/CloudTemplateErrorContractTest.php index 209deba6..7deb3520 100644 --- a/web/tests/CloudTemplateErrorContractTest.php +++ b/web/tests/CloudTemplateErrorContractTest.php @@ -86,6 +86,8 @@ $reviewedDownloadErrors = [ '官方云镜像下载服务器返回异常状态', '官方云镜像下载响应格式无效', '云镜像下载内容不完整', + '面板未安装 qemu-img,无法验证云镜像', + '云镜像完整性检查失败', ]; foreach ($reviewedDownloadErrors as $reviewed) { $assert(CloudEngineClient::safeVmTemplateMessage($reviewed) === $reviewed, @@ -108,9 +110,14 @@ $assert($allowlisted->invoke(null, '/internal/v1/vm-templates/import', ['error' $assert($allowlisted->invoke(null, '/internal/v1/vm-templates/import-jobs/12345678', ['detail' => '云镜像导入任务不存在']) === '云镜像导入任务不存在', 'The fixed import-job path did not pass its reviewed error.'); +$uploadMessage = '上传文件不是有效的 qcow2 镜像'; +$uploadPath = '/internal/v1/vm-templates/upload?name=Debian%2012&os_family=Debian&os_version=12'; +$assert($allowlisted->invoke(null, $uploadPath, ['error' => $uploadMessage]) + === $uploadMessage, 'The fixed cloud-template upload path did not pass its reviewed error.'); foreach ([ ['/internal/v1/vm-templates/import', ['error' => 'Traceback: /root/private/key']], + ['/internal/v1/vm-templates/import?unexpected=true', ['error' => $publicMessage]], ['/internal/v1/vm-templates/import/extra', ['error' => $publicMessage]], ['/internal/v1/hosts/1/vm-templates/import', ['error' => $publicMessage]], ['/internal/v1/vm-templates/import', ['error' => str_repeat('x', 501)]], diff --git a/web/tests/VirtualizationAssetsTest.php b/web/tests/VirtualizationAssetsTest.php index 425d3c4b..bcbdff36 100644 --- a/web/tests/VirtualizationAssetsTest.php +++ b/web/tests/VirtualizationAssetsTest.php @@ -510,6 +510,34 @@ $assert(str_contains($importScript, 'if (sha256) body.sha256 = sha256;') && !str_contains($importScript, 'host_id') && !str_contains($importScript, 'catalog-host'), 'Global cloud image imports must omit blank SHA-256 and all host selectors.'); +$uploadStart = strpos($adminScript, 'async function uploadCloudTemplate('); +$uploadEnd = $uploadStart === false ? false + : strpos($adminScript, 'async function uploadRescue(', $uploadStart); +$uploadScript = $uploadStart === false || $uploadEnd === false ? '' + : substr($adminScript, $uploadStart, $uploadEnd - $uploadStart); +$assert(str_contains($templateAdmin, 'id="template-upload"') + && str_contains($templateAdmin, 'id="template-upload-system"') + && str_contains($templateAdmin, 'id="template-upload-file"') + && str_contains($templateAdmin, 'accept=".qcow2,application/octet-stream"') + && str_contains($templateAdmin, '不含 backing file') + && str_contains($templateAdmin, '最大 4 GiB'), + 'Cloud template admin must offer a constrained local QCOW2 upload form.'); +$assert(str_contains($uploadScript, "file.name.toLowerCase().endsWith('.qcow2')") + && str_contains($uploadScript, '4 * 1024 ** 3') + && str_contains($uploadScript, "os_family: osFamily") + && str_contains($uploadScript, "os_version: osVersion") + && str_contains($uploadScript, '/api/v1/admin/cloud-templates/upload?') + && str_contains($adminScript, "request.setRequestHeader('Content-Type', 'application/octet-stream')") + && str_contains($adminScript, "request.upload.addEventListener('progress'"), + 'Browser QCOW2 upload must stream raw bytes with explicit metadata and progress.'); +$assert(str_contains($cloudRoutes, + "'/admin/cloud-templates/upload', 'App\\Controllers\\CloudAdminController:cloudTemplateUpload'") + && str_contains($cloudAdminController, 'public function cloudTemplateUpload(') + && str_contains($cloudAdminController, "'/internal/v1/vm-templates/upload?name='") + && str_contains($cloudAdminController, 'CLOUD_IMAGE_UPLOAD_MAX_BYTES') + && str_contains($cloudAdminController, 'CloudTemplateDeletionService::queueImport(') + && str_contains($cloudAdminController, "'upload'"), + 'PHP must stream administrator QCOW2 uploads through the signed private-engine path.'); $globalAdminStart = strpos($cloudAdminController, "if (\$kind === 'cloud')"); $globalAdminEnd = $globalAdminStart === false ? false : strpos($cloudAdminController, '// Rescue ISOs remain', $globalAdminStart); @@ -537,11 +565,18 @@ $assert(str_contains($cloudRoutes, "['PATCH', 'DELETE']") 'Global cloud templates must expose a confirmed, retryable admin deletion flow.'); $assert(str_contains($cloudSchema, 'CREATE TABLE IF NOT EXISTS cloud_template_retirements') && str_contains($cloudSchema, 'CREATE TABLE IF NOT EXISTS cloud_template_catalog_locks') + && str_contains($cloudSchema, 'lease_token CHAR(32)') + && str_contains($cloudSchema, 'lease_kind VARCHAR(16)') + && str_contains($cloudSchema, 'lease_until BIGINT UNSIGNED') && str_contains($cloudSchema, 'template_created_at BIGINT UNSIGNED') && str_contains($templateDeletionService, "'awaiting_payment', 'queued', 'running', 'uncertain'") && str_contains($templateDeletionService, "whereIn('kind', ['create', 'reinstall'])") && str_contains($templateDeletionService, 'lockForUpdate()') && str_contains($templateDeletionService, "where('state', 'deleting')->exists()") + && str_contains($templateDeletionService, 'IMPORT_LEASE_SECONDS') + && str_contains($templateDeletionService, 'reserveImport(') + && str_contains($templateDeletionService, 'clearImportLeaseSafely(') + && str_contains($templateDeletionService, 'assertNoActiveImportLeaseLocked(') && str_contains($templateDeletionService, 'reconcileAuthoritativeCatalog') && !str_contains($templateDeletionService, 'array_slice($templates') && str_contains($deploymentPageService, 'availableTemplates(')