"""Global cloud-template catalog, source file and import job tests.""" from __future__ import annotations import hashlib import json import os import stat import threading import time from pathlib import Path from types import SimpleNamespace from unittest.mock import patch import pytest from virsh_panel.cloud_template_runtime import ( CACHE_TOMBSTONE_KIND, CATALOG_KIND, IMPORT_JOB_KIND, CloudTemplateRuntime, GlobalCloudTemplateFiles, GlobalCloudTemplateStore, PERSISTENT_FREE_RESERVE_BYTES, cloud_template_id_for, ) from virsh_panel.cloud_templates import GIB, TemplateError, template_volume_name ALMA_10_URL = ( "https://repo.almalinux.org/almalinux/10/cloud/x86_64/images/" "AlmaLinux-10-GenericCloud-latest.x86_64.qcow2" ) ALMA_8_URL = ( "https://repo.almalinux.org/almalinux/8/cloud/x86_64/images/" "AlmaLinux-8-GenericCloud-latest.x86_64.qcow2" ) DEBIAN_12_URL = ( "https://cloud.debian.org/images/cloud/bookworm/latest/" "debian-12-generic-amd64.qcow2" ) UBUNTU_24_URL = ( "https://cloud-images.ubuntu.com/releases/24.04/release/" "ubuntu-24.04-server-cloudimg-amd64.img" ) CENTOS_STREAM_10_URL = ( "https://cloud.centos.org/centos/10-stream/x86_64/images/" "CentOS-Stream-GenericCloud-10-latest.x86_64.qcow2" ) class MemoryStore: def __init__(self) -> None: self.values: dict[tuple[str, str], str] = {} self.lock = threading.RLock() def list(self, kind: str) -> dict[str, str]: with self.lock: return { key: value for (record_kind, key), value in self.values.items() if record_kind == kind } def get(self, kind: str, key: str) -> str | None: with self.lock: return self.values.get((kind, key)) def set(self, kind: str, key: str, value: str) -> None: with self.lock: self.values[(kind, key)] = value def insert(self, kind: str, key: str, value: str) -> bool: with self.lock: identity = (kind, key) if identity in self.values: return False self.values[identity] = value return True def delete(self, kind: str, key: str) -> None: with self.lock: self.values.pop((kind, key), None) class CommitThenRaiseCatalogStore(MemoryStore): """Simulate a committed INSERT whose acknowledgement is lost.""" def insert(self, kind: str, key: str, value: str) -> bool: inserted = super().insert(kind, key, value) if kind == CATALOG_KIND and inserted: raise RuntimeError("catalog insert acknowledgement lost") return inserted class AmbiguousCatalogCommitStore(CommitThenRaiseCatalogStore): """Also fail the first catalog read after the ambiguous INSERT.""" def __init__(self) -> None: super().__init__() self.fail_next_catalog_read = False def list(self, kind: str) -> dict[str, str]: if kind == CATALOG_KIND and self.fail_next_catalog_read: self.fail_next_catalog_read = False raise RuntimeError("catalog read unavailable") return super().list(kind) def insert(self, kind: str, key: str, value: str) -> bool: try: return super().insert(kind, key, value) except RuntimeError: self.fail_next_catalog_read = True raise class LoseFirstCompletedJobWriteStore(MemoryStore): """Drop one terminal job update while retaining its running record.""" def __init__(self) -> None: super().__init__() self.lost_completed_write = False def set(self, kind: str, key: str, value: str) -> None: if kind == IMPORT_JOB_KIND: job = json.loads(value) if (job.get("status") == "completed" and not self.lost_completed_write): self.lost_completed_write = True raise RuntimeError("completed job write acknowledgement lost") super().set(kind, key, value) def wait_for_job(runtime: CloudTemplateRuntime, job_id: str) -> dict: for _ in range(200): job = runtime.get_job(job_id) if job["status"] in {"completed", "failed"}: return job time.sleep(0.01) raise AssertionError("cloud template import did not finish") def fake_download_bytes(payload: bytes): digest = hashlib.sha256(payload).hexdigest() def download(_url: str, expected_sha256: str, directory: Path, **_kwargs): assert expected_sha256 in {"", digest} source = Path(directory) / ".cloud-image-test.qcow2" source.write_bytes(payload) os.chmod(source, 0o600) return source, 2 * GIB, digest return digest, download def record_for(payload: bytes, *, name: str = "AlmaLinux 10") -> dict: digest = hashlib.sha256(payload).hexdigest() template_id = cloud_template_id_for(digest, "10") return { "id": template_id, "name": name, "os_family": "AlmaLinux", "os_version": "10", "sha256": digest, "volume_name": template_volume_name(template_id), "virtual_size": 2 * GIB, "image_bytes": len(payload), "created_at": 1, } def test_stable_global_id_and_catalog_use_an_independent_record_kind(): digest = "a" * 64 first = cloud_template_id_for(digest, "10") second = cloud_template_id_for(digest.upper(), "10") assert first == second assert first.startswith("alma10-") assert template_volume_name(first).endswith(".qcow2") store = MemoryStore() catalog = GlobalCloudTemplateStore(store) record = record_for(b"catalog-image") catalog.add(record) assert catalog.get(record["id"]) == record assert list(store.list(CATALOG_KIND)) == [record["id"]] assert store.list("template") == {} @pytest.mark.parametrize( "message", [ "官方云镜像地址无法解析", "官方云镜像地址解析结果无效", "官方云镜像地址解析到非公网地址", "无法连接官方云镜像下载地址", "官方云镜像重定向目标不安全", "官方云镜像重定向次数过多", "官方云镜像下载服务器返回异常状态", "官方云镜像下载响应格式无效", "云镜像下载内容不完整", ], ) def test_public_job_preserves_fixed_safe_download_errors(message): job = { "job_id": "safe-error", "status": "failed", "phase": "导入失败", "created_at": 1, "error": message, } assert CloudTemplateRuntime.public_job(job)["error"] == message def test_import_publishes_private_source_and_restores_global_catalog(tmp_path): store = MemoryStore() runtime = CloudTemplateRuntime(store, tmp_path) payload = b"verified-global-cloud-image" digest, downloader = fake_download_bytes(payload) try: with patch( "virsh_panel.cloud_template_runtime.download_verified_image", side_effect=downloader, ): queued = runtime.queue_import( ALMA_10_URL, "AlmaLinux 10", digest ) job = wait_for_job(runtime, queued["job_id"]) assert job["status"] == "completed", job template_id = job["template"]["id"] assert template_id == cloud_template_id_for(digest, "10") assert "volume_name" not in job["template"] listed = runtime.list() assert listed == [job["template"]] assert "active_import_job" not in runtime.status() root = tmp_path / "cloud-templates" source = root / template_volume_name(template_id) assert source.read_bytes() == payload assert stat.S_IMODE(root.stat().st_mode) == 0o700 assert stat.S_IMODE(source.stat().st_mode) == 0o600 assert store.get(IMPORT_JOB_KIND, queued["job_id"]) is not None # A new runtime reads the same host-independent catalog and verifies # the persisted source from disk instead of trusting process memory. restored = CloudTemplateRuntime(store, tmp_path) try: restored.restore_jobs() assert restored.list() == listed with restored.source(template_id) as (record, path): assert record["sha256"] == digest assert path == source assert restored.in_use[template_id] == 1 assert template_id not in restored.in_use finally: restored.close() finally: runtime.close() @pytest.mark.parametrize( ("url", "family", "version", "prefix"), [ (ALMA_8_URL, "AlmaLinux", "8", "alma8-"), (DEBIAN_12_URL, "Debian", "12", "debian12-"), (UBUNTU_24_URL, "Ubuntu", "24.04", "ubuntu2404-"), (CENTOS_STREAM_10_URL, "CentOS Stream", "10", "centosstream10-"), ], ) def test_import_derives_family_version_and_safe_source_name( tmp_path, url, family, version, prefix): runtime = CloudTemplateRuntime(MemoryStore(), tmp_path) payload = f"{family}-{version}-cloud-image".encode() digest, downloader = fake_download_bytes(payload) try: with patch( "virsh_panel.cloud_template_runtime.download_verified_image", side_effect=downloader, ): queued = runtime.queue_import(url, f"{family} {version}", digest) job = wait_for_job(runtime, queued["job_id"]) assert job["status"] == "completed", job template = job["template"] assert template["id"].startswith(prefix) assert template["os_family"] == family assert template["os_version"] == version internal = runtime.get(template["id"]) assert internal["volume_name"] == template_volume_name(template["id"]) assert ( tmp_path / "cloud-templates" / internal["volume_name"] ).read_bytes() == payload finally: runtime.close() def test_available_bytes_preserves_panel_disk_reserve(tmp_path): files = GlobalCloudTemplateFiles(tmp_path) with patch( "virsh_panel.cloud_template_runtime.shutil.disk_usage", return_value=SimpleNamespace( free=PERSISTENT_FREE_RESERVE_BYTES + 3 * 1024 * 1024 ), ): assert files.available_bytes() == 3 * 1024 * 1024 with patch( "virsh_panel.cloud_template_runtime.shutil.disk_usage", return_value=SimpleNamespace(free=PERSISTENT_FREE_RESERVE_BYTES - 1), ): assert files.available_bytes() == 0 def test_import_rejects_before_queue_when_only_reserved_space_remains(tmp_path): runtime = CloudTemplateRuntime(MemoryStore(), tmp_path) try: with patch.object( runtime.files, "available_bytes", return_value=1024 * 1024 - 1 ): with pytest.raises(TemplateError, match="空间不足") as insufficient: runtime.queue_import(ALMA_10_URL, "No Disk Space", "") assert insufficient.value.status == 507 assert runtime.jobs == {} assert runtime.pending == set() finally: runtime.close() def test_import_passes_only_space_above_reserve_to_downloader(tmp_path): runtime = CloudTemplateRuntime(MemoryStore(), tmp_path) payload = b"bounded-global-cloud-image" digest = hashlib.sha256(payload).hexdigest() limits: list[int] = [] def bounded_download(_url: str, _expected: str, directory: Path, *, max_bytes: int): limits.append(max_bytes) source = Path(directory) / ".cloud-image-bounded.qcow2" source.write_bytes(payload) os.chmod(source, 0o600) return source, 2 * GIB, digest try: with patch.object( runtime.files, "available_bytes", side_effect=[3 * 1024 * 1024, 2 * 1024 * 1024], ), patch( "virsh_panel.cloud_template_runtime.download_verified_image", side_effect=bounded_download, ): queued = runtime.queue_import(ALMA_10_URL, "Bounded", "") job = wait_for_job(runtime, queued["job_id"]) assert job["status"] == "completed", job assert limits == [2 * 1024 * 1024] finally: runtime.close() def test_duplicate_digest_and_casefold_name_are_rejected_globally(): store = MemoryStore() catalog = GlobalCloudTemplateStore(store) first = record_for(b"first-image", name="AlmaLinux Gold") catalog.add(first) with pytest.raises(TemplateError, match="已经导入") as duplicate_digest: catalog.add(first) assert duplicate_digest.value.status == 409 second = record_for(b"second-image", name="almalinux gold") with pytest.raises(TemplateError, match="同名") as duplicate_name: catalog.add(second) assert duplicate_name.value.status == 409 def test_rename_changes_only_display_name_and_keeps_source_identity(tmp_path): store = MemoryStore() catalog = GlobalCloudTemplateStore(store) record = record_for(b"rename-image") catalog.add(record) updated = catalog.rename(record["id"], "AlmaLinux 10 Production") assert updated["name"] == "AlmaLinux 10 Production" for key in ( "id", "sha256", "volume_name", "virtual_size", "image_bytes", "os_family", "os_version", "created_at", ): assert updated[key] == record[key] files = GlobalCloudTemplateFiles(tmp_path) temporary = files.directory() / ".cloud-image-rename.qcow2" temporary.write_bytes(b"rename-image") os.chmod(temporary, 0o600) source = files.publish( temporary, record["id"], expected_size=len(b"rename-image") ) assert source.name == record["volume_name"] def test_delete_removes_global_catalog_source_and_verification_cache(tmp_path): store = MemoryStore() runtime = CloudTemplateRuntime(store, tmp_path) payload = b"delete-global-cloud-image" record = record_for(payload) runtime.catalog.add(record) temporary = runtime.files.directory() / ".cloud-image-delete.qcow2" temporary.write_bytes(payload) os.chmod(temporary, 0o600) source = runtime.files.publish( temporary, record["id"], expected_size=len(payload) ) runtime.verified_sources[record["id"]] = (1, 2, 3, 4, record["sha256"]) try: runtime.delete(record["id"]) assert not source.exists() assert store.get(CATALOG_KIND, record["id"]) is None assert record["id"] not in runtime.verified_sources with pytest.raises(TemplateError) as missing: runtime.get(record["id"]) assert missing.value.status == 404 finally: runtime.close() def test_delete_rejects_import_or_active_source_lease(tmp_path): runtime = CloudTemplateRuntime(MemoryStore(), tmp_path) payload = b"leased-global-cloud-image" record = record_for(payload) runtime.catalog.add(record) temporary = runtime.files.directory() / ".cloud-image-leased.qcow2" temporary.write_bytes(payload) os.chmod(temporary, 0o600) source = runtime.files.publish( temporary, record["id"], expected_size=len(payload) ) try: runtime.pending.add("active-import") with pytest.raises(TemplateError, match="正在导入") as importing: runtime.delete(record["id"]) assert importing.value.status == 409 runtime.pending.clear() with runtime.source(record["id"]): with pytest.raises(TemplateError, match="正在同步") as leased: runtime.delete(record["id"]) assert leased.value.status == 409 assert source.exists() assert runtime.get(record["id"])["id"] == record["id"] finally: runtime.close() def test_reimport_waits_for_old_host_cache_gc_then_reuses_stable_id(tmp_path): store = MemoryStore() runtime = CloudTemplateRuntime(store, tmp_path) record = record_for(b"reimport-identical-image") tombstone = { "deletion_id": "a" * 32, "template_id": record["id"], "template_created_at": record["created_at"], "source_volume_name": record["volume_name"], "pending_hosts": ["offline-host"], "source_deleted": True, "created_at": 1, } store.set(CACHE_TOMBSTONE_KIND, record["id"], json.dumps(tombstone)) try: with pytest.raises(TemplateError, match="缓存仍在清理") as pending: runtime._prepare_reimport(record["id"]) assert pending.value.status == 409 assert store.get(CACHE_TOMBSTONE_KIND, record["id"]) is not None tombstone["pending_hosts"] = [] store.set(CACHE_TOMBSTONE_KIND, record["id"], json.dumps(tombstone)) previous_created_at = runtime._prepare_reimport(record["id"]) assert store.get(CACHE_TOMBSTONE_KIND, record["id"]) is None assert previous_created_at == record["created_at"] first_generation = runtime._next_generation( record["id"], previous_created_at) second_generation = runtime._next_generation(record["id"]) assert second_generation > first_generation > previous_created_at finally: runtime.close() def test_restore_cancels_provisional_delete_when_source_and_catalog_survive( tmp_path): store = MemoryStore() runtime = CloudTemplateRuntime(store, tmp_path) payload = b"delete-never-committed" record = record_for(payload) runtime.catalog.add(record) temporary = runtime.files.directory() / ".cloud-image-delete-intact.qcow2" temporary.write_bytes(payload) os.chmod(temporary, 0o600) source = runtime.files.publish( temporary, record["id"], expected_size=len(payload) ) tombstone = { "deletion_id": "c" * 32, "template_id": record["id"], "template_created_at": record["created_at"], "source_volume_name": record["volume_name"], "pending_hosts": ["host-one"], "source_deleted": False, "created_at": 1, } store.set(CACHE_TOMBSTONE_KIND, record["id"], json.dumps(tombstone)) try: runtime.restore_jobs() assert runtime.get(record["id"])["id"] == record["id"] assert source.is_file() assert store.get(CACHE_TOMBSTONE_KIND, record["id"]) is None finally: runtime.close() def test_restore_finishes_delete_after_source_unlink_crash_window(tmp_path): store = MemoryStore() runtime = CloudTemplateRuntime(store, tmp_path) payload = b"delete-unlink-committed" record = record_for(payload) runtime.catalog.add(record) tombstone = { "deletion_id": "d" * 32, "template_id": record["id"], "template_created_at": record["created_at"], "source_volume_name": record["volume_name"], "pending_hosts": ["host-one"], "source_deleted": False, "created_at": 1, } store.set(CACHE_TOMBSTONE_KIND, record["id"], json.dumps(tombstone)) try: assert runtime.list() == [] runtime.restore_jobs() with pytest.raises(TemplateError) as missing: runtime.get(record["id"]) assert missing.value.status == 404 restored = json.loads(store.get( CACHE_TOMBSTONE_KIND, record["id"] )) assert restored["source_deleted"] is True assert restored["pending_hosts"] == ["host-one"] finally: runtime.close() def test_source_rejects_permission_damage_and_content_tampering(tmp_path): store = MemoryStore() runtime = CloudTemplateRuntime(store, tmp_path) payload = b"source-integrity" record = record_for(payload) runtime.catalog.add(record) temporary = runtime.files.directory() / ".cloud-image-integrity.qcow2" temporary.write_bytes(payload) os.chmod(temporary, 0o600) source = runtime.files.publish( temporary, record["id"], expected_size=len(payload) ) try: os.chmod(source, 0o640) with pytest.raises(TemplateError, match="已损坏"): with runtime.source(record["id"]): pass os.chmod(source, 0o600) source.write_bytes(b"x" * len(payload)) with pytest.raises(TemplateError, match="SHA-256"): with runtime.source(record["id"]): pass finally: runtime.close() def test_concurrent_import_is_serialized_and_second_request_is_rejected(tmp_path): runtime = CloudTemplateRuntime(MemoryStore(), tmp_path) started = threading.Event() release = threading.Event() payload = b"blocking-image" digest = hashlib.sha256(payload).hexdigest() def blocking_download(_url: str, _expected: str, directory: Path, **_kwargs): started.set() assert release.wait(timeout=5) source = Path(directory) / ".cloud-image-blocking.qcow2" source.write_bytes(payload) os.chmod(source, 0o600) return source, 2 * GIB, digest try: with patch( "virsh_panel.cloud_template_runtime.download_verified_image", side_effect=blocking_download, ): first = runtime.queue_import(ALMA_10_URL, "First", "") assert started.wait(timeout=2) assert runtime.has_active_jobs() is True with pytest.raises(TemplateError, match="正在导入") as busy: runtime.queue_import(ALMA_10_URL, "Second", "") assert busy.value.status == 409 release.set() assert wait_for_job(runtime, first["job_id"])["status"] == "completed" assert runtime.has_active_jobs() is False finally: release.set() runtime.close() def test_restore_marks_interrupted_job_failed_and_removes_partial_files(tmp_path): store = MemoryStore() job_id = "interrupted-cloud-template" store.set(IMPORT_JOB_KIND, job_id, json.dumps({ "job_id": job_id, "status": "running", "phase": "正在下载并校验云镜像", "created_at": 1, })) files = GlobalCloudTemplateFiles(tmp_path) partial = files.directory() / ".cloud-image-abandoned.qcow2" partial.write_bytes(b"partial") os.chmod(partial, 0o600) runtime = CloudTemplateRuntime(store, tmp_path) try: runtime.restore_jobs() job = runtime.get_job(job_id) assert job["status"] == "failed" assert "重启" in job["phase"] assert "重启" in job["error"] assert not partial.exists() assert runtime.has_active_jobs() is False finally: runtime.close() def test_import_failure_redacts_internal_paths_and_releases_pending(tmp_path): runtime = CloudTemplateRuntime(MemoryStore(), tmp_path) try: with patch( "virsh_panel.cloud_template_runtime.download_verified_image", side_effect=TemplateError( "私有目录 /home/virt-cloud/data/cloud-templates 损坏", 500 ), ): queued = runtime.queue_import(ALMA_10_URL, "Broken", "") failed = wait_for_job(runtime, queued["job_id"]) assert failed["status"] == "failed" assert failed["error"] == "云镜像下载或保存失败,请联系管理员查看日志" assert "/home/" not in json.dumps(failed, ensure_ascii=False) assert runtime.pending == set() finally: runtime.close() def test_catalog_insert_commit_with_lost_ack_is_completed_not_rolled_back( tmp_path): store = CommitThenRaiseCatalogStore() runtime = CloudTemplateRuntime(store, tmp_path) payload = b"catalog-commit-ack-lost" digest, downloader = fake_download_bytes(payload) try: with patch( "virsh_panel.cloud_template_runtime.download_verified_image", side_effect=downloader, ): queued = runtime.queue_import(ALMA_10_URL, "Committed", digest) job = wait_for_job(runtime, queued["job_id"]) assert job["status"] == "completed" template_id = cloud_template_id_for(digest, "10") assert runtime.get(template_id)["sha256"] == digest source = tmp_path / "cloud-templates" / template_volume_name(template_id) assert source.read_bytes() == payload finally: runtime.close() def test_ambiguous_catalog_commit_preserves_source_for_startup_recovery( tmp_path): store = AmbiguousCatalogCommitStore() runtime = CloudTemplateRuntime(store, tmp_path) payload = b"ambiguous-catalog-commit" digest, downloader = fake_download_bytes(payload) try: with patch( "virsh_panel.cloud_template_runtime.download_verified_image", side_effect=downloader, ): queued = runtime.queue_import(ALMA_10_URL, "Ambiguous", digest) failed = wait_for_job(runtime, queued["job_id"]) assert failed["status"] == "failed" template_id = cloud_template_id_for(digest, "10") source = tmp_path / "cloud-templates" / template_volume_name(template_id) assert source.read_bytes() == payload finally: runtime.close() restored = CloudTemplateRuntime(store, tmp_path) try: restored.restore_jobs() recovered = restored.get_job(queued["job_id"]) assert recovered["status"] == "completed" assert recovered["template"]["id"] == template_id assert source.read_bytes() == payload finally: restored.close() def test_lost_completed_job_write_is_success_in_memory_and_repaired_on_restart( tmp_path): store = LoseFirstCompletedJobWriteStore() runtime = CloudTemplateRuntime(store, tmp_path) payload = b"lost-completed-job-write" digest, downloader = fake_download_bytes(payload) try: with patch( "virsh_panel.cloud_template_runtime.download_verified_image", side_effect=downloader, ): queued = runtime.queue_import(ALMA_10_URL, "Recoverable", digest) completed = wait_for_job(runtime, queued["job_id"]) assert completed["status"] == "completed" persisted = json.loads(store.get(IMPORT_JOB_KIND, queued["job_id"])) assert persisted["status"] == "running" assert persisted["candidate_template_id"] == cloud_template_id_for( digest, "10" ) finally: runtime.close() restored = CloudTemplateRuntime(store, tmp_path) try: restored.restore_jobs() repaired = restored.get_job(queued["job_id"]) assert repaired["status"] == "completed" persisted = json.loads(store.get(IMPORT_JOB_KIND, queued["job_id"])) assert persisted["status"] == "completed" finally: restored.close()