You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/engine/tests/test_cloud_template_runtime.py

593 lines
20 KiB
Python

"""Global cloud-template catalog, source file and import job tests."""
from __future__ import annotations
import hashlib
import json
import os
import stat
import threading
import time
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import patch
import pytest
from virsh_panel.cloud_template_runtime import (
CATALOG_KIND,
IMPORT_JOB_KIND,
CloudTemplateRuntime,
GlobalCloudTemplateFiles,
GlobalCloudTemplateStore,
PERSISTENT_FREE_RESERVE_BYTES,
cloud_template_id_for,
)
from virsh_panel.cloud_templates import GIB, TemplateError, template_volume_name
ALMA_10_URL = (
"https://repo.almalinux.org/almalinux/10/cloud/x86_64/images/"
"AlmaLinux-10-GenericCloud-latest.x86_64.qcow2"
)
ALMA_8_URL = (
"https://repo.almalinux.org/almalinux/8/cloud/x86_64/images/"
"AlmaLinux-8-GenericCloud-latest.x86_64.qcow2"
)
DEBIAN_12_URL = (
"https://cloud.debian.org/images/cloud/bookworm/latest/"
"debian-12-generic-amd64.qcow2"
)
UBUNTU_24_URL = (
"https://cloud-images.ubuntu.com/releases/24.04/release/"
"ubuntu-24.04-server-cloudimg-amd64.img"
)
CENTOS_STREAM_10_URL = (
"https://cloud.centos.org/centos/10-stream/x86_64/images/"
"CentOS-Stream-GenericCloud-10-latest.x86_64.qcow2"
)
class MemoryStore:
def __init__(self) -> None:
self.values: dict[tuple[str, str], str] = {}
self.lock = threading.RLock()
def list(self, kind: str) -> dict[str, str]:
with self.lock:
return {
key: value for (record_kind, key), value in self.values.items()
if record_kind == kind
}
def get(self, kind: str, key: str) -> str | None:
with self.lock:
return self.values.get((kind, key))
def set(self, kind: str, key: str, value: str) -> None:
with self.lock:
self.values[(kind, key)] = value
def insert(self, kind: str, key: str, value: str) -> bool:
with self.lock:
identity = (kind, key)
if identity in self.values:
return False
self.values[identity] = value
return True
def delete(self, kind: str, key: str) -> None:
with self.lock:
self.values.pop((kind, key), None)
class CommitThenRaiseCatalogStore(MemoryStore):
"""Simulate a committed INSERT whose acknowledgement is lost."""
def insert(self, kind: str, key: str, value: str) -> bool:
inserted = super().insert(kind, key, value)
if kind == CATALOG_KIND and inserted:
raise RuntimeError("catalog insert acknowledgement lost")
return inserted
class AmbiguousCatalogCommitStore(CommitThenRaiseCatalogStore):
"""Also fail the first catalog read after the ambiguous INSERT."""
def __init__(self) -> None:
super().__init__()
self.fail_next_catalog_read = False
def list(self, kind: str) -> dict[str, str]:
if kind == CATALOG_KIND and self.fail_next_catalog_read:
self.fail_next_catalog_read = False
raise RuntimeError("catalog read unavailable")
return super().list(kind)
def insert(self, kind: str, key: str, value: str) -> bool:
try:
return super().insert(kind, key, value)
except RuntimeError:
self.fail_next_catalog_read = True
raise
class LoseFirstCompletedJobWriteStore(MemoryStore):
"""Drop one terminal job update while retaining its running record."""
def __init__(self) -> None:
super().__init__()
self.lost_completed_write = False
def set(self, kind: str, key: str, value: str) -> None:
if kind == IMPORT_JOB_KIND:
job = json.loads(value)
if (job.get("status") == "completed"
and not self.lost_completed_write):
self.lost_completed_write = True
raise RuntimeError("completed job write acknowledgement lost")
super().set(kind, key, value)
def wait_for_job(runtime: CloudTemplateRuntime, job_id: str) -> dict:
for _ in range(200):
job = runtime.get_job(job_id)
if job["status"] in {"completed", "failed"}:
return job
time.sleep(0.01)
raise AssertionError("cloud template import did not finish")
def fake_download_bytes(payload: bytes):
digest = hashlib.sha256(payload).hexdigest()
def download(_url: str, expected_sha256: str, directory: Path, **_kwargs):
assert expected_sha256 in {"", digest}
source = Path(directory) / ".cloud-image-test.qcow2"
source.write_bytes(payload)
os.chmod(source, 0o600)
return source, 2 * GIB, digest
return digest, download
def record_for(payload: bytes, *, name: str = "AlmaLinux 10") -> dict:
digest = hashlib.sha256(payload).hexdigest()
template_id = cloud_template_id_for(digest, "10")
return {
"id": template_id,
"name": name,
"os_family": "AlmaLinux",
"os_version": "10",
"sha256": digest,
"volume_name": template_volume_name(template_id),
"virtual_size": 2 * GIB,
"image_bytes": len(payload),
"created_at": 1,
}
def test_stable_global_id_and_catalog_use_an_independent_record_kind():
digest = "a" * 64
first = cloud_template_id_for(digest, "10")
second = cloud_template_id_for(digest.upper(), "10")
assert first == second
assert first.startswith("alma10-")
assert template_volume_name(first).endswith(".qcow2")
store = MemoryStore()
catalog = GlobalCloudTemplateStore(store)
record = record_for(b"catalog-image")
catalog.add(record)
assert catalog.get(record["id"]) == record
assert list(store.list(CATALOG_KIND)) == [record["id"]]
assert store.list("template") == {}
@pytest.mark.parametrize(
"message",
[
"官方云镜像地址无法解析",
"官方云镜像地址解析结果无效",
"官方云镜像地址解析到非公网地址",
"无法连接官方云镜像下载地址",
"官方云镜像重定向目标不安全",
"官方云镜像重定向次数过多",
"官方云镜像下载服务器返回异常状态",
"官方云镜像下载响应格式无效",
"云镜像下载内容不完整",
],
)
def test_public_job_preserves_fixed_safe_download_errors(message):
job = {
"job_id": "safe-error",
"status": "failed",
"phase": "导入失败",
"created_at": 1,
"error": message,
}
assert CloudTemplateRuntime.public_job(job)["error"] == message
def test_import_publishes_private_source_and_restores_global_catalog(tmp_path):
store = MemoryStore()
runtime = CloudTemplateRuntime(store, tmp_path)
payload = b"verified-global-cloud-image"
digest, downloader = fake_download_bytes(payload)
try:
with patch(
"virsh_panel.cloud_template_runtime.download_verified_image",
side_effect=downloader,
):
queued = runtime.queue_import(
ALMA_10_URL, "AlmaLinux 10", digest
)
job = wait_for_job(runtime, queued["job_id"])
assert job["status"] == "completed", job
template_id = job["template"]["id"]
assert template_id == cloud_template_id_for(digest, "10")
assert "volume_name" not in job["template"]
listed = runtime.list()
assert listed == [job["template"]]
assert "active_import_job" not in runtime.status()
root = tmp_path / "cloud-templates"
source = root / template_volume_name(template_id)
assert source.read_bytes() == payload
assert stat.S_IMODE(root.stat().st_mode) == 0o700
assert stat.S_IMODE(source.stat().st_mode) == 0o600
assert store.get(IMPORT_JOB_KIND, queued["job_id"]) is not None
# A new runtime reads the same host-independent catalog and verifies
# the persisted source from disk instead of trusting process memory.
restored = CloudTemplateRuntime(store, tmp_path)
try:
restored.restore_jobs()
assert restored.list() == listed
with restored.source(template_id) as (record, path):
assert record["sha256"] == digest
assert path == source
assert restored.in_use[template_id] == 1
assert template_id not in restored.in_use
finally:
restored.close()
finally:
runtime.close()
@pytest.mark.parametrize(
("url", "family", "version", "prefix"),
[
(ALMA_8_URL, "AlmaLinux", "8", "alma8-"),
(DEBIAN_12_URL, "Debian", "12", "debian12-"),
(UBUNTU_24_URL, "Ubuntu", "24.04", "ubuntu2404-"),
(CENTOS_STREAM_10_URL, "CentOS Stream", "10", "centosstream10-"),
],
)
def test_import_derives_family_version_and_safe_source_name(
tmp_path, url, family, version, prefix):
runtime = CloudTemplateRuntime(MemoryStore(), tmp_path)
payload = f"{family}-{version}-cloud-image".encode()
digest, downloader = fake_download_bytes(payload)
try:
with patch(
"virsh_panel.cloud_template_runtime.download_verified_image",
side_effect=downloader,
):
queued = runtime.queue_import(url, f"{family} {version}", digest)
job = wait_for_job(runtime, queued["job_id"])
assert job["status"] == "completed", job
template = job["template"]
assert template["id"].startswith(prefix)
assert template["os_family"] == family
assert template["os_version"] == version
internal = runtime.get(template["id"])
assert internal["volume_name"] == template_volume_name(template["id"])
assert (
tmp_path / "cloud-templates" / internal["volume_name"]
).read_bytes() == payload
finally:
runtime.close()
def test_available_bytes_preserves_panel_disk_reserve(tmp_path):
files = GlobalCloudTemplateFiles(tmp_path)
with patch(
"virsh_panel.cloud_template_runtime.shutil.disk_usage",
return_value=SimpleNamespace(
free=PERSISTENT_FREE_RESERVE_BYTES + 3 * 1024 * 1024
),
):
assert files.available_bytes() == 3 * 1024 * 1024
with patch(
"virsh_panel.cloud_template_runtime.shutil.disk_usage",
return_value=SimpleNamespace(free=PERSISTENT_FREE_RESERVE_BYTES - 1),
):
assert files.available_bytes() == 0
def test_import_rejects_before_queue_when_only_reserved_space_remains(tmp_path):
runtime = CloudTemplateRuntime(MemoryStore(), tmp_path)
try:
with patch.object(
runtime.files, "available_bytes", return_value=1024 * 1024 - 1
):
with pytest.raises(TemplateError, match="空间不足") as insufficient:
runtime.queue_import(ALMA_10_URL, "No Disk Space", "")
assert insufficient.value.status == 507
assert runtime.jobs == {}
assert runtime.pending == set()
finally:
runtime.close()
def test_import_passes_only_space_above_reserve_to_downloader(tmp_path):
runtime = CloudTemplateRuntime(MemoryStore(), tmp_path)
payload = b"bounded-global-cloud-image"
digest = hashlib.sha256(payload).hexdigest()
limits: list[int] = []
def bounded_download(_url: str, _expected: str, directory: Path, *,
max_bytes: int):
limits.append(max_bytes)
source = Path(directory) / ".cloud-image-bounded.qcow2"
source.write_bytes(payload)
os.chmod(source, 0o600)
return source, 2 * GIB, digest
try:
with patch.object(
runtime.files, "available_bytes",
side_effect=[3 * 1024 * 1024, 2 * 1024 * 1024],
), patch(
"virsh_panel.cloud_template_runtime.download_verified_image",
side_effect=bounded_download,
):
queued = runtime.queue_import(ALMA_10_URL, "Bounded", "")
job = wait_for_job(runtime, queued["job_id"])
assert job["status"] == "completed", job
assert limits == [2 * 1024 * 1024]
finally:
runtime.close()
def test_duplicate_digest_and_casefold_name_are_rejected_globally():
store = MemoryStore()
catalog = GlobalCloudTemplateStore(store)
first = record_for(b"first-image", name="AlmaLinux Gold")
catalog.add(first)
with pytest.raises(TemplateError, match="已经导入") as duplicate_digest:
catalog.add(first)
assert duplicate_digest.value.status == 409
second = record_for(b"second-image", name="almalinux gold")
with pytest.raises(TemplateError, match="同名") as duplicate_name:
catalog.add(second)
assert duplicate_name.value.status == 409
def test_rename_changes_only_display_name_and_keeps_source_identity(tmp_path):
store = MemoryStore()
catalog = GlobalCloudTemplateStore(store)
record = record_for(b"rename-image")
catalog.add(record)
updated = catalog.rename(record["id"], "AlmaLinux 10 Production")
assert updated["name"] == "AlmaLinux 10 Production"
for key in (
"id", "sha256", "volume_name", "virtual_size", "image_bytes",
"os_family", "os_version", "created_at",
):
assert updated[key] == record[key]
files = GlobalCloudTemplateFiles(tmp_path)
temporary = files.directory() / ".cloud-image-rename.qcow2"
temporary.write_bytes(b"rename-image")
os.chmod(temporary, 0o600)
source = files.publish(
temporary, record["id"], expected_size=len(b"rename-image")
)
assert source.name == record["volume_name"]
def test_source_rejects_permission_damage_and_content_tampering(tmp_path):
store = MemoryStore()
runtime = CloudTemplateRuntime(store, tmp_path)
payload = b"source-integrity"
record = record_for(payload)
runtime.catalog.add(record)
temporary = runtime.files.directory() / ".cloud-image-integrity.qcow2"
temporary.write_bytes(payload)
os.chmod(temporary, 0o600)
source = runtime.files.publish(
temporary, record["id"], expected_size=len(payload)
)
try:
os.chmod(source, 0o640)
with pytest.raises(TemplateError, match="已损坏"):
with runtime.source(record["id"]):
pass
os.chmod(source, 0o600)
source.write_bytes(b"x" * len(payload))
with pytest.raises(TemplateError, match="SHA-256"):
with runtime.source(record["id"]):
pass
finally:
runtime.close()
def test_concurrent_import_is_serialized_and_second_request_is_rejected(tmp_path):
runtime = CloudTemplateRuntime(MemoryStore(), tmp_path)
started = threading.Event()
release = threading.Event()
payload = b"blocking-image"
digest = hashlib.sha256(payload).hexdigest()
def blocking_download(_url: str, _expected: str, directory: Path,
**_kwargs):
started.set()
assert release.wait(timeout=5)
source = Path(directory) / ".cloud-image-blocking.qcow2"
source.write_bytes(payload)
os.chmod(source, 0o600)
return source, 2 * GIB, digest
try:
with patch(
"virsh_panel.cloud_template_runtime.download_verified_image",
side_effect=blocking_download,
):
first = runtime.queue_import(ALMA_10_URL, "First", "")
assert started.wait(timeout=2)
assert runtime.has_active_jobs() is True
with pytest.raises(TemplateError, match="正在导入") as busy:
runtime.queue_import(ALMA_10_URL, "Second", "")
assert busy.value.status == 409
release.set()
assert wait_for_job(runtime, first["job_id"])["status"] == "completed"
assert runtime.has_active_jobs() is False
finally:
release.set()
runtime.close()
def test_restore_marks_interrupted_job_failed_and_removes_partial_files(tmp_path):
store = MemoryStore()
job_id = "interrupted-cloud-template"
store.set(IMPORT_JOB_KIND, job_id, json.dumps({
"job_id": job_id,
"status": "running",
"phase": "正在下载并校验云镜像",
"created_at": 1,
}))
files = GlobalCloudTemplateFiles(tmp_path)
partial = files.directory() / ".cloud-image-abandoned.qcow2"
partial.write_bytes(b"partial")
os.chmod(partial, 0o600)
runtime = CloudTemplateRuntime(store, tmp_path)
try:
runtime.restore_jobs()
job = runtime.get_job(job_id)
assert job["status"] == "failed"
assert "重启" in job["phase"]
assert "重启" in job["error"]
assert not partial.exists()
assert runtime.has_active_jobs() is False
finally:
runtime.close()
def test_import_failure_redacts_internal_paths_and_releases_pending(tmp_path):
runtime = CloudTemplateRuntime(MemoryStore(), tmp_path)
try:
with patch(
"virsh_panel.cloud_template_runtime.download_verified_image",
side_effect=TemplateError(
"私有目录 /home/virt-cloud/data/cloud-templates 损坏", 500
),
):
queued = runtime.queue_import(ALMA_10_URL, "Broken", "")
failed = wait_for_job(runtime, queued["job_id"])
assert failed["status"] == "failed"
assert failed["error"] == "云镜像下载或保存失败,请联系管理员查看日志"
assert "/home/" not in json.dumps(failed, ensure_ascii=False)
assert runtime.pending == set()
finally:
runtime.close()
def test_catalog_insert_commit_with_lost_ack_is_completed_not_rolled_back(
tmp_path):
store = CommitThenRaiseCatalogStore()
runtime = CloudTemplateRuntime(store, tmp_path)
payload = b"catalog-commit-ack-lost"
digest, downloader = fake_download_bytes(payload)
try:
with patch(
"virsh_panel.cloud_template_runtime.download_verified_image",
side_effect=downloader,
):
queued = runtime.queue_import(ALMA_10_URL, "Committed", digest)
job = wait_for_job(runtime, queued["job_id"])
assert job["status"] == "completed"
template_id = cloud_template_id_for(digest, "10")
assert runtime.get(template_id)["sha256"] == digest
source = tmp_path / "cloud-templates" / template_volume_name(template_id)
assert source.read_bytes() == payload
finally:
runtime.close()
def test_ambiguous_catalog_commit_preserves_source_for_startup_recovery(
tmp_path):
store = AmbiguousCatalogCommitStore()
runtime = CloudTemplateRuntime(store, tmp_path)
payload = b"ambiguous-catalog-commit"
digest, downloader = fake_download_bytes(payload)
try:
with patch(
"virsh_panel.cloud_template_runtime.download_verified_image",
side_effect=downloader,
):
queued = runtime.queue_import(ALMA_10_URL, "Ambiguous", digest)
failed = wait_for_job(runtime, queued["job_id"])
assert failed["status"] == "failed"
template_id = cloud_template_id_for(digest, "10")
source = tmp_path / "cloud-templates" / template_volume_name(template_id)
assert source.read_bytes() == payload
finally:
runtime.close()
restored = CloudTemplateRuntime(store, tmp_path)
try:
restored.restore_jobs()
recovered = restored.get_job(queued["job_id"])
assert recovered["status"] == "completed"
assert recovered["template"]["id"] == template_id
assert source.read_bytes() == payload
finally:
restored.close()
def test_lost_completed_job_write_is_success_in_memory_and_repaired_on_restart(
tmp_path):
store = LoseFirstCompletedJobWriteStore()
runtime = CloudTemplateRuntime(store, tmp_path)
payload = b"lost-completed-job-write"
digest, downloader = fake_download_bytes(payload)
try:
with patch(
"virsh_panel.cloud_template_runtime.download_verified_image",
side_effect=downloader,
):
queued = runtime.queue_import(ALMA_10_URL, "Recoverable", digest)
completed = wait_for_job(runtime, queued["job_id"])
assert completed["status"] == "completed"
persisted = json.loads(store.get(IMPORT_JOB_KIND, queued["job_id"]))
assert persisted["status"] == "running"
assert persisted["candidate_template_id"] == cloud_template_id_for(
digest, "10"
)
finally:
runtime.close()
restored = CloudTemplateRuntime(store, tmp_path)
try:
restored.restore_jobs()
repaired = restored.get_job(queued["job_id"])
assert repaired["status"] == "completed"
persisted = json.loads(store.get(IMPORT_JOB_KIND, queued["job_id"]))
assert persisted["status"] == "completed"
finally:
restored.close()