feat: share cloud images across regions
parent
59f262b9aa
commit
308977473f
@ -0,0 +1,515 @@
|
||||
"""Global cloud-template catalog, source file and import job tests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import stat
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from virsh_panel.cloud_template_runtime import (
|
||||
CATALOG_KIND,
|
||||
IMPORT_JOB_KIND,
|
||||
CloudTemplateRuntime,
|
||||
GlobalCloudTemplateFiles,
|
||||
GlobalCloudTemplateStore,
|
||||
PERSISTENT_FREE_RESERVE_BYTES,
|
||||
cloud_template_id_for,
|
||||
)
|
||||
from virsh_panel.cloud_templates import GIB, TemplateError, template_volume_name
|
||||
|
||||
|
||||
ALMA_10_URL = (
|
||||
"https://repo.almalinux.org/almalinux/10/cloud/x86_64/images/"
|
||||
"AlmaLinux-10-GenericCloud-latest.x86_64.qcow2"
|
||||
)
|
||||
|
||||
|
||||
class MemoryStore:
|
||||
def __init__(self) -> None:
|
||||
self.values: dict[tuple[str, str], str] = {}
|
||||
self.lock = threading.RLock()
|
||||
|
||||
def list(self, kind: str) -> dict[str, str]:
|
||||
with self.lock:
|
||||
return {
|
||||
key: value for (record_kind, key), value in self.values.items()
|
||||
if record_kind == kind
|
||||
}
|
||||
|
||||
def get(self, kind: str, key: str) -> str | None:
|
||||
with self.lock:
|
||||
return self.values.get((kind, key))
|
||||
|
||||
def set(self, kind: str, key: str, value: str) -> None:
|
||||
with self.lock:
|
||||
self.values[(kind, key)] = value
|
||||
|
||||
def insert(self, kind: str, key: str, value: str) -> bool:
|
||||
with self.lock:
|
||||
identity = (kind, key)
|
||||
if identity in self.values:
|
||||
return False
|
||||
self.values[identity] = value
|
||||
return True
|
||||
|
||||
def delete(self, kind: str, key: str) -> None:
|
||||
with self.lock:
|
||||
self.values.pop((kind, key), None)
|
||||
|
||||
|
||||
class CommitThenRaiseCatalogStore(MemoryStore):
|
||||
"""Simulate a committed INSERT whose acknowledgement is lost."""
|
||||
|
||||
def insert(self, kind: str, key: str, value: str) -> bool:
|
||||
inserted = super().insert(kind, key, value)
|
||||
if kind == CATALOG_KIND and inserted:
|
||||
raise RuntimeError("catalog insert acknowledgement lost")
|
||||
return inserted
|
||||
|
||||
|
||||
class AmbiguousCatalogCommitStore(CommitThenRaiseCatalogStore):
|
||||
"""Also fail the first catalog read after the ambiguous INSERT."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.fail_next_catalog_read = False
|
||||
|
||||
def list(self, kind: str) -> dict[str, str]:
|
||||
if kind == CATALOG_KIND and self.fail_next_catalog_read:
|
||||
self.fail_next_catalog_read = False
|
||||
raise RuntimeError("catalog read unavailable")
|
||||
return super().list(kind)
|
||||
|
||||
def insert(self, kind: str, key: str, value: str) -> bool:
|
||||
try:
|
||||
return super().insert(kind, key, value)
|
||||
except RuntimeError:
|
||||
self.fail_next_catalog_read = True
|
||||
raise
|
||||
|
||||
|
||||
class LoseFirstCompletedJobWriteStore(MemoryStore):
|
||||
"""Drop one terminal job update while retaining its running record."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.lost_completed_write = False
|
||||
|
||||
def set(self, kind: str, key: str, value: str) -> None:
|
||||
if kind == IMPORT_JOB_KIND:
|
||||
job = json.loads(value)
|
||||
if (job.get("status") == "completed"
|
||||
and not self.lost_completed_write):
|
||||
self.lost_completed_write = True
|
||||
raise RuntimeError("completed job write acknowledgement lost")
|
||||
super().set(kind, key, value)
|
||||
|
||||
|
||||
def wait_for_job(runtime: CloudTemplateRuntime, job_id: str) -> dict:
|
||||
for _ in range(200):
|
||||
job = runtime.get_job(job_id)
|
||||
if job["status"] in {"completed", "failed"}:
|
||||
return job
|
||||
time.sleep(0.01)
|
||||
raise AssertionError("cloud template import did not finish")
|
||||
|
||||
|
||||
def fake_download_bytes(payload: bytes):
|
||||
digest = hashlib.sha256(payload).hexdigest()
|
||||
|
||||
def download(_url: str, expected_sha256: str, directory: Path, **_kwargs):
|
||||
assert expected_sha256 in {"", digest}
|
||||
source = Path(directory) / ".cloud-image-test.qcow2"
|
||||
source.write_bytes(payload)
|
||||
os.chmod(source, 0o600)
|
||||
return source, 2 * GIB, digest
|
||||
|
||||
return digest, download
|
||||
|
||||
|
||||
def record_for(payload: bytes, *, name: str = "AlmaLinux 10") -> dict:
|
||||
digest = hashlib.sha256(payload).hexdigest()
|
||||
template_id = cloud_template_id_for(digest, "10")
|
||||
return {
|
||||
"id": template_id,
|
||||
"name": name,
|
||||
"os_family": "AlmaLinux",
|
||||
"os_version": "10",
|
||||
"sha256": digest,
|
||||
"volume_name": template_volume_name(template_id),
|
||||
"virtual_size": 2 * GIB,
|
||||
"image_bytes": len(payload),
|
||||
"created_at": 1,
|
||||
}
|
||||
|
||||
|
||||
def test_stable_global_id_and_catalog_use_an_independent_record_kind():
|
||||
digest = "a" * 64
|
||||
first = cloud_template_id_for(digest, "10")
|
||||
second = cloud_template_id_for(digest.upper(), "10")
|
||||
assert first == second
|
||||
assert first.startswith("alma10-")
|
||||
assert template_volume_name(first).endswith(".qcow2")
|
||||
|
||||
store = MemoryStore()
|
||||
catalog = GlobalCloudTemplateStore(store)
|
||||
record = record_for(b"catalog-image")
|
||||
catalog.add(record)
|
||||
|
||||
assert catalog.get(record["id"]) == record
|
||||
assert list(store.list(CATALOG_KIND)) == [record["id"]]
|
||||
assert store.list("template") == {}
|
||||
|
||||
|
||||
def test_import_publishes_private_source_and_restores_global_catalog(tmp_path):
|
||||
store = MemoryStore()
|
||||
runtime = CloudTemplateRuntime(store, tmp_path)
|
||||
payload = b"verified-global-cloud-image"
|
||||
digest, downloader = fake_download_bytes(payload)
|
||||
try:
|
||||
with patch(
|
||||
"virsh_panel.cloud_template_runtime.download_verified_image",
|
||||
side_effect=downloader,
|
||||
):
|
||||
queued = runtime.queue_import(
|
||||
ALMA_10_URL, "AlmaLinux 10", digest
|
||||
)
|
||||
job = wait_for_job(runtime, queued["job_id"])
|
||||
|
||||
assert job["status"] == "completed", job
|
||||
template_id = job["template"]["id"]
|
||||
assert template_id == cloud_template_id_for(digest, "10")
|
||||
assert "volume_name" not in job["template"]
|
||||
listed = runtime.list()
|
||||
assert listed == [job["template"]]
|
||||
assert "active_import_job" not in runtime.status()
|
||||
|
||||
root = tmp_path / "cloud-templates"
|
||||
source = root / template_volume_name(template_id)
|
||||
assert source.read_bytes() == payload
|
||||
assert stat.S_IMODE(root.stat().st_mode) == 0o700
|
||||
assert stat.S_IMODE(source.stat().st_mode) == 0o600
|
||||
assert store.get(IMPORT_JOB_KIND, queued["job_id"]) is not None
|
||||
|
||||
# A new runtime reads the same host-independent catalog and verifies
|
||||
# the persisted source from disk instead of trusting process memory.
|
||||
restored = CloudTemplateRuntime(store, tmp_path)
|
||||
try:
|
||||
restored.restore_jobs()
|
||||
assert restored.list() == listed
|
||||
with restored.source(template_id) as (record, path):
|
||||
assert record["sha256"] == digest
|
||||
assert path == source
|
||||
assert restored.in_use[template_id] == 1
|
||||
assert template_id not in restored.in_use
|
||||
finally:
|
||||
restored.close()
|
||||
finally:
|
||||
runtime.close()
|
||||
|
||||
|
||||
def test_available_bytes_preserves_panel_disk_reserve(tmp_path):
|
||||
files = GlobalCloudTemplateFiles(tmp_path)
|
||||
with patch(
|
||||
"virsh_panel.cloud_template_runtime.shutil.disk_usage",
|
||||
return_value=SimpleNamespace(
|
||||
free=PERSISTENT_FREE_RESERVE_BYTES + 3 * 1024 * 1024
|
||||
),
|
||||
):
|
||||
assert files.available_bytes() == 3 * 1024 * 1024
|
||||
|
||||
with patch(
|
||||
"virsh_panel.cloud_template_runtime.shutil.disk_usage",
|
||||
return_value=SimpleNamespace(free=PERSISTENT_FREE_RESERVE_BYTES - 1),
|
||||
):
|
||||
assert files.available_bytes() == 0
|
||||
|
||||
|
||||
def test_import_rejects_before_queue_when_only_reserved_space_remains(tmp_path):
|
||||
runtime = CloudTemplateRuntime(MemoryStore(), tmp_path)
|
||||
try:
|
||||
with patch.object(
|
||||
runtime.files, "available_bytes", return_value=1024 * 1024 - 1
|
||||
):
|
||||
with pytest.raises(TemplateError, match="空间不足") as insufficient:
|
||||
runtime.queue_import(ALMA_10_URL, "No Disk Space", "")
|
||||
assert insufficient.value.status == 507
|
||||
assert runtime.jobs == {}
|
||||
assert runtime.pending == set()
|
||||
finally:
|
||||
runtime.close()
|
||||
|
||||
|
||||
def test_import_passes_only_space_above_reserve_to_downloader(tmp_path):
|
||||
runtime = CloudTemplateRuntime(MemoryStore(), tmp_path)
|
||||
payload = b"bounded-global-cloud-image"
|
||||
digest = hashlib.sha256(payload).hexdigest()
|
||||
limits: list[int] = []
|
||||
|
||||
def bounded_download(_url: str, _expected: str, directory: Path, *,
|
||||
max_bytes: int):
|
||||
limits.append(max_bytes)
|
||||
source = Path(directory) / ".cloud-image-bounded.qcow2"
|
||||
source.write_bytes(payload)
|
||||
os.chmod(source, 0o600)
|
||||
return source, 2 * GIB, digest
|
||||
|
||||
try:
|
||||
with patch.object(
|
||||
runtime.files, "available_bytes",
|
||||
side_effect=[3 * 1024 * 1024, 2 * 1024 * 1024],
|
||||
), patch(
|
||||
"virsh_panel.cloud_template_runtime.download_verified_image",
|
||||
side_effect=bounded_download,
|
||||
):
|
||||
queued = runtime.queue_import(ALMA_10_URL, "Bounded", "")
|
||||
job = wait_for_job(runtime, queued["job_id"])
|
||||
assert job["status"] == "completed", job
|
||||
assert limits == [2 * 1024 * 1024]
|
||||
finally:
|
||||
runtime.close()
|
||||
|
||||
|
||||
def test_duplicate_digest_and_casefold_name_are_rejected_globally():
|
||||
store = MemoryStore()
|
||||
catalog = GlobalCloudTemplateStore(store)
|
||||
first = record_for(b"first-image", name="AlmaLinux Gold")
|
||||
catalog.add(first)
|
||||
|
||||
with pytest.raises(TemplateError, match="已经导入") as duplicate_digest:
|
||||
catalog.add(first)
|
||||
assert duplicate_digest.value.status == 409
|
||||
|
||||
second = record_for(b"second-image", name="almalinux gold")
|
||||
with pytest.raises(TemplateError, match="同名") as duplicate_name:
|
||||
catalog.add(second)
|
||||
assert duplicate_name.value.status == 409
|
||||
|
||||
|
||||
def test_rename_changes_only_display_name_and_keeps_source_identity(tmp_path):
|
||||
store = MemoryStore()
|
||||
catalog = GlobalCloudTemplateStore(store)
|
||||
record = record_for(b"rename-image")
|
||||
catalog.add(record)
|
||||
updated = catalog.rename(record["id"], "AlmaLinux 10 Production")
|
||||
|
||||
assert updated["name"] == "AlmaLinux 10 Production"
|
||||
for key in (
|
||||
"id", "sha256", "volume_name", "virtual_size", "image_bytes",
|
||||
"os_family", "os_version", "created_at",
|
||||
):
|
||||
assert updated[key] == record[key]
|
||||
|
||||
files = GlobalCloudTemplateFiles(tmp_path)
|
||||
temporary = files.directory() / ".cloud-image-rename.qcow2"
|
||||
temporary.write_bytes(b"rename-image")
|
||||
os.chmod(temporary, 0o600)
|
||||
source = files.publish(
|
||||
temporary, record["id"], expected_size=len(b"rename-image")
|
||||
)
|
||||
assert source.name == record["volume_name"]
|
||||
|
||||
|
||||
def test_source_rejects_permission_damage_and_content_tampering(tmp_path):
|
||||
store = MemoryStore()
|
||||
runtime = CloudTemplateRuntime(store, tmp_path)
|
||||
payload = b"source-integrity"
|
||||
record = record_for(payload)
|
||||
runtime.catalog.add(record)
|
||||
temporary = runtime.files.directory() / ".cloud-image-integrity.qcow2"
|
||||
temporary.write_bytes(payload)
|
||||
os.chmod(temporary, 0o600)
|
||||
source = runtime.files.publish(
|
||||
temporary, record["id"], expected_size=len(payload)
|
||||
)
|
||||
try:
|
||||
os.chmod(source, 0o640)
|
||||
with pytest.raises(TemplateError, match="已损坏"):
|
||||
with runtime.source(record["id"]):
|
||||
pass
|
||||
|
||||
os.chmod(source, 0o600)
|
||||
source.write_bytes(b"x" * len(payload))
|
||||
with pytest.raises(TemplateError, match="SHA-256"):
|
||||
with runtime.source(record["id"]):
|
||||
pass
|
||||
finally:
|
||||
runtime.close()
|
||||
|
||||
|
||||
def test_concurrent_import_is_serialized_and_second_request_is_rejected(tmp_path):
|
||||
runtime = CloudTemplateRuntime(MemoryStore(), tmp_path)
|
||||
started = threading.Event()
|
||||
release = threading.Event()
|
||||
payload = b"blocking-image"
|
||||
digest = hashlib.sha256(payload).hexdigest()
|
||||
|
||||
def blocking_download(_url: str, _expected: str, directory: Path,
|
||||
**_kwargs):
|
||||
started.set()
|
||||
assert release.wait(timeout=5)
|
||||
source = Path(directory) / ".cloud-image-blocking.qcow2"
|
||||
source.write_bytes(payload)
|
||||
os.chmod(source, 0o600)
|
||||
return source, 2 * GIB, digest
|
||||
|
||||
try:
|
||||
with patch(
|
||||
"virsh_panel.cloud_template_runtime.download_verified_image",
|
||||
side_effect=blocking_download,
|
||||
):
|
||||
first = runtime.queue_import(ALMA_10_URL, "First", "")
|
||||
assert started.wait(timeout=2)
|
||||
assert runtime.has_active_jobs() is True
|
||||
with pytest.raises(TemplateError, match="正在导入") as busy:
|
||||
runtime.queue_import(ALMA_10_URL, "Second", "")
|
||||
assert busy.value.status == 409
|
||||
release.set()
|
||||
assert wait_for_job(runtime, first["job_id"])["status"] == "completed"
|
||||
assert runtime.has_active_jobs() is False
|
||||
finally:
|
||||
release.set()
|
||||
runtime.close()
|
||||
|
||||
|
||||
def test_restore_marks_interrupted_job_failed_and_removes_partial_files(tmp_path):
|
||||
store = MemoryStore()
|
||||
job_id = "interrupted-cloud-template"
|
||||
store.set(IMPORT_JOB_KIND, job_id, json.dumps({
|
||||
"job_id": job_id,
|
||||
"status": "running",
|
||||
"phase": "正在下载并校验云镜像",
|
||||
"created_at": 1,
|
||||
}))
|
||||
files = GlobalCloudTemplateFiles(tmp_path)
|
||||
partial = files.directory() / ".cloud-image-abandoned.qcow2"
|
||||
partial.write_bytes(b"partial")
|
||||
os.chmod(partial, 0o600)
|
||||
|
||||
runtime = CloudTemplateRuntime(store, tmp_path)
|
||||
try:
|
||||
runtime.restore_jobs()
|
||||
job = runtime.get_job(job_id)
|
||||
assert job["status"] == "failed"
|
||||
assert "重启" in job["phase"]
|
||||
assert "重启" in job["error"]
|
||||
assert not partial.exists()
|
||||
assert runtime.has_active_jobs() is False
|
||||
finally:
|
||||
runtime.close()
|
||||
|
||||
|
||||
def test_import_failure_redacts_internal_paths_and_releases_pending(tmp_path):
|
||||
runtime = CloudTemplateRuntime(MemoryStore(), tmp_path)
|
||||
try:
|
||||
with patch(
|
||||
"virsh_panel.cloud_template_runtime.download_verified_image",
|
||||
side_effect=TemplateError(
|
||||
"私有目录 /home/virt-cloud/data/cloud-templates 损坏", 500
|
||||
),
|
||||
):
|
||||
queued = runtime.queue_import(ALMA_10_URL, "Broken", "")
|
||||
failed = wait_for_job(runtime, queued["job_id"])
|
||||
assert failed["status"] == "failed"
|
||||
assert failed["error"] == "云镜像下载或保存失败,请联系管理员查看日志"
|
||||
assert "/home/" not in json.dumps(failed, ensure_ascii=False)
|
||||
assert runtime.pending == set()
|
||||
finally:
|
||||
runtime.close()
|
||||
|
||||
|
||||
def test_catalog_insert_commit_with_lost_ack_is_completed_not_rolled_back(
|
||||
tmp_path):
|
||||
store = CommitThenRaiseCatalogStore()
|
||||
runtime = CloudTemplateRuntime(store, tmp_path)
|
||||
payload = b"catalog-commit-ack-lost"
|
||||
digest, downloader = fake_download_bytes(payload)
|
||||
try:
|
||||
with patch(
|
||||
"virsh_panel.cloud_template_runtime.download_verified_image",
|
||||
side_effect=downloader,
|
||||
):
|
||||
queued = runtime.queue_import(ALMA_10_URL, "Committed", digest)
|
||||
job = wait_for_job(runtime, queued["job_id"])
|
||||
|
||||
assert job["status"] == "completed"
|
||||
template_id = cloud_template_id_for(digest, "10")
|
||||
assert runtime.get(template_id)["sha256"] == digest
|
||||
source = tmp_path / "cloud-templates" / template_volume_name(template_id)
|
||||
assert source.read_bytes() == payload
|
||||
finally:
|
||||
runtime.close()
|
||||
|
||||
|
||||
def test_ambiguous_catalog_commit_preserves_source_for_startup_recovery(
|
||||
tmp_path):
|
||||
store = AmbiguousCatalogCommitStore()
|
||||
runtime = CloudTemplateRuntime(store, tmp_path)
|
||||
payload = b"ambiguous-catalog-commit"
|
||||
digest, downloader = fake_download_bytes(payload)
|
||||
try:
|
||||
with patch(
|
||||
"virsh_panel.cloud_template_runtime.download_verified_image",
|
||||
side_effect=downloader,
|
||||
):
|
||||
queued = runtime.queue_import(ALMA_10_URL, "Ambiguous", digest)
|
||||
failed = wait_for_job(runtime, queued["job_id"])
|
||||
|
||||
assert failed["status"] == "failed"
|
||||
template_id = cloud_template_id_for(digest, "10")
|
||||
source = tmp_path / "cloud-templates" / template_volume_name(template_id)
|
||||
assert source.read_bytes() == payload
|
||||
finally:
|
||||
runtime.close()
|
||||
|
||||
restored = CloudTemplateRuntime(store, tmp_path)
|
||||
try:
|
||||
restored.restore_jobs()
|
||||
recovered = restored.get_job(queued["job_id"])
|
||||
assert recovered["status"] == "completed"
|
||||
assert recovered["template"]["id"] == template_id
|
||||
assert source.read_bytes() == payload
|
||||
finally:
|
||||
restored.close()
|
||||
|
||||
|
||||
def test_lost_completed_job_write_is_success_in_memory_and_repaired_on_restart(
|
||||
tmp_path):
|
||||
store = LoseFirstCompletedJobWriteStore()
|
||||
runtime = CloudTemplateRuntime(store, tmp_path)
|
||||
payload = b"lost-completed-job-write"
|
||||
digest, downloader = fake_download_bytes(payload)
|
||||
try:
|
||||
with patch(
|
||||
"virsh_panel.cloud_template_runtime.download_verified_image",
|
||||
side_effect=downloader,
|
||||
):
|
||||
queued = runtime.queue_import(ALMA_10_URL, "Recoverable", digest)
|
||||
completed = wait_for_job(runtime, queued["job_id"])
|
||||
|
||||
assert completed["status"] == "completed"
|
||||
persisted = json.loads(store.get(IMPORT_JOB_KIND, queued["job_id"]))
|
||||
assert persisted["status"] == "running"
|
||||
assert persisted["candidate_template_id"] == cloud_template_id_for(
|
||||
digest, "10"
|
||||
)
|
||||
finally:
|
||||
runtime.close()
|
||||
|
||||
restored = CloudTemplateRuntime(store, tmp_path)
|
||||
try:
|
||||
restored.restore_jobs()
|
||||
repaired = restored.get_job(queued["job_id"])
|
||||
assert repaired["status"] == "completed"
|
||||
persisted = json.loads(store.get(IMPORT_JOB_KIND, queued["job_id"]))
|
||||
assert persisted["status"] == "completed"
|
||||
finally:
|
||||
restored.close()
|
||||
@ -0,0 +1,743 @@
|
||||
"""Engine-global cloud template catalog and persistent source images.
|
||||
|
||||
The legacy cloud-template flow downloads an image inside one host child app,
|
||||
uploads it immediately to that host, and then deletes the local download. A
|
||||
multi-host control plane needs a different ownership boundary: the engine owns
|
||||
one immutable source image and each hypervisor owns only a disposable replica.
|
||||
|
||||
This module deliberately contains no libvirt code. It provides the global
|
||||
catalog, import jobs and verified ``source()`` leases that a host-specific
|
||||
cache layer can consume later.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import stat
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterator
|
||||
|
||||
from .cloud_templates import (
|
||||
GIB,
|
||||
MAX_IMAGE_BYTES,
|
||||
MAX_VIRTUAL_BYTES,
|
||||
SHA256_RE,
|
||||
TemplateError,
|
||||
download_verified_image,
|
||||
image_version_for,
|
||||
template_id_for,
|
||||
template_volume_name,
|
||||
validate_template_name,
|
||||
)
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
# Keep the existing, already validated template/volume identifier grammar, but
|
||||
# replace its old host scope with one fixed engine-global namespace. The full
|
||||
# digest remains in the catalog and every source/replica marker must compare it;
|
||||
# the shortened digest in the public ID is never treated as integrity proof.
|
||||
GLOBAL_TEMPLATE_SCOPE = hashlib.sha256(
|
||||
b"virt-cloud/global-cloud-template/v1"
|
||||
).hexdigest()
|
||||
|
||||
CATALOG_KIND = "cloud_template"
|
||||
IMPORT_JOB_KIND = "cloud_template_import_job"
|
||||
SOURCE_DIRECTORY_NAME = "cloud-templates"
|
||||
PERSISTENT_FREE_RESERVE_BYTES = 256 * 1024**2
|
||||
MIN_DOWNLOAD_BYTES = 1024 * 1024
|
||||
SOURCE_FILE_RE = re.compile(
|
||||
r"panel-template-alma(?:9|10)-[0-9a-f]{8}-[0-9a-f]{16}\.qcow2\Z"
|
||||
)
|
||||
TEMPORARY_FILE_RE = re.compile(r"\.cloud-image-[A-Za-z0-9_-]+\.qcow2\Z")
|
||||
|
||||
|
||||
def cloud_template_id_for(digest: str, version: str) -> str:
|
||||
"""Return a stable ID that is independent of every target hypervisor."""
|
||||
if not isinstance(digest, str) or not SHA256_RE.fullmatch(digest):
|
||||
raise TemplateError("云镜像 SHA-256 格式无效")
|
||||
return template_id_for(GLOBAL_TEMPLATE_SCOPE, digest.lower(), version)
|
||||
|
||||
|
||||
class GlobalCloudTemplateStore:
|
||||
"""Strict global catalog backed by its own encrypted record kind."""
|
||||
|
||||
def __init__(self, store: Any | None) -> None:
|
||||
self.store = store
|
||||
self._memory: dict[str, str] = {}
|
||||
self._lock = threading.RLock()
|
||||
|
||||
@staticmethod
|
||||
def _validate(record: Any) -> dict[str, Any]:
|
||||
if not isinstance(record, dict):
|
||||
raise TemplateError("全局云镜像目录记录损坏", 500)
|
||||
try:
|
||||
template_id = record["id"]
|
||||
name = record["name"]
|
||||
os_family = record["os_family"]
|
||||
os_version = record["os_version"]
|
||||
digest = record["sha256"]
|
||||
volume_name = record["volume_name"]
|
||||
virtual_size = record["virtual_size"]
|
||||
image_bytes = record["image_bytes"]
|
||||
created_at = record["created_at"]
|
||||
except KeyError as exc:
|
||||
raise TemplateError("全局云镜像目录记录损坏", 500) from exc
|
||||
try:
|
||||
normalized_name = validate_template_name(name)
|
||||
except TemplateError as exc:
|
||||
raise TemplateError("全局云镜像目录记录损坏", 500) from exc
|
||||
valid = (
|
||||
isinstance(template_id, str)
|
||||
and isinstance(name, str)
|
||||
and normalized_name == name
|
||||
and os_family == "AlmaLinux"
|
||||
and os_version in {"9", "10"}
|
||||
and isinstance(digest, str)
|
||||
and SHA256_RE.fullmatch(digest) is not None
|
||||
and digest == digest.lower()
|
||||
and template_id == cloud_template_id_for(digest, os_version)
|
||||
and volume_name == template_volume_name(template_id)
|
||||
and not isinstance(virtual_size, bool)
|
||||
and isinstance(virtual_size, int)
|
||||
and GIB <= virtual_size <= MAX_VIRTUAL_BYTES
|
||||
and not isinstance(image_bytes, bool)
|
||||
and isinstance(image_bytes, int)
|
||||
and 0 < image_bytes <= MAX_IMAGE_BYTES
|
||||
and not isinstance(created_at, bool)
|
||||
and isinstance(created_at, int)
|
||||
and created_at > 0
|
||||
)
|
||||
if not valid:
|
||||
raise TemplateError("全局云镜像目录记录损坏", 500)
|
||||
# Reject unrecognised fields instead of accidentally persisting a URL,
|
||||
# local path, host identity or another future secret in public records.
|
||||
if set(record) != {
|
||||
"id", "name", "os_family", "os_version", "sha256",
|
||||
"volume_name", "virtual_size", "image_bytes", "created_at",
|
||||
}:
|
||||
raise TemplateError("全局云镜像目录记录损坏", 500)
|
||||
return record
|
||||
|
||||
def _source(self) -> dict[str, str]:
|
||||
return self.store.list(CATALOG_KIND) if self.store is not None else self._memory.copy()
|
||||
|
||||
def _list(self) -> list[dict[str, Any]]:
|
||||
records = []
|
||||
for key, payload in self._source().items():
|
||||
try:
|
||||
record = json.loads(payload)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise TemplateError("全局云镜像目录记录损坏", 500) from exc
|
||||
record = self._validate(record)
|
||||
if key != record["id"]:
|
||||
raise TemplateError("全局云镜像目录记录损坏", 500)
|
||||
records.append(record)
|
||||
return sorted(
|
||||
records,
|
||||
key=lambda item: (item["created_at"], item["id"]),
|
||||
reverse=True,
|
||||
)
|
||||
|
||||
def list(self) -> list[dict[str, Any]]:
|
||||
with self._lock:
|
||||
return self._list()
|
||||
|
||||
def get(self, template_id: str) -> dict[str, Any]:
|
||||
# This validates the opaque identifier before consulting storage.
|
||||
template_volume_name(template_id)
|
||||
with self._lock:
|
||||
for record in self._list():
|
||||
if record["id"] == template_id:
|
||||
return record
|
||||
raise TemplateError("云镜像模板不存在", 404)
|
||||
|
||||
def add(self, record: dict[str, Any]) -> dict[str, Any]:
|
||||
record = self._validate(record.copy())
|
||||
encoded = json.dumps(record, ensure_ascii=False, separators=(",", ":"))
|
||||
with self._lock:
|
||||
current = self._list()
|
||||
if any(row["id"] == record["id"] for row in current):
|
||||
raise TemplateError("此云镜像已经导入", 409)
|
||||
if any(row["name"].casefold() == record["name"].casefold()
|
||||
for row in current):
|
||||
raise TemplateError("已有同名云镜像模板", 409)
|
||||
if self.store is not None:
|
||||
if not self.store.insert(CATALOG_KIND, record["id"], encoded):
|
||||
raise TemplateError("此云镜像已经导入", 409)
|
||||
else:
|
||||
self._memory[record["id"]] = encoded
|
||||
return record
|
||||
|
||||
def rename(self, template_id: str, name: str) -> dict[str, Any]:
|
||||
name = validate_template_name(name)
|
||||
with self._lock:
|
||||
current = self._list()
|
||||
target = next((row for row in current if row["id"] == template_id), None)
|
||||
if target is None:
|
||||
raise TemplateError("云镜像模板不存在", 404)
|
||||
if any(row["id"] != template_id
|
||||
and row["name"].casefold() == name.casefold()
|
||||
for row in current):
|
||||
raise TemplateError("已有同名云镜像模板", 409)
|
||||
updated = {**target, "name": name}
|
||||
encoded = json.dumps(updated, ensure_ascii=False, separators=(",", ":"))
|
||||
if self.store is not None:
|
||||
self.store.set(CATALOG_KIND, template_id, encoded)
|
||||
else:
|
||||
self._memory[template_id] = encoded
|
||||
return updated
|
||||
|
||||
|
||||
class GlobalCloudTemplateFiles:
|
||||
"""Private engine-owned source files, never paths supplied by a caller."""
|
||||
|
||||
def __init__(self, data_dir: Path) -> None:
|
||||
data_dir = Path(data_dir)
|
||||
if not data_dir.is_absolute() or data_dir == Path("/"):
|
||||
raise ValueError("cloud template data directory must be absolute and dedicated")
|
||||
self.data_dir = data_dir
|
||||
self.root = data_dir / SOURCE_DIRECTORY_NAME
|
||||
|
||||
@staticmethod
|
||||
def _ensure_directory(path: Path) -> None:
|
||||
try:
|
||||
info = path.lstat()
|
||||
except FileNotFoundError:
|
||||
path.mkdir(mode=0o700)
|
||||
info = path.lstat()
|
||||
if (not stat.S_ISDIR(info.st_mode) or stat.S_ISLNK(info.st_mode)
|
||||
or info.st_uid != os.geteuid()):
|
||||
raise TemplateError("云镜像持久目录不安全,请联系管理员", 500)
|
||||
if stat.S_IMODE(info.st_mode) != 0o700:
|
||||
os.chmod(path, 0o700)
|
||||
|
||||
def directory(self) -> Path:
|
||||
self.data_dir.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
self._ensure_directory(self.data_dir)
|
||||
self._ensure_directory(self.root)
|
||||
return self.root
|
||||
|
||||
def available_bytes(self) -> int:
|
||||
"""Return writable bytes while preserving space for panel operation."""
|
||||
free = int(shutil.disk_usage(self.directory()).free)
|
||||
return max(0, free - PERSISTENT_FREE_RESERVE_BYTES)
|
||||
|
||||
def path(self, template_id: str, *, expected_size: int | None = None) -> Path:
|
||||
path = self.directory() / template_volume_name(template_id)
|
||||
try:
|
||||
info = path.lstat()
|
||||
except FileNotFoundError as exc:
|
||||
raise TemplateError("云镜像源文件不存在,请重新导入", 409) from exc
|
||||
if (not stat.S_ISREG(info.st_mode) or stat.S_ISLNK(info.st_mode)
|
||||
or info.st_uid != os.geteuid()
|
||||
or stat.S_IMODE(info.st_mode) != 0o600
|
||||
or (expected_size is not None and info.st_size != expected_size)):
|
||||
raise TemplateError("云镜像源文件已损坏,请联系管理员", 500)
|
||||
return path
|
||||
|
||||
def publish(self, temporary: Path, template_id: str, *, expected_size: int) -> Path:
|
||||
temporary = Path(temporary)
|
||||
directory = self.directory()
|
||||
try:
|
||||
info = temporary.lstat()
|
||||
except FileNotFoundError as exc:
|
||||
raise TemplateError("云镜像下载临时文件丢失", 500) from exc
|
||||
if (not stat.S_ISREG(info.st_mode) or stat.S_ISLNK(info.st_mode)
|
||||
or info.st_uid != os.geteuid()
|
||||
or stat.S_IMODE(info.st_mode) != 0o600
|
||||
or temporary.parent != directory
|
||||
or info.st_size != expected_size
|
||||
or expected_size <= 0 or expected_size > MAX_IMAGE_BYTES):
|
||||
raise TemplateError("云镜像下载临时文件无效", 500)
|
||||
destination = directory / template_volume_name(template_id)
|
||||
try:
|
||||
# download_verified_image creates the temporary in this directory.
|
||||
# A hard link publishes atomically and cannot replace an existing
|
||||
# immutable source with attacker-controlled bytes.
|
||||
os.link(temporary, destination)
|
||||
os.chmod(destination, 0o600)
|
||||
temporary.unlink()
|
||||
descriptor = os.open(
|
||||
destination.parent,
|
||||
os.O_RDONLY | getattr(os, "O_DIRECTORY", 0),
|
||||
)
|
||||
try:
|
||||
os.fsync(descriptor)
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
except FileExistsError as exc:
|
||||
raise TemplateError("此云镜像已经导入", 409) from exc
|
||||
except OSError as exc:
|
||||
destination.unlink(missing_ok=True)
|
||||
raise TemplateError("云镜像无法保存到持久目录", 507) from exc
|
||||
return destination
|
||||
|
||||
def delete(self, template_id: str) -> None:
|
||||
path = self.directory() / template_volume_name(template_id)
|
||||
try:
|
||||
info = path.lstat()
|
||||
except FileNotFoundError:
|
||||
return
|
||||
if not stat.S_ISREG(info.st_mode) or stat.S_ISLNK(info.st_mode):
|
||||
raise TemplateError("云镜像源文件无法安全删除", 500)
|
||||
path.unlink()
|
||||
|
||||
def reconcile(self, catalog: GlobalCloudTemplateStore) -> None:
|
||||
"""Remove only provable interrupted downloads and uncataloged sources."""
|
||||
try:
|
||||
self.root.lstat()
|
||||
except FileNotFoundError:
|
||||
return
|
||||
self._ensure_directory(self.root)
|
||||
active = {row["volume_name"] for row in catalog.list()}
|
||||
for candidate in self.root.iterdir():
|
||||
try:
|
||||
info = candidate.lstat()
|
||||
except OSError:
|
||||
continue
|
||||
if (not stat.S_ISREG(info.st_mode) or stat.S_ISLNK(info.st_mode)
|
||||
or info.st_uid != os.geteuid()):
|
||||
continue
|
||||
interrupted = TEMPORARY_FILE_RE.fullmatch(candidate.name) is not None
|
||||
orphan = (SOURCE_FILE_RE.fullmatch(candidate.name) is not None
|
||||
and candidate.name not in active)
|
||||
if interrupted or orphan:
|
||||
candidate.unlink(missing_ok=True)
|
||||
|
||||
|
||||
class CloudTemplateRuntime:
|
||||
"""One engine-process runtime shared by all host child applications."""
|
||||
|
||||
def __init__(self, store: Any | None, data_dir: Path) -> None:
|
||||
self.store = store
|
||||
self.catalog = GlobalCloudTemplateStore(store)
|
||||
self.files = GlobalCloudTemplateFiles(data_dir)
|
||||
self.executor = ThreadPoolExecutor(
|
||||
max_workers=1, thread_name_prefix="global-cloud-template-import"
|
||||
)
|
||||
self.lock = threading.RLock()
|
||||
self.jobs: dict[str, dict[str, Any]] = {}
|
||||
self.pending: set[str] = set()
|
||||
self.in_use: dict[str, int] = {}
|
||||
self.verified_sources: dict[str, tuple[int, int, int, int, str]] = {}
|
||||
self._restored = False
|
||||
|
||||
@staticmethod
|
||||
def public_record(record: dict[str, Any]) -> dict[str, Any]:
|
||||
return {key: value for key, value in record.items() if key in {
|
||||
"id", "name", "os_family", "os_version", "sha256",
|
||||
"virtual_size", "image_bytes", "created_at",
|
||||
}}
|
||||
|
||||
@staticmethod
|
||||
def public_error(value: Any) -> str:
|
||||
safe = {
|
||||
"面板在云镜像导入期间重启;请重新提交导入",
|
||||
"云镜像地址无效",
|
||||
"仅支持 AlmaLinux 官方仓库的 9/10 x86_64 GenericCloud qcow2 镜像",
|
||||
"官方镜像地址发生重定向,请使用最终的官方 HTTPS 直链",
|
||||
"SHA-256 可留空;填写时必须为 64 位十六进制校验值",
|
||||
"下载官方云镜像失败,请检查面板服务器网络",
|
||||
"云镜像下载超过 60 分钟,请检查面板服务器网络",
|
||||
"云镜像超过 4 GiB 限制",
|
||||
"云镜像 SHA-256 不匹配,已丢弃下载内容",
|
||||
"下载文件不是有效的 qcow2 镜像",
|
||||
"云镜像 qcow2 版本或簇大小不受支持",
|
||||
"云镜像 qcow2 头部不完整",
|
||||
"云镜像包含不受支持或外部依赖的 qcow2 特性",
|
||||
"云镜像含外部依赖、加密或容量超出支持范围",
|
||||
"此云镜像已经导入",
|
||||
"已有同名云镜像模板",
|
||||
"云镜像无法保存到持久目录",
|
||||
"面板云镜像持久目录空间不足",
|
||||
"云镜像导入失败,请查看引擎日志",
|
||||
}
|
||||
if isinstance(value, str) and value in safe:
|
||||
return value
|
||||
return "云镜像下载或保存失败,请联系管理员查看日志"
|
||||
|
||||
@classmethod
|
||||
def public_job(cls, job: dict[str, Any]) -> dict[str, Any]:
|
||||
allowed = {
|
||||
"job_id", "status", "phase", "created_at", "started_at",
|
||||
"finished_at", "template", "error",
|
||||
}
|
||||
result = {key: value for key, value in job.items() if key in allowed}
|
||||
if isinstance(result.get("template"), dict):
|
||||
result["template"] = cls.public_record(result["template"])
|
||||
if "error" in result:
|
||||
result["error"] = cls.public_error(result["error"])
|
||||
return result
|
||||
|
||||
def list(self) -> list[dict[str, Any]]:
|
||||
"""Return the public, host-independent template list."""
|
||||
return [self.public_record(row) for row in self.catalog.list()]
|
||||
|
||||
def status(self) -> dict[str, Any]:
|
||||
"""Return the public list plus the current import, when one exists."""
|
||||
result: dict[str, Any] = {"templates": self.list()}
|
||||
with self.lock:
|
||||
active = [job for job in self.jobs.values()
|
||||
if job.get("status") in {"queued", "running"}]
|
||||
if active:
|
||||
current = max(active, key=lambda job: (
|
||||
int(job.get("created_at", 0)), str(job.get("job_id", ""))))
|
||||
result["active_import_job"] = self.public_job(current)
|
||||
return result
|
||||
|
||||
def get(self, template_id: str) -> dict[str, Any]:
|
||||
return self.catalog.get(template_id)
|
||||
|
||||
def get_job(self, job_id: str) -> dict[str, Any]:
|
||||
with self.lock:
|
||||
job = self.jobs.get(job_id)
|
||||
if job is None:
|
||||
raise TemplateError("云镜像导入任务不存在", 404)
|
||||
return self.public_job(job)
|
||||
|
||||
def rename(self, template_id: str, name: str) -> dict[str, Any]:
|
||||
with self.lock:
|
||||
if self.pending:
|
||||
raise TemplateError("云镜像正在导入,请稍后修改模板名称", 409)
|
||||
return self.public_record(self.catalog.rename(template_id, name))
|
||||
|
||||
def _remaining_download_bytes(self) -> int:
|
||||
maximum = min(MAX_IMAGE_BYTES, self.files.available_bytes())
|
||||
if maximum < MIN_DOWNLOAD_BYTES:
|
||||
raise TemplateError("面板云镜像持久目录空间不足", 507)
|
||||
return maximum
|
||||
|
||||
def queue_import(self, url: str, name: str,
|
||||
expected_sha256: str = "") -> dict[str, str]:
|
||||
version = image_version_for(url)
|
||||
name = validate_template_name(name)
|
||||
if (not isinstance(expected_sha256, str)
|
||||
or (expected_sha256 and not SHA256_RE.fullmatch(expected_sha256))):
|
||||
raise TemplateError(
|
||||
"SHA-256 可留空;填写时必须为 64 位十六进制校验值", 422
|
||||
)
|
||||
expected_sha256 = expected_sha256.lower()
|
||||
pending_key = expected_sha256 or hashlib.sha256(url.encode("utf-8")).hexdigest()
|
||||
with self.lock:
|
||||
existing = self.catalog.list()
|
||||
self._remaining_download_bytes()
|
||||
if expected_sha256 and any(
|
||||
row["id"] == cloud_template_id_for(expected_sha256, version)
|
||||
for row in existing):
|
||||
raise TemplateError("此云镜像已经导入", 409)
|
||||
if any(row["name"].casefold() == name.casefold() for row in existing):
|
||||
raise TemplateError("已有同名云镜像模板", 409)
|
||||
if pending_key in self.pending:
|
||||
raise TemplateError("该云镜像正在导入", 409)
|
||||
if self.pending:
|
||||
raise TemplateError(
|
||||
"已有云镜像正在导入,请等待完成后再提交", 409
|
||||
)
|
||||
job_id = str(uuid.uuid4())
|
||||
job = {
|
||||
"job_id": job_id,
|
||||
"status": "queued",
|
||||
"phase": "等待下载",
|
||||
"created_at": int(time.time()),
|
||||
}
|
||||
self.pending.add(pending_key)
|
||||
try:
|
||||
self.save_job(job_id, job)
|
||||
self.executor.submit(
|
||||
self._run_import,
|
||||
job_id,
|
||||
url,
|
||||
name,
|
||||
expected_sha256,
|
||||
pending_key,
|
||||
)
|
||||
except Exception as exc:
|
||||
self.pending.discard(pending_key)
|
||||
self.delete_job(job_id)
|
||||
if isinstance(exc, RuntimeError):
|
||||
raise TemplateError(
|
||||
"引擎正在停止,无法开始云镜像导入", 503
|
||||
) from exc
|
||||
raise
|
||||
return {"job_id": job_id, "status": "queued"}
|
||||
|
||||
def _run_import(self, job_id: str, url: str, name: str,
|
||||
expected_sha256: str, pending_key: str) -> None:
|
||||
temporary: Path | None = None
|
||||
published = False
|
||||
cataloged = False
|
||||
commit_uncertain = False
|
||||
template_id = ""
|
||||
record: dict[str, Any] | None = None
|
||||
try:
|
||||
with self.lock:
|
||||
job = self.jobs[job_id]
|
||||
job.update(
|
||||
status="running",
|
||||
phase="正在下载并校验云镜像",
|
||||
started_at=int(time.time()),
|
||||
)
|
||||
self.save_job(job_id, job)
|
||||
directory = self.files.directory()
|
||||
maximum = self._remaining_download_bytes()
|
||||
temporary, virtual_size, actual_sha256 = download_verified_image(
|
||||
url, expected_sha256, directory, max_bytes=maximum
|
||||
)
|
||||
version = image_version_for(url)
|
||||
template_id = cloud_template_id_for(actual_sha256, version)
|
||||
size = temporary.stat().st_size
|
||||
record = {
|
||||
"id": template_id,
|
||||
"name": name,
|
||||
"os_family": "AlmaLinux",
|
||||
"os_version": version,
|
||||
"sha256": actual_sha256,
|
||||
"volume_name": template_volume_name(template_id),
|
||||
"virtual_size": virtual_size,
|
||||
"image_bytes": size,
|
||||
"created_at": int(time.time()),
|
||||
}
|
||||
with self.lock:
|
||||
if any(row["id"] == template_id for row in self.catalog.list()):
|
||||
raise TemplateError("此云镜像已经导入", 409)
|
||||
if any(row["name"].casefold() == name.casefold()
|
||||
for row in self.catalog.list()):
|
||||
raise TemplateError("已有同名云镜像模板", 409)
|
||||
job = self.jobs[job_id]
|
||||
job["phase"] = "正在保存云镜像源文件"
|
||||
# Persist the deterministic candidate before publishing. If
|
||||
# the process stops after the catalog commit but before the
|
||||
# terminal job write, startup can prove that this interrupted
|
||||
# job owns the committed catalog/source pair.
|
||||
job["candidate_template_id"] = template_id
|
||||
self.save_job(job_id, job)
|
||||
source = self.files.publish(
|
||||
temporary, template_id, expected_size=size
|
||||
)
|
||||
temporary = None
|
||||
published = True
|
||||
self._mark_verified(template_id, source, actual_sha256)
|
||||
self.catalog.add(record)
|
||||
cataloged = True
|
||||
self._save_completed_job(job_id, job, record)
|
||||
except Exception as exc:
|
||||
# A database insert can commit and still raise (for example when
|
||||
# the connection drops before its acknowledgement reaches this
|
||||
# process). Once the source was published, never delete it until
|
||||
# the catalog has been read back. An exact record is proof of a
|
||||
# completed import; an unavailable read is ambiguous and must
|
||||
# preserve the source for startup reconciliation.
|
||||
committed: dict[str, Any] | None = None
|
||||
if published and template_id and not cataloged:
|
||||
try:
|
||||
existing = self.catalog.get(template_id)
|
||||
except TemplateError as read_exc:
|
||||
if read_exc.status != 404:
|
||||
commit_uncertain = True
|
||||
except Exception:
|
||||
commit_uncertain = True
|
||||
LOG.exception(
|
||||
"Could not resolve cloud template catalog commit: job=%s",
|
||||
job_id,
|
||||
)
|
||||
else:
|
||||
cataloged = True
|
||||
if record is not None and existing == record:
|
||||
committed = existing
|
||||
if committed is not None:
|
||||
with self.lock:
|
||||
previous = self.jobs.get(job_id, {})
|
||||
self._save_completed_job(job_id, previous, committed)
|
||||
return
|
||||
if isinstance(exc, TemplateError):
|
||||
error = str(exc)
|
||||
else:
|
||||
LOG.exception("Global cloud template import failed: job=%s", job_id)
|
||||
error = "云镜像导入失败,请查看引擎日志"
|
||||
with self.lock:
|
||||
if cataloged or commit_uncertain:
|
||||
# A catalog write is the commit point. Do not remove a
|
||||
# committed (or ambiguously committed) template merely
|
||||
# because its acknowledgement failed. Startup can prove
|
||||
# the candidate from the persisted running job.
|
||||
pass
|
||||
elif published and template_id:
|
||||
try:
|
||||
self.files.delete(template_id)
|
||||
except Exception:
|
||||
LOG.exception(
|
||||
"Could not remove failed cloud template source: job=%s",
|
||||
job_id,
|
||||
)
|
||||
previous = self.jobs.get(job_id, {})
|
||||
failed = {
|
||||
"job_id": job_id,
|
||||
"status": "failed",
|
||||
"phase": "导入失败",
|
||||
"error": error,
|
||||
"created_at": previous.get("created_at", int(time.time())),
|
||||
"started_at": previous.get("started_at"),
|
||||
"finished_at": int(time.time()),
|
||||
}
|
||||
if template_id:
|
||||
failed["candidate_template_id"] = template_id
|
||||
if commit_uncertain:
|
||||
failed["commit_uncertain"] = True
|
||||
try:
|
||||
self.save_job(job_id, failed)
|
||||
except Exception:
|
||||
LOG.exception(
|
||||
"Could not persist cloud template import failure: job=%s",
|
||||
job_id,
|
||||
)
|
||||
finally:
|
||||
if temporary is not None:
|
||||
temporary.unlink(missing_ok=True)
|
||||
with self.lock:
|
||||
self.pending.discard(pending_key)
|
||||
|
||||
def _save_completed_job(self, job_id: str, previous: dict[str, Any],
|
||||
record: dict[str, Any]) -> None:
|
||||
"""Publish an in-memory success even if its final DB write fails.
|
||||
|
||||
The catalog record is the durable commit point. Reporting a failed
|
||||
import after that point invites a duplicate retry, so the running
|
||||
process must expose success while startup reconstructs the same result
|
||||
from the candidate ID if the terminal job write was lost.
|
||||
"""
|
||||
completed = {
|
||||
"job_id": job_id,
|
||||
"status": "completed",
|
||||
"phase": "云镜像已保存,可用于所有宿主机",
|
||||
"created_at": previous.get("created_at", int(time.time())),
|
||||
"started_at": previous.get("started_at"),
|
||||
"finished_at": int(time.time()),
|
||||
"template": record,
|
||||
}
|
||||
try:
|
||||
self.save_job(job_id, completed)
|
||||
except Exception:
|
||||
# save_job updates self.jobs before touching persistent storage.
|
||||
# Keep that completed state and let restore_jobs repair a stale
|
||||
# persisted running record on the next process start.
|
||||
self.jobs[job_id] = completed
|
||||
LOG.exception(
|
||||
"Could not persist completed cloud template import: job=%s",
|
||||
job_id,
|
||||
)
|
||||
|
||||
def _mark_verified(self, template_id: str, path: Path, digest: str) -> None:
|
||||
info = path.stat()
|
||||
self.verified_sources[template_id] = (
|
||||
info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns, digest
|
||||
)
|
||||
|
||||
def _verify_source(self, template_id: str, path: Path, digest: str) -> None:
|
||||
info = path.stat()
|
||||
identity = (info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns)
|
||||
cached = self.verified_sources.get(template_id)
|
||||
if cached is not None and cached[:4] == identity and cached[4] == digest:
|
||||
return
|
||||
checksum = hashlib.sha256()
|
||||
with path.open("rb") as source:
|
||||
while chunk := source.read(1024 * 1024):
|
||||
checksum.update(chunk)
|
||||
if checksum.hexdigest() != digest:
|
||||
raise TemplateError("云镜像源文件的 SHA-256 校验值不匹配", 500)
|
||||
self._mark_verified(template_id, path, digest)
|
||||
|
||||
@contextmanager
|
||||
def source(self, template_id: str) -> Iterator[tuple[dict[str, Any], Path]]:
|
||||
with self.lock:
|
||||
record = self.catalog.get(template_id)
|
||||
path = self.files.path(template_id, expected_size=record["image_bytes"])
|
||||
self._verify_source(template_id, path, record["sha256"])
|
||||
self.in_use[template_id] = self.in_use.get(template_id, 0) + 1
|
||||
try:
|
||||
yield record, path
|
||||
finally:
|
||||
with self.lock:
|
||||
count = self.in_use.get(template_id, 0)
|
||||
if count <= 1:
|
||||
self.in_use.pop(template_id, None)
|
||||
else:
|
||||
self.in_use[template_id] = count - 1
|
||||
|
||||
def save_job(self, job_id: str, job: dict[str, Any]) -> None:
|
||||
self.jobs[job_id] = job
|
||||
if self.store is not None:
|
||||
self.store.set(
|
||||
IMPORT_JOB_KIND,
|
||||
job_id,
|
||||
json.dumps(job, ensure_ascii=False, separators=(",", ":")),
|
||||
)
|
||||
|
||||
def delete_job(self, job_id: str) -> None:
|
||||
self.jobs.pop(job_id, None)
|
||||
if self.store is not None:
|
||||
self.store.delete(IMPORT_JOB_KIND, job_id)
|
||||
|
||||
def restore_jobs(self) -> None:
|
||||
"""Reconcile local files and fail imports interrupted by a restart."""
|
||||
with self.lock:
|
||||
if self._restored:
|
||||
return
|
||||
self.files.reconcile(self.catalog)
|
||||
source = self.store.list(IMPORT_JOB_KIND) if self.store is not None else {}
|
||||
for job_id, raw in source.items():
|
||||
try:
|
||||
job = json.loads(raw)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise TemplateError("云镜像导入任务记录损坏", 500) from exc
|
||||
if not isinstance(job, dict) or job.get("job_id") != job_id:
|
||||
raise TemplateError("云镜像导入任务记录损坏", 500)
|
||||
interrupted = job.get("status") in {"queued", "running"}
|
||||
uncertain = job.get("commit_uncertain") is True
|
||||
if interrupted or uncertain:
|
||||
candidate = job.get("candidate_template_id")
|
||||
committed: dict[str, Any] | None = None
|
||||
if isinstance(candidate, str):
|
||||
try:
|
||||
committed = self.catalog.get(candidate)
|
||||
path = self.files.path(
|
||||
candidate,
|
||||
expected_size=committed["image_bytes"],
|
||||
)
|
||||
self._verify_source(
|
||||
candidate, path, committed["sha256"]
|
||||
)
|
||||
except Exception:
|
||||
committed = None
|
||||
if committed is not None:
|
||||
self._save_completed_job(job_id, job, committed)
|
||||
continue
|
||||
job.update(
|
||||
status="failed",
|
||||
phase="引擎重启,任务已中断",
|
||||
error="面板在云镜像导入期间重启;请重新提交导入",
|
||||
finished_at=int(time.time()),
|
||||
)
|
||||
self.save_job(job_id, job)
|
||||
self.jobs[job_id] = job
|
||||
self._restored = True
|
||||
|
||||
def has_active_jobs(self) -> bool:
|
||||
with self.lock:
|
||||
return bool(self.pending) or any(
|
||||
job.get("status") in {"queued", "running"}
|
||||
for job in self.jobs.values()
|
||||
)
|
||||
|
||||
def close(self) -> None:
|
||||
self.executor.shutdown(wait=True, cancel_futures=False)
|
||||
@ -1 +1 @@
|
||||
{assign var=venix_asset_version value='2026092807' scope='global'}
|
||||
{assign var=venix_asset_version value='2026092809' scope='global'}
|
||||
|
||||
Loading…
Reference in New Issue