You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
593 lines
20 KiB
Python
593 lines
20 KiB
Python
"""Global cloud-template catalog, source file and import job tests."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import stat
|
|
import threading
|
|
import time
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
from unittest.mock import patch
|
|
|
|
import pytest
|
|
|
|
from virsh_panel.cloud_template_runtime import (
|
|
CATALOG_KIND,
|
|
IMPORT_JOB_KIND,
|
|
CloudTemplateRuntime,
|
|
GlobalCloudTemplateFiles,
|
|
GlobalCloudTemplateStore,
|
|
PERSISTENT_FREE_RESERVE_BYTES,
|
|
cloud_template_id_for,
|
|
)
|
|
from virsh_panel.cloud_templates import GIB, TemplateError, template_volume_name
|
|
|
|
|
|
ALMA_10_URL = (
|
|
"https://repo.almalinux.org/almalinux/10/cloud/x86_64/images/"
|
|
"AlmaLinux-10-GenericCloud-latest.x86_64.qcow2"
|
|
)
|
|
ALMA_8_URL = (
|
|
"https://repo.almalinux.org/almalinux/8/cloud/x86_64/images/"
|
|
"AlmaLinux-8-GenericCloud-latest.x86_64.qcow2"
|
|
)
|
|
DEBIAN_12_URL = (
|
|
"https://cloud.debian.org/images/cloud/bookworm/latest/"
|
|
"debian-12-generic-amd64.qcow2"
|
|
)
|
|
UBUNTU_24_URL = (
|
|
"https://cloud-images.ubuntu.com/releases/24.04/release/"
|
|
"ubuntu-24.04-server-cloudimg-amd64.img"
|
|
)
|
|
CENTOS_STREAM_10_URL = (
|
|
"https://cloud.centos.org/centos/10-stream/x86_64/images/"
|
|
"CentOS-Stream-GenericCloud-10-latest.x86_64.qcow2"
|
|
)
|
|
|
|
|
|
class MemoryStore:
|
|
def __init__(self) -> None:
|
|
self.values: dict[tuple[str, str], str] = {}
|
|
self.lock = threading.RLock()
|
|
|
|
def list(self, kind: str) -> dict[str, str]:
|
|
with self.lock:
|
|
return {
|
|
key: value for (record_kind, key), value in self.values.items()
|
|
if record_kind == kind
|
|
}
|
|
|
|
def get(self, kind: str, key: str) -> str | None:
|
|
with self.lock:
|
|
return self.values.get((kind, key))
|
|
|
|
def set(self, kind: str, key: str, value: str) -> None:
|
|
with self.lock:
|
|
self.values[(kind, key)] = value
|
|
|
|
def insert(self, kind: str, key: str, value: str) -> bool:
|
|
with self.lock:
|
|
identity = (kind, key)
|
|
if identity in self.values:
|
|
return False
|
|
self.values[identity] = value
|
|
return True
|
|
|
|
def delete(self, kind: str, key: str) -> None:
|
|
with self.lock:
|
|
self.values.pop((kind, key), None)
|
|
|
|
|
|
class CommitThenRaiseCatalogStore(MemoryStore):
|
|
"""Simulate a committed INSERT whose acknowledgement is lost."""
|
|
|
|
def insert(self, kind: str, key: str, value: str) -> bool:
|
|
inserted = super().insert(kind, key, value)
|
|
if kind == CATALOG_KIND and inserted:
|
|
raise RuntimeError("catalog insert acknowledgement lost")
|
|
return inserted
|
|
|
|
|
|
class AmbiguousCatalogCommitStore(CommitThenRaiseCatalogStore):
|
|
"""Also fail the first catalog read after the ambiguous INSERT."""
|
|
|
|
def __init__(self) -> None:
|
|
super().__init__()
|
|
self.fail_next_catalog_read = False
|
|
|
|
def list(self, kind: str) -> dict[str, str]:
|
|
if kind == CATALOG_KIND and self.fail_next_catalog_read:
|
|
self.fail_next_catalog_read = False
|
|
raise RuntimeError("catalog read unavailable")
|
|
return super().list(kind)
|
|
|
|
def insert(self, kind: str, key: str, value: str) -> bool:
|
|
try:
|
|
return super().insert(kind, key, value)
|
|
except RuntimeError:
|
|
self.fail_next_catalog_read = True
|
|
raise
|
|
|
|
|
|
class LoseFirstCompletedJobWriteStore(MemoryStore):
|
|
"""Drop one terminal job update while retaining its running record."""
|
|
|
|
def __init__(self) -> None:
|
|
super().__init__()
|
|
self.lost_completed_write = False
|
|
|
|
def set(self, kind: str, key: str, value: str) -> None:
|
|
if kind == IMPORT_JOB_KIND:
|
|
job = json.loads(value)
|
|
if (job.get("status") == "completed"
|
|
and not self.lost_completed_write):
|
|
self.lost_completed_write = True
|
|
raise RuntimeError("completed job write acknowledgement lost")
|
|
super().set(kind, key, value)
|
|
|
|
|
|
def wait_for_job(runtime: CloudTemplateRuntime, job_id: str) -> dict:
|
|
for _ in range(200):
|
|
job = runtime.get_job(job_id)
|
|
if job["status"] in {"completed", "failed"}:
|
|
return job
|
|
time.sleep(0.01)
|
|
raise AssertionError("cloud template import did not finish")
|
|
|
|
|
|
def fake_download_bytes(payload: bytes):
|
|
digest = hashlib.sha256(payload).hexdigest()
|
|
|
|
def download(_url: str, expected_sha256: str, directory: Path, **_kwargs):
|
|
assert expected_sha256 in {"", digest}
|
|
source = Path(directory) / ".cloud-image-test.qcow2"
|
|
source.write_bytes(payload)
|
|
os.chmod(source, 0o600)
|
|
return source, 2 * GIB, digest
|
|
|
|
return digest, download
|
|
|
|
|
|
def record_for(payload: bytes, *, name: str = "AlmaLinux 10") -> dict:
|
|
digest = hashlib.sha256(payload).hexdigest()
|
|
template_id = cloud_template_id_for(digest, "10")
|
|
return {
|
|
"id": template_id,
|
|
"name": name,
|
|
"os_family": "AlmaLinux",
|
|
"os_version": "10",
|
|
"sha256": digest,
|
|
"volume_name": template_volume_name(template_id),
|
|
"virtual_size": 2 * GIB,
|
|
"image_bytes": len(payload),
|
|
"created_at": 1,
|
|
}
|
|
|
|
|
|
def test_stable_global_id_and_catalog_use_an_independent_record_kind():
|
|
digest = "a" * 64
|
|
first = cloud_template_id_for(digest, "10")
|
|
second = cloud_template_id_for(digest.upper(), "10")
|
|
assert first == second
|
|
assert first.startswith("alma10-")
|
|
assert template_volume_name(first).endswith(".qcow2")
|
|
|
|
store = MemoryStore()
|
|
catalog = GlobalCloudTemplateStore(store)
|
|
record = record_for(b"catalog-image")
|
|
catalog.add(record)
|
|
|
|
assert catalog.get(record["id"]) == record
|
|
assert list(store.list(CATALOG_KIND)) == [record["id"]]
|
|
assert store.list("template") == {}
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"message",
|
|
[
|
|
"官方云镜像地址无法解析",
|
|
"官方云镜像地址解析结果无效",
|
|
"官方云镜像地址解析到非公网地址",
|
|
"无法连接官方云镜像下载地址",
|
|
"官方云镜像重定向目标不安全",
|
|
"官方云镜像重定向次数过多",
|
|
"官方云镜像下载服务器返回异常状态",
|
|
"官方云镜像下载响应格式无效",
|
|
"云镜像下载内容不完整",
|
|
],
|
|
)
|
|
def test_public_job_preserves_fixed_safe_download_errors(message):
|
|
job = {
|
|
"job_id": "safe-error",
|
|
"status": "failed",
|
|
"phase": "导入失败",
|
|
"created_at": 1,
|
|
"error": message,
|
|
}
|
|
assert CloudTemplateRuntime.public_job(job)["error"] == message
|
|
|
|
|
|
def test_import_publishes_private_source_and_restores_global_catalog(tmp_path):
|
|
store = MemoryStore()
|
|
runtime = CloudTemplateRuntime(store, tmp_path)
|
|
payload = b"verified-global-cloud-image"
|
|
digest, downloader = fake_download_bytes(payload)
|
|
try:
|
|
with patch(
|
|
"virsh_panel.cloud_template_runtime.download_verified_image",
|
|
side_effect=downloader,
|
|
):
|
|
queued = runtime.queue_import(
|
|
ALMA_10_URL, "AlmaLinux 10", digest
|
|
)
|
|
job = wait_for_job(runtime, queued["job_id"])
|
|
|
|
assert job["status"] == "completed", job
|
|
template_id = job["template"]["id"]
|
|
assert template_id == cloud_template_id_for(digest, "10")
|
|
assert "volume_name" not in job["template"]
|
|
listed = runtime.list()
|
|
assert listed == [job["template"]]
|
|
assert "active_import_job" not in runtime.status()
|
|
|
|
root = tmp_path / "cloud-templates"
|
|
source = root / template_volume_name(template_id)
|
|
assert source.read_bytes() == payload
|
|
assert stat.S_IMODE(root.stat().st_mode) == 0o700
|
|
assert stat.S_IMODE(source.stat().st_mode) == 0o600
|
|
assert store.get(IMPORT_JOB_KIND, queued["job_id"]) is not None
|
|
|
|
# A new runtime reads the same host-independent catalog and verifies
|
|
# the persisted source from disk instead of trusting process memory.
|
|
restored = CloudTemplateRuntime(store, tmp_path)
|
|
try:
|
|
restored.restore_jobs()
|
|
assert restored.list() == listed
|
|
with restored.source(template_id) as (record, path):
|
|
assert record["sha256"] == digest
|
|
assert path == source
|
|
assert restored.in_use[template_id] == 1
|
|
assert template_id not in restored.in_use
|
|
finally:
|
|
restored.close()
|
|
finally:
|
|
runtime.close()
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("url", "family", "version", "prefix"),
|
|
[
|
|
(ALMA_8_URL, "AlmaLinux", "8", "alma8-"),
|
|
(DEBIAN_12_URL, "Debian", "12", "debian12-"),
|
|
(UBUNTU_24_URL, "Ubuntu", "24.04", "ubuntu2404-"),
|
|
(CENTOS_STREAM_10_URL, "CentOS Stream", "10", "centosstream10-"),
|
|
],
|
|
)
|
|
def test_import_derives_family_version_and_safe_source_name(
|
|
tmp_path, url, family, version, prefix):
|
|
runtime = CloudTemplateRuntime(MemoryStore(), tmp_path)
|
|
payload = f"{family}-{version}-cloud-image".encode()
|
|
digest, downloader = fake_download_bytes(payload)
|
|
try:
|
|
with patch(
|
|
"virsh_panel.cloud_template_runtime.download_verified_image",
|
|
side_effect=downloader,
|
|
):
|
|
queued = runtime.queue_import(url, f"{family} {version}", digest)
|
|
job = wait_for_job(runtime, queued["job_id"])
|
|
|
|
assert job["status"] == "completed", job
|
|
template = job["template"]
|
|
assert template["id"].startswith(prefix)
|
|
assert template["os_family"] == family
|
|
assert template["os_version"] == version
|
|
internal = runtime.get(template["id"])
|
|
assert internal["volume_name"] == template_volume_name(template["id"])
|
|
assert (
|
|
tmp_path / "cloud-templates" / internal["volume_name"]
|
|
).read_bytes() == payload
|
|
finally:
|
|
runtime.close()
|
|
|
|
|
|
def test_available_bytes_preserves_panel_disk_reserve(tmp_path):
|
|
files = GlobalCloudTemplateFiles(tmp_path)
|
|
with patch(
|
|
"virsh_panel.cloud_template_runtime.shutil.disk_usage",
|
|
return_value=SimpleNamespace(
|
|
free=PERSISTENT_FREE_RESERVE_BYTES + 3 * 1024 * 1024
|
|
),
|
|
):
|
|
assert files.available_bytes() == 3 * 1024 * 1024
|
|
|
|
with patch(
|
|
"virsh_panel.cloud_template_runtime.shutil.disk_usage",
|
|
return_value=SimpleNamespace(free=PERSISTENT_FREE_RESERVE_BYTES - 1),
|
|
):
|
|
assert files.available_bytes() == 0
|
|
|
|
|
|
def test_import_rejects_before_queue_when_only_reserved_space_remains(tmp_path):
|
|
runtime = CloudTemplateRuntime(MemoryStore(), tmp_path)
|
|
try:
|
|
with patch.object(
|
|
runtime.files, "available_bytes", return_value=1024 * 1024 - 1
|
|
):
|
|
with pytest.raises(TemplateError, match="空间不足") as insufficient:
|
|
runtime.queue_import(ALMA_10_URL, "No Disk Space", "")
|
|
assert insufficient.value.status == 507
|
|
assert runtime.jobs == {}
|
|
assert runtime.pending == set()
|
|
finally:
|
|
runtime.close()
|
|
|
|
|
|
def test_import_passes_only_space_above_reserve_to_downloader(tmp_path):
|
|
runtime = CloudTemplateRuntime(MemoryStore(), tmp_path)
|
|
payload = b"bounded-global-cloud-image"
|
|
digest = hashlib.sha256(payload).hexdigest()
|
|
limits: list[int] = []
|
|
|
|
def bounded_download(_url: str, _expected: str, directory: Path, *,
|
|
max_bytes: int):
|
|
limits.append(max_bytes)
|
|
source = Path(directory) / ".cloud-image-bounded.qcow2"
|
|
source.write_bytes(payload)
|
|
os.chmod(source, 0o600)
|
|
return source, 2 * GIB, digest
|
|
|
|
try:
|
|
with patch.object(
|
|
runtime.files, "available_bytes",
|
|
side_effect=[3 * 1024 * 1024, 2 * 1024 * 1024],
|
|
), patch(
|
|
"virsh_panel.cloud_template_runtime.download_verified_image",
|
|
side_effect=bounded_download,
|
|
):
|
|
queued = runtime.queue_import(ALMA_10_URL, "Bounded", "")
|
|
job = wait_for_job(runtime, queued["job_id"])
|
|
assert job["status"] == "completed", job
|
|
assert limits == [2 * 1024 * 1024]
|
|
finally:
|
|
runtime.close()
|
|
|
|
|
|
def test_duplicate_digest_and_casefold_name_are_rejected_globally():
|
|
store = MemoryStore()
|
|
catalog = GlobalCloudTemplateStore(store)
|
|
first = record_for(b"first-image", name="AlmaLinux Gold")
|
|
catalog.add(first)
|
|
|
|
with pytest.raises(TemplateError, match="已经导入") as duplicate_digest:
|
|
catalog.add(first)
|
|
assert duplicate_digest.value.status == 409
|
|
|
|
second = record_for(b"second-image", name="almalinux gold")
|
|
with pytest.raises(TemplateError, match="同名") as duplicate_name:
|
|
catalog.add(second)
|
|
assert duplicate_name.value.status == 409
|
|
|
|
|
|
def test_rename_changes_only_display_name_and_keeps_source_identity(tmp_path):
|
|
store = MemoryStore()
|
|
catalog = GlobalCloudTemplateStore(store)
|
|
record = record_for(b"rename-image")
|
|
catalog.add(record)
|
|
updated = catalog.rename(record["id"], "AlmaLinux 10 Production")
|
|
|
|
assert updated["name"] == "AlmaLinux 10 Production"
|
|
for key in (
|
|
"id", "sha256", "volume_name", "virtual_size", "image_bytes",
|
|
"os_family", "os_version", "created_at",
|
|
):
|
|
assert updated[key] == record[key]
|
|
|
|
files = GlobalCloudTemplateFiles(tmp_path)
|
|
temporary = files.directory() / ".cloud-image-rename.qcow2"
|
|
temporary.write_bytes(b"rename-image")
|
|
os.chmod(temporary, 0o600)
|
|
source = files.publish(
|
|
temporary, record["id"], expected_size=len(b"rename-image")
|
|
)
|
|
assert source.name == record["volume_name"]
|
|
|
|
|
|
def test_source_rejects_permission_damage_and_content_tampering(tmp_path):
|
|
store = MemoryStore()
|
|
runtime = CloudTemplateRuntime(store, tmp_path)
|
|
payload = b"source-integrity"
|
|
record = record_for(payload)
|
|
runtime.catalog.add(record)
|
|
temporary = runtime.files.directory() / ".cloud-image-integrity.qcow2"
|
|
temporary.write_bytes(payload)
|
|
os.chmod(temporary, 0o600)
|
|
source = runtime.files.publish(
|
|
temporary, record["id"], expected_size=len(payload)
|
|
)
|
|
try:
|
|
os.chmod(source, 0o640)
|
|
with pytest.raises(TemplateError, match="已损坏"):
|
|
with runtime.source(record["id"]):
|
|
pass
|
|
|
|
os.chmod(source, 0o600)
|
|
source.write_bytes(b"x" * len(payload))
|
|
with pytest.raises(TemplateError, match="SHA-256"):
|
|
with runtime.source(record["id"]):
|
|
pass
|
|
finally:
|
|
runtime.close()
|
|
|
|
|
|
def test_concurrent_import_is_serialized_and_second_request_is_rejected(tmp_path):
|
|
runtime = CloudTemplateRuntime(MemoryStore(), tmp_path)
|
|
started = threading.Event()
|
|
release = threading.Event()
|
|
payload = b"blocking-image"
|
|
digest = hashlib.sha256(payload).hexdigest()
|
|
|
|
def blocking_download(_url: str, _expected: str, directory: Path,
|
|
**_kwargs):
|
|
started.set()
|
|
assert release.wait(timeout=5)
|
|
source = Path(directory) / ".cloud-image-blocking.qcow2"
|
|
source.write_bytes(payload)
|
|
os.chmod(source, 0o600)
|
|
return source, 2 * GIB, digest
|
|
|
|
try:
|
|
with patch(
|
|
"virsh_panel.cloud_template_runtime.download_verified_image",
|
|
side_effect=blocking_download,
|
|
):
|
|
first = runtime.queue_import(ALMA_10_URL, "First", "")
|
|
assert started.wait(timeout=2)
|
|
assert runtime.has_active_jobs() is True
|
|
with pytest.raises(TemplateError, match="正在导入") as busy:
|
|
runtime.queue_import(ALMA_10_URL, "Second", "")
|
|
assert busy.value.status == 409
|
|
release.set()
|
|
assert wait_for_job(runtime, first["job_id"])["status"] == "completed"
|
|
assert runtime.has_active_jobs() is False
|
|
finally:
|
|
release.set()
|
|
runtime.close()
|
|
|
|
|
|
def test_restore_marks_interrupted_job_failed_and_removes_partial_files(tmp_path):
|
|
store = MemoryStore()
|
|
job_id = "interrupted-cloud-template"
|
|
store.set(IMPORT_JOB_KIND, job_id, json.dumps({
|
|
"job_id": job_id,
|
|
"status": "running",
|
|
"phase": "正在下载并校验云镜像",
|
|
"created_at": 1,
|
|
}))
|
|
files = GlobalCloudTemplateFiles(tmp_path)
|
|
partial = files.directory() / ".cloud-image-abandoned.qcow2"
|
|
partial.write_bytes(b"partial")
|
|
os.chmod(partial, 0o600)
|
|
|
|
runtime = CloudTemplateRuntime(store, tmp_path)
|
|
try:
|
|
runtime.restore_jobs()
|
|
job = runtime.get_job(job_id)
|
|
assert job["status"] == "failed"
|
|
assert "重启" in job["phase"]
|
|
assert "重启" in job["error"]
|
|
assert not partial.exists()
|
|
assert runtime.has_active_jobs() is False
|
|
finally:
|
|
runtime.close()
|
|
|
|
|
|
def test_import_failure_redacts_internal_paths_and_releases_pending(tmp_path):
|
|
runtime = CloudTemplateRuntime(MemoryStore(), tmp_path)
|
|
try:
|
|
with patch(
|
|
"virsh_panel.cloud_template_runtime.download_verified_image",
|
|
side_effect=TemplateError(
|
|
"私有目录 /home/virt-cloud/data/cloud-templates 损坏", 500
|
|
),
|
|
):
|
|
queued = runtime.queue_import(ALMA_10_URL, "Broken", "")
|
|
failed = wait_for_job(runtime, queued["job_id"])
|
|
assert failed["status"] == "failed"
|
|
assert failed["error"] == "云镜像下载或保存失败,请联系管理员查看日志"
|
|
assert "/home/" not in json.dumps(failed, ensure_ascii=False)
|
|
assert runtime.pending == set()
|
|
finally:
|
|
runtime.close()
|
|
|
|
|
|
def test_catalog_insert_commit_with_lost_ack_is_completed_not_rolled_back(
|
|
tmp_path):
|
|
store = CommitThenRaiseCatalogStore()
|
|
runtime = CloudTemplateRuntime(store, tmp_path)
|
|
payload = b"catalog-commit-ack-lost"
|
|
digest, downloader = fake_download_bytes(payload)
|
|
try:
|
|
with patch(
|
|
"virsh_panel.cloud_template_runtime.download_verified_image",
|
|
side_effect=downloader,
|
|
):
|
|
queued = runtime.queue_import(ALMA_10_URL, "Committed", digest)
|
|
job = wait_for_job(runtime, queued["job_id"])
|
|
|
|
assert job["status"] == "completed"
|
|
template_id = cloud_template_id_for(digest, "10")
|
|
assert runtime.get(template_id)["sha256"] == digest
|
|
source = tmp_path / "cloud-templates" / template_volume_name(template_id)
|
|
assert source.read_bytes() == payload
|
|
finally:
|
|
runtime.close()
|
|
|
|
|
|
def test_ambiguous_catalog_commit_preserves_source_for_startup_recovery(
|
|
tmp_path):
|
|
store = AmbiguousCatalogCommitStore()
|
|
runtime = CloudTemplateRuntime(store, tmp_path)
|
|
payload = b"ambiguous-catalog-commit"
|
|
digest, downloader = fake_download_bytes(payload)
|
|
try:
|
|
with patch(
|
|
"virsh_panel.cloud_template_runtime.download_verified_image",
|
|
side_effect=downloader,
|
|
):
|
|
queued = runtime.queue_import(ALMA_10_URL, "Ambiguous", digest)
|
|
failed = wait_for_job(runtime, queued["job_id"])
|
|
|
|
assert failed["status"] == "failed"
|
|
template_id = cloud_template_id_for(digest, "10")
|
|
source = tmp_path / "cloud-templates" / template_volume_name(template_id)
|
|
assert source.read_bytes() == payload
|
|
finally:
|
|
runtime.close()
|
|
|
|
restored = CloudTemplateRuntime(store, tmp_path)
|
|
try:
|
|
restored.restore_jobs()
|
|
recovered = restored.get_job(queued["job_id"])
|
|
assert recovered["status"] == "completed"
|
|
assert recovered["template"]["id"] == template_id
|
|
assert source.read_bytes() == payload
|
|
finally:
|
|
restored.close()
|
|
|
|
|
|
def test_lost_completed_job_write_is_success_in_memory_and_repaired_on_restart(
|
|
tmp_path):
|
|
store = LoseFirstCompletedJobWriteStore()
|
|
runtime = CloudTemplateRuntime(store, tmp_path)
|
|
payload = b"lost-completed-job-write"
|
|
digest, downloader = fake_download_bytes(payload)
|
|
try:
|
|
with patch(
|
|
"virsh_panel.cloud_template_runtime.download_verified_image",
|
|
side_effect=downloader,
|
|
):
|
|
queued = runtime.queue_import(ALMA_10_URL, "Recoverable", digest)
|
|
completed = wait_for_job(runtime, queued["job_id"])
|
|
|
|
assert completed["status"] == "completed"
|
|
persisted = json.loads(store.get(IMPORT_JOB_KIND, queued["job_id"]))
|
|
assert persisted["status"] == "running"
|
|
assert persisted["candidate_template_id"] == cloud_template_id_for(
|
|
digest, "10"
|
|
)
|
|
finally:
|
|
runtime.close()
|
|
|
|
restored = CloudTemplateRuntime(store, tmp_path)
|
|
try:
|
|
restored.restore_jobs()
|
|
repaired = restored.get_job(queued["job_id"])
|
|
assert repaired["status"] == "completed"
|
|
persisted = json.loads(store.get(IMPORT_JOB_KIND, queued["job_id"]))
|
|
assert persisted["status"] == "completed"
|
|
finally:
|
|
restored.close()
|