"""Private engine tests: no PHP cookies, no real DB and no real hypervisor.""" from __future__ import annotations import base64 import asyncio import hashlib import json import os import struct import threading import time from concurrent.futures import ThreadPoolExecutor from contextlib import contextmanager from types import SimpleNamespace from unittest.mock import MagicMock, patch from urllib.parse import urlencode import pytest from fastapi.testclient import TestClient from test_app import (CLOUD_CREATE_BODY, CREATE_BODY, OFFICIAL_CLOUD_URL, FakeRemoteManager, FakeVMService, HOST_BODY, VM_ID) from virsh_panel.app import create_app from virsh_panel.cloud_templates import (TemplateError, template_id_for, template_scope, template_volume_name) from virsh_panel.config import Settings from virsh_panel.database import DatabaseError, DatabaseTemplateStore from virsh_panel.engine_context import (EngineComponents, ExternalOwnership, HostStore, LogStore, MainStore, TRUSTED_ACTOR) from virsh_panel.engine_security import (EngineAuthError, authenticate, encode_actor, signed_headers) from virsh_panel.internal import EngineApplication, HostRegistry from virsh_panel.libvirt_service import VMError from virsh_panel.metrics import host_scope from virsh_panel.user_isos import user_iso_scope SECRET = b"s" * 32 USER = {"user_id": 41, "is_admin": False, "is_super_admin": False, "can_manage": True, "session_id": "test-session"} SUPER = {**USER, "user_id": 1, "is_admin": True, "is_super_admin": True} ADMIN = {**SUPER, "is_super_admin": False} class MemoryStore: def __init__(self): self.records = {} self.nonces = set() self.lock = threading.RLock() self.daily_traffic_calls = [] def get(self, kind, key): with self.lock: return self.records.get((kind, key)) def set(self, kind, key, value): with self.lock: self.records[kind, key] = value def insert(self, kind, key, value): with self.lock: if (kind, key) in self.records: return False self.records[kind, key] = value return True def delete(self, kind, key): with self.lock: self.records.pop((kind, key), None) def list(self, kind): with self.lock: return {key: value for (row_kind, key), value in self.records.items() if row_kind == kind} def consume_nonce(self, nonce, expires_at, now): with self.lock: if nonce in self.nonces: return False self.nonces.add(nonce) return True def query_metric_samples(self, *args, **kwargs): return [{"timestamp": 123, "cpu_percent": 20.0}] def query_daily_traffic(self, scope, vm_id, **kwargs): self.daily_traffic_calls.append((scope, vm_id, kwargs)) day_start = kwargs["until"] - kwargs["until"] % 86400 return [{"day_start": day_start, "download_bytes": 4096, "upload_bytes": 1024, "total_bytes": 5120}] def get_value(self, key): return self.get("value", key) def set_value(self, key, value): return self.set("value", key, value) def delete_value(self, key): return self.delete("value", key) def get_secret(self, key): return self.get("secret", key) def set_secret(self, key, value): return self.set("secret", key, value) def delete_secret(self, key): return self.delete("secret", key) class FakeUpdater: def __init__(self): self.token = None self.active = None self.jobs = {} def settings(self): return { "repository": "https://g.luoli.live/root/virt.git", "branch": "venix", "token_configured": self.token is not None, "checkout_ready": True, "current_commit": "a" * 40, "last_job_id": None, "active_job_id": self.active["job_id"] if self.active else None, } def set_token(self, token): self.token = token def clear_token(self): self.token = None def active_job(self): return self.active def check(self): return {"repository": "https://g.luoli.live/root/virt.git", "branch": "venix", "current_commit": "a" * 40, "remote_commit": "b" * 40, "status": "update_available"} def create_job(self, *, expected_current_commit=None, expected_remote_commit=None): if self.token is None: from virsh_panel.updater import UpdateError raise UpdateError("请先在面板中设置 Gitea 访问令牌", 409) if expected_current_commit != "a" * 40 or expected_remote_commit != "b" * 40: from virsh_panel.updater import UpdateError raise UpdateError("面板本地版本已变化,请重新检查更新后再执行", 409) job = {"job_id": "update-job-1234567890", "status": "queued", "phase": "等待独立更新服务启动", "created_at": 1, "private": self.token} self.jobs[job["job_id"]] = job self.active = job return job.copy() def get_job(self, job_id): return self.jobs.get(job_id) def fail_job(self, job_id, message): self.jobs[job_id].update(status="failed", error=message) self.active = None return self.jobs[job_id].copy() class Service(FakeVMService): def __init__(self): super().__init__() self.create_failure = False self.absent = False self.absent_checks = [] self.cloud_template_delete_error = None self.vm.update(managed=True, os_source="iso") def create_vm(self, spec): self.calls.append(("create_vm", spec)) if self.create_failure: from virsh_panel.libvirt_service import VMError raise VMError("宿主机失联,回滚状态不明", 502) self.vm.update(uuid=spec["vm_uuid"], name=spec["name"]) return self.vm.copy() def provisioning_absent(self, vm_uuid, name, *, uuid_resources=False, legacy_name_collision=False): self.absent_checks.append( (vm_uuid, name, uuid_resources, legacy_name_collision)) return self.absent def delete_cloud_template_replicas(self, source_volume_name): self.calls.append(("delete_cloud_template_replicas", source_volume_name)) if self.cloud_template_delete_error is not None: raise self.cloud_template_delete_error return {"source_volume_name": source_volume_name, "deleted": True} @pytest.fixture def boundary(tmp_path): store = MemoryStore() logs = MemoryStore() config = {"service_secret": base64.b64encode(SECRET).decode(), "data_dir": str(tmp_path)} services = {} def factory(host_id, user_isos, cloud_templates): (tmp_path / host_id).mkdir(mode=0o700, exist_ok=True) scoped = HostStore(store, host_id) manager = FakeRemoteManager() manager.profile = {"host_id": host_id, "host": "host.example.test", "port": 22, "fingerprint": "SHA256:example", "key_locked": False, "storage_pool": "images", "iso_pool": "iso", "rescue_iso_pool": "rescue", "default_network": "default", "libvirt_uri": "fake", "ssh_target": "fake", "ssh_config_file": "/fake/config", "ssh_known_hosts_file": "/fake/known_hosts"} service = Service() services[host_id] = service components = EngineComponents(SimpleNamespace(users=ExternalOwnership(scoped)), manager, DatabaseTemplateStore(scoped), scoped, logs, user_isos, cloud_templates) app = create_app(Settings(data_dir=tmp_path / host_id), engine_components=components, service_factory=lambda profile: service) return components, app registry = HostRegistry(config, store, logs, factory=factory) async def active(*args): return True engine = EngineApplication(config, store, logs, registry=registry, introspection=active) engine._service = lambda profile: services[profile["host_id"]] with TestClient(engine, base_url="http://127.0.0.1") as client: yield client, engine, store, logs, services def call(client, method, path, actor=USER, value=None, raw=None, headers=None): body = raw if raw is not None else b"" if value is None else json.dumps(value, separators=(",", ":")).encode() signed = signed_headers(SECRET, method, path, body, actor) if value is not None: signed["Content-Type"] = "application/json" signed.update(headers or {}) return client.request(method, path, content=body, headers=signed) def import_user_iso_for_test(client, tmp_path, name, *, digest="e" * 64): download = tmp_path / (hashlib.sha256(name.encode()).hexdigest() + ".iso") download.write_bytes(b"i" * (1024 * 1024)) with patch("virsh_panel.user_isos.download_user_iso", return_value=(download, download.stat().st_size, digest)): queued = call(client, "POST", "/internal/v1/user-isos/import", USER, { "url": "https://downloads.example.test/os.iso", "name": name, "sha256": "", }) assert queued.status_code == 202, queued.text job_id = queued.json()["job_id"] for _ in range(100): job = call( client, "GET", f"/internal/v1/user-isos/import-jobs/{job_id}", ).json() if job["status"] in {"completed", "failed"}: break time.sleep(0.01) assert job["status"] == "completed", job return job["iso"]["id"] def import_cloud_template_for_test(client, tmp_path, name="Delete Cloud Image"): payload = name.encode("utf-8") digest = hashlib.sha256(payload).hexdigest() def fake_download(_url, _sha256, directory, **_kwargs): source = directory / ".cloud-image-delete-boundary.qcow2" source.write_bytes(payload) source.chmod(0o600) return source, 2 * 1024**3, digest with patch( "virsh_panel.cloud_template_runtime.download_verified_image", side_effect=fake_download): queued = call(client, "POST", "/internal/v1/vm-templates/import", SUPER, {"url": OFFICIAL_CLOUD_URL, "name": name, "sha256": ""}) assert queued.status_code == 202, queued.text job_id = queued.json()["job_id"] for _ in range(100): job = call( client, "GET", f"/internal/v1/vm-templates/import-jobs/{job_id}", SUPER, ).json() if job["status"] in {"completed", "failed"}: break time.sleep(0.01) assert job["status"] == "completed", job return job["template"], payload def test_global_cloud_template_is_visible_and_cached_once_per_host( boundary, tmp_path): client, engine, store, _logs, services = boundary payload = b"verified-global-cloud-template" digest = hashlib.sha256(payload).hexdigest() def fake_download(_url, _sha256, directory, **_kwargs): source = directory / ".cloud-image-boundary.qcow2" source.write_bytes(payload) source.chmod(0o600) return source, 2 * 1024**3, digest denied = call(client, "POST", "/internal/v1/vm-templates/import", USER, {"url": OFFICIAL_CLOUD_URL, "name": "Not Allowed", "sha256": ""}) assert denied.status_code == 403 with patch( "virsh_panel.cloud_template_runtime.download_verified_image", side_effect=fake_download): queued = call(client, "POST", "/internal/v1/vm-templates/import", SUPER, {"url": OFFICIAL_CLOUD_URL, "name": "AlmaLinux Global", "sha256": ""}) assert queued.status_code == 202, queued.text job_id = queued.json()["job_id"] for _ in range(100): job = call( client, "GET", f"/internal/v1/vm-templates/import-jobs/{job_id}", SUPER, ).json() if job["status"] in {"completed", "failed"}: break time.sleep(0.01) assert job["status"] == "completed", job template_id = job["template"]["id"] assert services == {} assert call(client, "GET", "/internal/v1/vm-templates", USER).json() == { "templates": [job["template"]], } provisioning = {**USER, "can_provision": True} body = {**CLOUD_CREATE_BODY, "template_id": template_id} hk = call( client, "POST", "/internal/v1/hosts/hk/vms", {**provisioning, "operation_id": "global-template-hk-0001"}, body, ) assert hk.status_code == 201, hk.text jp = call( client, "POST", "/internal/v1/hosts/jp/vms", {**provisioning, "operation_id": "global-template-jp-0001"}, {**body, "name": "cloud-vm-jp", "public_ip_cidr": "8.8.8.19/24"}, ) assert jp.status_code == 201, jp.text first_markers = { host_id: json.loads(HostStore(store, host_id).get( "cloud_template_replica", template_id)) for host_id in ("hk", "jp") } # The public catalog intentionally omits the private backing filename. source_volume_name = template_volume_name(template_id) for host_id, marker in first_markers.items(): assert marker["status"] == "ready" assert marker["sha256"] == digest assert len(marker["replica_identity"]) == 64 assert marker["replica_volume_name"] != source_volume_name assert marker["replica_volume_name"].startswith( source_volume_name.removesuffix(".qcow2") + "-") created = [row for row in services[host_id].calls if row[0] == "create_cloud_vm"] assert created[-1][2] == marker["replica_volume_name"] observed_markers = [] original_hk_cache = services["hk"].cache_template def inspect_hk_marker(*args, **kwargs): marker = json.loads(HostStore(store, "hk").get( "cloud_template_replica", template_id)) observed_markers.append(marker) return original_hk_cache(*args, **kwargs) services["hk"].cache_template = inspect_hk_marker hk_again = call( client, "POST", "/internal/v1/hosts/hk/vms", {**provisioning, "operation_id": "global-template-hk-0002"}, {**body, "name": "cloud-vm-hk-2", "public_ip_cidr": "8.8.8.20/24"}, ) assert hk_again.status_code == 201, hk_again.text services["hk"].cache_template = original_hk_cache assert [marker["status"] for marker in observed_markers] == ["syncing"] fallback_volume_name = observed_markers[0]["replica_volume_name"] assert fallback_volume_name != first_markers["hk"]["replica_volume_name"] hk_cache = [row for row in services["hk"].calls if row[0] == "cache_template"] jp_cache = [row for row in services["jp"].calls if row[0] == "cache_template"] assert hk_cache[0][4:] == (None, None) assert hk_cache[1][1] == fallback_volume_name assert hk_cache[1][4] == first_markers["hk"]["replica_volume_name"] assert hk_cache[1][5] == first_markers["hk"]["replica_identity"] assert jp_cache[0][4:] == (None, None) for host_id in ("hk", "jp"): marker = json.loads(HostStore(store, host_id).get( "cloud_template_replica", template_id)) assert marker["status"] == "ready" assert marker["sha256"] == digest assert len(marker["replica_identity"]) == 64 # A successful trusted reuse keeps the proven prior generation even # though the durable syncing marker contained a fresh fallback name. assert marker["replica_volume_name"] == ( first_markers[host_id]["replica_volume_name"]) # Rebinding the logical host record to a different SSH host identity must # invalidate its ready proof even when the pool name is unchanged. engine.registry.components["hk"].manager.profile["fingerprint"] = ( "SHA256:replacement-host") rebound = call( client, "POST", "/internal/v1/hosts/hk/vms", {**provisioning, "operation_id": "global-template-hk-0003"}, {**body, "name": "cloud-vm-hk-3", "public_ip_cidr": "8.8.8.21/24"}, ) assert rebound.status_code == 201, rebound.text hk_cache = [row for row in services["hk"].calls if row[0] == "cache_template"] assert hk_cache[-1][4:] == (None, None) rebound_marker = json.loads(HostStore(store, "hk").get( "cloud_template_replica", template_id)) assert rebound_marker["status"] == "ready" assert rebound_marker["replica_volume_name"] == hk_cache[-1][1] assert (rebound_marker["replica_volume_name"] != first_markers["hk"]["replica_volume_name"]) # A crash/failure after the syncing marker was stored must retry that # exact planned generation. Allocating yet another name would leak the # possibly partial remote volume and make recovery non-deterministic. failed_calls = [] def fail_hk_cache(image_path, volume_name, virtual_size, **kwargs): failed_calls.append((volume_name, kwargs, json.loads( HostStore(store, "hk").get( "cloud_template_replica", template_id)))) raise VMError("云镜像缓存同步中断", 502) services["hk"].cache_template = fail_hk_cache interrupted = call( client, "POST", "/internal/v1/hosts/hk/vms", {**provisioning, "operation_id": "global-template-hk-0004"}, {**body, "name": "cloud-vm-hk-4", "public_ip_cidr": "8.8.8.22/24"}, ) assert interrupted.status_code == 502, interrupted.text assert len(failed_calls) == 1 planned_volume_name, failed_kwargs, failed_marker = failed_calls[0] assert failed_marker["status"] == "syncing" assert failed_marker["replica_volume_name"] == planned_volume_name assert failed_kwargs["trusted_replica_volume_name"] == ( rebound_marker["replica_volume_name"]) assert failed_kwargs["trusted_replica_identity"] == ( rebound_marker["replica_identity"]) services["hk"].cache_template = original_hk_cache retried = call( client, "POST", "/internal/v1/hosts/hk/vms", {**provisioning, "operation_id": "global-template-hk-0005"}, {**body, "name": "cloud-vm-hk-5", "public_ip_cidr": "8.8.8.23/24"}, ) assert retried.status_code == 201, retried.text hk_cache = [row for row in services["hk"].calls if row[0] == "cache_template"] assert hk_cache[-1][1] == planned_volume_name assert hk_cache[-1][4:] == (None, None) recovered_marker = json.loads(HostStore(store, "hk").get( "cloud_template_replica", template_id)) assert recovered_marker["status"] == "ready" assert recovered_marker["replica_volume_name"] == planned_volume_name assert [row for row in services["hk"].calls if row[0] == "create_cloud_vm"][-1][2] == planned_volume_name assert (tmp_path / "cloud-templates" / engine.registry.cloud_templates.get(template_id)["volume_name"]).is_file() def test_legacy_host_template_id_remains_usable_for_an_already_queued_order( boundary): client, engine, store, _logs, services = boundary # Initialize the original target host, then seed the same host-scoped # catalog record an order created before the global-catalog upgrade holds. assert call(client, "GET", "/internal/v1/hosts/legacy/vms").status_code == 200 profile = engine.registry.components["legacy"].manager.profile scope = template_scope(profile) digest = "d" * 64 template_id = template_id_for(scope, digest, "10") DatabaseTemplateStore(HostStore(store, "legacy")).add(scope, { "id": template_id, "name": "Legacy AlmaLinux", "os_family": "AlmaLinux", "os_version": "10", "sha256": digest, "volume_name": template_volume_name(template_id), "virtual_size": 2 * 1024**3, "image_bytes": 1024, "created_at": 1, }) response = call( client, "POST", "/internal/v1/hosts/legacy/vms", {**USER, "can_provision": True, "operation_id": "legacy-template-order-0001"}, {**CLOUD_CREATE_BODY, "template_id": template_id, "name": "legacy-paid-order", "public_ip_cidr": "8.8.8.22/24"}, ) assert response.status_code == 201, response.text assert not any(row[0] == "cache_template" for row in services["legacy"].calls) create = next(row for row in services["legacy"].calls if row[0] == "create_cloud_vm") assert create[2] == template_volume_name(template_id) def test_global_cloud_template_delete_is_idempotent_and_retries_host_gc( boundary, tmp_path): client, engine, store, _logs, services = boundary imported, payload = import_cloud_template_for_test(client, tmp_path) template_id = imported["id"] template_created_at = imported["created_at"] source_volume_name = template_volume_name(template_id) source_path = tmp_path / "cloud-templates" / source_volume_name assert source_path.is_file() for host_id in ("hk", "jp"): assert call( client, "GET", f"/internal/v1/hosts/{host_id}/resources" ).status_code == 200 HostStore(store, host_id).set( "cloud_template_replica", template_id, "ready-marker") assert call( client, "DELETE", f"/internal/v1/vm-templates/{template_id}", ADMIN, {"created_at": template_created_at}, ).status_code == 403 services["hk"].cloud_template_delete_error = VMError( "云镜像仍被虚拟机引用,不能删除", 409) original_store_delete = store.delete failed_catalog_delete = False def fail_first_catalog_delete(kind, key): nonlocal failed_catalog_delete if (kind == "cloud_template" and key == template_id and not failed_catalog_delete): failed_catalog_delete = True raise RuntimeError("catalog delete failed before commit") return original_store_delete(kind, key) store.delete = fail_first_catalog_delete try: deleted = call( client, "DELETE", f"/internal/v1/vm-templates/{template_id}", SUPER, {"created_at": template_created_at}, ) finally: store.delete = original_store_delete assert failed_catalog_delete is True assert deleted.status_code == 200, deleted.text assert deleted.json() == { "deleted": True, "cache_cleanup_pending": True, } assert not source_path.exists() assert call(client, "GET", "/internal/v1/vm-templates", USER).json() == { "templates": [], } for _ in range(100): tombstone = json.loads(store.get( "cloud_template_tombstone", template_id)) if tombstone["pending_hosts"] == ["hk"]: break time.sleep(0.01) assert tombstone["source_deleted"] is True assert tombstone["source_volume_name"] == source_volume_name assert tombstone["pending_hosts"] == ["hk"] assert HostStore(store, "jp").get( "cloud_template_replica", template_id) is None assert HostStore(store, "hk").get( "cloud_template_replica", template_id) is not None services["hk"].cloud_template_delete_error = VMError( "存储池 images 不存在", 404) assert call(client, "GET", "/internal/v1/hosts/hk/resources").status_code == 200 still_pending = json.loads(store.get( "cloud_template_tombstone", template_id)) assert still_pending["pending_hosts"] == ["hk"] assert HostStore(store, "hk").get( "cloud_template_replica", template_id) is not None services["hk"].cloud_template_delete_error = None assert call(client, "GET", "/internal/v1/hosts/hk/resources").status_code == 200 terminal = json.loads(store.get("cloud_template_tombstone", template_id)) assert terminal["pending_hosts"] == [] assert HostStore(store, "hk").get( "cloud_template_replica", template_id) is None repeated = call( client, "DELETE", f"/internal/v1/vm-templates/{template_id}", SUPER, {"created_at": template_created_at}) assert repeated.status_code == 200 assert repeated.json() == { "deleted": True, "cache_cleanup_pending": False, } # A duplicate task from the old deletion must not touch a later deletion # generation (or the same-ID image/replica that may exist between them). old_deletion_id = terminal["deletion_id"] engine.registry.cloud_templates._prepare_reimport(template_id) reimported = { **imported, "volume_name": source_volume_name, "created_at": template_created_at + 1, } temporary = engine.registry.cloud_templates.files.directory() \ / ".cloud-image-reimport-boundary.qcow2" temporary.write_bytes(payload) temporary.chmod(0o600) engine.registry.cloud_templates.files.publish( temporary, template_id, expected_size=len(payload)) engine.registry.cloud_templates.catalog.add(reimported) stale_delete = call( client, "DELETE", f"/internal/v1/vm-templates/{template_id}", SUPER, {"created_at": template_created_at}) assert stale_delete.status_code == 409 assert engine.registry.cloud_templates.get( template_id)["created_at"] == template_created_at + 1 next_tombstone = { **terminal, "deletion_id": "b" * 32, "template_created_at": template_created_at + 1, "pending_hosts": ["hk"], } store.set("cloud_template_tombstone", template_id, json.dumps(next_tombstone)) calls_before = list(services["hk"].calls) engine._cleanup_cloud_template_tombstone_on_host_sync( template_id, old_deletion_id, "hk", services["hk"], engine.registry.components["hk"], ) assert services["hk"].calls == calls_before assert json.loads(store.get( "cloud_template_tombstone", template_id))["pending_hosts"] == ["hk"] def test_user_iso_routes_are_owner_scoped_and_create_uses_opaque_id(boundary, tmp_path): client, _engine, _store, _logs, services = boundary download = tmp_path / "download.iso" download.write_bytes(b"i" * (1024 * 1024)) def fake_download(_url, _sha256, _directory, *, max_bytes): assert max_bytes >= download.stat().st_size return download, download.stat().st_size, "a" * 64 with patch("virsh_panel.user_isos.download_user_iso", side_effect=fake_download): queued = call(client, "POST", "/internal/v1/user-isos/import", USER, {"url": "https://downloads.example.test/os.iso", "name": "My Installer", "sha256": ""}) assert queued.status_code == 202 job_id = queued.json()["job_id"] for _ in range(100): job = call(client, "GET", f"/internal/v1/user-isos/import-jobs/{job_id}").json() if job["status"] in {"completed", "failed"}: break time.sleep(0.01) assert job["status"] == "completed", job iso_id = job["iso"]["id"] assert "volume_name" not in job["iso"] listed = call(client, "GET", "/internal/v1/user-isos").json() assert [row["id"] for row in listed["isos"]] == [iso_id] assert "volume_name" not in listed["isos"][0] global_source = (tmp_path / "user-isos" / str(USER["user_id"]) / f"panel-user-iso-{iso_id}.iso") assert global_source.is_file() assert services == {} assert call( client, "GET", "/internal/v1/hosts/hk/user-isos", USER, ).status_code == 404 assert call( client, "POST", "/internal/v1/hosts/hk/user-isos/import", USER, { "url": "https://downloads.example.test/alias.iso", "name": "Host Alias Must Fail", "sha256": "", }, ).status_code == 404 assert call( client, "DELETE", f"/internal/v1/hosts/hk/user-isos/{iso_id}", USER, ).status_code == 404 assert global_source.is_file() other = {**USER, "user_id": 42} assert call(client, "GET", f"/internal/v1/user-isos/{iso_id}", other).status_code == 404 assert call(client, "GET", f"/internal/v1/user-isos/import-jobs/{job_id}", other).status_code == 404 assert call(client, "DELETE", f"/internal/v1/user-isos/{iso_id}", other).status_code == 404 provisioning = {**USER, "can_provision": True, "operation_id": "create-user-iso-0001"} body = {key: value for key, value in CREATE_BODY.items() if key != "iso"} body["user_iso_id"] = iso_id created = call(client, "POST", "/internal/v1/hosts/hk/vms", provisioning, body) assert created.status_code == 201, created.text create_calls = [row for row in services["hk"].calls if row[0] == "create_user_iso_vm"] assert len(create_calls) == 1 assert create_calls[0][2] == USER["user_id"] assert create_calls[0][4] == iso_id cache_calls = [row for row in services["hk"].calls if row[0] == "cache_user_iso"] assert len(cache_calls) == 1 assert cache_calls[0][4] is False other_host_actor = { **provisioning, "operation_id": "create-user-iso-0003", } other_host_body = {**body, "name": "other-host-installer"} other_host = call(client, "POST", "/internal/v1/hosts/jp/vms", other_host_actor, other_host_body) assert other_host.status_code == 201, other_host.text assert len([row for row in services["jp"].calls if row[0] == "cache_user_iso"]) == 1 assert global_source.is_file() ambiguous_actor = {**provisioning, "operation_id": "create-user-iso-0002"} ambiguous = call(client, "POST", "/internal/v1/hosts/hk/vms", ambiguous_actor, {**body, "iso": "legacy.iso"}) assert ambiguous.status_code == 422 def test_user_iso_writes_require_manage_capability(boundary): client, *_rest = boundary actor = {**USER, "can_manage": False} response = call(client, "POST", "/internal/v1/user-isos/import", actor, {"url": "https://downloads.example.test/os.iso", "name": "Installer", "sha256": ""}) assert response.status_code == 403 def test_concurrent_first_use_serializes_replica_marker_and_cache(boundary, tmp_path): client, _engine, _store, _logs, services = boundary download = tmp_path / "concurrent.iso" download.write_bytes(b"i" * (1024 * 1024)) with patch("virsh_panel.user_isos.download_user_iso", return_value=(download, download.stat().st_size, "1" * 64)): queued = call(client, "POST", "/internal/v1/user-isos/import", USER, { "url": "https://downloads.example.test/concurrent.iso", "name": "Concurrent Installer", "sha256": "", }) job_id = queued.json()["job_id"] for _ in range(100): job = call( client, "GET", f"/internal/v1/user-isos/import-jobs/{job_id}", ).json() if job["status"] in {"completed", "failed"}: break time.sleep(0.01) assert job["status"] == "completed", job iso_id = job["iso"]["id"] # Materialize the target child app before starting concurrent requests. assert call(client, "GET", "/internal/v1/hosts/hk/resources").status_code == 200 service = services["hk"] release = threading.Event() service.user_iso_cache_release = release base = {key: value for key, value in CREATE_BODY.items() if key != "iso"} base["user_iso_id"] = iso_id def create(number): actor = {**USER, "can_provision": True, "operation_id": f"concurrent-user-iso-{number:04d}"} return call(client, "POST", "/internal/v1/hosts/hk/vms", actor, {**base, "name": f"concurrent-vm-{number}"}) with ThreadPoolExecutor(max_workers=2) as executor: first = executor.submit(create, 1) assert service.user_iso_cache_started.wait(timeout=2) second = executor.submit(create, 2) time.sleep(0.05) release.set() responses = [first.result(timeout=5), second.result(timeout=5)] assert [response.status_code for response in responses] == [201, 201] cache_calls = [row for row in service.calls if row[0] == "cache_user_iso"] assert [row[4] for row in cache_calls] == [False, True] def test_user_iso_source_stays_in_use_through_vm_create(boundary, tmp_path): client, engine, _store, _logs, services = boundary iso_id = import_user_iso_for_test(client, tmp_path, "Create Lease Installer") assert call(client, "GET", "/internal/v1/hosts/hk/resources").status_code == 200 service = services["hk"] release = threading.Event() service.user_iso_create_release = release body = {key: value for key, value in CREATE_BODY.items() if key != "iso"} body["user_iso_id"] = iso_id actor = {**USER, "can_provision": True, "operation_id": "user-iso-source-lease-0001"} with ThreadPoolExecutor(max_workers=1) as executor: creating = executor.submit( call, client, "POST", "/internal/v1/hosts/hk/vms", actor, body) assert service.user_iso_create_started.wait(timeout=2) with pytest.raises(TemplateError, match="正在同步") as busy: engine.registry.user_isos.delete_for(USER["user_id"], iso_id) assert busy.value.status == 409 release.set() created = creating.result(timeout=5) assert created.status_code == 201, created.text assert engine.registry.user_isos.get_for(USER["user_id"], iso_id)["id"] == iso_id def test_ready_replica_marker_survives_failed_reuse(boundary, tmp_path): client, engine, store, _logs, services = boundary iso_id = import_user_iso_for_test(client, tmp_path, "Ready Marker Installer") body = {key: value for key, value in CREATE_BODY.items() if key != "iso"} body["user_iso_id"] = iso_id first = call( client, "POST", "/internal/v1/hosts/hk/vms", {**USER, "can_provision": True, "operation_id": "ready-marker-create-0001"}, body) assert first.status_code == 201, first.text marker_key = f"{USER['user_id']}:{iso_id}" scoped = HostStore(store, "hk") assert json.loads(scoped.get("user_iso_replica", marker_key))["status"] == "ready" with patch.object( engine.registry.user_isos, "_verify_source", side_effect=TemplateError("ISO 源文件的 SHA-256 校验值不匹配", 500)): failed = call( client, "POST", "/internal/v1/hosts/hk/vms", {**USER, "can_provision": True, "operation_id": "ready-marker-create-0002"}, {**body, "name": "ready-marker-failed"}) assert failed.status_code == 500 assert json.loads(scoped.get("user_iso_replica", marker_key))["status"] == "ready" recovered = call( client, "POST", "/internal/v1/hosts/hk/vms", {**USER, "can_provision": True, "operation_id": "ready-marker-create-0003"}, {**body, "name": "ready-marker-recovered"}) assert recovered.status_code == 201, recovered.text cache_calls = [row for row in services["hk"].calls if row[0] == "cache_user_iso"] assert cache_calls[-1][4] is True def test_global_delete_keeps_tombstone_until_host_cache_is_reclaimed( boundary, tmp_path): client, _engine, store, _logs, services = boundary download = tmp_path / "delete.iso" download.write_bytes(b"i" * (1024 * 1024)) with patch("virsh_panel.user_isos.download_user_iso", return_value=(download, download.stat().st_size, "2" * 64)): queued = call(client, "POST", "/internal/v1/user-isos/import", USER, { "url": "https://downloads.example.test/delete.iso", "name": "Delete Installer", "sha256": "", }) job_id = queued.json()["job_id"] for _ in range(100): job = call( client, "GET", f"/internal/v1/user-isos/import-jobs/{job_id}", ).json() if job["status"] in {"completed", "failed"}: break time.sleep(0.01) assert job["status"] == "completed", job iso_id = job["iso"]["id"] body = {key: value for key, value in CREATE_BODY.items() if key != "iso"} body["user_iso_id"] = iso_id created = call( client, "POST", "/internal/v1/hosts/hk/vms", {**USER, "can_provision": True, "operation_id": "delete-user-iso-cache-0001"}, body) assert created.status_code == 201, created.text from virsh_panel.libvirt_service import VMError services["hk"].user_iso_delete_error = VMError( "用户 ISO 仍被虚拟机安装过程引用,不能删除", 409) deleted = call( client, "DELETE", f"/internal/v1/user-isos/{iso_id}", USER) assert deleted.status_code == 200 assert deleted.json() == {"deleted": True, "cache_cleanup_pending": True} assert call(client, "GET", "/internal/v1/user-isos", USER).json()["isos"] == [] tombstone_key = f"{USER['user_id']}:{iso_id}" for _ in range(100): if any(row[0] == "delete_user_iso" for row in services["hk"].calls): break time.sleep(0.01) assert store.get("user_iso_tombstone", tombstone_key) is not None scoped = HostStore(store, "hk") assert scoped.get("user_iso_replica", tombstone_key) is not None services["hk"].user_iso_delete_error = None assert call(client, "GET", "/internal/v1/hosts/hk/resources").status_code == 200 assert store.get("user_iso_tombstone", tombstone_key) is None assert scoped.get("user_iso_replica", tombstone_key) is None def test_concurrent_global_deletes_preserve_committed_tombstone(boundary, tmp_path): client, engine, store, _logs, services = boundary iso_id = import_user_iso_for_test(client, tmp_path, "Concurrent Delete Installer") assert call(client, "GET", "/internal/v1/hosts/hk/resources").status_code == 200 record = engine.registry.user_isos.get_for(USER["user_id"], iso_id) marker_key = f"{USER['user_id']}:{iso_id}" HostStore(store, "hk").set("user_iso_replica", marker_key, json.dumps({ "status": "ready", "volume_name": record["volume_name"], "size_bytes": record["size_bytes"], "sha256": record["sha256"], })) from virsh_panel.libvirt_service import VMError services["hk"].user_iso_volumes.add((USER["user_id"], record["volume_name"])) services["hk"].user_iso_delete_error = VMError( "用户 ISO 仍被虚拟机安装过程引用,不能删除", 409) original_delete = engine.registry.user_isos.delete_for started = threading.Event() release = threading.Event() def blocking_delete(user_id, selected_iso_id): started.set() assert release.wait(timeout=5) return original_delete(user_id, selected_iso_id) try: with patch.object(engine.registry.user_isos, "delete_for", side_effect=blocking_delete) as delete_for: with ThreadPoolExecutor(max_workers=2) as executor: first = executor.submit( call, client, "DELETE", f"/internal/v1/user-isos/{iso_id}", USER) assert started.wait(timeout=2) second = executor.submit( call, client, "DELETE", f"/internal/v1/user-isos/{iso_id}", USER) time.sleep(0.05) release.set() responses = [first.result(timeout=5), second.result(timeout=5)] assert delete_for.call_count == 1 finally: release.set() assert [response.status_code for response in responses] == [200, 200] tombstone = json.loads(store.get("user_iso_tombstone", marker_key)) assert tombstone["source_deleted"] is True assert tombstone["pending_hosts"] == ["hk"] def test_user_iso_cache_cleanup_blocks_online_update(boundary, tmp_path): client, engine, store, _logs, services = boundary iso_id = import_user_iso_for_test(client, tmp_path, "Update Gate Installer") assert call(client, "GET", "/internal/v1/hosts/hk/resources").status_code == 200 record = engine.registry.user_isos.get_for(USER["user_id"], iso_id) marker_key = f"{USER['user_id']}:{iso_id}" HostStore(store, "hk").set("user_iso_replica", marker_key, json.dumps({ "status": "ready", "volume_name": record["volume_name"], "size_bytes": record["size_bytes"], "sha256": record["sha256"], })) service = services["hk"] service.user_iso_volumes.add((USER["user_id"], record["volume_name"])) release = threading.Event() service.user_iso_delete_release = release updater = FakeUpdater() updater.token = "valid-token-123456" engine.updater = updater try: deleted = call( client, "DELETE", f"/internal/v1/user-isos/{iso_id}", USER) assert deleted.status_code == 200 assert service.user_iso_delete_started.wait(timeout=2) blocked = call(client, "POST", "/internal/v1/update/apply", SUPER, { "expected_current_commit": "a" * 40, "expected_remote_commit": "b" * 40, }) assert blocked.status_code == 409 assert updater.active is None finally: release.set() for _ in range(100): if not engine.maintenance_tasks: break time.sleep(0.01) assert not engine.maintenance_tasks def test_host_get_skips_cache_cleanup_during_active_update(boundary): client, engine, store, _logs, services = boundary iso_id = "f" * 32 volume_name = f"panel-user-iso-{iso_id}.iso" marker_key = f"{USER['user_id']}:{iso_id}" HostStore(store, "hk").set("user_iso_replica", marker_key, json.dumps({ "status": "ready", "volume_name": volume_name, "size_bytes": 1024 * 1024, "sha256": "f" * 64, })) store.set("user_iso_tombstone", marker_key, json.dumps({ "user_id": USER["user_id"], "iso_id": iso_id, "volume_name": volume_name, "pending_hosts": ["hk"], "source_deleted": True, "created_at": 1, })) updater = FakeUpdater() updater.active = {"job_id": "update-job-1234567890", "status": "running"} engine.updater = updater response = call(client, "GET", "/internal/v1/hosts/hk/resources") assert response.status_code == 200 assert not any(row[0] == "delete_user_iso" for row in services["hk"].calls) assert store.get("user_iso_tombstone", marker_key) is not None def test_slow_tombstone_cleanup_does_not_block_engine_startup(tmp_path): class StartupRegistry: def __init__(self): self.components = {"offline": object()} self.user_isos = SimpleNamespace(has_active_jobs=lambda: False) self.started = False self.closed = False async def start(self): self.started = True async def close(self): self.closed = True async def scenario(): registry = StartupRegistry() updater = FakeUpdater() engine = EngineApplication( {"service_secret": base64.b64encode(SECRET).decode(), "data_dir": str(tmp_path)}, MemoryStore(), MemoryStore(), registry=registry, updater=updater, ) cleanup_started = asyncio.Event() cleanup_release = asyncio.Event() async def slow_cleanup(host_id): assert host_id == "offline" cleanup_started.set() await cleanup_release.wait() engine._cleanup_host_user_iso_tombstones = slow_cleanup incoming = asyncio.Queue() outgoing = asyncio.Queue() async def receive(): return await incoming.get() async def send(message): await outgoing.put(message) await incoming.put({"type": "lifespan.startup"}) lifespan = asyncio.create_task(engine._lifespan(receive, send)) startup = await asyncio.wait_for(outgoing.get(), timeout=0.5) assert startup == {"type": "lifespan.startup.complete"} await asyncio.wait_for(cleanup_started.wait(), timeout=0.5) assert registry.started is True assert engine.maintenance_tasks assert engine._long_running_host_job() is True cleanup_release.set() await incoming.put({"type": "lifespan.shutdown"}) shutdown = await asyncio.wait_for(outgoing.get(), timeout=0.5) assert shutdown == {"type": "lifespan.shutdown.complete"} await asyncio.wait_for(lifespan, timeout=0.5) assert registry.closed is True asyncio.run(scenario()) def test_user_iso_global_import_does_not_contact_a_host(boundary, tmp_path): client, _engine, _store, _logs, services = boundary download = tmp_path / "global.iso" download.write_bytes(b"i" * (1024 * 1024)) with patch("virsh_panel.user_isos.download_user_iso", return_value=(download, download.stat().st_size, "f" * 64)): response = call(client, "POST", "/internal/v1/user-isos/import", USER, {"url": "https://downloads.example.test/os.iso", "name": "Installer", "sha256": ""}) assert response.status_code == 202 job_id = response.json()["job_id"] for _ in range(100): job = call( client, "GET", f"/internal/v1/user-isos/import-jobs/{job_id}", ).json() if job["status"] in {"completed", "failed"}: break time.sleep(0.01) assert job["status"] == "completed", job assert services == {} def test_user_iso_download_is_bounded_by_panel_storage(boundary, tmp_path): client, engine, _store, _logs, _services = boundary download = tmp_path / "bounded.iso" download.write_bytes(b"i" * (1024 * 1024)) def bounded_download(_url, _sha256, _directory, *, max_bytes): assert max_bytes == 2 * 1024 * 1024 return download, download.stat().st_size, "d" * 64 with patch.object(engine.registry.user_isos.files, "available_bytes", return_value=2 * 1024 * 1024), patch( "virsh_panel.user_isos.download_user_iso", side_effect=bounded_download): response = call(client, "POST", "/internal/v1/user-isos/import", USER, {"url": "https://downloads.example.test/os.iso", "name": "Bounded Installer", "sha256": ""}) assert response.status_code == 202 job_id = response.json()["job_id"] for _ in range(100): job = call(client, "GET", f"/internal/v1/user-isos/import-jobs/{job_id}").json() if job["status"] in {"completed", "failed"}: break time.sleep(0.01) assert job["status"] == "completed", job def test_user_iso_job_hides_internal_paths_in_failures(boundary): client, _engine, _store, _logs, _services = boundary with patch("virsh_panel.user_isos.download_user_iso", side_effect=TemplateError("目录 /private/data/user-isos/41 损坏", 500)): response = call(client, "POST", "/internal/v1/user-isos/import", USER, {"url": "https://downloads.example.test/os.iso", "name": "Error Installer", "sha256": ""}) assert response.status_code == 202 job_id = response.json()["job_id"] for _ in range(100): job = call(client, "GET", f"/internal/v1/user-isos/import-jobs/{job_id}").json() if job["status"] in {"completed", "failed"}: break time.sleep(0.01) assert job["status"] == "failed" assert job["error"] == "ISO 下载或保存失败,请联系管理员查看日志" assert "/private/data" not in json.dumps(job, ensure_ascii=False) def test_user_iso_import_persistence_failure_does_not_stick_pending(boundary, tmp_path): client, _engine, store, _logs, _services = boundary assert call(client, "GET", "/internal/v1/user-isos").status_code == 200 original_set = store.set def fail_user_iso_job(kind, key, value): if kind == "user_iso_import_job": raise DatabaseError() return original_set(kind, key, value) body = {"url": "https://downloads.example.test/os.iso", "name": "Installer", "sha256": ""} with patch.object(store, "set", side_effect=fail_user_iso_job): failed = call(client, "POST", "/internal/v1/user-isos/import", USER, body) assert failed.status_code == 503 download = tmp_path / "retry.iso" download.write_bytes(b"i" * (1024 * 1024)) with patch("virsh_panel.user_isos.download_user_iso", return_value=(download, download.stat().st_size, "b" * 64)): retried = call(client, "POST", "/internal/v1/user-isos/import", USER, body) assert retried.status_code == 202, retried.text for _ in range(100): job = call( client, "GET", f"/internal/v1/user-isos/import-jobs/{retried.json()['job_id']}", ).json() if job["status"] in {"completed", "failed"}: break time.sleep(0.01) assert job["status"] == "completed", job def test_interrupted_user_iso_import_is_restored_as_failed(boundary): client, _engine, store, _logs, _services = boundary job_id = "interrupted-user-iso" scope = user_iso_scope(USER["user_id"]) store.set("user_iso_import_job", job_id, json.dumps({ "job_id": job_id, "scope": scope, "owner_user_id": USER["user_id"], "status": "running", "phase": "正在安全下载 ISO", "created_at": 1, })) _engine.registry.user_isos._restored = False _engine.registry.user_isos.restore_jobs() restored = call( client, "GET", f"/internal/v1/user-isos/import-jobs/{job_id}", ) assert restored.status_code == 200 assert restored.json()["status"] == "failed" assert "重启" in restored.json()["phase"] def test_user_iso_list_exposes_only_owners_active_import(boundary, tmp_path): client, engine, _store, _logs, _services = boundary download = tmp_path / "active.iso" download.write_bytes(b"i" * (1024 * 1024)) started = threading.Event() release = threading.Event() def blocking_download(_url, _sha256, _directory, *, max_bytes): assert max_bytes >= download.stat().st_size started.set() assert release.wait(timeout=5) return download, download.stat().st_size, "c" * 64 body = {"url": "https://downloads.example.test/os.iso", "name": "Active Installer", "sha256": ""} try: with patch("virsh_panel.user_isos.download_user_iso", side_effect=blocking_download): queued = call(client, "POST", "/internal/v1/user-isos/import", USER, body) assert queued.status_code == 202 assert started.wait(timeout=2) assert engine._long_running_host_job() is True own = call(client, "GET", "/internal/v1/user-isos", USER).json() active = own["active_import_job"] assert active["job_id"] == queued.json()["job_id"] assert active["status"] == "running" assert set(active) <= {"job_id", "status", "phase", "created_at", "started_at", "finished_at", "iso", "error"} assert "scope" not in active and "owner_user_id" not in active other = call(client, "GET", "/internal/v1/user-isos", {**USER, "user_id": 42}).json() assert "active_import_job" not in other release.set() for _ in range(100): job = call( client, "GET", f"/internal/v1/user-isos/import-jobs/{active['job_id']}", ).json() if job["status"] in {"completed", "failed"}: break time.sleep(0.01) finally: release.set() assert job["status"] == "completed", job assert "active_import_job" not in call( client, "GET", "/internal/v1/user-isos", USER).json() def test_hmac_binds_actor_payload_target_and_rejects_replay(): store = MemoryStore() body = b"{}" target = "/internal/v1/hosts/hk/vms?range=1h" headers = {key.lower(): value for key, value in signed_headers(SECRET, "POST", target, body, USER, timestamp=1000, nonce="n" * 24).items()} assert authenticate(headers, SECRET, "POST", target, hashlib.sha256(body).hexdigest(), store, now=1000) == USER with pytest.raises(EngineAuthError): authenticate(headers, SECRET, "POST", target, hashlib.sha256(body).hexdigest(), store, now=1000) changed = {**headers, "x-cloud-actor": encode_actor(SUPER)} with pytest.raises(EngineAuthError): authenticate(changed, SECRET, "POST", target, hashlib.sha256(body).hexdigest(), store, now=1000) with pytest.raises(EngineAuthError): authenticate(headers, SECRET, "POST", target + "x", hashlib.sha256(body).hexdigest(), store, now=1000) with pytest.raises(EngineAuthError): authenticate(headers, SECRET, "POST", target, hashlib.sha256(b"other").hexdigest(), store, now=1000) with pytest.raises(EngineAuthError): authenticate(headers, SECRET, "POST", target, hashlib.sha256(body).hexdigest(), store, now=1061) invalid_cleanup = {**USER, "system_cleanup": "true"} invalid_headers = {key.lower(): value for key, value in signed_headers( SECRET, "POST", target, body, invalid_cleanup, timestamp=1000, nonce="c" * 24, ).items()} with pytest.raises(EngineAuthError): authenticate( invalid_headers, SECRET, "POST", target, hashlib.sha256(body).hexdigest(), store, now=1000, ) def test_private_route_allowlist_and_no_browser_admin_cookie(boundary): client, *_ = boundary for path in ("/api/v1/vms", "/internal/v1/hosts/hk/register", "/internal/v1/hosts/hk/setup/admin", "/internal/v1/hosts/hk/update/apply", "/internal/v1/hosts/hk/smtp/settings"): assert call(client, "POST", path, SUPER, {}).status_code == 404 assert client.get("/internal/v1/hosts/hk/vms", cookies={"virsh_panel_session": "admin"}).status_code == 401 assert TRUSTED_ACTOR.get() is None def test_internal_update_routes_are_super_admin_only_and_never_echo_token(boundary): client, engine, *_ = boundary updater = FakeUpdater() triggered = [] engine.updater = updater def trigger(): triggered.append(True) updater.active["status"] = "running" engine.update_trigger = trigger settings_path = "/internal/v1/update/settings" assert client.get(settings_path, cookies={"venix": "admin"}).status_code == 401 assert call(client, "GET", settings_path, USER).status_code == 403 assert call(client, "GET", settings_path, ADMIN).status_code == 403 assert call(client, "GET", settings_path, {**SUPER, "can_manage": False}).status_code == 403 token = "gitea-private-token-123456" saved = call(client, "PUT", settings_path, SUPER, {"access_token": token}) assert saved.status_code == 200 assert saved.json()["token_configured"] is True assert saved.json()["repository"] == "https://g.luoli.live/root/virt.git" assert saved.json()["branch"] == "venix" assert token not in saved.text assert token not in call(client, "GET", settings_path, SUPER).text checked = call(client, "POST", "/internal/v1/update/check", SUPER, {}) assert checked.status_code == 200 assert checked.json()["status"] == "update_available" applied = call(client, "POST", "/internal/v1/update/apply", SUPER, { "expected_current_commit": "a" * 40, "expected_remote_commit": "b" * 40, }) assert applied.status_code == 202 assert applied.json()["job_id"] == "update-job-1234567890" assert "private" not in applied.json() and token not in applied.text assert triggered == [True] job = call(client, "GET", "/internal/v1/update/jobs/update-job-1234567890", SUPER) assert job.status_code == 200 and token not in job.text and "private" not in job.json() # While the persisted update job is active, reads remain available but no # host/VM mutation may begin and overlap the code/service restart. assert call(client, "GET", "/internal/v1/health", SUPER).status_code == 200 assert call(client, "POST", "/internal/v1/hosts/scan", SUPER, {"host": "host.example.test", "port": 22}).status_code == 409 def test_internal_update_validation_clear_and_trigger_failure(boundary): client, engine, *_ = boundary updater = FakeUpdater() engine.updater = updater settings_path = "/internal/v1/update/settings" assert call(client, "PUT", settings_path, SUPER, {}).status_code == 422 assert call(client, "PUT", settings_path, SUPER, {"access_token": "x", "unexpected": True}).status_code == 422 assert call(client, "POST", "/internal/v1/update/check", SUPER, {"repository": "https://evil.example/repo.git"}).status_code == 422 assert call(client, "POST", "/internal/v1/update/apply", SUPER, {"expected_current_commit": "a" * 40}).status_code == 422 assert call(client, "GET", "/internal/v1/update/jobs/missing-job-123456", SUPER).status_code == 404 call(client, "PUT", settings_path, SUPER, {"access_token": "valid-token-123456"}) cleared = call(client, "PUT", settings_path, SUPER, {"clear_token": True}) assert cleared.status_code == 200 and cleared.json()["token_configured"] is False call(client, "PUT", settings_path, SUPER, {"access_token": "valid-token-123456"}) def fail_trigger(): from virsh_panel.updater import UpdateError raise UpdateError("独立更新服务启动失败,请确认 virt-cloud-update.service 已安装", 503) engine.update_trigger = fail_trigger failed = call(client, "POST", "/internal/v1/update/apply", SUPER, { "expected_current_commit": "a" * 40, "expected_remote_commit": "b" * 40, }) assert failed.status_code == 503 job = updater.jobs["update-job-1234567890"] assert job["status"] == "failed" assert "令牌" not in job["error"] @pytest.mark.parametrize(("state", "expected"), [ ("activating", True), ("active", True), ("deactivating", True), ("inactive", False), ("failed", False), ("maintenance", None), ]) def test_update_oneshot_transitional_states_keep_maintenance_gate(state, expected): with patch("virsh_panel.internal.subprocess.run", return_value=SimpleNamespace(returncode=0, stdout=state + "\n")): assert EngineApplication._update_service_active() is expected def test_user_admin_super_admin_boundaries_and_cross_user_vm(boundary): client, engine, *_ = boundary path = "/internal/v1/hosts/hk" assert call(client, "GET", path + "/host").status_code == 403 assert call(client, "GET", path + "/host", ADMIN).status_code == 200 assert call(client, "POST", path + "/host/scan", ADMIN, {"host": "x", "port": 22}).status_code == 403 assert call(client, "POST", path + "/host/scan", SUPER, {"host": "x", "port": 22}).status_code == 200 assert call(client, "POST", "/internal/v1/hosts/scan", ADMIN, {"host": "x", "port": 22}).status_code == 403 component = engine.registry.components["hk"] component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41) assert call(client, "GET", path + f"/vms/{VM_ID}").status_code == 200 assert call(client, "GET", path + f"/vms/{VM_ID}", {**USER, "user_id": 42}).status_code == 404 assert call(client, "GET", path + "/vms", {**USER, "user_id": 42}).json()["items"] == [] assert call(client, "POST", path + f"/vms/{VM_ID}/actions", {**USER, "can_manage": False}, {"action": "reboot"}).status_code == 403 def test_existing_unmanaged_vms_are_claimed_once_for_configuring_admin(boundary): client, engine, _, _, services = boundary path = "/internal/v1/hosts/hk" assert call(client, "GET", path + "/host", ADMIN).status_code == 200 manual_uuid = "22222222-2222-4222-8222-222222222222" orphaned_managed_uuid = "33333333-3333-4333-8333-333333333333" services["hk"].other_vms = [ {"uuid": manual_uuid, "name": "manual-vm", "state": "shutoff", "vcpu": 2, "memory_mib": 2048, "disks": [], "managed": False}, {"uuid": orphaned_managed_uuid, "name": "unknown-panel-vm", "state": "shutoff", "vcpu": 2, "memory_mib": 2048, "disks": [], "managed": True}, ] body = {"known_owners": {VM_ID: 41}} assert call(client, "POST", path + "/host/discover-vms", ADMIN, body).status_code == 403 discovered = call(client, "POST", path + "/host/discover-vms", SUPER, body) assert discovered.status_code == 200, discovered.text rows = {row["uuid"]: row for row in discovered.json()["items"]} assert rows[VM_ID]["owner_user_id"] == 41 assert rows[manual_uuid]["owner_user_id"] == SUPER["user_id"] assert orphaned_managed_uuid not in rows assert discovered.json()["claimed_count"] == 2 assert discovered.json()["untracked_managed_count"] == 1 assert discovered.json()["total_count"] == 3 component = engine.registry.components["hk"] scope = host_scope(component.manager.profile) assert component.state.users.owner_of_vm(scope, manual_uuid) == SUPER["user_id"] adopted = call(client, "POST", path + "/host/discover-vms", SUPER, {**body, "claim_unowned_managed": True}) assert adopted.status_code == 200 adopted_rows = {row["uuid"]: row for row in adopted.json()["items"]} assert adopted_rows[orphaned_managed_uuid]["owner_user_id"] == SUPER["user_id"] assert adopted.json()["claimed_count"] == 1 assert adopted.json()["untracked_managed_count"] == 0 repeated = call(client, "POST", path + "/host/discover-vms", {**SUPER, "user_id": 2}, {**body, "claim_unowned_managed": True}) assert repeated.status_code == 200 repeated_rows = {row["uuid"]: row for row in repeated.json()["items"]} assert repeated_rows[manual_uuid]["owner_user_id"] == SUPER["user_id"] assert repeated_rows[orphaned_managed_uuid]["owner_user_id"] == SUPER["user_id"] assert repeated.json()["claimed_count"] == 0 assert component.state.users.owner_of_vm(scope, manual_uuid) == SUPER["user_id"] assert {row["uuid"] for row in call( client, "GET", path + "/vms", {**USER, "user_id": SUPER["user_id"]} ).json()["items"]} == {manual_uuid, orphaned_managed_uuid} assert call(client, "GET", path + "/vms", {**USER, "user_id": 2}).json()["items"] == [] def test_stable_host_identity_and_namespaces(): store = MemoryStore() first, second = HostStore(store, "a"), HostStore(store, "b") first.set("host", "current", "one") second.set("host", "current", "two") assert first.get("host", "current") == "one" assert second.get("host", "current") == "two" a = {"host_id": "a", "host": "1.2.3.4", "port": 22, "fingerprint": "old"} assert host_scope(a) == host_scope({**a, "host": "changed", "port": 2222, "fingerprint": "new"}) assert host_scope(a) != host_scope({**a, "host_id": "b"}) assert MainStore.RECORD_TABLE == "engine_records" assert LogStore.METRIC_TABLE == "engine_vm_metric_samples" def test_lightweight_vm_state_reads_are_allowlisted(boundary): client, _engine, _store, _logs, services = boundary listed = call(client, "GET", "/internal/v1/hosts/hk/vm-states", SUPER) assert listed.status_code == 200, listed.text assert listed.json()["items"] == [{ "uuid": VM_ID, "name": "alma-test", "state": "running", }] single = call( client, "GET", f"/internal/v1/hosts/hk/vms/{VM_ID}/state", SUPER, ) assert single.status_code == 200, single.text assert single.json() == { "uuid": VM_ID, "name": "alma-test", "state": "running", } assert ("list_vm_states",) in services["hk"].calls assert ("get_vm_state", VM_ID) in services["hk"].calls def test_selected_vm_metadata_is_allowlisted_and_owner_scoped(boundary): client, engine, _store, _logs, services = boundary path = f"/internal/v1/hosts/hk/vm-metadata?ids={VM_ID}" admin = call(client, "GET", path, SUPER) assert admin.status_code == 200, admin.text assert admin.json()["items"][0]["uuid"] == VM_ID assert services["hk"].calls[-1] == ("list_vm_metadata", (VM_ID,)) component = engine.registry.components["hk"] scope = host_scope(component.manager.profile) component.state.users.assign_vm(scope, VM_ID, USER["user_id"]) owner = call(client, "GET", path, USER) assert owner.status_code == 200, owner.text calls_after_owner = services["hk"].calls.count(("list_vm_metadata", (VM_ID,))) assert calls_after_owner == 2 stranger = call(client, "GET", path, {**USER, "user_id": 42}) assert stranger.status_code == 404 assert stranger.json()["error"] == "虚拟机不存在" assert services["hk"].calls.count(("list_vm_metadata", (VM_ID,))) == calls_after_owner def test_only_paid_operations_can_create_and_retry_never_duplicates(boundary): client, engine, store, _, services = boundary path = "/internal/v1/hosts/hk/vms" assert call(client, "POST", path, USER, CREATE_BODY).status_code == 403 provisioning = {**USER, "can_provision": True, "operation_id": "order-000001"} created = call(client, "POST", path, provisioning, CREATE_BODY) assert created.status_code == 201, created.text vm_id = created.json()["uuid"] assert created.json()["result"]["uuid"] == vm_id assert created.json()["status"] == "succeeded" component = engine.registry.components["hk"] assert component.state.users.owns_vm(host_scope(component.manager.profile), vm_id, 41) again = call(client, "POST", path, provisioning, CREATE_BODY) assert again.status_code == 201 and again.json()["uuid"] == vm_id assert len([row for row in services["hk"].calls if row[0] == "create_vm"]) == 1 assert call(client, "POST", path, provisioning, {**CREATE_BODY, "vcpu": 4}).status_code == 409 operation = call(client, "GET", "/internal/v1/operations/order-000001").json() assert operation["status"] == "succeeded" and operation["result"]["uuid"] == vm_id assert operation["safe_to_refund"] is False assert call(client, "GET", "/internal/v1/operations/order-000001", {**USER, "user_id": 42}).status_code == 404 def test_uncertain_creation_never_refunds_or_recreates(boundary): client, engine, store, _, services = boundary path = "/internal/v1/hosts/hk/vms" call(client, "GET", path) services["hk"].create_failure = True actor = {**USER, "can_provision": True, "operation_id": "order-uncertain"} failed = call(client, "POST", path, actor, CREATE_BODY) assert failed.status_code == 502 assert failed.json()["status"] == "uncertain" and not failed.json()["safe_to_refund"] record = json.loads(store.get("operation", "order-uncertain")) assert record["resource_naming"] == "uuid-v1" component = engine.registry.components["hk"] scope = host_scope(component.manager.profile) assert component.state.users.owns_vm(scope, record["vm_uuid"], 41) component.store.set("vm_credential", f"{scope}.{record['vm_uuid']}", "saved-password") component.store.set("ip_claim", f"{scope}.192.0.2.1", json.dumps({"vm_uuid": record["vm_uuid"]})) again = call(client, "POST", path, actor, CREATE_BODY) assert again.status_code == 202 and again.json()["status"] == "uncertain" assert len([row for row in services["hk"].calls if row[0] == "create_vm"]) == 1 services["hk"].absent = True operation = call(client, "GET", "/internal/v1/operations/order-uncertain").json() assert operation["status"] == "failed" and operation["safe_to_refund"] is True assert services["hk"].absent_checks[-1] == ( record["vm_uuid"], CREATE_BODY["name"], True, False) assert not component.state.users.owns_vm(scope, record["vm_uuid"], 41) assert component.store.get("vm_credential", f"{scope}.{record['vm_uuid']}") is None assert component.store.list("ip_claim") == {} def test_legacy_name_collision_operation_uses_legacy_absent_reconciliation(boundary): client, _engine, store, _logs, services = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") services["hk"].absent = True legacy_vm_id = "00000000-0000-4000-8000-000000000002" store.set("operation", "legacy-name-collision", json.dumps({ "operation_id": "legacy-name-collision", "host_id": "hk", "user_id": USER["user_id"], "request_hash": "legacy", "vm_uuid": legacy_vm_id, "vm_name": CREATE_BODY["name"], "kind": "create", "resource": "vms", "method": "POST", "status": "uncertain", "result": None, "error": "虚拟机名称已存在", "safe_to_refund": False, "created_at": 1, })) operation = call( client, "GET", "/internal/v1/operations/legacy-name-collision", USER, ).json() assert operation["status"] == "failed" assert operation["safe_to_refund"] is True assert services["hk"].absent_checks[-1] == ( legacy_vm_id, CREATE_BODY["name"], False, True) @pytest.mark.parametrize("payload", [{"name": "invalid"}, ["not-an-object"], {**CREATE_BODY, "guest_ssh_key_id": 3}]) def test_rejected_create_has_durable_safe_failure_even_when_response_is_lost(boundary, payload): client, engine, _, _, services = boundary path = "/internal/v1/hosts/hk/vms" actor = {**USER, "can_provision": True, "operation_id": "rejected-create"} response = call(client, "POST", path, actor, payload) assert response.status_code == 422 # Simulate a dispatcher losing the POST response and querying after restart. engine.active_operations.clear() saved = call(client, "GET", "/internal/v1/operations/rejected-create").json() assert saved["status"] == "failed" and saved["safe_to_refund"] is True assert call(client, "POST", path, actor, payload).status_code == 422 assert call(client, "POST", path, actor, CREATE_BODY).status_code == 409 assert not any(row[0] == "create_vm" for row in services["hk"].calls) def test_admin_created_vm_also_has_external_owner(boundary): client, engine, *_ = boundary actor = {**SUPER, "can_provision": True, "operation_id": "order-admin"} response = call(client, "POST", "/internal/v1/hosts/hk/vms", actor, CREATE_BODY) assert response.status_code == 201 components = engine.registry.components["hk"] assert components.state.users.owns_vm(host_scope(components.manager.profile), response.json()["uuid"], 1) def test_power_action_has_persistent_idempotency(boundary): client, engine, _, _, services = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") components = engine.registry.components["hk"] components.state.users.assign_vm(host_scope(components.manager.profile), VM_ID, 41) path = f"/internal/v1/hosts/hk/vms/{VM_ID}/actions" assert call(client, "POST", path, USER, {"action": "reboot"}).status_code == 403 actor = {**USER, "operation_id": "reboot-operation"} assert call(client, "POST", path, actor, {"action": "reboot"}).status_code == 200 assert call(client, "POST", path, actor, {"action": "reboot"}).status_code == 200 assert len([row for row in services["hk"].calls if row[0] == "action"]) == 1 assert call(client, "POST", path, actor, {"action": "poweroff"}).status_code == 409 operation = call(client, "GET", "/internal/v1/operations/reboot-operation").json() assert operation["status"] == "succeeded" and operation["safe_to_refund"] is False def test_only_signed_system_cleanup_can_delete_stopped_rescue_vm(boundary): client, engine, _, _, services = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") components = engine.registry.components["hk"] scope = host_scope(components.manager.profile) components.state.users.assign_vm(scope, VM_ID, USER["user_id"]) rescue_key = f"{scope}.{VM_ID}" components.store.set("vm_rescue", rescue_key, json.dumps({ "original_xml": "alma-test", "template_id": "0123456789abcdef0123456789abcdef", "volume_name": "panel-rescue-0123456789abcdef0123456789abcdef.iso", "status": "active", })) services["hk"].vm.update( state="shutoff", rescue_mode=True, rescue_staged=True, rescue_volume_name="panel-rescue-0123456789abcdef0123456789abcdef.iso", ) path = f"/internal/v1/hosts/hk/vms/{VM_ID}?delete_disks=true" ordinary = call( client, "DELETE", path, {**USER, "operation_id": "ordinary-rescue-delete", "system_cleanup": False}, ) assert ordinary.status_code == 409 assert components.store.get("vm_rescue", rescue_key) is not None assert not any(item[0] == "delete_vm" for item in services["hk"].calls) cleanup = call( client, "DELETE", path, {**USER, "operation_id": "expiry-rescue-delete", "system_cleanup": True}, ) assert cleanup.status_code == 200, cleanup.text assert cleanup.json()["status"] == "succeeded" assert services["hk"].calls[-1] == ("delete_vm", VM_ID, True) assert components.store.get("vm_rescue", rescue_key) is None assert components.state.users.owner_of_vm(scope, VM_ID) is None @pytest.mark.parametrize(("route", "method", "target"), [ ("service-suspend", "service_suspend", {"state": "shutoff", "autostart": False}), ("service-resume", "service_resume", {"autostart": True}), ("network-suspend", "network_suspend", {"network_suspended": True}), ("network-resume", "network_resume", {"network_suspended": False}), ]) def test_private_target_state_operations_are_owned_idempotent_mutations( boundary, route, method, target): client, engine, _, _, services = boundary base = f"/internal/v1/hosts/hk/vms/{VM_ID}/{route}" call(client, "GET", "/internal/v1/hosts/hk/vms") component = engine.registry.components["hk"] component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41) assert call(client, "POST", base, USER, {}).status_code == 403 assert call(client, "POST", base, {**USER, "can_manage": False, "operation_id": f"blocked-{route}"}, {}).status_code == 403 assert call(client, "POST", base, {**USER, "user_id": 42, "operation_id": f"foreign-{route}"}, {}).status_code == 404 actor = {**USER, "operation_id": f"target-{route}"} first = call(client, "POST", base, actor, {}) assert first.status_code == 200, first.text assert first.json()["status"] == "succeeded" assert all(first.json()[key] == value for key, value in target.items()) repeated = call(client, "POST", base, actor, {}) assert repeated.status_code == 200 assert len([item for item in services["hk"].calls if item[0] == method]) == 1 @pytest.mark.parametrize(("route", "method", "drift"), [ ("service-suspend", "service_suspend", {"state": "running", "autostart": True}), ("service-resume", "service_resume", {"autostart": False}), ("network-suspend", "network_suspend", {"network_suspended": False}), ("network-resume", "network_resume", {"network_suspended": True}), ]) def test_target_state_operation_replays_after_lost_response( boundary, route, method, drift): client, engine, store, _, services = boundary base = f"/internal/v1/hosts/hk/vms/{VM_ID}/{route}" call(client, "GET", "/internal/v1/hosts/hk/vms") component = engine.registry.components["hk"] component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41) actor = {**USER, "operation_id": f"replay-{route}"} assert call(client, "POST", base, actor, {}).status_code == 200 saved = json.loads(store.get("operation", actor["operation_id"])) saved.update(status="uncertain", result=None, error="response lost") store.set("operation", actor["operation_id"], json.dumps(saved)) services["hk"].vm.update(drift) engine.active_operations.clear() replayed = call(client, "GET", f"/internal/v1/operations/{actor['operation_id']}", actor) assert replayed.status_code == 200, replayed.text assert replayed.json()["status"] == "succeeded" assert len([item for item in services["hk"].calls if item[0] == method]) == 2 saved = json.loads(store.get("operation", actor["operation_id"])) saved.update(status="uncertain", result=None, error="response lost again") store.set("operation", actor["operation_id"], json.dumps(saved)) services["hk"].vm.update(drift) replayed_post = call(client, "POST", base, actor, {}) assert replayed_post.status_code == 200, replayed_post.text assert replayed_post.json()["status"] == "succeeded" assert len([item for item in services["hk"].calls if item[0] == method]) == 3 def test_non_reconciling_operation_lookup_never_replays_target_state(boundary): client, engine, store, _, services = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") component = engine.registry.components["hk"] component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41) path = f"/internal/v1/hosts/hk/vms/{VM_ID}/network-suspend" actor = {**USER, "operation_id": "peek-target-state"} assert call(client, "POST", path, actor, {}).status_code == 200 saved = json.loads(store.get("operation", actor["operation_id"])) saved.update(status="uncertain", result=None, error="response lost") store.set("operation", actor["operation_id"], json.dumps(saved)) services["hk"].vm["network_suspended"] = False engine.active_operations.clear() before = len(services["hk"].calls) peeked = call(client, "GET", "/internal/v1/operations/peek-target-state?reconcile=false", actor) assert peeked.status_code == 200 assert peeked.json()["status"] == "uncertain" assert peeked.json()["executor_active"] is False assert len(services["hk"].calls) == before assert services["hk"].vm["network_suspended"] is False engine.active_operations.add(actor["operation_id"]) active = call(client, "GET", "/internal/v1/operations/peek-target-state?reconcile=false", actor) assert active.status_code == 200 assert active.json()["executor_active"] is True assert len(services["hk"].calls) == before engine.active_operations.discard(actor["operation_id"]) def test_target_state_lock_serializes_same_vm_and_not_other_vms(boundary): _, engine, *_ = boundary async def exercise(): first = engine._target_lock("hk", VM_ID) same = engine._target_lock("hk", VM_ID) other = engine._target_lock("hk", "22222222-2222-4222-8222-222222222222") assert first is same assert first is not other entered = [] release = asyncio.Event() async def owner(): async with first: entered.append("owner") await release.wait() entered.append("owner-finished") async def waiter(): async with same: entered.append("waiter") owner_task = asyncio.create_task(owner()) await asyncio.sleep(0) waiter_task = asyncio.create_task(waiter()) await asyncio.sleep(0) assert entered == ["owner"] release.set() await asyncio.gather(owner_task, waiter_task) assert entered == ["owner", "owner-finished", "waiter"] errors = [] def run_in_isolated_loop(): try: asyncio.run(exercise()) except BaseException as exc: # propagate assertion failures to pytest errors.append(exc) thread = threading.Thread(target=run_in_isolated_loop) thread.start() thread.join(timeout=5) assert not thread.is_alive() if errors: raise errors[0] def test_expiry_target_waits_for_inflight_user_mutation_on_same_vm(boundary): client, engine, _, _, services = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") components = engine.registry.components["hk"] components.state.users.assign_vm( host_scope(components.manager.profile), VM_ID, USER["user_id"]) service = services["hk"] action_entered = threading.Event() release_action = threading.Event() network_entered = threading.Event() original_action = service.action original_network_suspend = service.network_suspend def slow_action(vm_id, action): action_entered.set() assert release_action.wait(timeout=5), "test did not release user mutation" return original_action(vm_id, action) def observed_network_suspend(vm_id): network_entered.set() return original_network_suspend(vm_id) service.action = slow_action service.network_suspend = observed_network_suspend responses = {} def user_request(): responses["action"] = call( client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/actions", {**USER, "operation_id": "concurrent-user-action"}, {"action": "reset"}) def policy_request(): responses["policy"] = call( client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/network-suspend", {**USER, "operation_id": "concurrent-expiry-policy"}, {}) first = threading.Thread(target=user_request) first.start() assert action_entered.wait(timeout=5) second = threading.Thread(target=policy_request) second.start() assert not network_entered.wait(timeout=0.2) release_action.set() first.join(timeout=5) second.join(timeout=5) assert not first.is_alive() and not second.is_alive() assert responses["action"].status_code == 200 assert responses["policy"].status_code == 200 assert network_entered.is_set() mutations = [item[0] for item in service.calls if item[0] in {"action", "network_suspend"}] assert mutations == ["action", "network_suspend"] assert service.vm["network_suspended"] is True def test_target_state_reconciliation_does_not_bypass_original_owner(boundary): client, engine, store, _, services = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") component = engine.registry.components["hk"] component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41) store.set("operation", "unsafe-target", json.dumps({ "operation_id": "unsafe-target", "host_id": "hk", "user_id": 42, "kind": "mutation", "resource": f"vms/{VM_ID}/network-suspend", "vm_uuid": VM_ID, "status": "uncertain", "result": None, "actor_can_manage": True, "actor_is_admin": False, "safe_to_refund": False, })) response = call(client, "GET", "/internal/v1/operations/unsafe-target", {**USER, "user_id": 42}) assert response.status_code == 200 assert response.json()["status"] == "failed" assert not any(item[0] == "network_suspend" for item in services["hk"].calls) def test_async_mutation_operation_survives_lost_job_response_and_does_not_retry(boundary): client, engine, store, *_ = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") components = engine.registry.components["hk"] job = {"operation_id": "lost-rescue-job", "owner": "php:41", "vm_id": VM_ID, "status": "running", "phase": "正在救援"} components.store.set("rescue_job", "job-abc", json.dumps(job)) operation = {"operation_id": "lost-rescue-job", "host_id": "hk", "user_id": 41, "kind": "mutation", "resource": f"vms/{VM_ID}/rescue", "vm_uuid": VM_ID, "status": "uncertain", "result": None, "safe_to_refund": False} store.set("operation", "lost-rescue-job", json.dumps(operation)) recovered = call(client, "GET", "/internal/v1/operations/lost-rescue-job").json() assert recovered["status"] == "running" and recovered["result"]["vm_id"] == VM_ID job.update(status="completed", phase="救援系统启动") components.store.set("rescue_job", "job-abc", json.dumps(job)) completed = call(client, "GET", "/internal/v1/operations/lost-rescue-job").json() assert completed["status"] == "succeeded" assert not completed["safe_to_refund"] def test_interrupted_async_job_becomes_terminal_without_replay(boundary): client, engine, store, *_ = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") components = engine.registry.components["hk"] components.store.set("reinstall_job", "job-abcd", json.dumps({ "operation_id": "interrupted-job", "owner": "php:41", "vm_id": VM_ID, "status": "failed", "phase": "面板重启,任务已中断"})) store.set("operation", "interrupted-job", json.dumps({"operation_id": "interrupted-job", "host_id": "hk", "user_id": 41, "kind": "mutation", "resource": f"vms/{VM_ID}/reinstall", "vm_uuid": VM_ID, "status": "pending", "result": None, "safe_to_refund": False})) recovered = call(client, "GET", "/internal/v1/operations/interrupted-job").json() assert recovered["status"] == "failed" assert "中断" in recovered["error"] @pytest.mark.parametrize("resource", [ f"vms/{VM_ID}/actions", f"vms/{VM_ID}", f"vms/{VM_ID}/reinstall", f"vms/{VM_ID}/rescue", ]) def test_orphaned_mutation_becomes_terminal_without_replay(boundary, resource): client, _, store, *_ = boundary operation_id = "orphan-" + resource.rsplit("/", 1)[-1].replace("?", "-") store.set("operation", operation_id, json.dumps({ "operation_id": operation_id, "host_id": "hk", "user_id": 41, "kind": "mutation", "resource": resource, "vm_uuid": VM_ID, "status": "running", "result": None, "safe_to_refund": False, "created_at": 1, })) recovered = call(client, "GET", f"/internal/v1/operations/{operation_id}").json() assert recovered["status"] == "failed" assert "不会自动重放" in recovered["error"] def test_service_resume_replays_exact_start_target_after_lost_response(boundary): client, engine, store, _, services = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") components = engine.registry.components["hk"] components.state.users.assign_vm( host_scope(components.manager.profile), VM_ID, USER["user_id"]) services["hk"].vm.update(state="shutoff", autostart=False) actor = {**USER, "operation_id": "resume-start-target"} path = f"/internal/v1/hosts/hk/vms/{VM_ID}/service-resume" first = call(client, "POST", path, actor, {"start_if_suspended": True}) assert first.status_code == 200, first.text assert services["hk"].vm["state"] == "running" assert services["hk"].calls[-1] == ("service_resume", VM_ID, True) saved = json.loads(store.get("operation", actor["operation_id"])) saved.update(status="uncertain", result=None, error="response lost") store.set("operation", actor["operation_id"], json.dumps(saved)) services["hk"].vm.update(state="shutoff", autostart=False) engine.active_operations.clear() replayed = call(client, "GET", "/internal/v1/operations/resume-start-target", actor) assert replayed.status_code == 200 assert replayed.json()["status"] == "succeeded" assert services["hk"].vm["state"] == "running" assert services["hk"].calls[-1] == ("service_resume", VM_ID, True) def test_service_suspend_replay_preserves_original_running_state(boundary): client, engine, store, _, services = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") components = engine.registry.components["hk"] components.state.users.assign_vm( host_scope(components.manager.profile), VM_ID, USER["user_id"]) actor = {**USER, "operation_id": "suspend-running-hint"} path = f"/internal/v1/hosts/hk/vms/{VM_ID}/service-suspend" assert call(client, "POST", path, actor, {}).status_code == 200 saved = json.loads(store.get("operation", actor["operation_id"])) assert saved["service_was_running_hint"] is True # Model a hard crash after libvirt stopped the guest but before the # successful result was durable. Reconciliation now observes shutoff, yet # must retain the original running state for the billing resume policy. saved.update(status="uncertain", result=None, error="engine crashed") store.set("operation", actor["operation_id"], json.dumps(saved)) services["hk"].vm.update(state="shutoff", autostart=False) engine.active_operations.clear() reconciled = call(client, "GET", "/internal/v1/operations/suspend-running-hint", actor) assert reconciled.status_code == 200 assert reconciled.json()["status"] == "succeeded" assert reconciled.json()["result"]["service_was_running"] is True def test_service_suspend_replay_without_running_hint_fails_closed(boundary): client, engine, store, _, services = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") components = engine.registry.components["hk"] components.state.users.assign_vm( host_scope(components.manager.profile), VM_ID, USER["user_id"]) operation_id = "suspend-missing-running-hint" store.set("operation", operation_id, json.dumps({ "operation_id": operation_id, "host_id": "hk", "user_id": USER["user_id"], "vm_uuid": VM_ID, "kind": "mutation", "resource": f"vms/{VM_ID}/service-suspend", "method": "POST", "status": "uncertain", "actor_can_manage": True, "actor_is_admin": False, "created_at": 1, })) before_calls = list(services["hk"].calls) response = call( client, "GET", f"/internal/v1/operations/{operation_id}", USER) assert response.status_code == 200 assert response.json()["status"] == "failed" assert "已取消服务暂停" in response.json()["error"] assert services["hk"].calls == before_calls def test_service_suspend_does_not_stop_vm_when_running_hint_cannot_persist( boundary, monkeypatch): client, engine, store, _, services = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") components = engine.registry.components["hk"] components.state.users.assign_vm( host_scope(components.manager.profile), VM_ID, USER["user_id"]) operation_id = "suspend-hint-write-failed" original_set = store.set def fail_operation_write(kind, key, value): if kind == "operation" and key == operation_id: raise RuntimeError("database unavailable") return original_set(kind, key, value) monkeypatch.setattr(store, "set", fail_operation_write) failed = call( client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/service-suspend", {**USER, "operation_id": operation_id}, {}) assert failed.status_code == 503 assert services["hk"].vm["state"] == "running" assert not any( item[0] == "service_suspend" for item in services["hk"].calls) monkeypatch.setattr(store, "set", original_set) recovered = call( client, "GET", f"/internal/v1/operations/{operation_id}", USER) assert recovered.status_code == 200 assert recovered.json()["status"] == "failed" assert services["hk"].vm["state"] == "running" assert not any( item[0] == "service_suspend" for item in services["hk"].calls) def test_metrics_query_uses_log_store_and_owner_permission(boundary): client, engine, _, logs, _ = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") component = engine.registry.components["hk"] scope = host_scope(component.manager.profile) component.state.users.assign_vm(scope, VM_ID, 41) response = call(client, "GET", f"/internal/v1/hosts/hk/vms/{VM_ID}/metrics?range=30d") assert response.status_code == 200 and response.json()["points"][0]["cpu_percent"] == 20 with patch("virsh_panel.app.time.time", return_value=3_000_000): traffic = call( client, "GET", f"/internal/v1/hosts/hk/vms/{VM_ID}/traffic/daily?days=2", ) assert traffic.status_code == 200 assert traffic.json()["points"][-1] == { "date": "1970-02-04", "day_start": 2_937_600, "download_bytes": 4096, "upload_bytes": 1024, "total_bytes": 5120, } assert logs.daily_traffic_calls == [(scope, VM_ID, { "host_id": "hk", "user_id": 41, "since": 2_851_200, "until": 3_000_000, "limit": 2, })] denied = call( client, "GET", f"/internal/v1/hosts/hk/vms/{VM_ID}/traffic/daily", {**USER, "user_id": 42}, ) assert denied.status_code == 404 assert len(logs.daily_traffic_calls) == 1 def test_console_ticket_is_one_use_session_bound_and_no_url_secret(boundary): client, engine, store, *_ = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") component = engine.registry.components["hk"] component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41) issued = call(client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/console-ticket", USER, {}) assert issued.status_code == 200, issued.text assert issued.json()["websocket_path"] == "/engine-console" ticket = issued.json()["ticket"] key = hashlib.sha256(ticket.encode()).hexdigest() record = json.loads(store.get("console_ticket", key)) assert record["actor"]["user_id"] == 41 and record["actor"]["session_id"] == "test-session" assert call(client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/console-ticket", {**USER, "user_id": 42}, {}).status_code == 404 from starlette.websockets import WebSocketDisconnect with pytest.raises(WebSocketDisconnect): with client.websocket_connect("wss://panel.example.test/engine-console", headers={"Origin": "https://evil.example"}, subprotocols=["cloud-ticket." + ticket]): pass assert store.get("console_ticket", key) is not None def test_console_consumes_ticket_and_closes_after_php_session_revocation(boundary): client, engine, store, *_ = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") component = engine.registry.components["hk"] component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41) checks = 0 async def active(*args): nonlocal checks checks += 1 return checks <= 2 # Ticket issuance + handshake; then PHP revokes. engine.introspection = active engine.console_check_interval = 0.01 issued = call(client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/console-ticket", USER, {}) ticket = issued.json()["ticket"] protocol = "cloud-ticket." + ticket async def relay(socket, **kwargs): await socket.send_bytes(b"vnc") await asyncio.Event().wait() from starlette.websockets import WebSocketDisconnect with patch("virsh_panel.console.relay_console", relay): with client.websocket_connect("wss://panel.example.test/engine-console", headers={"Origin": "https://panel.example.test"}, subprotocols=[protocol]) as socket: assert socket.accepted_subprotocol == protocol assert socket.receive_bytes() == b"vnc" with pytest.raises(WebSocketDisconnect): socket.receive_bytes() key = hashlib.sha256(ticket.encode()).hexdigest() assert store.get("console_ticket", key) is None assert store.get("ticket_used", key) == "1" with pytest.raises(WebSocketDisconnect): with client.websocket_connect("wss://panel.example.test/engine-console", headers={"Origin": "https://panel.example.test"}, subprotocols=[protocol]): pass def test_streamed_iso_digest_must_match_before_import_is_queued(boundary): client, engine, store, _, services = boundary path = "/internal/v1/hosts/hk/rescue-templates?name=rescue" signed_body = b"A" * (1024 * 1024) wrong_body = b"B" * len(signed_body) signed = signed_headers(SECRET, "POST", path, signed_body, SUPER) signed["X-Cloud-Content-SHA256"] = hashlib.sha256(signed_body).hexdigest() signed["Content-Type"] = "application/octet-stream" response = client.post(path, content=wrong_body, headers=signed) assert response.status_code == 401, response.text assert not any(row[0] == "import_rescue_iso" for row in services["hk"].calls) assert engine.registry.components["hk"].store.list("rescue_import_job") == {} def cloud_upload_fixture(size=1024 * 1024 + 257): header = bytearray(104) header[:4] = b"QFI\xfb" header[4:8] = struct.pack(">I", 3) header[20:24] = struct.pack(">I", 16) header[24:32] = struct.pack(">Q", 20 * 1024**3) header[100:104] = struct.pack(">I", 104) return bytes(header) + b"\0" * (size - len(header)) def wait_for_cloud_upload_job(client, job_id): path = f"/internal/v1/vm-templates/import-jobs/{job_id}" for _ in range(200): response = call(client, "GET", path, SUPER) assert response.status_code == 200, response.text job = response.json() if job["status"] in {"completed", "failed"}: return job time.sleep(0.01) raise AssertionError("uploaded cloud template import did not finish") def cloud_engine_scope(method, target, body, actor, *, headers=None): path, separator, query = target.partition("?") request_headers = signed_headers(SECRET, method, target, body, actor) request_headers.update(headers or {}) return { "type": "http", "asgi": {"version": "3.0"}, "http_version": "1.1", "method": method, "scheme": "http", "path": path, "raw_path": path.encode("ascii"), "query_string": query.encode("ascii") if separator else b"", "headers": [ (name.lower().encode("ascii"), value.encode("latin-1")) for name, value in request_headers.items() ], "client": ("testclient", 50000), "server": ("127.0.0.1", 80), } async def cloud_engine_request(engine, method, target, body, actor, *, headers=None, receive=None): scope = cloud_engine_scope( method, target, body, actor, headers=headers) delivered = False async def receive_body(): nonlocal delivered if delivered: return {"type": "http.disconnect"} delivered = True return {"type": "http.request", "body": body, "more_body": False} messages = [] async def send(message): messages.append(message) await engine(scope, receive or receive_body, send) status = next(message["status"] for message in messages if message["type"] == "http.response.start") raw = b"".join(message.get("body", b"") for message in messages if message["type"] == "http.response.body") return status, json.loads(raw) def test_global_cloud_template_upload_streams_past_json_limit(boundary): client, engine, *_ = boundary query = urlencode({ "name": "Debian 12 fixed", "os_family": "Debian", "os_version": "12", }) path = "/internal/v1/vm-templates/upload?" + query payload = cloud_upload_fixture() with patch("virsh_panel.cloud_templates._verify_qcow2_with_qemu"): response = call(client, "POST", path, SUPER, raw=payload, headers={ "X-Cloud-Content-SHA256": hashlib.sha256(payload).hexdigest(), "Content-Type": "application/octet-stream", }) assert response.status_code == 202, response.text job = wait_for_cloud_upload_job(client, response.json()["job_id"]) assert job["status"] == "completed", job assert job["template"]["name"] == "Debian 12 fixed" assert job["template"]["os_family"] == "Debian" assert job["template"]["os_version"] == "12" assert job["template"]["virtual_size"] == 20 * 1024**3 with engine.registry.cloud_templates.source( job["template"]["id"]) as (_record, source): assert source.stat().st_size == len(payload) assert hashlib.sha256(source.read_bytes()).digest() == hashlib.sha256( payload).digest() def test_global_cloud_template_upload_digest_must_match_before_queue(boundary): client, engine, *_ = boundary path = ("/internal/v1/vm-templates/upload?" "name=Debian+12&os_family=Debian&os_version=12") signed_body = cloud_upload_fixture() wrong_body = signed_body[:-1] + b"x" signed = signed_headers(SECRET, "POST", path, signed_body, SUPER) signed["X-Cloud-Content-SHA256"] = hashlib.sha256( signed_body).hexdigest() signed["Content-Type"] = "application/octet-stream" response = client.post(path, content=wrong_body, headers=signed) assert response.status_code == 401, response.text assert engine.registry.cloud_templates.jobs == {} assert engine.registry.cloud_templates.pending == set() root = engine.registry.cloud_templates.files.directory() assert not list(root.glob(".cloud-image-upload-*.qcow2")) @pytest.mark.parametrize( ("family", "version"), [ ("debian", "12"), ("Debian", "14"), ("CentOS", "10"), ("Ubuntu", "24"), ], ) def test_global_cloud_template_upload_rejects_noncanonical_metadata( boundary, family, version): client, engine, *_ = boundary path = "/internal/v1/vm-templates/upload?" + urlencode({ "name": "Invalid metadata", "os_family": family, "os_version": version, }) payload = cloud_upload_fixture() response = call(client, "POST", path, SUPER, raw=payload, headers={ "X-Cloud-Content-SHA256": hashlib.sha256(payload).hexdigest(), "Content-Type": "application/octet-stream", }) assert response.status_code == 422, response.text assert engine.registry.cloud_templates.jobs == {} def test_global_cloud_template_upload_requires_super_admin_and_octet_stream( boundary): client, engine, *_ = boundary path = ("/internal/v1/vm-templates/upload?" "name=Debian+12&os_family=Debian&os_version=12") payload = cloud_upload_fixture() digest = hashlib.sha256(payload).hexdigest() forbidden = call(client, "POST", path, ADMIN, raw=payload, headers={ "X-Cloud-Content-SHA256": digest, "Content-Type": "application/octet-stream", }) assert forbidden.status_code == 403, forbidden.text wrong_type = call(client, "POST", path, SUPER, raw=payload, headers={ "X-Cloud-Content-SHA256": digest, "Content-Type": "multipart/form-data", }) assert wrong_type.status_code == 415, wrong_type.text assert engine.registry.cloud_templates.jobs == {} def test_global_cloud_template_upload_enforces_minimum_size(boundary): client, engine, *_ = boundary path = ("/internal/v1/vm-templates/upload?" "name=Too+Small&os_family=Debian&os_version=12") payload = cloud_upload_fixture(104) response = call(client, "POST", path, SUPER, raw=payload, headers={ "X-Cloud-Content-SHA256": hashlib.sha256(payload).hexdigest(), "Content-Type": "application/octet-stream", }) assert response.status_code == 422, response.text assert engine.registry.cloud_templates.jobs == {} assert engine.registry.cloud_templates.pending == set() def test_global_cloud_template_upload_timeout_releases_reservation( boundary, monkeypatch): _client, engine, *_ = boundary query = b"name=Timeout&os_family=Debian&os_version=12" scope = { "query_string": query, "headers": [ (b"content-type", b"application/octet-stream"), (b"content-length", str(1024 * 1024).encode()), ], "engine_upload_validation": {"complete": False, "valid": False}, "engine_upload_digest": "0" * 64, } async def stalled_receive(): await asyncio.sleep(1) return {"type": "http.request", "body": b"", "more_body": False} async def send(_message): raise AssertionError("timeout must not start a response") monkeypatch.setattr("virsh_panel.internal.CLOUD_UPLOAD_IDLE_SECONDS", 0.01) with pytest.raises(TemplateError, match="上传超时") as timeout: asyncio.run(engine._upload_cloud_template(scope, stalled_receive, send)) assert timeout.value.status == 408 assert engine.registry.cloud_templates.pending == set() assert engine.registry.cloud_templates.upload_reservations == {} root = engine.registry.cloud_templates.files.directory() assert not list(root.glob(".cloud-image-upload-*.qcow2")) def test_stalled_cloud_upload_does_not_block_other_engine_writes( boundary, monkeypatch): _client, engine, *_ = boundary payload = cloud_upload_fixture() upload_path = "/internal/v1/vm-templates/upload?" + urlencode({ "name": "Stalled upload", "os_family": "Debian", "os_version": "12", }) upload_started = asyncio.Event() release_upload = asyncio.Event() class FastRemoteHostManager: def __init__(self, *_args, **_kwargs): pass @staticmethod def scan(host, port): return {"host": host, "port": port, "reachable": True} @staticmethod def close(): pass monkeypatch.setattr( "virsh_panel.remote_host.RemoteHostManager", FastRemoteHostManager) async def stalled_receive(): upload_started.set() await release_upload.wait() return {"type": "http.disconnect"} async def scenario(): upload = asyncio.create_task(cloud_engine_request( engine, "POST", upload_path, payload, SUPER, headers={ "X-Cloud-Content-SHA256": hashlib.sha256(payload).hexdigest(), "Content-Type": "application/octet-stream", "Content-Length": str(len(payload)), }, receive=stalled_receive, )) await asyncio.wait_for(upload_started.wait(), timeout=1) assert engine.registry.cloud_templates.pending scan_body = json.dumps({ "host": "host.example.test", "port": 22, }, separators=(",", ":")).encode() status, result = await asyncio.wait_for(cloud_engine_request( engine, "POST", "/internal/v1/hosts/scan", scan_body, SUPER, headers={"Content-Type": "application/json"}, ), timeout=0.5) assert status == 200 assert result["host"] == "host.example.test" release_upload.set() upload_status, _result = await asyncio.wait_for(upload, timeout=1) assert upload_status == 400 asyncio.run(scenario()) assert engine.registry.cloud_templates.pending == set() assert engine.registry.cloud_templates.upload_reservations == {} def test_cloud_upload_fsync_does_not_block_event_loop_or_engine_writes( boundary, monkeypatch): _client, engine, *_ = boundary payload = cloud_upload_fixture() upload_path = "/internal/v1/vm-templates/upload?" + urlencode({ "name": "Slow fsync upload", "os_family": "Debian", "os_version": "12", }) fsync_started = threading.Event() release_fsync = threading.Event() original_fsync = os.fsync def slow_fsync(descriptor): fsync_started.set() assert release_fsync.wait(timeout=2), "test did not release fsync" return original_fsync(descriptor) class FastRemoteHostManager: def __init__(self, *_args, **_kwargs): pass @staticmethod def scan(host, port): return {"host": host, "port": port, "reachable": True} @staticmethod def close(): pass monkeypatch.setattr("virsh_panel.internal.os.fsync", slow_fsync) monkeypatch.setattr( "virsh_panel.remote_host.RemoteHostManager", FastRemoteHostManager) monkeypatch.setattr( "virsh_panel.cloud_templates._verify_qcow2_with_qemu", lambda *_args: None) async def scenario(): safety_release = threading.Timer(1.5, release_fsync.set) safety_release.start() started_at = time.monotonic() try: upload = asyncio.create_task(cloud_engine_request( engine, "POST", upload_path, payload, SUPER, headers={ "X-Cloud-Content-SHA256": hashlib.sha256(payload).hexdigest(), "Content-Type": "application/octet-stream", "Content-Length": str(len(payload)), }, )) while not fsync_started.is_set(): await asyncio.sleep(0.001) scan_body = json.dumps({ "host": "host.example.test", "port": 22, }, separators=(",", ":")).encode() status, _result = await asyncio.wait_for(cloud_engine_request( engine, "POST", "/internal/v1/hosts/scan", scan_body, SUPER, headers={"Content-Type": "application/json"}, ), timeout=0.5) assert status == 200 assert time.monotonic() - started_at < 0.5 assert not release_fsync.is_set() release_fsync.set() upload_status, _result = await asyncio.wait_for(upload, timeout=2) assert upload_status == 202 finally: release_fsync.set() safety_release.cancel() asyncio.run(scenario()) def test_cloud_upload_rechecks_update_state_before_queue( boundary, monkeypatch): _client, engine, *_ = boundary updater = FakeUpdater() engine.updater = updater payload = cloud_upload_fixture() upload_path = "/internal/v1/vm-templates/upload?" + urlencode({ "name": "Update race upload", "os_family": "Debian", "os_version": "12", }) upload_started = asyncio.Event() release_upload = asyncio.Event() delivered = False monkeypatch.setattr( "virsh_panel.cloud_templates._verify_qcow2_with_qemu", lambda *_args: None) async def delayed_receive(): nonlocal delivered if delivered: return {"type": "http.disconnect"} delivered = True upload_started.set() await release_upload.wait() return {"type": "http.request", "body": payload, "more_body": False} async def scenario(): upload = asyncio.create_task(cloud_engine_request( engine, "POST", upload_path, payload, SUPER, headers={ "X-Cloud-Content-SHA256": hashlib.sha256(payload).hexdigest(), "Content-Type": "application/octet-stream", "Content-Length": str(len(payload)), }, receive=delayed_receive, )) await asyncio.wait_for(upload_started.wait(), timeout=1) assert engine.registry.cloud_templates.pending updater.active = { "job_id": "update-job-1234567890", "status": "running", } release_upload.set() status, result = await asyncio.wait_for(upload, timeout=2) assert status == 409 assert result["error"] == "面板正在更新,暂时不能执行虚拟化写操作" try: asyncio.run(scenario()) finally: updater.active = None assert engine.registry.cloud_templates.pending == set() assert engine.registry.cloud_templates.upload_reservations == {} assert engine.registry.cloud_templates.jobs == {} def test_upload_same_os_version_keeps_existing_template(boundary): client, engine, *_ = boundary def upload(name, payload): path = "/internal/v1/vm-templates/upload?" + urlencode({ "name": name, "os_family": "Debian", "os_version": "12", }) response = call(client, "POST", path, SUPER, raw=payload, headers={ "X-Cloud-Content-SHA256": hashlib.sha256(payload).hexdigest(), "Content-Type": "application/octet-stream", }) assert response.status_code == 202, response.text job = wait_for_cloud_upload_job(client, response.json()["job_id"]) assert job["status"] == "completed", job return job["template"] official_payload = cloud_upload_fixture()[:-1] + b"o" fixed_payload = cloud_upload_fixture()[:-1] + b"f" with patch("virsh_panel.cloud_templates._verify_qcow2_with_qemu"): official = upload("Debian 12 official", official_payload) fixed = upload("Debian 12 fixed", fixed_payload) assert official["id"] != fixed["id"] templates = call( client, "GET", "/internal/v1/vm-templates", SUPER).json()["templates"] assert {row["id"] for row in templates} == {official["id"], fixed["id"]} with engine.registry.cloud_templates.source( official["id"]) as (_record, source): assert hashlib.sha256(source.read_bytes()).hexdigest() == official["sha256"] def test_main_and_log_schema_do_not_create_python_accounts_or_cross_database_tables(): connection = MagicMock() cursor = connection.cursor.return_value.__enter__.return_value config = {"host": "db", "port": 3306, "database": "main", "username": "engine", "password": "test", "ssl_ca": "/private/test/ca.pem"} with patch("virsh_panel.database.pymysql.connect", return_value=connection): main = MainStore(config, b"m" * 32) statements = [call.args[0] for call in cursor.execute.call_args_list] assert any("engine_records" in statement for statement in statements) assert not any("CREATE TABLE IF NOT EXISTS users" in statement for statement in statements) assert not any("vm_metric_samples" in statement for statement in statements) assert main._config["ssl_verify_identity"] is True cursor.reset_mock() LogStore({**config, "database": "logs"}, b"m" * 32) statements = [call.args[0] for call in cursor.execute.call_args_list] assert "engine_vm_metric_samples" in statements[0] assert "engine_vm_traffic_totals" in statements[1] assert "engine_vm_traffic_checkpoints" in statements[2] assert all("ALTER TABLE engine_vm_metric_samples" in statement for statement in statements[3:]) assert LogStore.TRAFFIC_TABLE == "engine_vm_traffic_totals" assert LogStore.TRAFFIC_CHECKPOINT_TABLE == "engine_vm_traffic_checkpoints"