"""Private engine tests: no PHP cookies, no real DB and no real hypervisor.""" from __future__ import annotations import base64 import asyncio import hashlib import json import os import struct import threading import time from concurrent.futures import ThreadPoolExecutor from contextlib import asynccontextmanager, contextmanager from types import SimpleNamespace from unittest.mock import MagicMock, patch from urllib.parse import urlencode import pytest from fastapi.testclient import TestClient from test_app import (CLOUD_CREATE_BODY, CREATE_BODY, OFFICIAL_CLOUD_URL, FakeRemoteManager, FakeVMService, HOST_BODY, VM_ID) from virsh_panel.app import create_app from virsh_panel.cloud_templates import (TemplateError, template_id_for, template_scope, template_volume_name) from virsh_panel.config import Settings from virsh_panel.coordination import mark_remote_write_started from virsh_panel.database import DatabaseError, DatabaseTemplateStore from virsh_panel.engine_context import (EngineComponents, ExternalOwnership, HostStore, LogStore, MainStore, TRUSTED_ACTOR) from virsh_panel.engine_security import (EngineAuthError, authenticate, encode_actor, signed_headers) from virsh_panel.fabric import FabricError from virsh_panel.internal import EngineApplication, HostRegistry from virsh_panel.libvirt_service import ( FIREWALL_FILTER_COMPILER_VERSION, VMError, build_firewall_filter_xml, firewall_policy_digest, snapshot_volume_name, ) from virsh_panel.metrics import host_scope from virsh_panel.shutdown import ManagedThreadPoolExecutor from virsh_panel.user_isos import user_iso_scope from virsh_panel.vpc import parse_vpc_delete_plan SECRET = b"s" * 32 USER = {"user_id": 41, "is_admin": False, "is_super_admin": False, "can_manage": True, "session_id": "test-session"} SUPER = {**USER, "user_id": 1, "is_admin": True, "is_super_admin": True} ADMIN = {**SUPER, "is_super_admin": False} class MemoryStore: def __init__(self): self.records = {} self.nonces = set() self.lock = threading.RLock() self.daily_traffic_calls = [] def get(self, kind, key): with self.lock: return self.records.get((kind, key)) def set(self, kind, key, value): with self.lock: self.records[kind, key] = value def insert(self, kind, key, value): with self.lock: if (kind, key) in self.records: return False self.records[kind, key] = value return True def compare_and_set(self, kind, key, expected, value): with self.lock: if self.records.get((kind, key)) != expected: return False self.records[kind, key] = value return True def delete(self, kind, key): with self.lock: self.records.pop((kind, key), None) def list(self, kind): with self.lock: return {key: value for (row_kind, key), value in self.records.items() if row_kind == kind} def consume_nonce(self, nonce, expires_at, now): with self.lock: if nonce in self.nonces: return False self.nonces.add(nonce) return True def query_metric_samples(self, *args, **kwargs): return [{"timestamp": 123, "cpu_percent": 20.0}] def query_daily_traffic(self, scope, vm_id, **kwargs): self.daily_traffic_calls.append((scope, vm_id, kwargs)) day_start = kwargs["until"] - kwargs["until"] % 86400 return [{"day_start": day_start, "download_bytes": 4096, "upload_bytes": 1024, "total_bytes": 5120}] def get_value(self, key): return self.get("value", key) def set_value(self, key, value): return self.set("value", key, value) def delete_value(self, key): return self.delete("value", key) def get_secret(self, key): return self.get("secret", key) def set_secret(self, key, value): return self.set("secret", key, value) def delete_secret(self, key): return self.delete("secret", key) class FakeUpdater: def __init__(self): self.token = None self.active = None self.jobs = {} def settings(self): return { "repository": "https://g.luoli.live/root/virt.git", "branch": "venix", "token_configured": self.token is not None, "checkout_ready": True, "current_commit": "a" * 40, "last_job_id": None, "active_job_id": self.active["job_id"] if self.active else None, } def set_token(self, token): self.token = token def clear_token(self): self.token = None def active_job(self): return self.active def check(self): return {"repository": "https://g.luoli.live/root/virt.git", "branch": "venix", "current_commit": "a" * 40, "remote_commit": "b" * 40, "status": "update_available"} def preflight(self, *, expected_current_commit=None, expected_remote_commit=None): if self.token is None: from virsh_panel.updater import UpdateError raise UpdateError("请先在面板中设置 Gitea 访问令牌", 409) if expected_current_commit != "a" * 40 or expected_remote_commit != "b" * 40: from virsh_panel.updater import UpdateError raise UpdateError("面板本地版本已变化,请重新检查更新后再执行", 409) def create_job(self, *, expected_current_commit=None, expected_remote_commit=None, job_id=None): from virsh_panel.updater import UpdateError, UpdateNotQueued try: self.preflight(expected_current_commit=expected_current_commit, expected_remote_commit=expected_remote_commit) except UpdateError as exc: raise UpdateNotQueued(str(exc), exc.status) from exc job = {"job_id": job_id or "update-job-1234567890", "status": "queued", "phase": "等待独立更新服务启动", "created_at": 1, "private": self.token} self.jobs[job["job_id"]] = job self.active = job return job.copy() def get_job(self, job_id): return self.jobs.get(job_id) def fail_job(self, job_id, message): self.jobs[job_id].update(status="failed", error=message) self.active = None return self.jobs[job_id].copy() def test_engine_application_wires_one_catalog_mutation_lock(tmp_path): engine = EngineApplication( { "service_secret": base64.b64encode(SECRET).decode(), "data_dir": str(tmp_path), }, MemoryStore(), MemoryStore(), registry=SimpleNamespace(), updater=FakeUpdater(), ) try: application_lock = engine._network_catalog_mutation_lock() assert isinstance(application_lock, asyncio.Lock) assert engine.fabric.catalog_mutation_lock() is application_lock assert engine.vpc._catalog_mutation_lock() is application_lock finally: engine.fabric.close() class Service(FakeVMService): def __init__(self): super().__init__() self.create_failure = False self.absent = False self.absent_checks = [] self.cloud_template_delete_error = None self.vm.update(managed=True, os_source="iso") def create_vm(self, spec): self.calls.append(("create_vm", spec)) # This adapter skips the RPC transport; model its write boundary so # post-dispatch failures remain distinct from no-write preflights. mark_remote_write_started() if self.create_failure: from virsh_panel.libvirt_service import VMError raise VMError("宿主机失联,回滚状态不明", 502) self.vm.update(uuid=spec["vm_uuid"], name=spec["name"]) return self.vm.copy() def delete_vm(self, vm_id, delete_disks): mark_remote_write_started() return super().delete_vm(vm_id, delete_disks) def provisioning_absent(self, vm_uuid, name, *, uuid_resources=False, legacy_name_collision=False): self.absent_checks.append( (vm_uuid, name, uuid_resources, legacy_name_collision)) return self.absent def change_cloud_vm_network(self, vm_id, network, *, confirm_restart=False): self.calls.append(( "change_cloud_vm_network", vm_id, network["public_ip_cidr"], confirm_restart, )) if vm_id != self.vm["uuid"]: raise VMError("虚拟机不存在", 404) self.vm["public_ip_cidr"] = network["public_ip_cidr"] self.vm["public_pool_key"] = network.get("public_pool_key") return { "vm": self.vm.copy(), "public_pool_key": network.get("public_pool_key"), "public_ip_cidr": network["public_ip_cidr"], "network_changed": True, } def delete_cloud_template_replicas(self, source_volume_name): self.calls.append(("delete_cloud_template_replicas", source_volume_name)) if self.cloud_template_delete_error is not None: raise self.cloud_template_delete_error return {"source_volume_name": source_volume_name, "deleted": True} @pytest.fixture def boundary(tmp_path): store = MemoryStore() logs = MemoryStore() config = {"service_secret": base64.b64encode(SECRET).decode(), "data_dir": str(tmp_path)} services = {} def factory(host_id, user_isos, cloud_templates): (tmp_path / host_id).mkdir(mode=0o700, exist_ok=True) scoped = HostStore(store, host_id) manager = FakeRemoteManager() manager.host_id = host_id manager.profile = {"host_id": host_id, "host": "host.example.test", "port": 22, "fingerprint": "SHA256:example", "key_locked": False, "storage_pool": "images", "iso_pool": "iso", "rescue_iso_pool": "rescue", "default_network": "default", "libvirt_uri": "fake", "ssh_target": "fake", "ssh_config_file": "/fake/config", "ssh_known_hosts_file": "/fake/known_hosts"} service = Service() services[host_id] = service components = EngineComponents(SimpleNamespace(users=ExternalOwnership(scoped)), manager, DatabaseTemplateStore(scoped), scoped, logs, user_isos, cloud_templates) app = create_app(Settings(data_dir=tmp_path / host_id), engine_components=components, service_factory=lambda profile: service) return components, app registry = HostRegistry(config, store, logs, factory=factory) async def active(*args): return True engine = EngineApplication(config, store, logs, registry=registry, introspection=active) engine._service = lambda profile: services[profile["host_id"]] with TestClient(engine, base_url="http://127.0.0.1") as client: yield client, engine, store, logs, services def call(client, method, path, actor=USER, value=None, raw=None, headers=None): body = raw if raw is not None else b"" if value is None else json.dumps(value, separators=(",", ":")).encode() signed = signed_headers(SECRET, method, path, body, actor) if value is not None: signed["Content-Type"] = "application/json" signed.update(headers or {}) return client.request(method, path, content=body, headers=signed) FIREWALL_RULES = [ { "protocol": "icmp", "source_cidr": "0.0.0.0/0", "port_from": None, "port_to": None, "description": "Ping", }, { "protocol": "tcp", "source_cidr": "203.0.113.55/32", "port_from": 22, "port_to": 22, "description": "SSH", }, ] def firewall_body(*, revision=1, enabled=True, rules=None): rules = FIREWALL_RULES if rules is None else rules return { "enabled": enabled, "revision": revision, "policy_digest": firewall_policy_digest(enabled, rules), "rules": rules, } def test_legacy_firewall_compiler_digest_fixture_is_stable(): body = firewall_body() assert build_firewall_filter_xml( VM_ID, body["enabled"], body["rules"], compiler_version=1, )[3] == "42e1cd7c3e410dc6a0d80740753b19b6b4a67ca00c4b8265b2f0bb201f9af793" def test_host_capacity_endpoint_is_removed(boundary): client, _engine, _store, _logs, _services = boundary assert call(client, "GET", "/internal/v1/hosts/hk/host/capacity", USER).status_code == 404 assert call(client, "GET", "/internal/v1/hosts/hk/host/capacity", SUPER).status_code == 404 def import_user_iso_for_test(client, tmp_path, name, *, digest="e" * 64): download = tmp_path / (hashlib.sha256(name.encode()).hexdigest() + ".iso") download.write_bytes(b"i" * (1024 * 1024)) with patch("virsh_panel.user_isos.download_user_iso", return_value=(download, download.stat().st_size, digest)): queued = call(client, "POST", "/internal/v1/user-isos/import", USER, { "url": "https://downloads.example.test/os.iso", "name": name, "sha256": "", }) assert queued.status_code == 202, queued.text job_id = queued.json()["job_id"] for _ in range(100): job = call( client, "GET", f"/internal/v1/user-isos/import-jobs/{job_id}", ).json() if job["status"] in {"completed", "failed"}: break time.sleep(0.01) assert job["status"] == "completed", job return job["iso"]["id"] def test_host_network_options_endpoint_is_removed(boundary): client, _engine, _store, _logs, _services = boundary assert call(client, "GET", "/internal/v1/hosts/hk/host/network-options", USER).status_code == 404 assert call(client, "GET", "/internal/v1/hosts/hk/host/network-options", SUPER).status_code == 404 def test_fabric_is_engine_global_and_super_admin_write_only(boundary): client, engine, store, _logs, _services = boundary assert call(client, "GET", "/internal/v1/fabric", USER).status_code == 403 assert call(client, "GET", "/internal/v1/fabric", ADMIN).json() == [] assert call(client, "POST", "/internal/v1/fabric/apply", USER, {}).status_code == 403 assert call(client, "POST", "/internal/v1/fabric/apply", ADMIN, {}).status_code == 403 invalid = call(client, "POST", "/internal/v1/fabric/apply", SUPER, {}) assert invalid.status_code == 400 assert invalid.json()["error_code"] == "fabric_invalid" assert "failure_phase" not in invalid.json() async def failed_identity(_value): raise FabricError( "宿主机 Fabric 命令失败", 502, "fabric_remote_command_failed", "identity", ) with patch.object(engine.fabric, "enqueue_apply", failed_identity): identity_failed = call( client, "POST", "/internal/v1/fabric/apply", SUPER, {}, ) assert identity_failed.status_code == 502 assert identity_failed.json() == { "error": "宿主机 Fabric 命令失败", "error_code": "fabric_remote_command_failed", "failure_phase": "identity", } receipt = { "fabric_id": "fabric-hk", "operation": { "id": "a" * 32, "kind": "apply", "status": "queued", "started_at": None, "updated_at": 1, "finished_at": None, }, } async def accepted_apply(_value): return receipt with patch.object(engine.fabric, "enqueue_apply", accepted_apply): accepted = call( client, "POST", "/internal/v1/fabric/apply", SUPER, {}, ) assert accepted.status_code == 202 assert accepted.json() == receipt reconcile_receipt = { **receipt, "operation": {**receipt["operation"], "kind": "reconcile"}, } async def accepted_reconcile(fabric_id): assert fabric_id == "fabric-hk" return reconcile_receipt with patch.object( engine.fabric, "enqueue_reconcile", accepted_reconcile): accepted = call( client, "POST", "/internal/v1/fabric/fabric-hk/reconcile", SUPER, {}, ) assert accepted.status_code == 202 assert accepted.json() == reconcile_receipt assert call( client, "POST", "/internal/v1/fabric/probe", USER, {"host_ids": ["host-a"]}, ).status_code == 403 assert call( client, "POST", "/internal/v1/fabric/probe", ADMIN, {"host_ids": ["host-a"]}, ).status_code == 403 assert call( client, "POST", "/internal/v1/fabric/probe", {**SUPER, "can_manage": False}, {"host_ids": ["host-a"]}, ).status_code == 403 malformed = call( client, "POST", "/internal/v1/fabric/probe", SUPER, raw=b'{"host_ids":', ) assert malformed.status_code == 422 assert malformed.json()["error_code"] == "fabric_probe_invalid" extra = call( client, "POST", "/internal/v1/fabric/probe", SUPER, {"host_ids": ["host-a"], "unexpected": True}, ) assert extra.status_code == 422 assert extra.json()["error_code"] == "fabric_probe_invalid" store.set( "engine_host", "host-a", json.dumps({"host_id": "host-a"}), ) unsupported = call( client, "POST", "/internal/v1/fabric/probe", SUPER, {"host_ids": ["host-a"]}, ) assert unsupported.status_code == 200 assert unsupported.json()["hosts"][0]["error_code"] == ( "fabric_host_capability_probe_unsupported") manager = engine.registry.components["host-a"].manager manager.profile.update({ "host_id": "host-a", "host": "203.0.113.80", "region": "hk", }) capabilities = { "vpc_ready": True, "reasons": [], "os": {"id": "debian", "version_id": "13"}, "network": { "network_manager": "ifupdown", "network_manager_expected": "ifupdown", "network_manager_supported": True, "firewall_manager": "nftables", "firewall_mode": "native_nftables", "ip_forward": 1, "rp_filter": 2, "rp_filter_default": 2, "rp_filter_default_effective": 2, "rp_filter_interfaces": {"br0": 2}, "local_ipv4_addresses": ["203.0.113.80"], "local_ipv4_interfaces": {"203.0.113.80": ["br0"]}, }, "wireguard": { "deployment_mode": "ifupdown", "ready": True, "owned_by_panel": False, }, "nftables": { "ready": True, "service_active": True, "service_persistent": True, }, "network_roles": { "public": {"selector": "default", "verified": True}, }, } manager.reprobe_current_snapshot = lambda: (manager.profile, capabilities) probed = call( client, "POST", "/internal/v1/fabric/probe", SUPER, {"host_ids": ["host-a"]}, ) assert probed.status_code == 200 readiness = probed.json()["hosts"][0] assert readiness["configured"] is True assert readiness["reachable"] is True assert readiness["vpc_ready"] is True assert readiness["endpoint_ip"] == "203.0.113.80" assert readiness["capabilities"]["network_manager"] == "ifupdown" assert call( client, "GET", "/internal/v1/hosts/hk/fabric", SUPER ).status_code == 404 def test_vpc_control_plane_is_global_and_does_not_use_host_network_change(boundary): client, engine, store, _logs, _services = boundary assert engine.vpc_reconcile_task is not None assert not engine.vpc_reconcile_task.done() assert call(client, "GET", "/internal/v1/vpcs", USER).status_code == 403 assert call(client, "GET", "/internal/v1/vpcs", ADMIN).json() == [] assert call(client, "POST", "/internal/v1/vpcs/apply", USER, {}).status_code == 400 assert call(client, "POST", "/internal/v1/vpcs/apply", ADMIN, {}).status_code == 400 assert call(client, "POST", "/internal/v1/vpcs/apply", {**USER, "can_manage": False}, {}).status_code == 400 invalid = call(client, "POST", "/internal/v1/vpcs/apply", SUPER, {}) assert invalid.status_code == 400 assert invalid.json()["error_code"] == "vpc_operation_required" valid = { "schema_version": 1, "vpc_id": "vpc-boundary-1", "region": "hk", "fabric_id": "fabric-boundary", "fabric_revision": 1, "cidr": "10.30.0.0/24", "desired_revision": 1, "operation_id": "vpc-boundary-op1", "subnets": [{"subnet_id": "subnet-a", "cidr": "10.30.0.0/25"}], "hosts": [{"host_id": "host-a", "members": ["member-a"]}], "members": [{ "member_id": "member-a", "instance_uuid": "11111111-1111-4111-8111-111111111111", "host_id": "host-a", "private_ip_cidr": "10.30.0.10/32", "subnet_id": "subnet-a", }], "routes": [], "acls": [], } applied = call(client, "POST", "/internal/v1/vpcs/apply", {**USER, "can_manage": False}, valid) assert applied.status_code == 200, applied.text assert applied.json()["vpc_id"] == "vpc-boundary-1" assert applied.json()["next_stage_allowed"] is False assert "fabric_gate" in applied.json()["observed"] # The independent PHP VPC worker has a tenant actor and no browser # session. It must recover the stored receipt before replaying an apply. worker = {**USER, "session_id": ""} recovered = call(client, "GET", "/internal/v1/vpcs/vpc-boundary-1", worker) assert recovered.status_code == 200, recovered.text for field in ("vpc_id", "operation_id", "desired_revision", "desired_digest", "observed"): assert recovered.json()[field] == applied.json()[field] assert call(client, "GET", "/internal/v1/vpcs/missing-vpc", worker).status_code == 404 # A single signed PHP-authorized target does not grant inventory or # infrastructure administration access to the tenant worker. for path in ("/internal/v1/vpcs", "/internal/v1/fabric", "/internal/v1/hosts/hk/host", "/internal/v1/vm-templates/template-one/startup-script"): assert call(client, "GET", path, worker).status_code == 403 assert client.get("/internal/v1/vpcs/vpc-boundary-1").status_code == 401 assert call(client, "GET", "/internal/v1/vpcs", ADMIN).json()[0][ "vpc_id"] == "vpc-boundary-1" assert call(client, "GET", "/internal/v1/vpc", SUPER).status_code == 404 assert call(client, "GET", "/internal/v1/hosts/hk/vpcs", SUPER).status_code == 404 assert not store.list("operation") def test_vpc_root_delete_endpoint_returns_exact_durable_receipt(boundary): client, _engine, store, _logs, _services = boundary target = { "schema_version": 1, "vpc_id": "vpc-empty-delete", "region": "hk", "fabric_id": "fabric-boundary", "fabric_revision": 1, "cidr": "10.31.0.0/24", "desired_revision": 3, "operation_id": "vpc-empty-delete-op-0001", "subnets": [], "hosts": [], "members": [], "routes": [], "acls": [], "retiring_members": [], } target["desired_digest"] = parse_vpc_delete_plan(target)[0].desired_digest path = "/internal/v1/vpcs/vpc-empty-delete/delete" deleted = call( client, "POST", path, {**USER, "can_manage": False}, target) assert deleted.status_code == 200, deleted.text receipt = deleted.json() assert receipt == { "vpc_id": "vpc-empty-delete", "operation_id": "vpc-empty-delete-op-0001", "evidence_version": 2, "desired_revision": 3, "desired_digest": target["desired_digest"], "state": "deleted", "deleted": True, "readback": "confirmed", "removed_host_ids": [], "completed_at": receipt["completed_at"], } assert receipt["completed_at"] > 0 assert call(client, "POST", path, SUPER, target).json() == receipt assert store.get("vpc_deleted", "vpc-empty-delete") is not None assert store.get("vpc_desired", "vpc-empty-delete") is None missing_manifest = dict(target) missing_manifest.pop("retiring_members") invalid = call( client, "POST", "/internal/v1/vpcs/missing-manifest/delete", SUPER, {**missing_manifest, "vpc_id": "missing-manifest"}) assert invalid.status_code == 422 assert invalid.json()["error_code"] == "vpc_delete_manifest_invalid" def test_host_public_network_pools_are_revisioned_idempotent_and_aggregated( boundary): client, _engine, _store, _logs, services = boundary # Materialize the host child app before configuring its host-scoped # encrypted pool catalog. assert call( client, "GET", "/internal/v1/hosts/hk/host/network-pools", SUPER, ).json() == { "items": [], "count": 0, "maximum": 32, "compatibility_mode": False, } first_key = "a" * 32 second_key = "b" * 32 first_path = f"/internal/v1/hosts/hk/host/network-pools/{first_key}" first = { "revision": 1, "state": "active", "network_selector": "default", "cidr": "8.8.8.0/29", "gateway": "8.8.8.1", "dns_servers": ["1.1.1.1"], "reserved_ips": ["8.8.8.2"], "guest_prefix_length": 29, } assert call(client, "PUT", first_path, USER, first).status_code == 403 created = call(client, "PUT", first_path, SUPER, first) assert created.status_code == 200, created.text assert created.json()["applied"] is True assert created.json()["pool"]["pool_key"] == first_key repeated = call(client, "PUT", first_path, SUPER, first) assert repeated.status_code == 200 assert repeated.json()["idempotent"] is True conflict = call(client, "PUT", first_path, SUPER, { **first, "gateway": "8.8.8.2", }) assert conflict.status_code == 409 second_path = f"/internal/v1/hosts/hk/host/network-pools/{second_key}" second = { **first, "cidr": "9.9.9.0/29", "gateway": "9.9.9.1", "reserved_ips": ["9.9.9.2"], } assert call(client, "PUT", second_path, SUPER, second).status_code == 200 advanced = call(client, "PUT", first_path, SUPER, { **first, "revision": 2, "state": "draining", }) assert advanced.status_code == 200 assert advanced.json()["pool"]["state"] == "draining" assert call(client, "PUT", first_path, SUPER, first).status_code == 409 overlapping = call(client, "PUT", f"/internal/v1/hosts/hk/host/network-pools/{'c' * 32}", SUPER, {**first, "cidr": "8.8.8.0/30"}) assert overlapping.status_code == 400 catalog = call( client, "GET", "/internal/v1/hosts/hk/host/network-pools", SUPER, ).json() assert catalog["count"] == 2 assert [item["pool_key"] for item in catalog["items"]] == [ first_key, second_key, ] assert catalog["compatibility_mode"] is False assert call( client, "GET", "/internal/v1/hosts/hk/host/network-options", SUPER, ).status_code == 404 assert call( client, "GET", "/internal/v1/hosts/hk/host/capacity", SUPER, ).status_code == 404 disabled = call(client, "PUT", second_path, SUPER, { **second, "revision": 2, "state": "disabled", }) assert disabled.status_code == 200 assert call( client, "GET", "/internal/v1/hosts/hk/host/network-options", SUPER, ).status_code == 404 assert call( client, "GET", "/internal/v1/hosts/hk/host/capacity", SUPER, ).status_code == 404 def test_first_native_pool_cannot_orphan_occupied_legacy_network(boundary): client, engine, _store, _logs, services = boundary assert call(client, "GET", "/internal/v1/hosts/hk/vms").status_code == 200 components = engine.registry.components["hk"] profile = components.manager.profile profile.update({ "host": "8.8.4.4", "default_network": "bridge:br0", "public_pool_cidr": "8.8.8.0/29", "public_gateway": "8.8.8.1", "public_dns_servers": ["1.1.1.1"], "public_reserved_ips": ["8.8.8.2"], "public_vm_prefix": 29, }) scope = host_scope(profile) components.store.set("ip_claim", f"{scope}.8.8.8.3", json.dumps({ "vm_uuid": "00000000-0000-4000-8000-000000000003", })) pool_key = "d" * 32 path = f"/internal/v1/hosts/hk/host/network-pools/{pool_key}" base = { "revision": 1, "state": "active", "network_selector": "bridge:br0", "dns_servers": ["1.1.1.1"], "reserved_ips": ["9.9.9.2"], "guest_prefix_length": 29, } rejected = call(client, "PUT", path, SUPER, { **base, "cidr": "9.9.9.0/29", "gateway": "9.9.9.1", }) assert rejected.status_code == 409 assert call( client, "GET", "/internal/v1/hosts/hk/host/network-pools", SUPER, ).json()["compatibility_mode"] is True adopted = call(client, "PUT", path, SUPER, { **base, "cidr": "8.8.8.0/29", "gateway": "8.8.8.1", "reserved_ips": ["8.8.8.2"], }) assert adopted.status_code == 200, adopted.text assert adopted.json()["compatibility_mode"] is False def import_cloud_template_for_test(client, tmp_path, name="Delete Cloud Image"): payload = name.encode("utf-8") digest = hashlib.sha256(payload).hexdigest() def fake_download(_url, _sha256, directory, **_kwargs): source = directory / ".cloud-image-delete-boundary.qcow2" source.write_bytes(payload) source.chmod(0o600) return source, 2 * 1024**3, digest with patch( "virsh_panel.cloud_template_runtime.download_verified_image", side_effect=fake_download): queued = call(client, "POST", "/internal/v1/vm-templates/import", SUPER, {"url": OFFICIAL_CLOUD_URL, "name": name, "sha256": ""}) assert queued.status_code == 202, queued.text job_id = queued.json()["job_id"] for _ in range(100): job = call( client, "GET", f"/internal/v1/vm-templates/import-jobs/{job_id}", SUPER, ).json() if job["status"] in {"completed", "failed"}: break time.sleep(0.01) assert job["status"] == "completed", job return job["template"], payload def test_paid_scheduler_can_claim_exact_host_pool_ip(boundary, tmp_path): client, engine, _store, _logs, services = boundary template, _payload = import_cloud_template_for_test( client, tmp_path, "Scheduled Address Image", ) assert call( client, "GET", "/internal/v1/hosts/hk/resources", SUPER, ).status_code == 200 components = engine.registry.components["hk"] profile = components.manager.profile profile.update({ "host": "8.8.8.3", "default_network": "bridge:br0", "public_pool_cidr": "8.8.8.0/28", "public_gateway": "8.8.8.1", "public_dns_servers": ["1.1.1.1"], "public_reserved_ips": ["8.8.8.2"], "public_vm_prefix": 28, }) service = services["hk"] scope = host_scope(profile) components.store.set("ip_claim", f"{scope}.8.8.8.4", json.dumps({ "vm_uuid": "00000000-0000-4000-8000-000000000004", })) occupied_claim = f"{scope}.8.8.8.5" components.store.set("ip_claim", occupied_claim, json.dumps({ "vm_uuid": "00000000-0000-4000-8000-000000000005", })) base = { key: value for key, value in CLOUD_CREATE_BODY.items() if key not in {"gateway", "dns_servers"} } base["template_id"] = template["id"] path = "/internal/v1/hosts/hk/vms" unauthorized = call(client, "POST", path, { **USER, "operation_id": "scheduled-ip-unpaid-0001", }, {**base, "name": "unpaid-exact-ip", "public_ip_cidr": "8.8.8.8/28"}) assert unauthorized.status_code == 403 assert components.store.get("ip_claim", f"{scope}.8.8.8.8") is None service.absent = True outside = call(client, "POST", path, { **USER, "can_provision": True, "operation_id": "scheduled-ip-outside-0001", }, {**base, "name": "outside-exact-ip", "public_ip_cidr": "8.8.8.20/28"}) assert outside.status_code == 422 assert components.store.get("ip_claim", f"{scope}.8.8.8.20") is None occupied_vm = call(client, "POST", path, { **USER, "can_provision": True, "operation_id": "scheduled-ip-vm-used-0001", }, {**base, "name": "vm-used-exact-ip", "public_ip_cidr": "8.8.8.4/28"}) assert occupied_vm.status_code == 409 vm_claim = json.loads(components.store.get( "ip_claim", f"{scope}.8.8.8.4", )) assert vm_claim == { "vm_uuid": "00000000-0000-4000-8000-000000000004", } occupied_task = call(client, "POST", path, { **USER, "can_provision": True, "operation_id": "scheduled-ip-claim-used-0001", }, {**base, "name": "claim-used-exact-ip", "public_ip_cidr": "8.8.8.5/28"}) assert occupied_task.status_code == 409 assert components.store.get("ip_claim", occupied_claim) is not None created = call(client, "POST", path, { **USER, "can_provision": True, "operation_id": "scheduled-ip-success-0001", }, {**base, "name": "scheduled-exact-ip", "public_ip_cidr": "8.8.8.6/28"}) assert created.status_code == 201, created.text created_id = created.json()["uuid"] created_call = [ row for row in service.calls if row[0] == "create_cloud_vm" ][-1] assert created_call[1]["public_ip_cidr"] == "8.8.8.6/28" assert created_call[1]["public_pool_key"] == "legacy-default" assert created_call[1]["gateway"] == "8.8.8.1" assert created_call[1]["dns_servers"] == ["1.1.1.1"] assert not any(row[0] == "list_vms" for row in service.calls) created_claim = json.loads(components.store.get( "ip_claim", f"{scope}.8.8.8.6", )) assert created_claim["vm_uuid"] == created_id assert created_claim["pool_key"] == "legacy-default" def rolled_back_cloud_create(_spec, _volume_name): raise VMError("模拟创建失败,宿主机已完成回滚", 502) service.create_cloud_vm = rolled_back_cloud_create failed = call(client, "POST", path, { **USER, "can_provision": True, "operation_id": "scheduled-ip-rollback-0001", }, {**base, "name": "rolled-back-exact-ip", "public_ip_cidr": "8.8.8.7/28"}) assert failed.status_code == 502 assert failed.json()["status"] == "failed" assert failed.json()["safe_to_refund"] is True assert components.store.get("ip_claim", f"{scope}.8.8.8.7") is None def test_exact_public_ip_claim_is_atomic_without_inventory_scan( boundary, tmp_path): client, engine, _store, _logs, services = boundary template, _payload = import_cloud_template_for_test( client, tmp_path, "Concurrent Scheduled Address Image", ) assert call( client, "GET", "/internal/v1/hosts/hk/resources", SUPER, ).status_code == 200 components = engine.registry.components["hk"] service = services["hk"] # The exact-claim path must use only the panel-owned claim ledger. Any # fallback to a domain inventory scan turns this guard into a test failure. def forbidden_inventory_scan(): raise AssertionError("exact IP claim must not enumerate libvirt VMs") service.list_vms = forbidden_inventory_scan service.calls.clear() profile = components.manager.profile profile.update({ "host": "8.8.8.3", "default_network": "bridge:br0", "public_pool_cidr": "8.8.8.0/28", "public_gateway": "8.8.8.1", "public_dns_servers": ["1.1.1.1"], "public_reserved_ips": ["8.8.8.2"], "public_vm_prefix": 28, }) base = { key: value for key, value in CLOUD_CREATE_BODY.items() if key not in {"gateway", "dns_servers"} } base["template_id"] = template["id"] path = "/internal/v1/hosts/hk/vms" barrier = threading.Barrier(2) def create(number): barrier.wait(timeout=2) return call(client, "POST", path, { **USER, "can_provision": True, "operation_id": f"scheduled-ip-concurrent-{number:04d}", }, { **base, "name": f"scheduled-concurrent-{number}", "public_ip_cidr": "8.8.8.6/28", }) with ThreadPoolExecutor(max_workers=2) as executor: responses = [ future.result(timeout=5) for future in ( executor.submit(create, 1), executor.submit(create, 2), ) ] assert sorted(response.status_code for response in responses) == [201, 409] successful = next(response for response in responses if response.status_code == 201) scope = host_scope(profile) claim = json.loads(components.store.get( "ip_claim", f"{scope}.8.8.8.6", )) assert claim["vm_uuid"] == successful.json()["uuid"] assert not any(row[0] == "list_vms" for row in service.calls) def test_paid_scheduler_claims_the_requested_native_public_pool( boundary, tmp_path): client, engine, _store, _logs, services = boundary template, _payload = import_cloud_template_for_test( client, tmp_path, "Native Pool Address Image", ) assert call(client, "GET", "/internal/v1/hosts/hk/resources", SUPER).status_code == 200 components = engine.registry.components["hk"] profile = components.manager.profile profile["default_network"] = "bridge:br0" first_key = "1" * 32 second_key = "2" * 32 common = { "revision": 1, "state": "active", "network_selector": "bridge:br0", "dns_servers": ["1.1.1.1"], "reserved_ips": [], "guest_prefix_length": 29, } assert call( client, "PUT", f"/internal/v1/hosts/hk/host/network-pools/{first_key}", SUPER, {**common, "cidr": "8.8.8.0/29", "gateway": "8.8.8.1"}, ).status_code == 200 assert call( client, "PUT", f"/internal/v1/hosts/hk/host/network-pools/{second_key}", SUPER, {**common, "cidr": "9.9.9.0/29", "gateway": "9.9.9.1"}, ).status_code == 200 base = { key: value for key, value in CLOUD_CREATE_BODY.items() if key not in {"gateway", "dns_servers"} } base["template_id"] = template["id"] path = "/internal/v1/hosts/hk/vms" provisioning = {**USER, "can_provision": True} wrong = call(client, "POST", path, { **provisioning, "operation_id": "native-pool-mismatch-0001", }, { **base, "name": "wrong-native-pool", "public_pool_key": first_key, "public_ip_cidr": "9.9.9.3/29", }) assert wrong.status_code == 422 created = call(client, "POST", path, { **provisioning, "operation_id": "native-pool-success-0001", }, { **base, "name": "right-native-pool", "public_pool_key": second_key, "public_ip_cidr": "9.9.9.3/29", }) assert created.status_code == 201, created.text create_call = [ row for row in services["hk"].calls if row[0] == "create_cloud_vm" ][-1] assert create_call[1]["public_pool_key"] == second_key assert create_call[1]["gateway"] == "9.9.9.1" scope = host_scope(profile) claim = json.loads(components.store.get("ip_claim", f"{scope}.9.9.9.3")) assert claim["pool_key"] == second_key def test_global_cloud_template_is_visible_and_cached_once_per_host( boundary, tmp_path): client, engine, store, _logs, services = boundary payload = b"verified-global-cloud-template" digest = hashlib.sha256(payload).hexdigest() def fake_download(_url, _sha256, directory, **_kwargs): source = directory / ".cloud-image-boundary.qcow2" source.write_bytes(payload) source.chmod(0o600) return source, 2 * 1024**3, digest denied = call(client, "POST", "/internal/v1/vm-templates/import", USER, {"url": OFFICIAL_CLOUD_URL, "name": "Not Allowed", "sha256": ""}) assert denied.status_code == 403 with patch( "virsh_panel.cloud_template_runtime.download_verified_image", side_effect=fake_download): queued = call(client, "POST", "/internal/v1/vm-templates/import", SUPER, {"url": OFFICIAL_CLOUD_URL, "name": "AlmaLinux Global", "sha256": ""}) assert queued.status_code == 202, queued.text job_id = queued.json()["job_id"] for _ in range(100): job = call( client, "GET", f"/internal/v1/vm-templates/import-jobs/{job_id}", SUPER, ).json() if job["status"] in {"completed", "failed"}: break time.sleep(0.01) assert job["status"] == "completed", job template_id = job["template"]["id"] assert services == {} assert call(client, "GET", "/internal/v1/vm-templates", USER).json() == { "templates": [job["template"]], } provisioning = {**USER, "can_provision": True} body = {**CLOUD_CREATE_BODY, "template_id": template_id} hk = call( client, "POST", "/internal/v1/hosts/hk/vms", {**provisioning, "operation_id": "global-template-hk-0001"}, body, ) assert hk.status_code == 201, hk.text jp = call( client, "POST", "/internal/v1/hosts/jp/vms", {**provisioning, "operation_id": "global-template-jp-0001"}, {**body, "name": "cloud-vm-jp", "public_ip_cidr": "8.8.8.19/24"}, ) assert jp.status_code == 201, jp.text first_markers = { host_id: json.loads(HostStore(store, host_id).get( "cloud_template_replica", template_id)) for host_id in ("hk", "jp") } # The public catalog intentionally omits the private backing filename. source_volume_name = template_volume_name(template_id) for host_id, marker in first_markers.items(): assert marker["status"] == "ready" assert marker["sha256"] == digest assert len(marker["replica_identity"]) == 64 assert marker["replica_volume_name"] != source_volume_name assert marker["replica_volume_name"].startswith( source_volume_name.removesuffix(".qcow2") + "-") created = [row for row in services[host_id].calls if row[0] == "create_cloud_vm"] assert created[-1][2] == marker["replica_volume_name"] observed_markers = [] original_hk_cache = services["hk"].cache_template def inspect_hk_marker(*args, **kwargs): marker = json.loads(HostStore(store, "hk").get( "cloud_template_replica", template_id)) observed_markers.append(marker) return original_hk_cache(*args, **kwargs) services["hk"].cache_template = inspect_hk_marker hk_again = call( client, "POST", "/internal/v1/hosts/hk/vms", {**provisioning, "operation_id": "global-template-hk-0002"}, {**body, "name": "cloud-vm-hk-2", "public_ip_cidr": "8.8.8.20/24"}, ) assert hk_again.status_code == 201, hk_again.text services["hk"].cache_template = original_hk_cache assert [marker["status"] for marker in observed_markers] == ["syncing"] fallback_volume_name = observed_markers[0]["replica_volume_name"] assert fallback_volume_name != first_markers["hk"]["replica_volume_name"] hk_cache = [row for row in services["hk"].calls if row[0] == "cache_template"] jp_cache = [row for row in services["jp"].calls if row[0] == "cache_template"] assert hk_cache[0][4:] == (None, None) assert hk_cache[1][1] == fallback_volume_name assert hk_cache[1][4] == first_markers["hk"]["replica_volume_name"] assert hk_cache[1][5] == first_markers["hk"]["replica_identity"] assert jp_cache[0][4:] == (None, None) for host_id in ("hk", "jp"): marker = json.loads(HostStore(store, host_id).get( "cloud_template_replica", template_id)) assert marker["status"] == "ready" assert marker["sha256"] == digest assert len(marker["replica_identity"]) == 64 # A successful trusted reuse keeps the proven prior generation even # though the durable syncing marker contained a fresh fallback name. assert marker["replica_volume_name"] == ( first_markers[host_id]["replica_volume_name"]) # Rebinding the logical host record to a different SSH host identity must # invalidate its ready proof even when the pool name is unchanged. engine.registry.components["hk"].manager.profile["fingerprint"] = ( "SHA256:replacement-host") rebound = call( client, "POST", "/internal/v1/hosts/hk/vms", {**provisioning, "operation_id": "global-template-hk-0003"}, {**body, "name": "cloud-vm-hk-3", "public_ip_cidr": "8.8.8.21/24"}, ) assert rebound.status_code == 201, rebound.text hk_cache = [row for row in services["hk"].calls if row[0] == "cache_template"] assert hk_cache[-1][4:] == (None, None) rebound_marker = json.loads(HostStore(store, "hk").get( "cloud_template_replica", template_id)) assert rebound_marker["status"] == "ready" assert rebound_marker["replica_volume_name"] == hk_cache[-1][1] assert (rebound_marker["replica_volume_name"] != first_markers["hk"]["replica_volume_name"]) # A crash/failure after the syncing marker was stored must retry that # exact planned generation. Allocating yet another name would leak the # possibly partial remote volume and make recovery non-deterministic. failed_calls = [] def fail_hk_cache(image_path, volume_name, virtual_size, **kwargs): failed_calls.append((volume_name, kwargs, json.loads( HostStore(store, "hk").get( "cloud_template_replica", template_id)))) raise VMError("云镜像缓存同步中断", 502) services["hk"].cache_template = fail_hk_cache interrupted = call( client, "POST", "/internal/v1/hosts/hk/vms", {**provisioning, "operation_id": "global-template-hk-0004"}, {**body, "name": "cloud-vm-hk-4", "public_ip_cidr": "8.8.8.22/24"}, ) assert interrupted.status_code == 502, interrupted.text assert len(failed_calls) == 1 planned_volume_name, failed_kwargs, failed_marker = failed_calls[0] assert failed_marker["status"] == "syncing" assert failed_marker["replica_volume_name"] == planned_volume_name assert failed_kwargs["trusted_replica_volume_name"] == ( rebound_marker["replica_volume_name"]) assert failed_kwargs["trusted_replica_identity"] == ( rebound_marker["replica_identity"]) services["hk"].cache_template = original_hk_cache retried = call( client, "POST", "/internal/v1/hosts/hk/vms", {**provisioning, "operation_id": "global-template-hk-0005"}, {**body, "name": "cloud-vm-hk-5", "public_ip_cidr": "8.8.8.23/24"}, ) assert retried.status_code == 201, retried.text hk_cache = [row for row in services["hk"].calls if row[0] == "cache_template"] assert hk_cache[-1][1] == planned_volume_name assert hk_cache[-1][4:] == (None, None) recovered_marker = json.loads(HostStore(store, "hk").get( "cloud_template_replica", template_id)) assert recovered_marker["status"] == "ready" assert recovered_marker["replica_volume_name"] == planned_volume_name assert [row for row in services["hk"].calls if row[0] == "create_cloud_vm"][-1][2] == planned_volume_name assert (tmp_path / "cloud-templates" / engine.registry.cloud_templates.get(template_id)["volume_name"]).is_file() def test_legacy_host_template_id_remains_usable_for_an_already_queued_order( boundary): client, engine, store, _logs, services = boundary # Initialize the original target host, then seed the same host-scoped # catalog record an order created before the global-catalog upgrade holds. assert call(client, "GET", "/internal/v1/hosts/legacy/vms").status_code == 200 profile = engine.registry.components["legacy"].manager.profile scope = template_scope(profile) digest = "d" * 64 template_id = template_id_for(scope, digest, "10") DatabaseTemplateStore(HostStore(store, "legacy")).add(scope, { "id": template_id, "name": "Legacy AlmaLinux", "os_family": "AlmaLinux", "os_version": "10", "sha256": digest, "volume_name": template_volume_name(template_id), "virtual_size": 2 * 1024**3, "image_bytes": 1024, "created_at": 1, }) response = call( client, "POST", "/internal/v1/hosts/legacy/vms", {**USER, "can_provision": True, "operation_id": "legacy-template-order-0001"}, {**CLOUD_CREATE_BODY, "template_id": template_id, "name": "legacy-paid-order", "public_ip_cidr": "8.8.8.22/24"}, ) assert response.status_code == 201, response.text assert not any(row[0] == "cache_template" for row in services["legacy"].calls) create = next(row for row in services["legacy"].calls if row[0] == "create_cloud_vm") assert create[2] == template_volume_name(template_id) def test_global_cloud_template_delete_is_idempotent_and_retries_host_gc( boundary, tmp_path): client, engine, store, _logs, services = boundary imported, payload = import_cloud_template_for_test(client, tmp_path) template_id = imported["id"] template_created_at = imported["created_at"] source_volume_name = template_volume_name(template_id) source_path = tmp_path / "cloud-templates" / source_volume_name assert source_path.is_file() for host_id in ("hk", "jp"): assert call( client, "GET", f"/internal/v1/hosts/{host_id}/resources" ).status_code == 200 HostStore(store, host_id).set( "cloud_template_replica", template_id, "ready-marker") assert call( client, "DELETE", f"/internal/v1/vm-templates/{template_id}", ADMIN, {"created_at": template_created_at}, ).status_code == 403 services["hk"].cloud_template_delete_error = VMError( "云镜像仍被虚拟机引用,不能删除", 409) original_store_delete = store.delete failed_catalog_delete = False def fail_first_catalog_delete(kind, key): nonlocal failed_catalog_delete if (kind == "cloud_template" and key == template_id and not failed_catalog_delete): failed_catalog_delete = True raise RuntimeError("catalog delete failed before commit") return original_store_delete(kind, key) store.delete = fail_first_catalog_delete try: deleted = call( client, "DELETE", f"/internal/v1/vm-templates/{template_id}", SUPER, {"created_at": template_created_at}, ) finally: store.delete = original_store_delete assert failed_catalog_delete is True assert deleted.status_code == 200, deleted.text assert deleted.json() == { "deleted": True, "cache_cleanup_pending": True, } assert not source_path.exists() assert call(client, "GET", "/internal/v1/vm-templates", USER).json() == { "templates": [], } for _ in range(100): tombstone = json.loads(store.get( "cloud_template_tombstone", template_id)) if tombstone["pending_hosts"] == ["hk"]: break time.sleep(0.01) assert tombstone["source_deleted"] is True assert tombstone["source_volume_name"] == source_volume_name assert tombstone["pending_hosts"] == ["hk"] assert HostStore(store, "jp").get( "cloud_template_replica", template_id) is None assert HostStore(store, "hk").get( "cloud_template_replica", template_id) is not None # Catalog and host reads are side-effect free. The durable delete event's # maintenance supervisor retries independently of inventory/read traffic. calls_before = len([ row for row in services["hk"].calls if row[0] == "delete_cloud_template_replicas" ]) services["hk"].cloud_template_delete_error = VMError( "存储池 images 不存在", 404) with patch.object( engine, "_background", side_effect=AssertionError("catalog reads must not schedule cache GC")): assert call( client, "GET", "/internal/v1/vm-templates", USER ).status_code == 200 assert call(client, "GET", "/internal/v1/hosts/hk/resources").status_code == 200 assert len([ row for row in services["hk"].calls if row[0] == "delete_cloud_template_replicas" ]) == calls_before still_pending = json.loads(store.get( "cloud_template_tombstone", template_id)) assert still_pending["pending_hosts"] == ["hk"] assert HostStore(store, "hk").get( "cloud_template_replica", template_id) is not None engine.cache_cleanup_retry_seconds = 0.02 engine._wake_cache_cleanup() for _ in range(100): if len([ row for row in services["hk"].calls if row[0] == "delete_cloud_template_replicas" ]) > calls_before: break time.sleep(0.01) still_pending = json.loads(store.get( "cloud_template_tombstone", template_id)) assert still_pending["pending_hosts"] == ["hk"] services["hk"].cloud_template_delete_error = VMError( "云镜像宿主机缓存不存在", 404, error_code="cloud_template_cache_not_found", ) engine._wake_cache_cleanup() for _ in range(100): terminal = json.loads(store.get( "cloud_template_tombstone", template_id)) if terminal["pending_hosts"] == []: break time.sleep(0.01) terminal = json.loads(store.get("cloud_template_tombstone", template_id)) assert terminal["pending_hosts"] == [] assert HostStore(store, "hk").get( "cloud_template_replica", template_id) is None repeated = call( client, "DELETE", f"/internal/v1/vm-templates/{template_id}", SUPER, {"created_at": template_created_at}) assert repeated.status_code == 200 assert repeated.json() == { "deleted": True, "cache_cleanup_pending": False, } # A duplicate task from the old deletion must not touch a later deletion # generation (or the same-ID image/replica that may exist between them). old_deletion_id = terminal["deletion_id"] engine.registry.cloud_templates._prepare_reimport(template_id) reimported = { **imported, "volume_name": source_volume_name, "created_at": template_created_at + 1, } temporary = engine.registry.cloud_templates.files.directory() \ / ".cloud-image-reimport-boundary.qcow2" temporary.write_bytes(payload) temporary.chmod(0o600) engine.registry.cloud_templates.files.publish( temporary, template_id, expected_size=len(payload)) engine.registry.cloud_templates.catalog.add(reimported) stale_delete = call( client, "DELETE", f"/internal/v1/vm-templates/{template_id}", SUPER, {"created_at": template_created_at}) assert stale_delete.status_code == 409 assert engine.registry.cloud_templates.get( template_id)["created_at"] == template_created_at + 1 next_tombstone = { **terminal, "deletion_id": "b" * 32, "template_created_at": template_created_at + 1, "pending_hosts": ["hk"], } store.set("cloud_template_tombstone", template_id, json.dumps(next_tombstone)) calls_before = list(services["hk"].calls) engine._cleanup_cloud_template_tombstone_on_host_sync( template_id, old_deletion_id, "hk", services["hk"], engine.registry.components["hk"], ) assert services["hk"].calls == calls_before assert json.loads(store.get( "cloud_template_tombstone", template_id))["pending_hosts"] == ["hk"] def test_user_iso_routes_are_owner_scoped_and_create_uses_opaque_id(boundary, tmp_path): client, _engine, _store, _logs, services = boundary download = tmp_path / "download.iso" download.write_bytes(b"i" * (1024 * 1024)) def fake_download(_url, _sha256, _directory, *, max_bytes): assert max_bytes >= download.stat().st_size return download, download.stat().st_size, "a" * 64 with patch("virsh_panel.user_isos.download_user_iso", side_effect=fake_download): queued = call(client, "POST", "/internal/v1/user-isos/import", USER, {"url": "https://downloads.example.test/os.iso", "name": "My Installer", "sha256": ""}) assert queued.status_code == 202 job_id = queued.json()["job_id"] for _ in range(100): job = call(client, "GET", f"/internal/v1/user-isos/import-jobs/{job_id}").json() if job["status"] in {"completed", "failed"}: break time.sleep(0.01) assert job["status"] == "completed", job iso_id = job["iso"]["id"] assert "volume_name" not in job["iso"] listed = call(client, "GET", "/internal/v1/user-isos").json() assert [row["id"] for row in listed["isos"]] == [iso_id] assert "volume_name" not in listed["isos"][0] global_source = (tmp_path / "user-isos" / str(USER["user_id"]) / f"panel-user-iso-{iso_id}.iso") assert global_source.is_file() assert services == {} assert call( client, "GET", "/internal/v1/hosts/hk/user-isos", USER, ).status_code == 404 assert call( client, "POST", "/internal/v1/hosts/hk/user-isos/import", USER, { "url": "https://downloads.example.test/alias.iso", "name": "Host Alias Must Fail", "sha256": "", }, ).status_code == 404 assert call( client, "DELETE", f"/internal/v1/hosts/hk/user-isos/{iso_id}", USER, ).status_code == 404 assert global_source.is_file() other = {**USER, "user_id": 42} assert call(client, "GET", f"/internal/v1/user-isos/{iso_id}", other).status_code == 404 assert call(client, "GET", f"/internal/v1/user-isos/import-jobs/{job_id}", other).status_code == 404 assert call(client, "DELETE", f"/internal/v1/user-isos/{iso_id}", other).status_code == 404 provisioning = {**USER, "can_provision": True, "operation_id": "create-user-iso-0001"} body = {key: value for key, value in CREATE_BODY.items() if key != "iso"} body["user_iso_id"] = iso_id created = call(client, "POST", "/internal/v1/hosts/hk/vms", provisioning, body) assert created.status_code == 201, created.text create_calls = [row for row in services["hk"].calls if row[0] == "create_user_iso_vm"] assert len(create_calls) == 1 assert create_calls[0][2] == USER["user_id"] assert create_calls[0][4] == iso_id cache_calls = [row for row in services["hk"].calls if row[0] == "cache_user_iso"] assert len(cache_calls) == 1 assert cache_calls[0][4] is False other_host_actor = { **provisioning, "operation_id": "create-user-iso-0003", } other_host_body = {**body, "name": "other-host-installer"} other_host = call(client, "POST", "/internal/v1/hosts/jp/vms", other_host_actor, other_host_body) assert other_host.status_code == 201, other_host.text assert len([row for row in services["jp"].calls if row[0] == "cache_user_iso"]) == 1 assert global_source.is_file() ambiguous_actor = {**provisioning, "operation_id": "create-user-iso-0002"} ambiguous = call(client, "POST", "/internal/v1/hosts/hk/vms", ambiguous_actor, {**body, "iso": "legacy.iso"}) assert ambiguous.status_code == 422 def test_user_iso_writes_require_manage_capability(boundary): client, *_rest = boundary actor = {**USER, "can_manage": False} response = call(client, "POST", "/internal/v1/user-isos/import", actor, {"url": "https://downloads.example.test/os.iso", "name": "Installer", "sha256": ""}) assert response.status_code == 403 def test_concurrent_first_use_serializes_replica_marker_and_cache(boundary, tmp_path): client, _engine, _store, _logs, services = boundary download = tmp_path / "concurrent.iso" download.write_bytes(b"i" * (1024 * 1024)) with patch("virsh_panel.user_isos.download_user_iso", return_value=(download, download.stat().st_size, "1" * 64)): queued = call(client, "POST", "/internal/v1/user-isos/import", USER, { "url": "https://downloads.example.test/concurrent.iso", "name": "Concurrent Installer", "sha256": "", }) job_id = queued.json()["job_id"] for _ in range(100): job = call( client, "GET", f"/internal/v1/user-isos/import-jobs/{job_id}", ).json() if job["status"] in {"completed", "failed"}: break time.sleep(0.01) assert job["status"] == "completed", job iso_id = job["iso"]["id"] # Materialize the target child app before starting concurrent requests. assert call(client, "GET", "/internal/v1/hosts/hk/resources").status_code == 200 service = services["hk"] release = threading.Event() service.user_iso_cache_release = release base = {key: value for key, value in CREATE_BODY.items() if key != "iso"} base["user_iso_id"] = iso_id def create(number): actor = {**USER, "can_provision": True, "operation_id": f"concurrent-user-iso-{number:04d}"} return call(client, "POST", "/internal/v1/hosts/hk/vms", actor, {**base, "name": f"concurrent-vm-{number}"}) with ThreadPoolExecutor(max_workers=2) as executor: first = executor.submit(create, 1) assert service.user_iso_cache_started.wait(timeout=2) second = executor.submit(create, 2) time.sleep(0.05) release.set() responses = [first.result(timeout=5), second.result(timeout=5)] assert [response.status_code for response in responses] == [201, 201] cache_calls = [row for row in service.calls if row[0] == "cache_user_iso"] assert [row[4] for row in cache_calls] == [False, True] def test_user_iso_source_stays_in_use_through_vm_create(boundary, tmp_path): client, engine, _store, _logs, services = boundary iso_id = import_user_iso_for_test(client, tmp_path, "Create Lease Installer") assert call(client, "GET", "/internal/v1/hosts/hk/resources").status_code == 200 service = services["hk"] release = threading.Event() service.user_iso_create_release = release body = {key: value for key, value in CREATE_BODY.items() if key != "iso"} body["user_iso_id"] = iso_id actor = {**USER, "can_provision": True, "operation_id": "user-iso-source-lease-0001"} with ThreadPoolExecutor(max_workers=1) as executor: creating = executor.submit( call, client, "POST", "/internal/v1/hosts/hk/vms", actor, body) assert service.user_iso_create_started.wait(timeout=2) with pytest.raises(TemplateError, match="正在同步") as busy: engine.registry.user_isos.delete_for(USER["user_id"], iso_id) assert busy.value.status == 409 release.set() created = creating.result(timeout=5) assert created.status_code == 201, created.text assert engine.registry.user_isos.get_for(USER["user_id"], iso_id)["id"] == iso_id def test_ready_replica_marker_survives_failed_reuse(boundary, tmp_path): client, engine, store, _logs, services = boundary iso_id = import_user_iso_for_test(client, tmp_path, "Ready Marker Installer") body = {key: value for key, value in CREATE_BODY.items() if key != "iso"} body["user_iso_id"] = iso_id first = call( client, "POST", "/internal/v1/hosts/hk/vms", {**USER, "can_provision": True, "operation_id": "ready-marker-create-0001"}, body) assert first.status_code == 201, first.text marker_key = f"{USER['user_id']}:{iso_id}" scoped = HostStore(store, "hk") assert json.loads(scoped.get("user_iso_replica", marker_key))["status"] == "ready" with patch.object( engine.registry.user_isos, "_verify_source", side_effect=TemplateError("ISO 源文件的 SHA-256 校验值不匹配", 500)): failed = call( client, "POST", "/internal/v1/hosts/hk/vms", {**USER, "can_provision": True, "operation_id": "ready-marker-create-0002"}, {**body, "name": "ready-marker-failed"}) assert failed.status_code == 500 assert json.loads(scoped.get("user_iso_replica", marker_key))["status"] == "ready" recovered = call( client, "POST", "/internal/v1/hosts/hk/vms", {**USER, "can_provision": True, "operation_id": "ready-marker-create-0003"}, {**body, "name": "ready-marker-recovered"}) assert recovered.status_code == 201, recovered.text cache_calls = [row for row in services["hk"].calls if row[0] == "cache_user_iso"] assert cache_calls[-1][4] is True def test_global_delete_retries_tombstone_until_host_cache_is_reclaimed( boundary, tmp_path): client, engine, store, _logs, services = boundary download = tmp_path / "delete.iso" download.write_bytes(b"i" * (1024 * 1024)) with patch("virsh_panel.user_isos.download_user_iso", return_value=(download, download.stat().st_size, "2" * 64)): queued = call(client, "POST", "/internal/v1/user-isos/import", USER, { "url": "https://downloads.example.test/delete.iso", "name": "Delete Installer", "sha256": "", }) job_id = queued.json()["job_id"] for _ in range(100): job = call( client, "GET", f"/internal/v1/user-isos/import-jobs/{job_id}", ).json() if job["status"] in {"completed", "failed"}: break time.sleep(0.01) assert job["status"] == "completed", job iso_id = job["iso"]["id"] body = {key: value for key, value in CREATE_BODY.items() if key != "iso"} body["user_iso_id"] = iso_id created = call( client, "POST", "/internal/v1/hosts/hk/vms", {**USER, "can_provision": True, "operation_id": "delete-user-iso-cache-0001"}, body) assert created.status_code == 201, created.text from virsh_panel.libvirt_service import VMError services["hk"].user_iso_delete_error = VMError( "存储池 iso 不存在", 404) deleted = call( client, "DELETE", f"/internal/v1/user-isos/{iso_id}", USER) assert deleted.status_code == 200 assert deleted.json() == {"deleted": True, "cache_cleanup_pending": True} assert call(client, "GET", "/internal/v1/user-isos", USER).json()["isos"] == [] tombstone_key = f"{USER['user_id']}:{iso_id}" for _ in range(100): if any(row[0] == "delete_user_iso" for row in services["hk"].calls): break time.sleep(0.01) assert store.get("user_iso_tombstone", tombstone_key) is not None scoped = HostStore(store, "hk") assert scoped.get("user_iso_replica", tombstone_key) is not None services["hk"].user_iso_delete_error = VMError( "用户 ISO 在宿主机上不存在", 404, error_code="user_iso_volume_not_found", ) calls_before = len([ row for row in services["hk"].calls if row[0] == "delete_user_iso" ]) assert call(client, "GET", "/internal/v1/hosts/hk/resources").status_code == 200 assert len([ row for row in services["hk"].calls if row[0] == "delete_user_iso" ]) == calls_before engine.cache_cleanup_retry_seconds = 0.02 engine._wake_cache_cleanup() for _ in range(100): if store.get("user_iso_tombstone", tombstone_key) is None: break time.sleep(0.01) assert store.get("user_iso_tombstone", tombstone_key) is None assert scoped.get("user_iso_replica", tombstone_key) is None def test_concurrent_global_deletes_preserve_committed_tombstone(boundary, tmp_path): client, engine, store, _logs, services = boundary iso_id = import_user_iso_for_test(client, tmp_path, "Concurrent Delete Installer") assert call(client, "GET", "/internal/v1/hosts/hk/resources").status_code == 200 record = engine.registry.user_isos.get_for(USER["user_id"], iso_id) marker_key = f"{USER['user_id']}:{iso_id}" HostStore(store, "hk").set("user_iso_replica", marker_key, json.dumps({ "status": "ready", "volume_name": record["volume_name"], "size_bytes": record["size_bytes"], "sha256": record["sha256"], })) from virsh_panel.libvirt_service import VMError services["hk"].user_iso_volumes.add((USER["user_id"], record["volume_name"])) services["hk"].user_iso_delete_error = VMError( "用户 ISO 仍被虚拟机安装过程引用,不能删除", 409) original_delete = engine.registry.user_isos.delete_for started = threading.Event() release = threading.Event() def blocking_delete(user_id, selected_iso_id): started.set() assert release.wait(timeout=5) return original_delete(user_id, selected_iso_id) try: with patch.object(engine.registry.user_isos, "delete_for", side_effect=blocking_delete) as delete_for: with ThreadPoolExecutor(max_workers=2) as executor: first = executor.submit( call, client, "DELETE", f"/internal/v1/user-isos/{iso_id}", USER) assert started.wait(timeout=2) second = executor.submit( call, client, "DELETE", f"/internal/v1/user-isos/{iso_id}", USER) time.sleep(0.05) release.set() responses = [first.result(timeout=5), second.result(timeout=5)] assert delete_for.call_count == 1 finally: release.set() assert [response.status_code for response in responses] == [200, 200] tombstone = json.loads(store.get("user_iso_tombstone", marker_key)) assert tombstone["source_deleted"] is True assert tombstone["pending_hosts"] == ["hk"] def test_user_iso_cache_cleanup_blocks_online_update(boundary, tmp_path): client, engine, store, _logs, services = boundary iso_id = import_user_iso_for_test(client, tmp_path, "Update Gate Installer") assert call(client, "GET", "/internal/v1/hosts/hk/resources").status_code == 200 record = engine.registry.user_isos.get_for(USER["user_id"], iso_id) marker_key = f"{USER['user_id']}:{iso_id}" HostStore(store, "hk").set("user_iso_replica", marker_key, json.dumps({ "status": "ready", "volume_name": record["volume_name"], "size_bytes": record["size_bytes"], "sha256": record["sha256"], })) service = services["hk"] service.user_iso_volumes.add((USER["user_id"], record["volume_name"])) release = threading.Event() service.user_iso_delete_release = release updater = FakeUpdater() updater.token = "valid-token-123456" engine.updater = updater try: deleted = call( client, "DELETE", f"/internal/v1/user-isos/{iso_id}", USER) assert deleted.status_code == 200 assert service.user_iso_delete_started.wait(timeout=2) blocked = call(client, "POST", "/internal/v1/update/apply", SUPER, { "expected_current_commit": "a" * 40, "expected_remote_commit": "b" * 40, }) assert blocked.status_code == 409 assert updater.active is None finally: release.set() for _ in range(100): if not engine.maintenance_tasks: break time.sleep(0.01) assert not engine.maintenance_tasks def test_unrelated_host_requests_never_run_cache_cleanup(boundary): client, engine, store, _logs, services = boundary iso_id = "f" * 32 volume_name = f"panel-user-iso-{iso_id}.iso" marker_key = f"{USER['user_id']}:{iso_id}" HostStore(store, "hk").set("user_iso_replica", marker_key, json.dumps({ "status": "ready", "volume_name": volume_name, "size_bytes": 1024 * 1024, "sha256": "f" * 64, })) store.set("user_iso_tombstone", marker_key, json.dumps({ "user_id": USER["user_id"], "iso_id": iso_id, "volume_name": volume_name, "pending_hosts": ["hk"], "source_deleted": True, "created_at": 1, })) with patch.object( engine.updater, "active_job", side_effect=AssertionError("host reads must not repair updater state")), \ patch.object( engine, "_cleanup_host_user_iso_tombstones", side_effect=AssertionError("host reads must not run ISO GC")), \ patch.object( engine, "_cleanup_host_cloud_template_tombstones", side_effect=AssertionError("host reads must not run template GC")): response = call(client, "GET", "/internal/v1/hosts/hk/resources") assert response.status_code == 200 assert not any(row[0] == "delete_user_iso" for row in services["hk"].calls) assert store.get("user_iso_tombstone", marker_key) is not None with patch.object( engine, "_cleanup_host_user_iso_tombstones", side_effect=AssertionError("host writes must not run ISO GC")), \ patch.object( engine, "_cleanup_host_cloud_template_tombstones", side_effect=AssertionError("host writes must not run template GC")): invalid_write = call( client, "PUT", f"/internal/v1/hosts/hk/host/network-pools/{'a' * 32}", SUPER, {}, ) assert invalid_write.status_code == 422 def test_removed_inventory_routes_do_not_enter_dispatch(boundary): client, engine, *_ = boundary with patch.object( engine, "_update_lock", side_effect=AssertionError("removed inventory route reached mutation dispatch")), \ patch.object( engine.updater, "active_job", side_effect=AssertionError("removed inventory route reached updater")): capacity = call( client, "GET", "/internal/v1/hosts/hk/host/capacity", ADMIN) network = call( client, "GET", "/internal/v1/hosts/hk/host/network-options", ADMIN) assert capacity.status_code == 404 assert network.status_code == 404 def test_slow_tombstone_cleanup_does_not_block_engine_startup(tmp_path): class StartupRegistry: def __init__(self): self.components = {"offline": object()} self.user_isos = SimpleNamespace(has_active_jobs=lambda: False) self.started = False self.closed = False async def start(self): self.started = True async def close(self): self.closed = True async def scenario(): registry = StartupRegistry() updater = FakeUpdater() store = MemoryStore() iso_id = "f" * 32 tombstone_key = f"1:{iso_id}" store.set("user_iso_tombstone", tombstone_key, json.dumps({ "user_id": 1, "iso_id": iso_id, "volume_name": f"panel-user-iso-{iso_id}.iso", "pending_hosts": ["offline"], "source_deleted": True, "created_at": 1, })) engine = EngineApplication( {"service_secret": base64.b64encode(SECRET).decode(), "data_dir": str(tmp_path)}, store, MemoryStore(), registry=registry, updater=updater, ) cleanup_started = asyncio.Event() cleanup_release = asyncio.Event() async def slow_cleanup(host_id): assert host_id == "offline" cleanup_started.set() await cleanup_release.wait() engine._cleanup_host_user_iso_tombstones = slow_cleanup incoming = asyncio.Queue() outgoing = asyncio.Queue() async def receive(): return await incoming.get() async def send(message): await outgoing.put(message) await incoming.put({"type": "lifespan.startup"}) lifespan = asyncio.create_task(engine._lifespan(receive, send)) startup = await asyncio.wait_for(outgoing.get(), timeout=0.5) assert startup == {"type": "lifespan.startup.complete"} await asyncio.wait_for(cleanup_started.wait(), timeout=0.5) assert registry.started is True assert engine.maintenance_tasks assert engine._long_running_host_job() is True cleanup_release.set() await incoming.put({"type": "lifespan.shutdown"}) shutdown = await asyncio.wait_for(outgoing.get(), timeout=0.5) assert shutdown == {"type": "lifespan.shutdown.complete"} await asyncio.wait_for(lifespan, timeout=0.5) assert registry.closed is True asyncio.run(scenario()) def test_tombstone_cleanup_pauses_for_update_then_resumes(tmp_path): class StartupRegistry: def __init__(self): self.components = {"offline": object()} self.user_isos = SimpleNamespace(has_active_jobs=lambda: False) async def start(self): return None async def close(self): return None async def scenario(): store = MemoryStore() iso_id = "e" * 32 tombstone_key = f"1:{iso_id}" store.set("user_iso_tombstone", tombstone_key, json.dumps({ "user_id": 1, "iso_id": iso_id, "volume_name": f"panel-user-iso-{iso_id}.iso", "pending_hosts": ["offline"], "source_deleted": True, "created_at": 1, })) updater = FakeUpdater() updater.active = {"job_id": "update-job-running"} engine = EngineApplication( {"service_secret": base64.b64encode(SECRET).decode(), "data_dir": str(tmp_path)}, store, MemoryStore(), registry=StartupRegistry(), updater=updater, ) engine.cache_cleanup_retry_seconds = 0.02 cleanup_started = asyncio.Event() async def cleanup(host_id): assert host_id == "offline" cleanup_started.set() store.delete("user_iso_tombstone", tombstone_key) engine._cleanup_host_user_iso_tombstones = cleanup incoming = asyncio.Queue() outgoing = asyncio.Queue() async def receive(): return await incoming.get() async def send(message): await outgoing.put(message) await incoming.put({"type": "lifespan.startup"}) lifespan = asyncio.create_task(engine._lifespan(receive, send)) assert await asyncio.wait_for(outgoing.get(), timeout=0.5) == { "type": "lifespan.startup.complete", } await asyncio.sleep(0.06) assert not cleanup_started.is_set() updater.active = None await asyncio.wait_for(cleanup_started.wait(), timeout=0.5) assert store.get("user_iso_tombstone", tombstone_key) is None await incoming.put({"type": "lifespan.shutdown"}) assert await asyncio.wait_for(outgoing.get(), timeout=0.5) == { "type": "lifespan.shutdown.complete", } await asyncio.wait_for(lifespan, timeout=0.5) asyncio.run(scenario()) def test_user_iso_global_import_does_not_contact_a_host(boundary, tmp_path): client, _engine, _store, _logs, services = boundary download = tmp_path / "global.iso" download.write_bytes(b"i" * (1024 * 1024)) with patch("virsh_panel.user_isos.download_user_iso", return_value=(download, download.stat().st_size, "f" * 64)): response = call(client, "POST", "/internal/v1/user-isos/import", USER, {"url": "https://downloads.example.test/os.iso", "name": "Installer", "sha256": ""}) assert response.status_code == 202 job_id = response.json()["job_id"] for _ in range(100): job = call( client, "GET", f"/internal/v1/user-isos/import-jobs/{job_id}", ).json() if job["status"] in {"completed", "failed"}: break time.sleep(0.01) assert job["status"] == "completed", job assert services == {} def test_user_iso_download_is_bounded_by_panel_storage(boundary, tmp_path): client, engine, _store, _logs, _services = boundary download = tmp_path / "bounded.iso" download.write_bytes(b"i" * (1024 * 1024)) def bounded_download(_url, _sha256, _directory, *, max_bytes): assert max_bytes == 2 * 1024 * 1024 return download, download.stat().st_size, "d" * 64 with patch.object(engine.registry.user_isos.files, "available_bytes", return_value=2 * 1024 * 1024), patch( "virsh_panel.user_isos.download_user_iso", side_effect=bounded_download): response = call(client, "POST", "/internal/v1/user-isos/import", USER, {"url": "https://downloads.example.test/os.iso", "name": "Bounded Installer", "sha256": ""}) assert response.status_code == 202 job_id = response.json()["job_id"] for _ in range(100): job = call(client, "GET", f"/internal/v1/user-isos/import-jobs/{job_id}").json() if job["status"] in {"completed", "failed"}: break time.sleep(0.01) assert job["status"] == "completed", job def test_user_iso_job_hides_internal_paths_in_failures(boundary): client, _engine, _store, _logs, _services = boundary with patch("virsh_panel.user_isos.download_user_iso", side_effect=TemplateError("目录 /private/data/user-isos/41 损坏", 500)): response = call(client, "POST", "/internal/v1/user-isos/import", USER, {"url": "https://downloads.example.test/os.iso", "name": "Error Installer", "sha256": ""}) assert response.status_code == 202 job_id = response.json()["job_id"] for _ in range(100): job = call(client, "GET", f"/internal/v1/user-isos/import-jobs/{job_id}").json() if job["status"] in {"completed", "failed"}: break time.sleep(0.01) assert job["status"] == "failed" assert job["error"] == "ISO 下载或保存失败,请联系管理员查看日志" assert "/private/data" not in json.dumps(job, ensure_ascii=False) def test_user_iso_import_persistence_failure_does_not_stick_pending(boundary, tmp_path): client, _engine, store, _logs, _services = boundary assert call(client, "GET", "/internal/v1/user-isos").status_code == 200 original_set = store.set def fail_user_iso_job(kind, key, value): if kind == "user_iso_import_job": raise DatabaseError() return original_set(kind, key, value) body = {"url": "https://downloads.example.test/os.iso", "name": "Installer", "sha256": ""} with patch.object(store, "set", side_effect=fail_user_iso_job): failed = call(client, "POST", "/internal/v1/user-isos/import", USER, body) assert failed.status_code == 503 download = tmp_path / "retry.iso" download.write_bytes(b"i" * (1024 * 1024)) with patch("virsh_panel.user_isos.download_user_iso", return_value=(download, download.stat().st_size, "b" * 64)): retried = call(client, "POST", "/internal/v1/user-isos/import", USER, body) assert retried.status_code == 202, retried.text for _ in range(100): job = call( client, "GET", f"/internal/v1/user-isos/import-jobs/{retried.json()['job_id']}", ).json() if job["status"] in {"completed", "failed"}: break time.sleep(0.01) assert job["status"] == "completed", job def test_interrupted_user_iso_import_is_restored_as_failed(boundary): client, _engine, store, _logs, _services = boundary job_id = "interrupted-user-iso" scope = user_iso_scope(USER["user_id"]) store.set("user_iso_import_job", job_id, json.dumps({ "job_id": job_id, "scope": scope, "owner_user_id": USER["user_id"], "status": "running", "phase": "正在安全下载 ISO", "created_at": 1, })) _engine.registry.user_isos._restored = False _engine.registry.user_isos.restore_jobs() restored = call( client, "GET", f"/internal/v1/user-isos/import-jobs/{job_id}", ) assert restored.status_code == 200 assert restored.json()["status"] == "failed" assert "重启" in restored.json()["phase"] def test_user_iso_list_exposes_only_owners_active_import(boundary, tmp_path): client, engine, _store, _logs, _services = boundary download = tmp_path / "active.iso" download.write_bytes(b"i" * (1024 * 1024)) started = threading.Event() release = threading.Event() def blocking_download(_url, _sha256, _directory, *, max_bytes): assert max_bytes >= download.stat().st_size started.set() assert release.wait(timeout=5) return download, download.stat().st_size, "c" * 64 body = {"url": "https://downloads.example.test/os.iso", "name": "Active Installer", "sha256": ""} try: with patch("virsh_panel.user_isos.download_user_iso", side_effect=blocking_download): queued = call(client, "POST", "/internal/v1/user-isos/import", USER, body) assert queued.status_code == 202 assert started.wait(timeout=2) assert engine._long_running_host_job() is True own = call(client, "GET", "/internal/v1/user-isos", USER).json() active = own["active_import_job"] assert active["job_id"] == queued.json()["job_id"] assert active["status"] == "running" assert set(active) <= {"job_id", "status", "phase", "created_at", "started_at", "finished_at", "iso", "error"} assert "scope" not in active and "owner_user_id" not in active other = call(client, "GET", "/internal/v1/user-isos", {**USER, "user_id": 42}).json() assert "active_import_job" not in other release.set() for _ in range(100): job = call( client, "GET", f"/internal/v1/user-isos/import-jobs/{active['job_id']}", ).json() if job["status"] in {"completed", "failed"}: break time.sleep(0.01) finally: release.set() assert job["status"] == "completed", job assert "active_import_job" not in call( client, "GET", "/internal/v1/user-isos", USER).json() def test_hmac_binds_actor_payload_target_and_rejects_replay(): store = MemoryStore() body = b"{}" target = "/internal/v1/hosts/hk/vms?range=1h" headers = {key.lower(): value for key, value in signed_headers(SECRET, "POST", target, body, USER, timestamp=1000, nonce="n" * 24).items()} assert authenticate(headers, SECRET, "POST", target, hashlib.sha256(body).hexdigest(), store, now=1000) == USER with pytest.raises(EngineAuthError): authenticate(headers, SECRET, "POST", target, hashlib.sha256(body).hexdigest(), store, now=1000) changed = {**headers, "x-cloud-actor": encode_actor(SUPER)} with pytest.raises(EngineAuthError): authenticate(changed, SECRET, "POST", target, hashlib.sha256(body).hexdigest(), store, now=1000) with pytest.raises(EngineAuthError): authenticate(headers, SECRET, "POST", target + "x", hashlib.sha256(body).hexdigest(), store, now=1000) with pytest.raises(EngineAuthError): authenticate(headers, SECRET, "POST", target, hashlib.sha256(b"other").hexdigest(), store, now=1000) with pytest.raises(EngineAuthError): authenticate(headers, SECRET, "POST", target, hashlib.sha256(body).hexdigest(), store, now=1061) invalid_cleanup = {**USER, "system_cleanup": "true"} invalid_headers = {key.lower(): value for key, value in signed_headers( SECRET, "POST", target, body, invalid_cleanup, timestamp=1000, nonce="c" * 24, ).items()} with pytest.raises(EngineAuthError): authenticate( invalid_headers, SECRET, "POST", target, hashlib.sha256(body).hexdigest(), store, now=1000, ) invalid_vpc_reconcile = {**USER, "vpc_reconcile_explicit": "true"} invalid_headers = {key.lower(): value for key, value in signed_headers( SECRET, "POST", target, body, invalid_vpc_reconcile, timestamp=1000, nonce="v" * 24, ).items()} with pytest.raises(EngineAuthError): authenticate( invalid_headers, SECRET, "POST", target, hashlib.sha256(body).hexdigest(), store, now=1000, ) def test_private_route_allowlist_and_no_browser_admin_cookie(boundary): client, *_ = boundary for path in ("/api/v1/vms", "/internal/v1/hosts/hk/register", "/internal/v1/hosts/hk/setup/admin", "/internal/v1/hosts/hk/update/apply", "/internal/v1/hosts/hk/smtp/settings"): assert call(client, "POST", path, SUPER, {}).status_code == 404 assert client.get("/internal/v1/hosts/hk/vms", cookies={"virsh_panel_session": "admin"}).status_code == 401 assert TRUSTED_ACTOR.get() is None def test_internal_update_routes_are_super_admin_only_and_never_echo_token(boundary): client, engine, *_ = boundary updater = FakeUpdater() triggered = [] engine.updater = updater def trigger(): triggered.append(True) updater.active["status"] = "running" engine.update_trigger = trigger settings_path = "/internal/v1/update/settings" ready = call(client, "GET", "/internal/v1/health", SUPER) assert ready.status_code == 200 assert ready.json() == {"status": "ok", "accepting_mutations": True} assert client.get(settings_path, cookies={"venix": "admin"}).status_code == 401 assert call(client, "GET", settings_path, USER).status_code == 403 assert call(client, "GET", settings_path, ADMIN).status_code == 403 assert call(client, "GET", settings_path, {**SUPER, "can_manage": False}).status_code == 403 token = "gitea-private-token-123456" saved = call(client, "PUT", settings_path, SUPER, {"access_token": token}) assert saved.status_code == 200 assert saved.json()["token_configured"] is True assert saved.json()["repository"] == "https://g.luoli.live/root/virt.git" assert saved.json()["branch"] == "venix" assert token not in saved.text assert token not in call(client, "GET", settings_path, SUPER).text checked = call(client, "POST", "/internal/v1/update/check", SUPER, {}) assert checked.status_code == 200 assert checked.json()["status"] == "update_available" applied = call(client, "POST", "/internal/v1/update/apply", SUPER, { "expected_current_commit": "a" * 40, "expected_remote_commit": "b" * 40, }) assert applied.status_code == 202 job_id = applied.json()["job_id"] assert job_id in updater.jobs assert "private" not in applied.json() and token not in applied.text assert triggered == [True] job = call(client, "GET", f"/internal/v1/update/jobs/{job_id}", SUPER) assert job.status_code == 200 and token not in job.text and "private" not in job.json() # While the persisted update job is active, reads remain available but no # host/VM mutation may begin and overlap the code/service restart. maintenance = call(client, "GET", "/internal/v1/health", SUPER) assert maintenance.status_code == 200 assert maintenance.json() == {"status": "ok", "accepting_mutations": False} assert call(client, "POST", "/internal/v1/hosts/scan", SUPER, {"host": "host.example.test", "port": 22}).status_code == 409 def test_internal_update_validation_clear_and_trigger_failure(boundary): client, engine, *_ = boundary updater = FakeUpdater() engine.updater = updater settings_path = "/internal/v1/update/settings" assert call(client, "PUT", settings_path, SUPER, {}).status_code == 422 assert call(client, "PUT", settings_path, SUPER, {"access_token": "x", "unexpected": True}).status_code == 422 assert call(client, "POST", "/internal/v1/update/check", SUPER, {"repository": "https://evil.example/repo.git"}).status_code == 422 assert call(client, "POST", "/internal/v1/update/apply", SUPER, {"expected_current_commit": "a" * 40}).status_code == 422 assert call(client, "GET", "/internal/v1/update/jobs/missing-job-123456", SUPER).status_code == 404 call(client, "PUT", settings_path, SUPER, {"access_token": "valid-token-123456"}) cleared = call(client, "PUT", settings_path, SUPER, {"clear_token": True}) assert cleared.status_code == 200 and cleared.json()["token_configured"] is False call(client, "PUT", settings_path, SUPER, {"access_token": "valid-token-123456"}) def fail_trigger(): from virsh_panel.updater import UpdateError raise UpdateError("独立更新服务启动失败,请确认 virt-cloud-update.service 已安装", 503) engine.update_trigger = fail_trigger failed = call(client, "POST", "/internal/v1/update/apply", SUPER, { "expected_current_commit": "a" * 40, "expected_remote_commit": "b" * 40, }) assert failed.status_code == 503 assert len(updater.jobs) == 1 job = next(iter(updater.jobs.values())) assert job["status"] == "failed" assert "令牌" not in job["error"] @pytest.mark.parametrize(("state", "expected"), [ ("activating", True), ("active", True), ("deactivating", True), ("inactive", False), ("failed", False), ("maintenance", None), ]) def test_update_oneshot_transitional_states_keep_maintenance_gate(state, expected): with patch("virsh_panel.internal.subprocess.run", return_value=SimpleNamespace(returncode=0, stdout=state + "\n")): assert EngineApplication._update_service_active() is expected def test_user_admin_super_admin_boundaries_and_cross_user_vm(boundary): client, engine, *_ = boundary path = "/internal/v1/hosts/hk" assert call(client, "GET", path + "/host").status_code == 403 assert call(client, "GET", path + "/host", ADMIN).status_code == 200 assert call(client, "POST", path + "/host/scan", ADMIN, {"host": "x", "port": 22}).status_code == 403 assert call(client, "POST", path + "/host/scan", SUPER, {"host": "x", "port": 22}).status_code == 200 assert call(client, "POST", "/internal/v1/hosts/scan", ADMIN, {"host": "x", "port": 22}).status_code == 403 component = engine.registry.components["hk"] component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41) assert call(client, "GET", path + f"/vms/{VM_ID}").status_code == 200 assert call(client, "GET", path + f"/vms/{VM_ID}", {**USER, "user_id": 42}).status_code == 404 assert call(client, "GET", path + "/vms", {**USER, "user_id": 42}).json()["items"] == [] assert call(client, "POST", path + f"/vms/{VM_ID}/actions", {**USER, "can_manage": False}, {"action": "reboot"}).status_code == 403 def test_stable_host_identity_and_namespaces(): store = MemoryStore() first, second = HostStore(store, "a"), HostStore(store, "b") first.set("host", "current", "one") second.set("host", "current", "two") assert first.get("host", "current") == "one" assert second.get("host", "current") == "two" a = {"host_id": "a", "host": "1.2.3.4", "port": 22, "fingerprint": "old"} assert host_scope(a) == host_scope({**a, "host": "changed", "port": 2222, "fingerprint": "new"}) assert host_scope(a) != host_scope({**a, "host_id": "b"}) assert MainStore.RECORD_TABLE == "engine_records" assert LogStore.METRIC_TABLE == "engine_vm_metric_samples" def test_lightweight_vm_state_reads_are_allowlisted(boundary): client, _engine, _store, _logs, services = boundary listed = call(client, "GET", "/internal/v1/hosts/hk/vm-states", SUPER) assert listed.status_code == 200, listed.text assert listed.json()["items"] == [{ "uuid": VM_ID, "name": "alma-test", "state": "running", }] single = call( client, "GET", f"/internal/v1/hosts/hk/vms/{VM_ID}/state", SUPER, ) assert single.status_code == 200, single.text assert single.json() == { "uuid": VM_ID, "name": "alma-test", "state": "running", } assert ("list_vm_states",) in services["hk"].calls assert ("get_vm_state", VM_ID) in services["hk"].calls def test_selected_vm_metadata_is_allowlisted_and_owner_scoped(boundary): client, engine, _store, _logs, services = boundary path = f"/internal/v1/hosts/hk/vm-metadata?ids={VM_ID}" admin = call(client, "GET", path, SUPER) assert admin.status_code == 200, admin.text assert admin.json()["items"][0]["uuid"] == VM_ID assert services["hk"].calls[-1] == ("list_vm_metadata", (VM_ID,)) component = engine.registry.components["hk"] scope = host_scope(component.manager.profile) component.state.users.assign_vm(scope, VM_ID, USER["user_id"]) owner = call(client, "GET", path, USER) assert owner.status_code == 200, owner.text calls_after_owner = services["hk"].calls.count(("list_vm_metadata", (VM_ID,))) assert calls_after_owner == 2 stranger = call(client, "GET", path, {**USER, "user_id": 42}) assert stranger.status_code == 404 assert stranger.json()["error"] == "虚拟机不存在" assert services["hk"].calls.count(("list_vm_metadata", (VM_ID,))) == calls_after_owner def test_only_paid_operations_can_create_and_retry_never_duplicates(boundary): client, engine, store, _, services = boundary path = "/internal/v1/hosts/hk/vms" assert call(client, "POST", path, USER, CREATE_BODY).status_code == 403 provisioning = {**USER, "can_provision": True, "operation_id": "order-000001"} created = call(client, "POST", path, provisioning, CREATE_BODY) assert created.status_code == 201, created.text vm_id = created.json()["uuid"] assert created.json()["result"]["uuid"] == vm_id assert created.json()["status"] == "succeeded" component = engine.registry.components["hk"] assert component.state.users.owns_vm(host_scope(component.manager.profile), vm_id, 41) again = call(client, "POST", path, provisioning, CREATE_BODY) assert again.status_code == 201 and again.json()["uuid"] == vm_id assert len([row for row in services["hk"].calls if row[0] == "create_vm"]) == 1 assert call(client, "POST", path, provisioning, {**CREATE_BODY, "vcpu": 4}).status_code == 409 operation = call(client, "GET", "/internal/v1/operations/order-000001").json() assert operation["status"] == "succeeded" and operation["result"]["uuid"] == vm_id assert operation["safe_to_refund"] is False assert call(client, "GET", "/internal/v1/operations/order-000001", {**USER, "user_id": 42}).status_code == 404 def test_uncertain_creation_never_refunds_or_recreates(boundary): client, engine, store, _, services = boundary path = "/internal/v1/hosts/hk/vms" call(client, "GET", path) services["hk"].create_failure = True actor = {**USER, "can_provision": True, "operation_id": "order-uncertain"} failed = call(client, "POST", path, actor, CREATE_BODY) assert failed.status_code == 502 assert failed.json()["status"] == "uncertain" and not failed.json()["safe_to_refund"] record = json.loads(store.get("operation", "order-uncertain")) assert record["resource_naming"] == "uuid-v1" component = engine.registry.components["hk"] scope = host_scope(component.manager.profile) assert component.state.users.owns_vm(scope, record["vm_uuid"], 41) component.store.set("vm_credential", f"{scope}.{record['vm_uuid']}", "saved-password") component.store.set("ip_claim", f"{scope}.192.0.2.1", json.dumps({"vm_uuid": record["vm_uuid"]})) again = call(client, "POST", path, actor, CREATE_BODY) assert again.status_code == 202 and again.json()["status"] == "uncertain" assert len([row for row in services["hk"].calls if row[0] == "create_vm"]) == 1 services["hk"].absent = True absence_checks = list(services["hk"].absent_checks) operation = call(client, "GET", "/internal/v1/operations/order-uncertain").json() # A momentary absence cannot prove the original unknown writer has stopped. assert operation["status"] == "uncertain" and operation["safe_to_refund"] is False assert services["hk"].absent_checks == absence_checks assert component.state.users.owns_vm(scope, record["vm_uuid"], 41) assert component.store.get("vm_credential", f"{scope}.{record['vm_uuid']}") == "saved-password" assert component.store.get("ip_claim", f"{scope}.192.0.2.1") is not None @pytest.mark.parametrize("error_code", [ "libvirt_execution_uncertain", "libvirt_execution_blocked", ]) def test_isolated_creation_keeps_claims_and_never_refunds_on_temporary_absence( boundary, monkeypatch, error_code): client, engine, store, _, services = boundary path = "/internal/v1/hosts/hk/vms" call(client, "GET", path) service = services["hk"] service.absent = True attempts = [] def uncertain_create(spec): attempts.append(spec) raise VMError("远端执行状态未确认", 504, error_code=error_code) monkeypatch.setattr(service, "create_vm", uncertain_create) actor = {**USER, "can_provision": True, "operation_id": "isolated-create-unknown"} response = call(client, "POST", path, actor, CREATE_BODY) assert response.status_code == 504, response.text assert response.json()["status"] == "uncertain" assert response.json()["error_code"] == error_code assert response.json()["safe_to_refund"] is False assert service.absent_checks == [] record = json.loads(store.get("operation", actor["operation_id"])) component = engine.registry.components["hk"] scope = host_scope(component.manager.profile) credential_key = f"{scope}.{record['vm_uuid']}" component.store.set("vm_credential", credential_key, "saved-test-credential") claim_key = f"{scope}.192.0.2.1" component.store.set("ip_claim", claim_key, json.dumps({"vm_uuid": record["vm_uuid"]})) engine.active_operations.clear() polled = call(client, "GET", f"/internal/v1/operations/{actor['operation_id']}") repeated = call(client, "POST", path, actor, CREATE_BODY) assert polled.status_code == 200 assert repeated.status_code == 202 for result in (polled.json(), repeated.json()): assert result["status"] == "uncertain" assert result["error_code"] == error_code assert result["safe_to_refund"] is False assert "finished_at" not in result assert len(attempts) == 1 assert service.absent_checks == [] assert component.state.users.owns_vm(scope, record["vm_uuid"], USER["user_id"]) assert component.store.get("vm_credential", credential_key) == "saved-test-credential" assert component.store.get("ip_claim", claim_key) is not None def test_legacy_name_collision_operation_uses_legacy_absent_reconciliation(boundary): client, _engine, store, _logs, services = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") services["hk"].absent = True legacy_vm_id = "00000000-0000-4000-8000-000000000002" store.set("operation", "legacy-name-collision", json.dumps({ "operation_id": "legacy-name-collision", "host_id": "hk", "user_id": USER["user_id"], "request_hash": "legacy", "vm_uuid": legacy_vm_id, "vm_name": CREATE_BODY["name"], "kind": "create", "resource": "vms", "method": "POST", "status": "uncertain", "result": None, "error": "虚拟机名称已存在", "safe_to_refund": False, "created_at": 1, })) operation = call( client, "GET", "/internal/v1/operations/legacy-name-collision", USER, ).json() assert operation["status"] == "failed" assert operation["safe_to_refund"] is True assert services["hk"].absent_checks[-1] == ( legacy_vm_id, CREATE_BODY["name"], False, True) @pytest.mark.parametrize("payload", [{"name": "invalid"}, ["not-an-object"], {**CREATE_BODY, "guest_ssh_key_id": 3}]) def test_rejected_create_has_durable_safe_failure_even_when_response_is_lost(boundary, payload): client, engine, _, _, services = boundary path = "/internal/v1/hosts/hk/vms" actor = {**USER, "can_provision": True, "operation_id": "rejected-create"} response = call(client, "POST", path, actor, payload) assert response.status_code == 422 # Simulate a dispatcher losing the POST response and querying after restart. engine.active_operations.clear() saved = call(client, "GET", "/internal/v1/operations/rejected-create").json() assert saved["status"] == "failed" and saved["safe_to_refund"] is True assert call(client, "POST", path, actor, payload).status_code == 422 assert call(client, "POST", path, actor, CREATE_BODY).status_code == 409 assert not any(row[0] == "create_vm" for row in services["hk"].calls) def test_admin_created_vm_also_has_external_owner(boundary): client, engine, *_ = boundary actor = {**SUPER, "can_provision": True, "operation_id": "order-admin"} response = call(client, "POST", "/internal/v1/hosts/hk/vms", actor, CREATE_BODY) assert response.status_code == 201 components = engine.registry.components["hk"] assert components.state.users.owns_vm(host_scope(components.manager.profile), response.json()["uuid"], 1) def test_power_action_has_persistent_idempotency(boundary): client, engine, store, _, services = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") components = engine.registry.components["hk"] components.state.users.assign_vm(host_scope(components.manager.profile), VM_ID, 41) path = f"/internal/v1/hosts/hk/vms/{VM_ID}/actions" assert call(client, "POST", path, USER, {"action": "reboot"}).status_code == 403 actor = {**USER, "operation_id": "reboot-operation"} assert call(client, "POST", path, actor, {"action": "reboot"}).status_code == 200 assert call(client, "POST", path, actor, {"action": "reboot"}).status_code == 200 assert len([row for row in services["hk"].calls if row[0] == "action"]) == 1 assert call(client, "POST", path, actor, {"action": "poweroff"}).status_code == 409 operation = call(client, "GET", "/internal/v1/operations/reboot-operation").json() assert operation["status"] == "succeeded" and operation["safe_to_refund"] is False saved = json.loads(store.get("operation", "reboot-operation")) assert saved["target_action"] == "reboot" assert saved["mutation_started"] is True assert operation["mutation_started"] is True def test_acknowledged_shutdown_releases_target_before_guest_stops(boundary): client, engine, store, _, services = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") components = engine.registry.components["hk"] components.state.users.assign_vm(host_scope(components.manager.profile), VM_ID, 41) service = services["hk"] path = f"/internal/v1/hosts/hk/vms/{VM_ID}/actions" receipt = {"uuid": VM_ID, "action": "shutdown", "command_accepted": True} actor = {**USER, "operation_id": "accepted-shutdown"} response = call(client, "POST", path, actor, {"action": "shutdown"}) assert response.status_code == 200, response.text assert response.json()["status"] == "succeeded" assert response.json()["result"] == receipt assert service.vm["state"] == "running" # This guest ignores shutdown. assert engine.coordinator.operation_claim(actor["operation_id"]) is None # Poll/retry returns the durable receipt even when live observation fails. with patch.object(service, "get_vm", side_effect=VMError("observation unavailable", 502)), \ patch.object(service, "get_vm_state", side_effect=VMError("observation unavailable", 502)): operation = call(client, "GET", "/internal/v1/operations/accepted-shutdown", USER) assert operation.status_code == 200 assert operation.json()["result"] == receipt retried = call(client, "POST", path, actor, {"action": "shutdown"}) assert retried.status_code == 200 assert len([row for row in service.calls if row[0] == "action"]) == 1 forced = call(client, "POST", path, {**USER, "operation_id": "next-poweroff"}, {"action": "poweroff"}) assert forced.status_code == 200, forced.text assert forced.json()["status"] == "succeeded" assert ("action", VM_ID, "poweroff") in service.calls assert engine.coordinator.operation_claim("next-poweroff") is None def test_orphaned_power_action_reconciles_only_the_final_target_state(boundary): client, engine, store, _, services = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") components = engine.registry.components["hk"] components.state.users.assign_vm( host_scope(components.manager.profile), VM_ID, USER["user_id"]) services["hk"].vm["state"] = "stopped" operation = { "operation_id": "orphan-power-state", "host_id": "hk", "user_id": 41, "kind": "mutation", "resource": f"vms/{VM_ID}/actions", "vm_uuid": VM_ID, "status": "uncertain", "result": None, "safe_to_refund": False, "created_at": 1, "actor_can_manage": True, "actor_is_admin": False, "target_action": "shutdown", } store.set("operation", operation["operation_id"], json.dumps(operation)) recovered = call( client, "GET", "/internal/v1/operations/orphan-power-state", USER, ).json() assert recovered["status"] == "succeeded" assert recovered["result"] == { "uuid": VM_ID, "name": "alma-test", "state": "stopped", } assert ("get_vm_state", VM_ID) in services["hk"].calls assert not any(row[0] == "action" for row in services["hk"].calls) services["hk"].vm["state"] = "running" mismatched = {**operation, "operation_id": "orphan-power-mismatch", "target_action": "poweroff"} store.set("operation", mismatched["operation_id"], json.dumps(mismatched)) failed = call( client, "GET", "/internal/v1/operations/orphan-power-mismatch", USER, ).json() assert failed["status"] == "failed" assert "不会自动重放" in failed["error"] assert not any(row[0] == "action" for row in services["hk"].calls) for action in ("reboot", "reset"): ambiguous = { **operation, "operation_id": f"orphan-power-{action}", "target_action": action, } store.set("operation", ambiguous["operation_id"], json.dumps(ambiguous)) unresolved = call( client, "GET", f"/internal/v1/operations/{ambiguous['operation_id']}", USER, ).json() assert unresolved["status"] == "uncertain" assert unresolved["result"] is None assert "不会自动重放" in unresolved["error"] assert not any(row[0] == "action" for row in services["hk"].calls) services["hk"].vm["state"] = "stopped" reboot_off = { **operation, "operation_id": "orphan-power-reboot-off", "target_action": "reboot", } store.set("operation", reboot_off["operation_id"], json.dumps(reboot_off)) unresolved_off = call( client, "GET", "/internal/v1/operations/orphan-power-reboot-off", USER, ).json() assert unresolved_off["status"] == "uncertain" assert unresolved_off["result"] is None assert "不会自动重放" in unresolved_off["error"] assert not any(row[0] == "action" for row in services["hk"].calls) def test_only_signed_system_cleanup_can_delete_stopped_rescue_vm(boundary): client, engine, _, _, services = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") components = engine.registry.components["hk"] scope = host_scope(components.manager.profile) components.state.users.assign_vm(scope, VM_ID, USER["user_id"]) rescue_key = f"{scope}.{VM_ID}" components.store.set("vm_rescue", rescue_key, json.dumps({ "original_xml": "alma-test", "template_id": "0123456789abcdef0123456789abcdef", "volume_name": "panel-rescue-0123456789abcdef0123456789abcdef.iso", "status": "active", })) services["hk"].vm.update( state="shutoff", rescue_mode=True, rescue_staged=True, rescue_volume_name="panel-rescue-0123456789abcdef0123456789abcdef.iso", ) path = f"/internal/v1/hosts/hk/vms/{VM_ID}?delete_disks=true" ordinary = call( client, "DELETE", path, {**USER, "operation_id": "ordinary-rescue-delete", "system_cleanup": False}, ) assert ordinary.status_code == 409 assert components.store.get("vm_rescue", rescue_key) is not None assert not any(item[0] == "delete_vm" for item in services["hk"].calls) cleanup = call( client, "DELETE", path, {**USER, "operation_id": "expiry-rescue-delete", "system_cleanup": True}, ) assert cleanup.status_code == 200, cleanup.text assert cleanup.json()["status"] == "succeeded" assert ("delete_vm", VM_ID, True) in services["hk"].calls assert services["hk"].calls[-1] == ("delete_firewall_filter", VM_ID) assert components.store.get("vm_rescue", rescue_key) is None assert components.state.users.owner_of_vm(scope, VM_ID) is None def test_deleted_vm_reconciles_exact_snapshot_and_control_plane_cleanup( boundary): client, engine, store, _logs, services = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") components = engine.registry.components["hk"] profile = components.manager.profile scope = host_scope(profile) components.state.users.assign_vm(scope, VM_ID, USER["user_id"]) identity_key = f"{scope}.{VM_ID}" components.store.set("vm_credential", identity_key, json.dumps({ "auth_mode": "ssh_key", "password": None, "generation": "g1", })) components.store.set("vm_rescue", identity_key, json.dumps({ "original_xml": "alma-test", "template_id": "0123456789abcdef0123456789abcdef", "volume_name": "panel-rescue-0123456789abcdef0123456789abcdef.iso", "status": "active", })) claim_key = f"{scope}.8.8.8.4" components.store.set("ip_claim", claim_key, json.dumps({ "vm_uuid": VM_ID, })) historical_failed_key = "d" * 32 ready_snapshot_keys = ["e" * 32, "f" * 32] snapshot_keys = [historical_failed_key, *ready_snapshot_keys] volume_names = { key: snapshot_volume_name(VM_ID, key) for key in snapshot_keys } catalog_keys = {key: f"{VM_ID}:{key}" for key in snapshot_keys} for created_at, key in enumerate(snapshot_keys, start=1): components.store.set("vm_snapshot", catalog_keys[key], json.dumps({ "snapshot_key": key, "vm_id": VM_ID, "name": f"清退恢复点-{created_at}", "state": ("failed" if key == historical_failed_key else "ready"), "consistency": "crash", "volume_name": volume_names[key], "created_at": created_at, })) service = services["hk"] service.vm.update( state="shutoff", rescue_mode=True, rescue_staged=True, ) failed_once = False def flaky_snapshot_delete(vm_id, snapshot_key, volume_name, *, allow_missing_vm=False): nonlocal failed_once service.calls.append(( "delete_snapshot_volume", vm_id, snapshot_key, volume_name, allow_missing_vm, )) if snapshot_key == historical_failed_key and not failed_once: failed_once = True raise VMError("快照卷删除响应丢失", 502) return {"snapshot_key": snapshot_key, "deleted": True} service.delete_snapshot_volume = flaky_snapshot_delete operation_id = "delete-snapshot-cleanup" actor = { **USER, "operation_id": operation_id, "system_cleanup": True, } path = f"/internal/v1/hosts/hk/vms/{VM_ID}?delete_disks=true" first = call(client, "DELETE", path, actor) assert first.status_code == 503, first.text assert first.json()["status"] == "uncertain" assert first.json()["error_code"] == ( "vm_delete_snapshot_cleanup_pending") assert all(volume_name not in first.text for volume_name in volume_names.values()) saved = json.loads(store.get("operation", operation_id)) assert saved["cleanup_pending"] == "vm_delete" assert saved["vm_delete_committed"] is True assert saved["delete_cleanup_checkpoint"] == { "version": 1, "vm_uuid": VM_ID, "snapshots": [ { "snapshot_key": key, "volume_name": volume_names[key], } for key in snapshot_keys ], "system_cleanup": True, } assert len([item for item in service.calls if item[0] == "delete_vm"]) == 1 assert all(components.store.get( "vm_snapshot", catalog_keys[key]) is None for key in ready_snapshot_keys) assert components.store.get( "vm_snapshot", catalog_keys[historical_failed_key]) is not None assert components.state.users.owner_of_vm(scope, VM_ID) == USER["user_id"] assert components.store.get("vm_credential", identity_key) is not None assert components.store.get("vm_rescue", identity_key) is not None assert components.store.get("ip_claim", claim_key) is not None recovered = call( client, "GET", f"/internal/v1/operations/{operation_id}", actor) assert recovered.status_code == 200, recovered.text assert recovered.json()["status"] == "succeeded" assert recovered.json()["result"] == { "uuid": VM_ID, "deleted": True, "deleted_snapshots": snapshot_keys, "public_ip_claims_released": True, "firewall_cleaned": True, } assert all(volume_name not in recovered.text for volume_name in volume_names.values()) assert len([item for item in service.calls if item[0] == "delete_vm"]) == 1 assert all(components.store.get("vm_snapshot", catalog_keys[key]) is None for key in snapshot_keys) assert components.state.users.owner_of_vm(scope, VM_ID) is None assert components.store.get("vm_credential", identity_key) is None assert components.store.get("vm_rescue", identity_key) is None assert components.store.get("ip_claim", claim_key) is None def test_deleted_vm_reconciles_postcommit_control_plane_failure(boundary): client, engine, _store, _logs, services = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") components = engine.registry.components["hk"] scope = host_scope(components.manager.profile) identity_key = f"{scope}.{VM_ID}" components.state.users.assign_vm(scope, VM_ID, USER["user_id"]) components.store.set("vm_credential", identity_key, json.dumps({ "auth_mode": "ssh_key", "password": None, "generation": "g1", })) claim_key = f"{scope}.8.8.8.4" components.store.set("ip_claim", claim_key, json.dumps({ "vm_uuid": VM_ID, })) service = services["hk"] service.vm["state"] = "shutoff" original_delete = components.store.delete failed_once = False def flaky_delete(kind, key): nonlocal failed_once if kind == "vm_credential" and not failed_once: failed_once = True raise DatabaseError("credential delete interrupted") return original_delete(kind, key) components.store.delete = flaky_delete operation_id = "delete-control-cleanup" actor = { **USER, "operation_id": operation_id, "system_cleanup": True, } path = f"/internal/v1/hosts/hk/vms/{VM_ID}?delete_disks=true" first = call(client, "DELETE", path, actor) assert first.status_code == 503, first.text assert first.json()["error_code"] == ( "vm_delete_snapshot_cleanup_pending") assert len([item for item in service.calls if item[0] == "delete_vm"]) == 1 assert components.store.get("vm_credential", identity_key) is not None assert components.store.get("ip_claim", claim_key) is not None components.store.delete = original_delete recovered = call( client, "GET", f"/internal/v1/operations/{operation_id}", actor) assert recovered.status_code == 200 assert recovered.json()["status"] == "succeeded" assert len([item for item in service.calls if item[0] == "delete_vm"]) == 1 assert components.state.users.owner_of_vm(scope, VM_ID) is None assert components.store.get("vm_credential", identity_key) is None assert components.store.get("ip_claim", claim_key) is None def test_deleted_vm_volume_cleanup_reconciles_v2_manifest(boundary): client, engine, store, _logs, services = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") components = engine.registry.components["hk"] components.state.users.assign_vm( host_scope(components.manager.profile), VM_ID, USER["user_id"]) service = services["hk"] service.vm["state"] = "shutoff" service.delete_vm_error = VMError( "虚拟机定义已删除,但磁盘删除失败", 503, error_code="vm_delete_volume_cleanup_pending", ) manifest = { "vm_uuid": VM_ID, # Reinstall intentionally creates a fresh UUID seed unrelated to the # VM UUID. The persisted delete checkpoint must accept that identity. "seed_volume": "panel-seed-11111111-2222-4333-8444-555555555555.iso", "primary_volume": "panel-restore-0123456789abcdef0123456789abcdef0123456789abcdef.qcow2", } service.delete_vm_cleanup_manifest = lambda vm_id: manifest.copy() cleanup_calls = [] def cleanup_deleted_vm_volumes(vm_id, resources): cleanup_calls.append((vm_id, resources)) return {"deleted": [resources["primary_volume"]]} service.cleanup_deleted_vm_volumes = cleanup_deleted_vm_volumes operation_id = "delete-volume-cleanup-v2" actor = {**USER, "operation_id": operation_id} path = f"/internal/v1/hosts/hk/vms/{VM_ID}?delete_disks=true" first = call(client, "DELETE", path, actor) assert first.status_code == 503, first.text assert first.json()["error_code"] == "vm_delete_volume_cleanup_pending" saved = json.loads(store.get("operation", operation_id)) assert saved["cleanup_pending"] == "vm_delete" assert saved["vm_delete_committed"] is True assert saved["delete_cleanup_checkpoint"]["version"] == 2 assert saved["delete_cleanup_checkpoint"]["resources"] == { "seed_volume": manifest["seed_volume"], "primary_volume": manifest["primary_volume"], } recovered = call(client, "GET", f"/internal/v1/operations/{operation_id}", actor) assert recovered.status_code == 200, recovered.text assert recovered.json()["status"] == "succeeded" assert cleanup_calls == [( VM_ID, saved["delete_cleanup_checkpoint"]["resources"])] assert len([item for item in service.calls if item[0] == "delete_vm"]) == 1 def test_isolated_delete_cleanup_keeps_claims_and_stops_automatic_cleanup( boundary, monkeypatch): client, engine, store, _, services = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") component = engine.registry.components["hk"] scope = host_scope(component.manager.profile) component.state.users.assign_vm(scope, VM_ID, USER["user_id"]) claim_key = f"{scope}.192.0.2.1" component.store.set("ip_claim", claim_key, json.dumps({"vm_uuid": VM_ID})) operation_id = "isolated-delete-cleanup" store.set("operation", operation_id, json.dumps({ "operation_id": operation_id, "host_id": "hk", "user_id": 41, "kind": "mutation", "resource": f"vms/{VM_ID}", "method": "DELETE", "vm_uuid": VM_ID, "status": "uncertain", "safe_to_refund": False, "created_at": 1, "cleanup_pending": "vm_delete", "vm_delete_committed": True, "delete_ip_claim_keys": [claim_key], "delete_cleanup_checkpoint": { "version": 1, "vm_uuid": VM_ID, "snapshots": [], "system_cleanup": False, }, })) attempts = [] def uncertain_cleanup(vm_id): attempts.append(vm_id) raise VMError("过滤器删除结果未确认", 504, error_code="libvirt_execution_uncertain") monkeypatch.setattr(services["hk"], "delete_firewall_filter", uncertain_cleanup) for _ in range(2): response = call(client, "GET", f"/internal/v1/operations/{operation_id}") assert response.status_code == 200, response.text assert response.json()["status"] == "uncertain" assert response.json()["error_code"] == "libvirt_execution_uncertain" assert attempts == [VM_ID] assert component.state.users.owns_vm(scope, VM_ID, USER["user_id"]) assert component.store.get("ip_claim", claim_key) is not None saved = json.loads(store.get("operation", operation_id)) assert saved["cleanup_pending"] == "vm_delete" assert saved["delete_cleanup_checkpoint"]["vm_uuid"] == VM_ID @pytest.mark.parametrize(("committed", "checkpoint_volume"), [ (False, None), (True, "panel-snapshot-tampered.qcow2"), ]) def test_vm_delete_cleanup_requires_durable_commit_and_exact_identity( boundary, committed, checkpoint_volume): client, _engine, store, _logs, services = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") snapshot_key = "c" * 32 volume_name = snapshot_volume_name(VM_ID, snapshot_key) operation_id = ( "delete-cleanup-tampered" if checkpoint_volume else "delete-cleanup-not-committed" ) operation = { "operation_id": operation_id, "host_id": "hk", "user_id": USER["user_id"], "vm_uuid": VM_ID, "kind": "mutation", "resource": f"vms/{VM_ID}", "method": "DELETE", "status": "uncertain", "result": None, "safe_to_refund": False, "cleanup_pending": "vm_delete", "delete_ip_claim_keys": [], "delete_cleanup_checkpoint": { "version": 1, "vm_uuid": VM_ID, "snapshots": [{ "snapshot_key": snapshot_key, "volume_name": checkpoint_volume or volume_name, }], "system_cleanup": True, }, "created_at": 1, } if committed: operation["vm_delete_committed"] = True store.set("operation", operation_id, json.dumps(operation)) before = list(services["hk"].calls) response = call( client, "GET", f"/internal/v1/operations/{operation_id}", USER, ) assert response.status_code == 200 assert response.json()["status"] == "uncertain" assert "delete_cleanup_checkpoint" not in response.text assert volume_name not in response.text if checkpoint_volume is not None: assert checkpoint_volume not in response.text assert services["hk"].calls == before assert not any(item[0] in {"delete_vm", "delete_snapshot_volume"} for item in services["hk"].calls[len(before):]) @pytest.mark.parametrize(("route", "method", "target"), [ ("service-suspend", "service_suspend", {"state": "shutoff", "autostart": False}), ("service-resume", "service_resume", {"autostart": True}), ("network-suspend", "network_suspend", {"network_suspended": True}), ("network-resume", "network_resume", {"network_suspended": False}), ]) def test_private_target_state_operations_are_owned_idempotent_mutations( boundary, route, method, target): client, engine, _, _, services = boundary base = f"/internal/v1/hosts/hk/vms/{VM_ID}/{route}" call(client, "GET", "/internal/v1/hosts/hk/vms") component = engine.registry.components["hk"] component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41) assert call(client, "POST", base, USER, {}).status_code == 403 assert call(client, "POST", base, {**USER, "can_manage": False, "operation_id": f"blocked-{route}"}, {}).status_code == 403 assert call(client, "POST", base, {**USER, "user_id": 42, "operation_id": f"foreign-{route}"}, {}).status_code == 404 actor = {**USER, "operation_id": f"target-{route}"} first = call(client, "POST", base, actor, {}) assert first.status_code == 200, first.text assert first.json()["status"] == "succeeded" assert all(first.json()[key] == value for key, value in target.items()) repeated = call(client, "POST", base, actor, {}) assert repeated.status_code == 200 assert len([item for item in services["hk"].calls if item[0] == method]) == 1 @pytest.mark.parametrize(("route", "method"), [ ("service-suspend", "service_suspend"), ("service-resume", "service_resume"), ("network-suspend", "network_suspend"), ("network-resume", "network_resume"), ]) @pytest.mark.parametrize("rescue_marker", ["rescue_mode", "rescue_staged"]) def test_private_target_state_rejects_rescue_marker_before_mutation( boundary, route, method, rescue_marker): client, engine, _store, _logs, services = boundary base = f"/internal/v1/hosts/hk/vms/{VM_ID}/{route}" call(client, "GET", "/internal/v1/hosts/hk/vms") component = engine.registry.components["hk"] component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41) services["hk"].vm[rescue_marker] = True actor = {**USER, "operation_id": f"rescue-blocked-{route}-{rescue_marker}"} body = {"start_if_suspended": False} if route == "service-resume" else {} response = call(client, "POST", base, actor, body) assert response.status_code == 409, response.text assert response.json()["status"] == "failed" assert not any(item[0] == method for item in services["hk"].calls) @pytest.mark.parametrize(("route", "method", "drift"), [ ("service-suspend", "service_suspend", {"state": "running", "autostart": True}), ("service-resume", "service_resume", {"autostart": False}), ("network-suspend", "network_suspend", {"network_suspended": False}), ("network-resume", "network_resume", {"network_suspended": True}), ]) def test_target_state_operation_replays_after_lost_response( boundary, route, method, drift): client, engine, store, _, services = boundary base = f"/internal/v1/hosts/hk/vms/{VM_ID}/{route}" call(client, "GET", "/internal/v1/hosts/hk/vms") component = engine.registry.components["hk"] component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41) actor = {**USER, "operation_id": f"replay-{route}"} assert call(client, "POST", base, actor, {}).status_code == 200 saved = json.loads(store.get("operation", actor["operation_id"])) saved.update(status="uncertain", result=None, error="response lost") store.set("operation", actor["operation_id"], json.dumps(saved)) services["hk"].vm.update(drift) engine.active_operations.clear() replayed = call(client, "GET", f"/internal/v1/operations/{actor['operation_id']}", actor) assert replayed.status_code == 200, replayed.text assert replayed.json()["status"] == "succeeded" assert len([item for item in services["hk"].calls if item[0] == method]) == 2 saved = json.loads(store.get("operation", actor["operation_id"])) saved.update(status="uncertain", result=None, error="response lost again") store.set("operation", actor["operation_id"], json.dumps(saved)) services["hk"].vm.update(drift) replayed_post = call(client, "POST", base, actor, {}) assert replayed_post.status_code == 200, replayed_post.text assert replayed_post.json()["status"] == "succeeded" assert len([item for item in services["hk"].calls if item[0] == method]) == 3 @pytest.mark.parametrize(("route", "method"), [ ("service-suspend", "service_suspend"), ("service-resume", "service_resume"), ("network-suspend", "network_suspend"), ("network-resume", "network_resume"), ]) @pytest.mark.parametrize("error_code", [ "libvirt_execution_uncertain", "libvirt_execution_blocked", ]) def test_isolated_target_failure_is_durable_and_never_replayed( boundary, monkeypatch, route, method, error_code): client, engine, store, _, services = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") component = engine.registry.components["hk"] component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41) attempts = [] def uncertain_target(vm_id, **options): attempts.append((vm_id, options)) raise VMError("远端执行状态未确认", 504, error_code=error_code) monkeypatch.setattr(services["hk"], method, uncertain_target) path = f"/internal/v1/hosts/hk/vms/{VM_ID}/{route}" actor = {**USER, "operation_id": f"isolated-{route}"} first = call(client, "POST", path, actor, {}) assert first.status_code == 504, first.text assert first.json()["error_code"] == error_code engine.active_operations.clear() before = list(services["hk"].calls) polled = call(client, "GET", f"/internal/v1/operations/{actor['operation_id']}") repeated = call(client, "POST", path, actor, {}) for response in (first, polled, repeated): assert response.json()["status"] == "uncertain" assert response.json()["safe_to_refund"] is False assert response.json()["error_code"] == error_code saved = json.loads(store.get("operation", actor["operation_id"])) assert saved["error_code"] == error_code assert saved["mutation_started"] is True assert "finished_at" not in saved assert len(attempts) == 1 assert services["hk"].calls == before def test_target_recovery_persists_new_execution_uncertainty_before_next_poll( boundary, monkeypatch): client, engine, store, _, services = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") component = engine.registry.components["hk"] component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41) actor = {**USER, "operation_id": "isolated-recovery-target"} path = f"/internal/v1/hosts/hk/vms/{VM_ID}/service-resume" assert call(client, "POST", path, actor, {}).status_code == 200 saved = json.loads(store.get("operation", actor["operation_id"])) saved.update(status="uncertain", result=None) store.set("operation", actor["operation_id"], json.dumps(saved)) attempts = [] def uncertain_resume(vm_id, **options): attempts.append(vm_id) raise VMError("远端执行状态未确认", 504, error_code="libvirt_execution_uncertain") monkeypatch.setattr(services["hk"], "service_resume", uncertain_resume) for _ in range(2): recovered = call(client, "GET", f"/internal/v1/operations/{actor['operation_id']}") assert recovered.json()["status"] == "uncertain" assert recovered.json()["error_code"] == "libvirt_execution_uncertain" assert attempts == [VM_ID] saved = json.loads(store.get("operation", actor["operation_id"])) assert saved["error_code"] == "libvirt_execution_uncertain" assert "finished_at" not in saved def test_execution_failure_during_network_preflight_is_not_a_safe_terminal_failure( boundary, monkeypatch): client, engine, store, _, _ = boundary async def blocked_preflight(*args): raise VMError("宿主机有未决远端操作", 503, error_code="libvirt_execution_blocked") monkeypatch.setattr(engine, "_capture_network_transition", blocked_preflight) actor = {**USER, "operation_id": "isolated-network-preflight"} response = call( client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/network", actor, {}, ) assert response.status_code == 503 assert response.json()["error_code"] == "libvirt_execution_blocked" saved = json.loads(store.get("operation", actor["operation_id"])) assert saved["network_mutation_started"] is False assert saved["status"] == "uncertain" assert saved["safe_to_refund"] is False polled = call(client, "GET", f"/internal/v1/operations/{actor['operation_id']}") assert polled.json()["status"] == "uncertain" assert polled.json()["error_code"] == "libvirt_execution_blocked" @pytest.mark.parametrize("error_code", [ "libvirt_execution_timeout", "libvirt_execution_stopped", ]) def test_read_execution_error_is_preserved_in_operation_response( boundary, monkeypatch, error_code): client, engine, store, _, services = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") component = engine.registry.components["hk"] component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41) def unavailable_action(*args): raise VMError("执行器暂时不可用", 503, error_code=error_code) monkeypatch.setattr(services["hk"], "action", unavailable_action) actor = {**USER, "operation_id": "isolated-power-read-error"} response = call(client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/actions", actor, {"action": "start"}) assert response.status_code == 503 assert response.json()["error_code"] == error_code saved = json.loads(store.get("operation", actor["operation_id"])) assert saved["error_code"] == error_code @pytest.mark.parametrize(("resource", "kind"), [ (f"vms/{VM_ID}/reinstall", "reinstall_job"), (f"vms/{VM_ID}/rescue", "rescue_job"), (f"vms/{VM_ID}/snapshots", "snapshot_job"), ]) def test_isolated_background_job_does_not_become_terminal_after_restart( boundary, resource, kind): client, engine, store, _, services = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") component = engine.registry.components["hk"] operation_id = f"isolated-job-{kind}" job_id = "isolated-background-job" store.set("operation", operation_id, json.dumps({ "operation_id": operation_id, "host_id": "hk", "user_id": 41, "kind": "mutation", "resource": resource, "vm_uuid": VM_ID, "status": "pending", "result": None, "safe_to_refund": False, "created_at": 1, "job_id": job_id, })) component.store.set(kind, job_id, json.dumps({ "operation_id": operation_id, "owner": "php:41", "vm_id": VM_ID, "status": "uncertain", "phase": "面板重启后等待人工核查", "error_code": "libvirt_execution_uncertain", "error": "远端执行状态未确认", "credential_generation": "preserved", })) before = list(services["hk"].calls) response = call(client, "GET", f"/internal/v1/operations/{operation_id}") assert response.status_code == 200, response.text assert response.json()["status"] == "uncertain" assert response.json()["error_code"] == "libvirt_execution_uncertain" assert services["hk"].calls == before saved = json.loads(store.get("operation", operation_id)) assert saved["status"] == "uncertain" assert saved["error_code"] == "libvirt_execution_uncertain" assert "finished_at" not in saved @pytest.mark.parametrize(("marker", "expected_status"), [ ("live", "failed"), ("active", "failed"), ("queued", "uncertain"), ("failed", "uncertain"), ("corrupt", "uncertain"), ]) def test_target_state_replay_fails_closed_during_rescue( boundary, marker, expected_status): client, engine, store, _, services = boundary route, method = "network-resume", "network_resume" base = f"/internal/v1/hosts/hk/vms/{VM_ID}/{route}" call(client, "GET", "/internal/v1/hosts/hk/vms") component = engine.registry.components["hk"] component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41) actor = {**USER, "operation_id": f"replay-rescue-{marker}"} assert call(client, "POST", base, actor, {}).status_code == 200 before = len([item for item in services["hk"].calls if item[0] == method]) saved = json.loads(store.get("operation", actor["operation_id"])) saved.update(status="uncertain", result=None, error="response lost") store.set("operation", actor["operation_id"], json.dumps(saved)) if marker == "live": services["hk"].vm["rescue_mode"] = True elif marker == "corrupt": component.store.set( "vm_rescue", f"{host_scope(component.manager.profile)}.{VM_ID}", "not-json", ) elif marker in {"active", "queued", "failed"}: component.store.set( "vm_rescue", f"{host_scope(component.manager.profile)}.{VM_ID}", json.dumps({"vm_id": VM_ID, "status": marker}), ) engine.active_operations.clear() replayed = call( client, "GET", f"/internal/v1/operations/{actor['operation_id']}", actor, ) assert replayed.status_code == 200, replayed.text assert replayed.json()["status"] == expected_status assert len([item for item in services["hk"].calls if item[0] == method]) == before def test_non_reconciling_operation_lookup_never_replays_target_state(boundary): client, engine, store, _, services = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") component = engine.registry.components["hk"] component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41) path = f"/internal/v1/hosts/hk/vms/{VM_ID}/network-suspend" actor = {**USER, "operation_id": "peek-target-state"} assert call(client, "POST", path, actor, {}).status_code == 200 saved = json.loads(store.get("operation", actor["operation_id"])) saved.update(status="uncertain", result=None, error="response lost") store.set("operation", actor["operation_id"], json.dumps(saved)) services["hk"].vm["network_suspended"] = False engine.active_operations.clear() before = len(services["hk"].calls) peeked = call(client, "GET", "/internal/v1/operations/peek-target-state?reconcile=false", actor) assert peeked.status_code == 200 assert peeked.json()["status"] == "uncertain" assert peeked.json()["executor_active"] is False assert len(services["hk"].calls) == before assert services["hk"].vm["network_suspended"] is False claim = engine.coordinator.claim( operation_id=actor["operation_id"], resources=(f"vm:hk:{VM_ID}",)) active = call(client, "GET", "/internal/v1/operations/peek-target-state?reconcile=false", actor) assert active.status_code == 200 assert active.json()["executor_active"] is True assert len(services["hk"].calls) == before assert engine.coordinator.release(claim) def test_target_state_lock_serializes_same_vm_and_not_other_vms(boundary): _, engine, *_ = boundary async def exercise(): first = engine._target_lock("hk", VM_ID) same = engine._target_lock("hk", VM_ID) other = engine._target_lock("hk", "22222222-2222-4222-8222-222222222222") assert first is same assert first is not other entered = [] owner_entered = asyncio.Event() release = asyncio.Event() async def owner(): async with first: entered.append("owner") owner_entered.set() await release.wait() entered.append("owner-finished") async def waiter(): async with same: entered.append("waiter") owner_task = asyncio.create_task(owner()) await asyncio.wait_for(owner_entered.wait(), timeout=1) waiter_task = asyncio.create_task(waiter()) await asyncio.sleep(0) assert entered == ["owner"] release.set() await asyncio.gather(owner_task, waiter_task) assert entered == ["owner", "owner-finished", "waiter"] errors = [] def run_in_isolated_loop(): try: asyncio.run(exercise()) except BaseException as exc: # propagate assertion failures to pytest errors.append(exc) thread = threading.Thread(target=run_in_isolated_loop) thread.start() thread.join(timeout=5) assert not thread.is_alive() if errors: raise errors[0] def test_expiry_target_retries_after_inflight_user_mutation_on_same_vm(boundary): client, engine, _, _, services = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") components = engine.registry.components["hk"] components.state.users.assign_vm( host_scope(components.manager.profile), VM_ID, USER["user_id"]) service = services["hk"] action_entered = threading.Event() release_action = threading.Event() network_entered = threading.Event() original_action = service.action original_network_suspend = service.network_suspend def slow_action(vm_id, action): action_entered.set() assert release_action.wait(timeout=5), "test did not release user mutation" return original_action(vm_id, action) def observed_network_suspend(vm_id): network_entered.set() return original_network_suspend(vm_id) service.action = slow_action service.network_suspend = observed_network_suspend responses = {} def user_request(): responses["action"] = call( client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/actions", {**USER, "operation_id": "concurrent-user-action"}, {"action": "reset"}) def policy_request(): responses["policy"] = call( client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/network-suspend", {**USER, "operation_id": "concurrent-expiry-policy"}, {}) first = threading.Thread(target=user_request) first.start() assert action_entered.wait(timeout=5) second = threading.Thread(target=policy_request) second.start() try: second.join(timeout=2) assert not second.is_alive() assert responses["policy"].status_code == 409 assert responses["policy"].json()["error_code"] == "coordination_busy" assert not network_entered.is_set() finally: release_action.set() first.join(timeout=5) assert not first.is_alive() and not second.is_alive() assert responses["action"].status_code == 200 policy_request() assert responses["policy"].status_code == 200 assert network_entered.is_set() mutations = [item[0] for item in service.calls if item[0] in {"action", "network_suspend"}] assert mutations == ["action", "network_suspend"] assert service.vm["network_suspended"] is True def test_target_state_reconciliation_does_not_bypass_original_owner(boundary): client, engine, store, _, services = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") component = engine.registry.components["hk"] component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41) store.set("operation", "unsafe-target", json.dumps({ "operation_id": "unsafe-target", "host_id": "hk", "user_id": 42, "kind": "mutation", "resource": f"vms/{VM_ID}/network-suspend", "vm_uuid": VM_ID, "status": "uncertain", "result": None, "actor_can_manage": True, "actor_is_admin": False, "safe_to_refund": False, })) response = call(client, "GET", "/internal/v1/operations/unsafe-target", {**USER, "user_id": 42}) assert response.status_code == 200 assert response.json()["status"] == "failed" assert not any(item[0] == "network_suspend" for item in services["hk"].calls) def test_async_mutation_operation_survives_lost_job_response_and_does_not_retry(boundary): client, engine, store, *_ = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") components = engine.registry.components["hk"] job = {"operation_id": "lost-rescue-job", "owner": "php:41", "vm_id": VM_ID, "status": "running", "phase": "正在救援"} components.store.set("rescue_job", "job-abc", json.dumps(job)) operation = {"operation_id": "lost-rescue-job", "host_id": "hk", "user_id": 41, "kind": "mutation", "resource": f"vms/{VM_ID}/rescue", "vm_uuid": VM_ID, "status": "uncertain", "result": None, "safe_to_refund": False} store.set("operation", "lost-rescue-job", json.dumps(operation)) recovered = call(client, "GET", "/internal/v1/operations/lost-rescue-job").json() assert recovered["status"] == "running" and recovered["result"]["vm_id"] == VM_ID job.update(status="completed", phase="救援系统启动") components.store.set("rescue_job", "job-abc", json.dumps(job)) completed = call(client, "GET", "/internal/v1/operations/lost-rescue-job").json() assert completed["status"] == "succeeded" assert not completed["safe_to_refund"] def test_snapshot_mutation_operation_returns_only_safe_result(boundary): client, engine, _store, _logs, services = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") components = engine.registry.components["hk"] components.state.users.assign_vm( host_scope(components.manager.profile), VM_ID, USER["user_id"]) services["hk"].vm.update( managed=True, os_source="cloud_template", rescue_mode=False, rescue_staged=False, ) actor = {**USER, "operation_id": "snapshot-create-0001"} path = f"/internal/v1/hosts/hk/vms/{VM_ID}/snapshots" queued = call(client, "POST", path, actor, { "snapshot_key": "a" * 32, "name": "发布前恢复点", "consistency_mode": "crash", }) assert queued.status_code == 202, queued.text for _ in range(100): operation = call( client, "GET", "/internal/v1/operations/snapshot-create-0001", actor, ).json() if operation["status"] != "running" and operation["status"] != "pending": break time.sleep(0.01) assert operation["status"] == "succeeded" assert operation["result"]["snapshot_key"] == "a" * 32 assert operation["result"]["created"] is True assert operation["result"]["consistency"] == "crash" assert "volume_name" not in operation["result"] def test_snapshot_qga_failure_code_survives_operation_poll(boundary): client, engine, _store, _logs, services = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") components = engine.registry.components["hk"] components.state.users.assign_vm( host_scope(components.manager.profile), VM_ID, USER["user_id"]) services["hk"].vm.update( managed=True, os_source="cloud_template", rescue_mode=False, rescue_staged=False, ) services["hk"].snapshot_create_error = VMError( "QEMU Guest Agent 无法冻结文件系统,快照未创建", 409, error_code="snapshot_guest_agent_unavailable", ) actor = {**USER, "operation_id": "snapshot-create-qga1"} path = f"/internal/v1/hosts/hk/vms/{VM_ID}/snapshots" queued = call(client, "POST", path, actor, { "snapshot_key": "b" * 32, "name": "一致性恢复点", "consistency_mode": "filesystem", }) assert queued.status_code == 202, queued.text for _ in range(100): operation = call( client, "GET", "/internal/v1/operations/snapshot-create-qga1", actor, ).json() if operation["status"] not in {"running", "pending"}: break time.sleep(0.01) assert operation["status"] == "failed" assert operation["error_code"] == "snapshot_guest_agent_unavailable" def test_snapshot_restart_recovers_freeze_and_orphan_transition(boundary): client, _engine, store, _logs, services = boundary scoped = HostStore(store, "hk") freeze_key = "c" * 32 orphan_key = "d" * 32 for snapshot_key in (freeze_key, orphan_key): scoped.set("vm_snapshot", f"{VM_ID}:{snapshot_key}", json.dumps({ "snapshot_key": snapshot_key, "vm_id": VM_ID, "name": "中断恢复点", "state": "creating", "consistency": "filesystem", "volume_name": snapshot_volume_name(VM_ID, snapshot_key), "created_at": 1, })) scoped.set("snapshot_job", "snapshot-interrupted", json.dumps({ "operation_id": "snapshot-interrupted", "owner": "php:41", "vm_id": VM_ID, "snapshot_key": freeze_key, "action": "create", "consistency": "filesystem", "guest_freeze_state": "attempted", "status": "uncertain", "phase": "正在通过 QEMU Guest Agent 冻结访客文件系统", })) assert call(client, "GET", "/internal/v1/hosts/hk/vms").status_code == 200 interrupted = json.loads(scoped.get( "snapshot_job", "snapshot-interrupted")) assert interrupted["status"] == "uncertain" assert interrupted["guest_freeze_state"] == "recovered" assert interrupted["freeze_recovery"] == "confirmed" assert ("thaw_guest_filesystems", VM_ID) in services["hk"].calls record = json.loads(scoped.get( "vm_snapshot", f"{VM_ID}:{freeze_key}")) assert record["state"] == "uncertain" assert scoped.get( "vm_snapshot", f"{VM_ID}:{orphan_key}") is None @pytest.mark.parametrize(("initial_state", "expected_state"), [ ("creating", None), ("restoring", "ready"), ("deleting", "ready"), ]) def test_snapshot_registry_without_durable_job_rolls_back_before_mutation( boundary, initial_state, expected_state): client, _engine, store, _logs, services = boundary scoped = HostStore(store, "hk") snapshot_key = { "creating": "b", "restoring": "c", "deleting": "d", }[initial_state] * 32 scoped.set("vm_snapshot", f"{VM_ID}:{snapshot_key}", json.dumps({ "snapshot_key": snapshot_key, "vm_id": VM_ID, "name": "尚未入队恢复点", "state": initial_state, "consistency": "crash", "volume_name": snapshot_volume_name(VM_ID, snapshot_key), "created_at": 1, })) assert call(client, "GET", "/internal/v1/hosts/hk/vms").status_code == 200 raw = scoped.get("vm_snapshot", f"{VM_ID}:{snapshot_key}") if expected_state is None: assert raw is None else: assert json.loads(raw)["state"] == expected_state assert not any(call_[0] in { "create_live_snapshot", "restore_live_snapshot", "delete_snapshot_volume", } for call_ in services["hk"].calls) @pytest.mark.parametrize(("action", "initial_state", "expected_state"), [ ("create", "creating", None), ("restore", "restoring", "ready"), ("delete", "deleting", "ready"), ]) def test_snapshot_cancel_tombstone_finishes_catalog_rollback_after_restart( boundary, action, initial_state, expected_state): client, _engine, store, _logs, services = boundary scoped = HostStore(store, "hk") snapshot_key = { "create": "e", "restore": "f", "delete": "a", }[action] * 32 catalog_key = f"{VM_ID}:{snapshot_key}" scoped.set("vm_snapshot", catalog_key, json.dumps({ "snapshot_key": snapshot_key, "vm_id": VM_ID, "name": "取消中的排队恢复点", "state": initial_state, "consistency": "crash", "volume_name": snapshot_volume_name(VM_ID, snapshot_key), "created_at": 1, })) job_id = f"cancelled-before-start-{action}" operation_id = f"snapshot-cancel-restart-{action}" scoped.set("snapshot_job", job_id, json.dumps({ "job_id": job_id, "operation_id": operation_id, "scope": host_scope({"host_id": "hk"}), "owner": "php:41", "vm_id": VM_ID, "snapshot_key": snapshot_key, "action": action, "consistency": "crash", "status": "queued", "phase": "正在取消尚未开始的排队快照", "created_at": 1, "cancelled_before_start": True, "cancel_reason": "network_suspend", "cancel_requested_at": 1, })) resource = f"vms/{VM_ID}/snapshots" method = "POST" if action != "create": resource += f"/{snapshot_key}" if action == "restore": resource += "/restore" else: method = "DELETE" store.set("operation", operation_id, json.dumps({ "operation_id": operation_id, "host_id": "hk", "user_id": USER["user_id"], "kind": "mutation", "resource": resource, "method": method, "vm_uuid": VM_ID, "status": "pending", "job_id": job_id, "result": {"job_id": job_id, "status": "queued"}, "safe_to_refund": False, "created_at": 1, })) assert call(client, "GET", "/internal/v1/hosts/hk/vms").status_code == 200 saved_job = json.loads(scoped.get("snapshot_job", job_id)) assert saved_job["status"] == "failed" assert saved_job["cancelled_before_start"] is True raw = scoped.get("vm_snapshot", catalog_key) if expected_state is None: assert raw is None else: assert json.loads(raw)["state"] == expected_state operation = call( client, "GET", f"/internal/v1/operations/{operation_id}", USER, ) assert operation.status_code == 200 assert operation.json()["status"] == "failed" assert not any(call_[0] in { "create_live_snapshot", "restore_live_snapshot", "delete_snapshot_volume", } for call_ in services["hk"].calls) def test_snapshot_uncertain_restore_hides_private_volume_evidence(boundary): client, engine, store, _logs, services = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") components = engine.registry.components["hk"] snapshot_key = "4" * 32 operation_id = "snapshot-restore-private" job_id = "snapshot-job-private" snapshot_volume = snapshot_volume_name(VM_ID, snapshot_key) components.store.set("vm_snapshot", f"{VM_ID}:{snapshot_key}", json.dumps({ "snapshot_key": snapshot_key, "vm_id": VM_ID, "name": "恢复点", "state": "uncertain", "consistency": "crash", "volume_name": snapshot_volume, "created_at": 1, })) private_old = f"panel-{VM_ID}.qcow2" private_replacement = "panel-restore-private.qcow2" components.store.set("snapshot_job", job_id, json.dumps({ "job_id": job_id, "operation_id": operation_id, "owner": "php:41", "vm_id": VM_ID, "snapshot_key": snapshot_key, "action": "restore", "consistency": "crash", "status": "uncertain", "phase": "结果待核查", "restore_old_volume": private_old, "restore_replacement_volume": private_replacement, "restore_was_running": True, })) services["hk"].snapshot_reconcile_restore = {"outcome": "uncertain"} store.set("operation", operation_id, json.dumps({ "operation_id": operation_id, "host_id": "hk", "user_id": 41, "kind": "mutation", "resource": f"vms/{VM_ID}/snapshots/{snapshot_key}/restore", "vm_uuid": VM_ID, "status": "uncertain", "result": None, "safe_to_refund": False, })) operation = call( client, "GET", f"/internal/v1/operations/{operation_id}").json() assert operation["status"] == "uncertain" serialized = json.dumps(operation, ensure_ascii=False) assert private_old not in serialized assert private_replacement not in serialized assert snapshot_volume not in serialized def test_snapshot_restore_without_persisted_plan_converges_failed_without_replay(boundary): client, engine, store, _logs, services = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") components = engine.registry.components["hk"] snapshot_key = "e" * 32 operation_id = "snapshot-restore-no-plan" job_id = "snapshot-job-no-plan" components.store.set("vm_snapshot", f"{VM_ID}:{snapshot_key}", json.dumps({ "snapshot_key": snapshot_key, "vm_id": VM_ID, "name": "恢复点", "state": "uncertain", "consistency": "crash", "volume_name": snapshot_volume_name(VM_ID, snapshot_key), "created_at": 1, })) components.store.set("snapshot_job", job_id, json.dumps({ "job_id": job_id, "operation_id": operation_id, "owner": "php:41", "vm_id": VM_ID, "snapshot_key": snapshot_key, "action": "restore", "consistency": "crash", "status": "uncertain", "phase": "面板重启", })) store.set("operation", operation_id, json.dumps({ "operation_id": operation_id, "host_id": "hk", "user_id": 41, "kind": "mutation", "resource": f"vms/{VM_ID}/snapshots/{snapshot_key}/restore", "vm_uuid": VM_ID, "status": "uncertain", "result": None, "safe_to_refund": False, })) operation = call( client, "GET", f"/internal/v1/operations/{operation_id}").json() assert operation["status"] == "failed" record = json.loads(components.store.get( "vm_snapshot", f"{VM_ID}:{snapshot_key}")) assert record["state"] == "ready" assert not any(call_[0] in { "restore_live_snapshot", "reconcile_live_snapshot_restore", } for call_ in services["hk"].calls) def test_snapshot_uncertain_delete_without_catalog_converges_by_original_operation(boundary): client, engine, store, _logs, services = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") components = engine.registry.components["hk"] snapshot_key = "5" * 32 operation_id = "snapshot-delete-reconcile" job_id = "snapshot-job-delete" components.store.set("snapshot_job", job_id, json.dumps({ "job_id": job_id, "operation_id": operation_id, "owner": "php:41", "vm_id": VM_ID, "snapshot_key": snapshot_key, "action": "delete", "consistency": "filesystem", "status": "uncertain", "phase": "结果待核查", })) store.set("operation", operation_id, json.dumps({ "operation_id": operation_id, "host_id": "hk", "user_id": 41, "kind": "mutation", "resource": f"vms/{VM_ID}/snapshots/{snapshot_key}", "vm_uuid": VM_ID, "status": "uncertain", "result": None, "safe_to_refund": False, })) operation = call( client, "GET", f"/internal/v1/operations/{operation_id}").json() assert operation["status"] == "succeeded" assert operation["result"] == { "snapshot_key": snapshot_key, "deleted": True, "consistency": "filesystem", } saved_job = json.loads(components.store.get("snapshot_job", job_id)) assert saved_job["status"] == "succeeded" assert not any(call_[0] == "delete_snapshot_volume" for call_ in services["hk"].calls) assert len([call_ for call_ in services["hk"].calls if call_[0] == "reconcile_live_snapshot_delete"]) == 1 def test_snapshot_uncertain_create_reconciles_with_consistency(boundary): client, engine, store, _logs, services = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") components = engine.registry.components["hk"] snapshot_key = "6" * 32 operation_id = "snapshot-create-reconcile" job_id = "snapshot-job-create" components.store.set("vm_snapshot", f"{VM_ID}:{snapshot_key}", json.dumps({ "snapshot_key": snapshot_key, "vm_id": VM_ID, "name": "恢复点", "state": "uncertain", "consistency": "crash", "volume_name": snapshot_volume_name(VM_ID, snapshot_key), "created_at": 1, })) components.store.set("snapshot_job", job_id, json.dumps({ "job_id": job_id, "operation_id": operation_id, "owner": "php:41", "vm_id": VM_ID, "snapshot_key": snapshot_key, "action": "create", "consistency": "crash", "status": "uncertain", "phase": "结果待核查", "backup_verified_capacity": 20 * 1024**3, "backup_verified_identity": "a" * 64, })) store.set("operation", operation_id, json.dumps({ "operation_id": operation_id, "host_id": "hk", "user_id": 41, "kind": "mutation", "resource": f"vms/{VM_ID}/snapshots", "vm_uuid": VM_ID, "status": "uncertain", "result": None, "safe_to_refund": False, })) operation = call( client, "GET", f"/internal/v1/operations/{operation_id}").json() assert operation["status"] == "succeeded" assert operation["result"]["created"] is True assert operation["result"]["consistency"] == "crash" assert "volume_name" not in operation["result"] assert not any(call_[0] == "create_live_snapshot" for call_ in services["hk"].calls) assert len([call_ for call_ in services["hk"].calls if call_[0] == "reconcile_live_snapshot_create"]) == 1 def test_snapshot_confirmed_failed_create_removes_hidden_catalog(boundary): client, engine, store, _logs, services = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") components = engine.registry.components["hk"] snapshot_key = "b" * 32 operation_id = "snapshot-create-confirmed-failed" job_id = "snapshot-job-create-failed" catalog_key = f"{VM_ID}:{snapshot_key}" components.store.set("vm_snapshot", catalog_key, json.dumps({ "snapshot_key": snapshot_key, "vm_id": VM_ID, "name": "失败恢复点", "state": "uncertain", "consistency": "crash", "volume_name": snapshot_volume_name(VM_ID, snapshot_key), "created_at": 1, })) components.store.set("snapshot_job", job_id, json.dumps({ "job_id": job_id, "operation_id": operation_id, "owner": "php:41", "vm_id": VM_ID, "snapshot_key": snapshot_key, "action": "create", "consistency": "crash", "status": "uncertain", "phase": "结果待核查", })) store.set("operation", operation_id, json.dumps({ "operation_id": operation_id, "host_id": "hk", "user_id": 41, "kind": "mutation", "resource": f"vms/{VM_ID}/snapshots", "vm_uuid": VM_ID, "status": "uncertain", "result": None, "safe_to_refund": False, })) services["hk"].snapshot_reconcile_create = { "outcome": "failed", "snapshot_key": snapshot_key, } operation = call( client, "GET", f"/internal/v1/operations/{operation_id}").json() assert operation["status"] == "failed" assert components.store.get("vm_snapshot", catalog_key) is None saved_job = json.loads(components.store.get("snapshot_job", job_id)) assert saved_job["status"] == "failed" @pytest.mark.parametrize( "snapshot_state", ["creating", "restoring", "deleting", "uncertain"]) def test_transitional_snapshot_catalog_blocks_power_mutation_without_pending_job( boundary, snapshot_state): client, engine, _store, _logs, services = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") components = engine.registry.components["hk"] components.state.users.assign_vm( host_scope(components.manager.profile), VM_ID, USER["user_id"]) snapshot_key = "7" * 32 components.store.set("vm_snapshot", f"{VM_ID}:{snapshot_key}", json.dumps({ "snapshot_key": snapshot_key, "vm_id": VM_ID, "name": "中断恢复点", "state": snapshot_state, "consistency": "crash", "volume_name": snapshot_volume_name(VM_ID, snapshot_key), "created_at": 1, })) actor = {**USER, "operation_id": "power-blocked-by-snapshot"} response = call( client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/actions", actor, {"action": "poweroff"}) assert response.status_code == 409 assert not any(call_[0] == "action" for call_ in services["hk"].calls) def test_network_suspend_bypasses_durable_snapshot_uncertain_when_no_backup( boundary): route, method = "network-suspend", "network_suspend" client, engine, _store, _logs, services = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") components = engine.registry.components["hk"] components.state.users.assign_vm( host_scope(components.manager.profile), VM_ID, USER["user_id"]) snapshot_key = "8" * 32 components.store.set("vm_snapshot", f"{VM_ID}:{snapshot_key}", json.dumps({ "snapshot_key": snapshot_key, "vm_id": VM_ID, "name": "待核查恢复点", "state": "uncertain", "consistency": "crash", "volume_name": snapshot_volume_name(VM_ID, snapshot_key), "created_at": 1, })) actor = {**USER, "operation_id": f"expiry-uncertain-{route}"} response = call( client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/{route}", actor, {}) assert response.status_code == 200, response.text assert response.json()["status"] == "succeeded" assert len([item for item in services["hk"].calls if item[0] == method]) == 1 def test_network_suspend_waits_while_snapshot_backup_is_still_active( boundary): route, method = "network-suspend", "network_suspend" client, engine, _store, _logs, services = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") components = engine.registry.components["hk"] components.state.users.assign_vm( host_scope(components.manager.profile), VM_ID, USER["user_id"]) snapshot_key = "9" * 32 components.store.set("vm_snapshot", f"{VM_ID}:{snapshot_key}", json.dumps({ "snapshot_key": snapshot_key, "vm_id": VM_ID, "name": "复制中的恢复点", "state": "uncertain", "consistency": "filesystem", "volume_name": snapshot_volume_name(VM_ID, snapshot_key), "created_at": 1, })) services["hk"].snapshot_backup_active = True actor = {**USER, "operation_id": f"expiry-active-{route}"} response = call( client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/{route}", actor, {}) assert response.status_code == 409, response.text assert response.json()["status"] == "failed" assert len([item for item in services["hk"].calls if item[0] == method]) == 1 @pytest.mark.parametrize(("action", "method"), [ ("create", "create_live_snapshot"), ("restore", "restore_live_snapshot"), ("delete", "delete_snapshot_volume"), ]) def test_network_suspend_cancels_same_vm_snapshot_still_queued_behind_host_job( boundary, action, method): client, engine, _store, _logs, services = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") components = engine.registry.components["hk"] profile = components.manager.profile scope = host_scope(profile) service = services["hk"] queued_vm = "22222222-2222-4222-8222-222222222222" components.state.users.assign_vm(scope, VM_ID, USER["user_id"]) components.state.users.assign_vm(scope, queued_vm, USER["user_id"]) service.vm.update( managed=True, os_source="cloud_template", rescue_mode=False, rescue_staged=False, ) queued_vm_record = { **service.vm, "uuid": queued_vm, "name": "queued-vm", "network_suspended": False, } original_get_vm = service.get_vm original_create = service.create_live_snapshot original_network_suspend = service.network_suspend def get_vm(vm_id): if vm_id == queued_vm: service.calls.append(("get_vm", vm_id)) return dict(queued_vm_record) return original_get_vm(vm_id) blocker_started = threading.Event() blocker_release = threading.Event() def create_live_snapshot(vm_id, snapshot_key, *, consistency, progress=None, recovery_job_id=None): if vm_id == VM_ID: blocker_started.set() assert blocker_release.wait(timeout=5) return original_create( vm_id, snapshot_key, consistency=consistency, progress=progress) def network_suspend(vm_id): if vm_id == queued_vm: service.calls.append(("network_suspend", vm_id)) queued_vm_record["network_suspended"] = True return dict(queued_vm_record) return original_network_suspend(vm_id) service.get_vm = get_vm service.create_live_snapshot = create_live_snapshot service.network_suspend = network_suspend blocker_actor = {**USER, "operation_id": f"snapshot-blocker-{action}"} blocker_path = f"/internal/v1/hosts/hk/vms/{VM_ID}/snapshots" blocker = call(client, "POST", blocker_path, blocker_actor, { "snapshot_key": "a" * 32, "name": "占用宿主快照执行器", "consistency_mode": "crash", }) assert blocker.status_code == 202, blocker.text assert blocker_started.wait(timeout=2) snapshot_key = "b" * 32 catalog_key = f"{queued_vm}:{snapshot_key}" if action != "create": components.store.set("vm_snapshot", catalog_key, json.dumps({ "snapshot_key": snapshot_key, "vm_id": queued_vm, "name": "待操作恢复点", "state": "ready", "consistency": "crash", "volume_name": snapshot_volume_name(queued_vm, snapshot_key), "created_at": 1, })) snapshot_actor = { **USER, "operation_id": f"queued-{action}-cancel", } if action == "create": queued_path = f"/internal/v1/hosts/hk/vms/{queued_vm}/snapshots" queued_body = { "snapshot_key": snapshot_key, "name": "应被到期策略取消", "consistency_mode": "crash", } queued_method = "POST" else: queued_path = ( f"/internal/v1/hosts/hk/vms/{queued_vm}/snapshots/" f"{snapshot_key}" + ("/restore" if action == "restore" else "") ) queued_body = {} queued_method = "POST" if action == "restore" else "DELETE" try: queued = call( client, queued_method, queued_path, snapshot_actor, queued_body) assert queued.status_code == 202, queued.text suspend_actor = { **USER, "operation_id": f"expiry-network-{action}", } suspended = call( client, "POST", f"/internal/v1/hosts/hk/vms/{queued_vm}/network-suspend", suspend_actor, {}, ) assert suspended.status_code == 200, suspended.text assert suspended.json()["status"] == "succeeded" snapshot_operation = call( client, "GET", f"/internal/v1/operations/{snapshot_actor['operation_id']}", snapshot_actor, ) assert snapshot_operation.status_code == 200 assert snapshot_operation.json()["status"] == "failed" assert "未调用虚拟化操作" in snapshot_operation.json()["error"] assert not any( item[0] == method and len(item) > 1 and item[1] == queued_vm for item in service.calls ) raw = components.store.get("vm_snapshot", catalog_key) if action == "create": assert raw is None else: assert json.loads(raw)["state"] == "ready" finally: blocker_release.set() def test_network_suspend_rejects_snapshot_future_already_running(boundary): client, engine, _store, _logs, services = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") components = engine.registry.components["hk"] components.state.users.assign_vm( host_scope(components.manager.profile), VM_ID, USER["user_id"]) service = services["hk"] service.vm.update( managed=True, os_source="cloud_template", rescue_mode=False, rescue_staged=False, ) original_create = service.create_live_snapshot started = threading.Event() release = threading.Event() def running_snapshot(vm_id, snapshot_key, *, consistency, progress=None, recovery_job_id=None): started.set() assert release.wait(timeout=5) return original_create( vm_id, snapshot_key, consistency=consistency, progress=progress, recovery_job_id=recovery_job_id) service.create_live_snapshot = running_snapshot snapshot_actor = {**USER, "operation_id": "running-snapshot-guard"} queued = call( client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/snapshots", snapshot_actor, { "snapshot_key": "c" * 32, "name": "已开始的快照", "consistency_mode": "crash", }, ) assert queued.status_code == 202 assert started.wait(timeout=2) try: before = len([item for item in service.calls if item[0] == "network_suspend"]) suspended = call( client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/network-suspend", {**USER, "operation_id": "running-snapshot-network"}, {}, ) assert suspended.status_code == 409 assert suspended.json()["error_code"] == "coordination_busy" assert _store.get("operation", "running-snapshot-network") is None assert len([item for item in service.calls if item[0] == "network_suspend"]) == before finally: release.set() def test_cancelled_snapshot_phase_one_save_failure_converges_on_poll(boundary): client, engine, _store, _logs, services = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") components = engine.registry.components["hk"] scope = host_scope(components.manager.profile) service = services["hk"] queued_vm = "33333333-3333-4333-8333-333333333333" components.state.users.assign_vm(scope, VM_ID, USER["user_id"]) components.state.users.assign_vm(scope, queued_vm, USER["user_id"]) service.vm.update( managed=True, os_source="cloud_template", rescue_mode=False, rescue_staged=False, ) queued_vm_record = { **service.vm, "uuid": queued_vm, "name": "queued-save-failure", } original_get_vm = service.get_vm original_create = service.create_live_snapshot original_network_suspend = service.network_suspend def get_vm(vm_id): return (dict(queued_vm_record) if vm_id == queued_vm else original_get_vm(vm_id)) blocker_started = threading.Event() blocker_release = threading.Event() def create_live_snapshot(vm_id, snapshot_key, *, consistency, progress=None, recovery_job_id=None): if vm_id == VM_ID: blocker_started.set() assert blocker_release.wait(timeout=5) return original_create( vm_id, snapshot_key, consistency=consistency, progress=progress) def network_suspend(vm_id): if vm_id == queued_vm: service.calls.append(("network_suspend", vm_id)) return {**queued_vm_record, "network_suspended": True} return original_network_suspend(vm_id) service.get_vm = get_vm service.create_live_snapshot = create_live_snapshot service.network_suspend = network_suspend blocker = call( client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/snapshots", {**USER, "operation_id": "phase-one-blocker"}, { "snapshot_key": "d" * 32, "name": "占用执行器", "consistency_mode": "crash", }, ) assert blocker.status_code == 202 assert blocker_started.wait(timeout=2) snapshot_actor = {**USER, "operation_id": "phase-one-cancelled"} snapshot_key = "e" * 32 queued = call( client, "POST", f"/internal/v1/hosts/hk/vms/{queued_vm}/snapshots", snapshot_actor, { "snapshot_key": snapshot_key, "name": "取消写入故障", "consistency_mode": "crash", }, ) assert queued.status_code == 202 original_set = components.store.set failed_once = False def flaky_set(kind, key, value): nonlocal failed_once if kind == "snapshot_job" and not failed_once: payload = json.loads(value) if (payload.get("cancelled_before_start") is True and payload.get("status") == "queued"): failed_once = True raise DatabaseError("phase-one tombstone write failed") return original_set(kind, key, value) components.store.set = flaky_set try: first_suspend = call( client, "POST", f"/internal/v1/hosts/hk/vms/{queued_vm}/network-suspend", {**USER, "operation_id": "phase-one-network-first"}, {}, ) assert first_suspend.status_code == 503 assert failed_once is True snapshot_operation = call( client, "GET", f"/internal/v1/operations/{snapshot_actor['operation_id']}", snapshot_actor, ) assert snapshot_operation.status_code == 200 assert snapshot_operation.json()["status"] == "failed" assert components.store.get( "vm_snapshot", f"{queued_vm}:{snapshot_key}") is None assert not any( item[0] == "create_live_snapshot" and item[1] == queued_vm for item in service.calls ) second_suspend = call( client, "POST", f"/internal/v1/hosts/hk/vms/{queued_vm}/network-suspend", {**USER, "operation_id": "phase-one-network-retry"}, {}, ) assert second_suspend.status_code == 200, second_suspend.text finally: components.store.set = original_set blocker_release.set() def test_network_suspend_barrier_rejects_snapshot_admission_until_target_lands( boundary): client, engine, _store, _logs, services = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") components = engine.registry.components["hk"] scope = host_scope(components.manager.profile) service = services["hk"] vm_id = "44444444-4444-4444-8444-444444444444" components.state.users.assign_vm(scope, vm_id, USER["user_id"]) vm = { **service.vm, "uuid": vm_id, "name": "barrier-vm", "managed": True, "os_source": "cloud_template", "rescue_mode": False, "rescue_staged": False, } original_get_vm = service.get_vm original_network_suspend = service.network_suspend def get_vm(target): return dict(vm) if target == vm_id else original_get_vm(target) suspend_started = threading.Event() suspend_release = threading.Event() def network_suspend(target): if target == vm_id: service.calls.append(("network_suspend", target)) suspend_started.set() assert suspend_release.wait(timeout=5) return {**vm, "network_suspended": True} return original_network_suspend(target) service.get_vm = get_vm service.network_suspend = network_suspend suspend_actor = {**USER, "operation_id": "barrier-network-suspend"} direct = TestClient( engine.registry.apps["hk"], base_url="http://127.0.0.1") snapshot_key = "f" * 32 try: with ThreadPoolExecutor(max_workers=1) as executor: pending_suspend = executor.submit( call, client, "POST", f"/internal/v1/hosts/hk/vms/{vm_id}/network-suspend", suspend_actor, {}, ) assert suspend_started.wait(timeout=2) token = TRUSTED_ACTOR.set({ **USER, "operation_id": "barrier-snapshot-rejected", }) try: rejected = direct.post( f"/api/v1/vms/{vm_id}/snapshots", json={ "snapshot_key": snapshot_key, "name": "不应越过暂停屏障", "consistency_mode": "crash", }, ) finally: TRUSTED_ACTOR.reset(token) assert rejected.status_code == 409, rejected.text assert components.store.get( "vm_snapshot", f"{vm_id}:{snapshot_key}") is None assert not any( json.loads(raw).get("snapshot_key") == snapshot_key for raw in components.store.list("snapshot_job").values() ) suspend_release.set() suspended = pending_suspend.result(timeout=5) assert suspended.status_code == 200, suspended.text finally: suspend_release.set() direct.close() @pytest.mark.parametrize("action", ["create", "restore", "delete"]) def test_initial_snapshot_job_save_failure_leaves_no_network_suspend_ghost( boundary, action): client, engine, _store, _logs, services = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") components = engine.registry.components["hk"] scope = host_scope(components.manager.profile) components.state.users.assign_vm(scope, VM_ID, USER["user_id"]) service = services["hk"] service.vm.update( managed=True, os_source="cloud_template", rescue_mode=False, rescue_staged=False, ) snapshot_key = { "create": "7", "restore": "8", "delete": "9", }[action] * 32 catalog_key = f"{VM_ID}:{snapshot_key}" if action != "create": components.store.set("vm_snapshot", catalog_key, json.dumps({ "snapshot_key": snapshot_key, "vm_id": VM_ID, "name": "初始写入故障恢复点", "state": "ready", "consistency": "crash", "volume_name": snapshot_volume_name(VM_ID, snapshot_key), "created_at": 1, })) original_set = components.store.set failed_once = False def flaky_set(kind, key, value): nonlocal failed_once if kind == "snapshot_job" and not failed_once: failed_once = True raise DatabaseError("initial snapshot job write failed") return original_set(kind, key, value) components.store.set = flaky_set actor = {**USER, "operation_id": f"initial-save-failure-{action}"} if action == "create": path = f"/internal/v1/hosts/hk/vms/{VM_ID}/snapshots" body = { "snapshot_key": snapshot_key, "name": "初始写入故障", "consistency_mode": "crash", } method = "POST" else: path = f"/internal/v1/hosts/hk/vms/{VM_ID}/snapshots/{snapshot_key}" if action == "restore": path += "/restore" method = "POST" else: method = "DELETE" body = {} try: rejected = call(client, method, path, actor, body) assert rejected.status_code == 503 assert failed_once is True finally: components.store.set = original_set raw = components.store.get("vm_snapshot", catalog_key) if action == "create": assert raw is None else: assert json.loads(raw)["state"] == "ready" assert not any(call_[0] in { "create_live_snapshot", "restore_live_snapshot", "delete_snapshot_volume", } for call_ in service.calls) suspended = call( client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/network-suspend", {**USER, "operation_id": f"network-after-save-failure-{action}"}, {}, ) assert suspended.status_code == 200, suspended.text def test_snapshot_executor_submit_failure_is_terminal_and_does_not_block_suspend( boundary): client, engine, _store, _logs, services = boundary class FailingSnapshotExecutor: def submit(self, *_args, **_kwargs): raise RuntimeError("executor stopped") def shutdown(self, **_kwargs): return None def live_workers(self): return () real_executor = ManagedThreadPoolExecutor def executor_factory(*args, **kwargs): if kwargs.get("thread_name_prefix") == "vm-snapshot": return FailingSnapshotExecutor() return real_executor(*args, **kwargs) with patch("virsh_panel.app.ManagedThreadPoolExecutor", side_effect=executor_factory): call(client, "GET", "/internal/v1/hosts/hk/vms") components = engine.registry.components["hk"] components.state.users.assign_vm( host_scope(components.manager.profile), VM_ID, USER["user_id"]) services["hk"].vm.update( managed=True, os_source="cloud_template", rescue_mode=False, rescue_staged=False, ) actor = {**USER, "operation_id": "snapshot-submit-terminal"} snapshot_key = "6" * 32 rejected = call( client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/snapshots", actor, { "snapshot_key": snapshot_key, "name": "执行器停止", "consistency_mode": "crash", }, ) assert rejected.status_code == 503 assert components.store.get( "vm_snapshot", f"{VM_ID}:{snapshot_key}") is None operation = call( client, "GET", f"/internal/v1/operations/{actor['operation_id']}", actor, ) assert operation.status_code == 200 assert operation.json()["status"] == "failed" assert "未调用虚拟化操作" in operation.json()["error"] assert not any(item[0] == "create_live_snapshot" for item in services["hk"].calls) suspended = call( client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/network-suspend", {**USER, "operation_id": "network-after-submit-failure"}, {}, ) assert suspended.status_code == 200, suspended.text @pytest.mark.parametrize(("route", "method"), [ ("service-suspend", "service_suspend"), ("service-resume", "service_resume"), ("network-resume", "network_resume"), ]) def test_non_fail_safe_targets_remain_blocked_by_snapshot_uncertain( boundary, route, method): client, engine, _store, _logs, services = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") components = engine.registry.components["hk"] components.state.users.assign_vm( host_scope(components.manager.profile), VM_ID, USER["user_id"]) snapshot_key = "a" * 32 components.store.set("vm_snapshot", f"{VM_ID}:{snapshot_key}", json.dumps({ "snapshot_key": snapshot_key, "vm_id": VM_ID, "name": "待核查恢复点", "state": "uncertain", "consistency": "crash", "volume_name": snapshot_volume_name(VM_ID, snapshot_key), "created_at": 1, })) actor = {**USER, "operation_id": f"resume-blocked-{route}"} body = {"start_if_suspended": False} if route == "service-resume" else {} response = call( client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/{route}", actor, body) assert response.status_code == 409, response.text assert not any(item[0] == method for item in services["hk"].calls) def test_interrupted_async_job_becomes_terminal_without_replay(boundary): client, engine, store, *_ = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") components = engine.registry.components["hk"] components.store.set("reinstall_job", "job-abcd", json.dumps({ "operation_id": "interrupted-job", "owner": "php:41", "vm_id": VM_ID, "status": "failed", "phase": "面板重启,任务已中断"})) store.set("operation", "interrupted-job", json.dumps({"operation_id": "interrupted-job", "host_id": "hk", "user_id": 41, "kind": "mutation", "resource": f"vms/{VM_ID}/reinstall", "vm_uuid": VM_ID, "status": "pending", "result": None, "safe_to_refund": False})) recovered = call(client, "GET", "/internal/v1/operations/interrupted-job").json() assert recovered["status"] == "failed" assert "中断" in recovered["error"] def test_reinstall_cleanup_pending_stays_uncertain_without_replay(boundary): client, engine, store, *_ = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") components = engine.registry.components["hk"] components.store.set("reinstall_job", "job-cleanup-pending", json.dumps({ "operation_id": "reinstall-cleanup-pending", "owner": "php:41", "vm_id": VM_ID, "status": "uncertain", "phase": "重装已完成,但旧系统盘清理待核对", "error": "重装已完成,但旧系统盘清理待核对", "error_code": "reinstall_cleanup_pending", "reinstall_committed": True, "cleanup_pending": True, "vm": {"uuid": VM_ID, "name": "cloud-vm", "state": "running"}, "warning": "旧存储卷 panel-old.qcow2 删除失败,请检查宿主机", })) store.set("operation", "reinstall-cleanup-pending", json.dumps({ "operation_id": "reinstall-cleanup-pending", "host_id": "hk", "user_id": 41, "kind": "mutation", "resource": f"vms/{VM_ID}/reinstall", "vm_uuid": VM_ID, "status": "pending", "result": None, "safe_to_refund": False, })) recovered = call( client, "GET", "/internal/v1/operations/reinstall-cleanup-pending", USER, ) payload = recovered.json() assert recovered.status_code == 200 assert payload["status"] == "uncertain" assert payload["error_code"] == "reinstall_cleanup_pending" assert payload["result"]["reinstall_committed"] is True assert payload["result"]["cleanup_pending"] is True assert payload["result"]["vm"]["state"] == "running" def test_committed_reinstall_restart_marker_is_not_downgraded(boundary): client, engine, store, *_ = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") components = engine.registry.components["hk"] components.store.set("reinstall_job", "job-restart-committed", json.dumps({ "operation_id": "reinstall-restart-committed", "owner": "php:41", "vm_id": VM_ID, "status": "failed", "phase": "面板重启,任务已中断", "error": "面板在任务执行期间重启;请检查宿主机实际状态,勿直接重复提交", "reinstall_committed": True, })) store.set("operation", "reinstall-restart-committed", json.dumps({ "operation_id": "reinstall-restart-committed", "host_id": "hk", "user_id": 41, "kind": "mutation", "resource": f"vms/{VM_ID}/reinstall", "vm_uuid": VM_ID, "status": "pending", "result": None, "safe_to_refund": False, })) recovered = call( client, "GET", "/internal/v1/operations/reinstall-restart-committed", USER, ) payload = recovered.json() assert recovered.status_code == 200 assert payload["status"] == "uncertain" assert payload["error_code"] == "reinstall_result_uncertain" def test_committed_running_reinstall_marker_is_uncertain(boundary): client, engine, store, *_ = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") components = engine.registry.components["hk"] components.store.set("reinstall_job", "job-running-committed", json.dumps({ "operation_id": "reinstall-running-committed", "owner": "php:41", "vm_id": VM_ID, "status": "running", "phase": "cleaning_up", "reinstall_committed": True, })) store.set("operation", "reinstall-running-committed", json.dumps({ "operation_id": "reinstall-running-committed", "host_id": "hk", "user_id": 41, "kind": "mutation", "resource": f"vms/{VM_ID}/reinstall", "vm_uuid": VM_ID, "status": "running", "result": None, "safe_to_refund": False, })) recovered = call( client, "GET", "/internal/v1/operations/reinstall-running-committed", USER, ) payload = recovered.json() assert recovered.status_code == 200 assert payload["status"] == "uncertain" assert payload["error_code"] == "reinstall_result_uncertain" def test_legacy_reinstall_post_commit_phase_is_uncertain(boundary): client, engine, store, *_ = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") components = engine.registry.components["hk"] components.store.set("reinstall_job", "job-legacy-phase", json.dumps({ "operation_id": "reinstall-legacy-phase", "owner": "php:41", "vm_id": VM_ID, "status": "succeeded", "phase": "cleaning_up", "generated_password": "should-remain-private", })) store.set("operation", "reinstall-legacy-phase", json.dumps({ "operation_id": "reinstall-legacy-phase", "host_id": "hk", "user_id": 41, "kind": "mutation", "resource": f"vms/{VM_ID}/reinstall", "vm_uuid": VM_ID, "status": "pending", "result": None, "safe_to_refund": False, })) recovered = call( client, "GET", "/internal/v1/operations/reinstall-legacy-phase", USER, ) payload = recovered.json() assert recovered.status_code == 200 assert payload["status"] == "uncertain" assert payload["error_code"] == "reinstall_result_uncertain" assert payload["result"]["phase"] == "cleaning_up" @pytest.mark.parametrize("resource", [ f"vms/{VM_ID}/actions", f"vms/{VM_ID}/reinstall", f"vms/{VM_ID}/rescue", ]) def test_orphaned_mutation_becomes_terminal_without_replay(boundary, resource): client, _, store, *_ = boundary operation_id = "orphan-" + resource.rsplit("/", 1)[-1].replace("?", "-") store.set("operation", operation_id, json.dumps({ "operation_id": operation_id, "host_id": "hk", "user_id": 41, "kind": "mutation", "resource": resource, "vm_uuid": VM_ID, "status": "running", "result": None, "safe_to_refund": False, "created_at": 1, })) recovered = call(client, "GET", f"/internal/v1/operations/{operation_id}").json() assert recovered["status"] == "failed" assert "不会自动重放" in recovered["error"] def test_orphaned_delete_remains_uncertain_without_replay(boundary): client, _engine, store, _logs, services = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") operation_id = "orphan-destructive-delete" store.set("operation", operation_id, json.dumps({ "operation_id": operation_id, "host_id": "hk", "user_id": 41, "kind": "mutation", "resource": f"vms/{VM_ID}", "method": "DELETE", "vm_uuid": VM_ID, "status": "running", "result": None, "safe_to_refund": False, "delete_ip_claim_keys": [], "created_at": 1, })) before = list(services["hk"].calls) first = call( client, "GET", f"/internal/v1/operations/{operation_id}", USER, ).json() second = call( client, "GET", f"/internal/v1/operations/{operation_id}", USER, ).json() assert first["status"] == second["status"] == "uncertain" assert "原操作" in second["error"] assert services["hk"].calls == before def test_network_change_lost_response_keeps_both_addresses_uncertain(boundary): client, engine, store, _, services = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") components = engine.registry.components["hk"] profile = components.manager.profile profile.update({ "host": "8.8.8.3", "default_network": "bridge:br0", "public_pool_cidr": "8.8.8.0/28", "public_gateway": "8.8.8.1", "public_dns_servers": ["1.1.1.1"], "public_reserved_ips": ["8.8.8.2"], "public_vm_prefix": 28, }) service = services["hk"] service.vm.update( managed=True, os_source="cloud_template", public_ip_cidr="8.8.8.4/28", ) scope = host_scope(profile) components.state.users.assign_vm(scope, VM_ID, USER["user_id"]) actor = {**USER, "operation_id": "network-change-lost-response"} path = f"/internal/v1/hosts/hk/vms/{VM_ID}/network" changed = call(client, "POST", path, actor, { "public_ip_cidr": "8.8.8.6/28", "confirm_restart": True, }) assert changed.status_code == 200, changed.text saved = json.loads(store.get("operation", actor["operation_id"])) assert saved["network_source_cidr"] == "8.8.8.4/28" assert saved["network_target_cidr"] == "8.8.8.6/28" assert saved["network_mutation_started"] is True # Model a lost response after libvirt may already have switched. A single # metadata read cannot prove that XML, cloud-init and nwfilter all agree, # so automatic reconciliation must retain both address claims. saved.update(status="uncertain", result=None, error="response lost") store.set("operation", actor["operation_id"], json.dumps(saved)) components.store.set("ip_claim", f"{scope}.8.8.8.4", json.dumps({ "vm_uuid": VM_ID, })) engine.active_operations.clear() reconciled = call( client, "GET", f"/internal/v1/operations/{actor['operation_id']}", actor, ) assert reconciled.status_code == 200 assert reconciled.json()["status"] == "uncertain" assert "人工核查" in reconciled.json()["error"] assert components.store.get("ip_claim", f"{scope}.8.8.8.4") is not None source_record = { **saved, "operation_id": "network-change-still-source", "network_source_cidr": "8.8.8.6/28", "network_target_cidr": "8.8.8.7/28", "status": "uncertain", } store.set("operation", source_record["operation_id"], json.dumps(source_record)) components.store.set("ip_claim", f"{scope}.8.8.8.7", json.dumps({ "vm_uuid": VM_ID, })) source = call( client, "GET", f"/internal/v1/operations/{source_record['operation_id']}", actor, ) assert source.json()["status"] == "uncertain" assert components.store.get("ip_claim", f"{scope}.8.8.8.7") is not None service.vm["public_ip_cidr"] = "8.8.8.8/28" drift_record = { **source_record, "operation_id": "network-change-third-address", "status": "uncertain", } store.set("operation", drift_record["operation_id"], json.dumps(drift_record)) drift = call( client, "GET", f"/internal/v1/operations/{drift_record['operation_id']}", actor, ) assert drift.json()["status"] == "uncertain" assert "人工核查" in drift.json()["error"] def test_network_change_can_move_between_cidrs_on_the_same_selector(boundary): client, engine, store, _, services = boundary assert call(client, "GET", "/internal/v1/hosts/hk/vms").status_code == 200 components = engine.registry.components["hk"] profile = components.manager.profile profile["default_network"] = "bridge:br0" first_key = "3" * 32 second_key = "4" * 32 common = { "revision": 1, "state": "active", "network_selector": "bridge:br0", "dns_servers": ["1.1.1.1"], "reserved_ips": [], "guest_prefix_length": 29, } assert call( client, "PUT", f"/internal/v1/hosts/hk/host/network-pools/{first_key}", SUPER, {**common, "cidr": "8.8.8.0/29", "gateway": "8.8.8.1"}, ).status_code == 200 assert call( client, "PUT", f"/internal/v1/hosts/hk/host/network-pools/{second_key}", SUPER, {**common, "cidr": "9.9.9.0/29", "gateway": "9.9.9.1"}, ).status_code == 200 service = services["hk"] service.vm.update( managed=True, os_source="cloud_template", public_ip_cidr="8.8.8.3/29", public_pool_key=first_key, ) scope = host_scope(profile) components.state.users.assign_vm(scope, VM_ID, USER["user_id"]) actor = {**USER, "operation_id": "network-cross-cidr-0001"} changed = call( client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/network", actor, { "public_pool_key": second_key, "public_ip_cidr": "9.9.9.3/29", "confirm_restart": True, }, ) assert changed.status_code == 200, changed.text assert changed.json()["public_pool_key"] == second_key operation = json.loads(store.get("operation", actor["operation_id"])) assert operation["network_source_pool_key"] == first_key assert operation["network_target_pool_key"] == second_key claim = json.loads(components.store.get( "ip_claim", f"{scope}.9.9.9.3")) assert claim["pool_key"] == second_key def test_network_change_upgrades_legacy_firewall_before_switching_ip(boundary): client, engine, _store, _, services = boundary assert call(client, "GET", "/internal/v1/hosts/hk/vms").status_code == 200 components = engine.registry.components["hk"] profile = components.manager.profile profile.update({ "host": "8.8.8.3", "default_network": "bridge:br0", "public_pool_cidr": "8.8.8.0/28", "public_gateway": "8.8.8.1", "public_dns_servers": ["1.1.1.1"], "public_reserved_ips": ["8.8.8.2"], "public_vm_prefix": 28, }) service = services["hk"] service.vm.update( managed=True, os_source="cloud_template", public_ip_cidr="8.8.8.4/28", ) scope = host_scope(profile) components.state.users.assign_vm(scope, VM_ID, USER["user_id"]) firewall_path = f"/internal/v1/hosts/hk/vms/{VM_ID}/firewall" body = firewall_body() applied = call( client, "PUT", firewall_path, {**USER, "operation_id": "network-firewall-upgrade-0001"}, body, ) assert applied.status_code == 200, applied.text record_key = f"{scope}.{VM_ID}" legacy_digest = build_firewall_filter_xml( VM_ID, body["enabled"], body["rules"], compiler_version=1, )[3] legacy_record = json.loads( components.store.get("vm_firewall", record_key)) legacy_record.pop("filter_version") legacy_record["filter_digest"] = legacy_digest components.store.set( "vm_firewall", record_key, json.dumps(legacy_record, ensure_ascii=False, separators=(",", ":")), ) service.firewall_filter_digest = legacy_digest applies = sum(row[0] == "apply_firewall_policy" for row in service.calls) changed = call( client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/network", {**USER, "operation_id": "network-firewall-upgrade-0002"}, { "public_ip_cidr": "8.8.8.6/28", "confirm_restart": True, }, ) assert changed.status_code == 200, changed.text assert changed.json()["network_changed"] is True assert sum(row[0] == "apply_firewall_policy" for row in service.calls) == applies + 1 call_names = [row[0] for row in service.calls] assert call_names.index( "apply_firewall_policy", call_names.index("apply_firewall_policy") + 1, ) < call_names.index("change_cloud_vm_network") upgraded = json.loads(components.store.get("vm_firewall", record_key)) assert upgraded["filter_version"] == FIREWALL_FILTER_COMPILER_VERSION assert upgraded["filter_digest"] == build_firewall_filter_xml( VM_ID, body["enabled"], body["rules"])[3] def test_network_change_rejects_drifted_legacy_firewall(boundary): client, engine, _store, _, services = boundary assert call(client, "GET", "/internal/v1/hosts/hk/vms").status_code == 200 components = engine.registry.components["hk"] profile = components.manager.profile profile.update({ "host": "8.8.8.3", "default_network": "bridge:br0", "public_pool_cidr": "8.8.8.0/28", "public_gateway": "8.8.8.1", "public_dns_servers": ["1.1.1.1"], "public_reserved_ips": ["8.8.8.2"], "public_vm_prefix": 28, }) service = services["hk"] service.vm.update( managed=True, os_source="cloud_template", public_ip_cidr="8.8.8.4/28", ) scope = host_scope(profile) components.state.users.assign_vm(scope, VM_ID, USER["user_id"]) firewall_path = f"/internal/v1/hosts/hk/vms/{VM_ID}/firewall" body = firewall_body() applied = call( client, "PUT", firewall_path, {**USER, "operation_id": "network-firewall-drift-0001"}, body, ) assert applied.status_code == 200, applied.text record_key = f"{scope}.{VM_ID}" legacy_digest = build_firewall_filter_xml( VM_ID, body["enabled"], body["rules"], compiler_version=1, )[3] legacy_record = json.loads( components.store.get("vm_firewall", record_key)) legacy_record.pop("filter_version") legacy_record["filter_digest"] = legacy_digest components.store.set( "vm_firewall", record_key, json.dumps(legacy_record, ensure_ascii=False, separators=(",", ":")), ) service.firewall_filter_digest = "e" * 64 applies = sum(row[0] == "apply_firewall_policy" for row in service.calls) changed = call( client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/network", {**USER, "operation_id": "network-firewall-drift-0002"}, { "public_ip_cidr": "8.8.8.6/28", "confirm_restart": True, }, ) assert changed.status_code == 409, changed.text assert changed.json()["error_code"] == "firewall_drift_detected" assert sum(row[0] == "apply_firewall_policy" for row in service.calls) == applies assert not any(row[0] == "change_cloud_vm_network" for row in service.calls) @pytest.mark.parametrize("tracked", [False, True]) def test_network_change_rejects_untracked_or_current_firewall_drift( boundary, tracked, ): client, engine, _store, _, services = boundary assert call(client, "GET", "/internal/v1/hosts/hk/vms").status_code == 200 components = engine.registry.components["hk"] profile = components.manager.profile profile.update({ "host": "8.8.8.3", "default_network": "bridge:br0", "public_pool_cidr": "8.8.8.0/28", "public_gateway": "8.8.8.1", "public_dns_servers": ["1.1.1.1"], "public_reserved_ips": ["8.8.8.2"], "public_vm_prefix": 28, }) service = services["hk"] service.vm.update( managed=True, os_source="cloud_template", public_ip_cidr="8.8.8.4/28", ) scope = host_scope(profile) components.state.users.assign_vm(scope, VM_ID, USER["user_id"]) if tracked: body = firewall_body() applied = call( client, "PUT", f"/internal/v1/hosts/hk/vms/{VM_ID}/firewall", {**USER, "operation_id": "network-current-drift-0001"}, body, ) assert applied.status_code == 200, applied.text service.firewall_filter_digest = "d" * 64 applies = sum(row[0] == "apply_firewall_policy" for row in service.calls) changed = call( client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/network", {**USER, "operation_id": "network-current-drift-0002"}, { "public_ip_cidr": "8.8.8.6/28", "confirm_restart": True, }, ) assert changed.status_code == 409, changed.text assert changed.json()["error_code"] == "firewall_drift_detected" assert sum(row[0] == "apply_firewall_policy" for row in service.calls) == applies assert not any(row[0] == "change_cloud_vm_network" for row in service.calls) @pytest.mark.parametrize("error_code,status", [ ("firewall_apply_failed", 502), ("firewall_result_uncertain", 503), ]) def test_network_change_firewall_preflight_failure_is_not_ip_uncertain( boundary, error_code, status, ): client, engine, store, _, services = boundary assert call(client, "GET", "/internal/v1/hosts/hk/vms").status_code == 200 components = engine.registry.components["hk"] profile = components.manager.profile profile.update({ "host": "8.8.8.3", "default_network": "bridge:br0", "public_pool_cidr": "8.8.8.0/28", "public_gateway": "8.8.8.1", "public_dns_servers": ["1.1.1.1"], "public_reserved_ips": ["8.8.8.2"], "public_vm_prefix": 28, }) service = services["hk"] service.vm.update( managed=True, os_source="cloud_template", public_ip_cidr="8.8.8.4/28", ) scope = host_scope(profile) components.state.users.assign_vm(scope, VM_ID, USER["user_id"]) body = firewall_body() applied = call( client, "PUT", f"/internal/v1/hosts/hk/vms/{VM_ID}/firewall", {**USER, "operation_id": f"network-preflight-{status}-0001"}, body, ) assert applied.status_code == 200, applied.text record_key = f"{scope}.{VM_ID}" legacy_digest = build_firewall_filter_xml( VM_ID, body["enabled"], body["rules"], compiler_version=1, )[3] legacy_record = json.loads( components.store.get("vm_firewall", record_key)) legacy_record.pop("filter_version") legacy_record["filter_digest"] = legacy_digest components.store.set( "vm_firewall", record_key, json.dumps(legacy_record, ensure_ascii=False, separators=(",", ":")), ) service.firewall_filter_digest = legacy_digest service.firewall_apply_error = VMError( "firewall preflight failed", status, error_code=error_code) operation_id = f"network-preflight-{status}-0002" changed = call( client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/network", {**USER, "operation_id": operation_id}, { "public_ip_cidr": "8.8.8.6/28", "confirm_restart": True, }, ) assert changed.status_code == status, changed.text assert changed.json()["status"] == "failed" assert changed.json()["error_code"] == error_code operation = json.loads(store.get("operation", operation_id)) assert operation["status"] == "failed" assert operation["network_mutation_started"] is False polled = call( client, "GET", f"/internal/v1/operations/{operation_id}", USER) assert polled.status_code == 200, polled.text assert polled.json()["status"] == "failed" assert polled.json()["error_code"] == error_code assert components.store.get("ip_claim", f"{scope}.8.8.8.6") is None assert not any(row[0] == "change_cloud_vm_network" for row in service.calls) def test_network_change_capture_failure_is_terminal_before_ip_claim(boundary): client, engine, store, _, services = boundary assert call(client, "GET", "/internal/v1/hosts/hk/vms").status_code == 200 components = engine.registry.components["hk"] profile = components.manager.profile profile.update({ "host": "8.8.8.3", "default_network": "bridge:br0", "public_pool_cidr": "8.8.8.0/28", "public_gateway": "8.8.8.1", "public_dns_servers": ["1.1.1.1"], "public_reserved_ips": ["8.8.8.2"], "public_vm_prefix": 28, }) service = services["hk"] service.vm.update( managed=True, os_source="cloud_template", public_ip_cidr="8.8.8.4/28", ) scope = host_scope(profile) components.state.users.assign_vm(scope, VM_ID, USER["user_id"]) service.get_vm = MagicMock(side_effect=RuntimeError("capture failed")) operation_id = "network-capture-failed-0001" changed = call( client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/network", {**USER, "operation_id": operation_id}, { "public_ip_cidr": "8.8.8.6/28", "confirm_restart": True, }, ) assert changed.status_code == 503, changed.text operation = json.loads(store.get("operation", operation_id)) assert operation["status"] == "failed" assert operation["network_mutation_started"] is False polled = call( client, "GET", f"/internal/v1/operations/{operation_id}", USER) assert polled.status_code == 200, polled.text assert polled.json()["status"] == "failed" assert "error_code" not in polled.json() assert components.store.get("ip_claim", f"{scope}.8.8.8.6") is None assert not any(row[0] == "change_cloud_vm_network" for row in service.calls) def test_deleted_vm_retries_only_public_ip_claim_cleanup(boundary): client, engine, store, _, _services = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") components = engine.registry.components["hk"] scope = host_scope(components.manager.profile) claim_key = f"{scope}.8.8.8.4" components.store.set("ip_claim", claim_key, json.dumps({ "vm_uuid": VM_ID, })) operation = { "operation_id": "delete-claim-cleanup", "host_id": "hk", "user_id": USER["user_id"], "vm_uuid": VM_ID, "kind": "mutation", "resource": f"vms/{VM_ID}", "method": "DELETE", "status": "uncertain", "result": None, "safe_to_refund": False, "cleanup_pending": "public_ip_claim", "delete_ip_claim_keys": [claim_key], "created_at": 1, } store.set("operation", operation["operation_id"], json.dumps(operation)) recovered = call( client, "GET", f"/internal/v1/operations/{operation['operation_id']}", USER, ) assert recovered.status_code == 200 assert recovered.json()["status"] == "succeeded" assert recovered.json()["result"]["public_ip_claims_released"] is True assert components.store.get("ip_claim", claim_key) is None foreign_key = f"{scope}.8.8.8.5" components.store.set("ip_claim", foreign_key, json.dumps({ "vm_uuid": "22222222-2222-4222-8222-222222222222", })) foreign = { **operation, "operation_id": "delete-foreign-claim", "status": "uncertain", "cleanup_pending": "public_ip_claim", "delete_ip_claim_keys": [foreign_key], } store.set("operation", foreign["operation_id"], json.dumps(foreign)) blocked = call( client, "GET", f"/internal/v1/operations/{foreign['operation_id']}", USER, ) assert blocked.json()["status"] == "uncertain" assert components.store.get("ip_claim", foreign_key) is not None def test_service_resume_replays_exact_start_target_after_lost_response(boundary): client, engine, store, _, services = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") components = engine.registry.components["hk"] components.state.users.assign_vm( host_scope(components.manager.profile), VM_ID, USER["user_id"]) services["hk"].vm.update(state="shutoff", autostart=False) actor = {**USER, "operation_id": "resume-start-target"} path = f"/internal/v1/hosts/hk/vms/{VM_ID}/service-resume" first = call(client, "POST", path, actor, {"start_if_suspended": True}) assert first.status_code == 200, first.text assert services["hk"].vm["state"] == "running" assert services["hk"].calls[-1] == ("service_resume", VM_ID, True) saved = json.loads(store.get("operation", actor["operation_id"])) saved.update(status="uncertain", result=None, error="response lost") store.set("operation", actor["operation_id"], json.dumps(saved)) services["hk"].vm.update(state="shutoff", autostart=False) engine.active_operations.clear() replayed = call(client, "GET", "/internal/v1/operations/resume-start-target", actor) assert replayed.status_code == 200 assert replayed.json()["status"] == "succeeded" assert services["hk"].vm["state"] == "running" assert services["hk"].calls[-1] == ("service_resume", VM_ID, True) def test_service_suspend_replay_preserves_original_running_state(boundary): client, engine, store, _, services = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") components = engine.registry.components["hk"] components.state.users.assign_vm( host_scope(components.manager.profile), VM_ID, USER["user_id"]) actor = {**USER, "operation_id": "suspend-running-hint"} path = f"/internal/v1/hosts/hk/vms/{VM_ID}/service-suspend" assert call(client, "POST", path, actor, {}).status_code == 200 saved = json.loads(store.get("operation", actor["operation_id"])) assert saved["service_was_running_hint"] is True # Model a hard crash after libvirt stopped the guest but before the # successful result was durable. Reconciliation now observes shutoff, yet # must retain the original running state for the billing resume policy. saved.update(status="uncertain", result=None, error="engine crashed") store.set("operation", actor["operation_id"], json.dumps(saved)) services["hk"].vm.update(state="shutoff", autostart=False) engine.active_operations.clear() reconciled = call(client, "GET", "/internal/v1/operations/suspend-running-hint", actor) assert reconciled.status_code == 200 assert reconciled.json()["status"] == "succeeded" assert reconciled.json()["result"]["service_was_running"] is True def test_service_suspend_replay_without_running_hint_fails_closed(boundary): client, engine, store, _, services = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") components = engine.registry.components["hk"] components.state.users.assign_vm( host_scope(components.manager.profile), VM_ID, USER["user_id"]) operation_id = "suspend-missing-running-hint" store.set("operation", operation_id, json.dumps({ "operation_id": operation_id, "host_id": "hk", "user_id": USER["user_id"], "vm_uuid": VM_ID, "kind": "mutation", "resource": f"vms/{VM_ID}/service-suspend", "method": "POST", "status": "uncertain", "actor_can_manage": True, "actor_is_admin": False, "created_at": 1, })) before_calls = list(services["hk"].calls) response = call( client, "GET", f"/internal/v1/operations/{operation_id}", USER) assert response.status_code == 200 assert response.json()["status"] == "failed" assert "已取消服务暂停" in response.json()["error"] assert services["hk"].calls == before_calls def test_service_suspend_does_not_stop_vm_when_running_hint_cannot_persist( boundary, monkeypatch): client, engine, store, _, services = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") components = engine.registry.components["hk"] components.state.users.assign_vm( host_scope(components.manager.profile), VM_ID, USER["user_id"]) operation_id = "suspend-hint-write-failed" original_set = store.set def fail_operation_write(kind, key, value): if kind == "operation" and key == operation_id: raise RuntimeError("database unavailable") return original_set(kind, key, value) monkeypatch.setattr(store, "set", fail_operation_write) failed = call( client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/service-suspend", {**USER, "operation_id": operation_id}, {}) assert failed.status_code == 503 assert services["hk"].vm["state"] == "running" assert not any( item[0] == "service_suspend" for item in services["hk"].calls) monkeypatch.setattr(store, "set", original_set) recovered = call( client, "GET", f"/internal/v1/operations/{operation_id}", USER) assert recovered.status_code == 200 assert recovered.json()["status"] == "failed" assert services["hk"].vm["state"] == "running" assert not any( item[0] == "service_suspend" for item in services["hk"].calls) def test_metrics_query_uses_log_store_and_owner_permission(boundary): client, engine, _, logs, _ = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") component = engine.registry.components["hk"] scope = host_scope(component.manager.profile) component.state.users.assign_vm(scope, VM_ID, 41) response = call(client, "GET", f"/internal/v1/hosts/hk/vms/{VM_ID}/metrics?range=30d") assert response.status_code == 200 and response.json()["points"][0]["cpu_percent"] == 20 with patch("virsh_panel.app.time.time", return_value=3_000_000): traffic = call( client, "GET", f"/internal/v1/hosts/hk/vms/{VM_ID}/traffic/daily?days=2", ) assert traffic.status_code == 200 assert traffic.json()["points"][-1] == { "date": "1970-02-04", "day_start": 2_937_600, "download_bytes": 4096, "upload_bytes": 1024, "total_bytes": 5120, } assert logs.daily_traffic_calls == [(scope, VM_ID, { "host_id": "hk", "user_id": 41, "since": 2_851_200, "until": 3_000_000, "limit": 2, })] denied = call( client, "GET", f"/internal/v1/hosts/hk/vms/{VM_ID}/traffic/daily", {**USER, "user_id": 42}, ) assert denied.status_code == 404 assert len(logs.daily_traffic_calls) == 1 def test_console_ticket_is_one_use_session_bound_and_no_url_secret(boundary): client, engine, store, *_ = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") component = engine.registry.components["hk"] component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41) issued = call(client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/console-ticket", USER, {}) assert issued.status_code == 200, issued.text assert issued.json()["websocket_path"] == "/engine-console" ticket = issued.json()["ticket"] key = hashlib.sha256(ticket.encode()).hexdigest() record = json.loads(store.get("console_ticket", key)) assert record["actor"]["user_id"] == 41 and record["actor"]["session_id"] == "test-session" assert call(client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/console-ticket", {**USER, "user_id": 42}, {}).status_code == 404 from starlette.websockets import WebSocketDisconnect with pytest.raises(WebSocketDisconnect): with client.websocket_connect("wss://panel.example.test/engine-console", headers={"Origin": "https://evil.example"}, subprotocols=["cloud-ticket." + ticket]): pass assert store.get("console_ticket", key) is not None def test_console_consumes_ticket_and_closes_after_php_session_revocation(boundary): client, engine, store, *_ = boundary call(client, "GET", "/internal/v1/hosts/hk/vms") component = engine.registry.components["hk"] component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41) checks = 0 async def active(*args): nonlocal checks checks += 1 return checks <= 2 # Ticket issuance + handshake; then PHP revokes. engine.introspection = active engine.console_check_interval = 0.01 issued = call(client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/console-ticket", USER, {}) ticket = issued.json()["ticket"] protocol = "cloud-ticket." + ticket async def relay(socket, **kwargs): await socket.send_bytes(b"vnc") await asyncio.Event().wait() from starlette.websockets import WebSocketDisconnect with patch("virsh_panel.console.relay_console", relay): with client.websocket_connect("wss://panel.example.test/engine-console", headers={"Origin": "https://panel.example.test"}, subprotocols=[protocol]) as socket: assert socket.accepted_subprotocol == protocol assert socket.receive_bytes() == b"vnc" with pytest.raises(WebSocketDisconnect): socket.receive_bytes() key = hashlib.sha256(ticket.encode()).hexdigest() assert store.get("console_ticket", key) is None assert store.get("ticket_used", key) == "1" with pytest.raises(WebSocketDisconnect): with client.websocket_connect("wss://panel.example.test/engine-console", headers={"Origin": "https://panel.example.test"}, subprotocols=[protocol]): pass def test_streamed_iso_digest_must_match_before_import_is_queued(boundary): client, engine, store, _, services = boundary path = "/internal/v1/hosts/hk/rescue-templates?name=rescue" signed_body = b"A" * (1024 * 1024) wrong_body = b"B" * len(signed_body) signed = signed_headers(SECRET, "POST", path, signed_body, SUPER) signed["X-Cloud-Content-SHA256"] = hashlib.sha256(signed_body).hexdigest() signed["Content-Type"] = "application/octet-stream" response = client.post(path, content=wrong_body, headers=signed) assert response.status_code == 401, response.text assert not any(row[0] == "import_rescue_iso" for row in services["hk"].calls) assert engine.registry.components["hk"].store.list("rescue_import_job") == {} def cloud_upload_fixture(size=1024 * 1024 + 257): header = bytearray(104) header[:4] = b"QFI\xfb" header[4:8] = struct.pack(">I", 3) header[20:24] = struct.pack(">I", 16) header[24:32] = struct.pack(">Q", 20 * 1024**3) header[100:104] = struct.pack(">I", 104) return bytes(header) + b"\0" * (size - len(header)) def wait_for_cloud_upload_job(client, job_id): path = f"/internal/v1/vm-templates/import-jobs/{job_id}" for _ in range(200): response = call(client, "GET", path, SUPER) assert response.status_code == 200, response.text job = response.json() if job["status"] in {"completed", "failed"}: return job time.sleep(0.01) raise AssertionError("uploaded cloud template import did not finish") def cloud_engine_scope(method, target, body, actor, *, headers=None): path, separator, query = target.partition("?") request_headers = signed_headers(SECRET, method, target, body, actor) request_headers.update(headers or {}) return { "type": "http", "asgi": {"version": "3.0"}, "http_version": "1.1", "method": method, "scheme": "http", "path": path, "raw_path": path.encode("ascii"), "query_string": query.encode("ascii") if separator else b"", "headers": [ (name.lower().encode("ascii"), value.encode("latin-1")) for name, value in request_headers.items() ], "client": ("testclient", 50000), "server": ("127.0.0.1", 80), } async def cloud_engine_request(engine, method, target, body, actor, *, headers=None, receive=None): scope = cloud_engine_scope( method, target, body, actor, headers=headers) delivered = False async def receive_body(): nonlocal delivered if delivered: return {"type": "http.disconnect"} delivered = True return {"type": "http.request", "body": body, "more_body": False} messages = [] async def send(message): messages.append(message) await engine(scope, receive or receive_body, send) status = next(message["status"] for message in messages if message["type"] == "http.response.start") raw = b"".join(message.get("body", b"") for message in messages if message["type"] == "http.response.body") return status, json.loads(raw) def test_global_cloud_template_upload_streams_past_json_limit(boundary): client, engine, *_ = boundary query = urlencode({ "name": "Debian 12 fixed", "os_family": "Debian", "os_version": "12", }) path = "/internal/v1/vm-templates/upload?" + query payload = cloud_upload_fixture() with patch("virsh_panel.cloud_templates._verify_qcow2_with_qemu"): response = call(client, "POST", path, SUPER, raw=payload, headers={ "X-Cloud-Content-SHA256": hashlib.sha256(payload).hexdigest(), "Content-Type": "application/octet-stream", }) assert response.status_code == 202, response.text job = wait_for_cloud_upload_job(client, response.json()["job_id"]) assert job["status"] == "completed", job assert job["template"]["name"] == "Debian 12 fixed" assert job["template"]["os_family"] == "Debian" assert job["template"]["os_version"] == "12" assert job["template"]["virtual_size"] == 20 * 1024**3 with engine.registry.cloud_templates.source( job["template"]["id"]) as (_record, source): assert source.stat().st_size == len(payload) assert hashlib.sha256(source.read_bytes()).digest() == hashlib.sha256( payload).digest() def test_global_cloud_template_upload_digest_must_match_before_queue(boundary): client, engine, *_ = boundary path = ("/internal/v1/vm-templates/upload?" "name=Debian+12&os_family=Debian&os_version=12") signed_body = cloud_upload_fixture() wrong_body = signed_body[:-1] + b"x" signed = signed_headers(SECRET, "POST", path, signed_body, SUPER) signed["X-Cloud-Content-SHA256"] = hashlib.sha256( signed_body).hexdigest() signed["Content-Type"] = "application/octet-stream" response = client.post(path, content=wrong_body, headers=signed) assert response.status_code == 401, response.text assert engine.registry.cloud_templates.jobs == {} assert engine.registry.cloud_templates.pending == set() root = engine.registry.cloud_templates.files.directory() assert not list(root.glob(".cloud-image-upload-*.qcow2")) @pytest.mark.parametrize( ("family", "version"), [ ("debian", "12"), ("Debian", "14"), ("CentOS", "10"), ("Ubuntu", "24"), ], ) def test_global_cloud_template_upload_rejects_noncanonical_metadata( boundary, family, version): client, engine, *_ = boundary path = "/internal/v1/vm-templates/upload?" + urlencode({ "name": "Invalid metadata", "os_family": family, "os_version": version, }) payload = cloud_upload_fixture() response = call(client, "POST", path, SUPER, raw=payload, headers={ "X-Cloud-Content-SHA256": hashlib.sha256(payload).hexdigest(), "Content-Type": "application/octet-stream", }) assert response.status_code == 422, response.text assert engine.registry.cloud_templates.jobs == {} def test_global_cloud_template_upload_requires_super_admin_and_octet_stream( boundary): client, engine, *_ = boundary path = ("/internal/v1/vm-templates/upload?" "name=Debian+12&os_family=Debian&os_version=12") payload = cloud_upload_fixture() digest = hashlib.sha256(payload).hexdigest() forbidden = call(client, "POST", path, ADMIN, raw=payload, headers={ "X-Cloud-Content-SHA256": digest, "Content-Type": "application/octet-stream", }) assert forbidden.status_code == 403, forbidden.text wrong_type = call(client, "POST", path, SUPER, raw=payload, headers={ "X-Cloud-Content-SHA256": digest, "Content-Type": "multipart/form-data", }) assert wrong_type.status_code == 415, wrong_type.text assert engine.registry.cloud_templates.jobs == {} def test_global_cloud_template_upload_enforces_minimum_size(boundary): client, engine, *_ = boundary path = ("/internal/v1/vm-templates/upload?" "name=Too+Small&os_family=Debian&os_version=12") payload = cloud_upload_fixture(104) response = call(client, "POST", path, SUPER, raw=payload, headers={ "X-Cloud-Content-SHA256": hashlib.sha256(payload).hexdigest(), "Content-Type": "application/octet-stream", }) assert response.status_code == 422, response.text assert engine.registry.cloud_templates.jobs == {} assert engine.registry.cloud_templates.pending == set() def test_global_cloud_template_upload_timeout_releases_reservation( boundary, monkeypatch): _client, engine, *_ = boundary query = b"name=Timeout&os_family=Debian&os_version=12" scope = { "query_string": query, "headers": [ (b"content-type", b"application/octet-stream"), (b"content-length", str(1024 * 1024).encode()), ], "engine_upload_validation": {"complete": False, "valid": False}, "engine_upload_digest": "0" * 64, } async def stalled_receive(): await asyncio.sleep(1) return {"type": "http.request", "body": b"", "more_body": False} async def send(_message): raise AssertionError("timeout must not start a response") monkeypatch.setattr("virsh_panel.internal.CLOUD_UPLOAD_IDLE_SECONDS", 0.01) with pytest.raises(TemplateError, match="上传超时") as timeout: asyncio.run(engine._upload_cloud_template(scope, stalled_receive, send)) assert timeout.value.status == 408 assert engine.registry.cloud_templates.pending == set() assert engine.registry.cloud_templates.upload_reservations == {} root = engine.registry.cloud_templates.files.directory() assert not list(root.glob(".cloud-image-upload-*.qcow2")) def test_stalled_cloud_upload_does_not_block_other_engine_writes( boundary, monkeypatch): _client, engine, *_ = boundary payload = cloud_upload_fixture() upload_path = "/internal/v1/vm-templates/upload?" + urlencode({ "name": "Stalled upload", "os_family": "Debian", "os_version": "12", }) upload_started = asyncio.Event() release_upload = asyncio.Event() class FastRemoteHostManager: def __init__(self, *_args, **_kwargs): pass @staticmethod def scan(host, port): return {"host": host, "port": port, "reachable": True} @staticmethod def close(): pass monkeypatch.setattr( "virsh_panel.remote_host.RemoteHostManager", FastRemoteHostManager) async def stalled_receive(): upload_started.set() await release_upload.wait() return {"type": "http.disconnect"} async def scenario(): upload = asyncio.create_task(cloud_engine_request( engine, "POST", upload_path, payload, SUPER, headers={ "X-Cloud-Content-SHA256": hashlib.sha256(payload).hexdigest(), "Content-Type": "application/octet-stream", "Content-Length": str(len(payload)), }, receive=stalled_receive, )) await asyncio.wait_for(upload_started.wait(), timeout=1) assert engine.registry.cloud_templates.pending scan_body = json.dumps({ "host": "host.example.test", "port": 22, }, separators=(",", ":")).encode() status, result = await asyncio.wait_for(cloud_engine_request( engine, "POST", "/internal/v1/hosts/scan", scan_body, SUPER, headers={"Content-Type": "application/json"}, ), timeout=0.5) assert status == 200 assert result["host"] == "host.example.test" release_upload.set() upload_status, _result = await asyncio.wait_for(upload, timeout=1) assert upload_status == 400 asyncio.run(scenario()) assert engine.registry.cloud_templates.pending == set() assert engine.registry.cloud_templates.upload_reservations == {} def test_cloud_upload_fsync_does_not_block_event_loop_or_engine_writes( boundary, monkeypatch): _client, engine, *_ = boundary payload = cloud_upload_fixture() upload_path = "/internal/v1/vm-templates/upload?" + urlencode({ "name": "Slow fsync upload", "os_family": "Debian", "os_version": "12", }) fsync_started = threading.Event() release_fsync = threading.Event() original_fsync = os.fsync def slow_fsync(descriptor): fsync_started.set() assert release_fsync.wait(timeout=2), "test did not release fsync" return original_fsync(descriptor) class FastRemoteHostManager: def __init__(self, *_args, **_kwargs): pass @staticmethod def scan(host, port): return {"host": host, "port": port, "reachable": True} @staticmethod def close(): pass monkeypatch.setattr("virsh_panel.internal.os.fsync", slow_fsync) monkeypatch.setattr( "virsh_panel.remote_host.RemoteHostManager", FastRemoteHostManager) monkeypatch.setattr( "virsh_panel.cloud_templates._verify_qcow2_with_qemu", lambda *_args: None) async def scenario(): safety_release = threading.Timer(1.5, release_fsync.set) safety_release.start() started_at = time.monotonic() try: upload = asyncio.create_task(cloud_engine_request( engine, "POST", upload_path, payload, SUPER, headers={ "X-Cloud-Content-SHA256": hashlib.sha256(payload).hexdigest(), "Content-Type": "application/octet-stream", "Content-Length": str(len(payload)), }, )) while not fsync_started.is_set(): await asyncio.sleep(0.001) scan_body = json.dumps({ "host": "host.example.test", "port": 22, }, separators=(",", ":")).encode() status, _result = await asyncio.wait_for(cloud_engine_request( engine, "POST", "/internal/v1/hosts/scan", scan_body, SUPER, headers={"Content-Type": "application/json"}, ), timeout=0.5) assert status == 200 assert time.monotonic() - started_at < 0.5 assert not release_fsync.is_set() release_fsync.set() upload_status, _result = await asyncio.wait_for(upload, timeout=2) assert upload_status == 202 finally: release_fsync.set() safety_release.cancel() asyncio.run(scenario()) def test_cloud_upload_rechecks_update_state_before_queue( boundary, monkeypatch): _client, engine, *_ = boundary updater = FakeUpdater() engine.updater = updater payload = cloud_upload_fixture() upload_path = "/internal/v1/vm-templates/upload?" + urlencode({ "name": "Update race upload", "os_family": "Debian", "os_version": "12", }) upload_started = asyncio.Event() release_upload = asyncio.Event() delivered = False monkeypatch.setattr( "virsh_panel.cloud_templates._verify_qcow2_with_qemu", lambda *_args: None) async def delayed_receive(): nonlocal delivered if delivered: return {"type": "http.disconnect"} delivered = True upload_started.set() await release_upload.wait() return {"type": "http.request", "body": payload, "more_body": False} async def scenario(): upload = asyncio.create_task(cloud_engine_request( engine, "POST", upload_path, payload, SUPER, headers={ "X-Cloud-Content-SHA256": hashlib.sha256(payload).hexdigest(), "Content-Type": "application/octet-stream", "Content-Length": str(len(payload)), }, receive=delayed_receive, )) await asyncio.wait_for(upload_started.wait(), timeout=1) assert engine.registry.cloud_templates.pending updater.active = { "job_id": "update-job-1234567890", "status": "running", } release_upload.set() status, result = await asyncio.wait_for(upload, timeout=2) assert status == 409 assert result["error"] == "面板正在更新,暂时不能执行虚拟化写操作" try: asyncio.run(scenario()) finally: updater.active = None assert engine.registry.cloud_templates.pending == set() assert engine.registry.cloud_templates.upload_reservations == {} assert engine.registry.cloud_templates.jobs == {} def test_upload_same_os_version_keeps_existing_template(boundary): client, engine, *_ = boundary def upload(name, payload): path = "/internal/v1/vm-templates/upload?" + urlencode({ "name": name, "os_family": "Debian", "os_version": "12", }) response = call(client, "POST", path, SUPER, raw=payload, headers={ "X-Cloud-Content-SHA256": hashlib.sha256(payload).hexdigest(), "Content-Type": "application/octet-stream", }) assert response.status_code == 202, response.text job = wait_for_cloud_upload_job(client, response.json()["job_id"]) assert job["status"] == "completed", job return job["template"] official_payload = cloud_upload_fixture()[:-1] + b"o" fixed_payload = cloud_upload_fixture()[:-1] + b"f" with patch("virsh_panel.cloud_templates._verify_qcow2_with_qemu"): official = upload("Debian 12 official", official_payload) fixed = upload("Debian 12 fixed", fixed_payload) assert official["id"] != fixed["id"] templates = call( client, "GET", "/internal/v1/vm-templates", SUPER).json()["templates"] assert {row["id"] for row in templates} == {official["id"], fixed["id"]} with engine.registry.cloud_templates.source( official["id"]) as (_record, source): assert hashlib.sha256(source.read_bytes()).hexdigest() == official["sha256"] def test_firewall_contract_persists_round_trips_and_is_revision_idempotent( boundary): client, engine, _store, _logs, services = boundary assert call( client, "GET", "/internal/v1/hosts/hk/vms", SUPER, ).status_code == 200 components = engine.registry.components["hk"] scope = host_scope(components.manager.profile) components.state.users.assign_vm(scope, VM_ID, USER["user_id"]) path = f"/internal/v1/hosts/hk/vms/{VM_ID}/firewall" baseline = call(client, "GET", path, USER) assert baseline.status_code == 200, baseline.text assert baseline.json() == { "enabled": False, "revision": 0, "policy_digest": firewall_policy_digest(False, []), "filter_digest": None, "rules": [], "sync_state": "active", } rejected = call( client, "PUT", path, {**USER, "operation_id": "firewall-bad-digest"}, {**firewall_body(), "policy_digest": "0" * 64}, ) assert rejected.status_code == 422, rejected.text assert rejected.json()["error_code"] == "firewall_rule_invalid" assert not any(row[0] == "apply_firewall_policy" for row in services["hk"].calls) class ForbiddenUpdateLock: async def __aenter__(self): raise AssertionError("firewall acquired the engine-wide update lock") async def __aexit__(self, *_args): return False original_target_lock = engine._target_lock target_entries = [] @asynccontextmanager async def observed_target_lock(host_id, vm_id): target_entries.append((host_id, vm_id)) async with original_target_lock(host_id, vm_id): yield engine._update_lock = lambda: ForbiddenUpdateLock() engine._target_lock = observed_target_lock body = firewall_body() assert body["policy_digest"] == ( "46c4d40eef4410edaf998e17981ae91d161348e2fc6572b2bd96c8b5cbf9bfc3" ) applied = call( client, "PUT", path, {**USER, "operation_id": "firewall-apply-0001"}, body, ) assert applied.status_code == 200, applied.text assert applied.json()["revision"] == 1 assert applied.json()["policy_digest"] == body["policy_digest"] assert target_entries == [("hk", VM_ID)] record_key = f"{scope}.{VM_ID}" saved = json.loads(components.store.get("vm_firewall", record_key)) assert saved["revision"] == 1 assert saved["policy_digest"] == body["policy_digest"] fetched = call(client, "GET", path, USER) assert fetched.status_code == 200, fetched.text assert fetched.json()["revision"] == 1 assert fetched.json()["rules"] == FIREWALL_RULES applies = sum(row[0] == "apply_firewall_policy" for row in services["hk"].calls) repeated = call( client, "PUT", path, {**USER, "operation_id": "firewall-apply-0002"}, body, ) assert repeated.status_code == 200, repeated.text assert repeated.json()["revision"] == 1 assert sum(row[0] == "apply_firewall_policy" for row in services["hk"].calls) == applies jumped = firewall_body(revision=3) advanced = call( client, "PUT", path, {**USER, "operation_id": "firewall-apply-0003"}, jumped, ) assert advanced.status_code == 200, advanced.text assert advanced.json()["revision"] == 3 conflicting = call( client, "PUT", path, {**USER, "operation_id": "firewall-apply-conflict"}, firewall_body(revision=3, enabled=False, rules=[]), ) assert conflicting.status_code == 409 rollback = call( client, "PUT", path, {**USER, "operation_id": "firewall-apply-0004"}, firewall_body(revision=2), ) assert rollback.status_code == 409 def test_firewall_get_detects_drift_and_tampered_local_evidence(boundary): client, engine, _store, _logs, services = boundary assert call( client, "GET", "/internal/v1/hosts/hk/vms", SUPER, ).status_code == 200 components = engine.registry.components["hk"] scope = host_scope(components.manager.profile) components.state.users.assign_vm(scope, VM_ID, USER["user_id"]) path = f"/internal/v1/hosts/hk/vms/{VM_ID}/firewall" body = firewall_body() response = call( client, "PUT", path, {**USER, "operation_id": "firewall-drift-0001"}, body, ) assert response.status_code == 200, response.text services["hk"].firewall_filter_digest = "f" * 64 drifted = call(client, "GET", path, USER) assert drifted.status_code == 200, drifted.text assert drifted.json()["sync_state"] == "drifted" assert drifted.json()["revision"] is None same_revision = call( client, "PUT", path, {**USER, "operation_id": "firewall-drift-0002"}, body, ) assert same_revision.status_code == 409 assert same_revision.json()["error_code"] == "firewall_drift_detected" key = f"{scope}.{VM_ID}" tampered = json.loads(components.store.get("vm_firewall", key)) tampered["rules"][0]["description"] = "Tampered" components.store.set( "vm_firewall", key, json.dumps(tampered, ensure_ascii=False, separators=(",", ":")), ) corrupt = call(client, "GET", path, USER) assert corrupt.status_code == 503 assert corrupt.json()["error"] == "虚拟机防火墙记录损坏" def test_firewall_same_revision_upgrades_legacy_filter_once(boundary): client, engine, _store, _logs, services = boundary assert call( client, "GET", "/internal/v1/hosts/hk/vms", SUPER, ).status_code == 200 components = engine.registry.components["hk"] scope = host_scope(components.manager.profile) components.state.users.assign_vm(scope, VM_ID, USER["user_id"]) path = f"/internal/v1/hosts/hk/vms/{VM_ID}/firewall" body = firewall_body() applied = call( client, "PUT", path, {**USER, "operation_id": "firewall-upgrade-0001"}, body, ) assert applied.status_code == 200, applied.text record_key = f"{scope}.{VM_ID}" legacy_digest = build_firewall_filter_xml( VM_ID, body["enabled"], body["rules"], compiler_version=1, )[3] legacy_record = json.loads( components.store.get("vm_firewall", record_key)) legacy_record.pop("filter_version") legacy_record["filter_digest"] = legacy_digest components.store.set( "vm_firewall", record_key, json.dumps(legacy_record, ensure_ascii=False, separators=(",", ":")), ) services["hk"].firewall_filter_digest = legacy_digest applies = sum(row[0] == "apply_firewall_policy" for row in services["hk"].calls) upgraded = call( client, "PUT", path, {**USER, "operation_id": "firewall-upgrade-0002"}, body, ) assert upgraded.status_code == 200, upgraded.text assert sum(row[0] == "apply_firewall_policy" for row in services["hk"].calls) == applies + 1 upgraded_record = json.loads( components.store.get("vm_firewall", record_key)) current_digest = build_firewall_filter_xml( VM_ID, body["enabled"], body["rules"])[3] assert upgraded_record["filter_version"] == FIREWALL_FILTER_COMPILER_VERSION assert upgraded_record["filter_digest"] == current_digest repeated = call( client, "PUT", path, {**USER, "operation_id": "firewall-upgrade-0003"}, body, ) assert repeated.status_code == 200, repeated.text assert sum(row[0] == "apply_firewall_policy" for row in services["hk"].calls) == applies + 1 def test_firewall_uncertain_operation_reconciles_exact_host_evidence(boundary): client, engine, store, _logs, services = boundary assert call( client, "GET", "/internal/v1/hosts/hk/vms", SUPER, ).status_code == 200 components = engine.registry.components["hk"] scope = host_scope(components.manager.profile) components.state.users.assign_vm(scope, VM_ID, USER["user_id"]) path = f"/internal/v1/hosts/hk/vms/{VM_ID}/firewall" actor = {**USER, "operation_id": "firewall-lost-0001"} body = firewall_body() first = call(client, "PUT", path, actor, body) assert first.status_code == 200, first.text apply_count = sum(row[0] == "apply_firewall_policy" for row in services["hk"].calls) operation = json.loads(store.get("operation", actor["operation_id"])) operation.update(status="uncertain", result=None) operation.pop("finished_at", None) store.set("operation", actor["operation_id"], json.dumps(operation)) record_key = f"{scope}.{VM_ID}" components.store.delete("vm_firewall", record_key) class ForbiddenUpdateLock: async def __aenter__(self): raise AssertionError("firewall reconciliation acquired global lock") async def __aexit__(self, *_args): return False engine._update_lock = lambda: ForbiddenUpdateLock() recovered = call( client, "GET", f"/internal/v1/operations/{actor['operation_id']}", USER, ) assert recovered.status_code == 200, recovered.text assert recovered.json()["status"] == "succeeded" assert recovered.json()["result"]["revision"] == 1 assert recovered.json()["result"]["policy_digest"] == body["policy_digest"] assert components.store.get("vm_firewall", record_key) is not None assert sum(row[0] == "apply_firewall_policy" for row in services["hk"].calls) == apply_count old_body = body target = firewall_body(revision=2, enabled=False, rules=[]) old_operation = { **operation, "operation_id": "firewall-old-0002", "status": "uncertain", "result": None, "firewall_target": target, } store.set( "operation", old_operation["operation_id"], json.dumps(old_operation)) old_state = call( client, "GET", f"/internal/v1/operations/{old_operation['operation_id']}", USER, ) assert old_state.status_code == 200 assert old_state.json()["status"] == "failed" mixed_operation = { **old_operation, "operation_id": "firewall-mixed-0003", "status": "uncertain", } services["hk"].firewall_filter_digest = "e" * 64 store.set( "operation", mixed_operation["operation_id"], json.dumps(mixed_operation), ) mixed = call( client, "GET", f"/internal/v1/operations/{mixed_operation['operation_id']}", USER, ) assert mixed.status_code == 200 assert mixed.json()["status"] == "uncertain" assert mixed.json()["error_code"] == "firewall_result_uncertain" def test_firewall_uncertain_legacy_operation_survives_compiler_upgrade(boundary): client, engine, store, _logs, services = boundary assert call( client, "GET", "/internal/v1/hosts/hk/vms", SUPER, ).status_code == 200 components = engine.registry.components["hk"] scope = host_scope(components.manager.profile) components.state.users.assign_vm(scope, VM_ID, USER["user_id"]) path = f"/internal/v1/hosts/hk/vms/{VM_ID}/firewall" actor = {**USER, "operation_id": "firewall-legacy-lost-0001"} body = firewall_body() applied = call(client, "PUT", path, actor, body) assert applied.status_code == 200, applied.text legacy_digest = build_firewall_filter_xml( VM_ID, body["enabled"], body["rules"], compiler_version=1, )[3] record_key = f"{scope}.{VM_ID}" legacy_record = json.loads( components.store.get("vm_firewall", record_key)) legacy_record.pop("filter_version") legacy_record["filter_digest"] = legacy_digest components.store.set( "vm_firewall", record_key, json.dumps(legacy_record, ensure_ascii=False, separators=(",", ":")), ) services["hk"].firewall_filter_digest = legacy_digest operation = json.loads(store.get("operation", actor["operation_id"])) operation["firewall_target"].pop("filter_version") operation.update(status="uncertain", result=None) operation.pop("finished_at", None) store.set("operation", actor["operation_id"], json.dumps(operation)) apply_count = sum( row[0] == "apply_firewall_policy" for row in services["hk"].calls) recovered = call( client, "GET", f"/internal/v1/operations/{actor['operation_id']}", USER, ) assert recovered.status_code == 200, recovered.text assert recovered.json()["status"] == "succeeded" assert recovered.json()["result"]["filter_version"] == 1 recovered_record = json.loads( components.store.get("vm_firewall", record_key)) assert recovered_record["filter_version"] == 1 assert recovered_record["filter_digest"] == legacy_digest assert sum(row[0] == "apply_firewall_policy" for row in services["hk"].calls) == apply_count def test_firewall_old_uncertain_operation_cannot_downgrade_new_record(boundary): client, engine, store, _logs, services = boundary assert call( client, "GET", "/internal/v1/hosts/hk/vms", SUPER, ).status_code == 200 components = engine.registry.components["hk"] scope = host_scope(components.manager.profile) components.state.users.assign_vm(scope, VM_ID, USER["user_id"]) path = f"/internal/v1/hosts/hk/vms/{VM_ID}/firewall" body_v1 = firewall_body(revision=1) actor_v1 = {**USER, "operation_id": "firewall-old-lost-0001"} first = call(client, "PUT", path, actor_v1, body_v1) assert first.status_code == 200, first.text old_operation = json.loads( store.get("operation", actor_v1["operation_id"])) body_v2 = firewall_body(revision=2) second = call( client, "PUT", path, {**USER, "operation_id": "firewall-newer-0002"}, body_v2, ) assert second.status_code == 200, second.text record_key = f"{scope}.{VM_ID}" newer_record = components.store.get("vm_firewall", record_key) apply_count = sum( row[0] == "apply_firewall_policy" for row in services["hk"].calls) old_operation.update(status="uncertain", result=None) old_operation.pop("finished_at", None) store.set( "operation", actor_v1["operation_id"], json.dumps(old_operation)) reconciled = call( client, "GET", f"/internal/v1/operations/{actor_v1['operation_id']}", USER, ) assert reconciled.status_code == 200, reconciled.text assert reconciled.json()["status"] == "uncertain" assert reconciled.json()["error_code"] == "firewall_result_uncertain" assert components.store.get("vm_firewall", record_key) == newer_record assert json.loads(newer_record)["revision"] == 2 assert sum(row[0] == "apply_firewall_policy" for row in services["hk"].calls) == apply_count def test_deleted_vm_retries_firewall_cleanup_without_replaying_delete(boundary): client, engine, _store, _logs, services = boundary assert call( client, "GET", "/internal/v1/hosts/hk/vms", SUPER, ).status_code == 200 components = engine.registry.components["hk"] scope = host_scope(components.manager.profile) components.state.users.assign_vm(scope, VM_ID, USER["user_id"]) firewall_path = f"/internal/v1/hosts/hk/vms/{VM_ID}/firewall" applied = call( client, "PUT", firewall_path, {**USER, "operation_id": "firewall-before-delete"}, firewall_body(), ) assert applied.status_code == 200, applied.text record_key = f"{scope}.{VM_ID}" assert components.store.get("vm_firewall", record_key) is not None services["hk"].firewall_cleanup_error = VMError( "cleanup unavailable", 502, error_code="firewall_cleanup_pending") actor = {**USER, "operation_id": "delete-firewall-pending"} deleted = call( client, "DELETE", f"/internal/v1/hosts/hk/vms/{VM_ID}?delete_disks=true", actor, ) assert deleted.status_code == 503, deleted.text assert deleted.json()["error_code"] == "vm_delete_firewall_cleanup_pending" assert sum(row[0] == "delete_vm" for row in services["hk"].calls) == 1 services["hk"].firewall_cleanup_error = None recovered = call( client, "GET", f"/internal/v1/operations/{actor['operation_id']}", USER, ) assert recovered.status_code == 200, recovered.text assert recovered.json()["status"] == "succeeded" assert recovered.json()["result"]["firewall_cleaned"] is True assert components.store.get("vm_firewall", record_key) is None assert sum(row[0] == "delete_vm" for row in services["hk"].calls) == 1 def test_main_and_log_schema_do_not_create_python_accounts_or_cross_database_tables(): connection = MagicMock() cursor = connection.cursor.return_value.__enter__.return_value config = {"host": "db", "port": 3306, "database": "main", "username": "engine", "password": "test", "ssl_ca": "/private/test/ca.pem"} with patch("virsh_panel.database.pymysql.connect", return_value=connection): main = MainStore(config, b"m" * 32) statements = [call.args[0] for call in cursor.execute.call_args_list] assert any("engine_records" in statement for statement in statements) assert not any("CREATE TABLE IF NOT EXISTS users" in statement for statement in statements) assert not any("vm_metric_samples" in statement for statement in statements) assert main._config["ssl_verify_identity"] is True cursor.reset_mock() LogStore({**config, "database": "logs"}, b"m" * 32) statements = [call.args[0] for call in cursor.execute.call_args_list] assert "engine_vm_metric_samples" in statements[0] assert "engine_vm_traffic_totals" in statements[1] assert "engine_vm_traffic_checkpoints" in statements[2] assert all("ALTER TABLE engine_vm_metric_samples" in statement for statement in statements[3:]) assert LogStore.TRAFFIC_TABLE == "engine_vm_traffic_totals" assert LogStore.TRAFFIC_CHECKPOINT_TABLE == "engine_vm_traffic_checkpoints"