"""Real ASGI cancellation and durable admission, without a remote host.""" from __future__ import annotations import asyncio import base64 import gc import json import threading import time import uuid from contextlib import asynccontextmanager from types import SimpleNamespace import pytest from test_engine_boundary import ( CREATE_BODY, SECRET, SUPER, VM_ID, DatabaseTemplateStore, EngineApplication, EngineComponents, ExternalOwnership, FakeRemoteManager, FakeUpdater, HostRegistry, HostStore, MemoryStore, Service, Settings, cloud_engine_request, create_app, ) from virsh_panel.database import DatabaseError from virsh_panel.metrics import host_scope class _NoMetrics: def __init__(self, *_args, **_kwargs): pass def start(self): pass def stop(self): pass async def aclose(self): return True @pytest.fixture(autouse=True) def local_backends(monkeypatch): monkeypatch.setattr("virsh_panel.app.MetricsCollector", _NoMetrics) # These services are injected fakes. Their lifecycle must not close an # unrelated process-wide RPC runtime created by another test fixture. monkeypatch.setattr("virsh_panel.libvirt_rpc.get_runtime", lambda: None) @asynccontextmanager async def running_engine(directory, store=None): """Exercise startup, requests and shutdown on the same owning loop.""" directory.mkdir(parents=True, exist_ok=True) store = MemoryStore() if store is None else store logs = MemoryStore() config = { "service_secret": base64.b64encode(SECRET).decode(), "data_dir": str(directory), "shutdown_timeout_seconds": 0.5, } services = {} def factory(host_id, user_isos, cloud_templates): host_directory = directory / host_id host_directory.mkdir(exist_ok=True) scoped = HostStore(store, host_id) manager = FakeRemoteManager() manager.host_id = host_id manager.profile = { "host_id": host_id, "host": host_id + ".example.test", "port": 22, "fingerprint": "SHA256:example", "key_locked": False, "storage_pool": "images", "iso_pool": "iso", "rescue_iso_pool": "rescue", "default_network": "default", "libvirt_uri": "fake", "ssh_target": "fake", "ssh_config_file": "/fake/config", "ssh_known_hosts_file": "/fake/known_hosts", } service = services.setdefault(host_id, Service()) components = EngineComponents( SimpleNamespace(users=ExternalOwnership(scoped)), manager, DatabaseTemplateStore(scoped), scoped, logs, user_isos, cloud_templates, ) app = create_app( Settings(data_dir=host_directory), engine_components=components, service_factory=lambda _profile: service, ) return components, app registry = HostRegistry(config, store, logs, factory=factory) updater = FakeUpdater() updater.token = "test-only-token" async def active(*_args): return True engine = EngineApplication( config, store, logs, registry=registry, introspection=active, updater=updater, ) engine._service = lambda profile: services[profile["host_id"]] incoming, outgoing = asyncio.Queue(), asyncio.Queue() lifecycle = asyncio.create_task(engine._lifespan(incoming.get, outgoing.put)) await incoming.put({"type": "lifespan.startup"}) started = await asyncio.wait_for(outgoing.get(), 3) assert started["type"] == "lifespan.startup.complete", started try: yield engine, store, services finally: await incoming.put({"type": "lifespan.shutdown"}) stopped = await asyncio.wait_for(outgoing.get(), 3) assert stopped["type"] == "lifespan.shutdown.complete", stopped await lifecycle async def request(engine, method, target, value=None, *, operation_id=None): actor = dict(SUPER) if operation_id is not None: actor.update(operation_id=operation_id, can_provision=True) body = b"" if value is None else json.dumps(value).encode() return await cloud_engine_request( engine, method, target, body, actor, headers={"Content-Type": "application/json"}, ) async def wait_until(predicate, *, timeout=2.0): deadline = time.monotonic() + timeout while not predicate(): if time.monotonic() >= deadline: pytest.fail("test condition did not become true before its deadline") await asyncio.sleep(0.005) async def prepare_hosts(engine, *hosts): for host in hosts: status, payload = await request(engine, "GET", f"/internal/v1/hosts/{host}/vms") assert status == 200, payload def vm_path(host, resource="actions"): return f"/internal/v1/hosts/{host}/vms/{VM_ID}/{resource}" def test_stalled_host_mutation_does_not_serialize_other_hosts(tmp_path): async def run(): async with running_engine(tmp_path) as (engine, _store, services): await prepare_hosts(engine, "host-a", "host-b") entered, release = threading.Event(), threading.Event() action = services["host-a"].action def stalled_action(*args): entered.set() assert release.wait(3), "test did not release host A" return action(*args) services["host-a"].action = stalled_action first = asyncio.create_task(request( engine, "POST", vm_path("host-a"), {"action": "reset"}, operation_id="coord-host-a-reset", )) try: await wait_until(entered.is_set) read_status, _ = await asyncio.wait_for(request( engine, "GET", f"/internal/v1/hosts/host-a/vms"), 0.75) assert read_status == 200 other_status, other = await asyncio.wait_for(request( engine, "POST", vm_path("host-b"), {"action": "reset"}, operation_id="coord-host-b-reset", ), 0.75) assert other_status == 200, other assert not release.is_set() assert not first.done() finally: release.set() await asyncio.wait_for(first, 3) asyncio.run(run()) def test_stalled_vm_action_does_not_serialize_other_vms_on_same_host(tmp_path): async def run(): async with running_engine(tmp_path) as (engine, _store, services): await prepare_hosts(engine, "host-a") service = services["host-a"] other_id = "22222222-2222-4222-8222-222222222222" other_vm = {**service.vm, "uuid": other_id, "name": "second-vm"} entered, release = threading.Event(), threading.Event() get_vm, action = service.get_vm, service.action def get_either_vm(vm_id): if vm_id == other_id: service.calls.append(("get_vm", vm_id)) return other_vm.copy() return get_vm(vm_id) def stalled_action(vm_id, target): if vm_id == VM_ID: entered.set() assert release.wait(3), "test did not release the first VM" return action(vm_id, target) assert vm_id == other_id service.calls.append(("action", vm_id, target)) return other_vm.copy() service.get_vm = get_either_vm service.action = stalled_action first = asyncio.create_task(request( engine, "POST", vm_path("host-a"), {"action": "reset"}, operation_id="coord-same-host-first-vm", )) try: await wait_until(entered.is_set) other_status, other = await asyncio.wait_for(request( engine, "POST", f"/internal/v1/hosts/host-a/vms/{other_id}/actions", {"action": "reset"}, operation_id="coord-same-host-second-vm", ), 0.75) assert other_status == 200, other assert other["uuid"] == other_id assert ("action", other_id, "reset") in service.calls assert not first.done() assert not release.is_set() finally: release.set() first_status, first_result = await asyncio.wait_for(first, 3) assert first_status == 200, first_result asyncio.run(run()) def test_http_cancellation_keeps_executor_vm_claim_and_update_gate(tmp_path): async def run(): async with running_engine(tmp_path) as (engine, store, services): await prepare_hosts(engine, "host-a") service = services["host-a"] entered, release = threading.Event(), threading.Event() finished = threading.Event() suspend = service.network_suspend calls = [] def stalled_suspend(*args): calls.append(args) entered.set() assert release.wait(3), "test did not release suspended worker" try: return suspend(*args) finally: finished.set() service.network_suspend = stalled_suspend op_id = "coord-cancelled-suspend" path = vm_path("host-a", "network-suspend") first = asyncio.create_task(request( engine, "POST", path, {}, operation_id=op_id)) try: await wait_until(entered.is_set) first.cancel() with pytest.raises(asyncio.CancelledError): await first peek_status, peek = await request( engine, "GET", f"/internal/v1/operations/{op_id}?reconcile=false") assert peek_status == 200 assert peek["executor_active"] is True retry_status, retry = await request( engine, "POST", path, {}, operation_id=op_id) assert retry_status == 202, retry assert len(calls) == 1 conflict_status, conflict = await request( engine, "POST", vm_path("host-a", "network-resume"), {}, operation_id="coord-opposite-after-cancel", ) assert conflict_status == 409, conflict assert not any(call[0] == "network_resume" for call in service.calls) update_status, update = await request( engine, "POST", "/internal/v1/update/apply", { "expected_current_commit": "a" * 40, "expected_remote_commit": "b" * 40, }) assert update_status == 409, update finally: release.set() await wait_until(finished.is_set) await wait_until(lambda: json.loads( store.get("operation", op_id))["status"] == "succeeded") status, operation = await request( engine, "GET", f"/internal/v1/operations/{op_id}?reconcile=false") assert status == 200 assert operation["executor_active"] is False assert len(calls) == 1 asyncio.run(run()) def test_same_operation_retry_cannot_execute_before_first_record_insert(tmp_path): async def run(): async with running_engine(tmp_path) as (engine, store, services): await prepare_hosts(engine, "host-a") op_id = "coord-before-operation-insert" entered, release = threading.Event(), threading.Event() original_insert = store.insert first_insert = True def gated_insert(kind, key, value): nonlocal first_insert if kind == "operation" and key == op_id and first_insert: first_insert = False entered.set() assert release.wait(3), "test did not release operation insert" return original_insert(kind, key, value) store.insert = gated_insert body = {"action": "reset"} first = asyncio.create_task(request( engine, "POST", vm_path("host-a"), body, operation_id=op_id)) try: await wait_until(entered.is_set) assert store.get("operation", op_id) is None assert engine.coordinator.operation_claim(op_id) is not None status, retry = await asyncio.wait_for(request( engine, "POST", vm_path("host-a"), body, operation_id=op_id), 0.75) assert status == 409, retry assert retry["error_code"] == "coordination_busy" assert not any(call[0] == "action" for call in services["host-a"].calls) finally: release.set() first_status, first_result = await asyncio.wait_for(first, 3) assert first_status == 200, first_result assert sum(call[0] == "action" for call in services["host-a"].calls) == 1 asyncio.run(run()) def test_pending_snapshot_keeps_vm_claim_until_background_receipt(tmp_path): async def run(): async with running_engine(tmp_path) as (engine, _store, services): await prepare_hosts(engine, "host-a") service = services["host-a"] service.vm.update(managed=True, os_source="cloud_template", rescue_mode=False, rescue_staged=False) entered, release = threading.Event(), threading.Event() original = service.create_live_snapshot def stalled_snapshot(*args, **kwargs): entered.set() assert release.wait(3), "test did not release snapshot worker" return original(*args, **kwargs) service.create_live_snapshot = stalled_snapshot op_id = "coord-background-snapshot" status, queued = await request( engine, "POST", vm_path("host-a", "snapshots"), { "snapshot_key": "a" * 32, "name": "coordination recovery point", "consistency_mode": "crash", }, operation_id=op_id, ) assert status == 202, queued try: await wait_until(entered.is_set) assert engine.coordinator.operation_claim(op_id) is not None peek_status, peek = await request( engine, "GET", f"/internal/v1/operations/{op_id}?reconcile=false") assert peek_status == 200, peek assert peek["executor_active"] is True conflict_status, conflict = await request( engine, "POST", vm_path("host-a"), {"action": "reset"}, operation_id="coord-conflict-with-snapshot", ) assert conflict_status == 409, conflict assert not any(call[0] == "action" for call in service.calls) finally: release.set() await wait_until(lambda: engine.coordinator.operation_claim(op_id) is None) status, completed = await request( engine, "GET", f"/internal/v1/operations/{op_id}") assert status == 200, completed assert completed["status"] == "succeeded" assert completed["result"]["created"] is True assert sum(call[0] == "create_live_snapshot" for call in service.calls) == 1 asyncio.run(run()) class _FailedResultStore(MemoryStore): failed_operation = "coord-result-save-failure" reject_result = True def set(self, kind, key, value): if (self.reject_result and kind == "operation" and key == self.failed_operation and json.loads(value).get("status") == "succeeded"): raise DatabaseError("injected final operation persistence failure") super().set(kind, key, value) @pytest.mark.parametrize("restart", [False, True]) def test_result_save_failure_retains_unknown_claim_and_never_replays(tmp_path, restart): async def check_blocked(engine, services, op_id): await prepare_hosts(engine, "host-a") before = list(services["host-a"].calls) for method, path, value in ( ("GET", f"/internal/v1/operations/{op_id}", None), ("POST", vm_path("host-a"), {"action": "reset"}), ): status, outcome = await request( engine, method, path, value, operation_id=op_id if method == "POST" else None, ) assert status == (200 if method == "GET" else 202), outcome assert outcome["status"] == "uncertain" assert outcome["safe_to_refund"] is False assert services["host-a"].calls == before changed_status, changed = await request( engine, "POST", vm_path("host-a"), {"action": "poweroff"}, operation_id=op_id, ) assert changed_status == 409, changed assert services["host-a"].calls == before status, blocked = await request( engine, "POST", vm_path("host-a"), {"action": "poweroff"}, operation_id="coord-new-conflicting-operation", ) assert status == 503, blocked assert services["host-a"].calls == before async def run(): store = _FailedResultStore() op_id = store.failed_operation async with running_engine(tmp_path, store) as (engine, _store, services): await prepare_hosts(engine, "host-a") status, outcome = await request( engine, "POST", vm_path("host-a"), {"action": "reset"}, operation_id=op_id, ) assert status >= 500, outcome assert sum(call[0] == "action" for call in services["host-a"].calls) == 1 store.reject_result = False if not restart: await check_blocked(engine, services, op_id) if restart: async with running_engine(tmp_path, store) as (engine, _store, services): await check_blocked(engine, services, op_id) asyncio.run(run()) @pytest.mark.parametrize("legacy", [False, True]) def test_restarted_running_create_without_error_code_cannot_refund_or_release_ip(tmp_path, legacy): """A crash can leave only the old owner claim and mutation checkpoint.""" async def run(): store = MemoryStore() op_id = "coord-crashed-create" async with running_engine(tmp_path, store) as (engine, _store, _services): status, created = await request( engine, "POST", "/internal/v1/hosts/host-a/vms", CREATE_BODY, operation_id=op_id, ) assert status == 201, created row = json.loads(store.get("operation", op_id)) vm_id = row["vm_uuid"] component = engine.registry.components["host-a"] scope = host_scope(component.manager.profile) credential_key = f"{scope}.{vm_id}" ip_key = f"{scope}.192.0.2.42" component.store.set("vm_credential", credential_key, "test-credential") component.store.set("ip_claim", ip_key, json.dumps({"vm_uuid": vm_id})) # Reproduce the durable state immediately after a write began but # before its receipt/error was committed. There is deliberately # no libvirt_execution_* error code to trigger the old shortcut. row.update(status="running", result=None, created_at=1, mutation_started=True, safe_to_refund=False) row.pop("error_code", None) row.pop("finished_at", None) if legacy: row.pop("execution_id", None) store.set("operation", op_id, json.dumps(row)) owner_before_restart = engine.coordinator.owner_id if not legacy: engine.coordinator.claim(operation_id=op_id) async with running_engine(tmp_path, store) as (engine, _store, services): assert engine.coordinator.owner_id != owner_before_restart await prepare_hosts(engine, "host-a") service = services["host-a"] service.absent = True before = list(service.calls) status, outcome = await request( engine, "GET", f"/internal/v1/operations/{op_id}") assert status == 200, outcome assert outcome["status"] == "uncertain" assert outcome["safe_to_refund"] is False assert outcome["executor_active"] is False assert "finished_at" not in outcome repeated_status, repeated = await request( engine, "POST", "/internal/v1/hosts/host-a/vms", CREATE_BODY, operation_id=op_id, ) assert repeated_status == 202, repeated assert repeated["status"] == "uncertain" assert service.calls == before assert service.absent_checks == [] component = engine.registry.components["host-a"] assert component.state.users.owns_vm(scope, vm_id, SUPER["user_id"]) assert component.store.get("vm_credential", credential_key) == "test-credential" assert component.store.get("ip_claim", ip_key) is not None if legacy: conflict_status, conflict = await request( engine, "POST", f"/internal/v1/hosts/host-a/vms/{vm_id}/actions", {"action": "poweroff"}, operation_id="coord-new-after-legacy-crash", ) assert conflict_status == 503, conflict assert service.calls == before asyncio.run(run()) def test_vm_lock_registry_reclaims_idle_locks_without_splitting_waiters(tmp_path): async def run(): async with running_engine(tmp_path) as (engine, _store, _services): for number in range(1000): async with engine._target_lock("host-a", str(uuid.UUID(int=number))): pass gc.collect() assert len(engine.target_locks) == 0 entered = asyncio.Event() release = asyncio.Event() async def holder(): async with engine._target_lock("host-a", VM_ID): entered.set() await release.wait() owned = asyncio.create_task(holder()) await entered.wait() lock = engine._target_lock("host-a", VM_ID) waiter = asyncio.create_task(lock.acquire()) await asyncio.sleep(0) try: gc.collect() assert engine._target_lock("host-a", VM_ID) is lock assert lock.locked() assert not waiter.done() waiter.cancel() with pytest.raises(asyncio.CancelledError): await waiter assert engine._target_lock("host-a", VM_ID) is lock finally: release.set() await owned del lock, waiter, owned gc.collect() assert len(engine.target_locks) == 0 asyncio.run(run())