"""Engine lifespan shutdown shares one budget across unrelated blockers.""" import asyncio import threading import time from types import SimpleNamespace import pytest # Installs the native-libvirt test adapter before importing EngineApplication. from test_engine_boundary import EngineApplication, MemoryStore from virsh_panel import libvirt_rpc from virsh_panel.coordination import DurableCoordinator @pytest.mark.parametrize("blocked", [ "maintenance", "registry", "fabric", "fabric_operations", "poller", ]) def test_engine_shutdown_deadline_stops_rpc_before_waiting(monkeypatch, blocked): events = [] thread_release = threading.Event() runtime = SimpleNamespace(stopped=False) def stop_rpc(): runtime.stopped = True events.append("rpc-stopped") def close_rpc(timeout): events.append("rpc-closed") return True runtime.begin_shutdown = stop_rpc runtime.close = close_rpc monkeypatch.setattr(libvirt_rpc, "get_runtime", lambda: runtime) async def run(): async_release = asyncio.Event() owned = [] async def block_async(name): events.append(name) try: await async_release.wait() except asyncio.CancelledError: # Simulate an operation that still owes cleanup after its # requester was cancelled; wait_for would exceed the budget. await async_release.wait() async def close_registry(): assert runtime.stopped if blocked == "registry": await block_async("registry-blocked") else: events.append("registry-closed") async def shutdown_fabric_operations(): assert runtime.stopped if blocked == "fabric_operations": await block_async("fabric-operations-blocked") else: events.append("fabric-operations-stopped") def close_fabric(): assert runtime.stopped if blocked == "fabric": events.append("fabric-blocked") thread_release.wait() else: events.append("fabric-closed") # Exercise the production lifespan shutdown branch while keeping its # unrelated startup/database dependencies outside this fault test. engine = object.__new__(EngineApplication) engine.config = {"shutdown_timeout_seconds": 0.04} engine.stopping = False engine.registry = SimpleNamespace( begin_shutdown=lambda: events.append("registry-stopped"), close=close_registry, ) engine.fabric = SimpleNamespace( shutdown_operations=shutdown_fabric_operations, close=close_fabric, ) engine.maintenance_tasks = set() engine.request_tasks = set() engine.vpc_reconcile_task = None engine.fabric_reconcile_task = None engine.cache_cleanup_supervisor = None engine.cache_cleanup_event = asyncio.Event() if blocked == "maintenance": task = asyncio.create_task(block_async("maintenance-started")) engine.maintenance_tasks.add(task) owned.append(task) if blocked == "poller": task = asyncio.create_task(block_async("poller-started")) engine.vpc_reconcile_task = task owned.append(task) await asyncio.sleep(0) sent = [] async def receive(): return {"type": "lifespan.shutdown"} async def send(message): sent.append(message) before = time.monotonic() try: await engine._lifespan(receive, send) elapsed = time.monotonic() - before assert sent == [{"type": "lifespan.shutdown.complete"}] assert engine.stopping is True assert runtime.stopped assert elapsed < 0.3 assert engine.vpc_reconcile_task is None assert engine.fabric_reconcile_task is None assert engine.cache_cleanup_supervisor is None assert engine.cache_cleanup_event is None fabric_event = ("fabric-operations-blocked" if blocked == "fabric_operations" else "fabric-operations-stopped") assert events.index("rpc-stopped") < events.index(fabric_event) finally: async_release.set() thread_release.set() # The tests deliberately supply cancellation-resistant fake # coroutines. Release and collect them before closing this loop. await asyncio.sleep(0) remaining = [task for task in asyncio.all_tasks() if task is not asyncio.current_task() and not task.done()] await asyncio.gather(*owned, *remaining, return_exceptions=True) asyncio.run(run()) def test_engine_default_executor_covers_startup_and_stays_open_for_resource_closers(monkeypatch): monkeypatch.setattr(libvirt_rpc, "get_runtime", lambda: None) observed = [] async def run(): async def start_registry(): # The startup path must replace asyncio's default pool before # the registry can create its first worker. observed.append(await asyncio.to_thread( lambda: ("started", threading.current_thread().daemon))) async def close_registry(): observed.append(await asyncio.to_thread(lambda: ("closed", True))) async def poller(): await asyncio.Event().wait() async def shutdown_operations(): return None engine = object.__new__(EngineApplication) engine.config = {"shutdown_timeout_seconds": 0.2} engine.store = MemoryStore() engine.coordinator = DurableCoordinator(engine.store) engine._legacy_fence_lock = threading.Lock() engine._legacy_fences_loaded = False engine.registry = SimpleNamespace(start=start_registry, begin_shutdown=lambda: None, close=close_registry) engine.fabric = SimpleNamespace(recover=lambda: None, reconcile_loop=poller, shutdown_operations=shutdown_operations, close=lambda: None) engine.vpc = SimpleNamespace(recover=lambda: None, reconcile_loop=poller) engine._cache_cleanup_loop = poller engine.maintenance_tasks = set() engine.request_tasks = set() incoming = iter(("lifespan.startup", "lifespan.shutdown")) sent = [] async def receive(): return {"type": next(incoming)} async def send(message): sent.append(message["type"]) await engine._lifespan(receive, send) assert sent == ["lifespan.startup.complete", "lifespan.shutdown.complete"] assert observed == [("started", True), ("closed", True)] assert engine._shutdown_executor.live_workers() == () with pytest.raises(RuntimeError, match="after shutdown"): await asyncio.to_thread(lambda: None) asyncio.run(run()) def test_engine_shutdown_uses_one_budget_for_successive_slow_closers(monkeypatch): observed = [] runtime = SimpleNamespace(begin_shutdown=lambda: observed.append("rpc-stopped"), close=lambda timeout: True) monkeypatch.setattr(libvirt_rpc, "get_runtime", lambda: runtime) async def run(): gate = asyncio.Event() async def shutdown_operations(): await asyncio.sleep(0.025) async def close_registry(): observed.append("registry-closing") await gate.wait() def close_fabric(): observed.append("fabric-closing") time.sleep(0.025) engine = object.__new__(EngineApplication) engine.config = {"shutdown_timeout_seconds": 0.06} engine.registry = SimpleNamespace(begin_shutdown=lambda: None, close=close_registry) engine.fabric = SimpleNamespace(shutdown_operations=shutdown_operations, close=close_fabric) engine.maintenance_tasks = set() engine.request_tasks = set() engine.vpc_reconcile_task = engine.fabric_reconcile_task = None engine.cache_cleanup_supervisor = engine.cache_cleanup_event = None async def receive(): return {"type": "lifespan.shutdown"} async def send(message): assert message["type"] == "lifespan.shutdown.complete" before = time.monotonic() try: await engine._lifespan(receive, send) assert time.monotonic() - before < 0.13 assert observed[0] == "rpc-stopped" assert "fabric-closing" in observed assert "registry-closing" in observed finally: gate.set() await asyncio.sleep(0) asyncio.run(run())