You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/engine/tests/test_engine_shutdown.py

240 lines
8.9 KiB
Python

"""Engine lifespan shutdown shares one budget across unrelated blockers."""
import asyncio
import threading
import time
from types import SimpleNamespace
import pytest
# Installs the native-libvirt test adapter before importing EngineApplication.
from test_engine_boundary import EngineApplication, MemoryStore
from virsh_panel import libvirt_rpc
from virsh_panel.coordination import DurableCoordinator
@pytest.mark.parametrize("blocked", [
"maintenance", "registry", "fabric", "fabric_operations", "poller",
])
def test_engine_shutdown_deadline_stops_rpc_before_waiting(monkeypatch, blocked):
events = []
thread_release = threading.Event()
runtime = SimpleNamespace(stopped=False)
def stop_rpc():
runtime.stopped = True
events.append("rpc-stopped")
def close_rpc(timeout):
events.append("rpc-closed")
return True
runtime.begin_shutdown = stop_rpc
runtime.close = close_rpc
monkeypatch.setattr(libvirt_rpc, "get_runtime", lambda: runtime)
async def run():
async_release = asyncio.Event()
owned = []
async def block_async(name):
events.append(name)
try:
await async_release.wait()
except asyncio.CancelledError:
# Simulate an operation that still owes cleanup after its
# requester was cancelled; wait_for would exceed the budget.
await async_release.wait()
async def close_registry():
assert runtime.stopped
if blocked == "registry":
await block_async("registry-blocked")
else:
events.append("registry-closed")
async def shutdown_fabric_operations():
assert runtime.stopped
if blocked == "fabric_operations":
await block_async("fabric-operations-blocked")
else:
events.append("fabric-operations-stopped")
def close_fabric():
assert runtime.stopped
if blocked == "fabric":
events.append("fabric-blocked")
thread_release.wait()
else:
events.append("fabric-closed")
# Exercise the production lifespan shutdown branch while keeping its
# unrelated startup/database dependencies outside this fault test.
engine = object.__new__(EngineApplication)
engine.config = {"shutdown_timeout_seconds": 0.04}
engine.stopping = False
engine.registry = SimpleNamespace(
begin_shutdown=lambda: events.append("registry-stopped"),
close=close_registry,
)
engine.fabric = SimpleNamespace(
shutdown_operations=shutdown_fabric_operations,
close=close_fabric,
)
engine.maintenance_tasks = set()
engine.request_tasks = set()
engine.vpc_reconcile_task = None
engine.fabric_reconcile_task = None
engine.cache_cleanup_supervisor = None
engine.cache_cleanup_event = asyncio.Event()
if blocked == "maintenance":
task = asyncio.create_task(block_async("maintenance-started"))
engine.maintenance_tasks.add(task)
owned.append(task)
if blocked == "poller":
task = asyncio.create_task(block_async("poller-started"))
engine.vpc_reconcile_task = task
owned.append(task)
await asyncio.sleep(0)
sent = []
async def receive():
return {"type": "lifespan.shutdown"}
async def send(message):
sent.append(message)
before = time.monotonic()
try:
await engine._lifespan(receive, send)
elapsed = time.monotonic() - before
assert sent == [{"type": "lifespan.shutdown.complete"}]
assert engine.stopping is True
assert runtime.stopped
assert elapsed < 0.3
assert engine.vpc_reconcile_task is None
assert engine.fabric_reconcile_task is None
assert engine.cache_cleanup_supervisor is None
assert engine.cache_cleanup_event is None
fabric_event = ("fabric-operations-blocked" if blocked == "fabric_operations"
else "fabric-operations-stopped")
assert events.index("rpc-stopped") < events.index(fabric_event)
finally:
async_release.set()
thread_release.set()
# The tests deliberately supply cancellation-resistant fake
# coroutines. Release and collect them before closing this loop.
await asyncio.sleep(0)
remaining = [task for task in asyncio.all_tasks()
if task is not asyncio.current_task() and not task.done()]
await asyncio.gather(*owned, *remaining, return_exceptions=True)
asyncio.run(run())
def test_engine_default_executor_covers_startup_and_stays_open_for_resource_closers(monkeypatch):
monkeypatch.setattr(libvirt_rpc, "get_runtime", lambda: None)
observed = []
async def run():
async def start_registry():
# The startup path must replace asyncio's default pool before
# the registry can create its first worker.
observed.append(await asyncio.to_thread(
lambda: ("started", threading.current_thread().daemon)))
async def close_registry():
observed.append(await asyncio.to_thread(lambda: ("closed", True)))
async def poller():
await asyncio.Event().wait()
async def shutdown_operations():
return None
engine = object.__new__(EngineApplication)
engine.config = {"shutdown_timeout_seconds": 0.2}
engine.store = MemoryStore()
engine.coordinator = DurableCoordinator(engine.store)
engine._legacy_fence_lock = threading.Lock()
engine._legacy_fences_loaded = False
engine.registry = SimpleNamespace(start=start_registry,
begin_shutdown=lambda: None,
close=close_registry)
engine.fabric = SimpleNamespace(recover=lambda: None,
reconcile_loop=poller,
shutdown_operations=shutdown_operations,
close=lambda: None)
engine.vpc = SimpleNamespace(recover=lambda: None, reconcile_loop=poller)
engine._cache_cleanup_loop = poller
engine.maintenance_tasks = set()
engine.request_tasks = set()
incoming = iter(("lifespan.startup", "lifespan.shutdown"))
sent = []
async def receive():
return {"type": next(incoming)}
async def send(message):
sent.append(message["type"])
await engine._lifespan(receive, send)
assert sent == ["lifespan.startup.complete", "lifespan.shutdown.complete"]
assert observed == [("started", True), ("closed", True)]
assert engine._shutdown_executor.live_workers() == ()
with pytest.raises(RuntimeError, match="after shutdown"):
await asyncio.to_thread(lambda: None)
asyncio.run(run())
def test_engine_shutdown_uses_one_budget_for_successive_slow_closers(monkeypatch):
observed = []
runtime = SimpleNamespace(begin_shutdown=lambda: observed.append("rpc-stopped"),
close=lambda timeout: True)
monkeypatch.setattr(libvirt_rpc, "get_runtime", lambda: runtime)
async def run():
gate = asyncio.Event()
async def shutdown_operations():
await asyncio.sleep(0.025)
async def close_registry():
observed.append("registry-closing")
await gate.wait()
def close_fabric():
observed.append("fabric-closing")
time.sleep(0.025)
engine = object.__new__(EngineApplication)
engine.config = {"shutdown_timeout_seconds": 0.06}
engine.registry = SimpleNamespace(begin_shutdown=lambda: None,
close=close_registry)
engine.fabric = SimpleNamespace(shutdown_operations=shutdown_operations,
close=close_fabric)
engine.maintenance_tasks = set()
engine.request_tasks = set()
engine.vpc_reconcile_task = engine.fabric_reconcile_task = None
engine.cache_cleanup_supervisor = engine.cache_cleanup_event = None
async def receive():
return {"type": "lifespan.shutdown"}
async def send(message):
assert message["type"] == "lifespan.shutdown.complete"
before = time.monotonic()
try:
await engine._lifespan(receive, send)
assert time.monotonic() - before < 0.13
assert observed[0] == "rpc-stopped"
assert "fabric-closing" in observed
assert "registry-closing" in observed
finally:
gate.set()
await asyncio.sleep(0)
asyncio.run(run())