You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/engine/tests/test_shutdown.py

262 lines
8.3 KiB
Python

"""Shutdown deadlines must bound both the event loop and interpreter exit."""
import asyncio
import json
import os
import subprocess
import sys
import threading
import time
from pathlib import Path
import pytest
from virsh_panel.shutdown import (
ManagedThreadPoolExecutor, close_executors, close_executors_async,
install_asyncio_executor, remaining_shutdown_seconds, run_bounded_close,
shutdown_scope,
)
def test_shutdown_cancels_queued_work_without_claiming_running_work_stopped():
executor = ManagedThreadPoolExecutor(1, "shutdown-cancel-test")
started = threading.Event()
release = threading.Event()
queued_ran = threading.Event()
def running():
started.set()
release.wait()
return "finished"
active = executor.submit(running)
assert started.wait(1)
queued = executor.submit(queued_ran.set)
try:
with shutdown_scope(0.03):
assert close_executors((executor,)) is False
assert active.running()
assert queued.cancelled()
assert not queued_ran.is_set()
with pytest.raises(RuntimeError, match="after shutdown"):
executor.submit(lambda: None)
finally:
release.set()
assert active.result(timeout=1) == "finished"
assert close_executors((executor,)) is True
assert not queued_ran.is_set()
def test_multiple_pools_and_nested_closers_share_one_deadline():
release = threading.Event()
started = [threading.Event(), threading.Event()]
pools = [ManagedThreadPoolExecutor(1, f"shutdown-budget-{index}")
for index in range(2)]
def run(index):
started[index].set()
release.wait()
futures = [pool.submit(run, index) for index, pool in enumerate(pools)]
assert all(event.wait(1) for event in started)
try:
before = time.monotonic()
with shutdown_scope(0.06):
assert close_executors((pools[0],)) is False
with shutdown_scope(30):
assert remaining_shutdown_seconds() == 0
assert close_executors((pools[1],)) is False
assert time.monotonic() - before < 0.35
finally:
release.set()
for future in futures:
future.result(timeout=1)
assert close_executors(pools) is True
def test_async_shutdown_keeps_event_loop_responsive():
executor = ManagedThreadPoolExecutor(1, "shutdown-async-test")
release = threading.Event()
future = executor.submit(release.wait)
ticks = []
async def run():
async def ticker():
for _ in range(3):
await asyncio.sleep(0.005)
ticks.append(True)
with shutdown_scope(0.05):
drained, _ = await asyncio.gather(
close_executors_async((executor,)), ticker(),
)
return drained
try:
assert asyncio.run(run()) is False
assert len(ticks) == 3
finally:
release.set()
future.result(timeout=1)
assert close_executors((executor,)) is True
def test_blocked_resource_closer_has_a_deadline_and_inherits_budget():
release = threading.Event()
observed = []
def close():
observed.append(remaining_shutdown_seconds())
release.wait()
async def run():
with shutdown_scope(0.04):
return await run_bounded_close(close, name="blocked-test")
try:
assert asyncio.run(run()) is False
assert 0 < observed[0] <= 0.04
finally:
release.set()
def test_expired_deadline_does_not_begin_resource_teardown():
called = []
async def run():
with shutdown_scope(0):
return await run_bounded_close(lambda: called.append(True), name="late-test")
assert asyncio.run(run()) is False
assert called == []
def test_unresponsive_job_does_not_block_interpreter_exit():
# This is the behavior wait=False on ThreadPoolExecutor cannot provide:
# its interpreter-exit hook still joins a running non-daemon job forever.
source = """
import threading
from virsh_panel.shutdown import ManagedThreadPoolExecutor, close_executors, shutdown_scope
started = threading.Event()
never = threading.Event()
def blocked():
started.set()
never.wait()
pool = ManagedThreadPoolExecutor(1, 'exit-test')
pool.submit(blocked)
assert started.wait(1)
with shutdown_scope(0.03):
assert close_executors((pool,)) is False
print('bounded shutdown returned', flush=True)
"""
env = {**os.environ, "PYTHONPATH": str(Path(__file__).resolve().parents[1]),
"PYTHONDONTWRITEBYTECODE": "1"}
result = subprocess.run(
[sys.executable, "-c", source], env=env,
capture_output=True, text=True, timeout=3,
)
assert result.returncode == 0, result.stderr
assert "bounded shutdown returned" in result.stdout
def test_asyncio_run_exits_with_an_unresponsive_to_thread_call():
source = """
import asyncio
import threading
from virsh_panel.shutdown import close_executors_async, install_asyncio_executor, shutdown_scope
started = threading.Event()
never = threading.Event()
def blocked():
started.set()
never.wait()
async def main():
pool = install_asyncio_executor()
task = asyncio.create_task(asyncio.to_thread(blocked))
while not started.is_set():
await asyncio.sleep(0.001)
task.cancel()
await asyncio.gather(task, return_exceptions=True)
with shutdown_scope(0.03):
assert not await close_executors_async((pool,))
asyncio.run(main())
print('asyncio.run exited', flush=True)
"""
env = {**os.environ, "PYTHONPATH": str(Path(__file__).resolve().parents[1]),
"PYTHONDONTWRITEBYTECODE": "1"}
result = subprocess.run([sys.executable, "-c", source], env=env,
capture_output=True, text=True, timeout=3)
assert result.returncode == 0, result.stderr
assert "asyncio.run exited" in result.stdout
def test_asyncio_executor_preserves_to_thread_context_and_results():
import contextvars
value = contextvars.ContextVar("shutdown-test-value", default="missing")
async def run():
pool = install_asyncio_executor()
value.set("propagated")
assert await asyncio.to_thread(value.get) == "propagated"
assert await close_executors_async((pool,)) is True
with pytest.raises(RuntimeError, match="after shutdown"):
await asyncio.to_thread(lambda: None)
asyncio.run(run())
def test_worker_propagates_error_and_keeps_accepting_jobs():
executor = ManagedThreadPoolExecutor(1, "shutdown-future-test")
def fail():
raise ValueError("job failed")
try:
with pytest.raises(ValueError, match="job failed"):
executor.submit(fail).result(timeout=1)
assert executor.submit(lambda: 42).result(timeout=1) == 42
finally:
assert close_executors((executor,)) is True
@pytest.mark.parametrize("kind", ["cloud_template_import_job", "user_iso_import_job"])
def test_cancelled_queued_import_stays_durable_and_is_not_replayed(tmp_path, kind):
from test_user_isos import MemoryStore
from virsh_panel.cloud_template_runtime import CloudTemplateRuntime
from virsh_panel.user_isos import UserISORuntime
runtime_type = (CloudTemplateRuntime if kind == "cloud_template_import_job"
else UserISORuntime)
store = MemoryStore()
runtime = runtime_type(store, tmp_path)
started = threading.Event()
release = threading.Event()
queued_ran = threading.Event()
def running():
started.set()
release.wait()
active = runtime.executor.submit(running)
assert started.wait(1)
runtime.save_job("queued-import", {"job_id": "queued-import", "status": "queued"})
queued = runtime.executor.submit(queued_ran.set)
try:
with shutdown_scope(0.02):
assert runtime.close() is False
assert queued.cancelled()
assert json.loads(store.get(kind, "queued-import"))["status"] == "queued"
finally:
release.set()
active.result(timeout=1)
assert runtime.close() is True
restored = runtime_type(store, tmp_path)
try:
restored.restore_jobs()
assert json.loads(store.get(kind, "queued-import"))["status"] == "failed"
assert not queued_ran.is_set()
assert restored.executor.live_workers() == ()
finally:
assert restored.close() is True