You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
262 lines
8.3 KiB
Python
262 lines
8.3 KiB
Python
"""Shutdown deadlines must bound both the event loop and interpreter exit."""
|
|
|
|
import asyncio
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from virsh_panel.shutdown import (
|
|
ManagedThreadPoolExecutor, close_executors, close_executors_async,
|
|
install_asyncio_executor, remaining_shutdown_seconds, run_bounded_close,
|
|
shutdown_scope,
|
|
)
|
|
|
|
|
|
def test_shutdown_cancels_queued_work_without_claiming_running_work_stopped():
|
|
executor = ManagedThreadPoolExecutor(1, "shutdown-cancel-test")
|
|
started = threading.Event()
|
|
release = threading.Event()
|
|
queued_ran = threading.Event()
|
|
|
|
def running():
|
|
started.set()
|
|
release.wait()
|
|
return "finished"
|
|
|
|
active = executor.submit(running)
|
|
assert started.wait(1)
|
|
queued = executor.submit(queued_ran.set)
|
|
try:
|
|
with shutdown_scope(0.03):
|
|
assert close_executors((executor,)) is False
|
|
assert active.running()
|
|
assert queued.cancelled()
|
|
assert not queued_ran.is_set()
|
|
with pytest.raises(RuntimeError, match="after shutdown"):
|
|
executor.submit(lambda: None)
|
|
finally:
|
|
release.set()
|
|
assert active.result(timeout=1) == "finished"
|
|
assert close_executors((executor,)) is True
|
|
assert not queued_ran.is_set()
|
|
|
|
|
|
def test_multiple_pools_and_nested_closers_share_one_deadline():
|
|
release = threading.Event()
|
|
started = [threading.Event(), threading.Event()]
|
|
pools = [ManagedThreadPoolExecutor(1, f"shutdown-budget-{index}")
|
|
for index in range(2)]
|
|
|
|
def run(index):
|
|
started[index].set()
|
|
release.wait()
|
|
|
|
futures = [pool.submit(run, index) for index, pool in enumerate(pools)]
|
|
assert all(event.wait(1) for event in started)
|
|
try:
|
|
before = time.monotonic()
|
|
with shutdown_scope(0.06):
|
|
assert close_executors((pools[0],)) is False
|
|
with shutdown_scope(30):
|
|
assert remaining_shutdown_seconds() == 0
|
|
assert close_executors((pools[1],)) is False
|
|
assert time.monotonic() - before < 0.35
|
|
finally:
|
|
release.set()
|
|
for future in futures:
|
|
future.result(timeout=1)
|
|
assert close_executors(pools) is True
|
|
|
|
|
|
def test_async_shutdown_keeps_event_loop_responsive():
|
|
executor = ManagedThreadPoolExecutor(1, "shutdown-async-test")
|
|
release = threading.Event()
|
|
future = executor.submit(release.wait)
|
|
ticks = []
|
|
|
|
async def run():
|
|
async def ticker():
|
|
for _ in range(3):
|
|
await asyncio.sleep(0.005)
|
|
ticks.append(True)
|
|
|
|
with shutdown_scope(0.05):
|
|
drained, _ = await asyncio.gather(
|
|
close_executors_async((executor,)), ticker(),
|
|
)
|
|
return drained
|
|
|
|
try:
|
|
assert asyncio.run(run()) is False
|
|
assert len(ticks) == 3
|
|
finally:
|
|
release.set()
|
|
future.result(timeout=1)
|
|
assert close_executors((executor,)) is True
|
|
|
|
|
|
def test_blocked_resource_closer_has_a_deadline_and_inherits_budget():
|
|
release = threading.Event()
|
|
observed = []
|
|
|
|
def close():
|
|
observed.append(remaining_shutdown_seconds())
|
|
release.wait()
|
|
|
|
async def run():
|
|
with shutdown_scope(0.04):
|
|
return await run_bounded_close(close, name="blocked-test")
|
|
|
|
try:
|
|
assert asyncio.run(run()) is False
|
|
assert 0 < observed[0] <= 0.04
|
|
finally:
|
|
release.set()
|
|
|
|
|
|
def test_expired_deadline_does_not_begin_resource_teardown():
|
|
called = []
|
|
|
|
async def run():
|
|
with shutdown_scope(0):
|
|
return await run_bounded_close(lambda: called.append(True), name="late-test")
|
|
|
|
assert asyncio.run(run()) is False
|
|
assert called == []
|
|
|
|
|
|
def test_unresponsive_job_does_not_block_interpreter_exit():
|
|
# This is the behavior wait=False on ThreadPoolExecutor cannot provide:
|
|
# its interpreter-exit hook still joins a running non-daemon job forever.
|
|
source = """
|
|
import threading
|
|
from virsh_panel.shutdown import ManagedThreadPoolExecutor, close_executors, shutdown_scope
|
|
started = threading.Event()
|
|
never = threading.Event()
|
|
def blocked():
|
|
started.set()
|
|
never.wait()
|
|
pool = ManagedThreadPoolExecutor(1, 'exit-test')
|
|
pool.submit(blocked)
|
|
assert started.wait(1)
|
|
with shutdown_scope(0.03):
|
|
assert close_executors((pool,)) is False
|
|
print('bounded shutdown returned', flush=True)
|
|
"""
|
|
env = {**os.environ, "PYTHONPATH": str(Path(__file__).resolve().parents[1]),
|
|
"PYTHONDONTWRITEBYTECODE": "1"}
|
|
result = subprocess.run(
|
|
[sys.executable, "-c", source], env=env,
|
|
capture_output=True, text=True, timeout=3,
|
|
)
|
|
assert result.returncode == 0, result.stderr
|
|
assert "bounded shutdown returned" in result.stdout
|
|
|
|
|
|
def test_asyncio_run_exits_with_an_unresponsive_to_thread_call():
|
|
source = """
|
|
import asyncio
|
|
import threading
|
|
from virsh_panel.shutdown import close_executors_async, install_asyncio_executor, shutdown_scope
|
|
started = threading.Event()
|
|
never = threading.Event()
|
|
def blocked():
|
|
started.set()
|
|
never.wait()
|
|
async def main():
|
|
pool = install_asyncio_executor()
|
|
task = asyncio.create_task(asyncio.to_thread(blocked))
|
|
while not started.is_set():
|
|
await asyncio.sleep(0.001)
|
|
task.cancel()
|
|
await asyncio.gather(task, return_exceptions=True)
|
|
with shutdown_scope(0.03):
|
|
assert not await close_executors_async((pool,))
|
|
asyncio.run(main())
|
|
print('asyncio.run exited', flush=True)
|
|
"""
|
|
env = {**os.environ, "PYTHONPATH": str(Path(__file__).resolve().parents[1]),
|
|
"PYTHONDONTWRITEBYTECODE": "1"}
|
|
result = subprocess.run([sys.executable, "-c", source], env=env,
|
|
capture_output=True, text=True, timeout=3)
|
|
assert result.returncode == 0, result.stderr
|
|
assert "asyncio.run exited" in result.stdout
|
|
|
|
|
|
def test_asyncio_executor_preserves_to_thread_context_and_results():
|
|
import contextvars
|
|
|
|
value = contextvars.ContextVar("shutdown-test-value", default="missing")
|
|
|
|
async def run():
|
|
pool = install_asyncio_executor()
|
|
value.set("propagated")
|
|
assert await asyncio.to_thread(value.get) == "propagated"
|
|
assert await close_executors_async((pool,)) is True
|
|
with pytest.raises(RuntimeError, match="after shutdown"):
|
|
await asyncio.to_thread(lambda: None)
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_worker_propagates_error_and_keeps_accepting_jobs():
|
|
executor = ManagedThreadPoolExecutor(1, "shutdown-future-test")
|
|
|
|
def fail():
|
|
raise ValueError("job failed")
|
|
|
|
try:
|
|
with pytest.raises(ValueError, match="job failed"):
|
|
executor.submit(fail).result(timeout=1)
|
|
assert executor.submit(lambda: 42).result(timeout=1) == 42
|
|
finally:
|
|
assert close_executors((executor,)) is True
|
|
|
|
|
|
@pytest.mark.parametrize("kind", ["cloud_template_import_job", "user_iso_import_job"])
|
|
def test_cancelled_queued_import_stays_durable_and_is_not_replayed(tmp_path, kind):
|
|
from test_user_isos import MemoryStore
|
|
from virsh_panel.cloud_template_runtime import CloudTemplateRuntime
|
|
from virsh_panel.user_isos import UserISORuntime
|
|
|
|
runtime_type = (CloudTemplateRuntime if kind == "cloud_template_import_job"
|
|
else UserISORuntime)
|
|
store = MemoryStore()
|
|
runtime = runtime_type(store, tmp_path)
|
|
started = threading.Event()
|
|
release = threading.Event()
|
|
queued_ran = threading.Event()
|
|
|
|
def running():
|
|
started.set()
|
|
release.wait()
|
|
|
|
active = runtime.executor.submit(running)
|
|
assert started.wait(1)
|
|
runtime.save_job("queued-import", {"job_id": "queued-import", "status": "queued"})
|
|
queued = runtime.executor.submit(queued_ran.set)
|
|
try:
|
|
with shutdown_scope(0.02):
|
|
assert runtime.close() is False
|
|
assert queued.cancelled()
|
|
assert json.loads(store.get(kind, "queued-import"))["status"] == "queued"
|
|
finally:
|
|
release.set()
|
|
active.result(timeout=1)
|
|
assert runtime.close() is True
|
|
|
|
restored = runtime_type(store, tmp_path)
|
|
try:
|
|
restored.restore_jobs()
|
|
assert json.loads(store.get(kind, "queued-import"))["status"] == "failed"
|
|
assert not queued_ran.is_set()
|
|
assert restored.executor.live_workers() == ()
|
|
finally:
|
|
assert restored.close() is True
|