You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/engine/tests/test_inventory_diagnostics.py

863 lines
34 KiB
Python

"""Durable inventory evidence must not mutate or rescan the observed host."""
from __future__ import annotations
import asyncio
import base64
import gc
import json
import os
import stat
import threading
import weakref
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import AsyncMock, patch
import pytest
from test_engine_boundary import (
ADMIN, SECRET, USER, FakeUpdater, MemoryStore, boundary, call,
cloud_engine_request,
)
from virsh_panel.internal import INVENTORY_IO_EXECUTOR, EngineApplication
from virsh_panel.inventory_diagnostics import (
CURRENT, MAX_BYTES, MAX_STAGES, InventoryDiagnostics, inventory_result, inventory_stage,
)
from virsh_panel.libvirt_service import VMError, VMService
ATTEMPT = "a" * 32
OTHER = "b" * 32
CAPACITY = "/internal/v1/hosts/hk/host/capacity"
DIAGNOSTICS = f"/internal/v1/hosts/hk/host/inventory-diagnostics/{ATTEMPT}"
def test_endpoint_authentication_and_host_isolation(boundary):
client, engine, *_ = boundary
assert client.get(DIAGNOSTICS).status_code == 401
assert call(client, "GET", DIAGNOSTICS, USER).status_code == 403
assert call(client, "GET", CAPACITY + "?inventory_attempt_id=" + ATTEMPT, USER).status_code == 403
assert engine.inventory_diagnostics.read("hk", ATTEMPT)["traces"] == []
assert call(client, "GET", CAPACITY + "?inventory_attempt_id=" + ATTEMPT, ADMIN).status_code == 200
assert call(client, "GET", "/internal/v1/hosts/hk/host/network-options?inventory_attempt_id=" + ATTEMPT, ADMIN).status_code == 200
result = call(client, "GET", DIAGNOSTICS, ADMIN).json()
assert result["diagnostics_available"] is True
assert [item["resource"] for item in result["traces"]] == ["capacity", "network"]
assert all(item["state"] == "succeeded" for item in result["traces"])
assert all(item["http_status"] == 200 for item in result["traces"])
assert "public_ip_claims" in [item["stage"] for item in result["traces"][0]["stages"]]
for secret in ("host.example.test", "SHA256:example", "/fake/config", "_process", "_started"):
assert secret not in json.dumps(result)
assert call(client, "GET", DIAGNOSTICS.replace("/hk/", "/other/"), ADMIN).json()["traces"] == []
@pytest.mark.parametrize("query", ["", "A" * 32, "../bad", "a" * 31, ATTEMPT + "&inventory_attempt_id=" + OTHER])
def test_invalid_correlation_is_rejected_after_authentication(boundary, query):
client, engine, *_ = boundary
assert call(client, "GET", CAPACITY + "?inventory_attempt_id=" + query, ADMIN).status_code == 400
assert engine.inventory_diagnostics.read("hk", ATTEMPT)["traces"] == []
def test_query_cannot_be_added_to_unsigned_target(boundary):
client, engine, *_ = boundary
from virsh_panel.engine_security import signed_headers
headers = signed_headers(SECRET, "GET", CAPACITY, b"", ADMIN)
result = client.get(CAPACITY + "?inventory_attempt_id=" + ATTEMPT, headers=headers)
assert result.status_code == 401
assert engine.inventory_diagnostics.read("hk", ATTEMPT)["traces"] == []
def test_read_bypasses_registry_lock_and_gc(boundary):
client, engine, *_ = boundary
with patch.object(engine.registry, "get", side_effect=AssertionError("must not load host")), \
patch.object(engine, "_update_lock", side_effect=AssertionError("must not wait")), \
patch.object(engine, "_cleanup_host_user_iso_tombstones", side_effect=AssertionError("must not gc")), \
patch.object(engine, "_cleanup_host_cloud_template_tombstones", side_effect=AssertionError("must not gc")):
assert call(client, "GET", DIAGNOSTICS, ADMIN).status_code == 200
def test_failed_profile_records_fixed_stage_without_secret(boundary):
client, engine, *_ = boundary
assert call(client, "GET", CAPACITY, ADMIN).status_code == 200
engine.registry.components["hk"].manager.profile["key_locked"] = True
assert call(client, "GET", CAPACITY + "?inventory_attempt_id=" + ATTEMPT, ADMIN).status_code == 423
trace = call(client, "GET", DIAGNOSTICS, ADMIN).json()["traces"][0]
assert (trace["state"], trace["stage"], trace["error_code"], trace["http_status"]) == (
"failed", "profile", "profile_failed", 423)
assert trace["stages"][-1]["state"] == "failed"
def make_engine(tmp_path, app):
registry = SimpleNamespace(get=AsyncMock(return_value=app))
engine = EngineApplication(
{"service_secret": base64.b64encode(SECRET).decode(), "data_dir": str(tmp_path)},
MemoryStore(), MemoryStore(), registry=registry, updater=FakeUpdater(),
)
return engine
def test_inventory_read_completes_while_global_mutation_lock_is_held(tmp_path):
async def scenario():
called = False
async def endpoint(scope, receive, send):
nonlocal called
called = True
await send({"type": "http.response.start", "status": 200})
await send({"type": "http.response.body", "body": b"{}"})
engine = make_engine(tmp_path, endpoint)
lock = engine._update_lock()
await lock.acquire()
try:
status, _ = await asyncio.wait_for(cloud_engine_request(
engine, "GET", CAPACITY + "?inventory_attempt_id=" + ATTEMPT,
b"", ADMIN), timeout=1)
assert status == 200 and called
status, result = await asyncio.wait_for(cloud_engine_request(
engine, "GET", DIAGNOSTICS, b"", ADMIN), timeout=1)
assert status == 200
assert result["traces"][0]["state"] == "succeeded"
stages = {item["stage"] for item in result["traces"][0]["stages"]}
assert not stages & {
"update_lock_wait", "update_check", "iso_gc", "template_gc",
}
finally:
lock.release()
asyncio.run(scenario())
def test_cancellation_during_registry_load_is_interrupted_not_forever_running(tmp_path):
async def scenario():
endpoint = AsyncMock()
engine = make_engine(tmp_path, endpoint)
entered, release = asyncio.Event(), asyncio.Event()
async def load_host(_host):
entered.set()
await release.wait()
return endpoint
engine.registry.get = load_host
request = asyncio.create_task(cloud_engine_request(
engine, "GET", CAPACITY + "?inventory_attempt_id=" + ATTEMPT, b"", ADMIN))
try:
await asyncio.wait_for(entered.wait(), timeout=1)
request.cancel()
with pytest.raises(asyncio.CancelledError):
await request
for _ in range(200):
traces = engine.inventory_diagnostics.read(
"hk", ATTEMPT)["traces"]
if traces and traces[0]["state"] == "interrupted":
break
await asyncio.sleep(0.001)
else:
pytest.fail("registry cancellation evidence was not persisted")
result = traces[0]
assert (result["state"], result["error_code"]) == ("interrupted", "request_interrupted")
assert result["stage"] == "registry_profile"
assert result["stages"][-1]["state"] == "interrupted"
assert result["finished_at"] is None and result["http_status"] is None
endpoint.assert_not_called()
finally:
release.set()
asyncio.run(scenario())
def test_cancellation_during_slow_initial_write_is_immediately_interrupted(tmp_path):
async def scenario():
endpoint = AsyncMock()
engine = make_engine(tmp_path, endpoint)
write_entered, write_release = threading.Event(), threading.Event()
registry_entered, registry_release = asyncio.Event(), asyncio.Event()
original = engine.inventory_diagnostics._write
def slow_write(path, value):
write_entered.set()
assert write_release.wait(2)
original(path, value)
async def load_host(_host):
registry_entered.set()
await registry_release.wait()
return endpoint
engine.registry.get = load_host
with patch.object(
engine.inventory_diagnostics, "_write",
side_effect=slow_write):
request = asyncio.create_task(cloud_engine_request(
engine, "GET",
CAPACITY + "?inventory_attempt_id=" + ATTEMPT,
b"", ADMIN))
try:
await asyncio.wait_for(registry_entered.wait(), timeout=0.5)
for _ in range(200):
if write_entered.is_set():
break
await asyncio.sleep(0.001)
else:
pytest.fail("dedicated diagnostic writer did not start")
request.cancel()
with pytest.raises(asyncio.CancelledError):
await request
traces = engine.inventory_diagnostics.read(
"hk", ATTEMPT)["traces"]
assert traces and traces[0]["state"] == "interrupted"
finally:
registry_release.set()
write_release.set()
for _ in range(200):
if (engine.inventory_io_pending == 0
and not engine.inventory_diagnostics.dirty):
break
await asyncio.sleep(0.001)
else:
pytest.fail("cancelled diagnostic evidence did not drain")
assert traces[0]["error_code"] == "request_interrupted"
assert traces[0]["finished_at"] is None
endpoint.assert_not_called()
asyncio.run(scenario())
def test_slow_diagnostic_write_does_not_starve_default_executor(tmp_path):
async def scenario():
async def endpoint(scope, receive, send):
await send({"type": "http.response.start", "status": 200})
await send({"type": "http.response.body", "body": b"{}"})
engine = make_engine(tmp_path, endpoint)
loop = asyncio.get_running_loop()
loop.set_default_executor(ThreadPoolExecutor(
max_workers=1, thread_name_prefix="test-default"))
entered, release = threading.Event(), threading.Event()
original = engine.inventory_diagnostics._write
def slow_write(path, value):
entered.set()
assert release.wait(2)
original(path, value)
with patch.object(
engine.inventory_diagnostics, "_write",
side_effect=slow_write):
request = asyncio.create_task(cloud_engine_request(
engine, "GET",
CAPACITY + "?inventory_attempt_id=" + ATTEMPT,
b"", ADMIN))
for _ in range(200):
if entered.is_set():
break
await asyncio.sleep(0.005)
else:
pytest.fail("dedicated diagnostic worker did not start")
try:
# The inventory response itself and a one-thread default
# executor both remain available while diagnostic fsync hangs.
status, _ = await asyncio.wait_for(request, timeout=0.5)
assert status == 200
assert await asyncio.wait_for(
asyncio.to_thread(lambda: "available"), timeout=0.5
) == "available"
trace = engine.inventory_diagnostics.read(
"hk", ATTEMPT)["traces"][0]
assert trace["state"] == "succeeded"
finally:
release.set()
for _ in range(200):
if (engine.inventory_io_pending == 0
and not engine.inventory_diagnostics.dirty):
break
await asyncio.sleep(0.001)
else:
pytest.fail("diagnostic worker did not drain")
asyncio.run(scenario())
def test_diagnostic_io_admission_is_bounded(tmp_path):
async def scenario():
engine = make_engine(tmp_path, AsyncMock())
entered, release = threading.Event(), threading.Event()
def occupy_writer():
entered.set()
assert release.wait(2)
assert engine._submit_inventory_io(occupy_writer) is not None
for _ in range(200):
if entered.is_set():
break
await asyncio.sleep(0.001)
else:
pytest.fail("diagnostic writer reservation did not start")
engine.inventory_diagnostics.max_pending_traces = 1
trace = engine.inventory_diagnostics.begin_deferred(
"hk", ATTEMPT, "capacity")
assert trace is not None
assert engine.inventory_diagnostics.begin_deferred(
"hk", OTHER, "network") is None
item = trace.start("registry_profile")
trace.end(item)
trace.finish(200)
assert len(engine.inventory_diagnostics.dirty) == 1
assert engine.inventory_diagnostics.read(
"hk", ATTEMPT)["traces"][0]["state"] == "succeeded"
try:
# One explicit blocker plus exactly one coalesced flush task.
assert engine.inventory_io_pending == 2
finally:
release.set()
for _ in range(200):
if (engine.inventory_io_pending == 0
and not engine.inventory_diagnostics.dirty):
break
await asyncio.sleep(0.001)
else:
pytest.fail("bounded diagnostic writer did not drain")
assert engine.inventory_diagnostics.read(
"hk", ATTEMPT)["traces"][0]["state"] == "succeeded"
asyncio.run(scenario())
def test_inflight_trace_keeps_its_slot_for_terminal_state(tmp_path):
async def scenario():
engine = make_engine(tmp_path, AsyncMock())
engine.inventory_diagnostics.max_pending_traces = 1
entered, release = threading.Event(), threading.Event()
original = engine.inventory_diagnostics._write
def slow_write(path, value):
entered.set()
assert release.wait(2)
original(path, value)
with patch.object(
engine.inventory_diagnostics, "_write",
side_effect=slow_write):
trace = engine.inventory_diagnostics.begin_deferred(
"hk", ATTEMPT, "capacity")
assert trace is not None
for _ in range(200):
if entered.is_set():
break
await asyncio.sleep(0.001)
else:
pytest.fail("diagnostic write did not become inflight")
path = engine.inventory_diagnostics._path(
"hk", ATTEMPT, "capacity")
assert path in engine.inventory_diagnostics.inflight
assert engine.inventory_diagnostics.begin_deferred(
"hk", OTHER, "network") is None
trace.finish(200)
assert path in engine.inventory_diagnostics.dirty
release.set()
for _ in range(200):
if (engine.inventory_io_pending == 0
and not engine.inventory_diagnostics.dirty
and not engine.inventory_diagnostics.inflight):
break
await asyncio.sleep(0.001)
else:
pytest.fail("terminal diagnostic state did not drain")
restarted = InventoryDiagnostics(
tmp_path, "unknown", process_id=engine.inventory_diagnostics.process_id)
result = restarted.read("hk", ATTEMPT)["traces"][0]
assert result["state"] == "succeeded"
assert result["http_status"] == 200
asyncio.run(scenario())
def test_clean_active_trace_terminal_survives_full_pending_batch(tmp_path):
async def scenario():
engine = make_engine(tmp_path, AsyncMock())
engine.inventory_diagnostics.max_pending_traces = 1
first = engine.inventory_diagnostics.begin_deferred(
"hk", ATTEMPT, "capacity")
assert first is not None
for _ in range(200):
if (engine.inventory_io_pending == 0
and not engine.inventory_diagnostics.dirty
and not engine.inventory_diagnostics.inflight):
break
await asyncio.sleep(0.001)
else:
pytest.fail("initial running evidence did not drain")
entered, release = threading.Event(), threading.Event()
original = engine.inventory_diagnostics._write
def slow_write(path, value):
entered.set()
assert release.wait(2)
original(path, value)
with patch.object(
engine.inventory_diagnostics, "_write",
side_effect=slow_write):
second = engine.inventory_diagnostics.begin_deferred(
"hk", OTHER, "network")
assert second is not None
for _ in range(200):
if entered.is_set():
break
await asyncio.sleep(0.001)
else:
pytest.fail("second trace did not occupy the pending window")
# A is already clean on disk, while B consumes the complete new-
# trace admission window. A's terminal update must still be dirty.
first.finish(200)
first_path = engine.inventory_diagnostics._path(
"hk", ATTEMPT, "capacity")
assert first_path in engine.inventory_diagnostics.dirty
release.set()
for _ in range(200):
if (engine.inventory_io_pending == 0
and not engine.inventory_diagnostics.dirty
and not engine.inventory_diagnostics.inflight):
break
await asyncio.sleep(0.001)
else:
pytest.fail("full-window terminal update did not drain")
restarted = InventoryDiagnostics(
tmp_path, "unknown", process_id=engine.inventory_diagnostics.process_id)
result = restarted.read("hk", ATTEMPT)["traces"][0]
assert result["state"] == "succeeded"
assert result["http_status"] == 200
asyncio.run(scenario())
def test_diagnostic_writes_never_run_on_event_loop(tmp_path):
async def scenario():
async def endpoint(scope, receive, send):
await send({"type": "http.response.start", "status": 200})
await send({"type": "http.response.body", "body": b"{}"})
engine = make_engine(tmp_path, endpoint)
original = engine.inventory_diagnostics._write
event_loop_writes = []
writes = []
def checked_write(path, value):
try:
asyncio.get_running_loop()
except RuntimeError:
pass
else:
event_loop_writes.append(value.get("stage"))
writes.append(threading.current_thread().name)
original(path, value)
with patch.object(
engine.inventory_diagnostics, "_write",
side_effect=checked_write):
status, _ = await cloud_engine_request(
engine, "GET",
CAPACITY + "?inventory_attempt_id=" + ATTEMPT,
b"", ADMIN)
for _ in range(200):
if engine.inventory_io_pending == 0:
break
await asyncio.sleep(0.001)
assert status == 200
assert writes
assert event_loop_writes == []
asyncio.run(scenario())
def test_terminal_trace_releases_raw_exception_and_traceback(tmp_path):
class SecretPayload:
pass
store = InventoryDiagnostics(tmp_path, "unknown")
trace = store.begin("hk", ATTEMPT, "capacity")
item = trace.start("libvirt_connect")
payload = SecretPayload()
payload_reference = weakref.ref(payload)
try:
raise RuntimeError("private failure", payload)
except RuntimeError as exception:
trace.fail_stage(item, exception)
trace.finish(502)
del payload
gc.collect()
assert trace.failure_exception is None
assert trace.active_stages == []
assert payload_reference() is None
def test_late_failure_cannot_reattach_exception_after_interruption(tmp_path):
class SecretPayload:
pass
store = InventoryDiagnostics(tmp_path, "unknown")
trace = store.begin("hk", ATTEMPT, "capacity")
item = trace.start("registry_profile", asynchronous=True)
trace.waiter_cancelled()
payload = SecretPayload()
payload_reference = weakref.ref(payload)
try:
raise RuntimeError("late private failure", payload)
except RuntimeError as exception:
trace.fail_stage(item, exception)
del payload
gc.collect()
assert trace.value["state"] == "interrupted"
assert trace.failure_exception is None
assert payload_reference() is None
def test_diagnostic_writer_is_daemonized_for_process_shutdown():
assert INVENTORY_IO_EXECUTOR.thread.is_alive()
assert INVENTORY_IO_EXECUTOR.thread.daemon is True
def test_registry_failure_records_only_fixed_codes(tmp_path):
async def scenario():
engine = make_engine(tmp_path, AsyncMock())
async def load_host(_host):
raise RuntimeError("SECRET-password-/private/key")
engine.registry.get = load_host
status, _ = await cloud_engine_request(
engine, "GET", CAPACITY + "?inventory_attempt_id=" + ATTEMPT, b"", ADMIN)
assert status == 503
trace = engine.inventory_diagnostics.read("hk", ATTEMPT)["traces"][0]
assert trace["stage"] == "registry_profile"
assert trace["error_code"] == "registry_profile_failed"
assert "SECRET" not in json.dumps(trace)
asyncio.run(scenario())
def test_worker_finishes_after_http_waiter_cancelled(tmp_path):
async def scenario():
started, release = threading.Event(), threading.Event()
@inventory_result
def worker():
with inventory_stage("domain_scan"):
started.set()
assert release.wait(2)
return {}
async def endpoint(scope, receive, send):
await asyncio.to_thread(worker)
await send({"type": "http.response.start", "status": 200})
engine = make_engine(tmp_path, endpoint)
request = asyncio.create_task(cloud_engine_request(
engine, "GET", CAPACITY + "?inventory_attempt_id=" + ATTEMPT, b"", ADMIN))
try:
assert await asyncio.to_thread(started.wait, 1)
request.cancel()
with pytest.raises(asyncio.CancelledError):
await request
for _ in range(200):
traces = engine.inventory_diagnostics.read(
"hk", ATTEMPT)["traces"]
if traces:
break
await asyncio.sleep(0.001)
else:
pytest.fail("worker cancellation evidence was not readable")
trace = traces[0]
assert (trace["state"], trace["stage"]) == ("running", "domain_scan")
finally:
release.set()
for _ in range(200):
trace = engine.inventory_diagnostics.read("hk", ATTEMPT)["traces"][0]
if trace["state"] == "succeeded":
break
await asyncio.sleep(0.001)
assert trace["state"] == "succeeded"
asyncio.run(scenario())
def test_queued_worker_starting_after_waiter_cancelled_records_real_result(tmp_path):
store = InventoryDiagnostics(tmp_path, "unknown")
trace = store.begin("hk", ATTEMPT, "capacity")
trace.waiter_cancelled()
assert trace.value["state"] == "interrupted"
token = CURRENT.set(trace)
try:
@inventory_result
def worker():
with inventory_stage("host_info"):
assert trace.value["state"] == "running"
return {}
worker()
finally:
CURRENT.reset(token)
result = store.read("hk", ATTEMPT)["traces"][0]
assert result["state"] == "succeeded"
assert result["error_code"] == ""
assert result["finished_at"] is not None
def test_atomic_summary_permissions_retention_and_restart(tmp_path):
store = InventoryDiagnostics(tmp_path, "a" * 40, max_traces=2)
first = store.begin("hk", ATTEMPT, "capacity")
first.start("domain_scan")
path = store._path("hk", ATTEMPT, "capacity")
assert stat.S_IMODE(store.directory.stat().st_mode) == 0o700
assert stat.S_IMODE(path.stat().st_mode) == 0o600
assert path.stat().st_size < MAX_BYTES
assert store.begin("hk", ATTEMPT, "capacity") is None
restarted = InventoryDiagnostics(tmp_path, "b" * 40, process_id="another-process")
trace = restarted.read("hk", ATTEMPT)["traces"][0]
assert trace["state"] == "interrupted"
assert trace["error_code"] == "request_interrupted"
assert trace["finished_at"] is None
assert trace["engine_version"] == "a" * 40
store.begin("hk", OTHER, "capacity").finish()
store.begin("other", OTHER, "capacity").finish()
assert len(list(store.directory.glob("*.json"))) == 2
assert store.read("hk", ATTEMPT)["traces"] == []
first.finish()
assert not path.exists() # evicted late work cannot exceed retention bound
for current in store.directory.glob("*.json"):
os.utime(current, (1, 1))
store.last_pruned = 0
store.begin("hk", "c" * 32, "capacity").finish()
assert len(list(store.directory.glob("*.json"))) == 1
def test_storage_failure_does_not_fail_inventory(boundary):
client, engine, *_ = boundary
with patch.object(engine.inventory_diagnostics, "_write", side_effect=OSError("SECRET path")):
result = call(client, "GET", CAPACITY + "?inventory_attempt_id=" + ATTEMPT, ADMIN)
for _ in range(200):
if (engine.inventory_io_pending == 0
and not engine.inventory_diagnostics.dirty
and not engine.inventory_diagnostics.inflight):
break
time.sleep(0.001)
assert result.status_code == 200
assert call(client, "GET", DIAGNOSTICS, ADMIN).json()["traces"] == []
restarted = InventoryDiagnostics(
engine.inventory_diagnostics.directory.parent, "unknown",
process_id="restarted")
assert restarted.read("hk", ATTEMPT)["traces"] == []
def test_failed_deferred_publications_release_bounded_live_slots(tmp_path):
async def scenario():
engine = make_engine(tmp_path, AsyncMock())
engine.inventory_diagnostics.max_pending_traces = 2
with patch.object(
engine.inventory_diagnostics, "_write",
side_effect=OSError("disk unavailable")):
for batch in range(3):
for offset in range(2):
attempt = f"{batch * 2 + offset + 1:032x}"
assert engine.inventory_diagnostics.begin_deferred(
"hk", attempt, "capacity") is not None
for _ in range(200):
if (engine.inventory_io_pending == 0
and not engine.inventory_diagnostics.dirty
and not engine.inventory_diagnostics.inflight):
break
await asyncio.sleep(0.001)
else:
pytest.fail("failed diagnostic publications did not drain")
assert not engine.inventory_diagnostics.live
assert not engine.inventory_diagnostics.known_paths
assert list(engine.inventory_diagnostics.directory.glob("*.json")) == []
asyncio.run(scenario())
def test_failed_new_summary_does_not_evict_previous_evidence(tmp_path):
store = InventoryDiagnostics(tmp_path, "unknown", max_traces=1)
first = store.begin("hk", ATTEMPT, "capacity")
first.finish()
first_path = store._path("hk", ATTEMPT, "capacity")
with patch.object(store, "_write", side_effect=OSError("disk unavailable")):
assert store.begin("hk", OTHER, "capacity") is None
assert first_path.is_file()
result = store.read("hk", ATTEMPT)["traces"]
assert len(result) == 1 and result[0]["state"] == "succeeded"
def test_eviction_failure_cannot_grow_trace_directory_without_bound(tmp_path):
store = InventoryDiagnostics(tmp_path, "unknown", max_traces=1)
first = store.begin("hk", ATTEMPT, "capacity")
first.finish()
first_path = store._path("hk", ATTEMPT, "capacity")
original_unlink = Path.unlink
def protected_unlink(path, *args, **kwargs):
if path == first_path:
raise OSError("old evidence cannot be deleted")
return original_unlink(path, *args, **kwargs)
with patch.object(Path, "unlink", new=protected_unlink):
assert store.begin("hk", OTHER, "capacity") is None
assert store.begin("hk", "c" * 32, "capacity") is None
assert store.eviction_blocked is True
assert list(store.records) == [first_path]
assert list(store.directory.glob("*.json")) == [first_path]
def test_double_unlink_failure_stops_deferred_batch_at_one_extra_file(tmp_path):
async def scenario():
engine = make_engine(tmp_path, AsyncMock())
store = engine.inventory_diagnostics
store.max_traces = 1
store.max_pending_traces = 3
assert store.begin("hk", ATTEMPT, "capacity") is not None
entered, release = threading.Event(), threading.Event()
def occupy_writer():
entered.set()
assert release.wait(2)
assert engine._submit_inventory_io(occupy_writer) is not None
for _ in range(200):
if entered.is_set():
break
await asyncio.sleep(0.001)
else:
pytest.fail("diagnostic writer blocker did not start")
original_unlink = Path.unlink
def reject_json_unlink(path, *args, **kwargs):
if path.suffix == ".json":
raise OSError("diagnostic json cannot be deleted")
return original_unlink(path, *args, **kwargs)
with patch.object(Path, "unlink", new=reject_json_unlink):
for index in range(3):
assert store.begin_deferred(
"hk", f"{index + 10:032x}", "capacity") is not None
release.set()
for _ in range(300):
if (engine.inventory_io_pending == 0
and not store.dirty and not store.inflight):
break
await asyncio.sleep(0.001)
else:
pytest.fail("blocked eviction batch did not drain")
assert store.begin_deferred(
"hk", "f" * 32, "capacity") is None
assert store.eviction_blocked is True
assert len(list(store.directory.glob("*.json"))) == 2
assert len(store.records) == 2
asyncio.run(scenario())
def test_real_service_stages_and_translated_http_status(tmp_path):
from virsh_panel import libvirt_service
connection = SimpleNamespace(
getInfo=lambda: ("x86_64", 8192, 4), listAllDomains=lambda _: [],
storagePoolLookupByName=lambda _: SimpleNamespace(isActive=lambda: True, info=lambda: (0, 100, 0, 100)),
close=lambda: None,
)
store = InventoryDiagnostics(tmp_path, "unknown")
trace = store.begin("hk", ATTEMPT, "capacity")
token = CURRENT.set(trace)
try:
with patch.object(libvirt_service.libvirt, "open", return_value=connection, create=True):
VMService("SECRET-uri", "images", "iso", "default").host_capacity()
trace.finish()
finally:
CURRENT.reset(token)
stages = [item["stage"] for item in store.read("hk", ATTEMPT)["traces"][0]["stages"]]
assert stages == ["libvirt_connect", "host_info", "domain_list", "domain_scan", "storage_scan"]
trace = store.begin("hk", OTHER, "capacity")
token = CURRENT.set(trace)
try:
@inventory_result
def fail():
VMService("SECRET-uri", "images", "iso", "default").host_capacity()
with patch.object(libvirt_service.libvirt, "open", side_effect=libvirt_service.libvirt.libvirtError("SECRET"), create=True):
with pytest.raises(VMError):
fail()
finally:
CURRENT.reset(token)
result = store.read("hk", OTHER)["traces"][0]
assert (result["stage"], result["http_status"]) == ("libvirt_connect", 502)
assert "SECRET" not in json.dumps(result)
def test_nested_failed_span_does_not_fail_caught_gc_request(tmp_path):
store = InventoryDiagnostics(tmp_path, "unknown")
trace = store.begin("hk", ATTEMPT, "capacity")
token = CURRENT.set(trace)
try:
with inventory_stage("iso_gc"):
try:
with inventory_stage("libvirt_connect"):
running = store.read("hk", ATTEMPT)["traces"][0]
assert [item["state"] for item in running["stages"]] == ["running", "running"]
raise VMError("secret connection error", 502)
except VMError:
pass
assert trace.value["state"] == "running"
with inventory_stage("host_info"):
pass
trace.finish(200)
finally:
CURRENT.reset(token)
result = store.read("hk", ATTEMPT)["traces"][0]
assert result["state"] == "succeeded"
assert result["error_code"] == ""
assert [item["state"] for item in result["stages"]] == ["succeeded", "failed", "succeeded"]
assert result["stages"][0]["duration_ms"] >= result["stages"][1]["duration_ms"]
def test_escaping_nested_failure_preserves_deepest_stage(tmp_path):
store = InventoryDiagnostics(tmp_path, "unknown")
trace = store.begin("hk", ATTEMPT, "capacity")
token = CURRENT.set(trace)
try:
with pytest.raises(VMError):
with inventory_stage("iso_gc"):
try:
with inventory_stage("libvirt_connect"):
raise RuntimeError("secret")
except RuntimeError as exc:
raise VMError("redacted", 502) from exc
trace.finish(502)
finally:
CURRENT.reset(token)
result = store.read("hk", ATTEMPT)["traces"][0]
assert result["state"] == "failed"
assert result["stage"] == "libvirt_connect"
assert [item["state"] for item in result["stages"]] == ["failed", "failed"]
def test_stage_limit_keeps_latest_blocking_evidence(tmp_path):
store = InventoryDiagnostics(tmp_path, "unknown")
trace = store.begin("hk", ATTEMPT, "capacity")
token = CURRENT.set(trace)
try:
with inventory_stage("iso_gc"):
for _ in range(MAX_STAGES + 5):
with inventory_stage("libvirt_connect"):
pass
item = trace.start("storage_scan")
result = store.read("hk", ATTEMPT)["traces"][0]
assert len(result["stages"]) == MAX_STAGES
assert result["stages"][-1]["stage"] == "storage_scan"
assert result["stages"][-1]["state"] == "running"
trace.end(item)
trace.finish(200)
finally:
CURRENT.reset(token)