You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
863 lines
34 KiB
Python
863 lines
34 KiB
Python
"""Durable inventory evidence must not mutate or rescan the observed host."""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import base64
|
|
import gc
|
|
import json
|
|
import os
|
|
import stat
|
|
import threading
|
|
import weakref
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
from unittest.mock import AsyncMock, patch
|
|
|
|
import pytest
|
|
|
|
from test_engine_boundary import (
|
|
ADMIN, SECRET, USER, FakeUpdater, MemoryStore, boundary, call,
|
|
cloud_engine_request,
|
|
)
|
|
from virsh_panel.internal import INVENTORY_IO_EXECUTOR, EngineApplication
|
|
from virsh_panel.inventory_diagnostics import (
|
|
CURRENT, MAX_BYTES, MAX_STAGES, InventoryDiagnostics, inventory_result, inventory_stage,
|
|
)
|
|
from virsh_panel.libvirt_service import VMError, VMService
|
|
|
|
ATTEMPT = "a" * 32
|
|
OTHER = "b" * 32
|
|
CAPACITY = "/internal/v1/hosts/hk/host/capacity"
|
|
DIAGNOSTICS = f"/internal/v1/hosts/hk/host/inventory-diagnostics/{ATTEMPT}"
|
|
|
|
|
|
def test_endpoint_authentication_and_host_isolation(boundary):
|
|
client, engine, *_ = boundary
|
|
assert client.get(DIAGNOSTICS).status_code == 401
|
|
assert call(client, "GET", DIAGNOSTICS, USER).status_code == 403
|
|
assert call(client, "GET", CAPACITY + "?inventory_attempt_id=" + ATTEMPT, USER).status_code == 403
|
|
assert engine.inventory_diagnostics.read("hk", ATTEMPT)["traces"] == []
|
|
|
|
assert call(client, "GET", CAPACITY + "?inventory_attempt_id=" + ATTEMPT, ADMIN).status_code == 200
|
|
assert call(client, "GET", "/internal/v1/hosts/hk/host/network-options?inventory_attempt_id=" + ATTEMPT, ADMIN).status_code == 200
|
|
result = call(client, "GET", DIAGNOSTICS, ADMIN).json()
|
|
assert result["diagnostics_available"] is True
|
|
assert [item["resource"] for item in result["traces"]] == ["capacity", "network"]
|
|
assert all(item["state"] == "succeeded" for item in result["traces"])
|
|
assert all(item["http_status"] == 200 for item in result["traces"])
|
|
assert "public_ip_claims" in [item["stage"] for item in result["traces"][0]["stages"]]
|
|
for secret in ("host.example.test", "SHA256:example", "/fake/config", "_process", "_started"):
|
|
assert secret not in json.dumps(result)
|
|
assert call(client, "GET", DIAGNOSTICS.replace("/hk/", "/other/"), ADMIN).json()["traces"] == []
|
|
|
|
|
|
@pytest.mark.parametrize("query", ["", "A" * 32, "../bad", "a" * 31, ATTEMPT + "&inventory_attempt_id=" + OTHER])
|
|
def test_invalid_correlation_is_rejected_after_authentication(boundary, query):
|
|
client, engine, *_ = boundary
|
|
assert call(client, "GET", CAPACITY + "?inventory_attempt_id=" + query, ADMIN).status_code == 400
|
|
assert engine.inventory_diagnostics.read("hk", ATTEMPT)["traces"] == []
|
|
|
|
|
|
def test_query_cannot_be_added_to_unsigned_target(boundary):
|
|
client, engine, *_ = boundary
|
|
from virsh_panel.engine_security import signed_headers
|
|
headers = signed_headers(SECRET, "GET", CAPACITY, b"", ADMIN)
|
|
result = client.get(CAPACITY + "?inventory_attempt_id=" + ATTEMPT, headers=headers)
|
|
assert result.status_code == 401
|
|
assert engine.inventory_diagnostics.read("hk", ATTEMPT)["traces"] == []
|
|
|
|
|
|
def test_read_bypasses_registry_lock_and_gc(boundary):
|
|
client, engine, *_ = boundary
|
|
with patch.object(engine.registry, "get", side_effect=AssertionError("must not load host")), \
|
|
patch.object(engine, "_update_lock", side_effect=AssertionError("must not wait")), \
|
|
patch.object(engine, "_cleanup_host_user_iso_tombstones", side_effect=AssertionError("must not gc")), \
|
|
patch.object(engine, "_cleanup_host_cloud_template_tombstones", side_effect=AssertionError("must not gc")):
|
|
assert call(client, "GET", DIAGNOSTICS, ADMIN).status_code == 200
|
|
|
|
|
|
def test_failed_profile_records_fixed_stage_without_secret(boundary):
|
|
client, engine, *_ = boundary
|
|
assert call(client, "GET", CAPACITY, ADMIN).status_code == 200
|
|
engine.registry.components["hk"].manager.profile["key_locked"] = True
|
|
assert call(client, "GET", CAPACITY + "?inventory_attempt_id=" + ATTEMPT, ADMIN).status_code == 423
|
|
trace = call(client, "GET", DIAGNOSTICS, ADMIN).json()["traces"][0]
|
|
assert (trace["state"], trace["stage"], trace["error_code"], trace["http_status"]) == (
|
|
"failed", "profile", "profile_failed", 423)
|
|
assert trace["stages"][-1]["state"] == "failed"
|
|
|
|
|
|
def make_engine(tmp_path, app):
|
|
registry = SimpleNamespace(get=AsyncMock(return_value=app))
|
|
engine = EngineApplication(
|
|
{"service_secret": base64.b64encode(SECRET).decode(), "data_dir": str(tmp_path)},
|
|
MemoryStore(), MemoryStore(), registry=registry, updater=FakeUpdater(),
|
|
)
|
|
return engine
|
|
|
|
|
|
def test_inventory_read_completes_while_global_mutation_lock_is_held(tmp_path):
|
|
async def scenario():
|
|
called = False
|
|
async def endpoint(scope, receive, send):
|
|
nonlocal called
|
|
called = True
|
|
await send({"type": "http.response.start", "status": 200})
|
|
await send({"type": "http.response.body", "body": b"{}"})
|
|
engine = make_engine(tmp_path, endpoint)
|
|
lock = engine._update_lock()
|
|
await lock.acquire()
|
|
try:
|
|
status, _ = await asyncio.wait_for(cloud_engine_request(
|
|
engine, "GET", CAPACITY + "?inventory_attempt_id=" + ATTEMPT,
|
|
b"", ADMIN), timeout=1)
|
|
assert status == 200 and called
|
|
status, result = await asyncio.wait_for(cloud_engine_request(
|
|
engine, "GET", DIAGNOSTICS, b"", ADMIN), timeout=1)
|
|
assert status == 200
|
|
assert result["traces"][0]["state"] == "succeeded"
|
|
stages = {item["stage"] for item in result["traces"][0]["stages"]}
|
|
assert not stages & {
|
|
"update_lock_wait", "update_check", "iso_gc", "template_gc",
|
|
}
|
|
finally:
|
|
lock.release()
|
|
asyncio.run(scenario())
|
|
|
|
|
|
def test_cancellation_during_registry_load_is_interrupted_not_forever_running(tmp_path):
|
|
async def scenario():
|
|
endpoint = AsyncMock()
|
|
engine = make_engine(tmp_path, endpoint)
|
|
entered, release = asyncio.Event(), asyncio.Event()
|
|
|
|
async def load_host(_host):
|
|
entered.set()
|
|
await release.wait()
|
|
return endpoint
|
|
|
|
engine.registry.get = load_host
|
|
request = asyncio.create_task(cloud_engine_request(
|
|
engine, "GET", CAPACITY + "?inventory_attempt_id=" + ATTEMPT, b"", ADMIN))
|
|
try:
|
|
await asyncio.wait_for(entered.wait(), timeout=1)
|
|
request.cancel()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await request
|
|
for _ in range(200):
|
|
traces = engine.inventory_diagnostics.read(
|
|
"hk", ATTEMPT)["traces"]
|
|
if traces and traces[0]["state"] == "interrupted":
|
|
break
|
|
await asyncio.sleep(0.001)
|
|
else:
|
|
pytest.fail("registry cancellation evidence was not persisted")
|
|
result = traces[0]
|
|
assert (result["state"], result["error_code"]) == ("interrupted", "request_interrupted")
|
|
assert result["stage"] == "registry_profile"
|
|
assert result["stages"][-1]["state"] == "interrupted"
|
|
assert result["finished_at"] is None and result["http_status"] is None
|
|
endpoint.assert_not_called()
|
|
finally:
|
|
release.set()
|
|
asyncio.run(scenario())
|
|
|
|
|
|
def test_cancellation_during_slow_initial_write_is_immediately_interrupted(tmp_path):
|
|
async def scenario():
|
|
endpoint = AsyncMock()
|
|
engine = make_engine(tmp_path, endpoint)
|
|
write_entered, write_release = threading.Event(), threading.Event()
|
|
registry_entered, registry_release = asyncio.Event(), asyncio.Event()
|
|
original = engine.inventory_diagnostics._write
|
|
|
|
def slow_write(path, value):
|
|
write_entered.set()
|
|
assert write_release.wait(2)
|
|
original(path, value)
|
|
|
|
async def load_host(_host):
|
|
registry_entered.set()
|
|
await registry_release.wait()
|
|
return endpoint
|
|
|
|
engine.registry.get = load_host
|
|
with patch.object(
|
|
engine.inventory_diagnostics, "_write",
|
|
side_effect=slow_write):
|
|
request = asyncio.create_task(cloud_engine_request(
|
|
engine, "GET",
|
|
CAPACITY + "?inventory_attempt_id=" + ATTEMPT,
|
|
b"", ADMIN))
|
|
try:
|
|
await asyncio.wait_for(registry_entered.wait(), timeout=0.5)
|
|
for _ in range(200):
|
|
if write_entered.is_set():
|
|
break
|
|
await asyncio.sleep(0.001)
|
|
else:
|
|
pytest.fail("dedicated diagnostic writer did not start")
|
|
request.cancel()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await request
|
|
traces = engine.inventory_diagnostics.read(
|
|
"hk", ATTEMPT)["traces"]
|
|
assert traces and traces[0]["state"] == "interrupted"
|
|
finally:
|
|
registry_release.set()
|
|
write_release.set()
|
|
for _ in range(200):
|
|
if (engine.inventory_io_pending == 0
|
|
and not engine.inventory_diagnostics.dirty):
|
|
break
|
|
await asyncio.sleep(0.001)
|
|
else:
|
|
pytest.fail("cancelled diagnostic evidence did not drain")
|
|
assert traces[0]["error_code"] == "request_interrupted"
|
|
assert traces[0]["finished_at"] is None
|
|
endpoint.assert_not_called()
|
|
|
|
asyncio.run(scenario())
|
|
|
|
|
|
def test_slow_diagnostic_write_does_not_starve_default_executor(tmp_path):
|
|
async def scenario():
|
|
async def endpoint(scope, receive, send):
|
|
await send({"type": "http.response.start", "status": 200})
|
|
await send({"type": "http.response.body", "body": b"{}"})
|
|
|
|
engine = make_engine(tmp_path, endpoint)
|
|
loop = asyncio.get_running_loop()
|
|
loop.set_default_executor(ThreadPoolExecutor(
|
|
max_workers=1, thread_name_prefix="test-default"))
|
|
entered, release = threading.Event(), threading.Event()
|
|
original = engine.inventory_diagnostics._write
|
|
|
|
def slow_write(path, value):
|
|
entered.set()
|
|
assert release.wait(2)
|
|
original(path, value)
|
|
|
|
with patch.object(
|
|
engine.inventory_diagnostics, "_write",
|
|
side_effect=slow_write):
|
|
request = asyncio.create_task(cloud_engine_request(
|
|
engine, "GET",
|
|
CAPACITY + "?inventory_attempt_id=" + ATTEMPT,
|
|
b"", ADMIN))
|
|
for _ in range(200):
|
|
if entered.is_set():
|
|
break
|
|
await asyncio.sleep(0.005)
|
|
else:
|
|
pytest.fail("dedicated diagnostic worker did not start")
|
|
try:
|
|
# The inventory response itself and a one-thread default
|
|
# executor both remain available while diagnostic fsync hangs.
|
|
status, _ = await asyncio.wait_for(request, timeout=0.5)
|
|
assert status == 200
|
|
assert await asyncio.wait_for(
|
|
asyncio.to_thread(lambda: "available"), timeout=0.5
|
|
) == "available"
|
|
trace = engine.inventory_diagnostics.read(
|
|
"hk", ATTEMPT)["traces"][0]
|
|
assert trace["state"] == "succeeded"
|
|
finally:
|
|
release.set()
|
|
for _ in range(200):
|
|
if (engine.inventory_io_pending == 0
|
|
and not engine.inventory_diagnostics.dirty):
|
|
break
|
|
await asyncio.sleep(0.001)
|
|
else:
|
|
pytest.fail("diagnostic worker did not drain")
|
|
|
|
asyncio.run(scenario())
|
|
|
|
|
|
def test_diagnostic_io_admission_is_bounded(tmp_path):
|
|
async def scenario():
|
|
engine = make_engine(tmp_path, AsyncMock())
|
|
entered, release = threading.Event(), threading.Event()
|
|
|
|
def occupy_writer():
|
|
entered.set()
|
|
assert release.wait(2)
|
|
|
|
assert engine._submit_inventory_io(occupy_writer) is not None
|
|
for _ in range(200):
|
|
if entered.is_set():
|
|
break
|
|
await asyncio.sleep(0.001)
|
|
else:
|
|
pytest.fail("diagnostic writer reservation did not start")
|
|
|
|
engine.inventory_diagnostics.max_pending_traces = 1
|
|
trace = engine.inventory_diagnostics.begin_deferred(
|
|
"hk", ATTEMPT, "capacity")
|
|
assert trace is not None
|
|
assert engine.inventory_diagnostics.begin_deferred(
|
|
"hk", OTHER, "network") is None
|
|
item = trace.start("registry_profile")
|
|
trace.end(item)
|
|
trace.finish(200)
|
|
assert len(engine.inventory_diagnostics.dirty) == 1
|
|
assert engine.inventory_diagnostics.read(
|
|
"hk", ATTEMPT)["traces"][0]["state"] == "succeeded"
|
|
try:
|
|
# One explicit blocker plus exactly one coalesced flush task.
|
|
assert engine.inventory_io_pending == 2
|
|
finally:
|
|
release.set()
|
|
for _ in range(200):
|
|
if (engine.inventory_io_pending == 0
|
|
and not engine.inventory_diagnostics.dirty):
|
|
break
|
|
await asyncio.sleep(0.001)
|
|
else:
|
|
pytest.fail("bounded diagnostic writer did not drain")
|
|
assert engine.inventory_diagnostics.read(
|
|
"hk", ATTEMPT)["traces"][0]["state"] == "succeeded"
|
|
|
|
asyncio.run(scenario())
|
|
|
|
|
|
def test_inflight_trace_keeps_its_slot_for_terminal_state(tmp_path):
|
|
async def scenario():
|
|
engine = make_engine(tmp_path, AsyncMock())
|
|
engine.inventory_diagnostics.max_pending_traces = 1
|
|
entered, release = threading.Event(), threading.Event()
|
|
original = engine.inventory_diagnostics._write
|
|
|
|
def slow_write(path, value):
|
|
entered.set()
|
|
assert release.wait(2)
|
|
original(path, value)
|
|
|
|
with patch.object(
|
|
engine.inventory_diagnostics, "_write",
|
|
side_effect=slow_write):
|
|
trace = engine.inventory_diagnostics.begin_deferred(
|
|
"hk", ATTEMPT, "capacity")
|
|
assert trace is not None
|
|
for _ in range(200):
|
|
if entered.is_set():
|
|
break
|
|
await asyncio.sleep(0.001)
|
|
else:
|
|
pytest.fail("diagnostic write did not become inflight")
|
|
path = engine.inventory_diagnostics._path(
|
|
"hk", ATTEMPT, "capacity")
|
|
assert path in engine.inventory_diagnostics.inflight
|
|
assert engine.inventory_diagnostics.begin_deferred(
|
|
"hk", OTHER, "network") is None
|
|
trace.finish(200)
|
|
assert path in engine.inventory_diagnostics.dirty
|
|
release.set()
|
|
for _ in range(200):
|
|
if (engine.inventory_io_pending == 0
|
|
and not engine.inventory_diagnostics.dirty
|
|
and not engine.inventory_diagnostics.inflight):
|
|
break
|
|
await asyncio.sleep(0.001)
|
|
else:
|
|
pytest.fail("terminal diagnostic state did not drain")
|
|
|
|
restarted = InventoryDiagnostics(
|
|
tmp_path, "unknown", process_id=engine.inventory_diagnostics.process_id)
|
|
result = restarted.read("hk", ATTEMPT)["traces"][0]
|
|
assert result["state"] == "succeeded"
|
|
assert result["http_status"] == 200
|
|
|
|
asyncio.run(scenario())
|
|
|
|
|
|
def test_clean_active_trace_terminal_survives_full_pending_batch(tmp_path):
|
|
async def scenario():
|
|
engine = make_engine(tmp_path, AsyncMock())
|
|
engine.inventory_diagnostics.max_pending_traces = 1
|
|
first = engine.inventory_diagnostics.begin_deferred(
|
|
"hk", ATTEMPT, "capacity")
|
|
assert first is not None
|
|
for _ in range(200):
|
|
if (engine.inventory_io_pending == 0
|
|
and not engine.inventory_diagnostics.dirty
|
|
and not engine.inventory_diagnostics.inflight):
|
|
break
|
|
await asyncio.sleep(0.001)
|
|
else:
|
|
pytest.fail("initial running evidence did not drain")
|
|
|
|
entered, release = threading.Event(), threading.Event()
|
|
original = engine.inventory_diagnostics._write
|
|
|
|
def slow_write(path, value):
|
|
entered.set()
|
|
assert release.wait(2)
|
|
original(path, value)
|
|
|
|
with patch.object(
|
|
engine.inventory_diagnostics, "_write",
|
|
side_effect=slow_write):
|
|
second = engine.inventory_diagnostics.begin_deferred(
|
|
"hk", OTHER, "network")
|
|
assert second is not None
|
|
for _ in range(200):
|
|
if entered.is_set():
|
|
break
|
|
await asyncio.sleep(0.001)
|
|
else:
|
|
pytest.fail("second trace did not occupy the pending window")
|
|
# A is already clean on disk, while B consumes the complete new-
|
|
# trace admission window. A's terminal update must still be dirty.
|
|
first.finish(200)
|
|
first_path = engine.inventory_diagnostics._path(
|
|
"hk", ATTEMPT, "capacity")
|
|
assert first_path in engine.inventory_diagnostics.dirty
|
|
release.set()
|
|
for _ in range(200):
|
|
if (engine.inventory_io_pending == 0
|
|
and not engine.inventory_diagnostics.dirty
|
|
and not engine.inventory_diagnostics.inflight):
|
|
break
|
|
await asyncio.sleep(0.001)
|
|
else:
|
|
pytest.fail("full-window terminal update did not drain")
|
|
|
|
restarted = InventoryDiagnostics(
|
|
tmp_path, "unknown", process_id=engine.inventory_diagnostics.process_id)
|
|
result = restarted.read("hk", ATTEMPT)["traces"][0]
|
|
assert result["state"] == "succeeded"
|
|
assert result["http_status"] == 200
|
|
|
|
asyncio.run(scenario())
|
|
|
|
|
|
def test_diagnostic_writes_never_run_on_event_loop(tmp_path):
|
|
async def scenario():
|
|
async def endpoint(scope, receive, send):
|
|
await send({"type": "http.response.start", "status": 200})
|
|
await send({"type": "http.response.body", "body": b"{}"})
|
|
|
|
engine = make_engine(tmp_path, endpoint)
|
|
original = engine.inventory_diagnostics._write
|
|
event_loop_writes = []
|
|
writes = []
|
|
|
|
def checked_write(path, value):
|
|
try:
|
|
asyncio.get_running_loop()
|
|
except RuntimeError:
|
|
pass
|
|
else:
|
|
event_loop_writes.append(value.get("stage"))
|
|
writes.append(threading.current_thread().name)
|
|
original(path, value)
|
|
|
|
with patch.object(
|
|
engine.inventory_diagnostics, "_write",
|
|
side_effect=checked_write):
|
|
status, _ = await cloud_engine_request(
|
|
engine, "GET",
|
|
CAPACITY + "?inventory_attempt_id=" + ATTEMPT,
|
|
b"", ADMIN)
|
|
for _ in range(200):
|
|
if engine.inventory_io_pending == 0:
|
|
break
|
|
await asyncio.sleep(0.001)
|
|
assert status == 200
|
|
assert writes
|
|
assert event_loop_writes == []
|
|
|
|
asyncio.run(scenario())
|
|
|
|
|
|
def test_terminal_trace_releases_raw_exception_and_traceback(tmp_path):
|
|
class SecretPayload:
|
|
pass
|
|
|
|
store = InventoryDiagnostics(tmp_path, "unknown")
|
|
trace = store.begin("hk", ATTEMPT, "capacity")
|
|
item = trace.start("libvirt_connect")
|
|
payload = SecretPayload()
|
|
payload_reference = weakref.ref(payload)
|
|
try:
|
|
raise RuntimeError("private failure", payload)
|
|
except RuntimeError as exception:
|
|
trace.fail_stage(item, exception)
|
|
trace.finish(502)
|
|
|
|
del payload
|
|
gc.collect()
|
|
|
|
assert trace.failure_exception is None
|
|
assert trace.active_stages == []
|
|
assert payload_reference() is None
|
|
|
|
|
|
def test_late_failure_cannot_reattach_exception_after_interruption(tmp_path):
|
|
class SecretPayload:
|
|
pass
|
|
|
|
store = InventoryDiagnostics(tmp_path, "unknown")
|
|
trace = store.begin("hk", ATTEMPT, "capacity")
|
|
item = trace.start("registry_profile", asynchronous=True)
|
|
trace.waiter_cancelled()
|
|
payload = SecretPayload()
|
|
payload_reference = weakref.ref(payload)
|
|
try:
|
|
raise RuntimeError("late private failure", payload)
|
|
except RuntimeError as exception:
|
|
trace.fail_stage(item, exception)
|
|
|
|
del payload
|
|
gc.collect()
|
|
|
|
assert trace.value["state"] == "interrupted"
|
|
assert trace.failure_exception is None
|
|
assert payload_reference() is None
|
|
|
|
|
|
def test_diagnostic_writer_is_daemonized_for_process_shutdown():
|
|
assert INVENTORY_IO_EXECUTOR.thread.is_alive()
|
|
assert INVENTORY_IO_EXECUTOR.thread.daemon is True
|
|
|
|
|
|
def test_registry_failure_records_only_fixed_codes(tmp_path):
|
|
async def scenario():
|
|
engine = make_engine(tmp_path, AsyncMock())
|
|
async def load_host(_host):
|
|
raise RuntimeError("SECRET-password-/private/key")
|
|
|
|
engine.registry.get = load_host
|
|
status, _ = await cloud_engine_request(
|
|
engine, "GET", CAPACITY + "?inventory_attempt_id=" + ATTEMPT, b"", ADMIN)
|
|
assert status == 503
|
|
trace = engine.inventory_diagnostics.read("hk", ATTEMPT)["traces"][0]
|
|
assert trace["stage"] == "registry_profile"
|
|
assert trace["error_code"] == "registry_profile_failed"
|
|
assert "SECRET" not in json.dumps(trace)
|
|
asyncio.run(scenario())
|
|
|
|
|
|
def test_worker_finishes_after_http_waiter_cancelled(tmp_path):
|
|
async def scenario():
|
|
started, release = threading.Event(), threading.Event()
|
|
@inventory_result
|
|
def worker():
|
|
with inventory_stage("domain_scan"):
|
|
started.set()
|
|
assert release.wait(2)
|
|
return {}
|
|
async def endpoint(scope, receive, send):
|
|
await asyncio.to_thread(worker)
|
|
await send({"type": "http.response.start", "status": 200})
|
|
engine = make_engine(tmp_path, endpoint)
|
|
request = asyncio.create_task(cloud_engine_request(
|
|
engine, "GET", CAPACITY + "?inventory_attempt_id=" + ATTEMPT, b"", ADMIN))
|
|
try:
|
|
assert await asyncio.to_thread(started.wait, 1)
|
|
request.cancel()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await request
|
|
for _ in range(200):
|
|
traces = engine.inventory_diagnostics.read(
|
|
"hk", ATTEMPT)["traces"]
|
|
if traces:
|
|
break
|
|
await asyncio.sleep(0.001)
|
|
else:
|
|
pytest.fail("worker cancellation evidence was not readable")
|
|
trace = traces[0]
|
|
assert (trace["state"], trace["stage"]) == ("running", "domain_scan")
|
|
finally:
|
|
release.set()
|
|
for _ in range(200):
|
|
trace = engine.inventory_diagnostics.read("hk", ATTEMPT)["traces"][0]
|
|
if trace["state"] == "succeeded":
|
|
break
|
|
await asyncio.sleep(0.001)
|
|
assert trace["state"] == "succeeded"
|
|
asyncio.run(scenario())
|
|
|
|
|
|
def test_queued_worker_starting_after_waiter_cancelled_records_real_result(tmp_path):
|
|
store = InventoryDiagnostics(tmp_path, "unknown")
|
|
trace = store.begin("hk", ATTEMPT, "capacity")
|
|
trace.waiter_cancelled()
|
|
assert trace.value["state"] == "interrupted"
|
|
token = CURRENT.set(trace)
|
|
try:
|
|
@inventory_result
|
|
def worker():
|
|
with inventory_stage("host_info"):
|
|
assert trace.value["state"] == "running"
|
|
return {}
|
|
worker()
|
|
finally:
|
|
CURRENT.reset(token)
|
|
result = store.read("hk", ATTEMPT)["traces"][0]
|
|
assert result["state"] == "succeeded"
|
|
assert result["error_code"] == ""
|
|
assert result["finished_at"] is not None
|
|
|
|
|
|
def test_atomic_summary_permissions_retention_and_restart(tmp_path):
|
|
store = InventoryDiagnostics(tmp_path, "a" * 40, max_traces=2)
|
|
first = store.begin("hk", ATTEMPT, "capacity")
|
|
first.start("domain_scan")
|
|
path = store._path("hk", ATTEMPT, "capacity")
|
|
assert stat.S_IMODE(store.directory.stat().st_mode) == 0o700
|
|
assert stat.S_IMODE(path.stat().st_mode) == 0o600
|
|
assert path.stat().st_size < MAX_BYTES
|
|
assert store.begin("hk", ATTEMPT, "capacity") is None
|
|
restarted = InventoryDiagnostics(tmp_path, "b" * 40, process_id="another-process")
|
|
trace = restarted.read("hk", ATTEMPT)["traces"][0]
|
|
assert trace["state"] == "interrupted"
|
|
assert trace["error_code"] == "request_interrupted"
|
|
assert trace["finished_at"] is None
|
|
assert trace["engine_version"] == "a" * 40
|
|
store.begin("hk", OTHER, "capacity").finish()
|
|
store.begin("other", OTHER, "capacity").finish()
|
|
assert len(list(store.directory.glob("*.json"))) == 2
|
|
assert store.read("hk", ATTEMPT)["traces"] == []
|
|
first.finish()
|
|
assert not path.exists() # evicted late work cannot exceed retention bound
|
|
for current in store.directory.glob("*.json"):
|
|
os.utime(current, (1, 1))
|
|
store.last_pruned = 0
|
|
store.begin("hk", "c" * 32, "capacity").finish()
|
|
assert len(list(store.directory.glob("*.json"))) == 1
|
|
|
|
|
|
def test_storage_failure_does_not_fail_inventory(boundary):
|
|
client, engine, *_ = boundary
|
|
with patch.object(engine.inventory_diagnostics, "_write", side_effect=OSError("SECRET path")):
|
|
result = call(client, "GET", CAPACITY + "?inventory_attempt_id=" + ATTEMPT, ADMIN)
|
|
for _ in range(200):
|
|
if (engine.inventory_io_pending == 0
|
|
and not engine.inventory_diagnostics.dirty
|
|
and not engine.inventory_diagnostics.inflight):
|
|
break
|
|
time.sleep(0.001)
|
|
assert result.status_code == 200
|
|
assert call(client, "GET", DIAGNOSTICS, ADMIN).json()["traces"] == []
|
|
restarted = InventoryDiagnostics(
|
|
engine.inventory_diagnostics.directory.parent, "unknown",
|
|
process_id="restarted")
|
|
assert restarted.read("hk", ATTEMPT)["traces"] == []
|
|
|
|
|
|
def test_failed_deferred_publications_release_bounded_live_slots(tmp_path):
|
|
async def scenario():
|
|
engine = make_engine(tmp_path, AsyncMock())
|
|
engine.inventory_diagnostics.max_pending_traces = 2
|
|
with patch.object(
|
|
engine.inventory_diagnostics, "_write",
|
|
side_effect=OSError("disk unavailable")):
|
|
for batch in range(3):
|
|
for offset in range(2):
|
|
attempt = f"{batch * 2 + offset + 1:032x}"
|
|
assert engine.inventory_diagnostics.begin_deferred(
|
|
"hk", attempt, "capacity") is not None
|
|
for _ in range(200):
|
|
if (engine.inventory_io_pending == 0
|
|
and not engine.inventory_diagnostics.dirty
|
|
and not engine.inventory_diagnostics.inflight):
|
|
break
|
|
await asyncio.sleep(0.001)
|
|
else:
|
|
pytest.fail("failed diagnostic publications did not drain")
|
|
assert not engine.inventory_diagnostics.live
|
|
assert not engine.inventory_diagnostics.known_paths
|
|
assert list(engine.inventory_diagnostics.directory.glob("*.json")) == []
|
|
|
|
asyncio.run(scenario())
|
|
|
|
|
|
def test_failed_new_summary_does_not_evict_previous_evidence(tmp_path):
|
|
store = InventoryDiagnostics(tmp_path, "unknown", max_traces=1)
|
|
first = store.begin("hk", ATTEMPT, "capacity")
|
|
first.finish()
|
|
first_path = store._path("hk", ATTEMPT, "capacity")
|
|
|
|
with patch.object(store, "_write", side_effect=OSError("disk unavailable")):
|
|
assert store.begin("hk", OTHER, "capacity") is None
|
|
|
|
assert first_path.is_file()
|
|
result = store.read("hk", ATTEMPT)["traces"]
|
|
assert len(result) == 1 and result[0]["state"] == "succeeded"
|
|
|
|
|
|
def test_eviction_failure_cannot_grow_trace_directory_without_bound(tmp_path):
|
|
store = InventoryDiagnostics(tmp_path, "unknown", max_traces=1)
|
|
first = store.begin("hk", ATTEMPT, "capacity")
|
|
first.finish()
|
|
first_path = store._path("hk", ATTEMPT, "capacity")
|
|
original_unlink = Path.unlink
|
|
|
|
def protected_unlink(path, *args, **kwargs):
|
|
if path == first_path:
|
|
raise OSError("old evidence cannot be deleted")
|
|
return original_unlink(path, *args, **kwargs)
|
|
|
|
with patch.object(Path, "unlink", new=protected_unlink):
|
|
assert store.begin("hk", OTHER, "capacity") is None
|
|
assert store.begin("hk", "c" * 32, "capacity") is None
|
|
|
|
assert store.eviction_blocked is True
|
|
assert list(store.records) == [first_path]
|
|
assert list(store.directory.glob("*.json")) == [first_path]
|
|
|
|
|
|
def test_double_unlink_failure_stops_deferred_batch_at_one_extra_file(tmp_path):
|
|
async def scenario():
|
|
engine = make_engine(tmp_path, AsyncMock())
|
|
store = engine.inventory_diagnostics
|
|
store.max_traces = 1
|
|
store.max_pending_traces = 3
|
|
assert store.begin("hk", ATTEMPT, "capacity") is not None
|
|
entered, release = threading.Event(), threading.Event()
|
|
|
|
def occupy_writer():
|
|
entered.set()
|
|
assert release.wait(2)
|
|
|
|
assert engine._submit_inventory_io(occupy_writer) is not None
|
|
for _ in range(200):
|
|
if entered.is_set():
|
|
break
|
|
await asyncio.sleep(0.001)
|
|
else:
|
|
pytest.fail("diagnostic writer blocker did not start")
|
|
|
|
original_unlink = Path.unlink
|
|
|
|
def reject_json_unlink(path, *args, **kwargs):
|
|
if path.suffix == ".json":
|
|
raise OSError("diagnostic json cannot be deleted")
|
|
return original_unlink(path, *args, **kwargs)
|
|
|
|
with patch.object(Path, "unlink", new=reject_json_unlink):
|
|
for index in range(3):
|
|
assert store.begin_deferred(
|
|
"hk", f"{index + 10:032x}", "capacity") is not None
|
|
release.set()
|
|
for _ in range(300):
|
|
if (engine.inventory_io_pending == 0
|
|
and not store.dirty and not store.inflight):
|
|
break
|
|
await asyncio.sleep(0.001)
|
|
else:
|
|
pytest.fail("blocked eviction batch did not drain")
|
|
assert store.begin_deferred(
|
|
"hk", "f" * 32, "capacity") is None
|
|
|
|
assert store.eviction_blocked is True
|
|
assert len(list(store.directory.glob("*.json"))) == 2
|
|
assert len(store.records) == 2
|
|
|
|
asyncio.run(scenario())
|
|
|
|
|
|
def test_real_service_stages_and_translated_http_status(tmp_path):
|
|
from virsh_panel import libvirt_service
|
|
connection = SimpleNamespace(
|
|
getInfo=lambda: ("x86_64", 8192, 4), listAllDomains=lambda _: [],
|
|
storagePoolLookupByName=lambda _: SimpleNamespace(isActive=lambda: True, info=lambda: (0, 100, 0, 100)),
|
|
close=lambda: None,
|
|
)
|
|
store = InventoryDiagnostics(tmp_path, "unknown")
|
|
trace = store.begin("hk", ATTEMPT, "capacity")
|
|
token = CURRENT.set(trace)
|
|
try:
|
|
with patch.object(libvirt_service.libvirt, "open", return_value=connection, create=True):
|
|
VMService("SECRET-uri", "images", "iso", "default").host_capacity()
|
|
trace.finish()
|
|
finally:
|
|
CURRENT.reset(token)
|
|
stages = [item["stage"] for item in store.read("hk", ATTEMPT)["traces"][0]["stages"]]
|
|
assert stages == ["libvirt_connect", "host_info", "domain_list", "domain_scan", "storage_scan"]
|
|
trace = store.begin("hk", OTHER, "capacity")
|
|
token = CURRENT.set(trace)
|
|
try:
|
|
@inventory_result
|
|
def fail():
|
|
VMService("SECRET-uri", "images", "iso", "default").host_capacity()
|
|
with patch.object(libvirt_service.libvirt, "open", side_effect=libvirt_service.libvirt.libvirtError("SECRET"), create=True):
|
|
with pytest.raises(VMError):
|
|
fail()
|
|
finally:
|
|
CURRENT.reset(token)
|
|
result = store.read("hk", OTHER)["traces"][0]
|
|
assert (result["stage"], result["http_status"]) == ("libvirt_connect", 502)
|
|
assert "SECRET" not in json.dumps(result)
|
|
|
|
|
|
def test_nested_failed_span_does_not_fail_caught_gc_request(tmp_path):
|
|
store = InventoryDiagnostics(tmp_path, "unknown")
|
|
trace = store.begin("hk", ATTEMPT, "capacity")
|
|
token = CURRENT.set(trace)
|
|
try:
|
|
with inventory_stage("iso_gc"):
|
|
try:
|
|
with inventory_stage("libvirt_connect"):
|
|
running = store.read("hk", ATTEMPT)["traces"][0]
|
|
assert [item["state"] for item in running["stages"]] == ["running", "running"]
|
|
raise VMError("secret connection error", 502)
|
|
except VMError:
|
|
pass
|
|
assert trace.value["state"] == "running"
|
|
with inventory_stage("host_info"):
|
|
pass
|
|
trace.finish(200)
|
|
finally:
|
|
CURRENT.reset(token)
|
|
result = store.read("hk", ATTEMPT)["traces"][0]
|
|
assert result["state"] == "succeeded"
|
|
assert result["error_code"] == ""
|
|
assert [item["state"] for item in result["stages"]] == ["succeeded", "failed", "succeeded"]
|
|
assert result["stages"][0]["duration_ms"] >= result["stages"][1]["duration_ms"]
|
|
|
|
|
|
def test_escaping_nested_failure_preserves_deepest_stage(tmp_path):
|
|
store = InventoryDiagnostics(tmp_path, "unknown")
|
|
trace = store.begin("hk", ATTEMPT, "capacity")
|
|
token = CURRENT.set(trace)
|
|
try:
|
|
with pytest.raises(VMError):
|
|
with inventory_stage("iso_gc"):
|
|
try:
|
|
with inventory_stage("libvirt_connect"):
|
|
raise RuntimeError("secret")
|
|
except RuntimeError as exc:
|
|
raise VMError("redacted", 502) from exc
|
|
trace.finish(502)
|
|
finally:
|
|
CURRENT.reset(token)
|
|
result = store.read("hk", ATTEMPT)["traces"][0]
|
|
assert result["state"] == "failed"
|
|
assert result["stage"] == "libvirt_connect"
|
|
assert [item["state"] for item in result["stages"]] == ["failed", "failed"]
|
|
|
|
|
|
def test_stage_limit_keeps_latest_blocking_evidence(tmp_path):
|
|
store = InventoryDiagnostics(tmp_path, "unknown")
|
|
trace = store.begin("hk", ATTEMPT, "capacity")
|
|
token = CURRENT.set(trace)
|
|
try:
|
|
with inventory_stage("iso_gc"):
|
|
for _ in range(MAX_STAGES + 5):
|
|
with inventory_stage("libvirt_connect"):
|
|
pass
|
|
item = trace.start("storage_scan")
|
|
result = store.read("hk", ATTEMPT)["traces"][0]
|
|
assert len(result["stages"]) == MAX_STAGES
|
|
assert result["stages"][-1]["stage"] == "storage_scan"
|
|
assert result["stages"][-1]["state"] == "running"
|
|
trace.end(item)
|
|
trace.finish(200)
|
|
finally:
|
|
CURRENT.reset(token)
|