You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/engine/tests/test_inventory_diagnostics.py

407 lines
18 KiB
Python

"""Durable inventory evidence must not mutate or rescan the observed host."""
from __future__ import annotations
import asyncio
import base64
import json
import os
import stat
import threading
from types import SimpleNamespace
from unittest.mock import AsyncMock, patch
import pytest
from test_engine_boundary import (
ADMIN, SECRET, USER, FakeUpdater, MemoryStore, boundary, call,
cloud_engine_request,
)
from virsh_panel.internal import EngineApplication
from virsh_panel.inventory_diagnostics import (
CURRENT, MAX_BYTES, MAX_STAGES, InventoryDiagnostics, inventory_result, inventory_stage,
)
from virsh_panel.libvirt_service import VMError, VMService
ATTEMPT = "a" * 32
OTHER = "b" * 32
CAPACITY = "/internal/v1/hosts/hk/host/capacity"
DIAGNOSTICS = f"/internal/v1/hosts/hk/host/inventory-diagnostics/{ATTEMPT}"
def test_endpoint_authentication_and_host_isolation(boundary):
client, engine, *_ = boundary
assert client.get(DIAGNOSTICS).status_code == 401
assert call(client, "GET", DIAGNOSTICS, USER).status_code == 403
assert call(client, "GET", CAPACITY + "?inventory_attempt_id=" + ATTEMPT, USER).status_code == 403
assert engine.inventory_diagnostics.read("hk", ATTEMPT)["traces"] == []
assert call(client, "GET", CAPACITY + "?inventory_attempt_id=" + ATTEMPT, ADMIN).status_code == 200
assert call(client, "GET", "/internal/v1/hosts/hk/host/network-options?inventory_attempt_id=" + ATTEMPT, ADMIN).status_code == 200
result = call(client, "GET", DIAGNOSTICS, ADMIN).json()
assert result["diagnostics_available"] is True
assert [item["resource"] for item in result["traces"]] == ["capacity", "network"]
assert all(item["state"] == "succeeded" for item in result["traces"])
assert all(item["http_status"] == 200 for item in result["traces"])
assert "public_ip_claims" in [item["stage"] for item in result["traces"][0]["stages"]]
for secret in ("host.example.test", "SHA256:example", "/fake/config", "_process", "_started"):
assert secret not in json.dumps(result)
assert call(client, "GET", DIAGNOSTICS.replace("/hk/", "/other/"), ADMIN).json()["traces"] == []
@pytest.mark.parametrize("query", ["", "A" * 32, "../bad", "a" * 31, ATTEMPT + "&inventory_attempt_id=" + OTHER])
def test_invalid_correlation_is_rejected_after_authentication(boundary, query):
client, engine, *_ = boundary
assert call(client, "GET", CAPACITY + "?inventory_attempt_id=" + query, ADMIN).status_code == 400
assert engine.inventory_diagnostics.read("hk", ATTEMPT)["traces"] == []
def test_query_cannot_be_added_to_unsigned_target(boundary):
client, engine, *_ = boundary
from virsh_panel.engine_security import signed_headers
headers = signed_headers(SECRET, "GET", CAPACITY, b"", ADMIN)
result = client.get(CAPACITY + "?inventory_attempt_id=" + ATTEMPT, headers=headers)
assert result.status_code == 401
assert engine.inventory_diagnostics.read("hk", ATTEMPT)["traces"] == []
def test_read_bypasses_registry_lock_and_gc(boundary):
client, engine, *_ = boundary
with patch.object(engine.registry, "get", side_effect=AssertionError("must not load host")), \
patch.object(engine, "_update_lock", side_effect=AssertionError("must not wait")), \
patch.object(engine, "_cleanup_host_user_iso_tombstones", side_effect=AssertionError("must not gc")), \
patch.object(engine, "_cleanup_host_cloud_template_tombstones", side_effect=AssertionError("must not gc")):
assert call(client, "GET", DIAGNOSTICS, ADMIN).status_code == 200
def test_failed_profile_records_fixed_stage_without_secret(boundary):
client, engine, *_ = boundary
assert call(client, "GET", CAPACITY, ADMIN).status_code == 200
engine.registry.components["hk"].manager.profile["key_locked"] = True
assert call(client, "GET", CAPACITY + "?inventory_attempt_id=" + ATTEMPT, ADMIN).status_code == 423
trace = call(client, "GET", DIAGNOSTICS, ADMIN).json()["traces"][0]
assert (trace["state"], trace["stage"], trace["error_code"], trace["http_status"]) == (
"failed", "profile", "profile_failed", 423)
assert trace["stages"][-1]["state"] == "failed"
def make_engine(tmp_path, app):
registry = SimpleNamespace(get=AsyncMock(return_value=app))
engine = EngineApplication(
{"service_secret": base64.b64encode(SECRET).decode(), "data_dir": str(tmp_path)},
MemoryStore(), MemoryStore(), registry=registry, updater=FakeUpdater(),
)
engine._cleanup_host_user_iso_tombstones = AsyncMock()
engine._cleanup_host_cloud_template_tombstones = AsyncMock()
return engine
def test_lock_wait_is_persisted_before_capacity_endpoint(tmp_path):
async def scenario():
called = False
async def endpoint(scope, receive, send):
nonlocal called
called = True
await send({"type": "http.response.start", "status": 200})
await send({"type": "http.response.body", "body": b"{}"})
engine = make_engine(tmp_path, endpoint)
lock = engine._update_lock()
await lock.acquire()
request = asyncio.create_task(cloud_engine_request(
engine, "GET", CAPACITY + "?inventory_attempt_id=" + ATTEMPT, b"", ADMIN))
try:
for _ in range(200):
traces = engine.inventory_diagnostics.read("hk", ATTEMPT)["traces"]
if traces and traces[0]["stage"] == "update_lock_wait":
break
await asyncio.sleep(0.001)
else:
pytest.fail("lock wait not persisted")
status, result = await asyncio.wait_for(cloud_engine_request(
engine, "GET", DIAGNOSTICS, b"", ADMIN), timeout=1)
assert status == 200
assert result["traces"][0]["state"] == "running"
assert result["traces"][0]["stages"][-1]["state"] == "running"
assert called is False
finally:
lock.release()
await request
assert engine.inventory_diagnostics.read("hk", ATTEMPT)["traces"][0]["state"] == "succeeded"
asyncio.run(scenario())
def test_cancellation_during_lock_wait_is_interrupted_not_forever_running(tmp_path):
async def scenario():
endpoint = AsyncMock()
engine = make_engine(tmp_path, endpoint)
lock = engine._update_lock()
await lock.acquire()
request = asyncio.create_task(cloud_engine_request(
engine, "GET", CAPACITY + "?inventory_attempt_id=" + ATTEMPT, b"", ADMIN))
try:
for _ in range(200):
traces = engine.inventory_diagnostics.read("hk", ATTEMPT)["traces"]
if traces and traces[0]["stage"] == "update_lock_wait":
break
await asyncio.sleep(0.001)
else:
pytest.fail("lock wait not persisted")
request.cancel()
with pytest.raises(asyncio.CancelledError):
await request
result = engine.inventory_diagnostics.read("hk", ATTEMPT)["traces"][0]
assert (result["state"], result["error_code"]) == ("interrupted", "request_interrupted")
assert result["stage"] == "update_lock_wait"
assert result["stages"][-1]["state"] == "interrupted"
assert result["finished_at"] is None and result["http_status"] is None
endpoint.assert_not_called()
finally:
lock.release()
asyncio.run(scenario())
def test_cancellation_during_gc_does_not_claim_worker_stopped(tmp_path):
async def scenario():
started, release, completed = threading.Event(), threading.Event(), threading.Event()
def gc_worker():
try:
with inventory_stage("libvirt_connect"):
started.set()
assert release.wait(2)
finally:
completed.set()
engine = make_engine(tmp_path, AsyncMock())
async def cleanup(_host):
await asyncio.to_thread(gc_worker)
engine._cleanup_host_user_iso_tombstones = cleanup
request = asyncio.create_task(cloud_engine_request(
engine, "GET", CAPACITY + "?inventory_attempt_id=" + ATTEMPT, b"", ADMIN))
try:
assert await asyncio.to_thread(started.wait, 1)
request.cancel()
with pytest.raises(asyncio.CancelledError):
await request
result = engine.inventory_diagnostics.read("hk", ATTEMPT)["traces"][0]
assert result["state"] == "interrupted"
assert result["finished_at"] is None and result["http_status"] is None
assert result["stage"] == "libvirt_connect"
assert result["stages"][-1]["state"] == "running"
assert result["stages"][-2]["state"] == "interrupted"
assert not completed.is_set()
finally:
release.set()
assert await asyncio.to_thread(completed.wait, 1)
result = engine.inventory_diagnostics.read("hk", ATTEMPT)["traces"][0]
assert result["state"] == "interrupted" # GC completion is not inventory success
assert result["finished_at"] is None
assert result["stages"][-1]["state"] == "succeeded"
asyncio.run(scenario())
@pytest.mark.parametrize("name", ["registry_profile", "iso_gc", "template_gc"])
def test_pre_dispatch_failures_record_only_fixed_codes(tmp_path, name):
async def scenario():
engine = make_engine(tmp_path, AsyncMock())
target = engine.registry if name == "registry_profile" else engine
attribute = {"registry_profile": "get", "iso_gc": "_cleanup_host_user_iso_tombstones",
"template_gc": "_cleanup_host_cloud_template_tombstones"}[name]
setattr(target, attribute, AsyncMock(side_effect=RuntimeError("SECRET-password-/private/key")))
status, _ = await cloud_engine_request(
engine, "GET", CAPACITY + "?inventory_attempt_id=" + ATTEMPT, b"", ADMIN)
assert status == 503
trace = engine.inventory_diagnostics.read("hk", ATTEMPT)["traces"][0]
assert trace["stage"] == name
assert trace["error_code"] == name + "_failed"
assert "SECRET" not in json.dumps(trace)
asyncio.run(scenario())
def test_worker_finishes_after_http_waiter_cancelled(tmp_path):
async def scenario():
started, release = threading.Event(), threading.Event()
@inventory_result
def worker():
with inventory_stage("domain_scan"):
started.set()
assert release.wait(2)
return {}
async def endpoint(scope, receive, send):
await asyncio.to_thread(worker)
await send({"type": "http.response.start", "status": 200})
engine = make_engine(tmp_path, endpoint)
request = asyncio.create_task(cloud_engine_request(
engine, "GET", CAPACITY + "?inventory_attempt_id=" + ATTEMPT, b"", ADMIN))
try:
assert await asyncio.to_thread(started.wait, 1)
request.cancel()
with pytest.raises(asyncio.CancelledError):
await request
trace = engine.inventory_diagnostics.read("hk", ATTEMPT)["traces"][0]
assert (trace["state"], trace["stage"]) == ("running", "domain_scan")
finally:
release.set()
for _ in range(200):
trace = engine.inventory_diagnostics.read("hk", ATTEMPT)["traces"][0]
if trace["state"] == "succeeded":
break
await asyncio.sleep(0.001)
assert trace["state"] == "succeeded"
asyncio.run(scenario())
def test_queued_worker_starting_after_waiter_cancelled_records_real_result(tmp_path):
store = InventoryDiagnostics(tmp_path, "unknown")
trace = store.begin("hk", ATTEMPT, "capacity")
trace.waiter_cancelled()
assert trace.value["state"] == "interrupted"
token = CURRENT.set(trace)
try:
@inventory_result
def worker():
with inventory_stage("host_info"):
assert trace.value["state"] == "running"
return {}
worker()
finally:
CURRENT.reset(token)
result = store.read("hk", ATTEMPT)["traces"][0]
assert result["state"] == "succeeded"
assert result["error_code"] == ""
assert result["finished_at"] is not None
def test_atomic_summary_permissions_retention_and_restart(tmp_path):
store = InventoryDiagnostics(tmp_path, "a" * 40, max_traces=2)
first = store.begin("hk", ATTEMPT, "capacity")
first.start("domain_scan")
path = store._path("hk", ATTEMPT, "capacity")
assert stat.S_IMODE(store.directory.stat().st_mode) == 0o700
assert stat.S_IMODE(path.stat().st_mode) == 0o600
assert path.stat().st_size < MAX_BYTES
assert store.begin("hk", ATTEMPT, "capacity") is None
restarted = InventoryDiagnostics(tmp_path, "b" * 40, process_id="another-process")
trace = restarted.read("hk", ATTEMPT)["traces"][0]
assert trace["state"] == "interrupted"
assert trace["error_code"] == "request_interrupted"
assert trace["finished_at"] is None
assert trace["engine_version"] == "a" * 40
store.begin("hk", OTHER, "capacity").finish()
store.begin("other", OTHER, "capacity").finish()
assert len(list(store.directory.glob("*.json"))) == 2
assert store.read("hk", ATTEMPT)["traces"] == []
first.finish()
assert not path.exists() # evicted late work cannot exceed retention bound
for current in store.directory.glob("*.json"):
os.utime(current, (1, 1))
store.last_pruned = 0
store.begin("hk", "c" * 32, "capacity").finish()
assert len(list(store.directory.glob("*.json"))) == 1
def test_storage_failure_does_not_fail_inventory(boundary):
client, engine, *_ = boundary
with patch.object(engine.inventory_diagnostics, "_write", side_effect=OSError("SECRET path")):
result = call(client, "GET", CAPACITY + "?inventory_attempt_id=" + ATTEMPT, ADMIN)
assert result.status_code == 200
assert call(client, "GET", DIAGNOSTICS, ADMIN).json()["traces"] == []
def test_real_service_stages_and_translated_http_status(tmp_path):
from virsh_panel import libvirt_service
connection = SimpleNamespace(
getInfo=lambda: ("x86_64", 8192, 4), listAllDomains=lambda _: [],
storagePoolLookupByName=lambda _: SimpleNamespace(isActive=lambda: True, info=lambda: (0, 100, 0, 100)),
close=lambda: None,
)
store = InventoryDiagnostics(tmp_path, "unknown")
trace = store.begin("hk", ATTEMPT, "capacity")
token = CURRENT.set(trace)
try:
with patch.object(libvirt_service.libvirt, "open", return_value=connection, create=True):
VMService("SECRET-uri", "images", "iso", "default").host_capacity()
trace.finish()
finally:
CURRENT.reset(token)
stages = [item["stage"] for item in store.read("hk", ATTEMPT)["traces"][0]["stages"]]
assert stages == ["libvirt_connect", "host_info", "domain_list", "domain_scan", "storage_scan"]
trace = store.begin("hk", OTHER, "capacity")
token = CURRENT.set(trace)
try:
@inventory_result
def fail():
VMService("SECRET-uri", "images", "iso", "default").host_capacity()
with patch.object(libvirt_service.libvirt, "open", side_effect=libvirt_service.libvirt.libvirtError("SECRET"), create=True):
with pytest.raises(VMError):
fail()
finally:
CURRENT.reset(token)
result = store.read("hk", OTHER)["traces"][0]
assert (result["stage"], result["http_status"]) == ("libvirt_connect", 502)
assert "SECRET" not in json.dumps(result)
def test_nested_failed_span_does_not_fail_caught_gc_request(tmp_path):
store = InventoryDiagnostics(tmp_path, "unknown")
trace = store.begin("hk", ATTEMPT, "capacity")
token = CURRENT.set(trace)
try:
with inventory_stage("iso_gc"):
try:
with inventory_stage("libvirt_connect"):
running = store.read("hk", ATTEMPT)["traces"][0]
assert [item["state"] for item in running["stages"]] == ["running", "running"]
raise VMError("secret connection error", 502)
except VMError:
pass
assert trace.value["state"] == "running"
with inventory_stage("host_info"):
pass
trace.finish(200)
finally:
CURRENT.reset(token)
result = store.read("hk", ATTEMPT)["traces"][0]
assert result["state"] == "succeeded"
assert result["error_code"] == ""
assert [item["state"] for item in result["stages"]] == ["succeeded", "failed", "succeeded"]
assert result["stages"][0]["duration_ms"] >= result["stages"][1]["duration_ms"]
def test_escaping_nested_failure_preserves_deepest_stage(tmp_path):
store = InventoryDiagnostics(tmp_path, "unknown")
trace = store.begin("hk", ATTEMPT, "capacity")
token = CURRENT.set(trace)
try:
with pytest.raises(VMError):
with inventory_stage("iso_gc"):
try:
with inventory_stage("libvirt_connect"):
raise RuntimeError("secret")
except RuntimeError as exc:
raise VMError("redacted", 502) from exc
trace.finish(502)
finally:
CURRENT.reset(token)
result = store.read("hk", ATTEMPT)["traces"][0]
assert result["state"] == "failed"
assert result["stage"] == "libvirt_connect"
assert [item["state"] for item in result["stages"]] == ["failed", "failed"]
def test_stage_limit_keeps_latest_blocking_evidence(tmp_path):
store = InventoryDiagnostics(tmp_path, "unknown")
trace = store.begin("hk", ATTEMPT, "capacity")
token = CURRENT.set(trace)
try:
with inventory_stage("iso_gc"):
for _ in range(MAX_STAGES + 5):
with inventory_stage("libvirt_connect"):
pass
item = trace.start("storage_scan")
result = store.read("hk", ATTEMPT)["traces"][0]
assert len(result["stages"]) == MAX_STAGES
assert result["stages"][-1]["stage"] == "storage_scan"
assert result["stages"][-1]["state"] == "running"
trace.end(item)
trace.finish(200)
finally:
CURRENT.reset(token)