You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
407 lines
18 KiB
Python
407 lines
18 KiB
Python
"""Durable inventory evidence must not mutate or rescan the observed host."""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import base64
|
|
import json
|
|
import os
|
|
import stat
|
|
import threading
|
|
from types import SimpleNamespace
|
|
from unittest.mock import AsyncMock, patch
|
|
|
|
import pytest
|
|
|
|
from test_engine_boundary import (
|
|
ADMIN, SECRET, USER, FakeUpdater, MemoryStore, boundary, call,
|
|
cloud_engine_request,
|
|
)
|
|
from virsh_panel.internal import EngineApplication
|
|
from virsh_panel.inventory_diagnostics import (
|
|
CURRENT, MAX_BYTES, MAX_STAGES, InventoryDiagnostics, inventory_result, inventory_stage,
|
|
)
|
|
from virsh_panel.libvirt_service import VMError, VMService
|
|
|
|
ATTEMPT = "a" * 32
|
|
OTHER = "b" * 32
|
|
CAPACITY = "/internal/v1/hosts/hk/host/capacity"
|
|
DIAGNOSTICS = f"/internal/v1/hosts/hk/host/inventory-diagnostics/{ATTEMPT}"
|
|
|
|
|
|
def test_endpoint_authentication_and_host_isolation(boundary):
|
|
client, engine, *_ = boundary
|
|
assert client.get(DIAGNOSTICS).status_code == 401
|
|
assert call(client, "GET", DIAGNOSTICS, USER).status_code == 403
|
|
assert call(client, "GET", CAPACITY + "?inventory_attempt_id=" + ATTEMPT, USER).status_code == 403
|
|
assert engine.inventory_diagnostics.read("hk", ATTEMPT)["traces"] == []
|
|
|
|
assert call(client, "GET", CAPACITY + "?inventory_attempt_id=" + ATTEMPT, ADMIN).status_code == 200
|
|
assert call(client, "GET", "/internal/v1/hosts/hk/host/network-options?inventory_attempt_id=" + ATTEMPT, ADMIN).status_code == 200
|
|
result = call(client, "GET", DIAGNOSTICS, ADMIN).json()
|
|
assert result["diagnostics_available"] is True
|
|
assert [item["resource"] for item in result["traces"]] == ["capacity", "network"]
|
|
assert all(item["state"] == "succeeded" for item in result["traces"])
|
|
assert all(item["http_status"] == 200 for item in result["traces"])
|
|
assert "public_ip_claims" in [item["stage"] for item in result["traces"][0]["stages"]]
|
|
for secret in ("host.example.test", "SHA256:example", "/fake/config", "_process", "_started"):
|
|
assert secret not in json.dumps(result)
|
|
assert call(client, "GET", DIAGNOSTICS.replace("/hk/", "/other/"), ADMIN).json()["traces"] == []
|
|
|
|
|
|
@pytest.mark.parametrize("query", ["", "A" * 32, "../bad", "a" * 31, ATTEMPT + "&inventory_attempt_id=" + OTHER])
|
|
def test_invalid_correlation_is_rejected_after_authentication(boundary, query):
|
|
client, engine, *_ = boundary
|
|
assert call(client, "GET", CAPACITY + "?inventory_attempt_id=" + query, ADMIN).status_code == 400
|
|
assert engine.inventory_diagnostics.read("hk", ATTEMPT)["traces"] == []
|
|
|
|
|
|
def test_query_cannot_be_added_to_unsigned_target(boundary):
|
|
client, engine, *_ = boundary
|
|
from virsh_panel.engine_security import signed_headers
|
|
headers = signed_headers(SECRET, "GET", CAPACITY, b"", ADMIN)
|
|
result = client.get(CAPACITY + "?inventory_attempt_id=" + ATTEMPT, headers=headers)
|
|
assert result.status_code == 401
|
|
assert engine.inventory_diagnostics.read("hk", ATTEMPT)["traces"] == []
|
|
|
|
|
|
def test_read_bypasses_registry_lock_and_gc(boundary):
|
|
client, engine, *_ = boundary
|
|
with patch.object(engine.registry, "get", side_effect=AssertionError("must not load host")), \
|
|
patch.object(engine, "_update_lock", side_effect=AssertionError("must not wait")), \
|
|
patch.object(engine, "_cleanup_host_user_iso_tombstones", side_effect=AssertionError("must not gc")), \
|
|
patch.object(engine, "_cleanup_host_cloud_template_tombstones", side_effect=AssertionError("must not gc")):
|
|
assert call(client, "GET", DIAGNOSTICS, ADMIN).status_code == 200
|
|
|
|
|
|
def test_failed_profile_records_fixed_stage_without_secret(boundary):
|
|
client, engine, *_ = boundary
|
|
assert call(client, "GET", CAPACITY, ADMIN).status_code == 200
|
|
engine.registry.components["hk"].manager.profile["key_locked"] = True
|
|
assert call(client, "GET", CAPACITY + "?inventory_attempt_id=" + ATTEMPT, ADMIN).status_code == 423
|
|
trace = call(client, "GET", DIAGNOSTICS, ADMIN).json()["traces"][0]
|
|
assert (trace["state"], trace["stage"], trace["error_code"], trace["http_status"]) == (
|
|
"failed", "profile", "profile_failed", 423)
|
|
assert trace["stages"][-1]["state"] == "failed"
|
|
|
|
|
|
def make_engine(tmp_path, app):
|
|
registry = SimpleNamespace(get=AsyncMock(return_value=app))
|
|
engine = EngineApplication(
|
|
{"service_secret": base64.b64encode(SECRET).decode(), "data_dir": str(tmp_path)},
|
|
MemoryStore(), MemoryStore(), registry=registry, updater=FakeUpdater(),
|
|
)
|
|
engine._cleanup_host_user_iso_tombstones = AsyncMock()
|
|
engine._cleanup_host_cloud_template_tombstones = AsyncMock()
|
|
return engine
|
|
|
|
|
|
def test_lock_wait_is_persisted_before_capacity_endpoint(tmp_path):
|
|
async def scenario():
|
|
called = False
|
|
async def endpoint(scope, receive, send):
|
|
nonlocal called
|
|
called = True
|
|
await send({"type": "http.response.start", "status": 200})
|
|
await send({"type": "http.response.body", "body": b"{}"})
|
|
engine = make_engine(tmp_path, endpoint)
|
|
lock = engine._update_lock()
|
|
await lock.acquire()
|
|
request = asyncio.create_task(cloud_engine_request(
|
|
engine, "GET", CAPACITY + "?inventory_attempt_id=" + ATTEMPT, b"", ADMIN))
|
|
try:
|
|
for _ in range(200):
|
|
traces = engine.inventory_diagnostics.read("hk", ATTEMPT)["traces"]
|
|
if traces and traces[0]["stage"] == "update_lock_wait":
|
|
break
|
|
await asyncio.sleep(0.001)
|
|
else:
|
|
pytest.fail("lock wait not persisted")
|
|
status, result = await asyncio.wait_for(cloud_engine_request(
|
|
engine, "GET", DIAGNOSTICS, b"", ADMIN), timeout=1)
|
|
assert status == 200
|
|
assert result["traces"][0]["state"] == "running"
|
|
assert result["traces"][0]["stages"][-1]["state"] == "running"
|
|
assert called is False
|
|
finally:
|
|
lock.release()
|
|
await request
|
|
assert engine.inventory_diagnostics.read("hk", ATTEMPT)["traces"][0]["state"] == "succeeded"
|
|
asyncio.run(scenario())
|
|
|
|
|
|
def test_cancellation_during_lock_wait_is_interrupted_not_forever_running(tmp_path):
|
|
async def scenario():
|
|
endpoint = AsyncMock()
|
|
engine = make_engine(tmp_path, endpoint)
|
|
lock = engine._update_lock()
|
|
await lock.acquire()
|
|
request = asyncio.create_task(cloud_engine_request(
|
|
engine, "GET", CAPACITY + "?inventory_attempt_id=" + ATTEMPT, b"", ADMIN))
|
|
try:
|
|
for _ in range(200):
|
|
traces = engine.inventory_diagnostics.read("hk", ATTEMPT)["traces"]
|
|
if traces and traces[0]["stage"] == "update_lock_wait":
|
|
break
|
|
await asyncio.sleep(0.001)
|
|
else:
|
|
pytest.fail("lock wait not persisted")
|
|
request.cancel()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await request
|
|
result = engine.inventory_diagnostics.read("hk", ATTEMPT)["traces"][0]
|
|
assert (result["state"], result["error_code"]) == ("interrupted", "request_interrupted")
|
|
assert result["stage"] == "update_lock_wait"
|
|
assert result["stages"][-1]["state"] == "interrupted"
|
|
assert result["finished_at"] is None and result["http_status"] is None
|
|
endpoint.assert_not_called()
|
|
finally:
|
|
lock.release()
|
|
asyncio.run(scenario())
|
|
|
|
|
|
def test_cancellation_during_gc_does_not_claim_worker_stopped(tmp_path):
|
|
async def scenario():
|
|
started, release, completed = threading.Event(), threading.Event(), threading.Event()
|
|
def gc_worker():
|
|
try:
|
|
with inventory_stage("libvirt_connect"):
|
|
started.set()
|
|
assert release.wait(2)
|
|
finally:
|
|
completed.set()
|
|
engine = make_engine(tmp_path, AsyncMock())
|
|
async def cleanup(_host):
|
|
await asyncio.to_thread(gc_worker)
|
|
engine._cleanup_host_user_iso_tombstones = cleanup
|
|
request = asyncio.create_task(cloud_engine_request(
|
|
engine, "GET", CAPACITY + "?inventory_attempt_id=" + ATTEMPT, b"", ADMIN))
|
|
try:
|
|
assert await asyncio.to_thread(started.wait, 1)
|
|
request.cancel()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await request
|
|
result = engine.inventory_diagnostics.read("hk", ATTEMPT)["traces"][0]
|
|
assert result["state"] == "interrupted"
|
|
assert result["finished_at"] is None and result["http_status"] is None
|
|
assert result["stage"] == "libvirt_connect"
|
|
assert result["stages"][-1]["state"] == "running"
|
|
assert result["stages"][-2]["state"] == "interrupted"
|
|
assert not completed.is_set()
|
|
finally:
|
|
release.set()
|
|
assert await asyncio.to_thread(completed.wait, 1)
|
|
result = engine.inventory_diagnostics.read("hk", ATTEMPT)["traces"][0]
|
|
assert result["state"] == "interrupted" # GC completion is not inventory success
|
|
assert result["finished_at"] is None
|
|
assert result["stages"][-1]["state"] == "succeeded"
|
|
asyncio.run(scenario())
|
|
|
|
|
|
@pytest.mark.parametrize("name", ["registry_profile", "iso_gc", "template_gc"])
|
|
def test_pre_dispatch_failures_record_only_fixed_codes(tmp_path, name):
|
|
async def scenario():
|
|
engine = make_engine(tmp_path, AsyncMock())
|
|
target = engine.registry if name == "registry_profile" else engine
|
|
attribute = {"registry_profile": "get", "iso_gc": "_cleanup_host_user_iso_tombstones",
|
|
"template_gc": "_cleanup_host_cloud_template_tombstones"}[name]
|
|
setattr(target, attribute, AsyncMock(side_effect=RuntimeError("SECRET-password-/private/key")))
|
|
status, _ = await cloud_engine_request(
|
|
engine, "GET", CAPACITY + "?inventory_attempt_id=" + ATTEMPT, b"", ADMIN)
|
|
assert status == 503
|
|
trace = engine.inventory_diagnostics.read("hk", ATTEMPT)["traces"][0]
|
|
assert trace["stage"] == name
|
|
assert trace["error_code"] == name + "_failed"
|
|
assert "SECRET" not in json.dumps(trace)
|
|
asyncio.run(scenario())
|
|
|
|
|
|
def test_worker_finishes_after_http_waiter_cancelled(tmp_path):
|
|
async def scenario():
|
|
started, release = threading.Event(), threading.Event()
|
|
@inventory_result
|
|
def worker():
|
|
with inventory_stage("domain_scan"):
|
|
started.set()
|
|
assert release.wait(2)
|
|
return {}
|
|
async def endpoint(scope, receive, send):
|
|
await asyncio.to_thread(worker)
|
|
await send({"type": "http.response.start", "status": 200})
|
|
engine = make_engine(tmp_path, endpoint)
|
|
request = asyncio.create_task(cloud_engine_request(
|
|
engine, "GET", CAPACITY + "?inventory_attempt_id=" + ATTEMPT, b"", ADMIN))
|
|
try:
|
|
assert await asyncio.to_thread(started.wait, 1)
|
|
request.cancel()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await request
|
|
trace = engine.inventory_diagnostics.read("hk", ATTEMPT)["traces"][0]
|
|
assert (trace["state"], trace["stage"]) == ("running", "domain_scan")
|
|
finally:
|
|
release.set()
|
|
for _ in range(200):
|
|
trace = engine.inventory_diagnostics.read("hk", ATTEMPT)["traces"][0]
|
|
if trace["state"] == "succeeded":
|
|
break
|
|
await asyncio.sleep(0.001)
|
|
assert trace["state"] == "succeeded"
|
|
asyncio.run(scenario())
|
|
|
|
|
|
def test_queued_worker_starting_after_waiter_cancelled_records_real_result(tmp_path):
|
|
store = InventoryDiagnostics(tmp_path, "unknown")
|
|
trace = store.begin("hk", ATTEMPT, "capacity")
|
|
trace.waiter_cancelled()
|
|
assert trace.value["state"] == "interrupted"
|
|
token = CURRENT.set(trace)
|
|
try:
|
|
@inventory_result
|
|
def worker():
|
|
with inventory_stage("host_info"):
|
|
assert trace.value["state"] == "running"
|
|
return {}
|
|
worker()
|
|
finally:
|
|
CURRENT.reset(token)
|
|
result = store.read("hk", ATTEMPT)["traces"][0]
|
|
assert result["state"] == "succeeded"
|
|
assert result["error_code"] == ""
|
|
assert result["finished_at"] is not None
|
|
|
|
|
|
def test_atomic_summary_permissions_retention_and_restart(tmp_path):
|
|
store = InventoryDiagnostics(tmp_path, "a" * 40, max_traces=2)
|
|
first = store.begin("hk", ATTEMPT, "capacity")
|
|
first.start("domain_scan")
|
|
path = store._path("hk", ATTEMPT, "capacity")
|
|
assert stat.S_IMODE(store.directory.stat().st_mode) == 0o700
|
|
assert stat.S_IMODE(path.stat().st_mode) == 0o600
|
|
assert path.stat().st_size < MAX_BYTES
|
|
assert store.begin("hk", ATTEMPT, "capacity") is None
|
|
restarted = InventoryDiagnostics(tmp_path, "b" * 40, process_id="another-process")
|
|
trace = restarted.read("hk", ATTEMPT)["traces"][0]
|
|
assert trace["state"] == "interrupted"
|
|
assert trace["error_code"] == "request_interrupted"
|
|
assert trace["finished_at"] is None
|
|
assert trace["engine_version"] == "a" * 40
|
|
store.begin("hk", OTHER, "capacity").finish()
|
|
store.begin("other", OTHER, "capacity").finish()
|
|
assert len(list(store.directory.glob("*.json"))) == 2
|
|
assert store.read("hk", ATTEMPT)["traces"] == []
|
|
first.finish()
|
|
assert not path.exists() # evicted late work cannot exceed retention bound
|
|
for current in store.directory.glob("*.json"):
|
|
os.utime(current, (1, 1))
|
|
store.last_pruned = 0
|
|
store.begin("hk", "c" * 32, "capacity").finish()
|
|
assert len(list(store.directory.glob("*.json"))) == 1
|
|
|
|
|
|
def test_storage_failure_does_not_fail_inventory(boundary):
|
|
client, engine, *_ = boundary
|
|
with patch.object(engine.inventory_diagnostics, "_write", side_effect=OSError("SECRET path")):
|
|
result = call(client, "GET", CAPACITY + "?inventory_attempt_id=" + ATTEMPT, ADMIN)
|
|
assert result.status_code == 200
|
|
assert call(client, "GET", DIAGNOSTICS, ADMIN).json()["traces"] == []
|
|
|
|
|
|
def test_real_service_stages_and_translated_http_status(tmp_path):
|
|
from virsh_panel import libvirt_service
|
|
connection = SimpleNamespace(
|
|
getInfo=lambda: ("x86_64", 8192, 4), listAllDomains=lambda _: [],
|
|
storagePoolLookupByName=lambda _: SimpleNamespace(isActive=lambda: True, info=lambda: (0, 100, 0, 100)),
|
|
close=lambda: None,
|
|
)
|
|
store = InventoryDiagnostics(tmp_path, "unknown")
|
|
trace = store.begin("hk", ATTEMPT, "capacity")
|
|
token = CURRENT.set(trace)
|
|
try:
|
|
with patch.object(libvirt_service.libvirt, "open", return_value=connection, create=True):
|
|
VMService("SECRET-uri", "images", "iso", "default").host_capacity()
|
|
trace.finish()
|
|
finally:
|
|
CURRENT.reset(token)
|
|
stages = [item["stage"] for item in store.read("hk", ATTEMPT)["traces"][0]["stages"]]
|
|
assert stages == ["libvirt_connect", "host_info", "domain_list", "domain_scan", "storage_scan"]
|
|
trace = store.begin("hk", OTHER, "capacity")
|
|
token = CURRENT.set(trace)
|
|
try:
|
|
@inventory_result
|
|
def fail():
|
|
VMService("SECRET-uri", "images", "iso", "default").host_capacity()
|
|
with patch.object(libvirt_service.libvirt, "open", side_effect=libvirt_service.libvirt.libvirtError("SECRET"), create=True):
|
|
with pytest.raises(VMError):
|
|
fail()
|
|
finally:
|
|
CURRENT.reset(token)
|
|
result = store.read("hk", OTHER)["traces"][0]
|
|
assert (result["stage"], result["http_status"]) == ("libvirt_connect", 502)
|
|
assert "SECRET" not in json.dumps(result)
|
|
|
|
|
|
def test_nested_failed_span_does_not_fail_caught_gc_request(tmp_path):
|
|
store = InventoryDiagnostics(tmp_path, "unknown")
|
|
trace = store.begin("hk", ATTEMPT, "capacity")
|
|
token = CURRENT.set(trace)
|
|
try:
|
|
with inventory_stage("iso_gc"):
|
|
try:
|
|
with inventory_stage("libvirt_connect"):
|
|
running = store.read("hk", ATTEMPT)["traces"][0]
|
|
assert [item["state"] for item in running["stages"]] == ["running", "running"]
|
|
raise VMError("secret connection error", 502)
|
|
except VMError:
|
|
pass
|
|
assert trace.value["state"] == "running"
|
|
with inventory_stage("host_info"):
|
|
pass
|
|
trace.finish(200)
|
|
finally:
|
|
CURRENT.reset(token)
|
|
result = store.read("hk", ATTEMPT)["traces"][0]
|
|
assert result["state"] == "succeeded"
|
|
assert result["error_code"] == ""
|
|
assert [item["state"] for item in result["stages"]] == ["succeeded", "failed", "succeeded"]
|
|
assert result["stages"][0]["duration_ms"] >= result["stages"][1]["duration_ms"]
|
|
|
|
|
|
def test_escaping_nested_failure_preserves_deepest_stage(tmp_path):
|
|
store = InventoryDiagnostics(tmp_path, "unknown")
|
|
trace = store.begin("hk", ATTEMPT, "capacity")
|
|
token = CURRENT.set(trace)
|
|
try:
|
|
with pytest.raises(VMError):
|
|
with inventory_stage("iso_gc"):
|
|
try:
|
|
with inventory_stage("libvirt_connect"):
|
|
raise RuntimeError("secret")
|
|
except RuntimeError as exc:
|
|
raise VMError("redacted", 502) from exc
|
|
trace.finish(502)
|
|
finally:
|
|
CURRENT.reset(token)
|
|
result = store.read("hk", ATTEMPT)["traces"][0]
|
|
assert result["state"] == "failed"
|
|
assert result["stage"] == "libvirt_connect"
|
|
assert [item["state"] for item in result["stages"]] == ["failed", "failed"]
|
|
|
|
|
|
def test_stage_limit_keeps_latest_blocking_evidence(tmp_path):
|
|
store = InventoryDiagnostics(tmp_path, "unknown")
|
|
trace = store.begin("hk", ATTEMPT, "capacity")
|
|
token = CURRENT.set(trace)
|
|
try:
|
|
with inventory_stage("iso_gc"):
|
|
for _ in range(MAX_STAGES + 5):
|
|
with inventory_stage("libvirt_connect"):
|
|
pass
|
|
item = trace.start("storage_scan")
|
|
result = store.read("hk", ATTEMPT)["traces"][0]
|
|
assert len(result["stages"]) == MAX_STAGES
|
|
assert result["stages"][-1]["stage"] == "storage_scan"
|
|
assert result["stages"][-1]["state"] == "running"
|
|
trace.end(item)
|
|
trace.finish(200)
|
|
finally:
|
|
CURRENT.reset(token)
|