You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/engine/tests/test_connection_failure_adm...

379 lines
18 KiB
Python

"""Connection admission failures must not become permanent mutation fences.
Exercise the signed engine and real VMService/coordination boundaries. Only
SSH staging, remote objects and the RPC connection transport are simulated.
"""
import base64
import json
from concurrent.futures import ThreadPoolExecutor
from types import SimpleNamespace
import pytest
from fastapi.testclient import TestClient
from test_app import FakeRemoteManager, HOST_BODY, VM_ID
from test_engine_boundary import MemoryStore, SECRET, SUPER, call
from test_host_service_scope import StagedManager
from virsh_panel.app import create_app
from virsh_panel.config import Settings
from virsh_panel.coordination import (
CoordinationError, DurableCoordinator, connection_failure_before_write,
current_execution, execution_context, mark_remote_write_started,
resource_guard, update_current_claim,
)
from virsh_panel.database import DatabaseTemplateStore
from virsh_panel.durable_jobs import submit_durable
from virsh_panel.engine_context import EngineComponents, ExternalOwnership, HostStore
from virsh_panel.internal import EngineApplication, HostRegistry, json_response
from virsh_panel.libvirt_rpc import LibvirtRuntime, RPCSession
from virsh_panel.libvirt_service import GIB, VMError, VMService
from virsh_panel.metrics import MetricsCollector
from virsh_panel.remote_host import RemoteHostError
import virsh_panel.libvirt_service as service_module
@pytest.fixture
def signed_host(tmp_path, monkeypatch):
opened, managers = [], {}
behavior = SimpleNamespace(fail_open=True, fail_after_write=False)
def open_session(uri, scope, *, timeout):
opened.append(scope)
if behavior.fail_open:
raise service_module.libvirt.libvirtError("connection refused before open")
connection = SimpleNamespace(networkLookupByName=lambda _name:
SimpleNamespace(isActive=lambda: True))
return SimpleNamespace(connection=connection, failure=None, close=lambda: None)
runtime = SimpleNamespace(open=open_session, queue_timeout=0.02)
monkeypatch.setattr("virsh_panel.app.get_runtime", lambda: runtime)
monkeypatch.setattr(service_module, "get_runtime", lambda: runtime)
monkeypatch.setattr(MetricsCollector, "start", lambda _self: None)
def pool(service, conn, name, *_args):
if behavior.fail_after_write:
mark_remote_write_started()
# A completed first session's write is still part of this request.
behavior.fail_open = True
with service._connect():
pytest.fail("second connection should fail")
return SimpleNamespace(name=lambda: name, listVolumes=lambda: [],
info=lambda: (0, 100 * GIB, 0, 100 * GIB))
monkeypatch.setattr(VMService, "_active_pool", pool)
monkeypatch.setattr(VMService, "_ensure_pool", pool)
monkeypatch.setattr(VMService, "_refresh_directory_pool", lambda *_args: None)
monkeypatch.setattr(VMService, "_same_pool_location", lambda *_args: False)
store, logs = MemoryStore(), MemoryStore()
config = {"service_secret": base64.b64encode(SECRET).decode(),
"data_dir": str(tmp_path)}
def factory(host_id, user_isos, cloud_templates):
directory = tmp_path / host_id
directory.mkdir(mode=0o700, exist_ok=True)
scoped = HostStore(store, host_id)
manager = managers[host_id] = StagedManager()
manager.host_id = host_id
components = EngineComponents(
SimpleNamespace(users=ExternalOwnership(scoped)), manager,
DatabaseTemplateStore(scoped), scoped, logs, user_isos, cloud_templates)
return components, create_app(Settings(data_dir=directory),
engine_components=components)
registry = HostRegistry(config, store, logs, factory=factory)
engine = EngineApplication(config, store, logs, registry=registry)
with TestClient(engine, base_url="http://127.0.0.1") as client:
yield SimpleNamespace(client=client, engine=engine, behavior=behavior,
opened=opened, managers=managers)
def test_signed_host_save_connection_failure_releases_claims_and_allows_retry(signed_host):
host = signed_host
path = "/internal/v1/hosts/7/host"
response = call(host.client, "PUT", path, SUPER, HOST_BODY)
assert response.status_code == 502, response.text
assert response.json()["error_code"] == "libvirt_connection_failed"
assert host.managers["7"].current() is None
assert [row[0] for row in host.managers["7"].calls] == ["stage", "test", "abort"]
assert host.engine.coordinator.snapshot() == {}
# No orphan request/resource claim may block maintenance either.
maintenance = host.engine.coordinator.claim(maintenance=True, kind="update")
assert host.engine.coordinator.release(maintenance)
host.behavior.fail_open = False
response = call(host.client, "PUT", path, SUPER, HOST_BODY)
assert response.status_code == 200, response.text
assert host.managers["7"].current()["host_id"] == "7"
assert host.opened == ["7", "7", "7"]
assert host.engine.coordinator.snapshot() == {}
@pytest.mark.parametrize("receipt_lost", [False, True])
def test_signed_ssh_probe_failure_requires_durable_prestart_proof(
signed_host, monkeypatch, receipt_lost):
host = signed_host
path = "/internal/v1/hosts/7/host"
assert call(host.client, "GET", path, SUPER).status_code == 200
manager = host.managers["7"]
original_probe = manager.test
def refused(_staged):
raise RemoteHostError("SSH probe connection refused", 502)
monkeypatch.setattr(manager, "test", refused)
if receipt_lost:
def cannot_record(*_args, **_kwargs):
raise CoordinationError("coordination_unavailable", 503)
monkeypatch.setattr(host.engine.coordinator, "update_metadata", cannot_record)
response = call(host.client, "PUT", path, SUPER, HOST_BODY)
assert response.status_code == (503 if receipt_lost else 502), response.text
assert host.opened == []
assert manager.current() is None
if receipt_lost:
assert [row["state"] for row in host.engine.coordinator.snapshot().values()] == ["uncertain"]
with pytest.raises(CoordinationError):
host.engine.coordinator.claim(maintenance=True, kind="update")
return
assert host.engine.coordinator.snapshot() == {}
maintenance = host.engine.coordinator.claim(maintenance=True, kind="update")
assert host.engine.coordinator.release(maintenance)
monkeypatch.setattr(manager, "test", original_probe)
host.behavior.fail_open = False
assert call(host.client, "PUT", path, SUPER, HOST_BODY).status_code == 200
assert host.engine.coordinator.snapshot() == {}
@pytest.mark.parametrize("method,suffix", [("PUT", "host"), ("POST", "host/test")])
@pytest.mark.parametrize("receipt_lost", [False, True])
def test_signed_stage_scan_failure_requires_durable_prestart_proof(
signed_host, monkeypatch, method, suffix, receipt_lost):
host = signed_host
assert call(host.client, "GET", "/internal/v1/hosts/7/host", SUPER).status_code == 200
manager = host.managers["7"]
original_stage = manager.stage
def refused(*_args, **_kwargs):
raise RemoteHostError("keyscan failed during staging", 502)
monkeypatch.setattr(manager, "stage", refused)
if receipt_lost:
def cannot_record(*_args, **_kwargs):
raise CoordinationError("coordination_unavailable", 503)
monkeypatch.setattr(host.engine.coordinator, "update_metadata", cannot_record)
path = f"/internal/v1/hosts/7/{suffix}"
response = call(host.client, method, path, SUPER, HOST_BODY)
assert response.status_code == (503 if receipt_lost else 502), response.text
assert host.opened == []
assert manager.current() is None
if receipt_lost:
assert [row["state"] for row in host.engine.coordinator.snapshot().values()] == ["uncertain"]
with pytest.raises(CoordinationError):
host.engine.coordinator.claim(maintenance=True, kind="update")
return
assert host.engine.coordinator.snapshot() == {}
maintenance = host.engine.coordinator.claim(maintenance=True, kind="update")
assert host.engine.coordinator.release(maintenance)
monkeypatch.setattr(manager, "stage", original_stage)
host.behavior.fail_open = False
assert call(host.client, method, path, SUPER, HOST_BODY).status_code == 200
assert host.engine.coordinator.snapshot() == {}
@pytest.mark.parametrize("global_route", [False, True], ids=["host-scan", "discovery-scan"])
@pytest.mark.parametrize("receipt_lost", [False, True])
def test_signed_keyscan_failure_requires_durable_prestart_proof(
signed_host, monkeypatch, global_route, receipt_lost):
host = signed_host
if global_route:
manager = FakeRemoteManager()
manager.close = lambda: None
monkeypatch.setattr("virsh_panel.remote_host.RemoteHostManager", lambda *_args: manager)
path = "/internal/v1/hosts/scan"
else:
assert call(host.client, "GET", "/internal/v1/hosts/7/host", SUPER).status_code == 200
manager = host.managers["7"]
path = "/internal/v1/hosts/7/host/scan"
original_scan = manager.scan
def refused(*_args, **_kwargs):
raise RemoteHostError("keyscan timed out", 502)
monkeypatch.setattr(manager, "scan", refused)
if receipt_lost:
def cannot_record(*_args, **_kwargs):
raise CoordinationError("coordination_unavailable", 503)
monkeypatch.setattr(host.engine.coordinator, "update_metadata", cannot_record)
body = {"host": HOST_BODY["host"], "port": HOST_BODY["port"]}
response = call(host.client, "POST", path, SUPER, body)
# The global discovery route preserves its generic remote-error 503.
assert response.status_code == (503 if receipt_lost or global_route else 502), response.text
assert host.opened == []
if receipt_lost:
assert [row["state"] for row in host.engine.coordinator.snapshot().values()] == ["uncertain"]
with pytest.raises(CoordinationError):
host.engine.coordinator.claim(maintenance=True, kind="update")
return
assert host.engine.coordinator.snapshot() == {}
maintenance = host.engine.coordinator.claim(maintenance=True, kind="update")
assert host.engine.coordinator.release(maintenance)
monkeypatch.setattr(manager, "scan", original_scan)
assert call(host.client, "POST", path, SUPER, body).status_code == 200
assert host.engine.coordinator.snapshot() == {}
def test_signed_host_save_later_connection_failure_retains_prior_writes(signed_host):
host = signed_host
host.behavior.fail_open = False
host.behavior.fail_after_write = True
response = call(host.client, "PUT", "/internal/v1/hosts/7/host", SUPER, HOST_BODY)
assert response.status_code == 502, response.text
claims = host.engine.coordinator.snapshot().values()
assert {row["kind"] for row in claims if row["state"] == "uncertain"} == {
"request", "resource"}
assert any("storage:7" in row["resources"] for row in claims)
with pytest.raises(CoordinationError):
host.engine.coordinator.claim(maintenance=True, kind="update")
def test_signed_vm_connection_failure_does_not_keep_the_vm_locked(signed_host):
host = signed_host
host.behavior.fail_open = False
assert call(host.client, "PUT", "/internal/v1/hosts/7/host", SUPER,
HOST_BODY).status_code == 200
host.behavior.fail_open = True
response = call(host.client, "POST", f"/internal/v1/hosts/7/vms/{VM_ID}/actions",
{**SUPER, "operation_id": "start-connection-one"}, {"action": "start"})
assert response.status_code == 502, response.text
assert response.json()["status"] == "failed"
assert host.engine.coordinator.snapshot() == {}
next_operation = host.engine.coordinator.claim(resources=(f"vm:7:{VM_ID}",))
assert host.engine.coordinator.release(next_operation)
def test_mixed_ssh_request_does_not_use_missing_rpc_marker_as_proof(signed_host, monkeypatch):
host = signed_host
calls = []
async def mixed_handler(scope, receive, send, body, delegated, **_kwargs):
calls.append("SSH helper changed host")
assert not current_execution().connection_failure_safe
await json_response(send, 502, {"error_code": "libvirt_connection_failed"})
monkeypatch.setattr(host.engine, "_dispatch_regular", mixed_handler)
response = call(host.client, "POST", "/internal/v1/vpcs/apply", SUPER, {})
assert response.status_code == 502
assert calls == ["SSH helper changed host"]
assert [row["state"] for row in host.engine.coordinator.snapshot().values()] == ["uncertain"]
def test_service_guard_cannot_relax_a_mixed_workflows_unknown_side_effects(monkeypatch):
coordinator = DurableCoordinator(MemoryStore())
parent = coordinator.claim()
def refused(*_args, **_kwargs):
raise service_module.libvirt.libvirtError("connection refused")
monkeypatch.setattr(service_module, "get_runtime", lambda:
SimpleNamespace(open=refused, queue_timeout=0.02))
service = VMService("test", "images", "iso", "default", execution_scope="7")
with execution_context(coordinator, parent):
with pytest.raises(VMError):
service.ensure_storage_pools()
assert any(row["state"] == "uncertain" and row["resources"] == ["storage:7"]
for row in coordinator.snapshot().values())
@pytest.mark.parametrize("code", ["libvirt_connection_failed", "libvirt_execution_timeout",
"libvirt_execution_stopped", "libvirt_execution_uncertain"])
def test_nested_connection_failures_never_release_earlier_session_writes(code):
coordinator = DurableCoordinator(MemoryStore())
parent = coordinator.claim(resources=(f"vm:7:{VM_ID}",))
with execution_context(coordinator, parent, connection_failure_safe=True):
with resource_guard(("storage:7",)):
mark_remote_write_started()
updated = coordinator.add_resources(parent, ("allocation:7",))
update_current_claim(updated)
assert not connection_failure_before_write("libvirt_connection_failed")
with pytest.raises(VMError):
with resource_guard(("storage:7",), connection_failure_safe=True):
raise VMError("later session failed", 502, error_code=code)
assert any(row["kind"] == "resource" and row["state"] == "uncertain"
for row in coordinator.snapshot().values())
def test_background_write_evidence_returns_to_parent_thread_context():
coordinator = DurableCoordinator(MemoryStore())
parent = coordinator.claim(resources=(f"vm:7:{VM_ID}",))
with execution_context(coordinator, parent, connection_failure_safe=True):
with ThreadPoolExecutor(max_workers=1) as executor:
future = submit_durable(executor, mark_remote_write_started,
outcome=lambda: {"status": "completed"},
job_kind="snapshot_job", job_id="one")
future.result(timeout=2)
assert not connection_failure_before_write("libvirt_connection_failed")
assert current_execution().remote_write_started.is_set()
@pytest.mark.parametrize("earlier_write", [False, True])
def test_background_terminal_connection_failure_preserves_actual_execution_evidence(
monkeypatch, earlier_write):
store = MemoryStore()
coordinator = DurableCoordinator(store)
parent = coordinator.claim(resources=(f"vm:7:{VM_ID}",))
def refused(*_args, **_kwargs):
raise service_module.libvirt.libvirtError("connection refused")
monkeypatch.setattr(service_module, "get_runtime", lambda:
SimpleNamespace(open=refused, queue_timeout=0.02))
service = VMService("test", "images", "iso", "default", execution_scope="7")
def job():
if earlier_write:
mark_remote_write_started()
try:
service.ensure_storage_pools()
except VMError as exc:
store.set("reinstall_job", "one", json.dumps({
"status": "failed", "error_code": exc.error_code}))
with execution_context(coordinator, parent, connection_failure_safe=True):
with ThreadPoolExecutor(max_workers=1) as executor:
submit_durable(executor, job,
outcome=lambda: json.loads(store.get("reinstall_job", "one")),
job_kind="reinstall_job", job_id="one").result(timeout=2)
coordinator.release(parent)
claims = list(coordinator.snapshot().values())
if earlier_write:
assert {row["kind"] for row in claims if row["state"] == "uncertain"} == {
"background", "resource"}
else:
assert claims == []
def test_new_execution_does_not_inherit_another_requests_write_evidence():
coordinator = DurableCoordinator(MemoryStore())
first, second = coordinator.claim(), coordinator.claim()
with execution_context(coordinator, first, connection_failure_safe=True):
mark_remote_write_started()
with execution_context(coordinator, second, connection_failure_safe=True):
assert connection_failure_before_write("libvirt_connection_failed")
assert not connection_failure_before_write("libvirt_connection_failed")
def test_rpc_intent_hook_marks_parent_before_a_write(tmp_path):
coordinator = DurableCoordinator(MemoryStore())
parent = coordinator.claim()
runtime = LibvirtRuntime(tmp_path / "intents")
session = RPCSession(runtime, "test:///default", "7", 5)
with execution_context(coordinator, parent, connection_failure_safe=True):
sequence = runtime._before_write(session, "create")
assert session.intent.is_file()
assert current_execution().remote_write_started.is_set()
runtime._ack_write(session, sequence)
assert session.intent is None
# The RPC reply clears only its pending receipt, never the enclosing
# workflow's evidence that a remote write has already happened.
assert current_execution().remote_write_started.is_set()