You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
379 lines
18 KiB
Python
379 lines
18 KiB
Python
"""Connection admission failures must not become permanent mutation fences.
|
|
|
|
Exercise the signed engine and real VMService/coordination boundaries. Only
|
|
SSH staging, remote objects and the RPC connection transport are simulated.
|
|
"""
|
|
|
|
import base64
|
|
import json
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
from fastapi.testclient import TestClient
|
|
|
|
from test_app import FakeRemoteManager, HOST_BODY, VM_ID
|
|
from test_engine_boundary import MemoryStore, SECRET, SUPER, call
|
|
from test_host_service_scope import StagedManager
|
|
from virsh_panel.app import create_app
|
|
from virsh_panel.config import Settings
|
|
from virsh_panel.coordination import (
|
|
CoordinationError, DurableCoordinator, connection_failure_before_write,
|
|
current_execution, execution_context, mark_remote_write_started,
|
|
resource_guard, update_current_claim,
|
|
)
|
|
from virsh_panel.database import DatabaseTemplateStore
|
|
from virsh_panel.durable_jobs import submit_durable
|
|
from virsh_panel.engine_context import EngineComponents, ExternalOwnership, HostStore
|
|
from virsh_panel.internal import EngineApplication, HostRegistry, json_response
|
|
from virsh_panel.libvirt_rpc import LibvirtRuntime, RPCSession
|
|
from virsh_panel.libvirt_service import GIB, VMError, VMService
|
|
from virsh_panel.metrics import MetricsCollector
|
|
from virsh_panel.remote_host import RemoteHostError
|
|
import virsh_panel.libvirt_service as service_module
|
|
|
|
|
|
@pytest.fixture
|
|
def signed_host(tmp_path, monkeypatch):
|
|
opened, managers = [], {}
|
|
behavior = SimpleNamespace(fail_open=True, fail_after_write=False)
|
|
|
|
def open_session(uri, scope, *, timeout):
|
|
opened.append(scope)
|
|
if behavior.fail_open:
|
|
raise service_module.libvirt.libvirtError("connection refused before open")
|
|
connection = SimpleNamespace(networkLookupByName=lambda _name:
|
|
SimpleNamespace(isActive=lambda: True))
|
|
return SimpleNamespace(connection=connection, failure=None, close=lambda: None)
|
|
|
|
runtime = SimpleNamespace(open=open_session, queue_timeout=0.02)
|
|
monkeypatch.setattr("virsh_panel.app.get_runtime", lambda: runtime)
|
|
monkeypatch.setattr(service_module, "get_runtime", lambda: runtime)
|
|
monkeypatch.setattr(MetricsCollector, "start", lambda _self: None)
|
|
|
|
def pool(service, conn, name, *_args):
|
|
if behavior.fail_after_write:
|
|
mark_remote_write_started()
|
|
# A completed first session's write is still part of this request.
|
|
behavior.fail_open = True
|
|
with service._connect():
|
|
pytest.fail("second connection should fail")
|
|
return SimpleNamespace(name=lambda: name, listVolumes=lambda: [],
|
|
info=lambda: (0, 100 * GIB, 0, 100 * GIB))
|
|
|
|
monkeypatch.setattr(VMService, "_active_pool", pool)
|
|
monkeypatch.setattr(VMService, "_ensure_pool", pool)
|
|
monkeypatch.setattr(VMService, "_refresh_directory_pool", lambda *_args: None)
|
|
monkeypatch.setattr(VMService, "_same_pool_location", lambda *_args: False)
|
|
store, logs = MemoryStore(), MemoryStore()
|
|
config = {"service_secret": base64.b64encode(SECRET).decode(),
|
|
"data_dir": str(tmp_path)}
|
|
|
|
def factory(host_id, user_isos, cloud_templates):
|
|
directory = tmp_path / host_id
|
|
directory.mkdir(mode=0o700, exist_ok=True)
|
|
scoped = HostStore(store, host_id)
|
|
manager = managers[host_id] = StagedManager()
|
|
manager.host_id = host_id
|
|
components = EngineComponents(
|
|
SimpleNamespace(users=ExternalOwnership(scoped)), manager,
|
|
DatabaseTemplateStore(scoped), scoped, logs, user_isos, cloud_templates)
|
|
return components, create_app(Settings(data_dir=directory),
|
|
engine_components=components)
|
|
|
|
registry = HostRegistry(config, store, logs, factory=factory)
|
|
engine = EngineApplication(config, store, logs, registry=registry)
|
|
with TestClient(engine, base_url="http://127.0.0.1") as client:
|
|
yield SimpleNamespace(client=client, engine=engine, behavior=behavior,
|
|
opened=opened, managers=managers)
|
|
|
|
|
|
def test_signed_host_save_connection_failure_releases_claims_and_allows_retry(signed_host):
|
|
host = signed_host
|
|
path = "/internal/v1/hosts/7/host"
|
|
response = call(host.client, "PUT", path, SUPER, HOST_BODY)
|
|
assert response.status_code == 502, response.text
|
|
assert response.json()["error_code"] == "libvirt_connection_failed"
|
|
assert host.managers["7"].current() is None
|
|
assert [row[0] for row in host.managers["7"].calls] == ["stage", "test", "abort"]
|
|
assert host.engine.coordinator.snapshot() == {}
|
|
# No orphan request/resource claim may block maintenance either.
|
|
maintenance = host.engine.coordinator.claim(maintenance=True, kind="update")
|
|
assert host.engine.coordinator.release(maintenance)
|
|
|
|
host.behavior.fail_open = False
|
|
response = call(host.client, "PUT", path, SUPER, HOST_BODY)
|
|
assert response.status_code == 200, response.text
|
|
assert host.managers["7"].current()["host_id"] == "7"
|
|
assert host.opened == ["7", "7", "7"]
|
|
assert host.engine.coordinator.snapshot() == {}
|
|
|
|
|
|
@pytest.mark.parametrize("receipt_lost", [False, True])
|
|
def test_signed_ssh_probe_failure_requires_durable_prestart_proof(
|
|
signed_host, monkeypatch, receipt_lost):
|
|
host = signed_host
|
|
path = "/internal/v1/hosts/7/host"
|
|
assert call(host.client, "GET", path, SUPER).status_code == 200
|
|
manager = host.managers["7"]
|
|
original_probe = manager.test
|
|
|
|
def refused(_staged):
|
|
raise RemoteHostError("SSH probe connection refused", 502)
|
|
|
|
monkeypatch.setattr(manager, "test", refused)
|
|
if receipt_lost:
|
|
def cannot_record(*_args, **_kwargs):
|
|
raise CoordinationError("coordination_unavailable", 503)
|
|
monkeypatch.setattr(host.engine.coordinator, "update_metadata", cannot_record)
|
|
response = call(host.client, "PUT", path, SUPER, HOST_BODY)
|
|
assert response.status_code == (503 if receipt_lost else 502), response.text
|
|
assert host.opened == []
|
|
assert manager.current() is None
|
|
if receipt_lost:
|
|
assert [row["state"] for row in host.engine.coordinator.snapshot().values()] == ["uncertain"]
|
|
with pytest.raises(CoordinationError):
|
|
host.engine.coordinator.claim(maintenance=True, kind="update")
|
|
return
|
|
assert host.engine.coordinator.snapshot() == {}
|
|
maintenance = host.engine.coordinator.claim(maintenance=True, kind="update")
|
|
assert host.engine.coordinator.release(maintenance)
|
|
monkeypatch.setattr(manager, "test", original_probe)
|
|
host.behavior.fail_open = False
|
|
assert call(host.client, "PUT", path, SUPER, HOST_BODY).status_code == 200
|
|
assert host.engine.coordinator.snapshot() == {}
|
|
|
|
|
|
@pytest.mark.parametrize("method,suffix", [("PUT", "host"), ("POST", "host/test")])
|
|
@pytest.mark.parametrize("receipt_lost", [False, True])
|
|
def test_signed_stage_scan_failure_requires_durable_prestart_proof(
|
|
signed_host, monkeypatch, method, suffix, receipt_lost):
|
|
host = signed_host
|
|
assert call(host.client, "GET", "/internal/v1/hosts/7/host", SUPER).status_code == 200
|
|
manager = host.managers["7"]
|
|
original_stage = manager.stage
|
|
|
|
def refused(*_args, **_kwargs):
|
|
raise RemoteHostError("keyscan failed during staging", 502)
|
|
|
|
monkeypatch.setattr(manager, "stage", refused)
|
|
if receipt_lost:
|
|
def cannot_record(*_args, **_kwargs):
|
|
raise CoordinationError("coordination_unavailable", 503)
|
|
monkeypatch.setattr(host.engine.coordinator, "update_metadata", cannot_record)
|
|
path = f"/internal/v1/hosts/7/{suffix}"
|
|
response = call(host.client, method, path, SUPER, HOST_BODY)
|
|
assert response.status_code == (503 if receipt_lost else 502), response.text
|
|
assert host.opened == []
|
|
assert manager.current() is None
|
|
if receipt_lost:
|
|
assert [row["state"] for row in host.engine.coordinator.snapshot().values()] == ["uncertain"]
|
|
with pytest.raises(CoordinationError):
|
|
host.engine.coordinator.claim(maintenance=True, kind="update")
|
|
return
|
|
assert host.engine.coordinator.snapshot() == {}
|
|
maintenance = host.engine.coordinator.claim(maintenance=True, kind="update")
|
|
assert host.engine.coordinator.release(maintenance)
|
|
monkeypatch.setattr(manager, "stage", original_stage)
|
|
host.behavior.fail_open = False
|
|
assert call(host.client, method, path, SUPER, HOST_BODY).status_code == 200
|
|
assert host.engine.coordinator.snapshot() == {}
|
|
|
|
|
|
@pytest.mark.parametrize("global_route", [False, True], ids=["host-scan", "discovery-scan"])
|
|
@pytest.mark.parametrize("receipt_lost", [False, True])
|
|
def test_signed_keyscan_failure_requires_durable_prestart_proof(
|
|
signed_host, monkeypatch, global_route, receipt_lost):
|
|
host = signed_host
|
|
if global_route:
|
|
manager = FakeRemoteManager()
|
|
manager.close = lambda: None
|
|
monkeypatch.setattr("virsh_panel.remote_host.RemoteHostManager", lambda *_args: manager)
|
|
path = "/internal/v1/hosts/scan"
|
|
else:
|
|
assert call(host.client, "GET", "/internal/v1/hosts/7/host", SUPER).status_code == 200
|
|
manager = host.managers["7"]
|
|
path = "/internal/v1/hosts/7/host/scan"
|
|
original_scan = manager.scan
|
|
|
|
def refused(*_args, **_kwargs):
|
|
raise RemoteHostError("keyscan timed out", 502)
|
|
|
|
monkeypatch.setattr(manager, "scan", refused)
|
|
if receipt_lost:
|
|
def cannot_record(*_args, **_kwargs):
|
|
raise CoordinationError("coordination_unavailable", 503)
|
|
monkeypatch.setattr(host.engine.coordinator, "update_metadata", cannot_record)
|
|
body = {"host": HOST_BODY["host"], "port": HOST_BODY["port"]}
|
|
response = call(host.client, "POST", path, SUPER, body)
|
|
# The global discovery route preserves its generic remote-error 503.
|
|
assert response.status_code == (503 if receipt_lost or global_route else 502), response.text
|
|
assert host.opened == []
|
|
if receipt_lost:
|
|
assert [row["state"] for row in host.engine.coordinator.snapshot().values()] == ["uncertain"]
|
|
with pytest.raises(CoordinationError):
|
|
host.engine.coordinator.claim(maintenance=True, kind="update")
|
|
return
|
|
assert host.engine.coordinator.snapshot() == {}
|
|
maintenance = host.engine.coordinator.claim(maintenance=True, kind="update")
|
|
assert host.engine.coordinator.release(maintenance)
|
|
monkeypatch.setattr(manager, "scan", original_scan)
|
|
assert call(host.client, "POST", path, SUPER, body).status_code == 200
|
|
assert host.engine.coordinator.snapshot() == {}
|
|
|
|
|
|
def test_signed_host_save_later_connection_failure_retains_prior_writes(signed_host):
|
|
host = signed_host
|
|
host.behavior.fail_open = False
|
|
host.behavior.fail_after_write = True
|
|
response = call(host.client, "PUT", "/internal/v1/hosts/7/host", SUPER, HOST_BODY)
|
|
assert response.status_code == 502, response.text
|
|
claims = host.engine.coordinator.snapshot().values()
|
|
assert {row["kind"] for row in claims if row["state"] == "uncertain"} == {
|
|
"request", "resource"}
|
|
assert any("storage:7" in row["resources"] for row in claims)
|
|
with pytest.raises(CoordinationError):
|
|
host.engine.coordinator.claim(maintenance=True, kind="update")
|
|
|
|
|
|
def test_signed_vm_connection_failure_does_not_keep_the_vm_locked(signed_host):
|
|
host = signed_host
|
|
host.behavior.fail_open = False
|
|
assert call(host.client, "PUT", "/internal/v1/hosts/7/host", SUPER,
|
|
HOST_BODY).status_code == 200
|
|
host.behavior.fail_open = True
|
|
response = call(host.client, "POST", f"/internal/v1/hosts/7/vms/{VM_ID}/actions",
|
|
{**SUPER, "operation_id": "start-connection-one"}, {"action": "start"})
|
|
assert response.status_code == 502, response.text
|
|
assert response.json()["status"] == "failed"
|
|
assert host.engine.coordinator.snapshot() == {}
|
|
next_operation = host.engine.coordinator.claim(resources=(f"vm:7:{VM_ID}",))
|
|
assert host.engine.coordinator.release(next_operation)
|
|
|
|
|
|
def test_mixed_ssh_request_does_not_use_missing_rpc_marker_as_proof(signed_host, monkeypatch):
|
|
host = signed_host
|
|
calls = []
|
|
|
|
async def mixed_handler(scope, receive, send, body, delegated, **_kwargs):
|
|
calls.append("SSH helper changed host")
|
|
assert not current_execution().connection_failure_safe
|
|
await json_response(send, 502, {"error_code": "libvirt_connection_failed"})
|
|
|
|
monkeypatch.setattr(host.engine, "_dispatch_regular", mixed_handler)
|
|
response = call(host.client, "POST", "/internal/v1/vpcs/apply", SUPER, {})
|
|
assert response.status_code == 502
|
|
assert calls == ["SSH helper changed host"]
|
|
assert [row["state"] for row in host.engine.coordinator.snapshot().values()] == ["uncertain"]
|
|
|
|
|
|
def test_service_guard_cannot_relax_a_mixed_workflows_unknown_side_effects(monkeypatch):
|
|
coordinator = DurableCoordinator(MemoryStore())
|
|
parent = coordinator.claim()
|
|
|
|
def refused(*_args, **_kwargs):
|
|
raise service_module.libvirt.libvirtError("connection refused")
|
|
|
|
monkeypatch.setattr(service_module, "get_runtime", lambda:
|
|
SimpleNamespace(open=refused, queue_timeout=0.02))
|
|
service = VMService("test", "images", "iso", "default", execution_scope="7")
|
|
with execution_context(coordinator, parent):
|
|
with pytest.raises(VMError):
|
|
service.ensure_storage_pools()
|
|
assert any(row["state"] == "uncertain" and row["resources"] == ["storage:7"]
|
|
for row in coordinator.snapshot().values())
|
|
|
|
|
|
@pytest.mark.parametrize("code", ["libvirt_connection_failed", "libvirt_execution_timeout",
|
|
"libvirt_execution_stopped", "libvirt_execution_uncertain"])
|
|
def test_nested_connection_failures_never_release_earlier_session_writes(code):
|
|
coordinator = DurableCoordinator(MemoryStore())
|
|
parent = coordinator.claim(resources=(f"vm:7:{VM_ID}",))
|
|
with execution_context(coordinator, parent, connection_failure_safe=True):
|
|
with resource_guard(("storage:7",)):
|
|
mark_remote_write_started()
|
|
updated = coordinator.add_resources(parent, ("allocation:7",))
|
|
update_current_claim(updated)
|
|
assert not connection_failure_before_write("libvirt_connection_failed")
|
|
with pytest.raises(VMError):
|
|
with resource_guard(("storage:7",), connection_failure_safe=True):
|
|
raise VMError("later session failed", 502, error_code=code)
|
|
assert any(row["kind"] == "resource" and row["state"] == "uncertain"
|
|
for row in coordinator.snapshot().values())
|
|
|
|
|
|
def test_background_write_evidence_returns_to_parent_thread_context():
|
|
coordinator = DurableCoordinator(MemoryStore())
|
|
parent = coordinator.claim(resources=(f"vm:7:{VM_ID}",))
|
|
with execution_context(coordinator, parent, connection_failure_safe=True):
|
|
with ThreadPoolExecutor(max_workers=1) as executor:
|
|
future = submit_durable(executor, mark_remote_write_started,
|
|
outcome=lambda: {"status": "completed"},
|
|
job_kind="snapshot_job", job_id="one")
|
|
future.result(timeout=2)
|
|
assert not connection_failure_before_write("libvirt_connection_failed")
|
|
assert current_execution().remote_write_started.is_set()
|
|
|
|
|
|
@pytest.mark.parametrize("earlier_write", [False, True])
|
|
def test_background_terminal_connection_failure_preserves_actual_execution_evidence(
|
|
monkeypatch, earlier_write):
|
|
store = MemoryStore()
|
|
coordinator = DurableCoordinator(store)
|
|
parent = coordinator.claim(resources=(f"vm:7:{VM_ID}",))
|
|
|
|
def refused(*_args, **_kwargs):
|
|
raise service_module.libvirt.libvirtError("connection refused")
|
|
|
|
monkeypatch.setattr(service_module, "get_runtime", lambda:
|
|
SimpleNamespace(open=refused, queue_timeout=0.02))
|
|
service = VMService("test", "images", "iso", "default", execution_scope="7")
|
|
|
|
def job():
|
|
if earlier_write:
|
|
mark_remote_write_started()
|
|
try:
|
|
service.ensure_storage_pools()
|
|
except VMError as exc:
|
|
store.set("reinstall_job", "one", json.dumps({
|
|
"status": "failed", "error_code": exc.error_code}))
|
|
|
|
with execution_context(coordinator, parent, connection_failure_safe=True):
|
|
with ThreadPoolExecutor(max_workers=1) as executor:
|
|
submit_durable(executor, job,
|
|
outcome=lambda: json.loads(store.get("reinstall_job", "one")),
|
|
job_kind="reinstall_job", job_id="one").result(timeout=2)
|
|
coordinator.release(parent)
|
|
claims = list(coordinator.snapshot().values())
|
|
if earlier_write:
|
|
assert {row["kind"] for row in claims if row["state"] == "uncertain"} == {
|
|
"background", "resource"}
|
|
else:
|
|
assert claims == []
|
|
|
|
|
|
def test_new_execution_does_not_inherit_another_requests_write_evidence():
|
|
coordinator = DurableCoordinator(MemoryStore())
|
|
first, second = coordinator.claim(), coordinator.claim()
|
|
with execution_context(coordinator, first, connection_failure_safe=True):
|
|
mark_remote_write_started()
|
|
with execution_context(coordinator, second, connection_failure_safe=True):
|
|
assert connection_failure_before_write("libvirt_connection_failed")
|
|
assert not connection_failure_before_write("libvirt_connection_failed")
|
|
|
|
|
|
def test_rpc_intent_hook_marks_parent_before_a_write(tmp_path):
|
|
coordinator = DurableCoordinator(MemoryStore())
|
|
parent = coordinator.claim()
|
|
runtime = LibvirtRuntime(tmp_path / "intents")
|
|
session = RPCSession(runtime, "test:///default", "7", 5)
|
|
with execution_context(coordinator, parent, connection_failure_safe=True):
|
|
sequence = runtime._before_write(session, "create")
|
|
assert session.intent.is_file()
|
|
assert current_execution().remote_write_started.is_set()
|
|
runtime._ack_write(session, sequence)
|
|
assert session.intent is None
|
|
# The RPC reply clears only its pending receipt, never the enclosing
|
|
# workflow's evidence that a remote write has already happened.
|
|
assert current_execution().remote_write_started.is_set()
|