"""Failure ordering and durable-fence cleanup regressions for the RPC boundary.""" from __future__ import annotations from pathlib import Path import pytest from virsh_panel import libvirt_rpc from virsh_panel.libvirt_rpc import ExecutionFailure, LibvirtRuntime VM_ID = "00000000-0000-4000-8000-000000000001" @pytest.fixture def runtime(tmp_path, monkeypatch): monkeypatch.syspath_prepend(str(Path(__file__).with_name("fixtures"))) instance = LibvirtRuntime( tmp_path / "rpc", rpc_timeout=1.0, connect_timeout=3.0, backend_module="libvirt_rpc_backend", ) yield instance instance.close(timeout=2.0) def writing_session(runtime): session = runtime.open("test:///default", "host-a", timeout=10.0) domain = session.connection.lookupByUUIDString(VM_ID) domain.destroy() # The first RPC is acknowledged. These ordering tests start with the next # write prepared, before its response is received and durably committed. runtime._before_write(session, "destroy") assert session.intent is not None and session.intent.is_file() return session, domain @pytest.mark.parametrize("operation", ["call", "qga"]) def test_shutdown_admission_error_cannot_downgrade_an_existing_write(runtime, operation): session, domain = writing_session(runtime) # begin_shutdown sets closing before it marks each session failed. Exercise # the exact intervening state without depending on a scheduler race. with runtime._condition: runtime._closing = True with pytest.raises(ExecutionFailure) as raised: if operation == "call": domain.destroy() else: session.qemu_agent_command(domain, '{"execute":"guest-exec"}', 1, 0) assert raised.value.error_code == "libvirt_execution_uncertain" assert raised.value.outcome_unknown is True assert session.failure is raised.value session.close() assert session.intent.is_file() @pytest.mark.parametrize("operation", ["call", "qga"]) def test_admission_failure_does_not_overwrite_concurrent_unknown_failure( runtime, monkeypatch, operation): session = runtime.open("test:///default", "host-a", timeout=10.0) domain = session.connection.lookupByUUIDString(VM_ID) def interrupted_admission(*_args): session._fail("libvirt_execution_blocked", outcome_unknown=True) raise ExecutionFailure("libvirt_execution_stopped") monkeypatch.setattr(runtime, "_before_write", interrupted_admission) with pytest.raises(ExecutionFailure) as raised: if operation == "call": domain.destroy() else: session.qemu_agent_command(domain, '{"execute":"guest-exec"}', 1, 0) assert raised.value.error_code == "libvirt_execution_blocked" assert raised.value.outcome_unknown is True assert session._fail("libvirt_execution_timeout") is raised.value session.close() def test_qga_cannot_create_a_new_intent_after_a_read_session_failed(runtime, monkeypatch): session = runtime.open("test:///default", "host-a", timeout=10.0) domain = session.connection.lookupByUUIDString(VM_ID) failure = session._fail("libvirt_execution_timeout") attempted = [] monkeypatch.setattr(runtime, "_before_write", lambda *_args: attempted.append(True)) with pytest.raises(ExecutionFailure) as raised: session.qemu_agent_command(domain, '{"execute":"guest-exec"}', 1, 0) assert raised.value is failure assert attempted == [] assert session.intent is None session.close() def test_unlink_failure_keeps_fence_and_exposes_execution_failure(runtime, monkeypatch): session, _domain = writing_session(runtime) intent = session.intent original = intent.read_bytes() unlink = Path.unlink def fail_intent_unlink(path, *args, **kwargs): if path == intent: raise PermissionError("injected unlink failure") return unlink(path, *args, **kwargs) monkeypatch.setattr(Path, "unlink", fail_intent_unlink) with pytest.raises(ExecutionFailure) as unacknowledged: runtime._ack_write(session, session.pending_write["sequence"]) assert unacknowledged.value.outcome_unknown is True session.close() assert session.failure.error_code == "libvirt_execution_uncertain" assert intent.read_bytes() == original assert not session.process.is_alive() assert session not in runtime._sessions observer = runtime.open("test:///default", "host-a", timeout=5.0) assert observer.connection.getHostname() == "rpc-test-host" with pytest.raises(ExecutionFailure) as blocked: observer.connection.lookupByUUIDString(VM_ID).destroy() assert blocked.value.error_code == "libvirt_execution_blocked" observer.close() @pytest.mark.parametrize("persistent_failure", [False, True]) def test_directory_sync_failure_restores_deleted_fence( runtime, monkeypatch, persistent_failure): session, _domain = writing_session(runtime) intent = session.intent original = intent.read_bytes() sync = runtime._sync failures = [] def fail_sync(directory): if not failures or persistent_failure: failures.append(directory) raise OSError("injected directory sync failure") return sync(directory) monkeypatch.setattr(runtime, "_sync", fail_sync) with pytest.raises(ExecutionFailure) as unacknowledged: runtime._ack_write(session, session.pending_write["sequence"]) assert unacknowledged.value.outcome_unknown is True session.close() assert session.failure.error_code == "libvirt_execution_uncertain" assert session.failure.outcome_unknown is True assert intent.read_bytes() == original assert not session.process.is_alive() assert runtime._closing is persistent_failure def test_reap_failure_retains_worker_registration_and_fence(runtime, monkeypatch): session, _domain = writing_session(runtime) reap = session.reap def fail_reap(_deadline): raise PermissionError("injected process signalling failure") monkeypatch.setattr(session, "reap", fail_reap) session.close() assert session.failure.error_code == "libvirt_execution_uncertain" assert session.failure.outcome_unknown is True assert session.intent.is_file() assert session in runtime._sessions monkeypatch.setattr(session, "reap", reap) assert runtime.close(timeout=2.0) is True assert not session.process.is_alive() @pytest.mark.parametrize("write_completed", [False, True]) def test_signal_failure_does_not_mask_rpc_failure( runtime, monkeypatch, caplog, write_completed): session = runtime.open("test:///default", "host-a", timeout=10.0) if write_completed: session.connection.lookupByUUIDString(VM_ID).destroy() def lost_response(*_args): raise EOFError("injected lost receipt") def failed_signal(*_args): raise PermissionError("injected sensitive signal details") with monkeypatch.context() as fault: fault.setattr(libvirt_rpc, "_receive", lost_response) fault.setattr(session, "_signal", failed_signal) with pytest.raises(ExecutionFailure) as raised: session.connection.getHostname() assert raised.value.error_code == "libvirt_execution_timeout" assert raised.value.outcome_unknown is False assert "Unable to signal libvirt worker" in caplog.text assert "injected sensitive signal details" not in caplog.text session.close() assert not session.process.is_alive() assert session.intent is None def test_shutdown_continues_interrupting_after_one_session_fails(runtime, monkeypatch): first = runtime.open("test:///default", "host-a", timeout=10.0) second = runtime.open("test:///default", "host-b", timeout=10.0) interrupted = [] second_interrupt = second.interrupt def failed_interrupt(): interrupted.append(first) raise PermissionError("injected interrupt failure") def record_interrupt(): interrupted.append(second) second_interrupt() with monkeypatch.context() as fault: fault.setattr(first, "interrupt", failed_interrupt) fault.setattr(second, "interrupt", record_interrupt) runtime.begin_shutdown() assert set(interrupted) == {first, second} assert first.failure.error_code == "libvirt_execution_stopped" assert second.failure.error_code == "libvirt_execution_stopped" assert runtime.close(timeout=2.0) is True def test_runtime_close_reaps_other_groups_after_one_reap_fails(runtime, monkeypatch): first, _domain = writing_session(runtime) second = runtime.open("test:///default", "host-b", timeout=10.0) reaped = [] second_reap = second.reap def failed_reap(_deadline): reaped.append(first) raise PermissionError("injected reap failure") def record_reap(deadline): reaped.append(second) second_reap(deadline) with monkeypatch.context() as fault: fault.setattr(first, "reap", failed_reap) fault.setattr(second, "reap", record_reap) assert runtime.close(timeout=2.0) is False assert set(reaped) == {first, second} assert not second.process.is_alive() assert first.intent.is_file() assert first.failure.error_code == "libvirt_execution_uncertain" assert runtime.close(timeout=2.0) is True @pytest.mark.parametrize("operation", ["call", "qga"]) def test_concurrent_failure_before_admission_cannot_create_a_new_intent( runtime, monkeypatch, operation): session = runtime.open("test:///default", "host-a", timeout=10.0) domain = session.connection.lookupByUUIDString(VM_ID) before_write = runtime._before_write def failed_before_admission(*args): session._fail("libvirt_execution_timeout") before_write(*args) monkeypatch.setattr(runtime, "_before_write", failed_before_admission) with pytest.raises(ExecutionFailure) as raised: if operation == "call": domain.destroy() else: session.qemu_agent_command(domain, '{"execute":"guest-exec"}', 1, 0) assert raised.value.error_code == "libvirt_execution_timeout" assert raised.value.outcome_unknown is False assert session.intent is None session.close() assert not list(runtime.state_dir.rglob("*.json"))