You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/engine/tests/test_libvirt_rpc_failures.py

264 lines
10 KiB
Python

"""Failure ordering and durable-fence cleanup regressions for the RPC boundary."""
from __future__ import annotations
from pathlib import Path
import pytest
from virsh_panel import libvirt_rpc
from virsh_panel.libvirt_rpc import ExecutionFailure, LibvirtRuntime
VM_ID = "00000000-0000-4000-8000-000000000001"
@pytest.fixture
def runtime(tmp_path, monkeypatch):
monkeypatch.syspath_prepend(str(Path(__file__).with_name("fixtures")))
instance = LibvirtRuntime(
tmp_path / "rpc", rpc_timeout=1.0, connect_timeout=3.0,
backend_module="libvirt_rpc_backend",
)
yield instance
instance.close(timeout=2.0)
def writing_session(runtime):
session = runtime.open("test:///default", "host-a", timeout=10.0)
domain = session.connection.lookupByUUIDString(VM_ID)
domain.destroy()
# The first RPC is acknowledged. These ordering tests start with the next
# write prepared, before its response is received and durably committed.
runtime._before_write(session, "destroy")
assert session.intent is not None and session.intent.is_file()
return session, domain
@pytest.mark.parametrize("operation", ["call", "qga"])
def test_shutdown_admission_error_cannot_downgrade_an_existing_write(runtime, operation):
session, domain = writing_session(runtime)
# begin_shutdown sets closing before it marks each session failed. Exercise
# the exact intervening state without depending on a scheduler race.
with runtime._condition:
runtime._closing = True
with pytest.raises(ExecutionFailure) as raised:
if operation == "call":
domain.destroy()
else:
session.qemu_agent_command(domain, '{"execute":"guest-exec"}', 1, 0)
assert raised.value.error_code == "libvirt_execution_uncertain"
assert raised.value.outcome_unknown is True
assert session.failure is raised.value
session.close()
assert session.intent.is_file()
@pytest.mark.parametrize("operation", ["call", "qga"])
def test_admission_failure_does_not_overwrite_concurrent_unknown_failure(
runtime, monkeypatch, operation):
session = runtime.open("test:///default", "host-a", timeout=10.0)
domain = session.connection.lookupByUUIDString(VM_ID)
def interrupted_admission(*_args):
session._fail("libvirt_execution_blocked", outcome_unknown=True)
raise ExecutionFailure("libvirt_execution_stopped")
monkeypatch.setattr(runtime, "_before_write", interrupted_admission)
with pytest.raises(ExecutionFailure) as raised:
if operation == "call":
domain.destroy()
else:
session.qemu_agent_command(domain, '{"execute":"guest-exec"}', 1, 0)
assert raised.value.error_code == "libvirt_execution_blocked"
assert raised.value.outcome_unknown is True
assert session._fail("libvirt_execution_timeout") is raised.value
session.close()
def test_qga_cannot_create_a_new_intent_after_a_read_session_failed(runtime, monkeypatch):
session = runtime.open("test:///default", "host-a", timeout=10.0)
domain = session.connection.lookupByUUIDString(VM_ID)
failure = session._fail("libvirt_execution_timeout")
attempted = []
monkeypatch.setattr(runtime, "_before_write", lambda *_args: attempted.append(True))
with pytest.raises(ExecutionFailure) as raised:
session.qemu_agent_command(domain, '{"execute":"guest-exec"}', 1, 0)
assert raised.value is failure
assert attempted == []
assert session.intent is None
session.close()
def test_unlink_failure_keeps_fence_and_exposes_execution_failure(runtime, monkeypatch):
session, _domain = writing_session(runtime)
intent = session.intent
original = intent.read_bytes()
unlink = Path.unlink
def fail_intent_unlink(path, *args, **kwargs):
if path == intent:
raise PermissionError("injected unlink failure")
return unlink(path, *args, **kwargs)
monkeypatch.setattr(Path, "unlink", fail_intent_unlink)
with pytest.raises(ExecutionFailure) as unacknowledged:
runtime._ack_write(session, session.pending_write["sequence"])
assert unacknowledged.value.outcome_unknown is True
session.close()
assert session.failure.error_code == "libvirt_execution_uncertain"
assert intent.read_bytes() == original
assert not session.process.is_alive()
assert session not in runtime._sessions
observer = runtime.open("test:///default", "host-a", timeout=5.0)
assert observer.connection.getHostname() == "rpc-test-host"
with pytest.raises(ExecutionFailure) as blocked:
observer.connection.lookupByUUIDString(VM_ID).destroy()
assert blocked.value.error_code == "libvirt_execution_blocked"
observer.close()
@pytest.mark.parametrize("persistent_failure", [False, True])
def test_directory_sync_failure_restores_deleted_fence(
runtime, monkeypatch, persistent_failure):
session, _domain = writing_session(runtime)
intent = session.intent
original = intent.read_bytes()
sync = runtime._sync
failures = []
def fail_sync(directory):
if not failures or persistent_failure:
failures.append(directory)
raise OSError("injected directory sync failure")
return sync(directory)
monkeypatch.setattr(runtime, "_sync", fail_sync)
with pytest.raises(ExecutionFailure) as unacknowledged:
runtime._ack_write(session, session.pending_write["sequence"])
assert unacknowledged.value.outcome_unknown is True
session.close()
assert session.failure.error_code == "libvirt_execution_uncertain"
assert session.failure.outcome_unknown is True
assert intent.read_bytes() == original
assert not session.process.is_alive()
assert runtime._closing is persistent_failure
def test_reap_failure_retains_worker_registration_and_fence(runtime, monkeypatch):
session, _domain = writing_session(runtime)
reap = session.reap
def fail_reap(_deadline):
raise PermissionError("injected process signalling failure")
monkeypatch.setattr(session, "reap", fail_reap)
session.close()
assert session.failure.error_code == "libvirt_execution_uncertain"
assert session.failure.outcome_unknown is True
assert session.intent.is_file()
assert session in runtime._sessions
monkeypatch.setattr(session, "reap", reap)
assert runtime.close(timeout=2.0) is True
assert not session.process.is_alive()
@pytest.mark.parametrize("write_completed", [False, True])
def test_signal_failure_does_not_mask_rpc_failure(
runtime, monkeypatch, caplog, write_completed):
session = runtime.open("test:///default", "host-a", timeout=10.0)
if write_completed:
session.connection.lookupByUUIDString(VM_ID).destroy()
def lost_response(*_args):
raise EOFError("injected lost receipt")
def failed_signal(*_args):
raise PermissionError("injected sensitive signal details")
with monkeypatch.context() as fault:
fault.setattr(libvirt_rpc, "_receive", lost_response)
fault.setattr(session, "_signal", failed_signal)
with pytest.raises(ExecutionFailure) as raised:
session.connection.getHostname()
assert raised.value.error_code == "libvirt_execution_timeout"
assert raised.value.outcome_unknown is False
assert "Unable to signal libvirt worker" in caplog.text
assert "injected sensitive signal details" not in caplog.text
session.close()
assert not session.process.is_alive()
assert session.intent is None
def test_shutdown_continues_interrupting_after_one_session_fails(runtime, monkeypatch):
first = runtime.open("test:///default", "host-a", timeout=10.0)
second = runtime.open("test:///default", "host-b", timeout=10.0)
interrupted = []
second_interrupt = second.interrupt
def failed_interrupt():
interrupted.append(first)
raise PermissionError("injected interrupt failure")
def record_interrupt():
interrupted.append(second)
second_interrupt()
with monkeypatch.context() as fault:
fault.setattr(first, "interrupt", failed_interrupt)
fault.setattr(second, "interrupt", record_interrupt)
runtime.begin_shutdown()
assert set(interrupted) == {first, second}
assert first.failure.error_code == "libvirt_execution_stopped"
assert second.failure.error_code == "libvirt_execution_stopped"
assert runtime.close(timeout=2.0) is True
def test_runtime_close_reaps_other_groups_after_one_reap_fails(runtime, monkeypatch):
first, _domain = writing_session(runtime)
second = runtime.open("test:///default", "host-b", timeout=10.0)
reaped = []
second_reap = second.reap
def failed_reap(_deadline):
reaped.append(first)
raise PermissionError("injected reap failure")
def record_reap(deadline):
reaped.append(second)
second_reap(deadline)
with monkeypatch.context() as fault:
fault.setattr(first, "reap", failed_reap)
fault.setattr(second, "reap", record_reap)
assert runtime.close(timeout=2.0) is False
assert set(reaped) == {first, second}
assert not second.process.is_alive()
assert first.intent.is_file()
assert first.failure.error_code == "libvirt_execution_uncertain"
assert runtime.close(timeout=2.0) is True
@pytest.mark.parametrize("operation", ["call", "qga"])
def test_concurrent_failure_before_admission_cannot_create_a_new_intent(
runtime, monkeypatch, operation):
session = runtime.open("test:///default", "host-a", timeout=10.0)
domain = session.connection.lookupByUUIDString(VM_ID)
before_write = runtime._before_write
def failed_before_admission(*args):
session._fail("libvirt_execution_timeout")
before_write(*args)
monkeypatch.setattr(runtime, "_before_write", failed_before_admission)
with pytest.raises(ExecutionFailure) as raised:
if operation == "call":
domain.destroy()
else:
session.qemu_agent_command(domain, '{"execute":"guest-exec"}', 1, 0)
assert raised.value.error_code == "libvirt_execution_timeout"
assert raised.value.outcome_unknown is False
assert session.intent is None
session.close()
assert not list(runtime.state_dir.rglob("*.json"))