You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
264 lines
10 KiB
Python
264 lines
10 KiB
Python
"""Failure ordering and durable-fence cleanup regressions for the RPC boundary."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from virsh_panel import libvirt_rpc
|
|
from virsh_panel.libvirt_rpc import ExecutionFailure, LibvirtRuntime
|
|
|
|
|
|
VM_ID = "00000000-0000-4000-8000-000000000001"
|
|
|
|
|
|
@pytest.fixture
|
|
def runtime(tmp_path, monkeypatch):
|
|
monkeypatch.syspath_prepend(str(Path(__file__).with_name("fixtures")))
|
|
instance = LibvirtRuntime(
|
|
tmp_path / "rpc", rpc_timeout=1.0, connect_timeout=3.0,
|
|
backend_module="libvirt_rpc_backend",
|
|
)
|
|
yield instance
|
|
instance.close(timeout=2.0)
|
|
|
|
|
|
def writing_session(runtime):
|
|
session = runtime.open("test:///default", "host-a", timeout=10.0)
|
|
domain = session.connection.lookupByUUIDString(VM_ID)
|
|
domain.destroy()
|
|
# The first RPC is acknowledged. These ordering tests start with the next
|
|
# write prepared, before its response is received and durably committed.
|
|
runtime._before_write(session, "destroy")
|
|
assert session.intent is not None and session.intent.is_file()
|
|
return session, domain
|
|
|
|
|
|
@pytest.mark.parametrize("operation", ["call", "qga"])
|
|
def test_shutdown_admission_error_cannot_downgrade_an_existing_write(runtime, operation):
|
|
session, domain = writing_session(runtime)
|
|
# begin_shutdown sets closing before it marks each session failed. Exercise
|
|
# the exact intervening state without depending on a scheduler race.
|
|
with runtime._condition:
|
|
runtime._closing = True
|
|
with pytest.raises(ExecutionFailure) as raised:
|
|
if operation == "call":
|
|
domain.destroy()
|
|
else:
|
|
session.qemu_agent_command(domain, '{"execute":"guest-exec"}', 1, 0)
|
|
assert raised.value.error_code == "libvirt_execution_uncertain"
|
|
assert raised.value.outcome_unknown is True
|
|
assert session.failure is raised.value
|
|
session.close()
|
|
assert session.intent.is_file()
|
|
|
|
|
|
@pytest.mark.parametrize("operation", ["call", "qga"])
|
|
def test_admission_failure_does_not_overwrite_concurrent_unknown_failure(
|
|
runtime, monkeypatch, operation):
|
|
session = runtime.open("test:///default", "host-a", timeout=10.0)
|
|
domain = session.connection.lookupByUUIDString(VM_ID)
|
|
|
|
def interrupted_admission(*_args):
|
|
session._fail("libvirt_execution_blocked", outcome_unknown=True)
|
|
raise ExecutionFailure("libvirt_execution_stopped")
|
|
|
|
monkeypatch.setattr(runtime, "_before_write", interrupted_admission)
|
|
with pytest.raises(ExecutionFailure) as raised:
|
|
if operation == "call":
|
|
domain.destroy()
|
|
else:
|
|
session.qemu_agent_command(domain, '{"execute":"guest-exec"}', 1, 0)
|
|
assert raised.value.error_code == "libvirt_execution_blocked"
|
|
assert raised.value.outcome_unknown is True
|
|
assert session._fail("libvirt_execution_timeout") is raised.value
|
|
session.close()
|
|
|
|
|
|
def test_qga_cannot_create_a_new_intent_after_a_read_session_failed(runtime, monkeypatch):
|
|
session = runtime.open("test:///default", "host-a", timeout=10.0)
|
|
domain = session.connection.lookupByUUIDString(VM_ID)
|
|
failure = session._fail("libvirt_execution_timeout")
|
|
attempted = []
|
|
monkeypatch.setattr(runtime, "_before_write", lambda *_args: attempted.append(True))
|
|
with pytest.raises(ExecutionFailure) as raised:
|
|
session.qemu_agent_command(domain, '{"execute":"guest-exec"}', 1, 0)
|
|
assert raised.value is failure
|
|
assert attempted == []
|
|
assert session.intent is None
|
|
session.close()
|
|
|
|
|
|
def test_unlink_failure_keeps_fence_and_exposes_execution_failure(runtime, monkeypatch):
|
|
session, _domain = writing_session(runtime)
|
|
intent = session.intent
|
|
original = intent.read_bytes()
|
|
unlink = Path.unlink
|
|
|
|
def fail_intent_unlink(path, *args, **kwargs):
|
|
if path == intent:
|
|
raise PermissionError("injected unlink failure")
|
|
return unlink(path, *args, **kwargs)
|
|
|
|
monkeypatch.setattr(Path, "unlink", fail_intent_unlink)
|
|
with pytest.raises(ExecutionFailure) as unacknowledged:
|
|
runtime._ack_write(session, session.pending_write["sequence"])
|
|
assert unacknowledged.value.outcome_unknown is True
|
|
session.close()
|
|
assert session.failure.error_code == "libvirt_execution_uncertain"
|
|
assert intent.read_bytes() == original
|
|
assert not session.process.is_alive()
|
|
assert session not in runtime._sessions
|
|
observer = runtime.open("test:///default", "host-a", timeout=5.0)
|
|
assert observer.connection.getHostname() == "rpc-test-host"
|
|
with pytest.raises(ExecutionFailure) as blocked:
|
|
observer.connection.lookupByUUIDString(VM_ID).destroy()
|
|
assert blocked.value.error_code == "libvirt_execution_blocked"
|
|
observer.close()
|
|
|
|
|
|
@pytest.mark.parametrize("persistent_failure", [False, True])
|
|
def test_directory_sync_failure_restores_deleted_fence(
|
|
runtime, monkeypatch, persistent_failure):
|
|
session, _domain = writing_session(runtime)
|
|
intent = session.intent
|
|
original = intent.read_bytes()
|
|
sync = runtime._sync
|
|
failures = []
|
|
|
|
def fail_sync(directory):
|
|
if not failures or persistent_failure:
|
|
failures.append(directory)
|
|
raise OSError("injected directory sync failure")
|
|
return sync(directory)
|
|
|
|
monkeypatch.setattr(runtime, "_sync", fail_sync)
|
|
with pytest.raises(ExecutionFailure) as unacknowledged:
|
|
runtime._ack_write(session, session.pending_write["sequence"])
|
|
assert unacknowledged.value.outcome_unknown is True
|
|
session.close()
|
|
assert session.failure.error_code == "libvirt_execution_uncertain"
|
|
assert session.failure.outcome_unknown is True
|
|
assert intent.read_bytes() == original
|
|
assert not session.process.is_alive()
|
|
assert runtime._closing is persistent_failure
|
|
|
|
|
|
def test_reap_failure_retains_worker_registration_and_fence(runtime, monkeypatch):
|
|
session, _domain = writing_session(runtime)
|
|
reap = session.reap
|
|
|
|
def fail_reap(_deadline):
|
|
raise PermissionError("injected process signalling failure")
|
|
|
|
monkeypatch.setattr(session, "reap", fail_reap)
|
|
session.close()
|
|
assert session.failure.error_code == "libvirt_execution_uncertain"
|
|
assert session.failure.outcome_unknown is True
|
|
assert session.intent.is_file()
|
|
assert session in runtime._sessions
|
|
monkeypatch.setattr(session, "reap", reap)
|
|
assert runtime.close(timeout=2.0) is True
|
|
assert not session.process.is_alive()
|
|
|
|
|
|
@pytest.mark.parametrize("write_completed", [False, True])
|
|
def test_signal_failure_does_not_mask_rpc_failure(
|
|
runtime, monkeypatch, caplog, write_completed):
|
|
session = runtime.open("test:///default", "host-a", timeout=10.0)
|
|
if write_completed:
|
|
session.connection.lookupByUUIDString(VM_ID).destroy()
|
|
|
|
def lost_response(*_args):
|
|
raise EOFError("injected lost receipt")
|
|
|
|
def failed_signal(*_args):
|
|
raise PermissionError("injected sensitive signal details")
|
|
|
|
with monkeypatch.context() as fault:
|
|
fault.setattr(libvirt_rpc, "_receive", lost_response)
|
|
fault.setattr(session, "_signal", failed_signal)
|
|
with pytest.raises(ExecutionFailure) as raised:
|
|
session.connection.getHostname()
|
|
assert raised.value.error_code == "libvirt_execution_timeout"
|
|
assert raised.value.outcome_unknown is False
|
|
assert "Unable to signal libvirt worker" in caplog.text
|
|
assert "injected sensitive signal details" not in caplog.text
|
|
session.close()
|
|
assert not session.process.is_alive()
|
|
assert session.intent is None
|
|
|
|
|
|
def test_shutdown_continues_interrupting_after_one_session_fails(runtime, monkeypatch):
|
|
first = runtime.open("test:///default", "host-a", timeout=10.0)
|
|
second = runtime.open("test:///default", "host-b", timeout=10.0)
|
|
interrupted = []
|
|
second_interrupt = second.interrupt
|
|
|
|
def failed_interrupt():
|
|
interrupted.append(first)
|
|
raise PermissionError("injected interrupt failure")
|
|
|
|
def record_interrupt():
|
|
interrupted.append(second)
|
|
second_interrupt()
|
|
|
|
with monkeypatch.context() as fault:
|
|
fault.setattr(first, "interrupt", failed_interrupt)
|
|
fault.setattr(second, "interrupt", record_interrupt)
|
|
runtime.begin_shutdown()
|
|
assert set(interrupted) == {first, second}
|
|
assert first.failure.error_code == "libvirt_execution_stopped"
|
|
assert second.failure.error_code == "libvirt_execution_stopped"
|
|
assert runtime.close(timeout=2.0) is True
|
|
|
|
|
|
def test_runtime_close_reaps_other_groups_after_one_reap_fails(runtime, monkeypatch):
|
|
first, _domain = writing_session(runtime)
|
|
second = runtime.open("test:///default", "host-b", timeout=10.0)
|
|
reaped = []
|
|
second_reap = second.reap
|
|
|
|
def failed_reap(_deadline):
|
|
reaped.append(first)
|
|
raise PermissionError("injected reap failure")
|
|
|
|
def record_reap(deadline):
|
|
reaped.append(second)
|
|
second_reap(deadline)
|
|
|
|
with monkeypatch.context() as fault:
|
|
fault.setattr(first, "reap", failed_reap)
|
|
fault.setattr(second, "reap", record_reap)
|
|
assert runtime.close(timeout=2.0) is False
|
|
assert set(reaped) == {first, second}
|
|
assert not second.process.is_alive()
|
|
assert first.intent.is_file()
|
|
assert first.failure.error_code == "libvirt_execution_uncertain"
|
|
assert runtime.close(timeout=2.0) is True
|
|
|
|
|
|
@pytest.mark.parametrize("operation", ["call", "qga"])
|
|
def test_concurrent_failure_before_admission_cannot_create_a_new_intent(
|
|
runtime, monkeypatch, operation):
|
|
session = runtime.open("test:///default", "host-a", timeout=10.0)
|
|
domain = session.connection.lookupByUUIDString(VM_ID)
|
|
before_write = runtime._before_write
|
|
|
|
def failed_before_admission(*args):
|
|
session._fail("libvirt_execution_timeout")
|
|
before_write(*args)
|
|
|
|
monkeypatch.setattr(runtime, "_before_write", failed_before_admission)
|
|
with pytest.raises(ExecutionFailure) as raised:
|
|
if operation == "call":
|
|
domain.destroy()
|
|
else:
|
|
session.qemu_agent_command(domain, '{"execute":"guest-exec"}', 1, 0)
|
|
assert raised.value.error_code == "libvirt_execution_timeout"
|
|
assert raised.value.outcome_unknown is False
|
|
assert session.intent is None
|
|
session.close()
|
|
assert not list(runtime.state_dir.rglob("*.json"))
|