You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/engine/tests/test_rpc_write_receipts.py

518 lines
19 KiB
Python

"""A later observation failure cannot erase or invent a libvirt write receipt."""
from __future__ import annotations
import json
import os
import threading
import time
from pathlib import Path
import pytest
from virsh_panel import libvirt_rpc
from virsh_panel.libvirt_rpc import ExecutionFailure, LibvirtRuntime
VM_ID = "00000000-0000-4000-8000-000000000001"
OTHER_VM_ID = "00000000-0000-4000-8000-000000000002"
FIXTURES = Path(__file__).with_name("fixtures")
# Keep the fault injection in an importable module: calls below cross the real
# multiprocessing/socket boundary rather than replacing the supervisor's RPC.
BACKEND = '''
import json
import libvirt_rpc_backend as base
from libvirt_rpc_backend import libvirtError
class virDomain(base.virDomain):
def __init__(self, connection, uuid):
super().__init__(connection, uuid)
self.writes = 0
def destroy(self):
self.writes += 1
if self.connection.uri == "test:///second-write-hangs" and self.writes == 2:
self.connection.hang()
if self.connection.uri.startswith("test:///write-error"):
raise libvirtError("write failed after entering native API")
return 0
def create(self):
self.writes += 1
return 0
def info(self):
if self.connection.uri in {"test:///info-hangs", "test:///write-error-info-hangs"} and self.writes:
self.connection.hang()
return super().info()
class virStream(base.virStream):
def finish(self):
if self.connection.uri == "test:///stream-finish-hangs":
self.connection.hang()
return super().finish()
def abort(self):
if self.connection.uri == "test:///stream-abort-hangs":
self.connection.hang()
return super().abort()
class virConnect(base.virConnect):
def __init__(self, uri):
super().__init__(uri)
self.domains = {}
def lookupByUUIDString(self, uuid):
if uuid not in self.domains:
self.domains[uuid] = virDomain(self, uuid)
return self.domains[uuid]
def newStream(self, flags=0):
return virStream(self)
def close(self):
if self.uri == "test:///write-error-close-hangs":
self.hang()
return super().close()
def open(uri):
return virConnect(uri)
def qemuAgentCommand(domain, command, timeout, flags):
if json.loads(command).get("execute") == "guest-exec":
domain.writes += 1
return base.qemuAgentCommand(domain, command, timeout, flags)
'''
@pytest.fixture
def receipt_runtime(tmp_path, monkeypatch):
backend_dir = tmp_path / "backend"
backend_dir.mkdir()
(backend_dir / "rpc_receipt_backend.py").write_text(BACKEND, encoding="utf-8")
monkeypatch.syspath_prepend(str(FIXTURES))
monkeypatch.syspath_prepend(str(backend_dir))
monkeypatch.setenv("PYTHONPATH", os.pathsep.join(filter(None, [
str(backend_dir), str(FIXTURES), str(FIXTURES.parents[1]),
os.environ.get("PYTHONPATH", ""),
])))
runtimes = []
def create(**options):
runtime = LibvirtRuntime(
tmp_path / "rpc-state", rpc_timeout=options.pop("rpc_timeout", 0.35),
connect_timeout=3.0, backend_module="rpc_receipt_backend", **options,
)
runtimes.append(runtime)
return runtime
yield create
for runtime in reversed(runtimes):
runtime.close(timeout=2.0)
def assert_other_vm_writable(runtime):
other = runtime.open("test:///default", "host-a", timeout=5.0)
try:
assert other.connection.lookupByUUIDString(OTHER_VM_ID).destroy() == 0
finally:
other.close()
def assert_other_vm_blocked(runtime):
other = runtime.open("test:///default", "host-a", timeout=5.0)
try:
domain = other.connection.lookupByUUIDString(OTHER_VM_ID)
assert domain.info()[0] == 1 # Fences restrict writes, not observations.
with pytest.raises(ExecutionFailure) as blocked:
domain.destroy()
assert blocked.value.error_code == "libvirt_execution_blocked"
finally:
other.close()
@pytest.mark.parametrize("mutation", ["domain", "qga"])
def test_closed_handle_cannot_create_a_new_write_intent(receipt_runtime, mutation):
runtime = receipt_runtime()
session = runtime.open("test:///default", "host-a", timeout=5)
domain = session.connection.lookupByUUIDString(VM_ID)
domain.destroy()
session.close()
with pytest.raises(ExecutionFailure) as stopped:
if mutation == "domain":
domain.destroy()
else:
session.qemu_agent_command(domain, '{"execute":"guest-exec"}', 1, 0)
assert stopped.value.error_code == "libvirt_execution_stopped"
assert session.intent is None
assert not list(runtime.state_dir.rglob("*.json"))
assert_other_vm_writable(runtime)
@pytest.mark.parametrize("failure_phase", ["info", "close"])
@pytest.mark.parametrize("mutation", ["domain", "qga"])
def test_acknowledged_write_followed_by_read_or_close_timeout_has_no_host_fence(
receipt_runtime, failure_phase, mutation):
runtime = receipt_runtime()
session = runtime.open(f"test:///{failure_phase}-hangs", "host-a", timeout=5.0)
domain = session.connection.lookupByUUIDString(VM_ID)
if mutation == "domain":
assert domain.destroy() == 0
else:
result = session.qemu_agent_command(domain, '{"execute":"guest-exec"}', 1, 0)
assert json.loads(result)["return"]["execute"] == "guest-exec"
if failure_phase == "info":
with pytest.raises(ExecutionFailure) as timed_out:
domain.info()
assert timed_out.value.error_code == "libvirt_execution_timeout"
assert timed_out.value.outcome_unknown is False
session.close()
failure = session.cleanup_failure if failure_phase == "close" else session.failure
assert failure is not None
assert failure.outcome_unknown is False
if failure_phase == "close":
assert session.failure is None # Cleanup cannot replace a command receipt.
assert not session.process.is_alive()
assert not list(runtime.state_dir.rglob("*.json"))
assert_other_vm_writable(runtime)
runtime.close(timeout=2.0)
assert_other_vm_writable(receipt_runtime())
def test_lost_second_write_receipt_retains_fence_despite_first_success(receipt_runtime):
runtime = receipt_runtime()
session = runtime.open("test:///second-write-hangs", "host-a", timeout=5.0)
domain = session.connection.lookupByUUIDString(VM_ID)
assert domain.destroy() == 0
with pytest.raises(ExecutionFailure) as lost:
domain.destroy()
assert lost.value.error_code == "libvirt_execution_uncertain"
assert lost.value.outcome_unknown is True
session.close()
fences = list(runtime.state_dir.rglob("*.json"))
assert len(fences) == 1
evidence = json.loads(fences[0].read_text())
assert evidence["pending_write"] == {"sequence": 2, "method": "destroy"}
assert evidence["last_write_ack"] == 1
assert_other_vm_blocked(runtime)
runtime.close(timeout=2.0)
assert_other_vm_blocked(receipt_runtime())
@pytest.mark.parametrize("failure_phase", ["info", "close"])
def test_native_write_error_does_not_prove_a_rollback(receipt_runtime, failure_phase):
runtime = receipt_runtime()
session = runtime.open(f"test:///write-error-{failure_phase}-hangs", "host-a", timeout=5.0)
domain = session.connection.lookupByUUIDString(VM_ID)
with pytest.raises(Exception) as native:
domain.destroy()
assert not isinstance(native.value, ExecutionFailure)
assert session.failure is None # Native fallback/reconciliation can continue.
evidence = json.loads(session.intent.read_text())
assert evidence["write_error_receipt"] == {"sequence": 1, "method": "destroy"}
assert evidence["pending_write"] is None
assert evidence["last_write_ack"] == 0
if failure_phase == "info":
with pytest.raises(ExecutionFailure) as lost_observation:
domain.info()
assert lost_observation.value.outcome_unknown is True
session.close()
assert session.failure is not None
assert session.failure.outcome_unknown is True
assert len(list(runtime.state_dir.rglob("*.json"))) == 1
assert_other_vm_blocked(runtime)
@pytest.mark.parametrize("read_times_out", [False, True])
def test_successful_fallback_does_not_acknowledge_the_earlier_native_error(
receipt_runtime, read_times_out):
runtime = receipt_runtime()
uri = "test:///write-error-info-hangs" if read_times_out else "test:///write-error"
session = runtime.open(uri, "host-a", timeout=5.0)
domain = session.connection.lookupByUUIDString(VM_ID)
with pytest.raises(Exception) as native:
domain.destroy()
assert not isinstance(native.value, ExecutionFailure)
assert domain.create() == 0
evidence = json.loads(session.intent.read_text())
assert evidence["last_write_ack"] == 2
assert evidence["write_error_receipt"] == {"sequence": 1, "method": "destroy"}
assert evidence["pending_write"] is None
if read_times_out:
with pytest.raises(ExecutionFailure) as lost_observation:
domain.info()
assert lost_observation.value.outcome_unknown is True
else:
assert domain.info()[0] == 1
session.close()
if read_times_out:
assert_other_vm_blocked(runtime)
else:
assert session.failure is None
assert not list(runtime.state_dir.rglob("*.json"))
assert_other_vm_writable(runtime)
@pytest.mark.parametrize("completion", ["finish", "abort"])
def test_stream_completion_lost_receipt_remains_a_pending_write(
receipt_runtime, completion):
runtime = receipt_runtime()
session = runtime.open(f"test:///stream-{completion}-hangs", "host-a", timeout=5.0)
connection = session.connection
volume = connection.storagePoolLookupByName("images").storageVolLookupByName("disk.qcow2")
stream = connection.newStream(0)
assert volume.upload(stream, 0, 3, 0) == 0
assert stream.send(b"abc") == 3
with pytest.raises(ExecutionFailure) as incomplete:
getattr(stream, completion)()
assert incomplete.value.outcome_unknown is True
session.close()
fences = list(runtime.state_dir.rglob("*.json"))
assert len(fences) == 1
evidence = json.loads(fences[0].read_text())
assert evidence["last_write_ack"] == 2
assert evidence["pending_write"] == {"sequence": 3, "method": completion}
assert_other_vm_blocked(runtime)
@pytest.mark.parametrize("completion", ["finish", "abort"])
def test_download_stream_completion_timeout_never_creates_a_write_fence(
receipt_runtime, completion):
runtime = receipt_runtime()
session = runtime.open(
f"test:///stream-{completion}-hangs", "host-a", timeout=5.0)
connection = session.connection
volume = connection.storagePoolLookupByName(
"images").storageVolLookupByName("disk.qcow2")
stream = connection.newStream(0)
assert volume.download(stream, 0, volume.info()[2], 0) == 0
with pytest.raises(ExecutionFailure) as incomplete:
getattr(stream, completion)()
assert incomplete.value.error_code == "libvirt_execution_timeout"
assert incomplete.value.outcome_unknown is False
assert session.pending_write is None
assert session.intent is None
session.close()
assert not list(runtime.state_dir.rglob("*.json"))
assert_other_vm_writable(runtime)
def test_read_connection_cannot_clear_a_legacy_unconfirmed_write(receipt_runtime):
runtime = receipt_runtime()
directory = runtime._directory("host-a")
path = directory / "old-worker.json"
original = json.dumps({
"version": 1, "scope": "host-a", "session": "old-worker",
"method": "destroy", "started_at": 1,
}).encode()
path.write_bytes(original)
observer = runtime.open("test:///default", "host-a", timeout=5.0)
assert observer.connection.lookupByUUIDString(VM_ID).info()[0] == 1
observer.close()
assert_other_vm_blocked(runtime)
assert path.read_bytes() == original
def test_shutdown_after_acknowledged_write_does_not_quarantine_host(receipt_runtime):
runtime = receipt_runtime()
session = runtime.open("test:///default", "host-a", timeout=5.0)
assert session.connection.lookupByUUIDString(VM_ID).destroy() == 0
runtime.begin_shutdown()
assert session.failure is not None
assert session.failure.outcome_unknown is False
assert runtime.close(timeout=2.0) is True
assert not list(runtime.state_dir.rglob("*.json"))
assert_other_vm_writable(receipt_runtime())
def test_shutdown_during_next_write_preserves_its_unconfirmed_receipt(
receipt_runtime, tmp_path):
runtime = receipt_runtime(rpc_timeout=30.0)
session = runtime.open("test:///default", "host-a", timeout=60.0)
assert session.connection.lookupByUUIDString(VM_ID).destroy() == 0
marker = tmp_path / "second-write-started"
errors = []
def write_until_stopped():
try:
session.connection.hang(str(marker), True)
except BaseException as exc:
errors.append(exc)
caller = threading.Thread(target=write_until_stopped, daemon=True)
caller.start()
deadline = time.monotonic() + 3.0
while not marker.exists() and time.monotonic() < deadline:
time.sleep(0.01)
assert marker.exists(), "worker did not receive the next mutation"
runtime.begin_shutdown()
assert runtime.close(timeout=2.0) is True
caller.join(timeout=2.0)
assert not caller.is_alive()
assert len(errors) == 1
assert isinstance(errors[0], ExecutionFailure)
assert errors[0].outcome_unknown is True
assert len(list(runtime.state_dir.rglob("*.json"))) == 1
assert_other_vm_blocked(receipt_runtime())
@pytest.mark.parametrize("failure_kind", ["unlink", "fsync"])
def test_acknowledgement_durability_failure_keeps_the_write_fenced(
receipt_runtime, monkeypatch, failure_kind):
runtime = receipt_runtime()
session = runtime.open("test:///default", "host-a", timeout=5.0)
domain = session.connection.lookupByUUIDString(VM_ID)
receipt_received = threading.Event()
receive = libvirt_rpc._receive
sync = runtime._sync
unlink = Path.unlink
failures = []
def record_receipt(*args):
response = receive(*args)
receipt_received.set()
return response
def fail_ack_sync(directory):
if failure_kind == "fsync" and receipt_received.is_set() and not failures:
failures.append(directory)
raise OSError("injected write receipt durability failure")
return sync(directory)
def fail_ack_unlink(path, *args, **kwargs):
if (failure_kind == "unlink" and receipt_received.is_set()
and path == session.intent and not failures):
failures.append(path)
raise OSError("injected write receipt unlink failure")
return unlink(path, *args, **kwargs)
with monkeypatch.context() as fault:
fault.setattr(libvirt_rpc, "_receive", record_receipt)
fault.setattr(runtime, "_sync", fail_ack_sync)
fault.setattr(Path, "unlink", fail_ack_unlink)
with pytest.raises(ExecutionFailure) as unsettled:
domain.destroy()
assert receipt_received.is_set()
assert len(failures) == 1
assert unsettled.value.error_code == "libvirt_execution_uncertain"
assert unsettled.value.outcome_unknown is True
session.close()
assert len(list(runtime.state_dir.rglob("*.json"))) == 1
assert_other_vm_blocked(runtime)
def test_shutdown_before_received_write_receipt_is_committed_cannot_downgrade_unknown(
receipt_runtime, monkeypatch):
runtime = receipt_runtime()
session = runtime.open("test:///default", "host-a", timeout=5.0)
domain = session.connection.lookupByUUIDString(VM_ID)
receive = libvirt_rpc._receive
def stop_after_receive(*args):
response = receive(*args)
runtime.begin_shutdown()
return response
with monkeypatch.context() as fault:
fault.setattr(libvirt_rpc, "_receive", stop_after_receive)
with pytest.raises(ExecutionFailure) as interrupted:
domain.destroy()
assert interrupted.value.error_code == "libvirt_execution_uncertain"
assert interrupted.value.outcome_unknown is True
assert runtime.close(timeout=2.0) is True
assert len(list(runtime.state_dir.rglob("*.json"))) == 1
assert_other_vm_blocked(receipt_runtime())
@pytest.mark.parametrize("response", [
{"worker_error": True}, {"result": {"unsupported_value": "not decoded"}},
])
def test_malformed_write_response_cannot_be_committed_as_a_receipt(
receipt_runtime, monkeypatch, response):
runtime = receipt_runtime()
session = runtime.open("test:///default", "host-a", timeout=5.0)
domain = session.connection.lookupByUUIDString(VM_ID)
receive = libvirt_rpc._receive
def corrupt_received_result(*args):
receive(*args)
return response
with monkeypatch.context() as fault:
fault.setattr(libvirt_rpc, "_receive", corrupt_received_result)
with pytest.raises(ExecutionFailure) as malformed:
domain.destroy()
assert malformed.value.outcome_unknown is True
session.close()
assert len(list(runtime.state_dir.rglob("*.json"))) == 1
assert_other_vm_blocked(runtime)
def test_next_write_waits_for_previous_receipt_to_be_committed(
receipt_runtime, monkeypatch):
runtime = receipt_runtime(rpc_timeout=3.0)
session = runtime.open("test:///default", "host-a", timeout=10.0)
domain = session.connection.lookupByUUIDString(VM_ID)
before_write = runtime._before_write
ack_write = runtime._ack_write
first_receipt = threading.Event()
commit_receipt = threading.Event()
second_started = threading.Event()
prepared = []
results = []
errors = []
def record_prepare(*args):
prepared.append(args[-1])
return before_write(*args)
def pause_first_receipt(current_session, sequence):
if sequence == 1:
first_receipt.set()
assert commit_receipt.wait(timeout=3.0)
return ack_write(current_session, sequence)
def destroy(second=False):
if second:
second_started.set()
try:
results.append(domain.destroy())
except BaseException as exc:
errors.append(exc)
monkeypatch.setattr(runtime, "_before_write", record_prepare)
monkeypatch.setattr(runtime, "_ack_write", pause_first_receipt)
first = threading.Thread(target=destroy, daemon=True)
second = threading.Thread(target=destroy, args=(True,), daemon=True)
try:
first.start()
assert first_receipt.wait(timeout=3.0)
second.start()
assert second_started.wait(timeout=1.0)
time.sleep(0.05)
assert prepared == ["destroy"]
assert session.pending_write == {"sequence": 1, "method": "destroy"}
finally:
commit_receipt.set()
first.join(timeout=3.0)
if second.ident is not None:
second.join(timeout=3.0)
assert not first.is_alive() and not second.is_alive()
assert errors == []
assert results == [0, 0]
assert prepared == ["destroy", "destroy"]
assert session.last_write_ack == 2
assert session.pending_write is None
session.close()
assert_other_vm_writable(runtime)