You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
518 lines
19 KiB
Python
518 lines
19 KiB
Python
"""A later observation failure cannot erase or invent a libvirt write receipt."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import threading
|
|
import time
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from virsh_panel import libvirt_rpc
|
|
from virsh_panel.libvirt_rpc import ExecutionFailure, LibvirtRuntime
|
|
|
|
|
|
VM_ID = "00000000-0000-4000-8000-000000000001"
|
|
OTHER_VM_ID = "00000000-0000-4000-8000-000000000002"
|
|
FIXTURES = Path(__file__).with_name("fixtures")
|
|
|
|
# Keep the fault injection in an importable module: calls below cross the real
|
|
# multiprocessing/socket boundary rather than replacing the supervisor's RPC.
|
|
BACKEND = '''
|
|
import json
|
|
import libvirt_rpc_backend as base
|
|
from libvirt_rpc_backend import libvirtError
|
|
|
|
|
|
class virDomain(base.virDomain):
|
|
def __init__(self, connection, uuid):
|
|
super().__init__(connection, uuid)
|
|
self.writes = 0
|
|
|
|
def destroy(self):
|
|
self.writes += 1
|
|
if self.connection.uri == "test:///second-write-hangs" and self.writes == 2:
|
|
self.connection.hang()
|
|
if self.connection.uri.startswith("test:///write-error"):
|
|
raise libvirtError("write failed after entering native API")
|
|
return 0
|
|
|
|
def create(self):
|
|
self.writes += 1
|
|
return 0
|
|
|
|
def info(self):
|
|
if self.connection.uri in {"test:///info-hangs", "test:///write-error-info-hangs"} and self.writes:
|
|
self.connection.hang()
|
|
return super().info()
|
|
|
|
|
|
class virStream(base.virStream):
|
|
def finish(self):
|
|
if self.connection.uri == "test:///stream-finish-hangs":
|
|
self.connection.hang()
|
|
return super().finish()
|
|
|
|
def abort(self):
|
|
if self.connection.uri == "test:///stream-abort-hangs":
|
|
self.connection.hang()
|
|
return super().abort()
|
|
|
|
|
|
class virConnect(base.virConnect):
|
|
def __init__(self, uri):
|
|
super().__init__(uri)
|
|
self.domains = {}
|
|
|
|
def lookupByUUIDString(self, uuid):
|
|
if uuid not in self.domains:
|
|
self.domains[uuid] = virDomain(self, uuid)
|
|
return self.domains[uuid]
|
|
|
|
def newStream(self, flags=0):
|
|
return virStream(self)
|
|
|
|
def close(self):
|
|
if self.uri == "test:///write-error-close-hangs":
|
|
self.hang()
|
|
return super().close()
|
|
|
|
|
|
def open(uri):
|
|
return virConnect(uri)
|
|
|
|
|
|
def qemuAgentCommand(domain, command, timeout, flags):
|
|
if json.loads(command).get("execute") == "guest-exec":
|
|
domain.writes += 1
|
|
return base.qemuAgentCommand(domain, command, timeout, flags)
|
|
'''
|
|
|
|
|
|
@pytest.fixture
|
|
def receipt_runtime(tmp_path, monkeypatch):
|
|
backend_dir = tmp_path / "backend"
|
|
backend_dir.mkdir()
|
|
(backend_dir / "rpc_receipt_backend.py").write_text(BACKEND, encoding="utf-8")
|
|
monkeypatch.syspath_prepend(str(FIXTURES))
|
|
monkeypatch.syspath_prepend(str(backend_dir))
|
|
monkeypatch.setenv("PYTHONPATH", os.pathsep.join(filter(None, [
|
|
str(backend_dir), str(FIXTURES), str(FIXTURES.parents[1]),
|
|
os.environ.get("PYTHONPATH", ""),
|
|
])))
|
|
runtimes = []
|
|
|
|
def create(**options):
|
|
runtime = LibvirtRuntime(
|
|
tmp_path / "rpc-state", rpc_timeout=options.pop("rpc_timeout", 0.35),
|
|
connect_timeout=3.0, backend_module="rpc_receipt_backend", **options,
|
|
)
|
|
runtimes.append(runtime)
|
|
return runtime
|
|
|
|
yield create
|
|
for runtime in reversed(runtimes):
|
|
runtime.close(timeout=2.0)
|
|
|
|
|
|
def assert_other_vm_writable(runtime):
|
|
other = runtime.open("test:///default", "host-a", timeout=5.0)
|
|
try:
|
|
assert other.connection.lookupByUUIDString(OTHER_VM_ID).destroy() == 0
|
|
finally:
|
|
other.close()
|
|
|
|
|
|
def assert_other_vm_blocked(runtime):
|
|
other = runtime.open("test:///default", "host-a", timeout=5.0)
|
|
try:
|
|
domain = other.connection.lookupByUUIDString(OTHER_VM_ID)
|
|
assert domain.info()[0] == 1 # Fences restrict writes, not observations.
|
|
with pytest.raises(ExecutionFailure) as blocked:
|
|
domain.destroy()
|
|
assert blocked.value.error_code == "libvirt_execution_blocked"
|
|
finally:
|
|
other.close()
|
|
|
|
|
|
@pytest.mark.parametrize("mutation", ["domain", "qga"])
|
|
def test_closed_handle_cannot_create_a_new_write_intent(receipt_runtime, mutation):
|
|
runtime = receipt_runtime()
|
|
session = runtime.open("test:///default", "host-a", timeout=5)
|
|
domain = session.connection.lookupByUUIDString(VM_ID)
|
|
domain.destroy()
|
|
session.close()
|
|
with pytest.raises(ExecutionFailure) as stopped:
|
|
if mutation == "domain":
|
|
domain.destroy()
|
|
else:
|
|
session.qemu_agent_command(domain, '{"execute":"guest-exec"}', 1, 0)
|
|
assert stopped.value.error_code == "libvirt_execution_stopped"
|
|
assert session.intent is None
|
|
assert not list(runtime.state_dir.rglob("*.json"))
|
|
assert_other_vm_writable(runtime)
|
|
|
|
|
|
@pytest.mark.parametrize("failure_phase", ["info", "close"])
|
|
@pytest.mark.parametrize("mutation", ["domain", "qga"])
|
|
def test_acknowledged_write_followed_by_read_or_close_timeout_has_no_host_fence(
|
|
receipt_runtime, failure_phase, mutation):
|
|
runtime = receipt_runtime()
|
|
session = runtime.open(f"test:///{failure_phase}-hangs", "host-a", timeout=5.0)
|
|
domain = session.connection.lookupByUUIDString(VM_ID)
|
|
if mutation == "domain":
|
|
assert domain.destroy() == 0
|
|
else:
|
|
result = session.qemu_agent_command(domain, '{"execute":"guest-exec"}', 1, 0)
|
|
assert json.loads(result)["return"]["execute"] == "guest-exec"
|
|
|
|
if failure_phase == "info":
|
|
with pytest.raises(ExecutionFailure) as timed_out:
|
|
domain.info()
|
|
assert timed_out.value.error_code == "libvirt_execution_timeout"
|
|
assert timed_out.value.outcome_unknown is False
|
|
session.close()
|
|
|
|
failure = session.cleanup_failure if failure_phase == "close" else session.failure
|
|
assert failure is not None
|
|
assert failure.outcome_unknown is False
|
|
if failure_phase == "close":
|
|
assert session.failure is None # Cleanup cannot replace a command receipt.
|
|
assert not session.process.is_alive()
|
|
assert not list(runtime.state_dir.rglob("*.json"))
|
|
assert_other_vm_writable(runtime)
|
|
runtime.close(timeout=2.0)
|
|
assert_other_vm_writable(receipt_runtime())
|
|
|
|
|
|
def test_lost_second_write_receipt_retains_fence_despite_first_success(receipt_runtime):
|
|
runtime = receipt_runtime()
|
|
session = runtime.open("test:///second-write-hangs", "host-a", timeout=5.0)
|
|
domain = session.connection.lookupByUUIDString(VM_ID)
|
|
assert domain.destroy() == 0
|
|
with pytest.raises(ExecutionFailure) as lost:
|
|
domain.destroy()
|
|
assert lost.value.error_code == "libvirt_execution_uncertain"
|
|
assert lost.value.outcome_unknown is True
|
|
session.close()
|
|
fences = list(runtime.state_dir.rglob("*.json"))
|
|
assert len(fences) == 1
|
|
evidence = json.loads(fences[0].read_text())
|
|
assert evidence["pending_write"] == {"sequence": 2, "method": "destroy"}
|
|
assert evidence["last_write_ack"] == 1
|
|
assert_other_vm_blocked(runtime)
|
|
runtime.close(timeout=2.0)
|
|
assert_other_vm_blocked(receipt_runtime())
|
|
|
|
|
|
@pytest.mark.parametrize("failure_phase", ["info", "close"])
|
|
def test_native_write_error_does_not_prove_a_rollback(receipt_runtime, failure_phase):
|
|
runtime = receipt_runtime()
|
|
session = runtime.open(f"test:///write-error-{failure_phase}-hangs", "host-a", timeout=5.0)
|
|
domain = session.connection.lookupByUUIDString(VM_ID)
|
|
with pytest.raises(Exception) as native:
|
|
domain.destroy()
|
|
assert not isinstance(native.value, ExecutionFailure)
|
|
assert session.failure is None # Native fallback/reconciliation can continue.
|
|
evidence = json.loads(session.intent.read_text())
|
|
assert evidence["write_error_receipt"] == {"sequence": 1, "method": "destroy"}
|
|
assert evidence["pending_write"] is None
|
|
assert evidence["last_write_ack"] == 0
|
|
if failure_phase == "info":
|
|
with pytest.raises(ExecutionFailure) as lost_observation:
|
|
domain.info()
|
|
assert lost_observation.value.outcome_unknown is True
|
|
session.close()
|
|
assert session.failure is not None
|
|
assert session.failure.outcome_unknown is True
|
|
assert len(list(runtime.state_dir.rglob("*.json"))) == 1
|
|
assert_other_vm_blocked(runtime)
|
|
|
|
|
|
@pytest.mark.parametrize("read_times_out", [False, True])
|
|
def test_successful_fallback_does_not_acknowledge_the_earlier_native_error(
|
|
receipt_runtime, read_times_out):
|
|
runtime = receipt_runtime()
|
|
uri = "test:///write-error-info-hangs" if read_times_out else "test:///write-error"
|
|
session = runtime.open(uri, "host-a", timeout=5.0)
|
|
domain = session.connection.lookupByUUIDString(VM_ID)
|
|
with pytest.raises(Exception) as native:
|
|
domain.destroy()
|
|
assert not isinstance(native.value, ExecutionFailure)
|
|
assert domain.create() == 0
|
|
evidence = json.loads(session.intent.read_text())
|
|
assert evidence["last_write_ack"] == 2
|
|
assert evidence["write_error_receipt"] == {"sequence": 1, "method": "destroy"}
|
|
assert evidence["pending_write"] is None
|
|
if read_times_out:
|
|
with pytest.raises(ExecutionFailure) as lost_observation:
|
|
domain.info()
|
|
assert lost_observation.value.outcome_unknown is True
|
|
else:
|
|
assert domain.info()[0] == 1
|
|
session.close()
|
|
if read_times_out:
|
|
assert_other_vm_blocked(runtime)
|
|
else:
|
|
assert session.failure is None
|
|
assert not list(runtime.state_dir.rglob("*.json"))
|
|
assert_other_vm_writable(runtime)
|
|
|
|
|
|
@pytest.mark.parametrize("completion", ["finish", "abort"])
|
|
def test_stream_completion_lost_receipt_remains_a_pending_write(
|
|
receipt_runtime, completion):
|
|
runtime = receipt_runtime()
|
|
session = runtime.open(f"test:///stream-{completion}-hangs", "host-a", timeout=5.0)
|
|
connection = session.connection
|
|
volume = connection.storagePoolLookupByName("images").storageVolLookupByName("disk.qcow2")
|
|
stream = connection.newStream(0)
|
|
assert volume.upload(stream, 0, 3, 0) == 0
|
|
assert stream.send(b"abc") == 3
|
|
with pytest.raises(ExecutionFailure) as incomplete:
|
|
getattr(stream, completion)()
|
|
assert incomplete.value.outcome_unknown is True
|
|
session.close()
|
|
fences = list(runtime.state_dir.rglob("*.json"))
|
|
assert len(fences) == 1
|
|
evidence = json.loads(fences[0].read_text())
|
|
assert evidence["last_write_ack"] == 2
|
|
assert evidence["pending_write"] == {"sequence": 3, "method": completion}
|
|
assert_other_vm_blocked(runtime)
|
|
|
|
|
|
@pytest.mark.parametrize("completion", ["finish", "abort"])
|
|
def test_download_stream_completion_timeout_never_creates_a_write_fence(
|
|
receipt_runtime, completion):
|
|
runtime = receipt_runtime()
|
|
session = runtime.open(
|
|
f"test:///stream-{completion}-hangs", "host-a", timeout=5.0)
|
|
connection = session.connection
|
|
volume = connection.storagePoolLookupByName(
|
|
"images").storageVolLookupByName("disk.qcow2")
|
|
stream = connection.newStream(0)
|
|
assert volume.download(stream, 0, volume.info()[2], 0) == 0
|
|
|
|
with pytest.raises(ExecutionFailure) as incomplete:
|
|
getattr(stream, completion)()
|
|
|
|
assert incomplete.value.error_code == "libvirt_execution_timeout"
|
|
assert incomplete.value.outcome_unknown is False
|
|
assert session.pending_write is None
|
|
assert session.intent is None
|
|
session.close()
|
|
assert not list(runtime.state_dir.rglob("*.json"))
|
|
assert_other_vm_writable(runtime)
|
|
|
|
|
|
def test_read_connection_cannot_clear_a_legacy_unconfirmed_write(receipt_runtime):
|
|
runtime = receipt_runtime()
|
|
directory = runtime._directory("host-a")
|
|
path = directory / "old-worker.json"
|
|
original = json.dumps({
|
|
"version": 1, "scope": "host-a", "session": "old-worker",
|
|
"method": "destroy", "started_at": 1,
|
|
}).encode()
|
|
path.write_bytes(original)
|
|
observer = runtime.open("test:///default", "host-a", timeout=5.0)
|
|
assert observer.connection.lookupByUUIDString(VM_ID).info()[0] == 1
|
|
observer.close()
|
|
assert_other_vm_blocked(runtime)
|
|
assert path.read_bytes() == original
|
|
|
|
|
|
def test_shutdown_after_acknowledged_write_does_not_quarantine_host(receipt_runtime):
|
|
runtime = receipt_runtime()
|
|
session = runtime.open("test:///default", "host-a", timeout=5.0)
|
|
assert session.connection.lookupByUUIDString(VM_ID).destroy() == 0
|
|
runtime.begin_shutdown()
|
|
assert session.failure is not None
|
|
assert session.failure.outcome_unknown is False
|
|
assert runtime.close(timeout=2.0) is True
|
|
assert not list(runtime.state_dir.rglob("*.json"))
|
|
assert_other_vm_writable(receipt_runtime())
|
|
|
|
|
|
def test_shutdown_during_next_write_preserves_its_unconfirmed_receipt(
|
|
receipt_runtime, tmp_path):
|
|
runtime = receipt_runtime(rpc_timeout=30.0)
|
|
session = runtime.open("test:///default", "host-a", timeout=60.0)
|
|
assert session.connection.lookupByUUIDString(VM_ID).destroy() == 0
|
|
marker = tmp_path / "second-write-started"
|
|
errors = []
|
|
|
|
def write_until_stopped():
|
|
try:
|
|
session.connection.hang(str(marker), True)
|
|
except BaseException as exc:
|
|
errors.append(exc)
|
|
|
|
caller = threading.Thread(target=write_until_stopped, daemon=True)
|
|
caller.start()
|
|
deadline = time.monotonic() + 3.0
|
|
while not marker.exists() and time.monotonic() < deadline:
|
|
time.sleep(0.01)
|
|
assert marker.exists(), "worker did not receive the next mutation"
|
|
runtime.begin_shutdown()
|
|
assert runtime.close(timeout=2.0) is True
|
|
caller.join(timeout=2.0)
|
|
assert not caller.is_alive()
|
|
assert len(errors) == 1
|
|
assert isinstance(errors[0], ExecutionFailure)
|
|
assert errors[0].outcome_unknown is True
|
|
assert len(list(runtime.state_dir.rglob("*.json"))) == 1
|
|
assert_other_vm_blocked(receipt_runtime())
|
|
|
|
|
|
@pytest.mark.parametrize("failure_kind", ["unlink", "fsync"])
|
|
def test_acknowledgement_durability_failure_keeps_the_write_fenced(
|
|
receipt_runtime, monkeypatch, failure_kind):
|
|
runtime = receipt_runtime()
|
|
session = runtime.open("test:///default", "host-a", timeout=5.0)
|
|
domain = session.connection.lookupByUUIDString(VM_ID)
|
|
receipt_received = threading.Event()
|
|
receive = libvirt_rpc._receive
|
|
sync = runtime._sync
|
|
unlink = Path.unlink
|
|
failures = []
|
|
|
|
def record_receipt(*args):
|
|
response = receive(*args)
|
|
receipt_received.set()
|
|
return response
|
|
|
|
def fail_ack_sync(directory):
|
|
if failure_kind == "fsync" and receipt_received.is_set() and not failures:
|
|
failures.append(directory)
|
|
raise OSError("injected write receipt durability failure")
|
|
return sync(directory)
|
|
|
|
def fail_ack_unlink(path, *args, **kwargs):
|
|
if (failure_kind == "unlink" and receipt_received.is_set()
|
|
and path == session.intent and not failures):
|
|
failures.append(path)
|
|
raise OSError("injected write receipt unlink failure")
|
|
return unlink(path, *args, **kwargs)
|
|
|
|
with monkeypatch.context() as fault:
|
|
fault.setattr(libvirt_rpc, "_receive", record_receipt)
|
|
fault.setattr(runtime, "_sync", fail_ack_sync)
|
|
fault.setattr(Path, "unlink", fail_ack_unlink)
|
|
with pytest.raises(ExecutionFailure) as unsettled:
|
|
domain.destroy()
|
|
assert receipt_received.is_set()
|
|
assert len(failures) == 1
|
|
assert unsettled.value.error_code == "libvirt_execution_uncertain"
|
|
assert unsettled.value.outcome_unknown is True
|
|
session.close()
|
|
assert len(list(runtime.state_dir.rglob("*.json"))) == 1
|
|
assert_other_vm_blocked(runtime)
|
|
|
|
|
|
def test_shutdown_before_received_write_receipt_is_committed_cannot_downgrade_unknown(
|
|
receipt_runtime, monkeypatch):
|
|
runtime = receipt_runtime()
|
|
session = runtime.open("test:///default", "host-a", timeout=5.0)
|
|
domain = session.connection.lookupByUUIDString(VM_ID)
|
|
receive = libvirt_rpc._receive
|
|
|
|
def stop_after_receive(*args):
|
|
response = receive(*args)
|
|
runtime.begin_shutdown()
|
|
return response
|
|
|
|
with monkeypatch.context() as fault:
|
|
fault.setattr(libvirt_rpc, "_receive", stop_after_receive)
|
|
with pytest.raises(ExecutionFailure) as interrupted:
|
|
domain.destroy()
|
|
assert interrupted.value.error_code == "libvirt_execution_uncertain"
|
|
assert interrupted.value.outcome_unknown is True
|
|
assert runtime.close(timeout=2.0) is True
|
|
assert len(list(runtime.state_dir.rglob("*.json"))) == 1
|
|
assert_other_vm_blocked(receipt_runtime())
|
|
|
|
|
|
@pytest.mark.parametrize("response", [
|
|
{"worker_error": True}, {"result": {"unsupported_value": "not decoded"}},
|
|
])
|
|
def test_malformed_write_response_cannot_be_committed_as_a_receipt(
|
|
receipt_runtime, monkeypatch, response):
|
|
runtime = receipt_runtime()
|
|
session = runtime.open("test:///default", "host-a", timeout=5.0)
|
|
domain = session.connection.lookupByUUIDString(VM_ID)
|
|
receive = libvirt_rpc._receive
|
|
|
|
def corrupt_received_result(*args):
|
|
receive(*args)
|
|
return response
|
|
|
|
with monkeypatch.context() as fault:
|
|
fault.setattr(libvirt_rpc, "_receive", corrupt_received_result)
|
|
with pytest.raises(ExecutionFailure) as malformed:
|
|
domain.destroy()
|
|
assert malformed.value.outcome_unknown is True
|
|
session.close()
|
|
assert len(list(runtime.state_dir.rglob("*.json"))) == 1
|
|
assert_other_vm_blocked(runtime)
|
|
|
|
|
|
def test_next_write_waits_for_previous_receipt_to_be_committed(
|
|
receipt_runtime, monkeypatch):
|
|
runtime = receipt_runtime(rpc_timeout=3.0)
|
|
session = runtime.open("test:///default", "host-a", timeout=10.0)
|
|
domain = session.connection.lookupByUUIDString(VM_ID)
|
|
before_write = runtime._before_write
|
|
ack_write = runtime._ack_write
|
|
first_receipt = threading.Event()
|
|
commit_receipt = threading.Event()
|
|
second_started = threading.Event()
|
|
prepared = []
|
|
results = []
|
|
errors = []
|
|
|
|
def record_prepare(*args):
|
|
prepared.append(args[-1])
|
|
return before_write(*args)
|
|
|
|
def pause_first_receipt(current_session, sequence):
|
|
if sequence == 1:
|
|
first_receipt.set()
|
|
assert commit_receipt.wait(timeout=3.0)
|
|
return ack_write(current_session, sequence)
|
|
|
|
def destroy(second=False):
|
|
if second:
|
|
second_started.set()
|
|
try:
|
|
results.append(domain.destroy())
|
|
except BaseException as exc:
|
|
errors.append(exc)
|
|
|
|
monkeypatch.setattr(runtime, "_before_write", record_prepare)
|
|
monkeypatch.setattr(runtime, "_ack_write", pause_first_receipt)
|
|
first = threading.Thread(target=destroy, daemon=True)
|
|
second = threading.Thread(target=destroy, args=(True,), daemon=True)
|
|
try:
|
|
first.start()
|
|
assert first_receipt.wait(timeout=3.0)
|
|
second.start()
|
|
assert second_started.wait(timeout=1.0)
|
|
time.sleep(0.05)
|
|
assert prepared == ["destroy"]
|
|
assert session.pending_write == {"sequence": 1, "method": "destroy"}
|
|
finally:
|
|
commit_receipt.set()
|
|
first.join(timeout=3.0)
|
|
if second.ident is not None:
|
|
second.join(timeout=3.0)
|
|
assert not first.is_alive() and not second.is_alive()
|
|
assert errors == []
|
|
assert results == [0, 0]
|
|
assert prepared == ["destroy", "destroy"]
|
|
assert session.last_write_ack == 2
|
|
assert session.pending_write is None
|
|
session.close()
|
|
assert_other_vm_writable(runtime)
|