fix(engine): acknowledge libvirt writes before connection cleanup
parent
e55b26f1fd
commit
3ca8cfa131
@ -0,0 +1,91 @@
|
||||
"""Native error replies retain the existing same-session recovery contract."""
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from test_libvirt_rpc import VM_ID, runtime_factory # noqa: F401
|
||||
from virsh_panel.libvirt_rpc import ExecutionFailure
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def native_error_runtime(runtime_factory, tmp_path, monkeypatch):
|
||||
# Model the real storage-pool fallback: build can report an existing
|
||||
# directory, after which create and readback must remain callable.
|
||||
name = "libvirt_native_error_receipt_backend"
|
||||
(tmp_path / (name + ".py")).write_text('''
|
||||
import libvirt_rpc_backend as base
|
||||
|
||||
libvirtError = base.libvirtError
|
||||
|
||||
class virStoragePool(base.virStoragePool):
|
||||
def build(self, flags=0):
|
||||
raise libvirtError("directory already exists", (
|
||||
9, 18, "directory already exists", 2, None, None, None, 0, 0))
|
||||
|
||||
def create(self, flags=0):
|
||||
return 0
|
||||
|
||||
class virConnect(base.virConnect):
|
||||
def __init__(self, uri):
|
||||
super().__init__(uri)
|
||||
self.pool = virStoragePool(self)
|
||||
|
||||
def open(uri):
|
||||
return virConnect(uri)
|
||||
''', encoding="utf-8")
|
||||
monkeypatch.syspath_prepend(str(tmp_path))
|
||||
monkeypatch.setenv("PYTHONPATH", str(tmp_path) + os.pathsep
|
||||
+ os.environ.get("PYTHONPATH", ""))
|
||||
return runtime_factory(backend_module=name)
|
||||
|
||||
|
||||
def build_error(session):
|
||||
pool = session.connection.storagePoolLookupByName("images")
|
||||
with pytest.raises(Exception) as raised:
|
||||
pool.build(0)
|
||||
assert not isinstance(raised.value, ExecutionFailure)
|
||||
assert raised.value.get_error_code() == 9
|
||||
assert session.failure is None
|
||||
return pool
|
||||
|
||||
|
||||
def test_native_error_still_allows_existing_pool_start_and_readback(native_error_runtime):
|
||||
runtime = native_error_runtime
|
||||
session = runtime.open("test:///default", "host-a", timeout=5)
|
||||
pool = build_error(session)
|
||||
assert pool.create(0) == 0
|
||||
assert pool.info()[0] == 2
|
||||
# The error receipt is not a success ACK. A later successful write must
|
||||
# preserve its cleanup obligation until this healthy session closes.
|
||||
assert session.intent.is_file()
|
||||
session.close()
|
||||
assert session.failure is None
|
||||
assert not list(runtime.state_dir.rglob("*.json"))
|
||||
|
||||
later = runtime.open("test:///default", "host-a", timeout=5)
|
||||
assert later.connection.lookupByUUIDString(VM_ID).destroy() == 0
|
||||
later.close()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure", ["read", "close"])
|
||||
def test_native_error_obligation_survives_later_success_then_connection_failure(
|
||||
native_error_runtime, failure):
|
||||
runtime = native_error_runtime
|
||||
session = runtime.open("test:///" + failure + "-hangs", "host-a", timeout=5)
|
||||
assert build_error(session).create(0) == 0
|
||||
if failure == "read":
|
||||
with pytest.raises(ExecutionFailure) as raised:
|
||||
session.connection.getHostname()
|
||||
assert raised.value.outcome_unknown is True
|
||||
session.close()
|
||||
assert session.failure.error_code == "libvirt_execution_uncertain"
|
||||
assert session.failure.outcome_unknown is True
|
||||
assert session.intent.is_file()
|
||||
|
||||
observer = runtime.open("test:///default", "host-a", timeout=5)
|
||||
assert observer.connection.getVersion() > 0
|
||||
with pytest.raises(ExecutionFailure) as blocked:
|
||||
observer.connection.lookupByUUIDString(VM_ID).destroy()
|
||||
assert blocked.value.error_code == "libvirt_execution_blocked"
|
||||
observer.close()
|
||||
@ -0,0 +1,493 @@
|
||||
"""A later observation failure cannot erase or invent a libvirt write receipt."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from virsh_panel import libvirt_rpc
|
||||
from virsh_panel.libvirt_rpc import ExecutionFailure, LibvirtRuntime
|
||||
|
||||
|
||||
VM_ID = "00000000-0000-4000-8000-000000000001"
|
||||
OTHER_VM_ID = "00000000-0000-4000-8000-000000000002"
|
||||
FIXTURES = Path(__file__).with_name("fixtures")
|
||||
|
||||
# Keep the fault injection in an importable module: calls below cross the real
|
||||
# multiprocessing/socket boundary rather than replacing the supervisor's RPC.
|
||||
BACKEND = '''
|
||||
import json
|
||||
import libvirt_rpc_backend as base
|
||||
from libvirt_rpc_backend import libvirtError
|
||||
|
||||
|
||||
class virDomain(base.virDomain):
|
||||
def __init__(self, connection, uuid):
|
||||
super().__init__(connection, uuid)
|
||||
self.writes = 0
|
||||
|
||||
def destroy(self):
|
||||
self.writes += 1
|
||||
if self.connection.uri == "test:///second-write-hangs" and self.writes == 2:
|
||||
self.connection.hang()
|
||||
if self.connection.uri.startswith("test:///write-error"):
|
||||
raise libvirtError("write failed after entering native API")
|
||||
return 0
|
||||
|
||||
def create(self):
|
||||
self.writes += 1
|
||||
return 0
|
||||
|
||||
def info(self):
|
||||
if self.connection.uri in {"test:///info-hangs", "test:///write-error-info-hangs"} and self.writes:
|
||||
self.connection.hang()
|
||||
return super().info()
|
||||
|
||||
|
||||
class virStream(base.virStream):
|
||||
def finish(self):
|
||||
if self.connection.uri == "test:///stream-finish-hangs":
|
||||
self.connection.hang()
|
||||
return super().finish()
|
||||
|
||||
def abort(self):
|
||||
if self.connection.uri == "test:///stream-abort-hangs":
|
||||
self.connection.hang()
|
||||
return super().abort()
|
||||
|
||||
|
||||
class virConnect(base.virConnect):
|
||||
def __init__(self, uri):
|
||||
super().__init__(uri)
|
||||
self.domains = {}
|
||||
|
||||
def lookupByUUIDString(self, uuid):
|
||||
if uuid not in self.domains:
|
||||
self.domains[uuid] = virDomain(self, uuid)
|
||||
return self.domains[uuid]
|
||||
|
||||
def newStream(self, flags=0):
|
||||
return virStream(self)
|
||||
|
||||
def close(self):
|
||||
if self.uri == "test:///write-error-close-hangs":
|
||||
self.hang()
|
||||
return super().close()
|
||||
|
||||
|
||||
def open(uri):
|
||||
return virConnect(uri)
|
||||
|
||||
|
||||
def qemuAgentCommand(domain, command, timeout, flags):
|
||||
if json.loads(command).get("execute") == "guest-exec":
|
||||
domain.writes += 1
|
||||
return base.qemuAgentCommand(domain, command, timeout, flags)
|
||||
'''
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def receipt_runtime(tmp_path, monkeypatch):
|
||||
backend_dir = tmp_path / "backend"
|
||||
backend_dir.mkdir()
|
||||
(backend_dir / "rpc_receipt_backend.py").write_text(BACKEND, encoding="utf-8")
|
||||
monkeypatch.syspath_prepend(str(FIXTURES))
|
||||
monkeypatch.syspath_prepend(str(backend_dir))
|
||||
monkeypatch.setenv("PYTHONPATH", os.pathsep.join(filter(None, [
|
||||
str(backend_dir), str(FIXTURES), str(FIXTURES.parents[1]),
|
||||
os.environ.get("PYTHONPATH", ""),
|
||||
])))
|
||||
runtimes = []
|
||||
|
||||
def create(**options):
|
||||
runtime = LibvirtRuntime(
|
||||
tmp_path / "rpc-state", rpc_timeout=options.pop("rpc_timeout", 0.35),
|
||||
connect_timeout=3.0, backend_module="rpc_receipt_backend", **options,
|
||||
)
|
||||
runtimes.append(runtime)
|
||||
return runtime
|
||||
|
||||
yield create
|
||||
for runtime in reversed(runtimes):
|
||||
runtime.close(timeout=2.0)
|
||||
|
||||
|
||||
def assert_other_vm_writable(runtime):
|
||||
other = runtime.open("test:///default", "host-a", timeout=5.0)
|
||||
try:
|
||||
assert other.connection.lookupByUUIDString(OTHER_VM_ID).destroy() == 0
|
||||
finally:
|
||||
other.close()
|
||||
|
||||
|
||||
def assert_other_vm_blocked(runtime):
|
||||
other = runtime.open("test:///default", "host-a", timeout=5.0)
|
||||
try:
|
||||
domain = other.connection.lookupByUUIDString(OTHER_VM_ID)
|
||||
assert domain.info()[0] == 1 # Fences restrict writes, not observations.
|
||||
with pytest.raises(ExecutionFailure) as blocked:
|
||||
domain.destroy()
|
||||
assert blocked.value.error_code == "libvirt_execution_blocked"
|
||||
finally:
|
||||
other.close()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mutation", ["domain", "qga"])
|
||||
def test_closed_handle_cannot_create_a_new_write_intent(receipt_runtime, mutation):
|
||||
runtime = receipt_runtime()
|
||||
session = runtime.open("test:///default", "host-a", timeout=5)
|
||||
domain = session.connection.lookupByUUIDString(VM_ID)
|
||||
domain.destroy()
|
||||
session.close()
|
||||
with pytest.raises(ExecutionFailure) as stopped:
|
||||
if mutation == "domain":
|
||||
domain.destroy()
|
||||
else:
|
||||
session.qemu_agent_command(domain, '{"execute":"guest-exec"}', 1, 0)
|
||||
assert stopped.value.error_code == "libvirt_execution_stopped"
|
||||
assert session.intent is None
|
||||
assert not list(runtime.state_dir.rglob("*.json"))
|
||||
assert_other_vm_writable(runtime)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure_phase", ["info", "close"])
|
||||
@pytest.mark.parametrize("mutation", ["domain", "qga"])
|
||||
def test_acknowledged_write_followed_by_read_or_close_timeout_has_no_host_fence(
|
||||
receipt_runtime, failure_phase, mutation):
|
||||
runtime = receipt_runtime()
|
||||
session = runtime.open(f"test:///{failure_phase}-hangs", "host-a", timeout=5.0)
|
||||
domain = session.connection.lookupByUUIDString(VM_ID)
|
||||
if mutation == "domain":
|
||||
assert domain.destroy() == 0
|
||||
else:
|
||||
result = session.qemu_agent_command(domain, '{"execute":"guest-exec"}', 1, 0)
|
||||
assert json.loads(result)["return"]["execute"] == "guest-exec"
|
||||
|
||||
if failure_phase == "info":
|
||||
with pytest.raises(ExecutionFailure) as timed_out:
|
||||
domain.info()
|
||||
assert timed_out.value.error_code == "libvirt_execution_timeout"
|
||||
assert timed_out.value.outcome_unknown is False
|
||||
session.close()
|
||||
|
||||
failure = session.cleanup_failure if failure_phase == "close" else session.failure
|
||||
assert failure is not None
|
||||
assert failure.outcome_unknown is False
|
||||
if failure_phase == "close":
|
||||
assert session.failure is None # Cleanup cannot replace a command receipt.
|
||||
assert not session.process.is_alive()
|
||||
assert not list(runtime.state_dir.rglob("*.json"))
|
||||
assert_other_vm_writable(runtime)
|
||||
runtime.close(timeout=2.0)
|
||||
assert_other_vm_writable(receipt_runtime())
|
||||
|
||||
|
||||
def test_lost_second_write_receipt_retains_fence_despite_first_success(receipt_runtime):
|
||||
runtime = receipt_runtime()
|
||||
session = runtime.open("test:///second-write-hangs", "host-a", timeout=5.0)
|
||||
domain = session.connection.lookupByUUIDString(VM_ID)
|
||||
assert domain.destroy() == 0
|
||||
with pytest.raises(ExecutionFailure) as lost:
|
||||
domain.destroy()
|
||||
assert lost.value.error_code == "libvirt_execution_uncertain"
|
||||
assert lost.value.outcome_unknown is True
|
||||
session.close()
|
||||
fences = list(runtime.state_dir.rglob("*.json"))
|
||||
assert len(fences) == 1
|
||||
evidence = json.loads(fences[0].read_text())
|
||||
assert evidence["pending_write"] == {"sequence": 2, "method": "destroy"}
|
||||
assert evidence["last_write_ack"] == 1
|
||||
assert_other_vm_blocked(runtime)
|
||||
runtime.close(timeout=2.0)
|
||||
assert_other_vm_blocked(receipt_runtime())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure_phase", ["info", "close"])
|
||||
def test_native_write_error_does_not_prove_a_rollback(receipt_runtime, failure_phase):
|
||||
runtime = receipt_runtime()
|
||||
session = runtime.open(f"test:///write-error-{failure_phase}-hangs", "host-a", timeout=5.0)
|
||||
domain = session.connection.lookupByUUIDString(VM_ID)
|
||||
with pytest.raises(Exception) as native:
|
||||
domain.destroy()
|
||||
assert not isinstance(native.value, ExecutionFailure)
|
||||
assert session.failure is None # Native fallback/reconciliation can continue.
|
||||
evidence = json.loads(session.intent.read_text())
|
||||
assert evidence["write_error_receipt"] == {"sequence": 1, "method": "destroy"}
|
||||
assert evidence["pending_write"] is None
|
||||
assert evidence["last_write_ack"] == 0
|
||||
if failure_phase == "info":
|
||||
with pytest.raises(ExecutionFailure) as lost_observation:
|
||||
domain.info()
|
||||
assert lost_observation.value.outcome_unknown is True
|
||||
session.close()
|
||||
assert session.failure is not None
|
||||
assert session.failure.outcome_unknown is True
|
||||
assert len(list(runtime.state_dir.rglob("*.json"))) == 1
|
||||
assert_other_vm_blocked(runtime)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("read_times_out", [False, True])
|
||||
def test_successful_fallback_does_not_acknowledge_the_earlier_native_error(
|
||||
receipt_runtime, read_times_out):
|
||||
runtime = receipt_runtime()
|
||||
uri = "test:///write-error-info-hangs" if read_times_out else "test:///write-error"
|
||||
session = runtime.open(uri, "host-a", timeout=5.0)
|
||||
domain = session.connection.lookupByUUIDString(VM_ID)
|
||||
with pytest.raises(Exception) as native:
|
||||
domain.destroy()
|
||||
assert not isinstance(native.value, ExecutionFailure)
|
||||
assert domain.create() == 0
|
||||
evidence = json.loads(session.intent.read_text())
|
||||
assert evidence["last_write_ack"] == 2
|
||||
assert evidence["write_error_receipt"] == {"sequence": 1, "method": "destroy"}
|
||||
assert evidence["pending_write"] is None
|
||||
if read_times_out:
|
||||
with pytest.raises(ExecutionFailure) as lost_observation:
|
||||
domain.info()
|
||||
assert lost_observation.value.outcome_unknown is True
|
||||
else:
|
||||
assert domain.info()[0] == 1
|
||||
session.close()
|
||||
if read_times_out:
|
||||
assert_other_vm_blocked(runtime)
|
||||
else:
|
||||
assert session.failure is None
|
||||
assert not list(runtime.state_dir.rglob("*.json"))
|
||||
assert_other_vm_writable(runtime)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("completion", ["finish", "abort"])
|
||||
def test_stream_completion_lost_receipt_remains_a_pending_write(
|
||||
receipt_runtime, completion):
|
||||
runtime = receipt_runtime()
|
||||
session = runtime.open(f"test:///stream-{completion}-hangs", "host-a", timeout=5.0)
|
||||
connection = session.connection
|
||||
volume = connection.storagePoolLookupByName("images").storageVolLookupByName("disk.qcow2")
|
||||
stream = connection.newStream(0)
|
||||
assert volume.upload(stream, 0, 3, 0) == 0
|
||||
assert stream.send(b"abc") == 3
|
||||
with pytest.raises(ExecutionFailure) as incomplete:
|
||||
getattr(stream, completion)()
|
||||
assert incomplete.value.outcome_unknown is True
|
||||
session.close()
|
||||
fences = list(runtime.state_dir.rglob("*.json"))
|
||||
assert len(fences) == 1
|
||||
evidence = json.loads(fences[0].read_text())
|
||||
assert evidence["last_write_ack"] == 2
|
||||
assert evidence["pending_write"] == {"sequence": 3, "method": completion}
|
||||
assert_other_vm_blocked(runtime)
|
||||
|
||||
|
||||
def test_read_connection_cannot_clear_a_legacy_unconfirmed_write(receipt_runtime):
|
||||
runtime = receipt_runtime()
|
||||
directory = runtime._directory("host-a")
|
||||
path = directory / "old-worker.json"
|
||||
original = json.dumps({
|
||||
"version": 1, "scope": "host-a", "session": "old-worker",
|
||||
"method": "destroy", "started_at": 1,
|
||||
}).encode()
|
||||
path.write_bytes(original)
|
||||
observer = runtime.open("test:///default", "host-a", timeout=5.0)
|
||||
assert observer.connection.lookupByUUIDString(VM_ID).info()[0] == 1
|
||||
observer.close()
|
||||
assert_other_vm_blocked(runtime)
|
||||
assert path.read_bytes() == original
|
||||
|
||||
|
||||
def test_shutdown_after_acknowledged_write_does_not_quarantine_host(receipt_runtime):
|
||||
runtime = receipt_runtime()
|
||||
session = runtime.open("test:///default", "host-a", timeout=5.0)
|
||||
assert session.connection.lookupByUUIDString(VM_ID).destroy() == 0
|
||||
runtime.begin_shutdown()
|
||||
assert session.failure is not None
|
||||
assert session.failure.outcome_unknown is False
|
||||
assert runtime.close(timeout=2.0) is True
|
||||
assert not list(runtime.state_dir.rglob("*.json"))
|
||||
assert_other_vm_writable(receipt_runtime())
|
||||
|
||||
|
||||
def test_shutdown_during_next_write_preserves_its_unconfirmed_receipt(
|
||||
receipt_runtime, tmp_path):
|
||||
runtime = receipt_runtime(rpc_timeout=30.0)
|
||||
session = runtime.open("test:///default", "host-a", timeout=60.0)
|
||||
assert session.connection.lookupByUUIDString(VM_ID).destroy() == 0
|
||||
marker = tmp_path / "second-write-started"
|
||||
errors = []
|
||||
|
||||
def write_until_stopped():
|
||||
try:
|
||||
session.connection.hang(str(marker), True)
|
||||
except BaseException as exc:
|
||||
errors.append(exc)
|
||||
|
||||
caller = threading.Thread(target=write_until_stopped, daemon=True)
|
||||
caller.start()
|
||||
deadline = time.monotonic() + 3.0
|
||||
while not marker.exists() and time.monotonic() < deadline:
|
||||
time.sleep(0.01)
|
||||
assert marker.exists(), "worker did not receive the next mutation"
|
||||
runtime.begin_shutdown()
|
||||
assert runtime.close(timeout=2.0) is True
|
||||
caller.join(timeout=2.0)
|
||||
assert not caller.is_alive()
|
||||
assert len(errors) == 1
|
||||
assert isinstance(errors[0], ExecutionFailure)
|
||||
assert errors[0].outcome_unknown is True
|
||||
assert len(list(runtime.state_dir.rglob("*.json"))) == 1
|
||||
assert_other_vm_blocked(receipt_runtime())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure_kind", ["unlink", "fsync"])
|
||||
def test_acknowledgement_durability_failure_keeps_the_write_fenced(
|
||||
receipt_runtime, monkeypatch, failure_kind):
|
||||
runtime = receipt_runtime()
|
||||
session = runtime.open("test:///default", "host-a", timeout=5.0)
|
||||
domain = session.connection.lookupByUUIDString(VM_ID)
|
||||
receipt_received = threading.Event()
|
||||
receive = libvirt_rpc._receive
|
||||
sync = runtime._sync
|
||||
unlink = Path.unlink
|
||||
failures = []
|
||||
|
||||
def record_receipt(*args):
|
||||
response = receive(*args)
|
||||
receipt_received.set()
|
||||
return response
|
||||
|
||||
def fail_ack_sync(directory):
|
||||
if failure_kind == "fsync" and receipt_received.is_set() and not failures:
|
||||
failures.append(directory)
|
||||
raise OSError("injected write receipt durability failure")
|
||||
return sync(directory)
|
||||
|
||||
def fail_ack_unlink(path, *args, **kwargs):
|
||||
if (failure_kind == "unlink" and receipt_received.is_set()
|
||||
and path == session.intent and not failures):
|
||||
failures.append(path)
|
||||
raise OSError("injected write receipt unlink failure")
|
||||
return unlink(path, *args, **kwargs)
|
||||
|
||||
with monkeypatch.context() as fault:
|
||||
fault.setattr(libvirt_rpc, "_receive", record_receipt)
|
||||
fault.setattr(runtime, "_sync", fail_ack_sync)
|
||||
fault.setattr(Path, "unlink", fail_ack_unlink)
|
||||
with pytest.raises(ExecutionFailure) as unsettled:
|
||||
domain.destroy()
|
||||
assert receipt_received.is_set()
|
||||
assert len(failures) == 1
|
||||
assert unsettled.value.error_code == "libvirt_execution_uncertain"
|
||||
assert unsettled.value.outcome_unknown is True
|
||||
session.close()
|
||||
assert len(list(runtime.state_dir.rglob("*.json"))) == 1
|
||||
assert_other_vm_blocked(runtime)
|
||||
|
||||
|
||||
def test_shutdown_before_received_write_receipt_is_committed_cannot_downgrade_unknown(
|
||||
receipt_runtime, monkeypatch):
|
||||
runtime = receipt_runtime()
|
||||
session = runtime.open("test:///default", "host-a", timeout=5.0)
|
||||
domain = session.connection.lookupByUUIDString(VM_ID)
|
||||
receive = libvirt_rpc._receive
|
||||
|
||||
def stop_after_receive(*args):
|
||||
response = receive(*args)
|
||||
runtime.begin_shutdown()
|
||||
return response
|
||||
|
||||
with monkeypatch.context() as fault:
|
||||
fault.setattr(libvirt_rpc, "_receive", stop_after_receive)
|
||||
with pytest.raises(ExecutionFailure) as interrupted:
|
||||
domain.destroy()
|
||||
assert interrupted.value.error_code == "libvirt_execution_uncertain"
|
||||
assert interrupted.value.outcome_unknown is True
|
||||
assert runtime.close(timeout=2.0) is True
|
||||
assert len(list(runtime.state_dir.rglob("*.json"))) == 1
|
||||
assert_other_vm_blocked(receipt_runtime())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("response", [
|
||||
{"worker_error": True}, {"result": {"unsupported_value": "not decoded"}},
|
||||
])
|
||||
def test_malformed_write_response_cannot_be_committed_as_a_receipt(
|
||||
receipt_runtime, monkeypatch, response):
|
||||
runtime = receipt_runtime()
|
||||
session = runtime.open("test:///default", "host-a", timeout=5.0)
|
||||
domain = session.connection.lookupByUUIDString(VM_ID)
|
||||
receive = libvirt_rpc._receive
|
||||
|
||||
def corrupt_received_result(*args):
|
||||
receive(*args)
|
||||
return response
|
||||
|
||||
with monkeypatch.context() as fault:
|
||||
fault.setattr(libvirt_rpc, "_receive", corrupt_received_result)
|
||||
with pytest.raises(ExecutionFailure) as malformed:
|
||||
domain.destroy()
|
||||
assert malformed.value.outcome_unknown is True
|
||||
session.close()
|
||||
assert len(list(runtime.state_dir.rglob("*.json"))) == 1
|
||||
assert_other_vm_blocked(runtime)
|
||||
|
||||
|
||||
def test_next_write_waits_for_previous_receipt_to_be_committed(
|
||||
receipt_runtime, monkeypatch):
|
||||
runtime = receipt_runtime(rpc_timeout=3.0)
|
||||
session = runtime.open("test:///default", "host-a", timeout=10.0)
|
||||
domain = session.connection.lookupByUUIDString(VM_ID)
|
||||
before_write = runtime._before_write
|
||||
ack_write = runtime._ack_write
|
||||
first_receipt = threading.Event()
|
||||
commit_receipt = threading.Event()
|
||||
second_started = threading.Event()
|
||||
prepared = []
|
||||
results = []
|
||||
errors = []
|
||||
|
||||
def record_prepare(*args):
|
||||
prepared.append(args[-1])
|
||||
return before_write(*args)
|
||||
|
||||
def pause_first_receipt(current_session, sequence):
|
||||
if sequence == 1:
|
||||
first_receipt.set()
|
||||
assert commit_receipt.wait(timeout=3.0)
|
||||
return ack_write(current_session, sequence)
|
||||
|
||||
def destroy(second=False):
|
||||
if second:
|
||||
second_started.set()
|
||||
try:
|
||||
results.append(domain.destroy())
|
||||
except BaseException as exc:
|
||||
errors.append(exc)
|
||||
|
||||
monkeypatch.setattr(runtime, "_before_write", record_prepare)
|
||||
monkeypatch.setattr(runtime, "_ack_write", pause_first_receipt)
|
||||
first = threading.Thread(target=destroy, daemon=True)
|
||||
second = threading.Thread(target=destroy, args=(True,), daemon=True)
|
||||
try:
|
||||
first.start()
|
||||
assert first_receipt.wait(timeout=3.0)
|
||||
second.start()
|
||||
assert second_started.wait(timeout=1.0)
|
||||
time.sleep(0.05)
|
||||
assert prepared == ["destroy"]
|
||||
assert session.pending_write == {"sequence": 1, "method": "destroy"}
|
||||
finally:
|
||||
commit_receipt.set()
|
||||
first.join(timeout=3.0)
|
||||
if second.ident is not None:
|
||||
second.join(timeout=3.0)
|
||||
assert not first.is_alive() and not second.is_alive()
|
||||
assert errors == []
|
||||
assert results == [0, 0]
|
||||
assert prepared == ["destroy", "destroy"]
|
||||
assert session.last_write_ack == 2
|
||||
assert session.pending_write is None
|
||||
session.close()
|
||||
assert_other_vm_writable(runtime)
|
||||
Loading…
Reference in New Issue