You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/engine/tests/test_libvirt_rpc.py

404 lines
16 KiB
Python

"""Real subprocess tests for the libvirt RPC deadline and isolation boundary."""
from __future__ import annotations
import json
import os
import signal
import threading
import time
from pathlib import Path
import pytest
from virsh_panel.libvirt_rpc import ExecutionFailure, LibvirtRuntime
FIXTURES = Path(__file__).with_name("fixtures")
VM_ID = "00000000-0000-4000-8000-000000000001"
@pytest.mark.parametrize("option,value", [
("rpc_timeout", float("inf")), ("connect_timeout", float("nan")),
("queue_timeout", 0), ("max_connections", -1), ("max_per_host", 1.5),
])
def test_execution_limits_cannot_disable_deadlines_or_capacity(tmp_path, option, value):
with pytest.raises(ValueError):
LibvirtRuntime(tmp_path / "rpc-state", **{option: value})
def test_new_fence_directories_persist_their_entries_in_each_parent(tmp_path, monkeypatch):
synced = []
original = LibvirtRuntime._sync
def sync(directory):
synced.append(directory)
original(directory)
monkeypatch.setattr(LibvirtRuntime, "_sync", staticmethod(sync))
root = tmp_path / "new-data" / "rpc-state"
runtime = LibvirtRuntime(root)
scope = runtime._directory("host-a")
assert scope.is_dir()
assert synced == [tmp_path, root.parent, root]
runtime._directory("host-a")
assert len(synced) == 3 # repeated stream writes do not fsync unchanged parents
@pytest.fixture
def runtime_factory(tmp_path, monkeypatch):
# The supervisor receives the backend explicitly. This path merely makes
# that test module importable in a freshly spawned Python interpreter.
monkeypatch.syspath_prepend(str(FIXTURES))
monkeypatch.setenv(
"PYTHONPATH", os.pathsep.join(filter(None, [
str(FIXTURES), str(FIXTURES.parents[1]),
os.environ.get("PYTHONPATH", ""),
])),
)
runtimes = []
def create(**overrides):
options = {
"rpc_timeout": 0.35,
"connect_timeout": 3.0,
"queue_timeout": 0.15,
"backend_module": "libvirt_rpc_backend",
}
options.update(overrides)
runtime = LibvirtRuntime(tmp_path / "rpc-state", **options)
runtimes.append(runtime)
return runtime
yield create
for runtime in reversed(runtimes):
runtime.close(timeout=2.0)
def test_objects_nested_stats_and_values_cross_real_worker(runtime_factory):
runtime = runtime_factory()
session = runtime.open("test:///default", "host-a", timeout=5.0)
conn = session.connection
assert conn.process_id() == session.pid
assert session.pid != os.getpid()
assert conn.getHostname() == "rpc-test-host"
domain = conn.lookupByUUIDString(VM_ID)
assert domain.info() == [1, 2097152, 1048576, 2, 12345]
assert domain.state() == (1, 0)
stats = conn.getAllDomainStats(0, 0)
assert isinstance(stats[0], tuple)
assert stats[0][0].UUIDString() == VM_ID
assert stats[0][1] == {"cpu.time": 12345, "nested": [None, True, 1.25]}
assert conn.domainListGetStats([domain], 0, 0)[0][0].UUIDString() == VM_ID
snapshot = domain.snapshotLookupByName("before-upgrade", 0)
assert snapshot.getName() == "before-upgrade"
assert snapshot.getDomain().UUIDString() == VM_ID
value = {"tuple": (None, False, 2.5), "bytes": b"\x00\xffbinary", "list": [1]}
assert conn.echo(value) == value
session.close()
assert not session.process.is_alive()
def test_stream_handles_and_binary_short_writes_and_reads(runtime_factory):
runtime = runtime_factory()
session = runtime.open("test:///default", "host-a", timeout=5.0)
conn = session.connection
volume = conn.storagePoolLookupByName("images").storageVolLookupByName("disk.qcow2")
data = b"\x00\xff\x80volume-content\x00" * 3
stream = conn.newStream(0)
volume.upload(stream, 0, len(data), 0)
remaining = data
while remaining:
sent = stream.send(remaining)
assert 0 < sent <= len(remaining)
remaining = remaining[sent:]
stream.finish()
reader = conn.newStream(0)
volume.download(reader, 0, len(data), 0)
output = bytearray()
while chunk := reader.recv(65536):
output.extend(chunk)
reader.finish()
assert bytes(output) == data
assert volume.info()[2] == len(data)
session.close()
def test_native_error_code_and_full_fields_survive_rpc(runtime_factory):
runtime = runtime_factory()
session = runtime.open("test:///default", "host-a", timeout=5.0)
with pytest.raises(Exception) as raised:
session.connection.lookupByUUIDString("absent")
error = raised.value
assert not isinstance(error, ExecutionFailure)
assert error.get_error_code() == 42
assert tuple(error.err) == (
42, 10, "domain is absent", 2, "domain", "detail", None, 17, 23,
)
# A normal libvirt error must not poison a healthy connection.
assert session.connection.getHostname() == "rpc-test-host"
session.close()
def test_qemu_agent_command_uses_same_remote_domain(runtime_factory):
runtime = runtime_factory()
session = runtime.open("test:///default", "host-a", timeout=5.0)
domain = session.connection.lookupByUUIDString(VM_ID)
reply = session.qemu_agent_command(domain, '{"execute":"guest-ping"}', 3, 0)
assert json.loads(reply) == {"return": {
"uuid": VM_ID, "execute": "guest-ping", "timeout": 3, "flags": 0,
}}
session.close()
def test_read_timeout_reaps_child_and_sticky_session_cannot_resume(runtime_factory):
runtime = runtime_factory()
session = runtime.open("test:///read-hangs", "host-a", timeout=5.0)
started = time.monotonic()
with pytest.raises(ExecutionFailure) as raised:
session.connection.getHostname()
assert time.monotonic() - started < 2.0
assert raised.value.error_code == "libvirt_execution_timeout"
assert raised.value.outcome_unknown is False
assert session.failure is not None
with pytest.raises(ExecutionFailure):
session.connection.getVersion()
session.close()
assert not session.process.is_alive()
# A failed read carries no unconfirmed write and cannot quarantine a host.
healthy = runtime.open("test:///default", "host-a", timeout=5.0)
assert healthy.connection.lookupByUUIDString(VM_ID).destroy() == 0
healthy.close()
def test_absolute_session_deadline_is_not_extended_by_successful_calls(runtime_factory):
runtime = runtime_factory(rpc_timeout=3.0)
session = runtime.open("test:///default", "host-a", timeout=0.8)
assert session.connection.getHostname() == "rpc-test-host"
time.sleep(0.9)
started = time.monotonic()
with pytest.raises(ExecutionFailure):
session.connection.getHostname()
assert time.monotonic() - started < 1.0
session.close()
def test_open_timeout_is_bounded_and_releases_capacity(runtime_factory):
runtime = runtime_factory(connect_timeout=0.4, max_connections=1)
started = time.monotonic()
with pytest.raises(ExecutionFailure) as raised:
runtime.open("test:///open-hangs", "host-a", timeout=3.0)
assert time.monotonic() - started < 2.0
assert raised.value.outcome_unknown is False
healthy = runtime.open("test:///default", "host-b", timeout=5.0)
assert healthy.connection.getHostname() == "rpc-test-host"
healthy.close()
def test_native_open_error_releases_capacity_without_an_unknown_write(runtime_factory):
runtime = runtime_factory(max_connections=1)
with pytest.raises(Exception) as raised:
runtime.open("test:///open-error", "host-a", timeout=5.0)
assert not isinstance(raised.value, ExecutionFailure)
assert raised.value.get_error_code() == 9
assert not list(runtime.state_dir.rglob("*.json"))
healthy = runtime.open("test:///default", "host-a", timeout=5.0)
assert healthy.connection.lookupByUUIDString(VM_ID).destroy() == 0
healthy.close()
def test_child_crash_is_uncertain_and_cannot_be_reused(runtime_factory):
runtime = runtime_factory()
session = runtime.open("test:///default", "host-a", timeout=5.0)
with pytest.raises(ExecutionFailure) as raised:
session.connection.crash()
assert raised.value.error_code == "libvirt_execution_uncertain"
assert raised.value.outcome_unknown is True
with pytest.raises(ExecutionFailure):
session.connection.getHostname()
session.close()
assert not session.process.is_alive()
def test_unknown_write_barrier_survives_restart_but_allows_reads_and_other_hosts(runtime_factory):
runtime = runtime_factory()
session = runtime.open("test:///write-hangs", "host-a", timeout=5.0)
domain = session.connection.lookupByUUIDString(VM_ID)
with pytest.raises(ExecutionFailure) as raised:
domain.destroy()
assert raised.value.outcome_unknown is True
session.close()
runtime.close(timeout=2.0)
restarted = runtime_factory()
observation = restarted.open("test:///default", "host-a", timeout=5.0)
domain = observation.connection.lookupByUUIDString(VM_ID)
assert domain.state() == (1, 0)
with pytest.raises(ExecutionFailure) as blocked:
domain.destroy()
assert blocked.value.error_code == "libvirt_execution_blocked"
observation.close()
# Closing a read session must not silently erase the interrupted write.
observation = restarted.open("test:///default", "host-a", timeout=5.0)
with pytest.raises(ExecutionFailure):
observation.connection.lookupByUUIDString(VM_ID).destroy()
observation.close()
other = restarted.open("test:///default", "host-b", timeout=5.0)
assert other.connection.lookupByUUIDString(VM_ID).destroy() == 0
other.close()
def test_successful_write_and_close_clear_only_its_own_barrier(runtime_factory):
runtime = runtime_factory()
for _ in range(2):
session = runtime.open("test:///default", "host-a", timeout=5.0)
assert session.connection.lookupByUUIDString(VM_ID).destroy() == 0
session.close()
assert session.failure is None
def test_host_limit_leaves_capacity_for_another_host(runtime_factory):
runtime = runtime_factory(max_connections=2, max_per_host=1)
first = runtime.open("test:///default", "host-a", timeout=5.0)
started = time.monotonic()
with pytest.raises(ExecutionFailure):
runtime.open("test:///default", "host-a", timeout=5.0)
assert time.monotonic() - started < 1.0
other = runtime.open("test:///default", "host-b", timeout=5.0)
assert other.connection.getHostname() == "rpc-test-host"
first.close()
other.close()
def test_global_limit_is_bounded_and_capacity_returns_after_close(runtime_factory):
runtime = runtime_factory(max_connections=1, max_per_host=2)
first = runtime.open("test:///default", "host-a", timeout=5.0)
started = time.monotonic()
with pytest.raises(ExecutionFailure) as raised:
runtime.open("test:///default", "host-b", timeout=5.0)
assert raised.value.error_code == "libvirt_execution_timeout"
assert raised.value.outcome_unknown is False
assert time.monotonic() - started < 1.0
first.close()
second = runtime.open("test:///default", "host-b", timeout=5.0)
assert second.connection.getHostname() == "rpc-test-host"
second.close()
def test_shutdown_wakes_capacity_waiter_without_waiting_for_queue_deadline(runtime_factory):
runtime = runtime_factory(max_connections=1, queue_timeout=30.0)
first = runtime.open("test:///default", "host-a", timeout=5.0)
entered = threading.Event()
errors = []
def open_queued_session():
entered.set()
try:
runtime.open("test:///default", "host-b", timeout=5.0)
except BaseException as exc:
errors.append(exc)
waiter = threading.Thread(target=open_queued_session, daemon=True)
waiter.start()
assert entered.wait(timeout=1.0)
started = time.monotonic()
runtime.begin_shutdown()
waiter.join(timeout=1.0)
assert not waiter.is_alive()
assert len(errors) == 1
assert isinstance(errors[0], ExecutionFailure)
assert errors[0].error_code == "libvirt_execution_stopped"
assert runtime.close(timeout=1.0) is True
assert time.monotonic() - started < 2.0
assert not first.process.is_alive()
def test_shutdown_rejects_new_sessions_and_reaps_existing_workers(runtime_factory):
runtime = runtime_factory()
session = runtime.open("test:///default", "host-a", timeout=5.0)
started = time.monotonic()
runtime.begin_shutdown()
with pytest.raises(ExecutionFailure) as stopped:
runtime.open("test:///default", "host-b", timeout=5.0)
assert stopped.value.error_code == "libvirt_execution_stopped"
assert runtime.close(timeout=1.0) is True
assert time.monotonic() - started < 2.0
assert not session.process.is_alive()
with pytest.raises(ExecutionFailure):
session.connection.getHostname()
def test_connection_close_cannot_wait_forever(runtime_factory):
runtime = runtime_factory()
session = runtime.open("test:///close-hangs", "host-a", timeout=5.0)
started = time.monotonic()
session.close()
assert time.monotonic() - started < 3.0
assert session.failure is not None
assert not session.process.is_alive()
def test_cross_session_handles_are_rejected_before_remote_dispatch(runtime_factory):
runtime = runtime_factory()
first = runtime.open("test:///default", "host-a", timeout=5.0)
second = runtime.open("test:///default", "host-b", timeout=5.0)
foreign_domain = second.connection.lookupByUUIDString(VM_ID)
with pytest.raises((ValueError, ExecutionFailure)):
first.connection.domainListGetStats([foreign_domain], 0, 0)
assert second.connection.getHostname() == "rpc-test-host"
first.close()
second.close()
def test_shutdown_reaps_worker_ignoring_sigterm_while_rpc_is_running(runtime_factory, tmp_path):
runtime = runtime_factory(rpc_timeout=30.0)
session = runtime.open("test:///default", "host-a", timeout=60.0)
marker = tmp_path / "worker-entered-hang"
errors = []
def invoke_blocking_write():
try:
session.connection.hang(str(marker), True)
except BaseException as exc:
errors.append(exc)
caller = threading.Thread(target=invoke_blocking_write, daemon=True)
caller.start()
deadline = time.monotonic() + 3.0
while not marker.exists() and time.monotonic() < deadline:
time.sleep(0.01)
assert marker.exists(), "child did not reach the injected blocking call"
assert int(marker.read_text()) == session.pid
started = time.monotonic()
runtime.begin_shutdown()
assert runtime.close(timeout=1.0) is True
caller.join(timeout=1.0)
assert time.monotonic() - started < 2.0
assert not caller.is_alive()
assert not session.process.is_alive()
assert len(errors) == 1
assert isinstance(errors[0], ExecutionFailure)
assert errors[0].outcome_unknown is True
def test_worker_exit_does_not_leave_its_stubborn_descendant_running(runtime_factory, tmp_path):
runtime = runtime_factory(rpc_timeout=3.0)
session = runtime.open("test:///default", "host-a", timeout=10.0)
heartbeat = tmp_path / "descendant-heartbeat"
descendant_pid = session.connection.start_stubborn_child(str(heartbeat))
try:
session.close()
assert not session.process.is_alive()
# A killed but not yet reaped descendant can briefly remain a zombie;
# compare liveness evidence instead of mistaking kill(pid, 0) for work.
time.sleep(0.1)
stopped_at = heartbeat.stat().st_mtime_ns
time.sleep(0.15)
assert heartbeat.stat().st_mtime_ns == stopped_at
finally:
# Keep this regression safe even against the broken implementation.
try:
os.kill(descendant_pid, signal.SIGKILL)
except ProcessLookupError:
pass