You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
434 lines
17 KiB
Python
434 lines
17 KiB
Python
"""Real subprocess tests for the libvirt RPC deadline and isolation boundary."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import signal
|
|
import threading
|
|
import time
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from virsh_panel.libvirt_rpc import ExecutionFailure, LibvirtRuntime
|
|
|
|
|
|
FIXTURES = Path(__file__).with_name("fixtures")
|
|
VM_ID = "00000000-0000-4000-8000-000000000001"
|
|
|
|
|
|
@pytest.mark.parametrize("option,value", [
|
|
("rpc_timeout", float("inf")), ("connect_timeout", float("nan")),
|
|
("queue_timeout", 0), ("max_connections", -1), ("max_per_host", 1.5),
|
|
])
|
|
def test_execution_limits_cannot_disable_deadlines_or_capacity(tmp_path, option, value):
|
|
with pytest.raises(ValueError):
|
|
LibvirtRuntime(tmp_path / "rpc-state", **{option: value})
|
|
|
|
|
|
def test_new_fence_directories_persist_their_entries_in_each_parent(tmp_path, monkeypatch):
|
|
synced = []
|
|
original = LibvirtRuntime._sync
|
|
|
|
def sync(directory):
|
|
synced.append(directory)
|
|
original(directory)
|
|
|
|
monkeypatch.setattr(LibvirtRuntime, "_sync", staticmethod(sync))
|
|
root = tmp_path / "new-data" / "rpc-state"
|
|
runtime = LibvirtRuntime(root)
|
|
scope = runtime._directory("host-a")
|
|
assert scope.is_dir()
|
|
assert synced == [tmp_path, root.parent, root]
|
|
runtime._directory("host-a")
|
|
assert len(synced) == 3 # repeated stream writes do not fsync unchanged parents
|
|
|
|
|
|
@pytest.fixture
|
|
def runtime_factory(tmp_path, monkeypatch):
|
|
# The supervisor receives the backend explicitly. This path merely makes
|
|
# that test module importable in a freshly spawned Python interpreter.
|
|
monkeypatch.syspath_prepend(str(FIXTURES))
|
|
monkeypatch.setenv(
|
|
"PYTHONPATH", os.pathsep.join(filter(None, [
|
|
str(FIXTURES), str(FIXTURES.parents[1]),
|
|
os.environ.get("PYTHONPATH", ""),
|
|
])),
|
|
)
|
|
runtimes = []
|
|
|
|
def create(**overrides):
|
|
options = {
|
|
"rpc_timeout": 0.35,
|
|
"connect_timeout": 3.0,
|
|
"queue_timeout": 0.15,
|
|
"backend_module": "libvirt_rpc_backend",
|
|
}
|
|
options.update(overrides)
|
|
runtime = LibvirtRuntime(tmp_path / "rpc-state", **options)
|
|
runtimes.append(runtime)
|
|
return runtime
|
|
|
|
yield create
|
|
for runtime in reversed(runtimes):
|
|
runtime.close(timeout=2.0)
|
|
|
|
|
|
def test_objects_nested_stats_and_values_cross_real_worker(runtime_factory):
|
|
runtime = runtime_factory()
|
|
session = runtime.open("test:///default", "host-a", timeout=5.0)
|
|
conn = session.connection
|
|
assert conn.process_id() == session.pid
|
|
assert session.pid != os.getpid()
|
|
assert conn.getHostname() == "rpc-test-host"
|
|
domain = conn.lookupByUUIDString(VM_ID)
|
|
assert domain.info() == [1, 2097152, 1048576, 2, 12345]
|
|
assert domain.state() == (1, 0)
|
|
stats = conn.getAllDomainStats(0, 0)
|
|
assert isinstance(stats[0], tuple)
|
|
assert stats[0][0].UUIDString() == VM_ID
|
|
assert stats[0][1] == {"cpu.time": 12345, "nested": [None, True, 1.25]}
|
|
assert conn.domainListGetStats([domain], 0, 0)[0][0].UUIDString() == VM_ID
|
|
snapshot = domain.snapshotLookupByName("before-upgrade", 0)
|
|
assert snapshot.getName() == "before-upgrade"
|
|
assert snapshot.getDomain().UUIDString() == VM_ID
|
|
value = {"tuple": (None, False, 2.5), "bytes": b"\x00\xffbinary", "list": [1]}
|
|
assert conn.echo(value) == value
|
|
session.close()
|
|
assert not session.process.is_alive()
|
|
|
|
|
|
def test_stream_handles_and_binary_short_writes_and_reads(runtime_factory):
|
|
runtime = runtime_factory()
|
|
session = runtime.open("test:///default", "host-a", timeout=5.0)
|
|
conn = session.connection
|
|
volume = conn.storagePoolLookupByName("images").storageVolLookupByName("disk.qcow2")
|
|
data = b"\x00\xff\x80volume-content\x00" * 3
|
|
stream = conn.newStream(0)
|
|
volume.upload(stream, 0, len(data), 0)
|
|
remaining = data
|
|
while remaining:
|
|
sent = stream.send(remaining)
|
|
assert 0 < sent <= len(remaining)
|
|
remaining = remaining[sent:]
|
|
stream.finish()
|
|
reader = conn.newStream(0)
|
|
volume.download(reader, 0, len(data), 0)
|
|
output = bytearray()
|
|
while chunk := reader.recv(65536):
|
|
output.extend(chunk)
|
|
reader.finish()
|
|
assert bytes(output) == data
|
|
assert volume.info()[2] == len(data)
|
|
session.close()
|
|
|
|
|
|
def test_native_error_code_and_full_fields_survive_rpc(runtime_factory):
|
|
runtime = runtime_factory()
|
|
session = runtime.open("test:///default", "host-a", timeout=5.0)
|
|
with pytest.raises(Exception) as raised:
|
|
session.connection.lookupByUUIDString("absent")
|
|
error = raised.value
|
|
assert not isinstance(error, ExecutionFailure)
|
|
assert error.get_error_code() == 42
|
|
assert tuple(error.err) == (
|
|
42, 10, "domain is absent", 2, "domain", "detail", None, 17, 23,
|
|
)
|
|
# A normal libvirt error must not poison a healthy connection.
|
|
assert session.connection.getHostname() == "rpc-test-host"
|
|
session.close()
|
|
|
|
|
|
def test_qemu_agent_command_uses_same_remote_domain(runtime_factory):
|
|
runtime = runtime_factory()
|
|
session = runtime.open("test:///default", "host-a", timeout=5.0)
|
|
domain = session.connection.lookupByUUIDString(VM_ID)
|
|
reply = session.qemu_agent_command(domain, '{"execute":"guest-ping"}', 3, 0)
|
|
assert json.loads(reply) == {"return": {
|
|
"uuid": VM_ID, "execute": "guest-ping", "timeout": 3, "flags": 0,
|
|
}}
|
|
session.close()
|
|
|
|
|
|
def test_read_timeout_reaps_child_and_sticky_session_cannot_resume(runtime_factory):
|
|
runtime = runtime_factory()
|
|
session = runtime.open("test:///read-hangs", "host-a", timeout=5.0)
|
|
started = time.monotonic()
|
|
with pytest.raises(ExecutionFailure) as raised:
|
|
session.connection.getHostname()
|
|
assert time.monotonic() - started < 2.0
|
|
assert raised.value.error_code == "libvirt_execution_timeout"
|
|
assert raised.value.outcome_unknown is False
|
|
assert session.failure is not None
|
|
with pytest.raises(ExecutionFailure):
|
|
session.connection.getVersion()
|
|
session.close()
|
|
assert not session.process.is_alive()
|
|
# A failed read carries no unconfirmed write and cannot quarantine a host.
|
|
healthy = runtime.open("test:///default", "host-a", timeout=5.0)
|
|
assert healthy.connection.lookupByUUIDString(VM_ID).destroy() == 0
|
|
healthy.close()
|
|
|
|
|
|
def test_absolute_session_deadline_is_not_extended_by_successful_calls(runtime_factory):
|
|
runtime = runtime_factory(rpc_timeout=3.0)
|
|
session = runtime.open("test:///default", "host-a", timeout=0.8)
|
|
assert session.connection.getHostname() == "rpc-test-host"
|
|
time.sleep(0.9)
|
|
started = time.monotonic()
|
|
with pytest.raises(ExecutionFailure):
|
|
session.connection.getHostname()
|
|
assert time.monotonic() - started < 1.0
|
|
session.close()
|
|
|
|
|
|
def test_open_timeout_is_bounded_and_releases_capacity(runtime_factory):
|
|
runtime = runtime_factory(connect_timeout=0.4, max_connections=1)
|
|
started = time.monotonic()
|
|
with pytest.raises(ExecutionFailure) as raised:
|
|
runtime.open("test:///open-hangs", "host-a", timeout=3.0)
|
|
assert time.monotonic() - started < 2.0
|
|
assert raised.value.outcome_unknown is False
|
|
healthy = runtime.open("test:///default", "host-b", timeout=5.0)
|
|
assert healthy.connection.getHostname() == "rpc-test-host"
|
|
healthy.close()
|
|
|
|
|
|
def test_native_open_error_releases_capacity_without_an_unknown_write(runtime_factory):
|
|
runtime = runtime_factory(max_connections=1)
|
|
with pytest.raises(Exception) as raised:
|
|
runtime.open("test:///open-error", "host-a", timeout=5.0)
|
|
assert not isinstance(raised.value, ExecutionFailure)
|
|
assert raised.value.get_error_code() == 9
|
|
assert not list(runtime.state_dir.rglob("*.json"))
|
|
healthy = runtime.open("test:///default", "host-a", timeout=5.0)
|
|
assert healthy.connection.lookupByUUIDString(VM_ID).destroy() == 0
|
|
healthy.close()
|
|
|
|
|
|
def test_child_crash_is_uncertain_and_cannot_be_reused(runtime_factory):
|
|
runtime = runtime_factory()
|
|
session = runtime.open("test:///default", "host-a", timeout=5.0)
|
|
with pytest.raises(ExecutionFailure) as raised:
|
|
session.connection.crash()
|
|
assert raised.value.error_code == "libvirt_execution_uncertain"
|
|
assert raised.value.outcome_unknown is True
|
|
with pytest.raises(ExecutionFailure):
|
|
session.connection.getHostname()
|
|
session.close()
|
|
assert not session.process.is_alive()
|
|
|
|
|
|
def test_unknown_write_barrier_survives_restart_but_allows_reads_and_other_hosts(runtime_factory):
|
|
runtime = runtime_factory()
|
|
session = runtime.open("test:///write-hangs", "host-a", timeout=5.0)
|
|
domain = session.connection.lookupByUUIDString(VM_ID)
|
|
with pytest.raises(ExecutionFailure) as raised:
|
|
domain.destroy()
|
|
assert raised.value.outcome_unknown is True
|
|
session.close()
|
|
runtime.close(timeout=2.0)
|
|
|
|
restarted = runtime_factory()
|
|
observation = restarted.open("test:///default", "host-a", timeout=5.0)
|
|
domain = observation.connection.lookupByUUIDString(VM_ID)
|
|
assert domain.state() == (1, 0)
|
|
with pytest.raises(ExecutionFailure) as blocked:
|
|
domain.destroy()
|
|
assert blocked.value.error_code == "libvirt_execution_blocked"
|
|
observation.close()
|
|
# Closing a read session must not silently erase the interrupted write.
|
|
observation = restarted.open("test:///default", "host-a", timeout=5.0)
|
|
with pytest.raises(ExecutionFailure):
|
|
observation.connection.lookupByUUIDString(VM_ID).destroy()
|
|
observation.close()
|
|
other = restarted.open("test:///default", "host-b", timeout=5.0)
|
|
assert other.connection.lookupByUUIDString(VM_ID).destroy() == 0
|
|
other.close()
|
|
|
|
|
|
def test_successful_write_and_close_clear_only_its_own_barrier(runtime_factory):
|
|
runtime = runtime_factory()
|
|
for _ in range(2):
|
|
session = runtime.open("test:///default", "host-a", timeout=5.0)
|
|
assert session.connection.lookupByUUIDString(VM_ID).destroy() == 0
|
|
session.close()
|
|
assert session.failure is None
|
|
|
|
|
|
def test_host_limit_leaves_capacity_for_another_host(runtime_factory):
|
|
runtime = runtime_factory(max_connections=2, max_per_host=1)
|
|
first = runtime.open("test:///default", "host-a", timeout=5.0)
|
|
started = time.monotonic()
|
|
with pytest.raises(ExecutionFailure):
|
|
runtime.open("test:///default", "host-a", timeout=5.0)
|
|
assert time.monotonic() - started < 1.0
|
|
other = runtime.open("test:///default", "host-b", timeout=5.0)
|
|
assert other.connection.getHostname() == "rpc-test-host"
|
|
first.close()
|
|
other.close()
|
|
|
|
|
|
def test_global_limit_is_bounded_and_capacity_returns_after_close(runtime_factory):
|
|
runtime = runtime_factory(max_connections=1, max_per_host=2)
|
|
first = runtime.open("test:///default", "host-a", timeout=5.0)
|
|
started = time.monotonic()
|
|
with pytest.raises(ExecutionFailure) as raised:
|
|
runtime.open("test:///default", "host-b", timeout=5.0)
|
|
assert raised.value.error_code == "libvirt_execution_timeout"
|
|
assert raised.value.outcome_unknown is False
|
|
assert time.monotonic() - started < 1.0
|
|
first.close()
|
|
second = runtime.open("test:///default", "host-b", timeout=5.0)
|
|
assert second.connection.getHostname() == "rpc-test-host"
|
|
second.close()
|
|
|
|
|
|
def test_capacity_wait_is_deducted_from_callers_session_deadline(
|
|
runtime_factory):
|
|
runtime = runtime_factory(
|
|
max_connections=1, max_per_host=2, queue_timeout=30.0,
|
|
rpc_timeout=3.0)
|
|
first = runtime.open("test:///default", "host-a", timeout=5.0)
|
|
|
|
releaser = threading.Thread(
|
|
target=lambda: (time.sleep(0.25), first.close()), daemon=True)
|
|
releaser.start()
|
|
started = time.monotonic()
|
|
second = runtime.open("test:///default", "host-b", timeout=1.0)
|
|
queued_for = time.monotonic() - started
|
|
assert queued_for >= 0.15
|
|
|
|
# The session receives only the budget left after admission. Giving it
|
|
# a fresh one-second deadline here would make this read succeed.
|
|
time.sleep(0.8)
|
|
with pytest.raises(ExecutionFailure) as expired:
|
|
second.connection.getHostname()
|
|
assert expired.value.error_code == "libvirt_execution_timeout"
|
|
assert expired.value.outcome_unknown is False
|
|
assert time.monotonic() - started < 1.5
|
|
second.close()
|
|
releaser.join(timeout=1.0)
|
|
assert not releaser.is_alive()
|
|
|
|
|
|
def test_shutdown_wakes_capacity_waiter_without_waiting_for_queue_deadline(runtime_factory):
|
|
runtime = runtime_factory(max_connections=1, queue_timeout=30.0)
|
|
first = runtime.open("test:///default", "host-a", timeout=5.0)
|
|
entered = threading.Event()
|
|
errors = []
|
|
|
|
def open_queued_session():
|
|
entered.set()
|
|
try:
|
|
runtime.open("test:///default", "host-b", timeout=5.0)
|
|
except BaseException as exc:
|
|
errors.append(exc)
|
|
|
|
waiter = threading.Thread(target=open_queued_session, daemon=True)
|
|
waiter.start()
|
|
assert entered.wait(timeout=1.0)
|
|
started = time.monotonic()
|
|
runtime.begin_shutdown()
|
|
waiter.join(timeout=1.0)
|
|
assert not waiter.is_alive()
|
|
assert len(errors) == 1
|
|
assert isinstance(errors[0], ExecutionFailure)
|
|
assert errors[0].error_code == "libvirt_execution_stopped"
|
|
assert runtime.close(timeout=1.0) is True
|
|
assert time.monotonic() - started < 2.0
|
|
assert not first.process.is_alive()
|
|
|
|
|
|
def test_shutdown_rejects_new_sessions_and_reaps_existing_workers(runtime_factory):
|
|
runtime = runtime_factory()
|
|
session = runtime.open("test:///default", "host-a", timeout=5.0)
|
|
started = time.monotonic()
|
|
runtime.begin_shutdown()
|
|
with pytest.raises(ExecutionFailure) as stopped:
|
|
runtime.open("test:///default", "host-b", timeout=5.0)
|
|
assert stopped.value.error_code == "libvirt_execution_stopped"
|
|
assert runtime.close(timeout=1.0) is True
|
|
assert time.monotonic() - started < 2.0
|
|
assert not session.process.is_alive()
|
|
with pytest.raises(ExecutionFailure):
|
|
session.connection.getHostname()
|
|
|
|
|
|
def test_connection_close_cannot_wait_forever(runtime_factory):
|
|
runtime = runtime_factory()
|
|
session = runtime.open("test:///close-hangs", "host-a", timeout=5.0)
|
|
started = time.monotonic()
|
|
session.close()
|
|
assert time.monotonic() - started < 3.0
|
|
assert session.failure is None
|
|
assert session.cleanup_failure is not None
|
|
assert session.cleanup_failure.outcome_unknown is False
|
|
assert not session.process.is_alive()
|
|
|
|
|
|
def test_cross_session_handles_are_rejected_before_remote_dispatch(runtime_factory):
|
|
runtime = runtime_factory()
|
|
first = runtime.open("test:///default", "host-a", timeout=5.0)
|
|
second = runtime.open("test:///default", "host-b", timeout=5.0)
|
|
foreign_domain = second.connection.lookupByUUIDString(VM_ID)
|
|
with pytest.raises((ValueError, ExecutionFailure)):
|
|
first.connection.domainListGetStats([foreign_domain], 0, 0)
|
|
assert second.connection.getHostname() == "rpc-test-host"
|
|
first.close()
|
|
second.close()
|
|
|
|
|
|
def test_shutdown_reaps_worker_ignoring_sigterm_while_rpc_is_running(runtime_factory, tmp_path):
|
|
runtime = runtime_factory(rpc_timeout=30.0)
|
|
session = runtime.open("test:///default", "host-a", timeout=60.0)
|
|
marker = tmp_path / "worker-entered-hang"
|
|
errors = []
|
|
|
|
def invoke_blocking_write():
|
|
try:
|
|
session.connection.hang(str(marker), True)
|
|
except BaseException as exc:
|
|
errors.append(exc)
|
|
|
|
caller = threading.Thread(target=invoke_blocking_write, daemon=True)
|
|
caller.start()
|
|
deadline = time.monotonic() + 3.0
|
|
while not marker.exists() and time.monotonic() < deadline:
|
|
time.sleep(0.01)
|
|
assert marker.exists(), "child did not reach the injected blocking call"
|
|
assert int(marker.read_text()) == session.pid
|
|
started = time.monotonic()
|
|
runtime.begin_shutdown()
|
|
assert runtime.close(timeout=1.0) is True
|
|
caller.join(timeout=1.0)
|
|
assert time.monotonic() - started < 2.0
|
|
assert not caller.is_alive()
|
|
assert not session.process.is_alive()
|
|
assert len(errors) == 1
|
|
assert isinstance(errors[0], ExecutionFailure)
|
|
assert errors[0].outcome_unknown is True
|
|
|
|
|
|
def test_worker_exit_does_not_leave_its_stubborn_descendant_running(runtime_factory, tmp_path):
|
|
runtime = runtime_factory(rpc_timeout=3.0)
|
|
session = runtime.open("test:///default", "host-a", timeout=10.0)
|
|
heartbeat = tmp_path / "descendant-heartbeat"
|
|
descendant_pid = session.connection.start_stubborn_child(str(heartbeat))
|
|
try:
|
|
session.close()
|
|
assert not session.process.is_alive()
|
|
# A killed but not yet reaped descendant can briefly remain a zombie;
|
|
# compare liveness evidence instead of mistaking kill(pid, 0) for work.
|
|
time.sleep(0.1)
|
|
stopped_at = heartbeat.stat().st_mtime_ns
|
|
time.sleep(0.15)
|
|
assert heartbeat.stat().st_mtime_ns == stopped_at
|
|
finally:
|
|
# Keep this regression safe even against the broken implementation.
|
|
try:
|
|
os.kill(descendant_pid, signal.SIGKILL)
|
|
except ProcessLookupError:
|
|
pass
|