"""Real subprocess tests for the libvirt RPC deadline and isolation boundary.""" from __future__ import annotations import json import os import signal import threading import time from pathlib import Path import pytest from virsh_panel.libvirt_rpc import ExecutionFailure, LibvirtRuntime FIXTURES = Path(__file__).with_name("fixtures") VM_ID = "00000000-0000-4000-8000-000000000001" @pytest.mark.parametrize("option,value", [ ("rpc_timeout", float("inf")), ("connect_timeout", float("nan")), ("queue_timeout", 0), ("max_connections", -1), ("max_per_host", 1.5), ]) def test_execution_limits_cannot_disable_deadlines_or_capacity(tmp_path, option, value): with pytest.raises(ValueError): LibvirtRuntime(tmp_path / "rpc-state", **{option: value}) def test_new_fence_directories_persist_their_entries_in_each_parent(tmp_path, monkeypatch): synced = [] original = LibvirtRuntime._sync def sync(directory): synced.append(directory) original(directory) monkeypatch.setattr(LibvirtRuntime, "_sync", staticmethod(sync)) root = tmp_path / "new-data" / "rpc-state" runtime = LibvirtRuntime(root) scope = runtime._directory("host-a") assert scope.is_dir() assert synced == [tmp_path, root.parent, root] runtime._directory("host-a") assert len(synced) == 3 # repeated stream writes do not fsync unchanged parents @pytest.fixture def runtime_factory(tmp_path, monkeypatch): # The supervisor receives the backend explicitly. This path merely makes # that test module importable in a freshly spawned Python interpreter. monkeypatch.syspath_prepend(str(FIXTURES)) monkeypatch.setenv( "PYTHONPATH", os.pathsep.join(filter(None, [ str(FIXTURES), str(FIXTURES.parents[1]), os.environ.get("PYTHONPATH", ""), ])), ) runtimes = [] def create(**overrides): options = { "rpc_timeout": 0.35, "connect_timeout": 3.0, "queue_timeout": 0.15, "backend_module": "libvirt_rpc_backend", } options.update(overrides) runtime = LibvirtRuntime(tmp_path / "rpc-state", **options) runtimes.append(runtime) return runtime yield create for runtime in reversed(runtimes): runtime.close(timeout=2.0) def test_objects_nested_stats_and_values_cross_real_worker(runtime_factory): runtime = runtime_factory() session = runtime.open("test:///default", "host-a", timeout=5.0) conn = session.connection assert conn.process_id() == session.pid assert session.pid != os.getpid() assert conn.getHostname() == "rpc-test-host" domain = conn.lookupByUUIDString(VM_ID) assert domain.info() == [1, 2097152, 1048576, 2, 12345] assert domain.state() == (1, 0) stats = conn.getAllDomainStats(0, 0) assert isinstance(stats[0], tuple) assert stats[0][0].UUIDString() == VM_ID assert stats[0][1] == {"cpu.time": 12345, "nested": [None, True, 1.25]} assert conn.domainListGetStats([domain], 0, 0)[0][0].UUIDString() == VM_ID snapshot = domain.snapshotLookupByName("before-upgrade", 0) assert snapshot.getName() == "before-upgrade" assert snapshot.getDomain().UUIDString() == VM_ID value = {"tuple": (None, False, 2.5), "bytes": b"\x00\xffbinary", "list": [1]} assert conn.echo(value) == value session.close() assert not session.process.is_alive() def test_stream_handles_and_binary_short_writes_and_reads(runtime_factory): runtime = runtime_factory() session = runtime.open("test:///default", "host-a", timeout=5.0) conn = session.connection volume = conn.storagePoolLookupByName("images").storageVolLookupByName("disk.qcow2") data = b"\x00\xff\x80volume-content\x00" * 3 stream = conn.newStream(0) volume.upload(stream, 0, len(data), 0) remaining = data while remaining: sent = stream.send(remaining) assert 0 < sent <= len(remaining) remaining = remaining[sent:] stream.finish() reader = conn.newStream(0) volume.download(reader, 0, len(data), 0) output = bytearray() while chunk := reader.recv(65536): output.extend(chunk) reader.finish() assert bytes(output) == data assert volume.info()[2] == len(data) session.close() def test_native_error_code_and_full_fields_survive_rpc(runtime_factory): runtime = runtime_factory() session = runtime.open("test:///default", "host-a", timeout=5.0) with pytest.raises(Exception) as raised: session.connection.lookupByUUIDString("absent") error = raised.value assert not isinstance(error, ExecutionFailure) assert error.get_error_code() == 42 assert tuple(error.err) == ( 42, 10, "domain is absent", 2, "domain", "detail", None, 17, 23, ) # A normal libvirt error must not poison a healthy connection. assert session.connection.getHostname() == "rpc-test-host" session.close() def test_qemu_agent_command_uses_same_remote_domain(runtime_factory): runtime = runtime_factory() session = runtime.open("test:///default", "host-a", timeout=5.0) domain = session.connection.lookupByUUIDString(VM_ID) reply = session.qemu_agent_command(domain, '{"execute":"guest-ping"}', 3, 0) assert json.loads(reply) == {"return": { "uuid": VM_ID, "execute": "guest-ping", "timeout": 3, "flags": 0, }} session.close() def test_read_timeout_reaps_child_and_sticky_session_cannot_resume(runtime_factory): runtime = runtime_factory() session = runtime.open("test:///read-hangs", "host-a", timeout=5.0) started = time.monotonic() with pytest.raises(ExecutionFailure) as raised: session.connection.getHostname() assert time.monotonic() - started < 2.0 assert raised.value.error_code == "libvirt_execution_timeout" assert raised.value.outcome_unknown is False assert session.failure is not None with pytest.raises(ExecutionFailure): session.connection.getVersion() session.close() assert not session.process.is_alive() # A failed read carries no unconfirmed write and cannot quarantine a host. healthy = runtime.open("test:///default", "host-a", timeout=5.0) assert healthy.connection.lookupByUUIDString(VM_ID).destroy() == 0 healthy.close() def test_absolute_session_deadline_is_not_extended_by_successful_calls(runtime_factory): runtime = runtime_factory(rpc_timeout=3.0) session = runtime.open("test:///default", "host-a", timeout=0.8) assert session.connection.getHostname() == "rpc-test-host" time.sleep(0.9) started = time.monotonic() with pytest.raises(ExecutionFailure): session.connection.getHostname() assert time.monotonic() - started < 1.0 session.close() def test_open_timeout_is_bounded_and_releases_capacity(runtime_factory): runtime = runtime_factory(connect_timeout=0.4, max_connections=1) started = time.monotonic() with pytest.raises(ExecutionFailure) as raised: runtime.open("test:///open-hangs", "host-a", timeout=3.0) assert time.monotonic() - started < 2.0 assert raised.value.outcome_unknown is False healthy = runtime.open("test:///default", "host-b", timeout=5.0) assert healthy.connection.getHostname() == "rpc-test-host" healthy.close() def test_native_open_error_releases_capacity_without_an_unknown_write(runtime_factory): runtime = runtime_factory(max_connections=1) with pytest.raises(Exception) as raised: runtime.open("test:///open-error", "host-a", timeout=5.0) assert not isinstance(raised.value, ExecutionFailure) assert raised.value.get_error_code() == 9 assert not list(runtime.state_dir.rglob("*.json")) healthy = runtime.open("test:///default", "host-a", timeout=5.0) assert healthy.connection.lookupByUUIDString(VM_ID).destroy() == 0 healthy.close() def test_child_crash_is_uncertain_and_cannot_be_reused(runtime_factory): runtime = runtime_factory() session = runtime.open("test:///default", "host-a", timeout=5.0) with pytest.raises(ExecutionFailure) as raised: session.connection.crash() assert raised.value.error_code == "libvirt_execution_uncertain" assert raised.value.outcome_unknown is True with pytest.raises(ExecutionFailure): session.connection.getHostname() session.close() assert not session.process.is_alive() def test_unknown_write_barrier_survives_restart_but_allows_reads_and_other_hosts(runtime_factory): runtime = runtime_factory() session = runtime.open("test:///write-hangs", "host-a", timeout=5.0) domain = session.connection.lookupByUUIDString(VM_ID) with pytest.raises(ExecutionFailure) as raised: domain.destroy() assert raised.value.outcome_unknown is True session.close() runtime.close(timeout=2.0) restarted = runtime_factory() observation = restarted.open("test:///default", "host-a", timeout=5.0) domain = observation.connection.lookupByUUIDString(VM_ID) assert domain.state() == (1, 0) with pytest.raises(ExecutionFailure) as blocked: domain.destroy() assert blocked.value.error_code == "libvirt_execution_blocked" observation.close() # Closing a read session must not silently erase the interrupted write. observation = restarted.open("test:///default", "host-a", timeout=5.0) with pytest.raises(ExecutionFailure): observation.connection.lookupByUUIDString(VM_ID).destroy() observation.close() other = restarted.open("test:///default", "host-b", timeout=5.0) assert other.connection.lookupByUUIDString(VM_ID).destroy() == 0 other.close() def test_successful_write_and_close_clear_only_its_own_barrier(runtime_factory): runtime = runtime_factory() for _ in range(2): session = runtime.open("test:///default", "host-a", timeout=5.0) assert session.connection.lookupByUUIDString(VM_ID).destroy() == 0 session.close() assert session.failure is None def test_host_limit_leaves_capacity_for_another_host(runtime_factory): runtime = runtime_factory(max_connections=2, max_per_host=1) first = runtime.open("test:///default", "host-a", timeout=5.0) started = time.monotonic() with pytest.raises(ExecutionFailure): runtime.open("test:///default", "host-a", timeout=5.0) assert time.monotonic() - started < 1.0 other = runtime.open("test:///default", "host-b", timeout=5.0) assert other.connection.getHostname() == "rpc-test-host" first.close() other.close() def test_global_limit_is_bounded_and_capacity_returns_after_close(runtime_factory): runtime = runtime_factory(max_connections=1, max_per_host=2) first = runtime.open("test:///default", "host-a", timeout=5.0) started = time.monotonic() with pytest.raises(ExecutionFailure) as raised: runtime.open("test:///default", "host-b", timeout=5.0) assert raised.value.error_code == "libvirt_execution_timeout" assert raised.value.outcome_unknown is False assert time.monotonic() - started < 1.0 first.close() second = runtime.open("test:///default", "host-b", timeout=5.0) assert second.connection.getHostname() == "rpc-test-host" second.close() def test_capacity_wait_is_deducted_from_callers_session_deadline( runtime_factory): runtime = runtime_factory( max_connections=1, max_per_host=2, queue_timeout=30.0, rpc_timeout=3.0) first = runtime.open("test:///default", "host-a", timeout=5.0) releaser = threading.Thread( target=lambda: (time.sleep(0.25), first.close()), daemon=True) releaser.start() started = time.monotonic() second = runtime.open("test:///default", "host-b", timeout=1.0) queued_for = time.monotonic() - started assert queued_for >= 0.15 # The session receives only the budget left after admission. Giving it # a fresh one-second deadline here would make this read succeed. time.sleep(0.8) with pytest.raises(ExecutionFailure) as expired: second.connection.getHostname() assert expired.value.error_code == "libvirt_execution_timeout" assert expired.value.outcome_unknown is False assert time.monotonic() - started < 1.5 second.close() releaser.join(timeout=1.0) assert not releaser.is_alive() def test_shutdown_wakes_capacity_waiter_without_waiting_for_queue_deadline(runtime_factory): runtime = runtime_factory(max_connections=1, queue_timeout=30.0) first = runtime.open("test:///default", "host-a", timeout=5.0) entered = threading.Event() errors = [] def open_queued_session(): entered.set() try: runtime.open("test:///default", "host-b", timeout=5.0) except BaseException as exc: errors.append(exc) waiter = threading.Thread(target=open_queued_session, daemon=True) waiter.start() assert entered.wait(timeout=1.0) started = time.monotonic() runtime.begin_shutdown() waiter.join(timeout=1.0) assert not waiter.is_alive() assert len(errors) == 1 assert isinstance(errors[0], ExecutionFailure) assert errors[0].error_code == "libvirt_execution_stopped" assert runtime.close(timeout=1.0) is True assert time.monotonic() - started < 2.0 assert not first.process.is_alive() def test_shutdown_rejects_new_sessions_and_reaps_existing_workers(runtime_factory): runtime = runtime_factory() session = runtime.open("test:///default", "host-a", timeout=5.0) started = time.monotonic() runtime.begin_shutdown() with pytest.raises(ExecutionFailure) as stopped: runtime.open("test:///default", "host-b", timeout=5.0) assert stopped.value.error_code == "libvirt_execution_stopped" assert runtime.close(timeout=1.0) is True assert time.monotonic() - started < 2.0 assert not session.process.is_alive() with pytest.raises(ExecutionFailure): session.connection.getHostname() def test_connection_close_cannot_wait_forever(runtime_factory): runtime = runtime_factory() session = runtime.open("test:///close-hangs", "host-a", timeout=5.0) started = time.monotonic() session.close() assert time.monotonic() - started < 3.0 assert session.failure is None assert session.cleanup_failure is not None assert session.cleanup_failure.outcome_unknown is False assert not session.process.is_alive() def test_cross_session_handles_are_rejected_before_remote_dispatch(runtime_factory): runtime = runtime_factory() first = runtime.open("test:///default", "host-a", timeout=5.0) second = runtime.open("test:///default", "host-b", timeout=5.0) foreign_domain = second.connection.lookupByUUIDString(VM_ID) with pytest.raises((ValueError, ExecutionFailure)): first.connection.domainListGetStats([foreign_domain], 0, 0) assert second.connection.getHostname() == "rpc-test-host" first.close() second.close() def test_shutdown_reaps_worker_ignoring_sigterm_while_rpc_is_running(runtime_factory, tmp_path): runtime = runtime_factory(rpc_timeout=30.0) session = runtime.open("test:///default", "host-a", timeout=60.0) marker = tmp_path / "worker-entered-hang" errors = [] def invoke_blocking_write(): try: session.connection.hang(str(marker), True) except BaseException as exc: errors.append(exc) caller = threading.Thread(target=invoke_blocking_write, daemon=True) caller.start() deadline = time.monotonic() + 3.0 while not marker.exists() and time.monotonic() < deadline: time.sleep(0.01) assert marker.exists(), "child did not reach the injected blocking call" assert int(marker.read_text()) == session.pid started = time.monotonic() runtime.begin_shutdown() assert runtime.close(timeout=1.0) is True caller.join(timeout=1.0) assert time.monotonic() - started < 2.0 assert not caller.is_alive() assert not session.process.is_alive() assert len(errors) == 1 assert isinstance(errors[0], ExecutionFailure) assert errors[0].outcome_unknown is True def test_worker_exit_does_not_leave_its_stubborn_descendant_running(runtime_factory, tmp_path): runtime = runtime_factory(rpc_timeout=3.0) session = runtime.open("test:///default", "host-a", timeout=10.0) heartbeat = tmp_path / "descendant-heartbeat" descendant_pid = session.connection.start_stubborn_child(str(heartbeat)) try: session.close() assert not session.process.is_alive() # A killed but not yet reaped descendant can briefly remain a zombie; # compare liveness evidence instead of mistaking kill(pid, 0) for work. time.sleep(0.1) stopped_at = heartbeat.stat().st_mtime_ns time.sleep(0.15) assert heartbeat.stat().st_mtime_ns == stopped_at finally: # Keep this regression safe even against the broken implementation. try: os.kill(descendant_pid, signal.SIGKILL) except ProcessLookupError: pass