"""Host/VM isolation and reclamation at the local admission boundary.""" from concurrent.futures import Future from contextlib import contextmanager from types import SimpleNamespace import threading import time import pytest from virsh_panel.resource_locks import ResourceLockRegistry, ResourceLockTimeout def _start(call): result = Future() def run(): try: result.set_result(call()) except BaseException as exc: result.set_exception(exc) worker = threading.Thread(target=run, daemon=True) worker.start() return result def test_waiter_timeout_releases_partial_acquisition_and_registry_references(): registry = ResourceLockRegistry() with registry.hold(["b"]): def waiting(): with registry.hold(["b", "a"], timeout=0.03): pytest.fail("conflicting resource admitted") result = _start(waiting) with pytest.raises(ResourceLockTimeout): result.result(timeout=1) assert registry.entry_count() == 1 with registry.hold(["a"], timeout=0): assert registry.entry_count() == 2 assert registry.entry_count() == 0 def test_same_thread_can_reenter_without_replacing_waiters_lock(): registry = ResourceLockRegistry() acquired = threading.Event() def waiting(): with registry.hold(["vm"]): acquired.set() with registry.hold(["vm"]): with registry.hold(["vm", "vm"], timeout=0): assert registry.entry_count() == 1 waiter = _start(waiting) assert not acquired.wait(0.03) assert registry.entry_count() == 1 waiter.result(timeout=1) assert acquired.is_set() assert registry.entry_count() == 0 def test_reversed_resource_sets_share_one_order_and_do_not_deadlock(): registry = ResourceLockRegistry() start = threading.Barrier(2) values = [] def run(resources): start.wait(timeout=1) for _ in range(20): with registry.hold(resources): before = len(values) time.sleep(0.0005) assert len(values) == before values.append(True) first = _start(lambda: run(["storage", "vm", "allocation"])) second = _start(lambda: run(["allocation", "vm", "storage"])) first.result(timeout=2) second.result(timeout=2) assert len(values) == 40 assert registry.entry_count() == 0 def test_distinct_keys_progress_and_finished_keys_are_reclaimed(): registry = ResourceLockRegistry() def run(): for number in range(200): with registry.hold([f"vm:other-host:{number}"], timeout=0): pass with registry.hold(["vm:blocked-host:1"]): _start(run).result(timeout=1) assert registry.entry_count() == 1 assert registry.entry_count() == 0 def test_exception_releases_the_complete_resource_set(): registry = ResourceLockRegistry() with pytest.raises(RuntimeError, match="failed"): with registry.hold(["a", "b"]): raise RuntimeError("failed") assert registry.entry_count() == 0 with registry.hold(["a", "b"], timeout=0): pass class _ReachedConnection(Exception): pass @pytest.fixture def services(monkeypatch): # Reuse the native API adapter, never contact a real hypervisor. from test_libvirt_service import VMServiceTests from virsh_panel import coordination case = VMServiceTests() case.setUp() module = case.service_module monkeypatch.setattr(module, "get_runtime", lambda: SimpleNamespace(queue_timeout=0.04)) monkeypatch.setattr(coordination, "get_default_coordinator", lambda: None) monkeypatch.setattr(coordination, "current_execution", lambda: None) def make(host, pool="images", *, execution_scope=None): return module.VMService( f"qemu+ssh://panel@{host}/system", pool, "iso", "bridge:br0", execution_scope=execution_scope) return module, make def _connection_probe(service, monkeypatch, entered, release=None): @contextmanager def connect(**_kwargs): entered.set() if release is not None: assert release.wait(1) raise _ReachedConnection() yield # pragma: no cover - make this a context manager monkeypatch.setattr(service, "_connect", connect) def test_stalled_host_storage_does_not_block_another_host(services, monkeypatch): _module, make = services first, other = make("host-a"), make("host-b") entered, release = threading.Event(), threading.Event() other_entered = threading.Event() _connection_probe(first, monkeypatch, entered, release) _connection_probe(other, monkeypatch, other_entered) running = _start(first.ensure_storage_pools) assert entered.wait(1) try: with pytest.raises(_ReachedConnection): other.ensure_storage_pools() assert other_entered.is_set() finally: release.set() with pytest.raises(_ReachedConnection): running.result(timeout=1) def test_same_host_pool_aliases_share_storage_admission(services, monkeypatch): module, make = services first, alias = make("host-a", "images"), make("host-a", "/home/kvm") entered, release = threading.Event(), threading.Event() alias_entered = threading.Event() _connection_probe(first, monkeypatch, entered, release) _connection_probe(alias, monkeypatch, alias_entered) running = _start(first.ensure_storage_pools) assert entered.wait(1) try: with pytest.raises(module.VMError) as denied: alias.ensure_storage_pools() assert denied.value.status == 409 assert denied.value.error_code == "coordination_busy" assert not alias_entered.is_set() finally: release.set() with pytest.raises(_ReachedConnection): running.result(timeout=1) def test_snapshot_excludes_same_vm_power_but_not_other_vm_or_pure_reads(services, monkeypatch): module, make = services snapshot, power = make("host-a"), make("host-a") vm_id = "00000000-0000-4000-8000-000000000001" other_id = "00000000-0000-4000-8000-000000000002" entered, release = threading.Event(), threading.Event() power_entered = threading.Event() _connection_probe(snapshot, monkeypatch, entered, release) _connection_probe(power, monkeypatch, power_entered) running = _start(lambda: snapshot.create_live_snapshot(vm_id, "a" * 32)) assert entered.wait(1) try: with pytest.raises(module.VMError) as denied: power.action(vm_id, "poweroff") assert denied.value.error_code == "coordination_busy" assert not power_entered.is_set() with pytest.raises(_ReachedConnection): power.action(other_id, "poweroff") with pytest.raises(_ReachedConnection): power.get_vm_state(vm_id) finally: release.set() with pytest.raises(_ReachedConnection): running.result(timeout=1) def test_scoped_production_guard_fails_closed_without_coordinator(services): module, make = services service = make("host-a", execution_scope="host-a") with pytest.raises(module.VMError) as denied: with service._operation_guard(storage=True): pytest.fail("uncoordinated production write admitted") assert denied.value.status == 503 assert denied.value.error_code == "coordination_unavailable" assert module.RESOURCE_LOCKS.entry_count() == 0 def test_remote_uncertainty_releases_local_lock_but_retains_durable_resource(services, monkeypatch): from test_durable_jobs import MemoryCAS from virsh_panel import coordination module, make = services coordinator = coordination.DurableCoordinator(MemoryCAS()) monkeypatch.setattr(coordination, "get_default_coordinator", lambda: coordinator) service = make("host-a", execution_scope="host-a") vm_id = "00000000-0000-4000-8000-000000000001" with pytest.raises(module.VMError, match="unconfirmed"): with service._operation_guard(vm_id, storage=True): raise module.VMError("unconfirmed", 504, error_code="libvirt_execution_uncertain") assert module.RESOURCE_LOCKS.entry_count() == 0 claim = next(iter(coordinator.snapshot().values())) assert claim["state"] == "uncertain" assert claim["resources"] == ["storage:host-a", f"vm:host-a:{vm_id}"] with pytest.raises(module.VMError) as denied: with service._operation_guard(vm_id): pytest.fail("unknown remote mutation replayed") assert denied.value.status == 503 assert denied.value.error_code == "coordination_uncertain" assert module.RESOURCE_LOCKS.entry_count() == 0 def test_accepted_shutdown_releases_durable_vm_claim_for_forceoff(monkeypatch): from test_durable_jobs import MemoryCAS from test_libvirt_service import VMServiceTests from virsh_panel import coordination case = VMServiceTests() case.setUp() vm = case.service.create_vm(case.spec()) coordinator = coordination.DurableCoordinator(MemoryCAS()) monkeypatch.setattr(coordination, "get_default_coordinator", lambda: coordinator) case.backend.shutdown_stalls = True receipt = case.service.action(vm["uuid"], "shutdown") assert receipt == {"uuid": vm["uuid"], "action": "shutdown", "command_accepted": True} assert case.backend.domains[0].isActive() assert coordinator.snapshot() == {} assert case.service_module.RESOURCE_LOCKS.entry_count() == 0 assert case.service.action(vm["uuid"], "poweroff")["command_accepted"] is True assert not case.backend.domains[0].isActive() assert case.backend.shutdown_calls == case.backend.destroy_calls == 1 assert coordinator.snapshot() == {} def test_power_response_loss_still_retains_durable_vm_claim(monkeypatch): from test_durable_jobs import MemoryCAS from test_libvirt_service import FakeLibvirtError, VMServiceTests from virsh_panel import coordination case = VMServiceTests() case.setUp() vm = case.service.create_vm(case.spec()) coordinator = coordination.DurableCoordinator(MemoryCAS()) monkeypatch.setattr(coordination, "get_default_coordinator", lambda: coordinator) domain = case.backend.domains[0] shutdown = domain.shutdown def commit_then_lose_response(): shutdown() raise FakeLibvirtError("command response lost") monkeypatch.setattr(domain, "shutdown", commit_then_lose_response) with pytest.raises(case.error_type) as lost: case.service.action(vm["uuid"], "shutdown") assert lost.value.error_code == "power_action_result_uncertain" assert not domain.isActive() claim = next(iter(coordinator.snapshot().values())) assert claim["state"] == "uncertain" assert case.service_module.RESOURCE_LOCKS.entry_count() == 0 # A current shutoff state does not supply the missing command receipt. starts_before = case.backend.domain_starts with pytest.raises(case.error_type) as blocked: case.service.action(vm["uuid"], "start") assert blocked.value.error_code == "coordination_uncertain" assert case.backend.domain_starts == starts_before