You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/engine/tests/test_resource_locks.py

302 lines
11 KiB
Python

"""Host/VM isolation and reclamation at the local admission boundary."""
from concurrent.futures import Future
from contextlib import contextmanager
from types import SimpleNamespace
import threading
import time
import pytest
from virsh_panel.resource_locks import ResourceLockRegistry, ResourceLockTimeout
def _start(call):
result = Future()
def run():
try:
result.set_result(call())
except BaseException as exc:
result.set_exception(exc)
worker = threading.Thread(target=run, daemon=True)
worker.start()
return result
def test_waiter_timeout_releases_partial_acquisition_and_registry_references():
registry = ResourceLockRegistry()
with registry.hold(["b"]):
def waiting():
with registry.hold(["b", "a"], timeout=0.03):
pytest.fail("conflicting resource admitted")
result = _start(waiting)
with pytest.raises(ResourceLockTimeout):
result.result(timeout=1)
assert registry.entry_count() == 1
with registry.hold(["a"], timeout=0):
assert registry.entry_count() == 2
assert registry.entry_count() == 0
def test_same_thread_can_reenter_without_replacing_waiters_lock():
registry = ResourceLockRegistry()
acquired = threading.Event()
def waiting():
with registry.hold(["vm"]):
acquired.set()
with registry.hold(["vm"]):
with registry.hold(["vm", "vm"], timeout=0):
assert registry.entry_count() == 1
waiter = _start(waiting)
assert not acquired.wait(0.03)
assert registry.entry_count() == 1
waiter.result(timeout=1)
assert acquired.is_set()
assert registry.entry_count() == 0
def test_reversed_resource_sets_share_one_order_and_do_not_deadlock():
registry = ResourceLockRegistry()
start = threading.Barrier(2)
values = []
def run(resources):
start.wait(timeout=1)
for _ in range(20):
with registry.hold(resources):
before = len(values)
time.sleep(0.0005)
assert len(values) == before
values.append(True)
first = _start(lambda: run(["storage", "vm", "allocation"]))
second = _start(lambda: run(["allocation", "vm", "storage"]))
first.result(timeout=2)
second.result(timeout=2)
assert len(values) == 40
assert registry.entry_count() == 0
def test_distinct_keys_progress_and_finished_keys_are_reclaimed():
registry = ResourceLockRegistry()
def run():
for number in range(200):
with registry.hold([f"vm:other-host:{number}"], timeout=0):
pass
with registry.hold(["vm:blocked-host:1"]):
_start(run).result(timeout=1)
assert registry.entry_count() == 1
assert registry.entry_count() == 0
def test_exception_releases_the_complete_resource_set():
registry = ResourceLockRegistry()
with pytest.raises(RuntimeError, match="failed"):
with registry.hold(["a", "b"]):
raise RuntimeError("failed")
assert registry.entry_count() == 0
with registry.hold(["a", "b"], timeout=0):
pass
class _ReachedConnection(Exception):
pass
@pytest.fixture
def services(monkeypatch):
# Reuse the native API adapter, never contact a real hypervisor.
from test_libvirt_service import VMServiceTests
from virsh_panel import coordination
case = VMServiceTests()
case.setUp()
module = case.service_module
monkeypatch.setattr(module, "get_runtime", lambda: SimpleNamespace(queue_timeout=0.04))
monkeypatch.setattr(coordination, "get_default_coordinator", lambda: None)
monkeypatch.setattr(coordination, "current_execution", lambda: None)
def make(host, pool="images", *, execution_scope=None):
return module.VMService(
f"qemu+ssh://panel@{host}/system", pool, "iso", "bridge:br0",
execution_scope=execution_scope)
return module, make
def _connection_probe(service, monkeypatch, entered, release=None):
@contextmanager
def connect(**_kwargs):
entered.set()
if release is not None:
assert release.wait(1)
raise _ReachedConnection()
yield # pragma: no cover - make this a context manager
monkeypatch.setattr(service, "_connect", connect)
def test_stalled_host_storage_does_not_block_another_host(services, monkeypatch):
_module, make = services
first, other = make("host-a"), make("host-b")
entered, release = threading.Event(), threading.Event()
other_entered = threading.Event()
_connection_probe(first, monkeypatch, entered, release)
_connection_probe(other, monkeypatch, other_entered)
running = _start(first.ensure_storage_pools)
assert entered.wait(1)
try:
with pytest.raises(_ReachedConnection):
other.ensure_storage_pools()
assert other_entered.is_set()
finally:
release.set()
with pytest.raises(_ReachedConnection):
running.result(timeout=1)
def test_same_host_pool_aliases_share_storage_admission(services, monkeypatch):
module, make = services
first, alias = make("host-a", "images"), make("host-a", "/home/kvm")
entered, release = threading.Event(), threading.Event()
alias_entered = threading.Event()
_connection_probe(first, monkeypatch, entered, release)
_connection_probe(alias, monkeypatch, alias_entered)
running = _start(first.ensure_storage_pools)
assert entered.wait(1)
try:
with pytest.raises(module.VMError) as denied:
alias.ensure_storage_pools()
assert denied.value.status == 409
assert denied.value.error_code == "coordination_busy"
assert not alias_entered.is_set()
finally:
release.set()
with pytest.raises(_ReachedConnection):
running.result(timeout=1)
def test_snapshot_excludes_same_vm_power_but_not_other_vm_or_pure_reads(services, monkeypatch):
module, make = services
snapshot, power = make("host-a"), make("host-a")
vm_id = "00000000-0000-4000-8000-000000000001"
other_id = "00000000-0000-4000-8000-000000000002"
entered, release = threading.Event(), threading.Event()
power_entered = threading.Event()
_connection_probe(snapshot, monkeypatch, entered, release)
_connection_probe(power, monkeypatch, power_entered)
running = _start(lambda: snapshot.create_live_snapshot(vm_id, "a" * 32))
assert entered.wait(1)
try:
with pytest.raises(module.VMError) as denied:
power.action(vm_id, "poweroff")
assert denied.value.error_code == "coordination_busy"
assert not power_entered.is_set()
with pytest.raises(_ReachedConnection):
power.action(other_id, "poweroff")
with pytest.raises(_ReachedConnection):
power.get_vm_state(vm_id)
finally:
release.set()
with pytest.raises(_ReachedConnection):
running.result(timeout=1)
def test_scoped_production_guard_fails_closed_without_coordinator(services):
module, make = services
service = make("host-a", execution_scope="host-a")
with pytest.raises(module.VMError) as denied:
with service._operation_guard(storage=True):
pytest.fail("uncoordinated production write admitted")
assert denied.value.status == 503
assert denied.value.error_code == "coordination_unavailable"
assert module.RESOURCE_LOCKS.entry_count() == 0
def test_remote_uncertainty_releases_local_lock_but_retains_durable_resource(services, monkeypatch):
from test_durable_jobs import MemoryCAS
from virsh_panel import coordination
module, make = services
coordinator = coordination.DurableCoordinator(MemoryCAS())
monkeypatch.setattr(coordination, "get_default_coordinator", lambda: coordinator)
service = make("host-a", execution_scope="host-a")
vm_id = "00000000-0000-4000-8000-000000000001"
with pytest.raises(module.VMError, match="unconfirmed"):
with service._operation_guard(vm_id, storage=True):
raise module.VMError("unconfirmed", 504, error_code="libvirt_execution_uncertain")
assert module.RESOURCE_LOCKS.entry_count() == 0
claim = next(iter(coordinator.snapshot().values()))
assert claim["state"] == "uncertain"
assert claim["resources"] == ["storage:host-a", f"vm:host-a:{vm_id}"]
with pytest.raises(module.VMError) as denied:
with service._operation_guard(vm_id):
pytest.fail("unknown remote mutation replayed")
assert denied.value.status == 503
assert denied.value.error_code == "coordination_uncertain"
assert module.RESOURCE_LOCKS.entry_count() == 0
def test_accepted_shutdown_releases_durable_vm_claim_for_forceoff(monkeypatch):
from test_durable_jobs import MemoryCAS
from test_libvirt_service import VMServiceTests
from virsh_panel import coordination
case = VMServiceTests()
case.setUp()
vm = case.service.create_vm(case.spec())
coordinator = coordination.DurableCoordinator(MemoryCAS())
monkeypatch.setattr(coordination, "get_default_coordinator", lambda: coordinator)
case.backend.shutdown_stalls = True
receipt = case.service.action(vm["uuid"], "shutdown")
assert receipt == {"uuid": vm["uuid"], "action": "shutdown", "command_accepted": True}
assert case.backend.domains[0].isActive()
assert coordinator.snapshot() == {}
assert case.service_module.RESOURCE_LOCKS.entry_count() == 0
assert case.service.action(vm["uuid"], "poweroff")["command_accepted"] is True
assert not case.backend.domains[0].isActive()
assert case.backend.shutdown_calls == case.backend.destroy_calls == 1
assert coordinator.snapshot() == {}
def test_power_response_loss_still_retains_durable_vm_claim(monkeypatch):
from test_durable_jobs import MemoryCAS
from test_libvirt_service import FakeLibvirtError, VMServiceTests
from virsh_panel import coordination
case = VMServiceTests()
case.setUp()
vm = case.service.create_vm(case.spec())
coordinator = coordination.DurableCoordinator(MemoryCAS())
monkeypatch.setattr(coordination, "get_default_coordinator", lambda: coordinator)
domain = case.backend.domains[0]
shutdown = domain.shutdown
def commit_then_lose_response():
shutdown()
raise FakeLibvirtError("command response lost")
monkeypatch.setattr(domain, "shutdown", commit_then_lose_response)
with pytest.raises(case.error_type) as lost:
case.service.action(vm["uuid"], "shutdown")
assert lost.value.error_code == "power_action_result_uncertain"
assert not domain.isActive()
claim = next(iter(coordinator.snapshot().values()))
assert claim["state"] == "uncertain"
assert case.service_module.RESOURCE_LOCKS.entry_count() == 0
# A current shutoff state does not supply the missing command receipt.
starts_before = case.backend.domain_starts
with pytest.raises(case.error_type) as blocked:
case.service.action(vm["uuid"], "start")
assert blocked.value.error_code == "coordination_uncertain"
assert case.backend.domain_starts == starts_before