You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
302 lines
11 KiB
Python
302 lines
11 KiB
Python
"""Host/VM isolation and reclamation at the local admission boundary."""
|
|
|
|
from concurrent.futures import Future
|
|
from contextlib import contextmanager
|
|
from types import SimpleNamespace
|
|
import threading
|
|
import time
|
|
|
|
import pytest
|
|
|
|
from virsh_panel.resource_locks import ResourceLockRegistry, ResourceLockTimeout
|
|
|
|
|
|
def _start(call):
|
|
result = Future()
|
|
|
|
def run():
|
|
try:
|
|
result.set_result(call())
|
|
except BaseException as exc:
|
|
result.set_exception(exc)
|
|
|
|
worker = threading.Thread(target=run, daemon=True)
|
|
worker.start()
|
|
return result
|
|
|
|
|
|
def test_waiter_timeout_releases_partial_acquisition_and_registry_references():
|
|
registry = ResourceLockRegistry()
|
|
with registry.hold(["b"]):
|
|
def waiting():
|
|
with registry.hold(["b", "a"], timeout=0.03):
|
|
pytest.fail("conflicting resource admitted")
|
|
|
|
result = _start(waiting)
|
|
with pytest.raises(ResourceLockTimeout):
|
|
result.result(timeout=1)
|
|
assert registry.entry_count() == 1
|
|
with registry.hold(["a"], timeout=0):
|
|
assert registry.entry_count() == 2
|
|
assert registry.entry_count() == 0
|
|
|
|
|
|
def test_same_thread_can_reenter_without_replacing_waiters_lock():
|
|
registry = ResourceLockRegistry()
|
|
acquired = threading.Event()
|
|
|
|
def waiting():
|
|
with registry.hold(["vm"]):
|
|
acquired.set()
|
|
|
|
with registry.hold(["vm"]):
|
|
with registry.hold(["vm", "vm"], timeout=0):
|
|
assert registry.entry_count() == 1
|
|
waiter = _start(waiting)
|
|
assert not acquired.wait(0.03)
|
|
assert registry.entry_count() == 1
|
|
waiter.result(timeout=1)
|
|
assert acquired.is_set()
|
|
assert registry.entry_count() == 0
|
|
|
|
|
|
def test_reversed_resource_sets_share_one_order_and_do_not_deadlock():
|
|
registry = ResourceLockRegistry()
|
|
start = threading.Barrier(2)
|
|
values = []
|
|
|
|
def run(resources):
|
|
start.wait(timeout=1)
|
|
for _ in range(20):
|
|
with registry.hold(resources):
|
|
before = len(values)
|
|
time.sleep(0.0005)
|
|
assert len(values) == before
|
|
values.append(True)
|
|
|
|
first = _start(lambda: run(["storage", "vm", "allocation"]))
|
|
second = _start(lambda: run(["allocation", "vm", "storage"]))
|
|
first.result(timeout=2)
|
|
second.result(timeout=2)
|
|
assert len(values) == 40
|
|
assert registry.entry_count() == 0
|
|
|
|
|
|
def test_distinct_keys_progress_and_finished_keys_are_reclaimed():
|
|
registry = ResourceLockRegistry()
|
|
|
|
def run():
|
|
for number in range(200):
|
|
with registry.hold([f"vm:other-host:{number}"], timeout=0):
|
|
pass
|
|
|
|
with registry.hold(["vm:blocked-host:1"]):
|
|
_start(run).result(timeout=1)
|
|
assert registry.entry_count() == 1
|
|
assert registry.entry_count() == 0
|
|
|
|
|
|
def test_exception_releases_the_complete_resource_set():
|
|
registry = ResourceLockRegistry()
|
|
with pytest.raises(RuntimeError, match="failed"):
|
|
with registry.hold(["a", "b"]):
|
|
raise RuntimeError("failed")
|
|
assert registry.entry_count() == 0
|
|
with registry.hold(["a", "b"], timeout=0):
|
|
pass
|
|
|
|
|
|
class _ReachedConnection(Exception):
|
|
pass
|
|
|
|
|
|
@pytest.fixture
|
|
def services(monkeypatch):
|
|
# Reuse the native API adapter, never contact a real hypervisor.
|
|
from test_libvirt_service import VMServiceTests
|
|
from virsh_panel import coordination
|
|
|
|
case = VMServiceTests()
|
|
case.setUp()
|
|
module = case.service_module
|
|
monkeypatch.setattr(module, "get_runtime", lambda: SimpleNamespace(queue_timeout=0.04))
|
|
monkeypatch.setattr(coordination, "get_default_coordinator", lambda: None)
|
|
monkeypatch.setattr(coordination, "current_execution", lambda: None)
|
|
|
|
def make(host, pool="images", *, execution_scope=None):
|
|
return module.VMService(
|
|
f"qemu+ssh://panel@{host}/system", pool, "iso", "bridge:br0",
|
|
execution_scope=execution_scope)
|
|
|
|
return module, make
|
|
|
|
|
|
def _connection_probe(service, monkeypatch, entered, release=None):
|
|
@contextmanager
|
|
def connect(**_kwargs):
|
|
entered.set()
|
|
if release is not None:
|
|
assert release.wait(1)
|
|
raise _ReachedConnection()
|
|
yield # pragma: no cover - make this a context manager
|
|
|
|
monkeypatch.setattr(service, "_connect", connect)
|
|
|
|
|
|
def test_stalled_host_storage_does_not_block_another_host(services, monkeypatch):
|
|
_module, make = services
|
|
first, other = make("host-a"), make("host-b")
|
|
entered, release = threading.Event(), threading.Event()
|
|
other_entered = threading.Event()
|
|
_connection_probe(first, monkeypatch, entered, release)
|
|
_connection_probe(other, monkeypatch, other_entered)
|
|
running = _start(first.ensure_storage_pools)
|
|
assert entered.wait(1)
|
|
try:
|
|
with pytest.raises(_ReachedConnection):
|
|
other.ensure_storage_pools()
|
|
assert other_entered.is_set()
|
|
finally:
|
|
release.set()
|
|
with pytest.raises(_ReachedConnection):
|
|
running.result(timeout=1)
|
|
|
|
|
|
def test_same_host_pool_aliases_share_storage_admission(services, monkeypatch):
|
|
module, make = services
|
|
first, alias = make("host-a", "images"), make("host-a", "/home/kvm")
|
|
entered, release = threading.Event(), threading.Event()
|
|
alias_entered = threading.Event()
|
|
_connection_probe(first, monkeypatch, entered, release)
|
|
_connection_probe(alias, monkeypatch, alias_entered)
|
|
running = _start(first.ensure_storage_pools)
|
|
assert entered.wait(1)
|
|
try:
|
|
with pytest.raises(module.VMError) as denied:
|
|
alias.ensure_storage_pools()
|
|
assert denied.value.status == 409
|
|
assert denied.value.error_code == "coordination_busy"
|
|
assert not alias_entered.is_set()
|
|
finally:
|
|
release.set()
|
|
with pytest.raises(_ReachedConnection):
|
|
running.result(timeout=1)
|
|
|
|
|
|
def test_snapshot_excludes_same_vm_power_but_not_other_vm_or_pure_reads(services, monkeypatch):
|
|
module, make = services
|
|
snapshot, power = make("host-a"), make("host-a")
|
|
vm_id = "00000000-0000-4000-8000-000000000001"
|
|
other_id = "00000000-0000-4000-8000-000000000002"
|
|
entered, release = threading.Event(), threading.Event()
|
|
power_entered = threading.Event()
|
|
_connection_probe(snapshot, monkeypatch, entered, release)
|
|
_connection_probe(power, monkeypatch, power_entered)
|
|
running = _start(lambda: snapshot.create_live_snapshot(vm_id, "a" * 32))
|
|
assert entered.wait(1)
|
|
try:
|
|
with pytest.raises(module.VMError) as denied:
|
|
power.action(vm_id, "poweroff")
|
|
assert denied.value.error_code == "coordination_busy"
|
|
assert not power_entered.is_set()
|
|
with pytest.raises(_ReachedConnection):
|
|
power.action(other_id, "poweroff")
|
|
with pytest.raises(_ReachedConnection):
|
|
power.get_vm_state(vm_id)
|
|
finally:
|
|
release.set()
|
|
with pytest.raises(_ReachedConnection):
|
|
running.result(timeout=1)
|
|
|
|
|
|
def test_scoped_production_guard_fails_closed_without_coordinator(services):
|
|
module, make = services
|
|
service = make("host-a", execution_scope="host-a")
|
|
with pytest.raises(module.VMError) as denied:
|
|
with service._operation_guard(storage=True):
|
|
pytest.fail("uncoordinated production write admitted")
|
|
assert denied.value.status == 503
|
|
assert denied.value.error_code == "coordination_unavailable"
|
|
assert module.RESOURCE_LOCKS.entry_count() == 0
|
|
|
|
|
|
def test_remote_uncertainty_releases_local_lock_but_retains_durable_resource(services, monkeypatch):
|
|
from test_durable_jobs import MemoryCAS
|
|
from virsh_panel import coordination
|
|
|
|
module, make = services
|
|
coordinator = coordination.DurableCoordinator(MemoryCAS())
|
|
monkeypatch.setattr(coordination, "get_default_coordinator", lambda: coordinator)
|
|
service = make("host-a", execution_scope="host-a")
|
|
vm_id = "00000000-0000-4000-8000-000000000001"
|
|
with pytest.raises(module.VMError, match="unconfirmed"):
|
|
with service._operation_guard(vm_id, storage=True):
|
|
raise module.VMError("unconfirmed", 504, error_code="libvirt_execution_uncertain")
|
|
assert module.RESOURCE_LOCKS.entry_count() == 0
|
|
claim = next(iter(coordinator.snapshot().values()))
|
|
assert claim["state"] == "uncertain"
|
|
assert claim["resources"] == ["storage:host-a", f"vm:host-a:{vm_id}"]
|
|
with pytest.raises(module.VMError) as denied:
|
|
with service._operation_guard(vm_id):
|
|
pytest.fail("unknown remote mutation replayed")
|
|
assert denied.value.status == 503
|
|
assert denied.value.error_code == "coordination_uncertain"
|
|
assert module.RESOURCE_LOCKS.entry_count() == 0
|
|
|
|
|
|
def test_accepted_shutdown_releases_durable_vm_claim_for_forceoff(monkeypatch):
|
|
from test_durable_jobs import MemoryCAS
|
|
from test_libvirt_service import VMServiceTests
|
|
from virsh_panel import coordination
|
|
|
|
case = VMServiceTests()
|
|
case.setUp()
|
|
vm = case.service.create_vm(case.spec())
|
|
coordinator = coordination.DurableCoordinator(MemoryCAS())
|
|
monkeypatch.setattr(coordination, "get_default_coordinator", lambda: coordinator)
|
|
case.backend.shutdown_stalls = True
|
|
|
|
receipt = case.service.action(vm["uuid"], "shutdown")
|
|
|
|
assert receipt == {"uuid": vm["uuid"], "action": "shutdown", "command_accepted": True}
|
|
assert case.backend.domains[0].isActive()
|
|
assert coordinator.snapshot() == {}
|
|
assert case.service_module.RESOURCE_LOCKS.entry_count() == 0
|
|
assert case.service.action(vm["uuid"], "poweroff")["command_accepted"] is True
|
|
assert not case.backend.domains[0].isActive()
|
|
assert case.backend.shutdown_calls == case.backend.destroy_calls == 1
|
|
assert coordinator.snapshot() == {}
|
|
|
|
|
|
def test_power_response_loss_still_retains_durable_vm_claim(monkeypatch):
|
|
from test_durable_jobs import MemoryCAS
|
|
from test_libvirt_service import FakeLibvirtError, VMServiceTests
|
|
from virsh_panel import coordination
|
|
|
|
case = VMServiceTests()
|
|
case.setUp()
|
|
vm = case.service.create_vm(case.spec())
|
|
coordinator = coordination.DurableCoordinator(MemoryCAS())
|
|
monkeypatch.setattr(coordination, "get_default_coordinator", lambda: coordinator)
|
|
domain = case.backend.domains[0]
|
|
shutdown = domain.shutdown
|
|
|
|
def commit_then_lose_response():
|
|
shutdown()
|
|
raise FakeLibvirtError("command response lost")
|
|
|
|
monkeypatch.setattr(domain, "shutdown", commit_then_lose_response)
|
|
with pytest.raises(case.error_type) as lost:
|
|
case.service.action(vm["uuid"], "shutdown")
|
|
assert lost.value.error_code == "power_action_result_uncertain"
|
|
assert not domain.isActive()
|
|
claim = next(iter(coordinator.snapshot().values()))
|
|
assert claim["state"] == "uncertain"
|
|
assert case.service_module.RESOURCE_LOCKS.entry_count() == 0
|
|
# A current shutoff state does not supply the missing command receipt.
|
|
starts_before = case.backend.domain_starts
|
|
with pytest.raises(case.error_type) as blocked:
|
|
case.service.action(vm["uuid"], "start")
|
|
assert blocked.value.error_code == "coordination_uncertain"
|
|
assert case.backend.domain_starts == starts_before
|