You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/engine/tests/test_engine_coordination.py

553 lines
23 KiB
Python

"""Real ASGI cancellation and durable admission, without a remote host."""
from __future__ import annotations
import asyncio
import base64
import gc
import json
import threading
import time
import uuid
from contextlib import asynccontextmanager
from types import SimpleNamespace
import pytest
from test_engine_boundary import (
CREATE_BODY, SECRET, SUPER, VM_ID, DatabaseTemplateStore, EngineApplication,
EngineComponents, ExternalOwnership, FakeRemoteManager, FakeUpdater,
HostRegistry, HostStore, MemoryStore, Service, Settings, cloud_engine_request,
create_app,
)
from virsh_panel.database import DatabaseError
from virsh_panel.metrics import host_scope
class _NoMetrics:
def __init__(self, *_args, **_kwargs):
pass
def start(self):
pass
def stop(self):
pass
async def aclose(self):
return True
@pytest.fixture(autouse=True)
def local_backends(monkeypatch):
monkeypatch.setattr("virsh_panel.app.MetricsCollector", _NoMetrics)
# These services are injected fakes. Their lifecycle must not close an
# unrelated process-wide RPC runtime created by another test fixture.
monkeypatch.setattr("virsh_panel.libvirt_rpc.get_runtime", lambda: None)
@asynccontextmanager
async def running_engine(directory, store=None):
"""Exercise startup, requests and shutdown on the same owning loop."""
directory.mkdir(parents=True, exist_ok=True)
store = MemoryStore() if store is None else store
logs = MemoryStore()
config = {
"service_secret": base64.b64encode(SECRET).decode(),
"data_dir": str(directory),
"shutdown_timeout_seconds": 0.5,
}
services = {}
def factory(host_id, user_isos, cloud_templates):
host_directory = directory / host_id
host_directory.mkdir(exist_ok=True)
scoped = HostStore(store, host_id)
manager = FakeRemoteManager()
manager.host_id = host_id
manager.profile = {
"host_id": host_id, "host": host_id + ".example.test", "port": 22,
"fingerprint": "SHA256:example", "key_locked": False,
"storage_pool": "images", "iso_pool": "iso",
"rescue_iso_pool": "rescue", "default_network": "default",
"libvirt_uri": "fake", "ssh_target": "fake",
"ssh_config_file": "/fake/config", "ssh_known_hosts_file": "/fake/known_hosts",
}
service = services.setdefault(host_id, Service())
components = EngineComponents(
SimpleNamespace(users=ExternalOwnership(scoped)), manager,
DatabaseTemplateStore(scoped), scoped, logs, user_isos, cloud_templates,
)
app = create_app(
Settings(data_dir=host_directory), engine_components=components,
service_factory=lambda _profile: service,
)
return components, app
registry = HostRegistry(config, store, logs, factory=factory)
updater = FakeUpdater()
updater.token = "test-only-token"
async def active(*_args):
return True
engine = EngineApplication(
config, store, logs, registry=registry, introspection=active, updater=updater,
)
engine._service = lambda profile: services[profile["host_id"]]
incoming, outgoing = asyncio.Queue(), asyncio.Queue()
lifecycle = asyncio.create_task(engine._lifespan(incoming.get, outgoing.put))
await incoming.put({"type": "lifespan.startup"})
started = await asyncio.wait_for(outgoing.get(), 3)
assert started["type"] == "lifespan.startup.complete", started
try:
yield engine, store, services
finally:
await incoming.put({"type": "lifespan.shutdown"})
stopped = await asyncio.wait_for(outgoing.get(), 3)
assert stopped["type"] == "lifespan.shutdown.complete", stopped
await lifecycle
async def request(engine, method, target, value=None, *, operation_id=None):
actor = dict(SUPER)
if operation_id is not None:
actor.update(operation_id=operation_id, can_provision=True)
body = b"" if value is None else json.dumps(value).encode()
return await cloud_engine_request(
engine, method, target, body, actor,
headers={"Content-Type": "application/json"},
)
async def wait_until(predicate, *, timeout=2.0):
deadline = time.monotonic() + timeout
while not predicate():
if time.monotonic() >= deadline:
pytest.fail("test condition did not become true before its deadline")
await asyncio.sleep(0.005)
async def prepare_hosts(engine, *hosts):
for host in hosts:
status, payload = await request(engine, "GET", f"/internal/v1/hosts/{host}/vms")
assert status == 200, payload
def vm_path(host, resource="actions"):
return f"/internal/v1/hosts/{host}/vms/{VM_ID}/{resource}"
def test_stalled_host_mutation_does_not_serialize_other_hosts(tmp_path):
async def run():
async with running_engine(tmp_path) as (engine, _store, services):
await prepare_hosts(engine, "host-a", "host-b")
entered, release = threading.Event(), threading.Event()
action = services["host-a"].action
def stalled_action(*args):
entered.set()
assert release.wait(3), "test did not release host A"
return action(*args)
services["host-a"].action = stalled_action
first = asyncio.create_task(request(
engine, "POST", vm_path("host-a"), {"action": "reset"},
operation_id="coord-host-a-reset",
))
try:
await wait_until(entered.is_set)
read_status, _ = await asyncio.wait_for(request(
engine, "GET", f"/internal/v1/hosts/host-a/vms"), 0.75)
assert read_status == 200
other_status, other = await asyncio.wait_for(request(
engine, "POST", vm_path("host-b"), {"action": "reset"},
operation_id="coord-host-b-reset",
), 0.75)
assert other_status == 200, other
assert not release.is_set()
assert not first.done()
finally:
release.set()
await asyncio.wait_for(first, 3)
asyncio.run(run())
def test_stalled_vm_action_does_not_serialize_other_vms_on_same_host(tmp_path):
async def run():
async with running_engine(tmp_path) as (engine, _store, services):
await prepare_hosts(engine, "host-a")
service = services["host-a"]
other_id = "22222222-2222-4222-8222-222222222222"
other_vm = {**service.vm, "uuid": other_id, "name": "second-vm"}
entered, release = threading.Event(), threading.Event()
get_vm, action = service.get_vm, service.action
def get_either_vm(vm_id):
if vm_id == other_id:
service.calls.append(("get_vm", vm_id))
return other_vm.copy()
return get_vm(vm_id)
def stalled_action(vm_id, target):
if vm_id == VM_ID:
entered.set()
assert release.wait(3), "test did not release the first VM"
return action(vm_id, target)
assert vm_id == other_id
service.calls.append(("action", vm_id, target))
return other_vm.copy()
service.get_vm = get_either_vm
service.action = stalled_action
first = asyncio.create_task(request(
engine, "POST", vm_path("host-a"), {"action": "reset"},
operation_id="coord-same-host-first-vm",
))
try:
await wait_until(entered.is_set)
other_status, other = await asyncio.wait_for(request(
engine, "POST", f"/internal/v1/hosts/host-a/vms/{other_id}/actions",
{"action": "reset"}, operation_id="coord-same-host-second-vm",
), 0.75)
assert other_status == 200, other
assert other["uuid"] == other_id
assert ("action", other_id, "reset") in service.calls
assert not first.done()
assert not release.is_set()
finally:
release.set()
first_status, first_result = await asyncio.wait_for(first, 3)
assert first_status == 200, first_result
asyncio.run(run())
def test_http_cancellation_keeps_executor_vm_claim_and_update_gate(tmp_path):
async def run():
async with running_engine(tmp_path) as (engine, store, services):
await prepare_hosts(engine, "host-a")
service = services["host-a"]
entered, release = threading.Event(), threading.Event()
finished = threading.Event()
suspend = service.network_suspend
calls = []
def stalled_suspend(*args):
calls.append(args)
entered.set()
assert release.wait(3), "test did not release suspended worker"
try:
return suspend(*args)
finally:
finished.set()
service.network_suspend = stalled_suspend
op_id = "coord-cancelled-suspend"
path = vm_path("host-a", "network-suspend")
first = asyncio.create_task(request(
engine, "POST", path, {}, operation_id=op_id))
try:
await wait_until(entered.is_set)
first.cancel()
with pytest.raises(asyncio.CancelledError):
await first
peek_status, peek = await request(
engine, "GET", f"/internal/v1/operations/{op_id}?reconcile=false")
assert peek_status == 200
assert peek["executor_active"] is True
retry_status, retry = await request(
engine, "POST", path, {}, operation_id=op_id)
assert retry_status == 202, retry
assert len(calls) == 1
conflict_status, conflict = await request(
engine, "POST", vm_path("host-a", "network-resume"), {},
operation_id="coord-opposite-after-cancel",
)
assert conflict_status == 409, conflict
assert not any(call[0] == "network_resume" for call in service.calls)
update_status, update = await request(
engine, "POST", "/internal/v1/update/apply", {
"expected_current_commit": "a" * 40,
"expected_remote_commit": "b" * 40,
})
assert update_status == 409, update
finally:
release.set()
await wait_until(finished.is_set)
await wait_until(lambda: json.loads(
store.get("operation", op_id))["status"] == "succeeded")
status, operation = await request(
engine, "GET", f"/internal/v1/operations/{op_id}?reconcile=false")
assert status == 200
assert operation["executor_active"] is False
assert len(calls) == 1
asyncio.run(run())
def test_same_operation_retry_cannot_execute_before_first_record_insert(tmp_path):
async def run():
async with running_engine(tmp_path) as (engine, store, services):
await prepare_hosts(engine, "host-a")
op_id = "coord-before-operation-insert"
entered, release = threading.Event(), threading.Event()
original_insert = store.insert
first_insert = True
def gated_insert(kind, key, value):
nonlocal first_insert
if kind == "operation" and key == op_id and first_insert:
first_insert = False
entered.set()
assert release.wait(3), "test did not release operation insert"
return original_insert(kind, key, value)
store.insert = gated_insert
body = {"action": "reset"}
first = asyncio.create_task(request(
engine, "POST", vm_path("host-a"), body, operation_id=op_id))
try:
await wait_until(entered.is_set)
assert store.get("operation", op_id) is None
assert engine.coordinator.operation_claim(op_id) is not None
status, retry = await asyncio.wait_for(request(
engine, "POST", vm_path("host-a"), body, operation_id=op_id), 0.75)
assert status == 409, retry
assert retry["error_code"] == "coordination_busy"
assert not any(call[0] == "action" for call in services["host-a"].calls)
finally:
release.set()
first_status, first_result = await asyncio.wait_for(first, 3)
assert first_status == 200, first_result
assert sum(call[0] == "action" for call in services["host-a"].calls) == 1
asyncio.run(run())
def test_pending_snapshot_keeps_vm_claim_until_background_receipt(tmp_path):
async def run():
async with running_engine(tmp_path) as (engine, _store, services):
await prepare_hosts(engine, "host-a")
service = services["host-a"]
service.vm.update(managed=True, os_source="cloud_template",
rescue_mode=False, rescue_staged=False)
entered, release = threading.Event(), threading.Event()
original = service.create_live_snapshot
def stalled_snapshot(*args, **kwargs):
entered.set()
assert release.wait(3), "test did not release snapshot worker"
return original(*args, **kwargs)
service.create_live_snapshot = stalled_snapshot
op_id = "coord-background-snapshot"
status, queued = await request(
engine, "POST", vm_path("host-a", "snapshots"), {
"snapshot_key": "a" * 32, "name": "coordination recovery point",
"consistency_mode": "crash",
}, operation_id=op_id,
)
assert status == 202, queued
try:
await wait_until(entered.is_set)
assert engine.coordinator.operation_claim(op_id) is not None
peek_status, peek = await request(
engine, "GET", f"/internal/v1/operations/{op_id}?reconcile=false")
assert peek_status == 200, peek
assert peek["executor_active"] is True
conflict_status, conflict = await request(
engine, "POST", vm_path("host-a"), {"action": "reset"},
operation_id="coord-conflict-with-snapshot",
)
assert conflict_status == 409, conflict
assert not any(call[0] == "action" for call in service.calls)
finally:
release.set()
await wait_until(lambda: engine.coordinator.operation_claim(op_id) is None)
status, completed = await request(
engine, "GET", f"/internal/v1/operations/{op_id}")
assert status == 200, completed
assert completed["status"] == "succeeded"
assert completed["result"]["created"] is True
assert sum(call[0] == "create_live_snapshot" for call in service.calls) == 1
asyncio.run(run())
class _FailedResultStore(MemoryStore):
failed_operation = "coord-result-save-failure"
reject_result = True
def set(self, kind, key, value):
if (self.reject_result and kind == "operation"
and key == self.failed_operation
and json.loads(value).get("status") == "succeeded"):
raise DatabaseError("injected final operation persistence failure")
super().set(kind, key, value)
@pytest.mark.parametrize("restart", [False, True])
def test_result_save_failure_retains_unknown_claim_and_never_replays(tmp_path, restart):
async def check_blocked(engine, services, op_id):
await prepare_hosts(engine, "host-a")
before = list(services["host-a"].calls)
for method, path, value in (
("GET", f"/internal/v1/operations/{op_id}", None),
("POST", vm_path("host-a"), {"action": "reset"}),
):
status, outcome = await request(
engine, method, path, value,
operation_id=op_id if method == "POST" else None,
)
assert status == (200 if method == "GET" else 202), outcome
assert outcome["status"] == "uncertain"
assert outcome["safe_to_refund"] is False
assert services["host-a"].calls == before
changed_status, changed = await request(
engine, "POST", vm_path("host-a"), {"action": "poweroff"},
operation_id=op_id,
)
assert changed_status == 409, changed
assert services["host-a"].calls == before
status, blocked = await request(
engine, "POST", vm_path("host-a"), {"action": "poweroff"},
operation_id="coord-new-conflicting-operation",
)
assert status == 503, blocked
assert services["host-a"].calls == before
async def run():
store = _FailedResultStore()
op_id = store.failed_operation
async with running_engine(tmp_path, store) as (engine, _store, services):
await prepare_hosts(engine, "host-a")
status, outcome = await request(
engine, "POST", vm_path("host-a"), {"action": "reset"},
operation_id=op_id,
)
assert status >= 500, outcome
assert sum(call[0] == "action" for call in services["host-a"].calls) == 1
store.reject_result = False
if not restart:
await check_blocked(engine, services, op_id)
if restart:
async with running_engine(tmp_path, store) as (engine, _store, services):
await check_blocked(engine, services, op_id)
asyncio.run(run())
@pytest.mark.parametrize("legacy", [False, True])
def test_restarted_running_create_without_error_code_cannot_refund_or_release_ip(tmp_path, legacy):
"""A crash can leave only the old owner claim and mutation checkpoint."""
async def run():
store = MemoryStore()
op_id = "coord-crashed-create"
async with running_engine(tmp_path, store) as (engine, _store, _services):
status, created = await request(
engine, "POST", "/internal/v1/hosts/host-a/vms", CREATE_BODY,
operation_id=op_id,
)
assert status == 201, created
row = json.loads(store.get("operation", op_id))
vm_id = row["vm_uuid"]
component = engine.registry.components["host-a"]
scope = host_scope(component.manager.profile)
credential_key = f"{scope}.{vm_id}"
ip_key = f"{scope}.192.0.2.42"
component.store.set("vm_credential", credential_key, "test-credential")
component.store.set("ip_claim", ip_key, json.dumps({"vm_uuid": vm_id}))
# Reproduce the durable state immediately after a write began but
# before its receipt/error was committed. There is deliberately
# no libvirt_execution_* error code to trigger the old shortcut.
row.update(status="running", result=None, created_at=1,
mutation_started=True, safe_to_refund=False)
row.pop("error_code", None)
row.pop("finished_at", None)
if legacy:
row.pop("execution_id", None)
store.set("operation", op_id, json.dumps(row))
owner_before_restart = engine.coordinator.owner_id
if not legacy:
engine.coordinator.claim(operation_id=op_id)
async with running_engine(tmp_path, store) as (engine, _store, services):
assert engine.coordinator.owner_id != owner_before_restart
await prepare_hosts(engine, "host-a")
service = services["host-a"]
service.absent = True
before = list(service.calls)
status, outcome = await request(
engine, "GET", f"/internal/v1/operations/{op_id}")
assert status == 200, outcome
assert outcome["status"] == "uncertain"
assert outcome["safe_to_refund"] is False
assert outcome["executor_active"] is False
assert "finished_at" not in outcome
repeated_status, repeated = await request(
engine, "POST", "/internal/v1/hosts/host-a/vms", CREATE_BODY,
operation_id=op_id,
)
assert repeated_status == 202, repeated
assert repeated["status"] == "uncertain"
assert service.calls == before
assert service.absent_checks == []
component = engine.registry.components["host-a"]
assert component.state.users.owns_vm(scope, vm_id, SUPER["user_id"])
assert component.store.get("vm_credential", credential_key) == "test-credential"
assert component.store.get("ip_claim", ip_key) is not None
if legacy:
conflict_status, conflict = await request(
engine, "POST", f"/internal/v1/hosts/host-a/vms/{vm_id}/actions",
{"action": "poweroff"}, operation_id="coord-new-after-legacy-crash",
)
assert conflict_status == 503, conflict
assert service.calls == before
asyncio.run(run())
def test_vm_lock_registry_reclaims_idle_locks_without_splitting_waiters(tmp_path):
async def run():
async with running_engine(tmp_path) as (engine, _store, _services):
for number in range(1000):
async with engine._target_lock("host-a", str(uuid.UUID(int=number))):
pass
gc.collect()
assert len(engine.target_locks) == 0
entered = asyncio.Event()
release = asyncio.Event()
async def holder():
async with engine._target_lock("host-a", VM_ID):
entered.set()
await release.wait()
owned = asyncio.create_task(holder())
await entered.wait()
lock = engine._target_lock("host-a", VM_ID)
waiter = asyncio.create_task(lock.acquire())
await asyncio.sleep(0)
try:
gc.collect()
assert engine._target_lock("host-a", VM_ID) is lock
assert lock.locked()
assert not waiter.done()
waiter.cancel()
with pytest.raises(asyncio.CancelledError):
await waiter
assert engine._target_lock("host-a", VM_ID) is lock
finally:
release.set()
await owned
del lock, waiter, owned
gc.collect()
assert len(engine.target_locks) == 0
asyncio.run(run())