|
|
|
|
"""Real ASGI cancellation and durable admission, without a remote host."""
|
|
|
|
|
|
|
|
|
|
from __future__ import annotations
|
|
|
|
|
|
|
|
|
|
import asyncio
|
|
|
|
|
import base64
|
|
|
|
|
import gc
|
|
|
|
|
import json
|
|
|
|
|
import threading
|
|
|
|
|
import time
|
|
|
|
|
import uuid
|
|
|
|
|
from contextlib import asynccontextmanager
|
|
|
|
|
from types import SimpleNamespace
|
|
|
|
|
|
|
|
|
|
import pytest
|
|
|
|
|
|
|
|
|
|
from test_engine_boundary import (
|
|
|
|
|
CREATE_BODY, SECRET, SUPER, VM_ID, DatabaseTemplateStore, EngineApplication,
|
|
|
|
|
EngineComponents, ExternalOwnership, FakeRemoteManager, FakeUpdater,
|
|
|
|
|
HostRegistry, HostStore, MemoryStore, Service, Settings, cloud_engine_request,
|
|
|
|
|
create_app,
|
|
|
|
|
)
|
|
|
|
|
from virsh_panel.database import DatabaseError
|
|
|
|
|
from virsh_panel.metrics import host_scope
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class _NoMetrics:
|
|
|
|
|
def __init__(self, *_args, **_kwargs):
|
|
|
|
|
pass
|
|
|
|
|
|
|
|
|
|
def start(self):
|
|
|
|
|
pass
|
|
|
|
|
|
|
|
|
|
def stop(self):
|
|
|
|
|
pass
|
|
|
|
|
|
|
|
|
|
async def aclose(self):
|
|
|
|
|
return True
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
|
|
|
def local_backends(monkeypatch):
|
|
|
|
|
monkeypatch.setattr("virsh_panel.app.MetricsCollector", _NoMetrics)
|
|
|
|
|
# These services are injected fakes. Their lifecycle must not close an
|
|
|
|
|
# unrelated process-wide RPC runtime created by another test fixture.
|
|
|
|
|
monkeypatch.setattr("virsh_panel.libvirt_rpc.get_runtime", lambda: None)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@asynccontextmanager
|
|
|
|
|
async def running_engine(directory, store=None):
|
|
|
|
|
"""Exercise startup, requests and shutdown on the same owning loop."""
|
|
|
|
|
directory.mkdir(parents=True, exist_ok=True)
|
|
|
|
|
store = MemoryStore() if store is None else store
|
|
|
|
|
logs = MemoryStore()
|
|
|
|
|
config = {
|
|
|
|
|
"service_secret": base64.b64encode(SECRET).decode(),
|
|
|
|
|
"data_dir": str(directory),
|
|
|
|
|
"shutdown_timeout_seconds": 0.5,
|
|
|
|
|
}
|
|
|
|
|
services = {}
|
|
|
|
|
|
|
|
|
|
def factory(host_id, user_isos, cloud_templates):
|
|
|
|
|
host_directory = directory / host_id
|
|
|
|
|
host_directory.mkdir(exist_ok=True)
|
|
|
|
|
scoped = HostStore(store, host_id)
|
|
|
|
|
manager = FakeRemoteManager()
|
|
|
|
|
manager.host_id = host_id
|
|
|
|
|
manager.profile = {
|
|
|
|
|
"host_id": host_id, "host": host_id + ".example.test", "port": 22,
|
|
|
|
|
"fingerprint": "SHA256:example", "key_locked": False,
|
|
|
|
|
"storage_pool": "images", "iso_pool": "iso",
|
|
|
|
|
"rescue_iso_pool": "rescue", "default_network": "default",
|
|
|
|
|
"libvirt_uri": "fake", "ssh_target": "fake",
|
|
|
|
|
"ssh_config_file": "/fake/config", "ssh_known_hosts_file": "/fake/known_hosts",
|
|
|
|
|
}
|
|
|
|
|
service = services.setdefault(host_id, Service())
|
|
|
|
|
components = EngineComponents(
|
|
|
|
|
SimpleNamespace(users=ExternalOwnership(scoped)), manager,
|
|
|
|
|
DatabaseTemplateStore(scoped), scoped, logs, user_isos, cloud_templates,
|
|
|
|
|
)
|
|
|
|
|
app = create_app(
|
|
|
|
|
Settings(data_dir=host_directory), engine_components=components,
|
|
|
|
|
service_factory=lambda _profile: service,
|
|
|
|
|
)
|
|
|
|
|
return components, app
|
|
|
|
|
|
|
|
|
|
registry = HostRegistry(config, store, logs, factory=factory)
|
|
|
|
|
updater = FakeUpdater()
|
|
|
|
|
updater.token = "test-only-token"
|
|
|
|
|
|
|
|
|
|
async def active(*_args):
|
|
|
|
|
return True
|
|
|
|
|
|
|
|
|
|
engine = EngineApplication(
|
|
|
|
|
config, store, logs, registry=registry, introspection=active, updater=updater,
|
|
|
|
|
)
|
|
|
|
|
engine._service = lambda profile: services[profile["host_id"]]
|
|
|
|
|
incoming, outgoing = asyncio.Queue(), asyncio.Queue()
|
|
|
|
|
lifecycle = asyncio.create_task(engine._lifespan(incoming.get, outgoing.put))
|
|
|
|
|
await incoming.put({"type": "lifespan.startup"})
|
|
|
|
|
started = await asyncio.wait_for(outgoing.get(), 3)
|
|
|
|
|
assert started["type"] == "lifespan.startup.complete", started
|
|
|
|
|
try:
|
|
|
|
|
yield engine, store, services
|
|
|
|
|
finally:
|
|
|
|
|
await incoming.put({"type": "lifespan.shutdown"})
|
|
|
|
|
stopped = await asyncio.wait_for(outgoing.get(), 3)
|
|
|
|
|
assert stopped["type"] == "lifespan.shutdown.complete", stopped
|
|
|
|
|
await lifecycle
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
async def request(engine, method, target, value=None, *, operation_id=None):
|
|
|
|
|
actor = dict(SUPER)
|
|
|
|
|
if operation_id is not None:
|
|
|
|
|
actor.update(operation_id=operation_id, can_provision=True)
|
|
|
|
|
body = b"" if value is None else json.dumps(value).encode()
|
|
|
|
|
return await cloud_engine_request(
|
|
|
|
|
engine, method, target, body, actor,
|
|
|
|
|
headers={"Content-Type": "application/json"},
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
async def wait_until(predicate, *, timeout=2.0):
|
|
|
|
|
deadline = time.monotonic() + timeout
|
|
|
|
|
while not predicate():
|
|
|
|
|
if time.monotonic() >= deadline:
|
|
|
|
|
pytest.fail("test condition did not become true before its deadline")
|
|
|
|
|
await asyncio.sleep(0.005)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
async def prepare_hosts(engine, *hosts):
|
|
|
|
|
for host in hosts:
|
|
|
|
|
status, payload = await request(engine, "GET", f"/internal/v1/hosts/{host}/vms")
|
|
|
|
|
assert status == 200, payload
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def vm_path(host, resource="actions"):
|
|
|
|
|
return f"/internal/v1/hosts/{host}/vms/{VM_ID}/{resource}"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_stalled_host_mutation_does_not_serialize_other_hosts(tmp_path):
|
|
|
|
|
async def run():
|
|
|
|
|
async with running_engine(tmp_path) as (engine, _store, services):
|
|
|
|
|
await prepare_hosts(engine, "host-a", "host-b")
|
|
|
|
|
entered, release = threading.Event(), threading.Event()
|
|
|
|
|
action = services["host-a"].action
|
|
|
|
|
|
|
|
|
|
def stalled_action(*args):
|
|
|
|
|
entered.set()
|
|
|
|
|
assert release.wait(3), "test did not release host A"
|
|
|
|
|
return action(*args)
|
|
|
|
|
|
|
|
|
|
services["host-a"].action = stalled_action
|
|
|
|
|
first = asyncio.create_task(request(
|
|
|
|
|
engine, "POST", vm_path("host-a"), {"action": "reset"},
|
|
|
|
|
operation_id="coord-host-a-reset",
|
|
|
|
|
))
|
|
|
|
|
try:
|
|
|
|
|
await wait_until(entered.is_set)
|
|
|
|
|
read_status, _ = await asyncio.wait_for(request(
|
|
|
|
|
engine, "GET", f"/internal/v1/hosts/host-a/vms"), 0.75)
|
|
|
|
|
assert read_status == 200
|
|
|
|
|
other_status, other = await asyncio.wait_for(request(
|
|
|
|
|
engine, "POST", vm_path("host-b"), {"action": "reset"},
|
|
|
|
|
operation_id="coord-host-b-reset",
|
|
|
|
|
), 0.75)
|
|
|
|
|
assert other_status == 200, other
|
|
|
|
|
assert not release.is_set()
|
|
|
|
|
assert not first.done()
|
|
|
|
|
finally:
|
|
|
|
|
release.set()
|
|
|
|
|
await asyncio.wait_for(first, 3)
|
|
|
|
|
|
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_stalled_vm_action_does_not_serialize_other_vms_on_same_host(tmp_path):
|
|
|
|
|
async def run():
|
|
|
|
|
async with running_engine(tmp_path) as (engine, _store, services):
|
|
|
|
|
await prepare_hosts(engine, "host-a")
|
|
|
|
|
service = services["host-a"]
|
|
|
|
|
other_id = "22222222-2222-4222-8222-222222222222"
|
|
|
|
|
other_vm = {**service.vm, "uuid": other_id, "name": "second-vm"}
|
|
|
|
|
entered, release = threading.Event(), threading.Event()
|
|
|
|
|
get_vm, action = service.get_vm, service.action
|
|
|
|
|
|
|
|
|
|
def get_either_vm(vm_id):
|
|
|
|
|
if vm_id == other_id:
|
|
|
|
|
service.calls.append(("get_vm", vm_id))
|
|
|
|
|
return other_vm.copy()
|
|
|
|
|
return get_vm(vm_id)
|
|
|
|
|
|
|
|
|
|
def stalled_action(vm_id, target):
|
|
|
|
|
if vm_id == VM_ID:
|
|
|
|
|
entered.set()
|
|
|
|
|
assert release.wait(3), "test did not release the first VM"
|
|
|
|
|
return action(vm_id, target)
|
|
|
|
|
assert vm_id == other_id
|
|
|
|
|
service.calls.append(("action", vm_id, target))
|
|
|
|
|
return other_vm.copy()
|
|
|
|
|
|
|
|
|
|
service.get_vm = get_either_vm
|
|
|
|
|
service.action = stalled_action
|
|
|
|
|
first = asyncio.create_task(request(
|
|
|
|
|
engine, "POST", vm_path("host-a"), {"action": "reset"},
|
|
|
|
|
operation_id="coord-same-host-first-vm",
|
|
|
|
|
))
|
|
|
|
|
try:
|
|
|
|
|
await wait_until(entered.is_set)
|
|
|
|
|
other_status, other = await asyncio.wait_for(request(
|
|
|
|
|
engine, "POST", f"/internal/v1/hosts/host-a/vms/{other_id}/actions",
|
|
|
|
|
{"action": "reset"}, operation_id="coord-same-host-second-vm",
|
|
|
|
|
), 0.75)
|
|
|
|
|
assert other_status == 200, other
|
|
|
|
|
assert other["uuid"] == other_id
|
|
|
|
|
assert ("action", other_id, "reset") in service.calls
|
|
|
|
|
assert not first.done()
|
|
|
|
|
assert not release.is_set()
|
|
|
|
|
finally:
|
|
|
|
|
release.set()
|
|
|
|
|
first_status, first_result = await asyncio.wait_for(first, 3)
|
|
|
|
|
assert first_status == 200, first_result
|
|
|
|
|
|
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_http_cancellation_keeps_executor_vm_claim_and_update_gate(tmp_path):
|
|
|
|
|
async def run():
|
|
|
|
|
async with running_engine(tmp_path) as (engine, store, services):
|
|
|
|
|
await prepare_hosts(engine, "host-a")
|
|
|
|
|
service = services["host-a"]
|
|
|
|
|
entered, release = threading.Event(), threading.Event()
|
|
|
|
|
finished = threading.Event()
|
|
|
|
|
suspend = service.network_suspend
|
|
|
|
|
calls = []
|
|
|
|
|
|
|
|
|
|
def stalled_suspend(*args):
|
|
|
|
|
calls.append(args)
|
|
|
|
|
entered.set()
|
|
|
|
|
assert release.wait(3), "test did not release suspended worker"
|
|
|
|
|
try:
|
|
|
|
|
return suspend(*args)
|
|
|
|
|
finally:
|
|
|
|
|
finished.set()
|
|
|
|
|
|
|
|
|
|
service.network_suspend = stalled_suspend
|
|
|
|
|
op_id = "coord-cancelled-suspend"
|
|
|
|
|
path = vm_path("host-a", "network-suspend")
|
|
|
|
|
first = asyncio.create_task(request(
|
|
|
|
|
engine, "POST", path, {}, operation_id=op_id))
|
|
|
|
|
try:
|
|
|
|
|
await wait_until(entered.is_set)
|
|
|
|
|
first.cancel()
|
|
|
|
|
with pytest.raises(asyncio.CancelledError):
|
|
|
|
|
await first
|
|
|
|
|
peek_status, peek = await request(
|
|
|
|
|
engine, "GET", f"/internal/v1/operations/{op_id}?reconcile=false")
|
|
|
|
|
assert peek_status == 200
|
|
|
|
|
assert peek["executor_active"] is True
|
|
|
|
|
|
|
|
|
|
retry_status, retry = await request(
|
|
|
|
|
engine, "POST", path, {}, operation_id=op_id)
|
|
|
|
|
assert retry_status == 202, retry
|
|
|
|
|
assert len(calls) == 1
|
|
|
|
|
conflict_status, conflict = await request(
|
|
|
|
|
engine, "POST", vm_path("host-a", "network-resume"), {},
|
|
|
|
|
operation_id="coord-opposite-after-cancel",
|
|
|
|
|
)
|
|
|
|
|
assert conflict_status == 409, conflict
|
|
|
|
|
assert not any(call[0] == "network_resume" for call in service.calls)
|
|
|
|
|
update_status, update = await request(
|
|
|
|
|
engine, "POST", "/internal/v1/update/apply", {
|
|
|
|
|
"expected_current_commit": "a" * 40,
|
|
|
|
|
"expected_remote_commit": "b" * 40,
|
|
|
|
|
})
|
|
|
|
|
assert update_status == 409, update
|
|
|
|
|
finally:
|
|
|
|
|
release.set()
|
|
|
|
|
await wait_until(finished.is_set)
|
|
|
|
|
await wait_until(lambda: json.loads(
|
|
|
|
|
store.get("operation", op_id))["status"] == "succeeded")
|
|
|
|
|
status, operation = await request(
|
|
|
|
|
engine, "GET", f"/internal/v1/operations/{op_id}?reconcile=false")
|
|
|
|
|
assert status == 200
|
|
|
|
|
assert operation["executor_active"] is False
|
|
|
|
|
assert len(calls) == 1
|
|
|
|
|
|
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_same_operation_retry_cannot_execute_before_first_record_insert(tmp_path):
|
|
|
|
|
async def run():
|
|
|
|
|
async with running_engine(tmp_path) as (engine, store, services):
|
|
|
|
|
await prepare_hosts(engine, "host-a")
|
|
|
|
|
op_id = "coord-before-operation-insert"
|
|
|
|
|
entered, release = threading.Event(), threading.Event()
|
|
|
|
|
original_insert = store.insert
|
|
|
|
|
first_insert = True
|
|
|
|
|
|
|
|
|
|
def gated_insert(kind, key, value):
|
|
|
|
|
nonlocal first_insert
|
|
|
|
|
if kind == "operation" and key == op_id and first_insert:
|
|
|
|
|
first_insert = False
|
|
|
|
|
entered.set()
|
|
|
|
|
assert release.wait(3), "test did not release operation insert"
|
|
|
|
|
return original_insert(kind, key, value)
|
|
|
|
|
|
|
|
|
|
store.insert = gated_insert
|
|
|
|
|
body = {"action": "reset"}
|
|
|
|
|
first = asyncio.create_task(request(
|
|
|
|
|
engine, "POST", vm_path("host-a"), body, operation_id=op_id))
|
|
|
|
|
try:
|
|
|
|
|
await wait_until(entered.is_set)
|
|
|
|
|
assert store.get("operation", op_id) is None
|
|
|
|
|
assert engine.coordinator.operation_claim(op_id) is not None
|
|
|
|
|
status, retry = await asyncio.wait_for(request(
|
|
|
|
|
engine, "POST", vm_path("host-a"), body, operation_id=op_id), 0.75)
|
|
|
|
|
assert status == 409, retry
|
|
|
|
|
assert retry["error_code"] == "coordination_busy"
|
|
|
|
|
assert not any(call[0] == "action" for call in services["host-a"].calls)
|
|
|
|
|
finally:
|
|
|
|
|
release.set()
|
|
|
|
|
first_status, first_result = await asyncio.wait_for(first, 3)
|
|
|
|
|
assert first_status == 200, first_result
|
|
|
|
|
assert sum(call[0] == "action" for call in services["host-a"].calls) == 1
|
|
|
|
|
|
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_pending_snapshot_keeps_vm_claim_until_background_receipt(tmp_path):
|
|
|
|
|
async def run():
|
|
|
|
|
async with running_engine(tmp_path) as (engine, _store, services):
|
|
|
|
|
await prepare_hosts(engine, "host-a")
|
|
|
|
|
service = services["host-a"]
|
|
|
|
|
service.vm.update(managed=True, os_source="cloud_template",
|
|
|
|
|
rescue_mode=False, rescue_staged=False)
|
|
|
|
|
entered, release = threading.Event(), threading.Event()
|
|
|
|
|
original = service.create_live_snapshot
|
|
|
|
|
|
|
|
|
|
def stalled_snapshot(*args, **kwargs):
|
|
|
|
|
entered.set()
|
|
|
|
|
assert release.wait(3), "test did not release snapshot worker"
|
|
|
|
|
return original(*args, **kwargs)
|
|
|
|
|
|
|
|
|
|
service.create_live_snapshot = stalled_snapshot
|
|
|
|
|
op_id = "coord-background-snapshot"
|
|
|
|
|
status, queued = await request(
|
|
|
|
|
engine, "POST", vm_path("host-a", "snapshots"), {
|
|
|
|
|
"snapshot_key": "a" * 32, "name": "coordination recovery point",
|
|
|
|
|
"consistency_mode": "crash",
|
|
|
|
|
}, operation_id=op_id,
|
|
|
|
|
)
|
|
|
|
|
assert status == 202, queued
|
|
|
|
|
try:
|
|
|
|
|
await wait_until(entered.is_set)
|
|
|
|
|
assert engine.coordinator.operation_claim(op_id) is not None
|
|
|
|
|
peek_status, peek = await request(
|
|
|
|
|
engine, "GET", f"/internal/v1/operations/{op_id}?reconcile=false")
|
|
|
|
|
assert peek_status == 200, peek
|
|
|
|
|
assert peek["executor_active"] is True
|
|
|
|
|
conflict_status, conflict = await request(
|
|
|
|
|
engine, "POST", vm_path("host-a"), {"action": "reset"},
|
|
|
|
|
operation_id="coord-conflict-with-snapshot",
|
|
|
|
|
)
|
|
|
|
|
assert conflict_status == 409, conflict
|
|
|
|
|
assert not any(call[0] == "action" for call in service.calls)
|
|
|
|
|
finally:
|
|
|
|
|
release.set()
|
|
|
|
|
await wait_until(lambda: engine.coordinator.operation_claim(op_id) is None)
|
|
|
|
|
status, completed = await request(
|
|
|
|
|
engine, "GET", f"/internal/v1/operations/{op_id}")
|
|
|
|
|
assert status == 200, completed
|
|
|
|
|
assert completed["status"] == "succeeded"
|
|
|
|
|
assert completed["result"]["created"] is True
|
|
|
|
|
assert sum(call[0] == "create_live_snapshot" for call in service.calls) == 1
|
|
|
|
|
|
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class _FailedResultStore(MemoryStore):
|
|
|
|
|
failed_operation = "coord-result-save-failure"
|
|
|
|
|
reject_result = True
|
|
|
|
|
|
|
|
|
|
def set(self, kind, key, value):
|
|
|
|
|
if (self.reject_result and kind == "operation"
|
|
|
|
|
and key == self.failed_operation
|
|
|
|
|
and json.loads(value).get("status") == "succeeded"):
|
|
|
|
|
raise DatabaseError("injected final operation persistence failure")
|
|
|
|
|
super().set(kind, key, value)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.mark.parametrize("restart", [False, True])
|
|
|
|
|
def test_result_save_failure_retains_unknown_claim_and_never_replays(tmp_path, restart):
|
|
|
|
|
async def check_blocked(engine, services, op_id):
|
|
|
|
|
await prepare_hosts(engine, "host-a")
|
|
|
|
|
before = list(services["host-a"].calls)
|
|
|
|
|
for method, path, value in (
|
|
|
|
|
("GET", f"/internal/v1/operations/{op_id}", None),
|
|
|
|
|
("POST", vm_path("host-a"), {"action": "reset"}),
|
|
|
|
|
):
|
|
|
|
|
status, outcome = await request(
|
|
|
|
|
engine, method, path, value,
|
|
|
|
|
operation_id=op_id if method == "POST" else None,
|
|
|
|
|
)
|
|
|
|
|
assert status == (200 if method == "GET" else 202), outcome
|
|
|
|
|
assert outcome["status"] == "uncertain"
|
|
|
|
|
assert outcome["safe_to_refund"] is False
|
|
|
|
|
assert services["host-a"].calls == before
|
|
|
|
|
changed_status, changed = await request(
|
|
|
|
|
engine, "POST", vm_path("host-a"), {"action": "poweroff"},
|
|
|
|
|
operation_id=op_id,
|
|
|
|
|
)
|
|
|
|
|
assert changed_status == 409, changed
|
|
|
|
|
assert services["host-a"].calls == before
|
|
|
|
|
status, blocked = await request(
|
|
|
|
|
engine, "POST", vm_path("host-a"), {"action": "poweroff"},
|
|
|
|
|
operation_id="coord-new-conflicting-operation",
|
|
|
|
|
)
|
|
|
|
|
assert status == 503, blocked
|
|
|
|
|
assert services["host-a"].calls == before
|
|
|
|
|
|
|
|
|
|
async def run():
|
|
|
|
|
store = _FailedResultStore()
|
|
|
|
|
op_id = store.failed_operation
|
|
|
|
|
async with running_engine(tmp_path, store) as (engine, _store, services):
|
|
|
|
|
await prepare_hosts(engine, "host-a")
|
|
|
|
|
status, outcome = await request(
|
|
|
|
|
engine, "POST", vm_path("host-a"), {"action": "reset"},
|
|
|
|
|
operation_id=op_id,
|
|
|
|
|
)
|
|
|
|
|
assert status >= 500, outcome
|
|
|
|
|
assert sum(call[0] == "action" for call in services["host-a"].calls) == 1
|
|
|
|
|
store.reject_result = False
|
|
|
|
|
if not restart:
|
|
|
|
|
await check_blocked(engine, services, op_id)
|
|
|
|
|
if restart:
|
|
|
|
|
async with running_engine(tmp_path, store) as (engine, _store, services):
|
|
|
|
|
await check_blocked(engine, services, op_id)
|
|
|
|
|
|
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.mark.parametrize("legacy", [False, True])
|
|
|
|
|
def test_restarted_running_create_without_error_code_cannot_refund_or_release_ip(tmp_path, legacy):
|
|
|
|
|
"""A crash can leave only the old owner claim and mutation checkpoint."""
|
|
|
|
|
async def run():
|
|
|
|
|
store = MemoryStore()
|
|
|
|
|
op_id = "coord-crashed-create"
|
|
|
|
|
async with running_engine(tmp_path, store) as (engine, _store, _services):
|
|
|
|
|
status, created = await request(
|
|
|
|
|
engine, "POST", "/internal/v1/hosts/host-a/vms", CREATE_BODY,
|
|
|
|
|
operation_id=op_id,
|
|
|
|
|
)
|
|
|
|
|
assert status == 201, created
|
|
|
|
|
row = json.loads(store.get("operation", op_id))
|
|
|
|
|
vm_id = row["vm_uuid"]
|
|
|
|
|
component = engine.registry.components["host-a"]
|
|
|
|
|
scope = host_scope(component.manager.profile)
|
|
|
|
|
credential_key = f"{scope}.{vm_id}"
|
|
|
|
|
ip_key = f"{scope}.192.0.2.42"
|
|
|
|
|
component.store.set("vm_credential", credential_key, "test-credential")
|
|
|
|
|
component.store.set("ip_claim", ip_key, json.dumps({"vm_uuid": vm_id}))
|
|
|
|
|
|
|
|
|
|
# Reproduce the durable state immediately after a write began but
|
|
|
|
|
# before its receipt/error was committed. There is deliberately
|
|
|
|
|
# no libvirt_execution_* error code to trigger the old shortcut.
|
|
|
|
|
row.update(status="running", result=None, created_at=1,
|
|
|
|
|
mutation_started=True, safe_to_refund=False)
|
|
|
|
|
row.pop("error_code", None)
|
|
|
|
|
row.pop("finished_at", None)
|
|
|
|
|
if legacy:
|
|
|
|
|
row.pop("execution_id", None)
|
|
|
|
|
store.set("operation", op_id, json.dumps(row))
|
|
|
|
|
owner_before_restart = engine.coordinator.owner_id
|
|
|
|
|
if not legacy:
|
|
|
|
|
engine.coordinator.claim(operation_id=op_id)
|
|
|
|
|
|
|
|
|
|
async with running_engine(tmp_path, store) as (engine, _store, services):
|
|
|
|
|
assert engine.coordinator.owner_id != owner_before_restart
|
|
|
|
|
await prepare_hosts(engine, "host-a")
|
|
|
|
|
service = services["host-a"]
|
|
|
|
|
service.absent = True
|
|
|
|
|
before = list(service.calls)
|
|
|
|
|
status, outcome = await request(
|
|
|
|
|
engine, "GET", f"/internal/v1/operations/{op_id}")
|
|
|
|
|
assert status == 200, outcome
|
|
|
|
|
assert outcome["status"] == "uncertain"
|
|
|
|
|
assert outcome["safe_to_refund"] is False
|
|
|
|
|
assert outcome["executor_active"] is False
|
|
|
|
|
assert "finished_at" not in outcome
|
|
|
|
|
repeated_status, repeated = await request(
|
|
|
|
|
engine, "POST", "/internal/v1/hosts/host-a/vms", CREATE_BODY,
|
|
|
|
|
operation_id=op_id,
|
|
|
|
|
)
|
|
|
|
|
assert repeated_status == 202, repeated
|
|
|
|
|
assert repeated["status"] == "uncertain"
|
|
|
|
|
assert service.calls == before
|
|
|
|
|
assert service.absent_checks == []
|
|
|
|
|
component = engine.registry.components["host-a"]
|
|
|
|
|
assert component.state.users.owns_vm(scope, vm_id, SUPER["user_id"])
|
|
|
|
|
assert component.store.get("vm_credential", credential_key) == "test-credential"
|
|
|
|
|
assert component.store.get("ip_claim", ip_key) is not None
|
|
|
|
|
if legacy:
|
|
|
|
|
conflict_status, conflict = await request(
|
|
|
|
|
engine, "POST", f"/internal/v1/hosts/host-a/vms/{vm_id}/actions",
|
|
|
|
|
{"action": "poweroff"}, operation_id="coord-new-after-legacy-crash",
|
|
|
|
|
)
|
|
|
|
|
assert conflict_status == 503, conflict
|
|
|
|
|
assert service.calls == before
|
|
|
|
|
|
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_vm_lock_registry_reclaims_idle_locks_without_splitting_waiters(tmp_path):
|
|
|
|
|
async def run():
|
|
|
|
|
async with running_engine(tmp_path) as (engine, _store, _services):
|
|
|
|
|
for number in range(1000):
|
|
|
|
|
async with engine._target_lock("host-a", str(uuid.UUID(int=number))):
|
|
|
|
|
pass
|
|
|
|
|
gc.collect()
|
|
|
|
|
assert len(engine.target_locks) == 0
|
|
|
|
|
|
|
|
|
|
entered = asyncio.Event()
|
|
|
|
|
release = asyncio.Event()
|
|
|
|
|
|
|
|
|
|
async def holder():
|
|
|
|
|
async with engine._target_lock("host-a", VM_ID):
|
|
|
|
|
entered.set()
|
|
|
|
|
await release.wait()
|
|
|
|
|
|
|
|
|
|
owned = asyncio.create_task(holder())
|
|
|
|
|
await entered.wait()
|
|
|
|
|
lock = engine._target_lock("host-a", VM_ID)
|
|
|
|
|
waiter = asyncio.create_task(lock.acquire())
|
|
|
|
|
await asyncio.sleep(0)
|
|
|
|
|
try:
|
|
|
|
|
gc.collect()
|
|
|
|
|
assert engine._target_lock("host-a", VM_ID) is lock
|
|
|
|
|
assert lock.locked()
|
|
|
|
|
assert not waiter.done()
|
|
|
|
|
waiter.cancel()
|
|
|
|
|
with pytest.raises(asyncio.CancelledError):
|
|
|
|
|
await waiter
|
|
|
|
|
assert engine._target_lock("host-a", VM_ID) is lock
|
|
|
|
|
finally:
|
|
|
|
|
release.set()
|
|
|
|
|
await owned
|
|
|
|
|
del lock, waiter, owned
|
|
|
|
|
gc.collect()
|
|
|
|
|
assert len(engine.target_locks) == 0
|
|
|
|
|
|
|
|
|
|
asyncio.run(run())
|