You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/engine/tests/test_durable_coordination.py

428 lines
18 KiB
Python

"""Cross-process admission using the production encrypted CAS implementation."""
from __future__ import annotations
import asyncio
from dataclasses import replace
import json
import multiprocessing
import sqlite3
import time
from cryptography.hazmat.primitives.ciphers.aead import AESGCM
import pytest
from virsh_panel import coordination
from virsh_panel.coordination import (
CoordinationError, DurableCoordinator, RECORD_KEY, RECORD_KIND,
configure_default_coordinator, current_execution, execution_context,
get_default_coordinator, resource_guard, update_current_claim,
)
from virsh_panel.database import DatabaseStore, pymysql
class SQLiteCursor:
def __init__(self, raw):
self.raw = raw.cursor()
def __enter__(self):
return self
def __exit__(self, *_args):
self.raw.close()
def execute(self, statement, values=()):
try:
return self.raw.execute(statement.replace("%s", "?").replace(" FOR UPDATE", ""), values)
except sqlite3.IntegrityError as exc:
raise pymysql.err.IntegrityError(1062, "duplicate test row") from exc
def fetchone(self):
row = self.raw.fetchone()
return dict(row) if row is not None else None
def fetchall(self):
return [dict(row) for row in self.raw.fetchall()]
class SQLiteConnection:
def __init__(self, path):
self.raw = sqlite3.connect(path, timeout=5.0, isolation_level=None)
self.raw.row_factory = sqlite3.Row
# The SQLite writer lock substitutes for MariaDB's row lock. Production
# get/insert/CAS/encryption methods execute unchanged.
self.raw.execute("BEGIN IMMEDIATE")
def cursor(self):
return SQLiteCursor(self.raw)
def commit(self):
self.raw.commit()
def rollback(self):
self.raw.rollback()
def close(self):
self.raw.close()
class EncryptedSQLiteStore(DatabaseStore):
RECORD_TABLE = "engine_records"
def __init__(self, path):
self.path = str(path)
self._cipher = AESGCM(b"a" * 32)
with sqlite3.connect(self.path, timeout=5.0) as connection:
connection.execute("CREATE TABLE IF NOT EXISTS engine_records ("
"record_type TEXT, record_key TEXT, encrypted_value BLOB, "
"PRIMARY KEY(record_type,record_key))")
def _connect(self):
return SQLiteConnection(self.path)
@pytest.fixture
def coordinator(tmp_path):
return DurableCoordinator(EncryptedSQLiteStore(tmp_path / "state.sqlite"))
def _claim_in_process(path, options, ready, release, output):
instance = DurableCoordinator(EncryptedSQLiteStore(path))
ready.wait(timeout=10)
try:
claim = instance.claim(**options)
except CoordinationError as exc:
output.put(("rejected", exc.error_code))
else:
output.put(("admitted", claim.token))
release.wait(timeout=10)
instance.release(claim)
@pytest.mark.parametrize("first,second,admitted", [
({"resources": ("vm:a:one",)}, {"resources": ("vm:a:one",)}, 1),
({"resources": ("vm:a:one",)}, {"resources": ("vm:b:one",)}, 2),
({"operation_id": "operation-a", "resources": ("vm:a:one",)},
{"operation_id": "operation-a", "resources": ("vm:b:one",)}, 1),
({"resources": ("vm:a:one",)}, {"maintenance": True}, 1),
])
def test_independent_processes_share_atomic_admission(tmp_path, first, second, admitted):
path = tmp_path / "state.sqlite"
EncryptedSQLiteStore(path)
context = multiprocessing.get_context("spawn")
ready, release, output = context.Barrier(2), context.Event(), context.Queue()
processes = [context.Process(target=_claim_in_process,
args=(str(path), options, ready, release, output))
for options in (first, second)]
try:
for process in processes:
process.start()
results = [output.get(timeout=10) for _ in processes]
assert sum(result[0] == "admitted" for result in results) == admitted
observer = DurableCoordinator(EncryptedSQLiteStore(path))
assert len(observer.snapshot()) == admitted
finally:
release.set()
for process in processes:
process.join(timeout=5)
if process.is_alive():
process.kill()
process.join(timeout=2)
output.close()
output.join_thread()
assert all(process.exitcode == 0 for process in processes)
assert observer.snapshot() == {}
def test_same_execution_handoff_preserves_operation_until_last_claim(coordinator):
request = coordinator.claim(operation_id="operation-one", resources=("vm:a:one",))
job = coordinator.claim(operation_id="operation-one", resources=request.resources,
execution_id=request.execution_id, kind="background")
assert coordinator.release(request)
assert coordinator.operation_claim("operation-one")["token"] == job.token
assert coordinator.retain(job, "libvirt_execution_uncertain")
with pytest.raises(CoordinationError) as raised:
coordinator.claim(resources=job.resources, execution_id=job.execution_id)
assert raised.value.error_code == "coordination_uncertain"
assert coordinator.release(job) is False
def test_operation_lookup_prefers_unknown_sibling(coordinator):
request = coordinator.claim(operation_id="operation-one")
job = coordinator.claim(operation_id="operation-one", execution_id=request.execution_id)
coordinator.retain(job)
assert coordinator.operation_claim("operation-one")["state"] == "uncertain"
assert coordinator.release(request)
assert coordinator.operation_claim("operation-one")["token"] == job.token
def test_maintenance_never_ignores_its_own_execution(coordinator):
request = coordinator.claim(resources=("vm:a:one",))
with pytest.raises(CoordinationError):
coordinator.claim(maintenance=True, execution_id=request.execution_id)
coordinator.release(request)
maintenance = coordinator.claim(maintenance=True)
with pytest.raises(CoordinationError):
coordinator.claim(resources=("vm:a:two",), execution_id=maintenance.execution_id)
assert coordinator.release(maintenance)
@pytest.mark.parametrize("uncertain", [False, True])
def test_restart_and_old_timestamps_never_grant_execution(coordinator, monkeypatch, uncertain):
claim = coordinator.claim(operation_id="old-operation", resources=("vm:a:one",))
if uncertain:
coordinator.retain(claim)
monkeypatch.setattr(coordination.time, "time", lambda: 99999999999)
restarted = DurableCoordinator(EncryptedSQLiteStore(coordinator.store.path))
assert restarted.has_work()
with pytest.raises(CoordinationError):
restarted.claim(resources=claim.resources)
with pytest.raises(CoordinationError):
restarted.claim(maintenance=True)
assert restarted.snapshot()[claim.token]["owner_id"] == claim.owner_id
def test_identity_and_resolution_prevent_stale_release(coordinator):
first = coordinator.claim(resources=("vm:a:one",))
assert coordinator.release(replace(first, owner_id="z" * 32)) is False
coordinator.retain(first)
assert coordinator.release(first) is False
with pytest.raises(ValueError):
coordinator.resolve(first, " ")
assert coordinator.resolve(first, "review/host-a/verified-completed")
second = coordinator.claim(resources=first.resources)
assert coordinator.release(first) is False
assert coordinator.resolve(first, "old review") is False
assert second.token in coordinator.snapshot()
document = json.loads(coordinator.store.get(RECORD_KIND, RECORD_KEY))
assert document["resolutions"][-1]["token"] == first.token
def test_refresh_metadata_and_resource_expansion_preserve_identity(coordinator, monkeypatch):
claim = coordinator.claim(operation_id="operation-one", metadata={"phase": "queued"})
original = coordinator.snapshot()[claim.token]
monkeypatch.setattr(coordination.time, "time", lambda: original["updated_at"] + 60)
expanded = coordinator.add_resources(claim, ("vm:a:one",))
assert expanded.token == claim.token and expanded.execution_id == claim.execution_id
assert expanded.resources == ("vm:a:one",)
assert coordinator.refresh(expanded)
assert coordinator.update_metadata(expanded, {"job_id": "maintenance-job"})
coordinator.retain(expanded)
assert coordinator.refresh(expanded)
current = coordinator.snapshot()[claim.token]
assert current["metadata"] == {"phase": "queued", "job_id": "maintenance-job"}
assert current["updated_at"] == original["updated_at"] + 60
assert current["state"] == "uncertain"
assert coordinator.release(expanded) is False
@pytest.mark.parametrize("uncertain", [False, True])
def test_resource_expansion_cannot_overlap_another_execution(coordinator, uncertain):
blocker = coordinator.claim(resources=("vm:a:one",))
if uncertain:
coordinator.retain(blocker)
candidate = coordinator.claim(metadata={"keep": True})
with pytest.raises(CoordinationError) as raised:
coordinator.add_resources(candidate, blocker.resources)
assert raised.value.status == (503 if uncertain else 409)
assert coordinator.snapshot()[candidate.token]["resources"] == []
assert coordinator.snapshot()[candidate.token]["metadata"] == {"keep": True}
def test_cas_retry_preserves_competing_claim(coordinator, monkeypatch):
compare = coordinator.store.compare_and_set
injected = []
other = DurableCoordinator(EncryptedSQLiteStore(coordinator.store.path))
def competing_claim(kind, key, expected, value):
if not injected:
injected.append(other.claim(resources=("vm:a:other",)))
return compare(kind, key, expected, value)
monkeypatch.setattr(coordinator.store, "compare_and_set", competing_claim)
claim = coordinator.claim(resources=("vm:a:one",))
assert set(coordinator.snapshot()) == {claim.token, injected[0].token}
def test_missing_cas_and_exhausted_contention_fail_closed(coordinator, monkeypatch):
with pytest.raises(ValueError, match="atomic database CAS"):
DurableCoordinator(object())
monkeypatch.setattr(coordinator.store, "compare_and_set", lambda *_args: False)
with pytest.raises(CoordinationError) as raised:
coordinator.claim(resources=("vm:a:one",))
assert raised.value.error_code == "coordination_unavailable"
assert raised.value.status == 503
assert coordinator.snapshot() == {}
def test_corrupt_record_is_not_reinitialized(coordinator):
coordinator.store.insert(RECORD_KIND, RECORD_KEY, '{"version":1,"claims":null}')
with pytest.raises(CoordinationError):
coordinator.claim(resources=("vm:a:one",))
assert coordinator.store.get(RECORD_KIND, RECORD_KEY) == '{"version":1,"claims":null}'
def test_capacity_and_normal_release_do_not_accumulate_entries(coordinator):
limited = DurableCoordinator(coordinator.store, max_claims=1)
for _index in range(40):
claim = limited.claim(resources=("vm:a:one",))
with pytest.raises(CoordinationError):
limited.claim(resources=("vm:b:one",))
assert limited.release(claim)
assert limited.snapshot() == {}
assert len(coordinator.store.get(RECORD_KIND, RECORD_KEY)) < 100
def test_context_flows_to_thread_and_nested_guards_release_individually(coordinator):
outer = coordinator.claim(operation_id="operation-one", resources=("vm:a:one",),
metadata={"job_id": "durable-job"})
def guarded_thread():
assert current_execution().claim.token == outer.token
with resource_guard(("storage:a", "vm:a:one", "storage:a")) as inner:
assert inner.execution_id == outer.execution_id
assert inner.operation_id == outer.operation_id
assert inner.metadata == {"job_id": "durable-job"}
assert inner.resources == ("storage:a", "vm:a:one")
assert current_execution().claim.token == inner.token
with resource_guard(("vm:a:one",)):
assert len(coordinator.snapshot()) == 3
assert current_execution().claim.token == outer.token
async def run():
with execution_context(coordinator, outer):
await asyncio.to_thread(guarded_thread)
asyncio.run(run())
assert current_execution() is None
assert set(coordinator.snapshot()) == {outer.token}
assert coordinator.release(outer)
def test_default_guard_is_reentrant_and_missing_default_fails_closed(coordinator):
original = get_default_coordinator()
try:
configure_default_coordinator(None)
with pytest.raises(CoordinationError):
with resource_guard(("vm:a:one",)):
pytest.fail("No durable coordinator")
configure_default_coordinator(coordinator)
with resource_guard(("vm:a:one",)) as first:
with resource_guard(("vm:a:one",)) as second:
assert first.execution_id == second.execution_id
assert not coordinator.has_work()
finally:
configure_default_coordinator(original)
@pytest.mark.parametrize("status,error_code,uncertain", [
(404, None, False), (500, None, True),
(504, "libvirt_execution_timeout", False),
(503, "libvirt_execution_stopped", False),
(504, "libvirt_execution_uncertain", True),
(503, "libvirt_execution_blocked", True),
])
def test_guard_retains_only_unconfirmed_execution(coordinator, status, error_code, uncertain):
class ServiceError(Exception):
pass
failure = ServiceError("failure")
failure.status, failure.error_code = status, error_code
outer = coordinator.claim()
with execution_context(coordinator, outer):
with pytest.raises(ServiceError):
with resource_guard(("vm:a:one",)):
raise failure
coordinator.release(outer)
remaining = coordinator.snapshot()
assert bool(remaining) is uncertain
if uncertain:
assert next(iter(remaining.values()))["state"] == "uncertain"
def test_cancelled_guard_keeps_remote_execution_ownership(coordinator):
outer = coordinator.claim()
with execution_context(coordinator, outer):
with pytest.raises(asyncio.CancelledError):
with resource_guard(("vm:a:one",)):
raise asyncio.CancelledError
coordinator.release(outer)
assert next(iter(coordinator.snapshot().values()))["state"] == "uncertain"
def test_guard_contention_has_bounded_wait(coordinator):
blocker = coordinator.claim(resources=("vm:a:one",))
other = coordinator.claim()
started = time.monotonic()
with execution_context(coordinator, other):
with pytest.raises(CoordinationError) as raised:
with resource_guard((item for item in blocker.resources), timeout=0.05):
pytest.fail("Conflicting execution admitted")
assert 0.04 <= time.monotonic() - started < 0.5
assert raised.value.error_code == "coordination_busy"
assert set(coordinator.snapshot()) == {blocker.token, other.token}
def test_lost_claim_commit_receipt_keeps_durable_admission(coordinator, monkeypatch):
compare = coordinator.store.compare_and_set
def commit_then_disconnect(*args):
assert compare(*args)
raise ConnectionError("injected lost commit receipt")
monkeypatch.setattr(coordinator.store, "compare_and_set", commit_then_disconnect)
with pytest.raises(CoordinationError) as raised:
coordinator.claim(operation_id="lost-receipt", resources=("vm:a:one",))
assert raised.value.error_code == "coordination_unavailable"
restarted = DurableCoordinator(EncryptedSQLiteStore(coordinator.store.path))
assert restarted.operation_claim("lost-receipt") is not None
with pytest.raises(CoordinationError):
restarted.claim(resources=("vm:a:one",))
def test_guard_release_database_error_cannot_erase_ownership(coordinator, monkeypatch):
outer = coordinator.claim()
with execution_context(coordinator, outer):
with pytest.raises(CoordinationError) as raised:
with resource_guard(("vm:a:one",)):
def unavailable(*_args):
raise ConnectionError("injected database outage")
monkeypatch.setattr(coordinator.store, "compare_and_set", unavailable)
assert raised.value.error_code == "coordination_unavailable"
assert len(coordinator.snapshot()) == 2
def test_unknown_conflict_takes_precedence_over_an_active_sibling(coordinator):
active = coordinator.claim(operation_id="operation-one", resources=("vm:a:one",))
uncertain = coordinator.claim(operation_id="operation-one", resources=active.resources,
execution_id=active.execution_id)
coordinator.retain(uncertain)
with pytest.raises(CoordinationError) as raised:
coordinator.claim(resources=active.resources)
assert raised.value.error_code == "coordination_uncertain"
def test_current_claim_can_only_be_replaced_by_same_identity(coordinator):
original = coordinator.claim(operation_id="create-one")
expanded = coordinator.add_resources(original, ("vm:a:created",))
with execution_context(coordinator, original):
update_current_claim(expanded)
assert current_execution().claim.resources == ("vm:a:created",)
with pytest.raises(ValueError):
update_current_claim(replace(expanded, token="other" * 8))
assert current_execution() is None
def test_safe_error_cannot_hide_a_concurrently_retained_guard(coordinator):
class NotFound(Exception):
status = 404
outer = coordinator.claim()
with execution_context(coordinator, outer):
with pytest.raises(CoordinationError) as raised:
with resource_guard(("vm:a:one",)) as claim:
coordinator.retain(claim)
raise NotFound()
assert raised.value.error_code == "coordination_uncertain"
assert coordinator.snapshot()[claim.token]["state"] == "uncertain"