You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
428 lines
18 KiB
Python
428 lines
18 KiB
Python
"""Cross-process admission using the production encrypted CAS implementation."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
from dataclasses import replace
|
|
import json
|
|
import multiprocessing
|
|
import sqlite3
|
|
import time
|
|
|
|
from cryptography.hazmat.primitives.ciphers.aead import AESGCM
|
|
import pytest
|
|
|
|
from virsh_panel import coordination
|
|
from virsh_panel.coordination import (
|
|
CoordinationError, DurableCoordinator, RECORD_KEY, RECORD_KIND,
|
|
configure_default_coordinator, current_execution, execution_context,
|
|
get_default_coordinator, resource_guard, update_current_claim,
|
|
)
|
|
from virsh_panel.database import DatabaseStore, pymysql
|
|
|
|
|
|
class SQLiteCursor:
|
|
def __init__(self, raw):
|
|
self.raw = raw.cursor()
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *_args):
|
|
self.raw.close()
|
|
|
|
def execute(self, statement, values=()):
|
|
try:
|
|
return self.raw.execute(statement.replace("%s", "?").replace(" FOR UPDATE", ""), values)
|
|
except sqlite3.IntegrityError as exc:
|
|
raise pymysql.err.IntegrityError(1062, "duplicate test row") from exc
|
|
|
|
def fetchone(self):
|
|
row = self.raw.fetchone()
|
|
return dict(row) if row is not None else None
|
|
|
|
def fetchall(self):
|
|
return [dict(row) for row in self.raw.fetchall()]
|
|
|
|
|
|
class SQLiteConnection:
|
|
def __init__(self, path):
|
|
self.raw = sqlite3.connect(path, timeout=5.0, isolation_level=None)
|
|
self.raw.row_factory = sqlite3.Row
|
|
# The SQLite writer lock substitutes for MariaDB's row lock. Production
|
|
# get/insert/CAS/encryption methods execute unchanged.
|
|
self.raw.execute("BEGIN IMMEDIATE")
|
|
|
|
def cursor(self):
|
|
return SQLiteCursor(self.raw)
|
|
|
|
def commit(self):
|
|
self.raw.commit()
|
|
|
|
def rollback(self):
|
|
self.raw.rollback()
|
|
|
|
def close(self):
|
|
self.raw.close()
|
|
|
|
|
|
class EncryptedSQLiteStore(DatabaseStore):
|
|
RECORD_TABLE = "engine_records"
|
|
|
|
def __init__(self, path):
|
|
self.path = str(path)
|
|
self._cipher = AESGCM(b"a" * 32)
|
|
with sqlite3.connect(self.path, timeout=5.0) as connection:
|
|
connection.execute("CREATE TABLE IF NOT EXISTS engine_records ("
|
|
"record_type TEXT, record_key TEXT, encrypted_value BLOB, "
|
|
"PRIMARY KEY(record_type,record_key))")
|
|
|
|
def _connect(self):
|
|
return SQLiteConnection(self.path)
|
|
|
|
|
|
@pytest.fixture
|
|
def coordinator(tmp_path):
|
|
return DurableCoordinator(EncryptedSQLiteStore(tmp_path / "state.sqlite"))
|
|
|
|
|
|
def _claim_in_process(path, options, ready, release, output):
|
|
instance = DurableCoordinator(EncryptedSQLiteStore(path))
|
|
ready.wait(timeout=10)
|
|
try:
|
|
claim = instance.claim(**options)
|
|
except CoordinationError as exc:
|
|
output.put(("rejected", exc.error_code))
|
|
else:
|
|
output.put(("admitted", claim.token))
|
|
release.wait(timeout=10)
|
|
instance.release(claim)
|
|
|
|
|
|
@pytest.mark.parametrize("first,second,admitted", [
|
|
({"resources": ("vm:a:one",)}, {"resources": ("vm:a:one",)}, 1),
|
|
({"resources": ("vm:a:one",)}, {"resources": ("vm:b:one",)}, 2),
|
|
({"operation_id": "operation-a", "resources": ("vm:a:one",)},
|
|
{"operation_id": "operation-a", "resources": ("vm:b:one",)}, 1),
|
|
({"resources": ("vm:a:one",)}, {"maintenance": True}, 1),
|
|
])
|
|
def test_independent_processes_share_atomic_admission(tmp_path, first, second, admitted):
|
|
path = tmp_path / "state.sqlite"
|
|
EncryptedSQLiteStore(path)
|
|
context = multiprocessing.get_context("spawn")
|
|
ready, release, output = context.Barrier(2), context.Event(), context.Queue()
|
|
processes = [context.Process(target=_claim_in_process,
|
|
args=(str(path), options, ready, release, output))
|
|
for options in (first, second)]
|
|
try:
|
|
for process in processes:
|
|
process.start()
|
|
results = [output.get(timeout=10) for _ in processes]
|
|
assert sum(result[0] == "admitted" for result in results) == admitted
|
|
observer = DurableCoordinator(EncryptedSQLiteStore(path))
|
|
assert len(observer.snapshot()) == admitted
|
|
finally:
|
|
release.set()
|
|
for process in processes:
|
|
process.join(timeout=5)
|
|
if process.is_alive():
|
|
process.kill()
|
|
process.join(timeout=2)
|
|
output.close()
|
|
output.join_thread()
|
|
assert all(process.exitcode == 0 for process in processes)
|
|
assert observer.snapshot() == {}
|
|
|
|
|
|
def test_same_execution_handoff_preserves_operation_until_last_claim(coordinator):
|
|
request = coordinator.claim(operation_id="operation-one", resources=("vm:a:one",))
|
|
job = coordinator.claim(operation_id="operation-one", resources=request.resources,
|
|
execution_id=request.execution_id, kind="background")
|
|
assert coordinator.release(request)
|
|
assert coordinator.operation_claim("operation-one")["token"] == job.token
|
|
assert coordinator.retain(job, "libvirt_execution_uncertain")
|
|
with pytest.raises(CoordinationError) as raised:
|
|
coordinator.claim(resources=job.resources, execution_id=job.execution_id)
|
|
assert raised.value.error_code == "coordination_uncertain"
|
|
assert coordinator.release(job) is False
|
|
|
|
|
|
def test_operation_lookup_prefers_unknown_sibling(coordinator):
|
|
request = coordinator.claim(operation_id="operation-one")
|
|
job = coordinator.claim(operation_id="operation-one", execution_id=request.execution_id)
|
|
coordinator.retain(job)
|
|
assert coordinator.operation_claim("operation-one")["state"] == "uncertain"
|
|
assert coordinator.release(request)
|
|
assert coordinator.operation_claim("operation-one")["token"] == job.token
|
|
|
|
|
|
def test_maintenance_never_ignores_its_own_execution(coordinator):
|
|
request = coordinator.claim(resources=("vm:a:one",))
|
|
with pytest.raises(CoordinationError):
|
|
coordinator.claim(maintenance=True, execution_id=request.execution_id)
|
|
coordinator.release(request)
|
|
maintenance = coordinator.claim(maintenance=True)
|
|
with pytest.raises(CoordinationError):
|
|
coordinator.claim(resources=("vm:a:two",), execution_id=maintenance.execution_id)
|
|
assert coordinator.release(maintenance)
|
|
|
|
|
|
@pytest.mark.parametrize("uncertain", [False, True])
|
|
def test_restart_and_old_timestamps_never_grant_execution(coordinator, monkeypatch, uncertain):
|
|
claim = coordinator.claim(operation_id="old-operation", resources=("vm:a:one",))
|
|
if uncertain:
|
|
coordinator.retain(claim)
|
|
monkeypatch.setattr(coordination.time, "time", lambda: 99999999999)
|
|
restarted = DurableCoordinator(EncryptedSQLiteStore(coordinator.store.path))
|
|
assert restarted.has_work()
|
|
with pytest.raises(CoordinationError):
|
|
restarted.claim(resources=claim.resources)
|
|
with pytest.raises(CoordinationError):
|
|
restarted.claim(maintenance=True)
|
|
assert restarted.snapshot()[claim.token]["owner_id"] == claim.owner_id
|
|
|
|
|
|
def test_identity_and_resolution_prevent_stale_release(coordinator):
|
|
first = coordinator.claim(resources=("vm:a:one",))
|
|
assert coordinator.release(replace(first, owner_id="z" * 32)) is False
|
|
coordinator.retain(first)
|
|
assert coordinator.release(first) is False
|
|
with pytest.raises(ValueError):
|
|
coordinator.resolve(first, " ")
|
|
assert coordinator.resolve(first, "review/host-a/verified-completed")
|
|
second = coordinator.claim(resources=first.resources)
|
|
assert coordinator.release(first) is False
|
|
assert coordinator.resolve(first, "old review") is False
|
|
assert second.token in coordinator.snapshot()
|
|
document = json.loads(coordinator.store.get(RECORD_KIND, RECORD_KEY))
|
|
assert document["resolutions"][-1]["token"] == first.token
|
|
|
|
|
|
def test_refresh_metadata_and_resource_expansion_preserve_identity(coordinator, monkeypatch):
|
|
claim = coordinator.claim(operation_id="operation-one", metadata={"phase": "queued"})
|
|
original = coordinator.snapshot()[claim.token]
|
|
monkeypatch.setattr(coordination.time, "time", lambda: original["updated_at"] + 60)
|
|
expanded = coordinator.add_resources(claim, ("vm:a:one",))
|
|
assert expanded.token == claim.token and expanded.execution_id == claim.execution_id
|
|
assert expanded.resources == ("vm:a:one",)
|
|
assert coordinator.refresh(expanded)
|
|
assert coordinator.update_metadata(expanded, {"job_id": "maintenance-job"})
|
|
coordinator.retain(expanded)
|
|
assert coordinator.refresh(expanded)
|
|
current = coordinator.snapshot()[claim.token]
|
|
assert current["metadata"] == {"phase": "queued", "job_id": "maintenance-job"}
|
|
assert current["updated_at"] == original["updated_at"] + 60
|
|
assert current["state"] == "uncertain"
|
|
assert coordinator.release(expanded) is False
|
|
|
|
|
|
@pytest.mark.parametrize("uncertain", [False, True])
|
|
def test_resource_expansion_cannot_overlap_another_execution(coordinator, uncertain):
|
|
blocker = coordinator.claim(resources=("vm:a:one",))
|
|
if uncertain:
|
|
coordinator.retain(blocker)
|
|
candidate = coordinator.claim(metadata={"keep": True})
|
|
with pytest.raises(CoordinationError) as raised:
|
|
coordinator.add_resources(candidate, blocker.resources)
|
|
assert raised.value.status == (503 if uncertain else 409)
|
|
assert coordinator.snapshot()[candidate.token]["resources"] == []
|
|
assert coordinator.snapshot()[candidate.token]["metadata"] == {"keep": True}
|
|
|
|
|
|
def test_cas_retry_preserves_competing_claim(coordinator, monkeypatch):
|
|
compare = coordinator.store.compare_and_set
|
|
injected = []
|
|
other = DurableCoordinator(EncryptedSQLiteStore(coordinator.store.path))
|
|
|
|
def competing_claim(kind, key, expected, value):
|
|
if not injected:
|
|
injected.append(other.claim(resources=("vm:a:other",)))
|
|
return compare(kind, key, expected, value)
|
|
|
|
monkeypatch.setattr(coordinator.store, "compare_and_set", competing_claim)
|
|
claim = coordinator.claim(resources=("vm:a:one",))
|
|
assert set(coordinator.snapshot()) == {claim.token, injected[0].token}
|
|
|
|
|
|
def test_missing_cas_and_exhausted_contention_fail_closed(coordinator, monkeypatch):
|
|
with pytest.raises(ValueError, match="atomic database CAS"):
|
|
DurableCoordinator(object())
|
|
monkeypatch.setattr(coordinator.store, "compare_and_set", lambda *_args: False)
|
|
with pytest.raises(CoordinationError) as raised:
|
|
coordinator.claim(resources=("vm:a:one",))
|
|
assert raised.value.error_code == "coordination_unavailable"
|
|
assert raised.value.status == 503
|
|
assert coordinator.snapshot() == {}
|
|
|
|
|
|
def test_corrupt_record_is_not_reinitialized(coordinator):
|
|
coordinator.store.insert(RECORD_KIND, RECORD_KEY, '{"version":1,"claims":null}')
|
|
with pytest.raises(CoordinationError):
|
|
coordinator.claim(resources=("vm:a:one",))
|
|
assert coordinator.store.get(RECORD_KIND, RECORD_KEY) == '{"version":1,"claims":null}'
|
|
|
|
|
|
def test_capacity_and_normal_release_do_not_accumulate_entries(coordinator):
|
|
limited = DurableCoordinator(coordinator.store, max_claims=1)
|
|
for _index in range(40):
|
|
claim = limited.claim(resources=("vm:a:one",))
|
|
with pytest.raises(CoordinationError):
|
|
limited.claim(resources=("vm:b:one",))
|
|
assert limited.release(claim)
|
|
assert limited.snapshot() == {}
|
|
assert len(coordinator.store.get(RECORD_KIND, RECORD_KEY)) < 100
|
|
|
|
|
|
def test_context_flows_to_thread_and_nested_guards_release_individually(coordinator):
|
|
outer = coordinator.claim(operation_id="operation-one", resources=("vm:a:one",),
|
|
metadata={"job_id": "durable-job"})
|
|
|
|
def guarded_thread():
|
|
assert current_execution().claim.token == outer.token
|
|
with resource_guard(("storage:a", "vm:a:one", "storage:a")) as inner:
|
|
assert inner.execution_id == outer.execution_id
|
|
assert inner.operation_id == outer.operation_id
|
|
assert inner.metadata == {"job_id": "durable-job"}
|
|
assert inner.resources == ("storage:a", "vm:a:one")
|
|
assert current_execution().claim.token == inner.token
|
|
with resource_guard(("vm:a:one",)):
|
|
assert len(coordinator.snapshot()) == 3
|
|
assert current_execution().claim.token == outer.token
|
|
|
|
async def run():
|
|
with execution_context(coordinator, outer):
|
|
await asyncio.to_thread(guarded_thread)
|
|
|
|
asyncio.run(run())
|
|
assert current_execution() is None
|
|
assert set(coordinator.snapshot()) == {outer.token}
|
|
assert coordinator.release(outer)
|
|
|
|
|
|
def test_default_guard_is_reentrant_and_missing_default_fails_closed(coordinator):
|
|
original = get_default_coordinator()
|
|
try:
|
|
configure_default_coordinator(None)
|
|
with pytest.raises(CoordinationError):
|
|
with resource_guard(("vm:a:one",)):
|
|
pytest.fail("No durable coordinator")
|
|
configure_default_coordinator(coordinator)
|
|
with resource_guard(("vm:a:one",)) as first:
|
|
with resource_guard(("vm:a:one",)) as second:
|
|
assert first.execution_id == second.execution_id
|
|
assert not coordinator.has_work()
|
|
finally:
|
|
configure_default_coordinator(original)
|
|
|
|
|
|
@pytest.mark.parametrize("status,error_code,uncertain", [
|
|
(404, None, False), (500, None, True),
|
|
(504, "libvirt_execution_timeout", False),
|
|
(503, "libvirt_execution_stopped", False),
|
|
(504, "libvirt_execution_uncertain", True),
|
|
(503, "libvirt_execution_blocked", True),
|
|
])
|
|
def test_guard_retains_only_unconfirmed_execution(coordinator, status, error_code, uncertain):
|
|
class ServiceError(Exception):
|
|
pass
|
|
|
|
failure = ServiceError("failure")
|
|
failure.status, failure.error_code = status, error_code
|
|
outer = coordinator.claim()
|
|
with execution_context(coordinator, outer):
|
|
with pytest.raises(ServiceError):
|
|
with resource_guard(("vm:a:one",)):
|
|
raise failure
|
|
coordinator.release(outer)
|
|
remaining = coordinator.snapshot()
|
|
assert bool(remaining) is uncertain
|
|
if uncertain:
|
|
assert next(iter(remaining.values()))["state"] == "uncertain"
|
|
|
|
|
|
def test_cancelled_guard_keeps_remote_execution_ownership(coordinator):
|
|
outer = coordinator.claim()
|
|
with execution_context(coordinator, outer):
|
|
with pytest.raises(asyncio.CancelledError):
|
|
with resource_guard(("vm:a:one",)):
|
|
raise asyncio.CancelledError
|
|
coordinator.release(outer)
|
|
assert next(iter(coordinator.snapshot().values()))["state"] == "uncertain"
|
|
|
|
|
|
def test_guard_contention_has_bounded_wait(coordinator):
|
|
blocker = coordinator.claim(resources=("vm:a:one",))
|
|
other = coordinator.claim()
|
|
started = time.monotonic()
|
|
with execution_context(coordinator, other):
|
|
with pytest.raises(CoordinationError) as raised:
|
|
with resource_guard((item for item in blocker.resources), timeout=0.05):
|
|
pytest.fail("Conflicting execution admitted")
|
|
assert 0.04 <= time.monotonic() - started < 0.5
|
|
assert raised.value.error_code == "coordination_busy"
|
|
assert set(coordinator.snapshot()) == {blocker.token, other.token}
|
|
|
|
|
|
def test_lost_claim_commit_receipt_keeps_durable_admission(coordinator, monkeypatch):
|
|
compare = coordinator.store.compare_and_set
|
|
|
|
def commit_then_disconnect(*args):
|
|
assert compare(*args)
|
|
raise ConnectionError("injected lost commit receipt")
|
|
|
|
monkeypatch.setattr(coordinator.store, "compare_and_set", commit_then_disconnect)
|
|
with pytest.raises(CoordinationError) as raised:
|
|
coordinator.claim(operation_id="lost-receipt", resources=("vm:a:one",))
|
|
assert raised.value.error_code == "coordination_unavailable"
|
|
restarted = DurableCoordinator(EncryptedSQLiteStore(coordinator.store.path))
|
|
assert restarted.operation_claim("lost-receipt") is not None
|
|
with pytest.raises(CoordinationError):
|
|
restarted.claim(resources=("vm:a:one",))
|
|
|
|
|
|
def test_guard_release_database_error_cannot_erase_ownership(coordinator, monkeypatch):
|
|
outer = coordinator.claim()
|
|
with execution_context(coordinator, outer):
|
|
with pytest.raises(CoordinationError) as raised:
|
|
with resource_guard(("vm:a:one",)):
|
|
def unavailable(*_args):
|
|
raise ConnectionError("injected database outage")
|
|
monkeypatch.setattr(coordinator.store, "compare_and_set", unavailable)
|
|
assert raised.value.error_code == "coordination_unavailable"
|
|
assert len(coordinator.snapshot()) == 2
|
|
|
|
|
|
def test_unknown_conflict_takes_precedence_over_an_active_sibling(coordinator):
|
|
active = coordinator.claim(operation_id="operation-one", resources=("vm:a:one",))
|
|
uncertain = coordinator.claim(operation_id="operation-one", resources=active.resources,
|
|
execution_id=active.execution_id)
|
|
coordinator.retain(uncertain)
|
|
with pytest.raises(CoordinationError) as raised:
|
|
coordinator.claim(resources=active.resources)
|
|
assert raised.value.error_code == "coordination_uncertain"
|
|
|
|
|
|
def test_current_claim_can_only_be_replaced_by_same_identity(coordinator):
|
|
original = coordinator.claim(operation_id="create-one")
|
|
expanded = coordinator.add_resources(original, ("vm:a:created",))
|
|
with execution_context(coordinator, original):
|
|
update_current_claim(expanded)
|
|
assert current_execution().claim.resources == ("vm:a:created",)
|
|
with pytest.raises(ValueError):
|
|
update_current_claim(replace(expanded, token="other" * 8))
|
|
assert current_execution() is None
|
|
|
|
|
|
def test_safe_error_cannot_hide_a_concurrently_retained_guard(coordinator):
|
|
class NotFound(Exception):
|
|
status = 404
|
|
|
|
outer = coordinator.claim()
|
|
with execution_context(coordinator, outer):
|
|
with pytest.raises(CoordinationError) as raised:
|
|
with resource_guard(("vm:a:one",)) as claim:
|
|
coordinator.retain(claim)
|
|
raise NotFound()
|
|
assert raised.value.error_code == "coordination_uncertain"
|
|
assert coordinator.snapshot()[claim.token]["state"] == "uncertain"
|