"""Cross-process admission using the production encrypted CAS implementation.""" from __future__ import annotations import asyncio from dataclasses import replace import json import multiprocessing import sqlite3 import time from cryptography.hazmat.primitives.ciphers.aead import AESGCM import pytest from virsh_panel import coordination from virsh_panel.coordination import ( CoordinationError, DurableCoordinator, RECORD_KEY, RECORD_KIND, configure_default_coordinator, current_execution, execution_context, get_default_coordinator, resource_guard, update_current_claim, ) from virsh_panel.database import DatabaseStore, pymysql class SQLiteCursor: def __init__(self, raw): self.raw = raw.cursor() def __enter__(self): return self def __exit__(self, *_args): self.raw.close() def execute(self, statement, values=()): try: return self.raw.execute(statement.replace("%s", "?").replace(" FOR UPDATE", ""), values) except sqlite3.IntegrityError as exc: raise pymysql.err.IntegrityError(1062, "duplicate test row") from exc def fetchone(self): row = self.raw.fetchone() return dict(row) if row is not None else None def fetchall(self): return [dict(row) for row in self.raw.fetchall()] class SQLiteConnection: def __init__(self, path): self.raw = sqlite3.connect(path, timeout=5.0, isolation_level=None) self.raw.row_factory = sqlite3.Row # The SQLite writer lock substitutes for MariaDB's row lock. Production # get/insert/CAS/encryption methods execute unchanged. self.raw.execute("BEGIN IMMEDIATE") def cursor(self): return SQLiteCursor(self.raw) def commit(self): self.raw.commit() def rollback(self): self.raw.rollback() def close(self): self.raw.close() class EncryptedSQLiteStore(DatabaseStore): RECORD_TABLE = "engine_records" def __init__(self, path): self.path = str(path) self._cipher = AESGCM(b"a" * 32) with sqlite3.connect(self.path, timeout=5.0) as connection: connection.execute("CREATE TABLE IF NOT EXISTS engine_records (" "record_type TEXT, record_key TEXT, encrypted_value BLOB, " "PRIMARY KEY(record_type,record_key))") def _connect(self): return SQLiteConnection(self.path) @pytest.fixture def coordinator(tmp_path): return DurableCoordinator(EncryptedSQLiteStore(tmp_path / "state.sqlite")) def _claim_in_process(path, options, ready, release, output): instance = DurableCoordinator(EncryptedSQLiteStore(path)) ready.wait(timeout=10) try: claim = instance.claim(**options) except CoordinationError as exc: output.put(("rejected", exc.error_code)) else: output.put(("admitted", claim.token)) release.wait(timeout=10) instance.release(claim) @pytest.mark.parametrize("first,second,admitted", [ ({"resources": ("vm:a:one",)}, {"resources": ("vm:a:one",)}, 1), ({"resources": ("vm:a:one",)}, {"resources": ("vm:b:one",)}, 2), ({"operation_id": "operation-a", "resources": ("vm:a:one",)}, {"operation_id": "operation-a", "resources": ("vm:b:one",)}, 1), ({"resources": ("vm:a:one",)}, {"maintenance": True}, 1), ]) def test_independent_processes_share_atomic_admission(tmp_path, first, second, admitted): path = tmp_path / "state.sqlite" EncryptedSQLiteStore(path) context = multiprocessing.get_context("spawn") ready, release, output = context.Barrier(2), context.Event(), context.Queue() processes = [context.Process(target=_claim_in_process, args=(str(path), options, ready, release, output)) for options in (first, second)] try: for process in processes: process.start() results = [output.get(timeout=10) for _ in processes] assert sum(result[0] == "admitted" for result in results) == admitted observer = DurableCoordinator(EncryptedSQLiteStore(path)) assert len(observer.snapshot()) == admitted finally: release.set() for process in processes: process.join(timeout=5) if process.is_alive(): process.kill() process.join(timeout=2) output.close() output.join_thread() assert all(process.exitcode == 0 for process in processes) assert observer.snapshot() == {} def test_same_execution_handoff_preserves_operation_until_last_claim(coordinator): request = coordinator.claim(operation_id="operation-one", resources=("vm:a:one",)) job = coordinator.claim(operation_id="operation-one", resources=request.resources, execution_id=request.execution_id, kind="background") assert coordinator.release(request) assert coordinator.operation_claim("operation-one")["token"] == job.token assert coordinator.retain(job, "libvirt_execution_uncertain") with pytest.raises(CoordinationError) as raised: coordinator.claim(resources=job.resources, execution_id=job.execution_id) assert raised.value.error_code == "coordination_uncertain" assert coordinator.release(job) is False def test_operation_lookup_prefers_unknown_sibling(coordinator): request = coordinator.claim(operation_id="operation-one") job = coordinator.claim(operation_id="operation-one", execution_id=request.execution_id) coordinator.retain(job) assert coordinator.operation_claim("operation-one")["state"] == "uncertain" assert coordinator.release(request) assert coordinator.operation_claim("operation-one")["token"] == job.token def test_maintenance_never_ignores_its_own_execution(coordinator): request = coordinator.claim(resources=("vm:a:one",)) with pytest.raises(CoordinationError): coordinator.claim(maintenance=True, execution_id=request.execution_id) coordinator.release(request) maintenance = coordinator.claim(maintenance=True) with pytest.raises(CoordinationError): coordinator.claim(resources=("vm:a:two",), execution_id=maintenance.execution_id) assert coordinator.release(maintenance) @pytest.mark.parametrize("uncertain", [False, True]) def test_restart_and_old_timestamps_never_grant_execution(coordinator, monkeypatch, uncertain): claim = coordinator.claim(operation_id="old-operation", resources=("vm:a:one",)) if uncertain: coordinator.retain(claim) monkeypatch.setattr(coordination.time, "time", lambda: 99999999999) restarted = DurableCoordinator(EncryptedSQLiteStore(coordinator.store.path)) assert restarted.has_work() with pytest.raises(CoordinationError): restarted.claim(resources=claim.resources) with pytest.raises(CoordinationError): restarted.claim(maintenance=True) assert restarted.snapshot()[claim.token]["owner_id"] == claim.owner_id def test_identity_and_resolution_prevent_stale_release(coordinator): first = coordinator.claim(resources=("vm:a:one",)) assert coordinator.release(replace(first, owner_id="z" * 32)) is False coordinator.retain(first) assert coordinator.release(first) is False with pytest.raises(ValueError): coordinator.resolve(first, " ") assert coordinator.resolve(first, "review/host-a/verified-completed") second = coordinator.claim(resources=first.resources) assert coordinator.release(first) is False assert coordinator.resolve(first, "old review") is False assert second.token in coordinator.snapshot() document = json.loads(coordinator.store.get(RECORD_KIND, RECORD_KEY)) assert document["resolutions"][-1]["token"] == first.token def test_refresh_metadata_and_resource_expansion_preserve_identity(coordinator, monkeypatch): claim = coordinator.claim(operation_id="operation-one", metadata={"phase": "queued"}) original = coordinator.snapshot()[claim.token] monkeypatch.setattr(coordination.time, "time", lambda: original["updated_at"] + 60) expanded = coordinator.add_resources(claim, ("vm:a:one",)) assert expanded.token == claim.token and expanded.execution_id == claim.execution_id assert expanded.resources == ("vm:a:one",) assert coordinator.refresh(expanded) assert coordinator.update_metadata(expanded, {"job_id": "maintenance-job"}) coordinator.retain(expanded) assert coordinator.refresh(expanded) current = coordinator.snapshot()[claim.token] assert current["metadata"] == {"phase": "queued", "job_id": "maintenance-job"} assert current["updated_at"] == original["updated_at"] + 60 assert current["state"] == "uncertain" assert coordinator.release(expanded) is False @pytest.mark.parametrize("uncertain", [False, True]) def test_resource_expansion_cannot_overlap_another_execution(coordinator, uncertain): blocker = coordinator.claim(resources=("vm:a:one",)) if uncertain: coordinator.retain(blocker) candidate = coordinator.claim(metadata={"keep": True}) with pytest.raises(CoordinationError) as raised: coordinator.add_resources(candidate, blocker.resources) assert raised.value.status == (503 if uncertain else 409) assert coordinator.snapshot()[candidate.token]["resources"] == [] assert coordinator.snapshot()[candidate.token]["metadata"] == {"keep": True} def test_cas_retry_preserves_competing_claim(coordinator, monkeypatch): compare = coordinator.store.compare_and_set injected = [] other = DurableCoordinator(EncryptedSQLiteStore(coordinator.store.path)) def competing_claim(kind, key, expected, value): if not injected: injected.append(other.claim(resources=("vm:a:other",))) return compare(kind, key, expected, value) monkeypatch.setattr(coordinator.store, "compare_and_set", competing_claim) claim = coordinator.claim(resources=("vm:a:one",)) assert set(coordinator.snapshot()) == {claim.token, injected[0].token} def test_missing_cas_and_exhausted_contention_fail_closed(coordinator, monkeypatch): with pytest.raises(ValueError, match="atomic database CAS"): DurableCoordinator(object()) monkeypatch.setattr(coordinator.store, "compare_and_set", lambda *_args: False) with pytest.raises(CoordinationError) as raised: coordinator.claim(resources=("vm:a:one",)) assert raised.value.error_code == "coordination_unavailable" assert raised.value.status == 503 assert coordinator.snapshot() == {} def test_corrupt_record_is_not_reinitialized(coordinator): coordinator.store.insert(RECORD_KIND, RECORD_KEY, '{"version":1,"claims":null}') with pytest.raises(CoordinationError): coordinator.claim(resources=("vm:a:one",)) assert coordinator.store.get(RECORD_KIND, RECORD_KEY) == '{"version":1,"claims":null}' def test_capacity_and_normal_release_do_not_accumulate_entries(coordinator): limited = DurableCoordinator(coordinator.store, max_claims=1) for _index in range(40): claim = limited.claim(resources=("vm:a:one",)) with pytest.raises(CoordinationError): limited.claim(resources=("vm:b:one",)) assert limited.release(claim) assert limited.snapshot() == {} assert len(coordinator.store.get(RECORD_KIND, RECORD_KEY)) < 100 def test_context_flows_to_thread_and_nested_guards_release_individually(coordinator): outer = coordinator.claim(operation_id="operation-one", resources=("vm:a:one",), metadata={"job_id": "durable-job"}) def guarded_thread(): assert current_execution().claim.token == outer.token with resource_guard(("storage:a", "vm:a:one", "storage:a")) as inner: assert inner.execution_id == outer.execution_id assert inner.operation_id == outer.operation_id assert inner.metadata == {"job_id": "durable-job"} assert inner.resources == ("storage:a", "vm:a:one") assert current_execution().claim.token == inner.token with resource_guard(("vm:a:one",)): assert len(coordinator.snapshot()) == 3 assert current_execution().claim.token == outer.token async def run(): with execution_context(coordinator, outer): await asyncio.to_thread(guarded_thread) asyncio.run(run()) assert current_execution() is None assert set(coordinator.snapshot()) == {outer.token} assert coordinator.release(outer) def test_default_guard_is_reentrant_and_missing_default_fails_closed(coordinator): original = get_default_coordinator() try: configure_default_coordinator(None) with pytest.raises(CoordinationError): with resource_guard(("vm:a:one",)): pytest.fail("No durable coordinator") configure_default_coordinator(coordinator) with resource_guard(("vm:a:one",)) as first: with resource_guard(("vm:a:one",)) as second: assert first.execution_id == second.execution_id assert not coordinator.has_work() finally: configure_default_coordinator(original) @pytest.mark.parametrize("status,error_code,uncertain", [ (404, None, False), (500, None, True), (504, "libvirt_execution_timeout", False), (503, "libvirt_execution_stopped", False), (504, "libvirt_execution_uncertain", True), (503, "libvirt_execution_blocked", True), ]) def test_guard_retains_only_unconfirmed_execution(coordinator, status, error_code, uncertain): class ServiceError(Exception): pass failure = ServiceError("failure") failure.status, failure.error_code = status, error_code outer = coordinator.claim() with execution_context(coordinator, outer): with pytest.raises(ServiceError): with resource_guard(("vm:a:one",)): raise failure coordinator.release(outer) remaining = coordinator.snapshot() assert bool(remaining) is uncertain if uncertain: assert next(iter(remaining.values()))["state"] == "uncertain" def test_cancelled_guard_keeps_remote_execution_ownership(coordinator): outer = coordinator.claim() with execution_context(coordinator, outer): with pytest.raises(asyncio.CancelledError): with resource_guard(("vm:a:one",)): raise asyncio.CancelledError coordinator.release(outer) assert next(iter(coordinator.snapshot().values()))["state"] == "uncertain" def test_guard_contention_has_bounded_wait(coordinator): blocker = coordinator.claim(resources=("vm:a:one",)) other = coordinator.claim() started = time.monotonic() with execution_context(coordinator, other): with pytest.raises(CoordinationError) as raised: with resource_guard((item for item in blocker.resources), timeout=0.05): pytest.fail("Conflicting execution admitted") assert 0.04 <= time.monotonic() - started < 0.5 assert raised.value.error_code == "coordination_busy" assert set(coordinator.snapshot()) == {blocker.token, other.token} def test_lost_claim_commit_receipt_keeps_durable_admission(coordinator, monkeypatch): compare = coordinator.store.compare_and_set def commit_then_disconnect(*args): assert compare(*args) raise ConnectionError("injected lost commit receipt") monkeypatch.setattr(coordinator.store, "compare_and_set", commit_then_disconnect) with pytest.raises(CoordinationError) as raised: coordinator.claim(operation_id="lost-receipt", resources=("vm:a:one",)) assert raised.value.error_code == "coordination_unavailable" restarted = DurableCoordinator(EncryptedSQLiteStore(coordinator.store.path)) assert restarted.operation_claim("lost-receipt") is not None with pytest.raises(CoordinationError): restarted.claim(resources=("vm:a:one",)) def test_guard_release_database_error_cannot_erase_ownership(coordinator, monkeypatch): outer = coordinator.claim() with execution_context(coordinator, outer): with pytest.raises(CoordinationError) as raised: with resource_guard(("vm:a:one",)): def unavailable(*_args): raise ConnectionError("injected database outage") monkeypatch.setattr(coordinator.store, "compare_and_set", unavailable) assert raised.value.error_code == "coordination_unavailable" assert len(coordinator.snapshot()) == 2 def test_unknown_conflict_takes_precedence_over_an_active_sibling(coordinator): active = coordinator.claim(operation_id="operation-one", resources=("vm:a:one",)) uncertain = coordinator.claim(operation_id="operation-one", resources=active.resources, execution_id=active.execution_id) coordinator.retain(uncertain) with pytest.raises(CoordinationError) as raised: coordinator.claim(resources=active.resources) assert raised.value.error_code == "coordination_uncertain" def test_current_claim_can_only_be_replaced_by_same_identity(coordinator): original = coordinator.claim(operation_id="create-one") expanded = coordinator.add_resources(original, ("vm:a:created",)) with execution_context(coordinator, original): update_current_claim(expanded) assert current_execution().claim.resources == ("vm:a:created",) with pytest.raises(ValueError): update_current_claim(replace(expanded, token="other" * 8)) assert current_execution() is None def test_safe_error_cannot_hide_a_concurrently_retained_guard(coordinator): class NotFound(Exception): status = 404 outer = coordinator.claim() with execution_context(coordinator, outer): with pytest.raises(CoordinationError) as raised: with resource_guard(("vm:a:one",)) as claim: coordinator.retain(claim) raise NotFound() assert raised.value.error_code == "coordination_uncertain" assert coordinator.snapshot()[claim.token]["state"] == "uncertain"