"""Read-only update checks cannot leave a global maintenance fence. Use the real Updater and signed ASGI boundary. Only Git observations and the systemd worker are replaced; persistence and coordinator admission remain real. """ from __future__ import annotations import asyncio import json import threading import pytest from test_engine_coordination import ( local_backends, prepare_hosts, request, running_engine, vm_path, wait_until, ) from virsh_panel.updater import UpdateError, Updater UPDATE = "/internal/v1/update/apply" CONFIRMED = { "expected_current_commit": "a" * 40, "expected_remote_commit": "b" * 40, } def install_updater(engine, store, directory, monkeypatch): updater = Updater(directory, store) updater.set_token("test-only-update-token-123") monkeypatch.setattr(updater, "_checkout_head", lambda **_kwargs: "a" * 40) monkeypatch.setattr(updater, "_remote_head", lambda: "b" * 40) engine.updater = updater started = [] def start_worker(): job = updater.active_job() assert job is not None started.append(job["job_id"]) job.update(status="running", started_at=job["created_at"]) updater._save_job(job) engine.update_trigger = start_worker return updater, started def saved_jobs(store): return {key: json.loads(value) for key, value in store.list("value").items() if key.startswith("update.job.")} async def assert_unrelated_vm_can_write(engine, services, operation_id): status, payload = await asyncio.wait_for(request( engine, "POST", vm_path("host-b"), {"action": "reset"}, operation_id=operation_id, ), 1) assert status == 200, payload assert any(call[0] == "action" for call in services["host-b"].calls) def test_remote_preflight_502_creates_no_task_or_maintenance_claim(tmp_path, monkeypatch): async def run(): async with running_engine(tmp_path) as (engine, store, services): await prepare_hosts(engine, "host-a", "host-b") updater, started = install_updater(engine, store, tmp_path, monkeypatch) def unavailable(): raise UpdateError("remote Git unavailable", 502) monkeypatch.setattr(updater, "_remote_head", unavailable) status, payload = await request(engine, "POST", UPDATE, CONFIRMED) assert status == 502, payload assert saved_jobs(store) == {} assert store.get_value("update.active_job") is None assert started == [] await assert_unrelated_vm_can_write(engine, services, "preflight-502-unrelated-vm") assert engine.coordinator.snapshot() == {} asyncio.run(run()) def test_slow_preflight_does_not_serialize_unrelated_vm_writes(tmp_path, monkeypatch): async def run(): async with running_engine(tmp_path) as (engine, store, services): await prepare_hosts(engine, "host-a", "host-b") updater, started = install_updater(engine, store, tmp_path, monkeypatch) entered, release = threading.Event(), threading.Event() def unavailable(): entered.set() assert release.wait(5), "test did not release remote Git check" raise UpdateError("remote Git unavailable", 502) monkeypatch.setattr(updater, "_remote_head", unavailable) update = asyncio.create_task(request(engine, "POST", UPDATE, CONFIRMED)) try: await wait_until(entered.is_set) await assert_unrelated_vm_can_write(engine, services, "slow-preflight-unrelated-vm") assert engine.coordinator.snapshot() == {} assert not update.done() assert saved_jobs(store) == {} assert started == [] finally: release.set() status, payload = await asyncio.wait_for(update, 3) assert status == 502, payload assert engine.coordinator.snapshot() == {} asyncio.run(run()) @pytest.mark.parametrize("failure", ["local_changed", "remote_changed", "remote_502", "remote_exception"]) def test_final_version_recheck_failure_releases_only_its_unpublished_claim( tmp_path, monkeypatch, failure): async def run(): async with running_engine(tmp_path) as (engine, store, services): await prepare_hosts(engine, "host-b") updater, started = install_updater(engine, store, tmp_path, monkeypatch) calls, final_claims = [], [] def observe(): calls.append(True) if len(calls) == 1: assert engine.coordinator.snapshot() == {} return "a" * 40 if failure == "local_changed" else "b" * 40 claims = engine.coordinator.snapshot() assert len(claims) == 1 claim = next(iter(claims.values())) assert claim["maintenance"] is True assert claim["metadata"]["update_job_id"] final_claims.append(claim) if failure == "remote_502": raise UpdateError("second remote Git check unavailable", 502) if failure == "remote_exception": raise OSError("remote observation could not start") return "c" * 40 if failure == "local_changed": monkeypatch.setattr(updater, "_checkout_head", lambda **_kwargs: observe()) else: monkeypatch.setattr(updater, "_remote_head", observe) status, payload = await request(engine, "POST", UPDATE, CONFIRMED) assert status == (409 if failure.endswith("changed") else 502 if failure == "remote_502" else 503), payload assert len(final_claims) == 1 assert len(calls) == 2 assert engine.coordinator.snapshot() == {} assert saved_jobs(store) == {} assert store.get_value("update.active_job") is None assert started == [] await assert_unrelated_vm_can_write( engine, services, "final-preflight-" + failure) asyncio.run(run()) @pytest.mark.parametrize("lost_receipt", ["job", "active_pointer"]) @pytest.mark.parametrize("error_type", [OSError, UpdateError], ids=["io-error", "update-409"]) def test_published_update_with_lost_database_receipt_keeps_bound_maintenance_claim( tmp_path, monkeypatch, lost_receipt, error_type): async def run(): async with running_engine(tmp_path) as (engine, store, services): await prepare_hosts(engine, "host-b") updater, started = install_updater(engine, store, tmp_path, monkeypatch) persist = store.set_value def save_then_disconnect(key, value): persist(key, value) if ((lost_receipt == "job" and key.startswith("update.job.")) or (lost_receipt == "active_pointer" and key == "update.active_job")): if error_type is UpdateError: # HTTP status alone is not proof that publication did # not happen. Preserve the same fence for a 4xx reply. raise UpdateError("database committed before a conflict reply", 409) raise OSError("database committed but its reply was lost") monkeypatch.setattr(store, "set_value", save_then_disconnect) status, payload = await request(engine, "POST", UPDATE, CONFIRMED) assert status == (409 if error_type is UpdateError else 503), payload jobs = saved_jobs(store) assert len(jobs) == 1 job = next(iter(jobs.values())) claims = engine.coordinator.snapshot() assert len(claims) == 1 claim = next(iter(claims.values())) assert claim["state"] == "uncertain" assert claim["metadata"]["update_job_id"] == job["job_id"] assert updater.get_job(claim["metadata"]["update_job_id"]) == job assert started == [] status, payload = await request( engine, "POST", vm_path("host-b"), {"action": "reset"}, operation_id="published-response-lost-" + lost_receipt, ) assert status == 503, payload assert payload["error_code"] == "coordination_uncertain" assert not any(call[0] == "action" for call in services["host-b"].calls) asyncio.run(run()) def test_two_preflights_racing_publish_only_one_update(tmp_path, monkeypatch): async def run(): async with running_engine(tmp_path) as (engine, store, _services): updater, started = install_updater(engine, store, tmp_path, monkeypatch) barrier = threading.Barrier(2) lock = threading.Lock() calls = [] def remote_head(): with lock: calls.append(True) first_check = len(calls) <= 2 if first_check: barrier.wait(timeout=3) return "b" * 40 monkeypatch.setattr(updater, "_remote_head", remote_head) results = await asyncio.wait_for(asyncio.gather( request(engine, "POST", UPDATE, CONFIRMED), request(engine, "POST", UPDATE, CONFIRMED), ), 5) assert sorted(status for status, _payload in results) == [202, 409], results assert len(saved_jobs(store)) == 1 assert len(started) == 1 assert len(engine.coordinator.snapshot()) == 1 job = updater.active_job() assert job["job_id"] == started[0] claim = next(iter(engine.coordinator.snapshot().values())) assert claim["metadata"]["update_job_id"] == job["job_id"] asyncio.run(run()) def test_cancellation_during_final_check_cannot_release_claim_while_thread_can_publish( tmp_path, monkeypatch): async def run(): async with running_engine(tmp_path) as (engine, store, services): await prepare_hosts(engine, "host-b") updater, started = install_updater(engine, store, tmp_path, monkeypatch) entered, release = threading.Event(), threading.Event() calls = [] def remote_head(): calls.append(True) if len(calls) == 2: entered.set() assert release.wait(5), "test did not release final Git check" return "b" * 40 monkeypatch.setattr(updater, "_remote_head", remote_head) update = asyncio.create_task(request(engine, "POST", UPDATE, CONFIRMED)) try: await wait_until(entered.is_set) # Simulate engine shutdown cancelling its owned coroutine, not # merely a disconnected HTTP client (which is shielded). assert len(engine.request_tasks) == 1 next(iter(engine.request_tasks)).cancel() with pytest.raises(asyncio.CancelledError): await update claims = engine.coordinator.snapshot() assert len(claims) == 1 claim = next(iter(claims.values())) assert claim["state"] == "uncertain" assert claim["metadata"]["update_job_id"] assert saved_jobs(store) == {} finally: release.set() await wait_until(lambda: bool(saved_jobs(store))) assert next(iter(saved_jobs(store).values()))["job_id"] == claim["metadata"]["update_job_id"] assert started == [] status, payload = await request( engine, "POST", vm_path("host-b"), {"action": "reset"}, operation_id="cancelled-create-still-published", ) assert status == 503, payload assert payload["error_code"] == "coordination_uncertain" assert not any(call[0] == "action" for call in services["host-b"].calls) asyncio.run(run())