"""Maintenance admission shared by independent engine owners.""" from __future__ import annotations import asyncio from contextlib import asynccontextmanager import json import threading import pytest from test_engine_boundary import FakeUpdater, MemoryStore from test_engine_coordination import ( local_backends, prepare_hosts, request, running_engine, vm_path, wait_until, ) UPDATE = "/internal/v1/update/apply" CONFIRMED_UPDATE = { "expected_current_commit": "a" * 40, "expected_remote_commit": "b" * 40, } JOB_KIND = "test_update_job" def save_update_job(store, job): store.set(JOB_KIND, job["job_id"], json.dumps(job)) def read_update_job(store, job_id): raw = store.get(JOB_KIND, job_id) return json.loads(raw) if raw else None @asynccontextmanager async def two_engines(directory, store): # Separate owner identities model independent workers. Both applications # intentionally share durable storage and no active_operations/task sets. async with running_engine(directory / "a", store) as first: try: async with running_engine(directory / "b", store) as second: assert first[0].coordinator.owner_id != second[0].coordinator.owner_id yield first, second finally: # Production workers have different loops. Restore A's still-live # executor after the nested B harness closes its own executor. asyncio.get_running_loop().set_default_executor(first[0]._shutdown_executor) def test_remote_work_in_one_engine_rejects_another_engines_update(tmp_path): async def run(): store = MemoryStore() async with two_engines(tmp_path, store) as (first, second): engine_a, _, services_a = first engine_b, _, _services_b = second await prepare_hosts(engine_a, "host-a") entered, release = threading.Event(), threading.Event() original_action = services_a["host-a"].action def stalled_action(*args): entered.set() assert release.wait(5), "test did not release host work" return original_action(*args) services_a["host-a"].action = stalled_action operation = asyncio.create_task(request( engine_a, "POST", vm_path("host-a"), {"action": "reset"}, operation_id="maintenance-other-worker-live", )) try: await wait_until(entered.is_set) assert not engine_b.active_operations assert not engine_b.maintenance_tasks status, payload = await request(engine_b, "POST", UPDATE, CONFIRMED_UPDATE) assert status == 409, payload assert payload["error_code"] == "coordination_busy" assert engine_b.updater.jobs == {} assert not operation.done() finally: release.set() status, payload = await asyncio.wait_for(operation, 3) assert status == 200, payload asyncio.run(run()) @pytest.mark.parametrize("terminal", ["failed", "succeeded"]) def test_maintenance_claim_precedes_job_creation_and_terminal_receipt_releases_it( tmp_path, monkeypatch, terminal): store = MemoryStore() monkeypatch.setattr(FakeUpdater, "get_job", lambda self, job_id: read_update_job(store, job_id)) async def run(): async with two_engines(tmp_path, store) as (first, second): engine_a, _, _services_a = first engine_b, _, services_b = second await prepare_hosts(engine_b, "host-b") entered, release = threading.Event(), threading.Event() original_create = engine_a.updater.create_job def stalled_create(**kwargs): entered.set() assert release.wait(5), "test did not release job creation" job = original_create(**kwargs) save_update_job(store, job) return job def start_update(): job = engine_a.updater.active job["status"] = "running" save_update_job(store, job) engine_a.updater.create_job = stalled_create engine_a.update_trigger = start_update update = asyncio.create_task(request(engine_a, "POST", UPDATE, CONFIRMED_UPDATE)) try: await wait_until(entered.is_set) claims = engine_b.coordinator.snapshot() assert len(claims) == 1 maintenance = next(iter(claims.values())) assert maintenance["maintenance"] is True assert maintenance["owner_id"] == engine_a.coordinator.owner_id reserved_job_id = maintenance["metadata"]["update_job_id"] assert read_update_job(store, reserved_job_id) is None assert engine_b.updater.active_job() is None status, payload = await request( engine_b, "POST", vm_path("host-b"), {"action": "reset"}, operation_id="maintenance-before-job-blocks-write", ) assert status == 409, payload assert payload["error_code"] == "coordination_busy" assert not any(call[0] == "action" for call in services_b["host-b"].calls) read_status, _ = await request(engine_b, "GET", "/internal/v1/hosts/host-b/vms") assert read_status == 200 finally: release.set() status, payload = await asyncio.wait_for(update, 3) assert status == 202, payload claims = engine_b.coordinator.snapshot() assert len(claims) == 1 maintenance = next(iter(claims.values())) job_id = maintenance["metadata"]["update_job_id"] assert job_id == reserved_job_id assert read_update_job(store, job_id)["status"] == "running" status, payload = await request( engine_b, "POST", vm_path("host-b"), {"action": "reset"}, operation_id="maintenance-bound-job-blocks-write", ) assert status == 409, payload job = read_update_job(store, job_id) job["status"] = terminal save_update_job(store, job) status, payload = await request( engine_b, "POST", vm_path("host-b"), {"action": "reset"}, operation_id="maintenance-terminal-allows-write", ) assert status == 200, payload assert maintenance["token"] not in engine_b.coordinator.snapshot() assert any(call[0] == "action" for call in services_b["host-b"].calls) asyncio.run(run()) @pytest.mark.parametrize("bound,uncertain,job_status", [ (False, False, None), (False, True, None), (True, False, None), (True, False, "running"), (True, False, "blocked"), # A completed updater receipt cannot clear independently unknown work. (True, True, "succeeded"), (True, True, "failed"), ]) def test_unbound_or_unknown_maintenance_survives_restart( tmp_path, monkeypatch, bound, uncertain, job_status): store = MemoryStore() monkeypatch.setattr(FakeUpdater, "get_job", lambda self, job_id: read_update_job(store, job_id)) async def assert_blocked(engine, expected_token, op_id): await prepare_hosts(engine, "host-a") await engine._settle_update_claims() assert expected_token in engine.coordinator.snapshot() status, payload = await request( engine, "POST", vm_path("host-a"), {"action": "reset"}, operation_id=op_id, ) assert status == (503 if uncertain else 409), payload assert payload["error_code"] == ( "coordination_uncertain" if uncertain else "coordination_busy") status, health = await request(engine, "GET", "/internal/v1/health") assert status == 200 assert health["accepting_mutations"] is False async def run(): async with running_engine(tmp_path / "original", store) as (engine, _, services): claim = engine.coordinator.claim(maintenance=True, kind="update") if bound: engine.coordinator.update_metadata(claim, {"update_job_id": "persisted-update-job"}) if job_status is not None: save_update_job(store, {"job_id": "persisted-update-job", "status": job_status}) if uncertain: engine.coordinator.retain(claim) # Claim age is diagnostic; old timestamps are never a takeover. await assert_blocked(engine, claim.token, "maintenance-before-restart") original_owner = engine.coordinator.owner_id assert not any(call[0] == "action" for call in services["host-a"].calls) async with running_engine(tmp_path / "restarted", store) as (engine, _, services): assert engine.coordinator.owner_id != original_owner await assert_blocked(engine, claim.token, "maintenance-after-restart") assert not any(call[0] == "action" for call in services["host-a"].calls) assert engine.coordinator.snapshot()[claim.token]["owner_id"] == original_owner asyncio.run(run()) @pytest.mark.parametrize("status", ["running", "interrupted", "uncertain"]) def test_legacy_fabric_attempt_without_child_claim_blocks_update(tmp_path, status): async def run(): async with running_engine(tmp_path) as (engine, store, _services): store.set("fabric_desired", "legacy-fabric", json.dumps({ "operation": {"id": "a" * 32, "status": status}, "nodes": [], })) assert not engine.coordinator.has_work() response, _body = await request(engine, "POST", "/internal/v1/update/apply", { "expected_current_commit": "a" * 40, "expected_remote_commit": "b" * 40, }) assert response == 409 assert not engine.coordinator.has_work() asyncio.run(run())