You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
238 lines
10 KiB
Python
238 lines
10 KiB
Python
"""Maintenance admission shared by independent engine owners."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
from contextlib import asynccontextmanager
|
|
import json
|
|
import threading
|
|
|
|
import pytest
|
|
|
|
from test_engine_boundary import FakeUpdater, MemoryStore
|
|
from test_engine_coordination import (
|
|
local_backends, prepare_hosts, request, running_engine, vm_path, wait_until,
|
|
)
|
|
|
|
|
|
UPDATE = "/internal/v1/update/apply"
|
|
CONFIRMED_UPDATE = {
|
|
"expected_current_commit": "a" * 40,
|
|
"expected_remote_commit": "b" * 40,
|
|
}
|
|
JOB_KIND = "test_update_job"
|
|
|
|
|
|
def save_update_job(store, job):
|
|
store.set(JOB_KIND, job["job_id"], json.dumps(job))
|
|
|
|
|
|
def read_update_job(store, job_id):
|
|
raw = store.get(JOB_KIND, job_id)
|
|
return json.loads(raw) if raw else None
|
|
|
|
|
|
@asynccontextmanager
|
|
async def two_engines(directory, store):
|
|
# Separate owner identities model independent workers. Both applications
|
|
# intentionally share durable storage and no active_operations/task sets.
|
|
async with running_engine(directory / "a", store) as first:
|
|
try:
|
|
async with running_engine(directory / "b", store) as second:
|
|
assert first[0].coordinator.owner_id != second[0].coordinator.owner_id
|
|
yield first, second
|
|
finally:
|
|
# Production workers have different loops. Restore A's still-live
|
|
# executor after the nested B harness closes its own executor.
|
|
asyncio.get_running_loop().set_default_executor(first[0]._shutdown_executor)
|
|
|
|
|
|
def test_remote_work_in_one_engine_rejects_another_engines_update(tmp_path):
|
|
async def run():
|
|
store = MemoryStore()
|
|
async with two_engines(tmp_path, store) as (first, second):
|
|
engine_a, _, services_a = first
|
|
engine_b, _, _services_b = second
|
|
await prepare_hosts(engine_a, "host-a")
|
|
entered, release = threading.Event(), threading.Event()
|
|
original_action = services_a["host-a"].action
|
|
|
|
def stalled_action(*args):
|
|
entered.set()
|
|
assert release.wait(5), "test did not release host work"
|
|
return original_action(*args)
|
|
|
|
services_a["host-a"].action = stalled_action
|
|
operation = asyncio.create_task(request(
|
|
engine_a, "POST", vm_path("host-a"), {"action": "reset"},
|
|
operation_id="maintenance-other-worker-live",
|
|
))
|
|
try:
|
|
await wait_until(entered.is_set)
|
|
assert not engine_b.active_operations
|
|
assert not engine_b.maintenance_tasks
|
|
status, payload = await request(engine_b, "POST", UPDATE, CONFIRMED_UPDATE)
|
|
assert status == 409, payload
|
|
assert payload["error_code"] == "coordination_busy"
|
|
assert engine_b.updater.jobs == {}
|
|
assert not operation.done()
|
|
finally:
|
|
release.set()
|
|
status, payload = await asyncio.wait_for(operation, 3)
|
|
assert status == 200, payload
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
@pytest.mark.parametrize("terminal", ["failed", "succeeded"])
|
|
def test_maintenance_claim_precedes_job_creation_and_terminal_receipt_releases_it(
|
|
tmp_path, monkeypatch, terminal):
|
|
store = MemoryStore()
|
|
monkeypatch.setattr(FakeUpdater, "get_job",
|
|
lambda self, job_id: read_update_job(store, job_id))
|
|
|
|
async def run():
|
|
async with two_engines(tmp_path, store) as (first, second):
|
|
engine_a, _, _services_a = first
|
|
engine_b, _, services_b = second
|
|
await prepare_hosts(engine_b, "host-b")
|
|
entered, release = threading.Event(), threading.Event()
|
|
original_create = engine_a.updater.create_job
|
|
|
|
def stalled_create(**kwargs):
|
|
entered.set()
|
|
assert release.wait(5), "test did not release job creation"
|
|
job = original_create(**kwargs)
|
|
save_update_job(store, job)
|
|
return job
|
|
|
|
def start_update():
|
|
job = engine_a.updater.active
|
|
job["status"] = "running"
|
|
save_update_job(store, job)
|
|
|
|
engine_a.updater.create_job = stalled_create
|
|
engine_a.update_trigger = start_update
|
|
update = asyncio.create_task(request(engine_a, "POST", UPDATE, CONFIRMED_UPDATE))
|
|
try:
|
|
await wait_until(entered.is_set)
|
|
claims = engine_b.coordinator.snapshot()
|
|
assert len(claims) == 1
|
|
maintenance = next(iter(claims.values()))
|
|
assert maintenance["maintenance"] is True
|
|
assert maintenance["owner_id"] == engine_a.coordinator.owner_id
|
|
reserved_job_id = maintenance["metadata"]["update_job_id"]
|
|
assert read_update_job(store, reserved_job_id) is None
|
|
assert engine_b.updater.active_job() is None
|
|
status, payload = await request(
|
|
engine_b, "POST", vm_path("host-b"), {"action": "reset"},
|
|
operation_id="maintenance-before-job-blocks-write",
|
|
)
|
|
assert status == 409, payload
|
|
assert payload["error_code"] == "coordination_busy"
|
|
assert not any(call[0] == "action" for call in services_b["host-b"].calls)
|
|
read_status, _ = await request(engine_b, "GET", "/internal/v1/hosts/host-b/vms")
|
|
assert read_status == 200
|
|
finally:
|
|
release.set()
|
|
status, payload = await asyncio.wait_for(update, 3)
|
|
assert status == 202, payload
|
|
|
|
claims = engine_b.coordinator.snapshot()
|
|
assert len(claims) == 1
|
|
maintenance = next(iter(claims.values()))
|
|
job_id = maintenance["metadata"]["update_job_id"]
|
|
assert job_id == reserved_job_id
|
|
assert read_update_job(store, job_id)["status"] == "running"
|
|
status, payload = await request(
|
|
engine_b, "POST", vm_path("host-b"), {"action": "reset"},
|
|
operation_id="maintenance-bound-job-blocks-write",
|
|
)
|
|
assert status == 409, payload
|
|
|
|
job = read_update_job(store, job_id)
|
|
job["status"] = terminal
|
|
save_update_job(store, job)
|
|
status, payload = await request(
|
|
engine_b, "POST", vm_path("host-b"), {"action": "reset"},
|
|
operation_id="maintenance-terminal-allows-write",
|
|
)
|
|
assert status == 200, payload
|
|
assert maintenance["token"] not in engine_b.coordinator.snapshot()
|
|
assert any(call[0] == "action" for call in services_b["host-b"].calls)
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
@pytest.mark.parametrize("bound,uncertain,job_status", [
|
|
(False, False, None),
|
|
(False, True, None),
|
|
(True, False, None),
|
|
(True, False, "running"),
|
|
(True, False, "blocked"),
|
|
# A completed updater receipt cannot clear independently unknown work.
|
|
(True, True, "succeeded"),
|
|
(True, True, "failed"),
|
|
])
|
|
def test_unbound_or_unknown_maintenance_survives_restart(
|
|
tmp_path, monkeypatch, bound, uncertain, job_status):
|
|
store = MemoryStore()
|
|
monkeypatch.setattr(FakeUpdater, "get_job",
|
|
lambda self, job_id: read_update_job(store, job_id))
|
|
|
|
async def assert_blocked(engine, expected_token, op_id):
|
|
await prepare_hosts(engine, "host-a")
|
|
await engine._settle_update_claims()
|
|
assert expected_token in engine.coordinator.snapshot()
|
|
status, payload = await request(
|
|
engine, "POST", vm_path("host-a"), {"action": "reset"},
|
|
operation_id=op_id,
|
|
)
|
|
assert status == (503 if uncertain else 409), payload
|
|
assert payload["error_code"] == (
|
|
"coordination_uncertain" if uncertain else "coordination_busy")
|
|
status, health = await request(engine, "GET", "/internal/v1/health")
|
|
assert status == 200
|
|
assert health["accepting_mutations"] is False
|
|
|
|
async def run():
|
|
async with running_engine(tmp_path / "original", store) as (engine, _, services):
|
|
claim = engine.coordinator.claim(maintenance=True, kind="update")
|
|
if bound:
|
|
engine.coordinator.update_metadata(claim, {"update_job_id": "persisted-update-job"})
|
|
if job_status is not None:
|
|
save_update_job(store, {"job_id": "persisted-update-job", "status": job_status})
|
|
if uncertain:
|
|
engine.coordinator.retain(claim)
|
|
# Claim age is diagnostic; old timestamps are never a takeover.
|
|
await assert_blocked(engine, claim.token, "maintenance-before-restart")
|
|
original_owner = engine.coordinator.owner_id
|
|
assert not any(call[0] == "action" for call in services["host-a"].calls)
|
|
|
|
async with running_engine(tmp_path / "restarted", store) as (engine, _, services):
|
|
assert engine.coordinator.owner_id != original_owner
|
|
await assert_blocked(engine, claim.token, "maintenance-after-restart")
|
|
assert not any(call[0] == "action" for call in services["host-a"].calls)
|
|
assert engine.coordinator.snapshot()[claim.token]["owner_id"] == original_owner
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
|
|
@pytest.mark.parametrize("status", ["running", "interrupted", "uncertain"])
|
|
def test_legacy_fabric_attempt_without_child_claim_blocks_update(tmp_path, status):
|
|
async def run():
|
|
async with running_engine(tmp_path) as (engine, store, _services):
|
|
store.set("fabric_desired", "legacy-fabric", json.dumps({
|
|
"operation": {"id": "a" * 32, "status": status},
|
|
"nodes": [],
|
|
}))
|
|
assert not engine.coordinator.has_work()
|
|
response, _body = await request(engine, "POST", "/internal/v1/update/apply", {
|
|
"expected_current_commit": "a" * 40,
|
|
"expected_remote_commit": "b" * 40,
|
|
})
|
|
assert response == 409
|
|
assert not engine.coordinator.has_work()
|
|
asyncio.run(run())
|