You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/engine/tests/test_engine_maintenance_coo...

238 lines
10 KiB
Python

"""Maintenance admission shared by independent engine owners."""
from __future__ import annotations
import asyncio
from contextlib import asynccontextmanager
import json
import threading
import pytest
from test_engine_boundary import FakeUpdater, MemoryStore
from test_engine_coordination import (
local_backends, prepare_hosts, request, running_engine, vm_path, wait_until,
)
UPDATE = "/internal/v1/update/apply"
CONFIRMED_UPDATE = {
"expected_current_commit": "a" * 40,
"expected_remote_commit": "b" * 40,
}
JOB_KIND = "test_update_job"
def save_update_job(store, job):
store.set(JOB_KIND, job["job_id"], json.dumps(job))
def read_update_job(store, job_id):
raw = store.get(JOB_KIND, job_id)
return json.loads(raw) if raw else None
@asynccontextmanager
async def two_engines(directory, store):
# Separate owner identities model independent workers. Both applications
# intentionally share durable storage and no active_operations/task sets.
async with running_engine(directory / "a", store) as first:
try:
async with running_engine(directory / "b", store) as second:
assert first[0].coordinator.owner_id != second[0].coordinator.owner_id
yield first, second
finally:
# Production workers have different loops. Restore A's still-live
# executor after the nested B harness closes its own executor.
asyncio.get_running_loop().set_default_executor(first[0]._shutdown_executor)
def test_remote_work_in_one_engine_rejects_another_engines_update(tmp_path):
async def run():
store = MemoryStore()
async with two_engines(tmp_path, store) as (first, second):
engine_a, _, services_a = first
engine_b, _, _services_b = second
await prepare_hosts(engine_a, "host-a")
entered, release = threading.Event(), threading.Event()
original_action = services_a["host-a"].action
def stalled_action(*args):
entered.set()
assert release.wait(5), "test did not release host work"
return original_action(*args)
services_a["host-a"].action = stalled_action
operation = asyncio.create_task(request(
engine_a, "POST", vm_path("host-a"), {"action": "reset"},
operation_id="maintenance-other-worker-live",
))
try:
await wait_until(entered.is_set)
assert not engine_b.active_operations
assert not engine_b.maintenance_tasks
status, payload = await request(engine_b, "POST", UPDATE, CONFIRMED_UPDATE)
assert status == 409, payload
assert payload["error_code"] == "coordination_busy"
assert engine_b.updater.jobs == {}
assert not operation.done()
finally:
release.set()
status, payload = await asyncio.wait_for(operation, 3)
assert status == 200, payload
asyncio.run(run())
@pytest.mark.parametrize("terminal", ["failed", "succeeded"])
def test_maintenance_claim_precedes_job_creation_and_terminal_receipt_releases_it(
tmp_path, monkeypatch, terminal):
store = MemoryStore()
monkeypatch.setattr(FakeUpdater, "get_job",
lambda self, job_id: read_update_job(store, job_id))
async def run():
async with two_engines(tmp_path, store) as (first, second):
engine_a, _, _services_a = first
engine_b, _, services_b = second
await prepare_hosts(engine_b, "host-b")
entered, release = threading.Event(), threading.Event()
original_create = engine_a.updater.create_job
def stalled_create(**kwargs):
entered.set()
assert release.wait(5), "test did not release job creation"
job = original_create(**kwargs)
save_update_job(store, job)
return job
def start_update():
job = engine_a.updater.active
job["status"] = "running"
save_update_job(store, job)
engine_a.updater.create_job = stalled_create
engine_a.update_trigger = start_update
update = asyncio.create_task(request(engine_a, "POST", UPDATE, CONFIRMED_UPDATE))
try:
await wait_until(entered.is_set)
claims = engine_b.coordinator.snapshot()
assert len(claims) == 1
maintenance = next(iter(claims.values()))
assert maintenance["maintenance"] is True
assert maintenance["owner_id"] == engine_a.coordinator.owner_id
reserved_job_id = maintenance["metadata"]["update_job_id"]
assert read_update_job(store, reserved_job_id) is None
assert engine_b.updater.active_job() is None
status, payload = await request(
engine_b, "POST", vm_path("host-b"), {"action": "reset"},
operation_id="maintenance-before-job-blocks-write",
)
assert status == 409, payload
assert payload["error_code"] == "coordination_busy"
assert not any(call[0] == "action" for call in services_b["host-b"].calls)
read_status, _ = await request(engine_b, "GET", "/internal/v1/hosts/host-b/vms")
assert read_status == 200
finally:
release.set()
status, payload = await asyncio.wait_for(update, 3)
assert status == 202, payload
claims = engine_b.coordinator.snapshot()
assert len(claims) == 1
maintenance = next(iter(claims.values()))
job_id = maintenance["metadata"]["update_job_id"]
assert job_id == reserved_job_id
assert read_update_job(store, job_id)["status"] == "running"
status, payload = await request(
engine_b, "POST", vm_path("host-b"), {"action": "reset"},
operation_id="maintenance-bound-job-blocks-write",
)
assert status == 409, payload
job = read_update_job(store, job_id)
job["status"] = terminal
save_update_job(store, job)
status, payload = await request(
engine_b, "POST", vm_path("host-b"), {"action": "reset"},
operation_id="maintenance-terminal-allows-write",
)
assert status == 200, payload
assert maintenance["token"] not in engine_b.coordinator.snapshot()
assert any(call[0] == "action" for call in services_b["host-b"].calls)
asyncio.run(run())
@pytest.mark.parametrize("bound,uncertain,job_status", [
(False, False, None),
(False, True, None),
(True, False, None),
(True, False, "running"),
(True, False, "blocked"),
# A completed updater receipt cannot clear independently unknown work.
(True, True, "succeeded"),
(True, True, "failed"),
])
def test_unbound_or_unknown_maintenance_survives_restart(
tmp_path, monkeypatch, bound, uncertain, job_status):
store = MemoryStore()
monkeypatch.setattr(FakeUpdater, "get_job",
lambda self, job_id: read_update_job(store, job_id))
async def assert_blocked(engine, expected_token, op_id):
await prepare_hosts(engine, "host-a")
await engine._settle_update_claims()
assert expected_token in engine.coordinator.snapshot()
status, payload = await request(
engine, "POST", vm_path("host-a"), {"action": "reset"},
operation_id=op_id,
)
assert status == (503 if uncertain else 409), payload
assert payload["error_code"] == (
"coordination_uncertain" if uncertain else "coordination_busy")
status, health = await request(engine, "GET", "/internal/v1/health")
assert status == 200
assert health["accepting_mutations"] is False
async def run():
async with running_engine(tmp_path / "original", store) as (engine, _, services):
claim = engine.coordinator.claim(maintenance=True, kind="update")
if bound:
engine.coordinator.update_metadata(claim, {"update_job_id": "persisted-update-job"})
if job_status is not None:
save_update_job(store, {"job_id": "persisted-update-job", "status": job_status})
if uncertain:
engine.coordinator.retain(claim)
# Claim age is diagnostic; old timestamps are never a takeover.
await assert_blocked(engine, claim.token, "maintenance-before-restart")
original_owner = engine.coordinator.owner_id
assert not any(call[0] == "action" for call in services["host-a"].calls)
async with running_engine(tmp_path / "restarted", store) as (engine, _, services):
assert engine.coordinator.owner_id != original_owner
await assert_blocked(engine, claim.token, "maintenance-after-restart")
assert not any(call[0] == "action" for call in services["host-a"].calls)
assert engine.coordinator.snapshot()[claim.token]["owner_id"] == original_owner
asyncio.run(run())
@pytest.mark.parametrize("status", ["running", "interrupted", "uncertain"])
def test_legacy_fabric_attempt_without_child_claim_blocks_update(tmp_path, status):
async def run():
async with running_engine(tmp_path) as (engine, store, _services):
store.set("fabric_desired", "legacy-fabric", json.dumps({
"operation": {"id": "a" * 32, "status": status},
"nodes": [],
}))
assert not engine.coordinator.has_work()
response, _body = await request(engine, "POST", "/internal/v1/update/apply", {
"expected_current_commit": "a" * 40,
"expected_remote_commit": "b" * 40,
})
assert response == 409
assert not engine.coordinator.has_work()
asyncio.run(run())