You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/engine/tests/test_update_preflight_coord...

280 lines
12 KiB
Python

"""Read-only update checks cannot leave a global maintenance fence.
Use the real Updater and signed ASGI boundary. Only Git observations and the
systemd worker are replaced; persistence and coordinator admission remain real.
"""
from __future__ import annotations
import asyncio
import json
import threading
import pytest
from test_engine_coordination import (
local_backends, prepare_hosts, request, running_engine, vm_path, wait_until,
)
from virsh_panel.updater import UpdateError, Updater
UPDATE = "/internal/v1/update/apply"
CONFIRMED = {
"expected_current_commit": "a" * 40,
"expected_remote_commit": "b" * 40,
}
def install_updater(engine, store, directory, monkeypatch):
updater = Updater(directory, store)
updater.set_token("test-only-update-token-123")
monkeypatch.setattr(updater, "_checkout_head", lambda **_kwargs: "a" * 40)
monkeypatch.setattr(updater, "_remote_head", lambda: "b" * 40)
engine.updater = updater
started = []
def start_worker():
job = updater.active_job()
assert job is not None
started.append(job["job_id"])
job.update(status="running", started_at=job["created_at"])
updater._save_job(job)
engine.update_trigger = start_worker
return updater, started
def saved_jobs(store):
return {key: json.loads(value) for key, value in store.list("value").items()
if key.startswith("update.job.")}
async def assert_unrelated_vm_can_write(engine, services, operation_id):
status, payload = await asyncio.wait_for(request(
engine, "POST", vm_path("host-b"), {"action": "reset"},
operation_id=operation_id,
), 1)
assert status == 200, payload
assert any(call[0] == "action" for call in services["host-b"].calls)
def test_remote_preflight_502_creates_no_task_or_maintenance_claim(tmp_path, monkeypatch):
async def run():
async with running_engine(tmp_path) as (engine, store, services):
await prepare_hosts(engine, "host-a", "host-b")
updater, started = install_updater(engine, store, tmp_path, monkeypatch)
def unavailable():
raise UpdateError("remote Git unavailable", 502)
monkeypatch.setattr(updater, "_remote_head", unavailable)
status, payload = await request(engine, "POST", UPDATE, CONFIRMED)
assert status == 502, payload
assert saved_jobs(store) == {}
assert store.get_value("update.active_job") is None
assert started == []
await assert_unrelated_vm_can_write(engine, services, "preflight-502-unrelated-vm")
assert engine.coordinator.snapshot() == {}
asyncio.run(run())
def test_slow_preflight_does_not_serialize_unrelated_vm_writes(tmp_path, monkeypatch):
async def run():
async with running_engine(tmp_path) as (engine, store, services):
await prepare_hosts(engine, "host-a", "host-b")
updater, started = install_updater(engine, store, tmp_path, monkeypatch)
entered, release = threading.Event(), threading.Event()
def unavailable():
entered.set()
assert release.wait(5), "test did not release remote Git check"
raise UpdateError("remote Git unavailable", 502)
monkeypatch.setattr(updater, "_remote_head", unavailable)
update = asyncio.create_task(request(engine, "POST", UPDATE, CONFIRMED))
try:
await wait_until(entered.is_set)
await assert_unrelated_vm_can_write(engine, services, "slow-preflight-unrelated-vm")
assert engine.coordinator.snapshot() == {}
assert not update.done()
assert saved_jobs(store) == {}
assert started == []
finally:
release.set()
status, payload = await asyncio.wait_for(update, 3)
assert status == 502, payload
assert engine.coordinator.snapshot() == {}
asyncio.run(run())
@pytest.mark.parametrize("failure", ["local_changed", "remote_changed", "remote_502", "remote_exception"])
def test_final_version_recheck_failure_releases_only_its_unpublished_claim(
tmp_path, monkeypatch, failure):
async def run():
async with running_engine(tmp_path) as (engine, store, services):
await prepare_hosts(engine, "host-b")
updater, started = install_updater(engine, store, tmp_path, monkeypatch)
calls, final_claims = [], []
def observe():
calls.append(True)
if len(calls) == 1:
assert engine.coordinator.snapshot() == {}
return "a" * 40 if failure == "local_changed" else "b" * 40
claims = engine.coordinator.snapshot()
assert len(claims) == 1
claim = next(iter(claims.values()))
assert claim["maintenance"] is True
assert claim["metadata"]["update_job_id"]
final_claims.append(claim)
if failure == "remote_502":
raise UpdateError("second remote Git check unavailable", 502)
if failure == "remote_exception":
raise OSError("remote observation could not start")
return "c" * 40
if failure == "local_changed":
monkeypatch.setattr(updater, "_checkout_head", lambda **_kwargs: observe())
else:
monkeypatch.setattr(updater, "_remote_head", observe)
status, payload = await request(engine, "POST", UPDATE, CONFIRMED)
assert status == (409 if failure.endswith("changed") else
502 if failure == "remote_502" else 503), payload
assert len(final_claims) == 1
assert len(calls) == 2
assert engine.coordinator.snapshot() == {}
assert saved_jobs(store) == {}
assert store.get_value("update.active_job") is None
assert started == []
await assert_unrelated_vm_can_write(
engine, services, "final-preflight-" + failure)
asyncio.run(run())
@pytest.mark.parametrize("lost_receipt", ["job", "active_pointer"])
@pytest.mark.parametrize("error_type", [OSError, UpdateError], ids=["io-error", "update-409"])
def test_published_update_with_lost_database_receipt_keeps_bound_maintenance_claim(
tmp_path, monkeypatch, lost_receipt, error_type):
async def run():
async with running_engine(tmp_path) as (engine, store, services):
await prepare_hosts(engine, "host-b")
updater, started = install_updater(engine, store, tmp_path, monkeypatch)
persist = store.set_value
def save_then_disconnect(key, value):
persist(key, value)
if ((lost_receipt == "job" and key.startswith("update.job."))
or (lost_receipt == "active_pointer" and key == "update.active_job")):
if error_type is UpdateError:
# HTTP status alone is not proof that publication did
# not happen. Preserve the same fence for a 4xx reply.
raise UpdateError("database committed before a conflict reply", 409)
raise OSError("database committed but its reply was lost")
monkeypatch.setattr(store, "set_value", save_then_disconnect)
status, payload = await request(engine, "POST", UPDATE, CONFIRMED)
assert status == (409 if error_type is UpdateError else 503), payload
jobs = saved_jobs(store)
assert len(jobs) == 1
job = next(iter(jobs.values()))
claims = engine.coordinator.snapshot()
assert len(claims) == 1
claim = next(iter(claims.values()))
assert claim["state"] == "uncertain"
assert claim["metadata"]["update_job_id"] == job["job_id"]
assert updater.get_job(claim["metadata"]["update_job_id"]) == job
assert started == []
status, payload = await request(
engine, "POST", vm_path("host-b"), {"action": "reset"},
operation_id="published-response-lost-" + lost_receipt,
)
assert status == 503, payload
assert payload["error_code"] == "coordination_uncertain"
assert not any(call[0] == "action" for call in services["host-b"].calls)
asyncio.run(run())
def test_two_preflights_racing_publish_only_one_update(tmp_path, monkeypatch):
async def run():
async with running_engine(tmp_path) as (engine, store, _services):
updater, started = install_updater(engine, store, tmp_path, monkeypatch)
barrier = threading.Barrier(2)
lock = threading.Lock()
calls = []
def remote_head():
with lock:
calls.append(True)
first_check = len(calls) <= 2
if first_check:
barrier.wait(timeout=3)
return "b" * 40
monkeypatch.setattr(updater, "_remote_head", remote_head)
results = await asyncio.wait_for(asyncio.gather(
request(engine, "POST", UPDATE, CONFIRMED),
request(engine, "POST", UPDATE, CONFIRMED),
), 5)
assert sorted(status for status, _payload in results) == [202, 409], results
assert len(saved_jobs(store)) == 1
assert len(started) == 1
assert len(engine.coordinator.snapshot()) == 1
job = updater.active_job()
assert job["job_id"] == started[0]
claim = next(iter(engine.coordinator.snapshot().values()))
assert claim["metadata"]["update_job_id"] == job["job_id"]
asyncio.run(run())
def test_cancellation_during_final_check_cannot_release_claim_while_thread_can_publish(
tmp_path, monkeypatch):
async def run():
async with running_engine(tmp_path) as (engine, store, services):
await prepare_hosts(engine, "host-b")
updater, started = install_updater(engine, store, tmp_path, monkeypatch)
entered, release = threading.Event(), threading.Event()
calls = []
def remote_head():
calls.append(True)
if len(calls) == 2:
entered.set()
assert release.wait(5), "test did not release final Git check"
return "b" * 40
monkeypatch.setattr(updater, "_remote_head", remote_head)
update = asyncio.create_task(request(engine, "POST", UPDATE, CONFIRMED))
try:
await wait_until(entered.is_set)
# Simulate engine shutdown cancelling its owned coroutine, not
# merely a disconnected HTTP client (which is shielded).
assert len(engine.request_tasks) == 1
next(iter(engine.request_tasks)).cancel()
with pytest.raises(asyncio.CancelledError):
await update
claims = engine.coordinator.snapshot()
assert len(claims) == 1
claim = next(iter(claims.values()))
assert claim["state"] == "uncertain"
assert claim["metadata"]["update_job_id"]
assert saved_jobs(store) == {}
finally:
release.set()
await wait_until(lambda: bool(saved_jobs(store)))
assert next(iter(saved_jobs(store).values()))["job_id"] == claim["metadata"]["update_job_id"]
assert started == []
status, payload = await request(
engine, "POST", vm_path("host-b"), {"action": "reset"},
operation_id="cancelled-create-still-published",
)
assert status == 503, payload
assert payload["error_code"] == "coordination_uncertain"
assert not any(call[0] == "action" for call in services["host-b"].calls)
asyncio.run(run())