You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
280 lines
12 KiB
Python
280 lines
12 KiB
Python
"""Read-only update checks cannot leave a global maintenance fence.
|
|
|
|
Use the real Updater and signed ASGI boundary. Only Git observations and the
|
|
systemd worker are replaced; persistence and coordinator admission remain real.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
import threading
|
|
|
|
import pytest
|
|
|
|
from test_engine_coordination import (
|
|
local_backends, prepare_hosts, request, running_engine, vm_path, wait_until,
|
|
)
|
|
from virsh_panel.updater import UpdateError, Updater
|
|
|
|
|
|
UPDATE = "/internal/v1/update/apply"
|
|
CONFIRMED = {
|
|
"expected_current_commit": "a" * 40,
|
|
"expected_remote_commit": "b" * 40,
|
|
}
|
|
|
|
|
|
def install_updater(engine, store, directory, monkeypatch):
|
|
updater = Updater(directory, store)
|
|
updater.set_token("test-only-update-token-123")
|
|
monkeypatch.setattr(updater, "_checkout_head", lambda **_kwargs: "a" * 40)
|
|
monkeypatch.setattr(updater, "_remote_head", lambda: "b" * 40)
|
|
engine.updater = updater
|
|
started = []
|
|
|
|
def start_worker():
|
|
job = updater.active_job()
|
|
assert job is not None
|
|
started.append(job["job_id"])
|
|
job.update(status="running", started_at=job["created_at"])
|
|
updater._save_job(job)
|
|
|
|
engine.update_trigger = start_worker
|
|
return updater, started
|
|
|
|
|
|
def saved_jobs(store):
|
|
return {key: json.loads(value) for key, value in store.list("value").items()
|
|
if key.startswith("update.job.")}
|
|
|
|
|
|
async def assert_unrelated_vm_can_write(engine, services, operation_id):
|
|
status, payload = await asyncio.wait_for(request(
|
|
engine, "POST", vm_path("host-b"), {"action": "reset"},
|
|
operation_id=operation_id,
|
|
), 1)
|
|
assert status == 200, payload
|
|
assert any(call[0] == "action" for call in services["host-b"].calls)
|
|
|
|
|
|
def test_remote_preflight_502_creates_no_task_or_maintenance_claim(tmp_path, monkeypatch):
|
|
async def run():
|
|
async with running_engine(tmp_path) as (engine, store, services):
|
|
await prepare_hosts(engine, "host-a", "host-b")
|
|
updater, started = install_updater(engine, store, tmp_path, monkeypatch)
|
|
|
|
def unavailable():
|
|
raise UpdateError("remote Git unavailable", 502)
|
|
|
|
monkeypatch.setattr(updater, "_remote_head", unavailable)
|
|
status, payload = await request(engine, "POST", UPDATE, CONFIRMED)
|
|
assert status == 502, payload
|
|
assert saved_jobs(store) == {}
|
|
assert store.get_value("update.active_job") is None
|
|
assert started == []
|
|
await assert_unrelated_vm_can_write(engine, services, "preflight-502-unrelated-vm")
|
|
assert engine.coordinator.snapshot() == {}
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_slow_preflight_does_not_serialize_unrelated_vm_writes(tmp_path, monkeypatch):
|
|
async def run():
|
|
async with running_engine(tmp_path) as (engine, store, services):
|
|
await prepare_hosts(engine, "host-a", "host-b")
|
|
updater, started = install_updater(engine, store, tmp_path, monkeypatch)
|
|
entered, release = threading.Event(), threading.Event()
|
|
|
|
def unavailable():
|
|
entered.set()
|
|
assert release.wait(5), "test did not release remote Git check"
|
|
raise UpdateError("remote Git unavailable", 502)
|
|
|
|
monkeypatch.setattr(updater, "_remote_head", unavailable)
|
|
update = asyncio.create_task(request(engine, "POST", UPDATE, CONFIRMED))
|
|
try:
|
|
await wait_until(entered.is_set)
|
|
await assert_unrelated_vm_can_write(engine, services, "slow-preflight-unrelated-vm")
|
|
assert engine.coordinator.snapshot() == {}
|
|
assert not update.done()
|
|
assert saved_jobs(store) == {}
|
|
assert started == []
|
|
finally:
|
|
release.set()
|
|
status, payload = await asyncio.wait_for(update, 3)
|
|
assert status == 502, payload
|
|
assert engine.coordinator.snapshot() == {}
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
@pytest.mark.parametrize("failure", ["local_changed", "remote_changed", "remote_502", "remote_exception"])
|
|
def test_final_version_recheck_failure_releases_only_its_unpublished_claim(
|
|
tmp_path, monkeypatch, failure):
|
|
async def run():
|
|
async with running_engine(tmp_path) as (engine, store, services):
|
|
await prepare_hosts(engine, "host-b")
|
|
updater, started = install_updater(engine, store, tmp_path, monkeypatch)
|
|
calls, final_claims = [], []
|
|
|
|
def observe():
|
|
calls.append(True)
|
|
if len(calls) == 1:
|
|
assert engine.coordinator.snapshot() == {}
|
|
return "a" * 40 if failure == "local_changed" else "b" * 40
|
|
claims = engine.coordinator.snapshot()
|
|
assert len(claims) == 1
|
|
claim = next(iter(claims.values()))
|
|
assert claim["maintenance"] is True
|
|
assert claim["metadata"]["update_job_id"]
|
|
final_claims.append(claim)
|
|
if failure == "remote_502":
|
|
raise UpdateError("second remote Git check unavailable", 502)
|
|
if failure == "remote_exception":
|
|
raise OSError("remote observation could not start")
|
|
return "c" * 40
|
|
|
|
if failure == "local_changed":
|
|
monkeypatch.setattr(updater, "_checkout_head", lambda **_kwargs: observe())
|
|
else:
|
|
monkeypatch.setattr(updater, "_remote_head", observe)
|
|
status, payload = await request(engine, "POST", UPDATE, CONFIRMED)
|
|
assert status == (409 if failure.endswith("changed") else
|
|
502 if failure == "remote_502" else 503), payload
|
|
assert len(final_claims) == 1
|
|
assert len(calls) == 2
|
|
assert engine.coordinator.snapshot() == {}
|
|
assert saved_jobs(store) == {}
|
|
assert store.get_value("update.active_job") is None
|
|
assert started == []
|
|
await assert_unrelated_vm_can_write(
|
|
engine, services, "final-preflight-" + failure)
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
@pytest.mark.parametrize("lost_receipt", ["job", "active_pointer"])
|
|
@pytest.mark.parametrize("error_type", [OSError, UpdateError], ids=["io-error", "update-409"])
|
|
def test_published_update_with_lost_database_receipt_keeps_bound_maintenance_claim(
|
|
tmp_path, monkeypatch, lost_receipt, error_type):
|
|
async def run():
|
|
async with running_engine(tmp_path) as (engine, store, services):
|
|
await prepare_hosts(engine, "host-b")
|
|
updater, started = install_updater(engine, store, tmp_path, monkeypatch)
|
|
persist = store.set_value
|
|
|
|
def save_then_disconnect(key, value):
|
|
persist(key, value)
|
|
if ((lost_receipt == "job" and key.startswith("update.job."))
|
|
or (lost_receipt == "active_pointer" and key == "update.active_job")):
|
|
if error_type is UpdateError:
|
|
# HTTP status alone is not proof that publication did
|
|
# not happen. Preserve the same fence for a 4xx reply.
|
|
raise UpdateError("database committed before a conflict reply", 409)
|
|
raise OSError("database committed but its reply was lost")
|
|
|
|
monkeypatch.setattr(store, "set_value", save_then_disconnect)
|
|
status, payload = await request(engine, "POST", UPDATE, CONFIRMED)
|
|
assert status == (409 if error_type is UpdateError else 503), payload
|
|
jobs = saved_jobs(store)
|
|
assert len(jobs) == 1
|
|
job = next(iter(jobs.values()))
|
|
claims = engine.coordinator.snapshot()
|
|
assert len(claims) == 1
|
|
claim = next(iter(claims.values()))
|
|
assert claim["state"] == "uncertain"
|
|
assert claim["metadata"]["update_job_id"] == job["job_id"]
|
|
assert updater.get_job(claim["metadata"]["update_job_id"]) == job
|
|
assert started == []
|
|
status, payload = await request(
|
|
engine, "POST", vm_path("host-b"), {"action": "reset"},
|
|
operation_id="published-response-lost-" + lost_receipt,
|
|
)
|
|
assert status == 503, payload
|
|
assert payload["error_code"] == "coordination_uncertain"
|
|
assert not any(call[0] == "action" for call in services["host-b"].calls)
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_two_preflights_racing_publish_only_one_update(tmp_path, monkeypatch):
|
|
async def run():
|
|
async with running_engine(tmp_path) as (engine, store, _services):
|
|
updater, started = install_updater(engine, store, tmp_path, monkeypatch)
|
|
barrier = threading.Barrier(2)
|
|
lock = threading.Lock()
|
|
calls = []
|
|
|
|
def remote_head():
|
|
with lock:
|
|
calls.append(True)
|
|
first_check = len(calls) <= 2
|
|
if first_check:
|
|
barrier.wait(timeout=3)
|
|
return "b" * 40
|
|
|
|
monkeypatch.setattr(updater, "_remote_head", remote_head)
|
|
results = await asyncio.wait_for(asyncio.gather(
|
|
request(engine, "POST", UPDATE, CONFIRMED),
|
|
request(engine, "POST", UPDATE, CONFIRMED),
|
|
), 5)
|
|
assert sorted(status for status, _payload in results) == [202, 409], results
|
|
assert len(saved_jobs(store)) == 1
|
|
assert len(started) == 1
|
|
assert len(engine.coordinator.snapshot()) == 1
|
|
job = updater.active_job()
|
|
assert job["job_id"] == started[0]
|
|
claim = next(iter(engine.coordinator.snapshot().values()))
|
|
assert claim["metadata"]["update_job_id"] == job["job_id"]
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_cancellation_during_final_check_cannot_release_claim_while_thread_can_publish(
|
|
tmp_path, monkeypatch):
|
|
async def run():
|
|
async with running_engine(tmp_path) as (engine, store, services):
|
|
await prepare_hosts(engine, "host-b")
|
|
updater, started = install_updater(engine, store, tmp_path, monkeypatch)
|
|
entered, release = threading.Event(), threading.Event()
|
|
calls = []
|
|
|
|
def remote_head():
|
|
calls.append(True)
|
|
if len(calls) == 2:
|
|
entered.set()
|
|
assert release.wait(5), "test did not release final Git check"
|
|
return "b" * 40
|
|
|
|
monkeypatch.setattr(updater, "_remote_head", remote_head)
|
|
update = asyncio.create_task(request(engine, "POST", UPDATE, CONFIRMED))
|
|
try:
|
|
await wait_until(entered.is_set)
|
|
# Simulate engine shutdown cancelling its owned coroutine, not
|
|
# merely a disconnected HTTP client (which is shielded).
|
|
assert len(engine.request_tasks) == 1
|
|
next(iter(engine.request_tasks)).cancel()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await update
|
|
claims = engine.coordinator.snapshot()
|
|
assert len(claims) == 1
|
|
claim = next(iter(claims.values()))
|
|
assert claim["state"] == "uncertain"
|
|
assert claim["metadata"]["update_job_id"]
|
|
assert saved_jobs(store) == {}
|
|
finally:
|
|
release.set()
|
|
await wait_until(lambda: bool(saved_jobs(store)))
|
|
assert next(iter(saved_jobs(store).values()))["job_id"] == claim["metadata"]["update_job_id"]
|
|
assert started == []
|
|
status, payload = await request(
|
|
engine, "POST", vm_path("host-b"), {"action": "reset"},
|
|
operation_id="cancelled-create-still-published",
|
|
)
|
|
assert status == 503, payload
|
|
assert payload["error_code"] == "coordination_uncertain"
|
|
assert not any(call[0] == "action" for call in services["host-b"].calls)
|
|
|
|
asyncio.run(run())
|