You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/engine/tests/test_prewrite_execution_bou...

176 lines
8.3 KiB
Python

"""Signed operation receipts distinguish no dispatch from unknown execution."""
import asyncio
import json
import pytest
from test_engine_boundary import (
CREATE_BODY, USER, VM_ID, boundary, call, host_scope,
)
from virsh_panel.coordination import (
execution_context, mark_execution_unknown, mark_remote_write_started,
)
from virsh_panel.libvirt_service import VMError, VMService
def prepare(boundary):
client, engine, store, _, services = boundary
call(client, "GET", "/internal/v1/hosts/hk/vms")
component = engine.registry.components["hk"]
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
return client, engine, store, component, services["hk"]
@pytest.mark.parametrize("code", [None, "libvirt_execution_timeout", "libvirt_execution_stopped"])
def test_prewrite_502_finishes_and_allows_followup_power(boundary, monkeypatch, code):
client, engine, store, _, service = prepare(boundary)
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/actions"
action = service.action
attempts = []
tracked = VMService("test", "images", "iso", "default", execution_scope="hk")
def failed(vm_id, command):
attempts.append(command)
with tracked._operation_guard(vm_id):
raise VMError("state/XML read failed before dispatch", 502, error_code=code)
monkeypatch.setattr(service, "action", failed)
actor = {**USER, "operation_id": "prewrite-power-receipt"}
response = call(client, "POST", path, actor, {"action": "reboot"})
assert response.status_code == 502, response.text
assert response.json()["status"] == "failed"
assert response.json()["mutation_started"] is False
assert response.json()["execution_outcome"] == "not_dispatched"
assert engine.coordinator.snapshot() == {}
saved = store.get("operation", actor["operation_id"])
polled = call(client, "GET", f"/internal/v1/operations/{actor['operation_id']}")
assert polled.json()["execution_outcome"] == "not_dispatched"
assert call(client, "POST", path, actor, {"action": "reboot"}).json()["status"] == "failed"
assert store.get("operation", actor["operation_id"]) == saved
assert attempts == ["reboot"]
monkeypatch.setattr(service, "action", action)
forced = call(client, "POST", path, {**USER, "operation_id": "power-after-prewrite"},
{"action": "poweroff"})
assert forced.json()["status"] == "succeeded", forced.text
assert forced.json()["execution_outcome"] == "acknowledged"
def test_foreign_storage_fence_cannot_spread_to_rejected_vm(boundary, monkeypatch):
client, engine, _, _, service = prepare(boundary)
blocker = engine.coordinator.claim(resources=("storage:hk", "vm:hk:other"))
engine.coordinator.retain(blocker)
original = engine.coordinator.snapshot()
action = service.action
tracked = VMService("test", "images", "iso", "default", execution_scope="hk")
def rejected(vm_id, _action):
with tracked._operation_guard(vm_id, storage=True):
pytest.fail("A refused request must not execute")
monkeypatch.setattr(service, "action", rejected)
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/actions"
response = call(client, "POST", path, {**USER, "operation_id": "storage-rejected-vm"},
{"action": "start"})
assert response.status_code == 503, response.text
assert response.json()["status"] == "failed"
assert response.json()["execution_outcome"] == "not_dispatched"
assert engine.coordinator.snapshot() == original
monkeypatch.setattr(service, "action", action)
next_response = call(client, "POST", path, {**USER, "operation_id": "storage-next-power"},
{"action": "poweroff"})
assert next_response.status_code == 200, next_response.text
assert engine.coordinator.snapshot() == original
@pytest.mark.parametrize("absence", [True, False, "read_failed"])
def test_create_prewrite_failure_unlocks_but_refund_needs_cleanup(boundary, monkeypatch, absence):
client, engine, _, component, service = prepare(boundary)
tracked = VMService("test", "images", "iso", "default", execution_scope="hk")
def failed(spec):
with tracked._operation_guard(spec["vm_uuid"], storage=True, allocation=True):
raise VMError("traffic preflight failed", 502)
def check(*_args, **_kwargs):
if absence == "read_failed":
raise VMError("absence read unavailable", 504, error_code="libvirt_execution_timeout")
return absence
monkeypatch.setattr(service, "create_vm", failed)
monkeypatch.setattr(service, "provisioning_absent", check)
actor = {**USER, "can_provision": True, "operation_id": "create-prewrite-proof"}
response = call(client, "POST", "/internal/v1/hosts/hk/vms", actor, CREATE_BODY)
assert response.status_code == 502, response.text
assert response.json()["status"] == "failed"
assert response.json()["execution_outcome"] == "not_dispatched"
assert response.json()["safe_to_refund"] is (absence is True)
assert engine.coordinator.snapshot() == {}
@pytest.mark.parametrize("mode", ["write", "unproven_unknown", "database", "unknown_then_409"])
def test_actual_or_unproven_failure_keeps_vm_protected(boundary, monkeypatch, mode):
client, engine, _, _, service = prepare(boundary)
def failed(*_args):
if mode == "write":
mark_remote_write_started()
raise VMError("response lost after write", 502)
if mode == "database":
raise OSError("commit reply lost")
if mode == "unknown_then_409":
mark_execution_unknown()
raise VMError("later validation rejection cannot erase unknown commit", 409)
raise VMError("unproven admission outcome", 503, error_code="coordination_uncertain")
monkeypatch.setattr(service, "action", failed)
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/actions"
response = call(client, "POST", path, {**USER, "operation_id": "unknown-power-receipt"},
{"action": "reset"})
assert response.status_code >= 500 or mode == "unknown_then_409"
claims = engine.coordinator.snapshot()
assert any(row["state"] == "uncertain" and f"vm:hk:{VM_ID}" in row["resources"]
for row in claims.values())
blocked = call(client, "POST", path, {**USER, "operation_id": "power-still-protected"},
{"action": "poweroff"})
assert blocked.status_code == 503
@pytest.mark.parametrize("state", ["pending_child", "write_during_snapshot"])
def test_no_dispatch_proof_cannot_race_background_handoff(boundary, monkeypatch, state):
_, engine, _, _, _ = prepare(boundary)
parent = engine.coordinator.claim()
with execution_context(engine.coordinator, parent, tracked_mutations=True):
if state == "pending_child":
engine.coordinator.claim(execution_id=parent.execution_id, kind="background")
else:
snapshot = engine.coordinator.snapshot
def child_finished():
mark_remote_write_started()
return snapshot()
monkeypatch.setattr(engine.coordinator, "snapshot", child_finished)
assert not asyncio.run(engine._failed_before_dispatch({}, "libvirt_execution_timeout"))
@pytest.mark.parametrize("proof", [True, False])
def test_background_operation_uses_only_explicit_new_no_dispatch_receipt(boundary, proof):
client, engine, store, component, _ = prepare(boundary)
op_id, job_id = "prewrite-background", "job-prewrite"
job = {"operation_id": op_id, "owner": "php:41", "vm_id": VM_ID,
"status": "failed", "error_code": "coordination_uncertain", "error": "refused"}
if proof:
job.update(mutation_started=False, execution_outcome="not_dispatched")
component.store.set("reinstall_job", job_id, json.dumps(job))
store.set("operation", op_id, json.dumps({
"operation_id": op_id, "host_id": "hk", "user_id": 41, "kind": "mutation",
"resource": f"vms/{VM_ID}/reinstall", "vm_uuid": VM_ID, "job_id": job_id,
"status": "pending", "mutation_started": True, "execution_id": "a" * 64,
"result": None, "safe_to_refund": False,
}))
result = call(client, "GET", f"/internal/v1/operations/{op_id}").json()
assert result["status"] == ("failed" if proof else "uncertain"), result
assert result["execution_outcome"] == ("not_dispatched" if proof else "unknown")
assert result["mutation_started"] is (not proof)