You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
176 lines
8.3 KiB
Python
176 lines
8.3 KiB
Python
"""Signed operation receipts distinguish no dispatch from unknown execution."""
|
|
|
|
import asyncio
|
|
import json
|
|
|
|
import pytest
|
|
|
|
from test_engine_boundary import (
|
|
CREATE_BODY, USER, VM_ID, boundary, call, host_scope,
|
|
)
|
|
from virsh_panel.coordination import (
|
|
execution_context, mark_execution_unknown, mark_remote_write_started,
|
|
)
|
|
from virsh_panel.libvirt_service import VMError, VMService
|
|
|
|
|
|
def prepare(boundary):
|
|
client, engine, store, _, services = boundary
|
|
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
|
component = engine.registry.components["hk"]
|
|
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
|
|
return client, engine, store, component, services["hk"]
|
|
|
|
|
|
@pytest.mark.parametrize("code", [None, "libvirt_execution_timeout", "libvirt_execution_stopped"])
|
|
def test_prewrite_502_finishes_and_allows_followup_power(boundary, monkeypatch, code):
|
|
client, engine, store, _, service = prepare(boundary)
|
|
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/actions"
|
|
action = service.action
|
|
attempts = []
|
|
tracked = VMService("test", "images", "iso", "default", execution_scope="hk")
|
|
|
|
def failed(vm_id, command):
|
|
attempts.append(command)
|
|
with tracked._operation_guard(vm_id):
|
|
raise VMError("state/XML read failed before dispatch", 502, error_code=code)
|
|
|
|
monkeypatch.setattr(service, "action", failed)
|
|
actor = {**USER, "operation_id": "prewrite-power-receipt"}
|
|
response = call(client, "POST", path, actor, {"action": "reboot"})
|
|
assert response.status_code == 502, response.text
|
|
assert response.json()["status"] == "failed"
|
|
assert response.json()["mutation_started"] is False
|
|
assert response.json()["execution_outcome"] == "not_dispatched"
|
|
assert engine.coordinator.snapshot() == {}
|
|
saved = store.get("operation", actor["operation_id"])
|
|
polled = call(client, "GET", f"/internal/v1/operations/{actor['operation_id']}")
|
|
assert polled.json()["execution_outcome"] == "not_dispatched"
|
|
assert call(client, "POST", path, actor, {"action": "reboot"}).json()["status"] == "failed"
|
|
assert store.get("operation", actor["operation_id"]) == saved
|
|
assert attempts == ["reboot"]
|
|
monkeypatch.setattr(service, "action", action)
|
|
forced = call(client, "POST", path, {**USER, "operation_id": "power-after-prewrite"},
|
|
{"action": "poweroff"})
|
|
assert forced.json()["status"] == "succeeded", forced.text
|
|
assert forced.json()["execution_outcome"] == "acknowledged"
|
|
|
|
|
|
def test_foreign_storage_fence_cannot_spread_to_rejected_vm(boundary, monkeypatch):
|
|
client, engine, _, _, service = prepare(boundary)
|
|
blocker = engine.coordinator.claim(resources=("storage:hk", "vm:hk:other"))
|
|
engine.coordinator.retain(blocker)
|
|
original = engine.coordinator.snapshot()
|
|
action = service.action
|
|
tracked = VMService("test", "images", "iso", "default", execution_scope="hk")
|
|
|
|
def rejected(vm_id, _action):
|
|
with tracked._operation_guard(vm_id, storage=True):
|
|
pytest.fail("A refused request must not execute")
|
|
|
|
monkeypatch.setattr(service, "action", rejected)
|
|
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/actions"
|
|
response = call(client, "POST", path, {**USER, "operation_id": "storage-rejected-vm"},
|
|
{"action": "start"})
|
|
assert response.status_code == 503, response.text
|
|
assert response.json()["status"] == "failed"
|
|
assert response.json()["execution_outcome"] == "not_dispatched"
|
|
assert engine.coordinator.snapshot() == original
|
|
monkeypatch.setattr(service, "action", action)
|
|
next_response = call(client, "POST", path, {**USER, "operation_id": "storage-next-power"},
|
|
{"action": "poweroff"})
|
|
assert next_response.status_code == 200, next_response.text
|
|
assert engine.coordinator.snapshot() == original
|
|
|
|
|
|
@pytest.mark.parametrize("absence", [True, False, "read_failed"])
|
|
def test_create_prewrite_failure_unlocks_but_refund_needs_cleanup(boundary, monkeypatch, absence):
|
|
client, engine, _, component, service = prepare(boundary)
|
|
tracked = VMService("test", "images", "iso", "default", execution_scope="hk")
|
|
|
|
def failed(spec):
|
|
with tracked._operation_guard(spec["vm_uuid"], storage=True, allocation=True):
|
|
raise VMError("traffic preflight failed", 502)
|
|
|
|
def check(*_args, **_kwargs):
|
|
if absence == "read_failed":
|
|
raise VMError("absence read unavailable", 504, error_code="libvirt_execution_timeout")
|
|
return absence
|
|
|
|
monkeypatch.setattr(service, "create_vm", failed)
|
|
monkeypatch.setattr(service, "provisioning_absent", check)
|
|
actor = {**USER, "can_provision": True, "operation_id": "create-prewrite-proof"}
|
|
response = call(client, "POST", "/internal/v1/hosts/hk/vms", actor, CREATE_BODY)
|
|
assert response.status_code == 502, response.text
|
|
assert response.json()["status"] == "failed"
|
|
assert response.json()["execution_outcome"] == "not_dispatched"
|
|
assert response.json()["safe_to_refund"] is (absence is True)
|
|
assert engine.coordinator.snapshot() == {}
|
|
|
|
|
|
@pytest.mark.parametrize("mode", ["write", "unproven_unknown", "database", "unknown_then_409"])
|
|
def test_actual_or_unproven_failure_keeps_vm_protected(boundary, monkeypatch, mode):
|
|
client, engine, _, _, service = prepare(boundary)
|
|
|
|
def failed(*_args):
|
|
if mode == "write":
|
|
mark_remote_write_started()
|
|
raise VMError("response lost after write", 502)
|
|
if mode == "database":
|
|
raise OSError("commit reply lost")
|
|
if mode == "unknown_then_409":
|
|
mark_execution_unknown()
|
|
raise VMError("later validation rejection cannot erase unknown commit", 409)
|
|
raise VMError("unproven admission outcome", 503, error_code="coordination_uncertain")
|
|
|
|
monkeypatch.setattr(service, "action", failed)
|
|
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/actions"
|
|
response = call(client, "POST", path, {**USER, "operation_id": "unknown-power-receipt"},
|
|
{"action": "reset"})
|
|
assert response.status_code >= 500 or mode == "unknown_then_409"
|
|
claims = engine.coordinator.snapshot()
|
|
assert any(row["state"] == "uncertain" and f"vm:hk:{VM_ID}" in row["resources"]
|
|
for row in claims.values())
|
|
blocked = call(client, "POST", path, {**USER, "operation_id": "power-still-protected"},
|
|
{"action": "poweroff"})
|
|
assert blocked.status_code == 503
|
|
|
|
|
|
@pytest.mark.parametrize("state", ["pending_child", "write_during_snapshot"])
|
|
def test_no_dispatch_proof_cannot_race_background_handoff(boundary, monkeypatch, state):
|
|
_, engine, _, _, _ = prepare(boundary)
|
|
parent = engine.coordinator.claim()
|
|
with execution_context(engine.coordinator, parent, tracked_mutations=True):
|
|
if state == "pending_child":
|
|
engine.coordinator.claim(execution_id=parent.execution_id, kind="background")
|
|
else:
|
|
snapshot = engine.coordinator.snapshot
|
|
|
|
def child_finished():
|
|
mark_remote_write_started()
|
|
return snapshot()
|
|
|
|
monkeypatch.setattr(engine.coordinator, "snapshot", child_finished)
|
|
assert not asyncio.run(engine._failed_before_dispatch({}, "libvirt_execution_timeout"))
|
|
|
|
|
|
@pytest.mark.parametrize("proof", [True, False])
|
|
def test_background_operation_uses_only_explicit_new_no_dispatch_receipt(boundary, proof):
|
|
client, engine, store, component, _ = prepare(boundary)
|
|
op_id, job_id = "prewrite-background", "job-prewrite"
|
|
job = {"operation_id": op_id, "owner": "php:41", "vm_id": VM_ID,
|
|
"status": "failed", "error_code": "coordination_uncertain", "error": "refused"}
|
|
if proof:
|
|
job.update(mutation_started=False, execution_outcome="not_dispatched")
|
|
component.store.set("reinstall_job", job_id, json.dumps(job))
|
|
store.set("operation", op_id, json.dumps({
|
|
"operation_id": op_id, "host_id": "hk", "user_id": 41, "kind": "mutation",
|
|
"resource": f"vms/{VM_ID}/reinstall", "vm_uuid": VM_ID, "job_id": job_id,
|
|
"status": "pending", "mutation_started": True, "execution_id": "a" * 64,
|
|
"result": None, "safe_to_refund": False,
|
|
}))
|
|
result = call(client, "GET", f"/internal/v1/operations/{op_id}").json()
|
|
assert result["status"] == ("failed" if proof else "uncertain"), result
|
|
assert result["execution_outcome"] == ("not_dispatched" if proof else "unknown")
|
|
assert result["mutation_started"] is (not proof)
|