fix(engine): release proven undispatched execution claims
parent
2b87fe89f2
commit
e55b26f1fd
@ -0,0 +1,315 @@
|
||||
"""Live prewrite receipts settle only the worker that produced them."""
|
||||
|
||||
import inspect
|
||||
import json
|
||||
import threading
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
from test_app import (
|
||||
VM_ID, configure_host, configure_reinstall_case, configure_snapshot_case,
|
||||
setup_admin, web, # noqa: F401
|
||||
)
|
||||
from test_durable_jobs import MemoryCAS, jobs # noqa: F401
|
||||
from virsh_panel import app as app_module
|
||||
from virsh_panel.coordination import (
|
||||
CoordinationError, DurableCoordinator, execution_context,
|
||||
current_execution, execution_failure_before_write, mark_admission_rejected,
|
||||
mark_execution_unknown, mark_remote_write_started,
|
||||
)
|
||||
from virsh_panel.database import DatabaseError
|
||||
from virsh_panel.durable_jobs import read_job_outcome, submit_durable
|
||||
from virsh_panel.libvirt_service import VMError
|
||||
|
||||
|
||||
@pytest.mark.parametrize("error_code", [
|
||||
None, "libvirt_execution_timeout", "libvirt_execution_stopped",
|
||||
"libvirt_connection_failed", "coordination_uncertain",
|
||||
])
|
||||
def test_live_failed_worker_receipt_releases_claim(jobs, error_code):
|
||||
store, coordinator, parent, executor = jobs
|
||||
|
||||
def worker():
|
||||
if error_code == "coordination_uncertain":
|
||||
mark_admission_rejected()
|
||||
assert execution_failure_before_write(error_code)
|
||||
store.set("snapshot_job", "job-one", json.dumps({
|
||||
"status": "failed", "error_code": error_code,
|
||||
"mutation_started": False, "execution_outcome": "not_dispatched",
|
||||
}))
|
||||
|
||||
with execution_context(coordinator, parent, tracked_mutations=True):
|
||||
future = submit_durable(
|
||||
executor, worker,
|
||||
outcome=lambda: read_job_outcome(store, "snapshot_job", "job-one"),
|
||||
job_kind="snapshot_job", job_id="job-one")
|
||||
future.result(timeout=2)
|
||||
assert list(coordinator.snapshot()) == [parent.token]
|
||||
coordinator.release(parent)
|
||||
replacement = coordinator.claim(resources=parent.resources)
|
||||
assert coordinator.release(replacement)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("unsafe", ["untracked", "prior_write", "unknown", "old_record", "database"])
|
||||
def test_stored_prewrite_flag_does_not_override_missing_or_contrary_evidence(jobs, unsafe):
|
||||
store, coordinator, parent, executor = jobs
|
||||
record = {
|
||||
"status": "failed", "error_code": "libvirt_execution_timeout",
|
||||
"mutation_started": False, "execution_outcome": "not_dispatched",
|
||||
}
|
||||
if unsafe == "unknown":
|
||||
record["error_code"] = "libvirt_execution_uncertain"
|
||||
if unsafe == "old_record":
|
||||
record.pop("execution_outcome")
|
||||
|
||||
def worker():
|
||||
if unsafe == "prior_write":
|
||||
mark_remote_write_started()
|
||||
if unsafe == "database":
|
||||
mark_execution_unknown()
|
||||
store.set("snapshot_job", "job-one", json.dumps(record))
|
||||
|
||||
with execution_context(coordinator, parent, tracked_mutations=unsafe != "untracked"):
|
||||
future = submit_durable(
|
||||
executor, worker,
|
||||
outcome=lambda: read_job_outcome(store, "snapshot_job", "job-one"),
|
||||
job_kind="snapshot_job", job_id="job-one")
|
||||
future.result(timeout=2)
|
||||
child = next(row for row in coordinator.snapshot().values() if row["kind"] == "background")
|
||||
assert child["state"] == "uncertain"
|
||||
|
||||
|
||||
def test_tracked_unwritten_queued_and_running_worker_still_blocks_competitors(jobs):
|
||||
store, coordinator, parent, executor = jobs
|
||||
occupied, worker_started = threading.Event(), threading.Event()
|
||||
release_queue, release_worker = threading.Event(), threading.Event()
|
||||
|
||||
def occupy():
|
||||
occupied.set()
|
||||
assert release_queue.wait(2)
|
||||
|
||||
def worker():
|
||||
worker_started.set()
|
||||
assert execution_failure_before_write("libvirt_execution_timeout")
|
||||
assert release_worker.wait(2)
|
||||
store.set("snapshot_job", "job-one", json.dumps({"status": "completed"}))
|
||||
|
||||
blocker = executor.submit(occupy)
|
||||
assert occupied.wait(2)
|
||||
try:
|
||||
with execution_context(coordinator, parent, tracked_mutations=True):
|
||||
future = submit_durable(
|
||||
executor, worker,
|
||||
outcome=lambda: read_job_outcome(store, "snapshot_job", "job-one"),
|
||||
job_kind="snapshot_job", job_id="job-one")
|
||||
coordinator.release(parent)
|
||||
with pytest.raises(CoordinationError):
|
||||
coordinator.claim(resources=parent.resources)
|
||||
release_queue.set()
|
||||
blocker.result(timeout=2)
|
||||
assert worker_started.wait(2)
|
||||
with pytest.raises(CoordinationError):
|
||||
coordinator.claim(resources=parent.resources)
|
||||
finally:
|
||||
release_queue.set()
|
||||
release_worker.set()
|
||||
future.result(timeout=2)
|
||||
assert coordinator.snapshot() == {}
|
||||
|
||||
|
||||
def instrument_background(monkeypatch, *, write=False):
|
||||
"""Supply the live RPC evidence context to the app's fake host worker."""
|
||||
coordinator = DurableCoordinator(MemoryCAS())
|
||||
parent = coordinator.claim(resources=("vm:host-a:" + VM_ID,))
|
||||
original = app_module.submit_durable
|
||||
seen = {}
|
||||
|
||||
def submit(executor, fn, *args, **kwargs):
|
||||
kind = kwargs["job_kind"]
|
||||
seen[kind] = inspect.getclosurevars(fn).nonlocals
|
||||
|
||||
def run(*worker_args, **worker_kwargs):
|
||||
with execution_context(coordinator, parent, tracked_mutations=True):
|
||||
seen["execution"] = current_execution()
|
||||
if write:
|
||||
mark_remote_write_started()
|
||||
return fn(*worker_args, **worker_kwargs)
|
||||
|
||||
return original(executor, run, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(app_module, "submit_durable", submit)
|
||||
return seen
|
||||
|
||||
|
||||
def wait_job(seen, kind, job_id):
|
||||
jobs_by_kind = {
|
||||
"reinstall_job": "reinstall_jobs", "rescue_job": "rescue_jobs",
|
||||
"snapshot_job": "snapshot_jobs",
|
||||
}
|
||||
for _ in range(200):
|
||||
saved = dict(seen[kind][jobs_by_kind[kind]][job_id])
|
||||
if saved["status"] not in {"queued", "running"}:
|
||||
return saved
|
||||
time.sleep(0.01)
|
||||
pytest.fail("background worker did not settle")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("write", [False, True])
|
||||
def test_reinstall_preflight_timeout_does_not_create_new_pending_guard(web, monkeypatch, write):
|
||||
client, _state, _token, service, _manager, _ = web
|
||||
csrf, template_id, _ = configure_reinstall_case(web)
|
||||
seen = instrument_background(monkeypatch, write=write)
|
||||
|
||||
def fail(*_args, **_kwargs):
|
||||
raise VMError("preflight read timed out", 503, error_code="libvirt_execution_timeout")
|
||||
|
||||
service.reinstall_cloud_vm = fail
|
||||
headers = {"X-CSRF-Token": csrf}
|
||||
response = client.post(f"/api/v1/vms/{VM_ID}/reinstall", json={
|
||||
"template_id": template_id, "confirmation_name": "cloud-vm",
|
||||
}, headers=headers)
|
||||
assert response.status_code == 202, response.text
|
||||
result = wait_job(seen, "reinstall_job", response.json()["job_id"])
|
||||
assert result["status"] == ("uncertain" if write else "failed")
|
||||
assert (result.get("execution_outcome") == "not_dispatched") is (not write)
|
||||
power = client.post(f"/api/v1/vms/{VM_ID}/actions", json={"action": "poweroff"}, headers=headers)
|
||||
assert power.status_code == (409 if write else 200), power.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize("write", [False, True])
|
||||
def test_snapshot_preflight_timeout_rolls_back_catalog_before_releasing_guard(web, monkeypatch, write):
|
||||
client, service, csrf = configure_snapshot_case(web)
|
||||
seen = instrument_background(monkeypatch, write=write)
|
||||
service.snapshot_create_error = VMError(
|
||||
"preflight read timed out", 503, error_code="libvirt_execution_timeout")
|
||||
key = "d" * 32
|
||||
headers = {"X-CSRF-Token": csrf}
|
||||
response = client.post(f"/api/v1/vms/{VM_ID}/snapshots", json={
|
||||
"snapshot_key": key, "name": "preflight", "consistency_mode": "crash",
|
||||
}, headers=headers)
|
||||
assert response.status_code == 202, response.text
|
||||
result = wait_job(seen, "snapshot_job", response.json()["job_id"])
|
||||
assert result["status"] == ("uncertain" if write else "failed")
|
||||
assert (result.get("execution_outcome") == "not_dispatched") is (not write)
|
||||
snapshot = client.get(f"/api/v1/vms/{VM_ID}/snapshots/{key}")
|
||||
assert snapshot.status_code == (200 if write else 404), snapshot.text
|
||||
power = client.post(f"/api/v1/vms/{VM_ID}/actions", json={"action": "poweroff"}, headers=headers)
|
||||
assert power.status_code == (409 if write else 200), power.text
|
||||
|
||||
|
||||
def configure_rescue(web):
|
||||
client, _state, token, service, _manager, _ = web
|
||||
csrf = setup_admin(client, token)
|
||||
configure_host(client, csrf)
|
||||
headers = {"X-CSRF-Token": csrf}
|
||||
iso = bytearray(1024 * 1024)
|
||||
for sector, kind in ((16, 1), (17, 0), (18, 255)):
|
||||
offset = sector * 2048
|
||||
iso[offset] = kind
|
||||
iso[offset + 1:offset + 6] = b"CD001"
|
||||
iso[offset + 6] = 1
|
||||
iso[17 * 2048 + 7:17 * 2048 + 30] = b"EL TORITO SPECIFICATION"
|
||||
iso[17 * 2048 + 71:17 * 2048 + 75] = (20).to_bytes(4, "little")
|
||||
response = client.post("/api/v1/rescue-templates?name=Rescue", content=bytes(iso), headers=headers)
|
||||
assert response.status_code == 202, response.text
|
||||
for _ in range(200):
|
||||
imported = client.get(
|
||||
f"/api/v1/rescue-templates/import-jobs/{response.json()['job_id']}").json()
|
||||
if imported["status"] not in {"queued", "running"}:
|
||||
break
|
||||
time.sleep(0.01)
|
||||
assert imported["status"] == "completed"
|
||||
template = client.get("/api/v1/rescue-templates").json()["templates"][0]
|
||||
return client, service, headers, template
|
||||
|
||||
|
||||
@pytest.mark.parametrize("write", [False, True])
|
||||
def test_rescue_preflight_timeout_clears_only_unstarted_entry(web, monkeypatch, write):
|
||||
client, service, headers, template = configure_rescue(web)
|
||||
seen = instrument_background(monkeypatch, write=write)
|
||||
service.rescue_enter_error = VMError(
|
||||
"preflight read timed out", 503, error_code="libvirt_execution_timeout")
|
||||
response = client.post(f"/api/v1/vms/{VM_ID}/rescue", json={
|
||||
"template_id": template["id"], "confirmation_name": "alma-test",
|
||||
}, headers=headers)
|
||||
assert response.status_code == 202, response.text
|
||||
result = wait_job(seen, "rescue_job", response.json()["job_id"])
|
||||
assert result["status"] == ("uncertain" if write else "failed")
|
||||
assert (result.get("execution_outcome") == "not_dispatched") is (not write)
|
||||
assert sum(call[0] == "prepare_rescue_vm" for call in service.calls) == 1
|
||||
power = client.post(f"/api/v1/vms/{VM_ID}/actions", json={"action": "poweroff"}, headers=headers)
|
||||
assert power.status_code == (409 if write else 200), power.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind", ["reinstall", "snapshot", "rescue"])
|
||||
def test_database_failure_never_produces_worker_no_dispatch_receipt(web, monkeypatch, kind):
|
||||
error = DatabaseError("lost database commit response")
|
||||
if kind == "reinstall":
|
||||
client, _state, _token, service, _manager, _ = web
|
||||
csrf, template_id, _ = configure_reinstall_case(web)
|
||||
headers = {"X-CSRF-Token": csrf}
|
||||
body = {"template_id": template_id, "confirmation_name": "cloud-vm"}
|
||||
resource = "reinstall"
|
||||
|
||||
def fail(*_args, **_kwargs):
|
||||
raise error
|
||||
|
||||
service.reinstall_cloud_vm = fail
|
||||
elif kind == "snapshot":
|
||||
client, service, csrf = configure_snapshot_case(web)
|
||||
headers = {"X-CSRF-Token": csrf}
|
||||
body = {"snapshot_key": "a" * 32, "name": "test", "consistency_mode": "crash"}
|
||||
resource = "snapshots"
|
||||
service.snapshot_create_error = error
|
||||
else:
|
||||
client, service, headers, template = configure_rescue(web)
|
||||
body = {"template_id": template["id"], "confirmation_name": "alma-test"}
|
||||
resource = "rescue"
|
||||
service.rescue_enter_error = error
|
||||
seen = instrument_background(monkeypatch)
|
||||
response = client.post(f"/api/v1/vms/{VM_ID}/{resource}", json=body, headers=headers)
|
||||
assert response.status_code == 202, response.text
|
||||
result = wait_job(seen, kind + "_job", response.json()["job_id"])
|
||||
assert result.get("execution_outcome") != "not_dispatched"
|
||||
assert result.get("mutation_started") is not False
|
||||
assert seen["execution"].execution_unknown.is_set()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("action", ["force", "exit"])
|
||||
def test_unstarted_rescue_transition_restores_previous_recovery_row(web, monkeypatch, action):
|
||||
client, service, headers, template = configure_rescue(web)
|
||||
service.rescue_pending_manual_restart = action == "force"
|
||||
entered = client.post(f"/api/v1/vms/{VM_ID}/rescue", json={
|
||||
"template_id": template["id"], "confirmation_name": "alma-test",
|
||||
}, headers=headers)
|
||||
assert entered.status_code == 202, entered.text
|
||||
for _ in range(200):
|
||||
previous = client.get(f"/api/v1/vms/{VM_ID}/rescue").json()
|
||||
if previous["status"] in {"active", "staged"}:
|
||||
break
|
||||
time.sleep(0.01)
|
||||
assert previous["status"] == ("staged" if action == "force" else "active")
|
||||
seen = instrument_background(monkeypatch)
|
||||
|
||||
def fail(*_args, **_kwargs):
|
||||
raise VMError("read timeout", 503, error_code="libvirt_execution_timeout")
|
||||
|
||||
monkeypatch.setattr(service, "force_start_staged_rescue_vm" if action == "force"
|
||||
else "exit_rescue_vm", fail)
|
||||
resource = "force-start" if action == "force" else "exit"
|
||||
response = client.post(f"/api/v1/vms/{VM_ID}/rescue/{resource}", json={
|
||||
"confirmation_name": "alma-test",
|
||||
**({"ack_data_loss": True} if action == "force" else {}),
|
||||
}, headers=headers)
|
||||
assert response.status_code == 202, response.text
|
||||
result = wait_job(seen, "rescue_job", response.json()["job_id"])
|
||||
assert result["status"] == "failed"
|
||||
assert result["execution_outcome"] == "not_dispatched"
|
||||
restored = client.get(f"/api/v1/vms/{VM_ID}/rescue").json()
|
||||
assert restored["status"] == previous["status"]
|
||||
# The new failed transition must not replace the original recovery owner.
|
||||
namespace = seen["rescue_job"]
|
||||
profile = web[4].current()
|
||||
saved = namespace["load_rescue_state"](profile, VM_ID)
|
||||
assert saved["job_id"] == entered.json()["job_id"]
|
||||
@ -0,0 +1,136 @@
|
||||
"""A failed public-IP preflight releases only a confirmed local reservation."""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from test_engine_boundary import USER, VM_ID, boundary, call # noqa: F401
|
||||
from virsh_panel.coordination import mark_remote_write_started
|
||||
from virsh_panel.database import DatabaseError
|
||||
from virsh_panel.libvirt_service import VMError
|
||||
from virsh_panel.metrics import host_scope
|
||||
|
||||
|
||||
def test_database_error_handler_cannot_turn_serialized_503_into_no_dispatch(boundary, monkeypatch):
|
||||
client, engine, store, _logs, services = boundary
|
||||
assert call(client, "GET", "/internal/v1/hosts/hk/vms").status_code == 200
|
||||
components = engine.registry.components["hk"]
|
||||
components.state.users.assign_vm(host_scope(components.manager.profile), VM_ID, USER["user_id"])
|
||||
|
||||
def failed_action(*_args, **_kwargs):
|
||||
raise DatabaseError("lost database commit response")
|
||||
|
||||
monkeypatch.setattr(services["hk"], "action", failed_action)
|
||||
operation_id = "database-error-http-boundary"
|
||||
response = call(client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/actions", {
|
||||
**USER, "operation_id": operation_id,
|
||||
}, {"action": "poweroff"})
|
||||
assert response.status_code == 503, response.text
|
||||
record = json.loads(store.get("operation", operation_id))
|
||||
assert record["status"] == "uncertain", record
|
||||
assert record.get("execution_outcome") != "not_dispatched"
|
||||
assert engine.coordinator.operation_claim(operation_id) is not None
|
||||
|
||||
|
||||
def test_failed_snapshot_admission_receipt_does_not_hide_catalog_rollback_failure(boundary, monkeypatch):
|
||||
client, engine, store, _logs, services = boundary
|
||||
assert call(client, "GET", "/internal/v1/hosts/hk/vms").status_code == 200
|
||||
components = engine.registry.components["hk"]
|
||||
components.state.users.assign_vm(host_scope(components.manager.profile), VM_ID, USER["user_id"])
|
||||
services["hk"].vm.update(managed=True, os_source="cloud_template",
|
||||
rescue_mode=False, rescue_staged=False)
|
||||
original_set, original_delete = components.store.set, components.store.delete
|
||||
|
||||
def failed_job_save(kind, key, value):
|
||||
if kind == "snapshot_job":
|
||||
raise DatabaseError("initial job save failed")
|
||||
return original_set(kind, key, value)
|
||||
|
||||
def failed_catalog_rollback(kind, key):
|
||||
if kind == "vm_snapshot":
|
||||
raise DatabaseError("snapshot catalog rollback failed")
|
||||
return original_delete(kind, key)
|
||||
|
||||
monkeypatch.setattr(components.store, "set", failed_job_save)
|
||||
monkeypatch.setattr(components.store, "delete", failed_catalog_rollback)
|
||||
operation_id = "snapshot-local-rollback-unknown"
|
||||
response = call(client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/snapshots", {
|
||||
**USER, "operation_id": operation_id,
|
||||
}, {"snapshot_key": "e" * 32, "name": "rollback failure", "consistency_mode": "crash"})
|
||||
assert response.status_code == 503, response.text
|
||||
record = json.loads(store.get("operation", operation_id))
|
||||
assert record["status"] == "uncertain", record
|
||||
assert record.get("execution_outcome") != "not_dispatched"
|
||||
assert engine.coordinator.operation_claim(operation_id) is not None
|
||||
assert components.store.get("vm_snapshot", f"{VM_ID}:" + "e" * 32) is not None
|
||||
assert not any(row[0] == "create_live_snapshot" for row in services["hk"].calls)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure", ["read", "write", "rollback", "reservation"])
|
||||
def test_network_prewrite_failure_requires_successful_ip_rollback(boundary, monkeypatch, failure):
|
||||
client, engine, store, _logs, services = boundary
|
||||
assert call(client, "GET", "/internal/v1/hosts/hk/vms").status_code == 200
|
||||
components = engine.registry.components["hk"]
|
||||
profile = components.manager.profile
|
||||
profile.update({
|
||||
"host": "8.8.8.3", "default_network": "bridge:br0",
|
||||
"public_pool_cidr": "8.8.8.0/28", "public_gateway": "8.8.8.1",
|
||||
"public_dns_servers": ["1.1.1.1"], "public_reserved_ips": ["8.8.8.2"],
|
||||
"public_vm_prefix": 28,
|
||||
})
|
||||
service = services["hk"]
|
||||
service.vm.update(managed=True, os_source="cloud_template", public_ip_cidr="8.8.8.4/28")
|
||||
scope = host_scope(profile)
|
||||
components.state.users.assign_vm(scope, VM_ID, USER["user_id"])
|
||||
original_claim = json.dumps({"vm_uuid": VM_ID})
|
||||
components.store.set("ip_claim", f"{scope}.8.8.8.4", original_claim)
|
||||
expected_key = f"{scope}.8.8.8.6"
|
||||
calls = []
|
||||
|
||||
def fail_service(*_args, **_kwargs):
|
||||
calls.append("service")
|
||||
if failure == "write":
|
||||
mark_remote_write_started()
|
||||
raise VMError("preflight read failed", 502)
|
||||
|
||||
monkeypatch.setattr(service, "change_cloud_vm_network", fail_service)
|
||||
if failure == "rollback":
|
||||
original_delete = components.store.delete
|
||||
|
||||
def failed_delete(kind, key):
|
||||
if kind == "ip_claim" and key == expected_key:
|
||||
raise DatabaseError("reservation rollback failed")
|
||||
return original_delete(kind, key)
|
||||
|
||||
monkeypatch.setattr(components.store, "delete", failed_delete)
|
||||
if failure == "reservation":
|
||||
original_insert = components.store.insert
|
||||
|
||||
def lost_insert(kind, key, value):
|
||||
result = original_insert(kind, key, value)
|
||||
if kind == "ip_claim" and key == expected_key:
|
||||
raise DatabaseError("reservation commit response lost")
|
||||
return result
|
||||
|
||||
monkeypatch.setattr(components.store, "insert", lost_insert)
|
||||
|
||||
operation_id = "network-prewrite-" + failure
|
||||
response = call(client, "POST", f"/internal/v1/hosts/hk/vms/{VM_ID}/network", {
|
||||
**USER, "operation_id": operation_id,
|
||||
}, {"public_ip_cidr": "8.8.8.6/28", "confirm_restart": True})
|
||||
assert response.status_code >= 500, response.text
|
||||
record = json.loads(store.get("operation", operation_id))
|
||||
assert components.store.get("ip_claim", f"{scope}.8.8.8.4") == original_claim
|
||||
assert service.vm["public_ip_cidr"] == "8.8.8.4/28"
|
||||
if failure == "read":
|
||||
assert record["status"] == "failed", record
|
||||
assert record["network_mutation_started"] is False
|
||||
assert record["execution_outcome"] == "not_dispatched"
|
||||
assert components.store.get("ip_claim", expected_key) is None
|
||||
assert engine.coordinator.operation_claim(operation_id) is None
|
||||
else:
|
||||
assert record["status"] == "uncertain", record
|
||||
assert record.get("execution_outcome") != "not_dispatched"
|
||||
assert components.store.get("ip_claim", expected_key) is not None
|
||||
assert engine.coordinator.operation_claim(operation_id) is not None
|
||||
assert calls == ([] if failure == "reservation" else ["service"])
|
||||
@ -0,0 +1,193 @@
|
||||
"""No-dispatch failures release only this live VM execution's claims."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from test_connection_failure_admission import MemoryStore, VM_ID, VMError, VMService
|
||||
from virsh_panel import coordination, libvirt_service as service_module
|
||||
from virsh_panel.coordination import (
|
||||
CoordinationClaimRejected, CoordinationError, DurableCoordinator,
|
||||
execution_context, execution_failure_before_write, mark_admission_rejected,
|
||||
mark_remote_write_started, resource_guard,
|
||||
)
|
||||
from virsh_panel.libvirt_rpc import ExecutionFailure, LibvirtRuntime, RPCSession
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def tracked_service(monkeypatch):
|
||||
coordinator = DurableCoordinator(MemoryStore())
|
||||
connection = SimpleNamespace()
|
||||
session = SimpleNamespace(connection=connection, failure=None, close=lambda: None)
|
||||
runtime = SimpleNamespace(open=lambda *_args, **_kwargs: session, queue_timeout=0.01)
|
||||
monkeypatch.setattr(service_module, "get_runtime", lambda: runtime)
|
||||
service = VMService("test", "images", "iso", "default", execution_scope="7")
|
||||
return coordinator, service, connection, runtime
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stage", ["state", "clean-traffic", "queue"])
|
||||
def test_failed_vm_preflight_releases_vm_storage_and_allocation(tracked_service, stage):
|
||||
coordinator, service, connection, runtime = tracked_service
|
||||
request = coordinator.claim(resources=(f"vm:7:{VM_ID}",))
|
||||
writes = []
|
||||
|
||||
def read_failed():
|
||||
raise service_module.libvirt.libvirtError("read denied before any mutation")
|
||||
|
||||
connection.lookupByUUIDString = lambda _uuid: SimpleNamespace(
|
||||
isActive=read_failed, create=lambda: writes.append("create"))
|
||||
if stage == "queue":
|
||||
def queue_full(*_args, **_kwargs):
|
||||
raise ExecutionFailure("libvirt_execution_timeout")
|
||||
runtime.open = queue_full
|
||||
with execution_context(coordinator, request, tracked_mutations=True):
|
||||
with pytest.raises(VMError) as raised:
|
||||
if stage == "clean-traffic":
|
||||
with service._operation_guard(VM_ID, storage=True, allocation=True):
|
||||
service._require_clean_traffic_filter(connection)
|
||||
else:
|
||||
service.action(VM_ID, "start")
|
||||
assert execution_failure_before_write(raised.value.error_code)
|
||||
assert set(coordinator.snapshot()) == {request.token}
|
||||
assert writes == []
|
||||
assert coordinator.release(request)
|
||||
next_request = coordinator.claim(resources=("storage:7", "allocation:7", f"vm:7:{VM_ID}"))
|
||||
assert coordinator.release(next_request)
|
||||
|
||||
|
||||
def test_other_vm_storage_fence_does_not_contaminate_rejected_vm(tracked_service):
|
||||
coordinator, service, _connection, _runtime = tracked_service
|
||||
blocker = coordinator.claim(resources=("storage:7", "vm:7:other"))
|
||||
coordinator.retain(blocker)
|
||||
before = coordinator.snapshot()[blocker.token]
|
||||
request = coordinator.claim(resources=(f"vm:7:{VM_ID}",))
|
||||
with execution_context(coordinator, request, tracked_mutations=True):
|
||||
with pytest.raises(VMError) as raised:
|
||||
with service._operation_guard(VM_ID, storage=True):
|
||||
pytest.fail("No host work may begin after conflicting admission")
|
||||
assert raised.value.error_code == "coordination_uncertain"
|
||||
assert execution_failure_before_write(raised.value.error_code)
|
||||
assert not coordination._uncertain_exception(raised.value)
|
||||
assert coordinator.release(request)
|
||||
assert coordinator.snapshot() == {blocker.token: before}
|
||||
# VM B's later power action needs only its own VM resource.
|
||||
next_request = coordinator.claim()
|
||||
with execution_context(coordinator, next_request, tracked_mutations=True):
|
||||
with service._operation_guard(VM_ID):
|
||||
pass
|
||||
assert coordinator.release(next_request)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("untracked", ["native", "mixed"])
|
||||
def test_untracked_execution_cannot_infer_no_write(tracked_service, untracked):
|
||||
coordinator, service, _connection, _runtime = tracked_service
|
||||
if untracked == "native":
|
||||
service.execution_scope = None
|
||||
request = coordinator.claim()
|
||||
with execution_context(coordinator, request):
|
||||
with pytest.raises(VMError):
|
||||
with service._operation_guard(VM_ID):
|
||||
raise VMError("untracked failure", 502)
|
||||
assert any(row["state"] == "uncertain" for row in coordinator.snapshot().values())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("prior", ["write", "unknown", "cancel"])
|
||||
def test_no_write_marker_never_releases_actual_or_unknown_execution(tracked_service, prior):
|
||||
coordinator, service, _connection, _runtime = tracked_service
|
||||
request = coordinator.claim()
|
||||
with execution_context(coordinator, request, tracked_mutations=True):
|
||||
error = VMError("later read failed", 502)
|
||||
if prior == "write":
|
||||
mark_remote_write_started()
|
||||
elif prior == "unknown":
|
||||
error = VMError("unknown receipt", 502, error_code="libvirt_execution_uncertain")
|
||||
else:
|
||||
import asyncio
|
||||
error = asyncio.CancelledError()
|
||||
with pytest.raises(type(error)):
|
||||
with service._operation_guard(VM_ID):
|
||||
raise error
|
||||
assert not execution_failure_before_write(getattr(error, "error_code", None))
|
||||
assert any(row["state"] == "uncertain" for row in coordinator.snapshot().values())
|
||||
|
||||
|
||||
def test_error_code_alone_cannot_claim_definite_admission_rejection(tracked_service):
|
||||
coordinator, service, _connection, _runtime = tracked_service
|
||||
request = coordinator.claim()
|
||||
with execution_context(coordinator, request, tracked_mutations=True):
|
||||
with pytest.raises(VMError):
|
||||
with service._operation_guard(VM_ID):
|
||||
raise VMError("unproven result", 503, error_code="coordination_uncertain")
|
||||
assert not execution_failure_before_write("coordination_uncertain")
|
||||
assert any(row["state"] == "uncertain" for row in coordinator.snapshot().values())
|
||||
|
||||
|
||||
def test_lost_claim_commit_invalidates_earlier_admission_proof(tracked_service, monkeypatch):
|
||||
coordinator, _service, _connection, _runtime = tracked_service
|
||||
request = coordinator.claim()
|
||||
compare = coordinator.store.compare_and_set
|
||||
|
||||
def lose_reply(*args):
|
||||
assert compare(*args)
|
||||
raise OSError("commit succeeded but reply lost")
|
||||
|
||||
with execution_context(coordinator, request, tracked_mutations=True):
|
||||
mark_admission_rejected()
|
||||
assert execution_failure_before_write("coordination_uncertain")
|
||||
monkeypatch.setattr(coordinator.store, "compare_and_set", lose_reply)
|
||||
with pytest.raises(CoordinationError) as raised:
|
||||
with resource_guard((f"vm:7:{VM_ID}",)):
|
||||
pytest.fail("Unknown durable admission must not dispatch")
|
||||
assert raised.value.error_code == "coordination_unavailable"
|
||||
assert not execution_failure_before_write("coordination_uncertain")
|
||||
assert not execution_failure_before_write(None)
|
||||
assert coordination._uncertain_exception(VMError(
|
||||
"later queue timeout", 504, error_code="libvirt_execution_timeout"))
|
||||
assert len(coordinator.snapshot()) == 2
|
||||
|
||||
|
||||
def test_own_uncertain_claim_cannot_reuse_another_claims_rejection_receipt():
|
||||
coordinator = DurableCoordinator(MemoryStore())
|
||||
request = coordinator.claim()
|
||||
with execution_context(coordinator, request, tracked_mutations=True):
|
||||
mark_admission_rejected()
|
||||
coordinator.retain(request)
|
||||
with pytest.raises(CoordinationError) as raised:
|
||||
coordinator.add_resources(request, ("storage:7",))
|
||||
assert not isinstance(raised.value, CoordinationClaimRejected)
|
||||
assert not execution_failure_before_write(raised.value.error_code)
|
||||
assert not execution_failure_before_write(None)
|
||||
|
||||
|
||||
def test_confirmed_rpc_fence_rejection_keeps_original_fence_only(tmp_path):
|
||||
coordinator = DurableCoordinator(MemoryStore())
|
||||
request = coordinator.claim()
|
||||
runtime = LibvirtRuntime(tmp_path / "rpc")
|
||||
fence = runtime._directory("7") / ("a" * 32 + ".json")
|
||||
fence.write_text("previous unknown write")
|
||||
session = RPCSession(runtime, "test", "7", 5)
|
||||
with execution_context(coordinator, request, tracked_mutations=True):
|
||||
with pytest.raises(ExecutionFailure) as raised:
|
||||
session.call(1, "destroy", (), {})
|
||||
assert raised.value.error_code == "libvirt_execution_blocked"
|
||||
assert execution_failure_before_write(raised.value.error_code)
|
||||
assert session.intent is None
|
||||
assert list(runtime.state_dir.rglob("*.json")) == [fence]
|
||||
# A genuine subsequent unknown error cannot reuse that admission proof.
|
||||
other = RPCSession(runtime, "test", "7", 5)
|
||||
other._fail("libvirt_execution_uncertain", outcome_unknown=True)
|
||||
assert not execution_failure_before_write("libvirt_execution_blocked")
|
||||
assert fence.read_text() == "previous unknown write"
|
||||
|
||||
|
||||
def test_claim_rejection_in_mixed_workflow_does_not_erase_unknown_ssh_effects():
|
||||
coordinator = DurableCoordinator(MemoryStore())
|
||||
blocker = coordinator.claim(resources=("storage:7",))
|
||||
coordinator.retain(blocker)
|
||||
request = coordinator.claim()
|
||||
with execution_context(coordinator, request):
|
||||
with pytest.raises(CoordinationClaimRejected) as raised:
|
||||
with resource_guard(("storage:7",)):
|
||||
pytest.fail("Conflicting claim admitted")
|
||||
assert coordination._uncertain_exception(raised.value)
|
||||
assert not execution_failure_before_write(raised.value.error_code)
|
||||
@ -0,0 +1,175 @@
|
||||
"""Signed operation receipts distinguish no dispatch from unknown execution."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from test_engine_boundary import (
|
||||
CREATE_BODY, USER, VM_ID, boundary, call, host_scope,
|
||||
)
|
||||
from virsh_panel.coordination import (
|
||||
execution_context, mark_execution_unknown, mark_remote_write_started,
|
||||
)
|
||||
from virsh_panel.libvirt_service import VMError, VMService
|
||||
|
||||
|
||||
def prepare(boundary):
|
||||
client, engine, store, _, services = boundary
|
||||
call(client, "GET", "/internal/v1/hosts/hk/vms")
|
||||
component = engine.registry.components["hk"]
|
||||
component.state.users.assign_vm(host_scope(component.manager.profile), VM_ID, 41)
|
||||
return client, engine, store, component, services["hk"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("code", [None, "libvirt_execution_timeout", "libvirt_execution_stopped"])
|
||||
def test_prewrite_502_finishes_and_allows_followup_power(boundary, monkeypatch, code):
|
||||
client, engine, store, _, service = prepare(boundary)
|
||||
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/actions"
|
||||
action = service.action
|
||||
attempts = []
|
||||
tracked = VMService("test", "images", "iso", "default", execution_scope="hk")
|
||||
|
||||
def failed(vm_id, command):
|
||||
attempts.append(command)
|
||||
with tracked._operation_guard(vm_id):
|
||||
raise VMError("state/XML read failed before dispatch", 502, error_code=code)
|
||||
|
||||
monkeypatch.setattr(service, "action", failed)
|
||||
actor = {**USER, "operation_id": "prewrite-power-receipt"}
|
||||
response = call(client, "POST", path, actor, {"action": "reboot"})
|
||||
assert response.status_code == 502, response.text
|
||||
assert response.json()["status"] == "failed"
|
||||
assert response.json()["mutation_started"] is False
|
||||
assert response.json()["execution_outcome"] == "not_dispatched"
|
||||
assert engine.coordinator.snapshot() == {}
|
||||
saved = store.get("operation", actor["operation_id"])
|
||||
polled = call(client, "GET", f"/internal/v1/operations/{actor['operation_id']}")
|
||||
assert polled.json()["execution_outcome"] == "not_dispatched"
|
||||
assert call(client, "POST", path, actor, {"action": "reboot"}).json()["status"] == "failed"
|
||||
assert store.get("operation", actor["operation_id"]) == saved
|
||||
assert attempts == ["reboot"]
|
||||
monkeypatch.setattr(service, "action", action)
|
||||
forced = call(client, "POST", path, {**USER, "operation_id": "power-after-prewrite"},
|
||||
{"action": "poweroff"})
|
||||
assert forced.json()["status"] == "succeeded", forced.text
|
||||
assert forced.json()["execution_outcome"] == "acknowledged"
|
||||
|
||||
|
||||
def test_foreign_storage_fence_cannot_spread_to_rejected_vm(boundary, monkeypatch):
|
||||
client, engine, _, _, service = prepare(boundary)
|
||||
blocker = engine.coordinator.claim(resources=("storage:hk", "vm:hk:other"))
|
||||
engine.coordinator.retain(blocker)
|
||||
original = engine.coordinator.snapshot()
|
||||
action = service.action
|
||||
tracked = VMService("test", "images", "iso", "default", execution_scope="hk")
|
||||
|
||||
def rejected(vm_id, _action):
|
||||
with tracked._operation_guard(vm_id, storage=True):
|
||||
pytest.fail("A refused request must not execute")
|
||||
|
||||
monkeypatch.setattr(service, "action", rejected)
|
||||
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/actions"
|
||||
response = call(client, "POST", path, {**USER, "operation_id": "storage-rejected-vm"},
|
||||
{"action": "start"})
|
||||
assert response.status_code == 503, response.text
|
||||
assert response.json()["status"] == "failed"
|
||||
assert response.json()["execution_outcome"] == "not_dispatched"
|
||||
assert engine.coordinator.snapshot() == original
|
||||
monkeypatch.setattr(service, "action", action)
|
||||
next_response = call(client, "POST", path, {**USER, "operation_id": "storage-next-power"},
|
||||
{"action": "poweroff"})
|
||||
assert next_response.status_code == 200, next_response.text
|
||||
assert engine.coordinator.snapshot() == original
|
||||
|
||||
|
||||
@pytest.mark.parametrize("absence", [True, False, "read_failed"])
|
||||
def test_create_prewrite_failure_unlocks_but_refund_needs_cleanup(boundary, monkeypatch, absence):
|
||||
client, engine, _, component, service = prepare(boundary)
|
||||
tracked = VMService("test", "images", "iso", "default", execution_scope="hk")
|
||||
|
||||
def failed(spec):
|
||||
with tracked._operation_guard(spec["vm_uuid"], storage=True, allocation=True):
|
||||
raise VMError("traffic preflight failed", 502)
|
||||
|
||||
def check(*_args, **_kwargs):
|
||||
if absence == "read_failed":
|
||||
raise VMError("absence read unavailable", 504, error_code="libvirt_execution_timeout")
|
||||
return absence
|
||||
|
||||
monkeypatch.setattr(service, "create_vm", failed)
|
||||
monkeypatch.setattr(service, "provisioning_absent", check)
|
||||
actor = {**USER, "can_provision": True, "operation_id": "create-prewrite-proof"}
|
||||
response = call(client, "POST", "/internal/v1/hosts/hk/vms", actor, CREATE_BODY)
|
||||
assert response.status_code == 502, response.text
|
||||
assert response.json()["status"] == "failed"
|
||||
assert response.json()["execution_outcome"] == "not_dispatched"
|
||||
assert response.json()["safe_to_refund"] is (absence is True)
|
||||
assert engine.coordinator.snapshot() == {}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", ["write", "unproven_unknown", "database", "unknown_then_409"])
|
||||
def test_actual_or_unproven_failure_keeps_vm_protected(boundary, monkeypatch, mode):
|
||||
client, engine, _, _, service = prepare(boundary)
|
||||
|
||||
def failed(*_args):
|
||||
if mode == "write":
|
||||
mark_remote_write_started()
|
||||
raise VMError("response lost after write", 502)
|
||||
if mode == "database":
|
||||
raise OSError("commit reply lost")
|
||||
if mode == "unknown_then_409":
|
||||
mark_execution_unknown()
|
||||
raise VMError("later validation rejection cannot erase unknown commit", 409)
|
||||
raise VMError("unproven admission outcome", 503, error_code="coordination_uncertain")
|
||||
|
||||
monkeypatch.setattr(service, "action", failed)
|
||||
path = f"/internal/v1/hosts/hk/vms/{VM_ID}/actions"
|
||||
response = call(client, "POST", path, {**USER, "operation_id": "unknown-power-receipt"},
|
||||
{"action": "reset"})
|
||||
assert response.status_code >= 500 or mode == "unknown_then_409"
|
||||
claims = engine.coordinator.snapshot()
|
||||
assert any(row["state"] == "uncertain" and f"vm:hk:{VM_ID}" in row["resources"]
|
||||
for row in claims.values())
|
||||
blocked = call(client, "POST", path, {**USER, "operation_id": "power-still-protected"},
|
||||
{"action": "poweroff"})
|
||||
assert blocked.status_code == 503
|
||||
|
||||
|
||||
@pytest.mark.parametrize("state", ["pending_child", "write_during_snapshot"])
|
||||
def test_no_dispatch_proof_cannot_race_background_handoff(boundary, monkeypatch, state):
|
||||
_, engine, _, _, _ = prepare(boundary)
|
||||
parent = engine.coordinator.claim()
|
||||
with execution_context(engine.coordinator, parent, tracked_mutations=True):
|
||||
if state == "pending_child":
|
||||
engine.coordinator.claim(execution_id=parent.execution_id, kind="background")
|
||||
else:
|
||||
snapshot = engine.coordinator.snapshot
|
||||
|
||||
def child_finished():
|
||||
mark_remote_write_started()
|
||||
return snapshot()
|
||||
|
||||
monkeypatch.setattr(engine.coordinator, "snapshot", child_finished)
|
||||
assert not asyncio.run(engine._failed_before_dispatch({}, "libvirt_execution_timeout"))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("proof", [True, False])
|
||||
def test_background_operation_uses_only_explicit_new_no_dispatch_receipt(boundary, proof):
|
||||
client, engine, store, component, _ = prepare(boundary)
|
||||
op_id, job_id = "prewrite-background", "job-prewrite"
|
||||
job = {"operation_id": op_id, "owner": "php:41", "vm_id": VM_ID,
|
||||
"status": "failed", "error_code": "coordination_uncertain", "error": "refused"}
|
||||
if proof:
|
||||
job.update(mutation_started=False, execution_outcome="not_dispatched")
|
||||
component.store.set("reinstall_job", job_id, json.dumps(job))
|
||||
store.set("operation", op_id, json.dumps({
|
||||
"operation_id": op_id, "host_id": "hk", "user_id": 41, "kind": "mutation",
|
||||
"resource": f"vms/{VM_ID}/reinstall", "vm_uuid": VM_ID, "job_id": job_id,
|
||||
"status": "pending", "mutation_started": True, "execution_id": "a" * 64,
|
||||
"result": None, "safe_to_refund": False,
|
||||
}))
|
||||
result = call(client, "GET", f"/internal/v1/operations/{op_id}").json()
|
||||
assert result["status"] == ("failed" if proof else "uncertain"), result
|
||||
assert result["execution_outcome"] == ("not_dispatched" if proof else "unknown")
|
||||
assert result["mutation_started"] is (not proof)
|
||||
@ -0,0 +1,167 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services {
|
||||
final class Config
|
||||
{
|
||||
public static function get($name, $default = null) { return $default; }
|
||||
}
|
||||
}
|
||||
|
||||
namespace App\Services\Cloud {
|
||||
/** Isolated test keyring; never loaded by the application. */
|
||||
final class CloudSecrets
|
||||
{
|
||||
private static array $values = [];
|
||||
public static function key(string $purpose): string { return str_repeat('k', 32); }
|
||||
public static function encrypt(array $value): string
|
||||
{
|
||||
$key = bin2hex(random_bytes(16));
|
||||
self::$values[$key] = $value;
|
||||
return $key;
|
||||
}
|
||||
public static function decrypt(string $key): array { return self::$values[$key]; }
|
||||
}
|
||||
}
|
||||
|
||||
namespace {
|
||||
require dirname(__DIR__) . '/vendor/autoload.php';
|
||||
|
||||
use App\Models\CloudJob;
|
||||
use App\Services\Cloud\CloudEngineClient;
|
||||
use App\Services\Cloud\CloudSecrets;
|
||||
use App\Services\Cloud\CloudWorkerService;
|
||||
use GuzzleHttp\Client;
|
||||
use GuzzleHttp\Handler\MockHandler;
|
||||
use GuzzleHttp\HandlerStack;
|
||||
use GuzzleHttp\Middleware;
|
||||
use GuzzleHttp\Psr7\Response;
|
||||
use Illuminate\Database\Capsule\Manager as Capsule;
|
||||
use Illuminate\Database\Schema\Blueprint;
|
||||
|
||||
$assert = static function (bool $value, string $message): void {
|
||||
if (!$value) { throw new RuntimeException($message); }
|
||||
};
|
||||
$capsule = new Capsule();
|
||||
$capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'], 'default');
|
||||
$capsule->setAsGlobal();
|
||||
$capsule->bootEloquent();
|
||||
$schema = $capsule->schema();
|
||||
$schema->create('user', static function (Blueprint $t): void {
|
||||
$t->increments('id'); $t->string('money'); $t->integer('enable')->default(1);
|
||||
$t->boolean('is_admin')->default(false);
|
||||
});
|
||||
$schema->create('cloud_hosts', static function (Blueprint $t): void {
|
||||
$t->increments('id');
|
||||
});
|
||||
$schema->create('cloud_offers', static function (Blueprint $t): void {
|
||||
$t->increments('id'); $t->integer('stock'); $t->integer('updated_at')->default(0);
|
||||
});
|
||||
$schema->create('cloud_orders', static function (Blueprint $t): void {
|
||||
$t->increments('id'); $t->integer('user_id'); $t->integer('host_id');
|
||||
$t->integer('offer_id'); $t->string('amount'); $t->string('status');
|
||||
$t->integer('stock_reserved_at'); $t->integer('stock_released_at')->nullable();
|
||||
$t->integer('refunded_at')->nullable(); $t->integer('updated_at')->default(0);
|
||||
});
|
||||
$schema->create('cloud_invoices', static function (Blueprint $t): void {
|
||||
$t->increments('id'); $t->integer('order_id'); $t->string('purpose');
|
||||
$t->string('status'); $t->integer('updated_at')->default(0);
|
||||
});
|
||||
$schema->create('cloud_resource_reservations', static function (Blueprint $t): void {
|
||||
$t->increments('id'); $t->integer('order_id'); $t->integer('host_id');
|
||||
$t->integer('public_ip_id'); $t->integer('network_attachment_id');
|
||||
$t->integer('public_ip_pool_id'); $t->string('operation_id');
|
||||
$t->string('state'); $t->integer('released_at')->nullable();
|
||||
$t->string('release_reason')->nullable(); $t->integer('expires_at')->nullable();
|
||||
$t->integer('updated_at')->default(0);
|
||||
});
|
||||
$schema->create('cloud_public_ips', static function (Blueprint $t): void {
|
||||
$t->increments('id'); $t->integer('host_id'); $t->integer('public_ip_pool_id');
|
||||
$t->integer('reservation_id')->nullable(); $t->integer('instance_id')->nullable();
|
||||
$t->string('operation_id')->nullable(); $t->string('state');
|
||||
$t->integer('updated_at')->default(0);
|
||||
});
|
||||
$schema->create('cloud_jobs', static function (Blueprint $t): void {
|
||||
$t->increments('id'); $t->string('operation_id')->unique();
|
||||
$t->integer('order_id')->nullable(); $t->integer('instance_id')->nullable();
|
||||
$t->integer('snapshot_id')->nullable(); $t->integer('user_id'); $t->integer('host_id');
|
||||
$t->string('kind'); $t->string('status'); $t->text('payload_cipher');
|
||||
$t->integer('attempts')->default(0); $t->integer('dispatched_at')->nullable();
|
||||
$t->string('lease_token')->nullable(); $t->integer('lease_until')->default(0);
|
||||
$t->integer('available_at'); $t->integer('created_at'); $t->integer('updated_at');
|
||||
$t->string('error_code')->nullable(); $t->text('result_json')->nullable();
|
||||
});
|
||||
Capsule::table('user')->insert(['id' => 1, 'money' => '100.00']);
|
||||
Capsule::table('cloud_hosts')->insert(['id' => 7]);
|
||||
Capsule::table('cloud_offers')->insert(['id' => 1, 'stock' => 10]);
|
||||
$history = [];
|
||||
$mock = new MockHandler();
|
||||
$handler = HandlerStack::create($mock);
|
||||
$handler->push(Middleware::history($history));
|
||||
$worker = new CloudWorkerService(new CloudEngineClient(new Client(['handler' => $handler])));
|
||||
// Begin at the paid, reserved boundary. All subsequent transitions use
|
||||
// the real worker, finance services, Eloquent and HTTP outcome validation.
|
||||
$paidCreate = static function (int $id): CloudJob {
|
||||
$operation = bin2hex(random_bytes(16));
|
||||
$now = time();
|
||||
Capsule::table('user')->where('id', 1)->decrement('money', 9.99);
|
||||
Capsule::table('cloud_offers')->where('id', 1)->decrement('stock');
|
||||
Capsule::table('cloud_orders')->insert(['id' => $id, 'user_id' => 1,
|
||||
'host_id' => 7, 'offer_id' => 1, 'amount' => '9.99', 'status' => 'queued',
|
||||
'stock_reserved_at' => $now]);
|
||||
Capsule::table('cloud_invoices')->insert(['id' => $id, 'order_id' => $id,
|
||||
'purpose' => 'provision', 'status' => 'paid']);
|
||||
Capsule::table('cloud_resource_reservations')->insert(['id' => $id,
|
||||
'order_id' => $id, 'host_id' => 7, 'public_ip_id' => $id,
|
||||
'network_attachment_id' => 1, 'public_ip_pool_id' => 1,
|
||||
'operation_id' => $operation, 'state' => 'held', 'expires_at' => $now + 3600]);
|
||||
Capsule::table('cloud_public_ips')->insert(['id' => $id, 'host_id' => 7,
|
||||
'public_ip_pool_id' => 1, 'reservation_id' => $id,
|
||||
'operation_id' => $operation, 'state' => 'reserved']);
|
||||
Capsule::table('cloud_jobs')->insert(['id' => $id, 'operation_id' => $operation,
|
||||
'order_id' => $id, 'user_id' => 1, 'host_id' => 7,
|
||||
'kind' => 'create', 'status' => 'queued', 'payload_cipher' => CloudSecrets::encrypt([
|
||||
'method' => 'POST', 'resource' => 'vms', 'body' => ['name' => 'preflight-' . $id],
|
||||
]), 'available_at' => $now, 'created_at' => $now, 'updated_at' => $now]);
|
||||
return CloudJob::find($id);
|
||||
};
|
||||
$job = $paidCreate(1);
|
||||
$mock->append(new Response(503, [], json_encode([
|
||||
'operation_id' => $job->operation_id, 'status' => 'failed',
|
||||
'execution_outcome' => 'not_dispatched', 'mutation_started' => false,
|
||||
'safe_to_refund' => true, 'error_code' => 'execution_result_uncertain',
|
||||
])));
|
||||
$assert($worker->tick(), 'Preflight failure was not consumed.');
|
||||
$assert(CloudJob::find(1)->status === 'failed'
|
||||
&& (string) Capsule::table('user')->value('money') === '100.00'
|
||||
&& Capsule::table('cloud_offers')->value('stock') === 10
|
||||
&& Capsule::table('cloud_orders')->where('id', 1)->value('refunded_at') !== null
|
||||
&& Capsule::table('cloud_invoices')->where('id', 1)->value('status') === 'refunded'
|
||||
&& Capsule::table('cloud_resource_reservations')->where('id', 1)->value('state') === 'released'
|
||||
&& Capsule::table('cloud_public_ips')->where('id', 1)->value('state') === 'available',
|
||||
'Proven no-write 503 retained a create lock, wallet charge, stock or IP.');
|
||||
$assert(!$worker->tick() && count($history) === 1
|
||||
&& (string) Capsule::table('user')->value('money') === '100.00',
|
||||
'Terminal preflight failure was dispatched or refunded twice.');
|
||||
|
||||
foreach ([['unknown', 'uncertain', true], ['not_dispatched', 'failed', false]]
|
||||
as $index => [$outcome, $status, $started]) {
|
||||
$id = $index + 2;
|
||||
$job = $paidCreate($id);
|
||||
$mock->append(new Response(503, [], json_encode([
|
||||
'operation_id' => $job->operation_id, 'status' => $status,
|
||||
'execution_outcome' => $outcome, 'mutation_started' => $started,
|
||||
'safe_to_refund' => false, 'error_code' => 'libvirt_execution_uncertain',
|
||||
])));
|
||||
$worker->tick();
|
||||
$assert(CloudJob::find($id)->status === 'uncertain'
|
||||
&& Capsule::table('cloud_orders')->where('id', $id)->value('refunded_at') === null
|
||||
&& Capsule::table('cloud_invoices')->where('id', $id)->value('status') === 'paid'
|
||||
&& Capsule::table('cloud_resource_reservations')->where('id', $id)->value('state') === 'uncertain'
|
||||
&& Capsule::table('cloud_public_ips')->where('id', $id)->value('state') === 'uncertain'
|
||||
&& Capsule::table('cloud_offers')->value('stock') === 11 - $id,
|
||||
'Unknown write or missing absence proof released a create reservation.');
|
||||
}
|
||||
$assert((string) Capsule::table('user')->value('money') === '80.02',
|
||||
'Unproven creation outcomes refunded the wallet.');
|
||||
echo "Prewrite creation failure workflow tests passed.\n";
|
||||
}
|
||||
Loading…
Reference in New Issue