fix(engine): repair connection admission, snapshot thaw and VPC readback
parent
b82f3b855e
commit
71fc613b59
@ -0,0 +1,124 @@
|
||||
"""Persistent fake remote state for the real emergency-thaw RPC boundary."""
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
VIR_DOMAIN_XML_INACTIVE = 2
|
||||
VIR_DOMAIN_JOB_NONE = 0
|
||||
VIR_DOMAIN_JOB_UNBOUNDED = 2
|
||||
VIR_DOMAIN_JOB_OPERATION_BACKUP = 12
|
||||
|
||||
|
||||
class libvirtError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class virDomain:
|
||||
def __init__(self, connection):
|
||||
self.connection = connection
|
||||
|
||||
def UUIDString(self):
|
||||
return self.connection.read()['vm_id']
|
||||
|
||||
def ID(self):
|
||||
return self.connection.read().get('runtime_id', 7)
|
||||
|
||||
def isActive(self):
|
||||
return self.connection.read().get('running', True)
|
||||
|
||||
def XMLDesc(self, flags=0):
|
||||
state = self.connection.read()
|
||||
token = '' if flags else (
|
||||
'<qga-runtime xmlns="urn:virsh-panel:qga-runtime:1" '
|
||||
f'domain-uuid="{state["vm_id"]}" token="{state["token"]}"/>')
|
||||
return f'<domain><metadata>{token}</metadata></domain>'
|
||||
|
||||
def fsFreeze(self, mounts, flags):
|
||||
self.connection.update(frozen=True)
|
||||
if self.connection.read().get('freeze_hangs'):
|
||||
time.sleep(10)
|
||||
return 1
|
||||
|
||||
def backupBegin(self, xml, checkpoint, flags):
|
||||
state = self.connection.read()
|
||||
if not state.get('backup_missing'):
|
||||
self.connection.update(backup_active=True)
|
||||
if state.get('backup_hangs'):
|
||||
time.sleep(10)
|
||||
return 0
|
||||
|
||||
def fsThaw(self, mounts, flags):
|
||||
state = self.connection.read()
|
||||
self.connection.update(thaw_calls=state.get('thaw_calls', 0) + 1)
|
||||
if state.get('thaw_hangs'):
|
||||
time.sleep(10)
|
||||
self.connection.update(frozen=False)
|
||||
if state.get('thaw_invalid'):
|
||||
return -1
|
||||
if state.get('thaw_type_error'):
|
||||
raise TypeError('error after mutation')
|
||||
return 1
|
||||
|
||||
def jobStats(self, flags):
|
||||
state = self.connection.read()
|
||||
return {'type': 2 if state.get('backup_active') else 0,
|
||||
'operation': state.get('job_operation', 12)}
|
||||
|
||||
def backupGetXMLDesc(self, flags=0):
|
||||
state = self.connection.read()
|
||||
target = state.get('backup_path', state['target_path'])
|
||||
return ('<domainbackup mode="push"><disks>'
|
||||
f'<disk name="vda" backup="yes" type="file"><target file="{target}"/>'
|
||||
'</disk><disk name="vdb" backup="no"/></disks></domainbackup>')
|
||||
|
||||
def destroy(self):
|
||||
self.connection.update(destroyed=True)
|
||||
return 0
|
||||
|
||||
|
||||
class virStorageVol:
|
||||
def __init__(self, connection):
|
||||
self.connection = connection
|
||||
|
||||
def path(self):
|
||||
return self.connection.read()['target_path']
|
||||
|
||||
|
||||
class virStoragePool:
|
||||
def __init__(self, connection):
|
||||
self.connection = connection
|
||||
|
||||
def storageVolLookupByName(self, name):
|
||||
if name != self.connection.read()['volume_name']:
|
||||
raise libvirtError('wrong volume')
|
||||
return virStorageVol(self.connection)
|
||||
|
||||
|
||||
class virConnect:
|
||||
def __init__(self, uri):
|
||||
self.path = Path(uri)
|
||||
|
||||
def read(self):
|
||||
return json.loads(self.path.read_text())
|
||||
|
||||
def update(self, **values):
|
||||
state = self.read()
|
||||
state.update(values)
|
||||
self.path.write_text(json.dumps(state))
|
||||
|
||||
def lookupByUUIDString(self, vm_id):
|
||||
if vm_id != self.read()['vm_id']:
|
||||
raise libvirtError('wrong domain')
|
||||
return virDomain(self)
|
||||
|
||||
def storagePoolLookupByName(self, name):
|
||||
if name != 'images':
|
||||
raise libvirtError('wrong pool')
|
||||
return virStoragePool(self)
|
||||
|
||||
def close(self):
|
||||
return 0
|
||||
|
||||
|
||||
def open(uri):
|
||||
return virConnect(uri)
|
||||
@ -0,0 +1,373 @@
|
||||
"""Connection admission failures must not become permanent mutation fences.
|
||||
|
||||
Exercise the signed engine and real VMService/coordination boundaries. Only
|
||||
SSH staging, remote objects and the RPC connection transport are simulated.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import json
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from test_app import FakeRemoteManager, HOST_BODY, VM_ID
|
||||
from test_engine_boundary import MemoryStore, SECRET, SUPER, call
|
||||
from test_host_service_scope import StagedManager
|
||||
from virsh_panel.app import create_app
|
||||
from virsh_panel.config import Settings
|
||||
from virsh_panel.coordination import (
|
||||
CoordinationError, DurableCoordinator, connection_failure_before_write,
|
||||
current_execution, execution_context, mark_remote_write_started,
|
||||
resource_guard, update_current_claim,
|
||||
)
|
||||
from virsh_panel.database import DatabaseTemplateStore
|
||||
from virsh_panel.durable_jobs import submit_durable
|
||||
from virsh_panel.engine_context import EngineComponents, ExternalOwnership, HostStore
|
||||
from virsh_panel.internal import EngineApplication, HostRegistry, json_response
|
||||
from virsh_panel.libvirt_rpc import LibvirtRuntime
|
||||
from virsh_panel.libvirt_service import GIB, VMError, VMService
|
||||
from virsh_panel.metrics import MetricsCollector
|
||||
from virsh_panel.remote_host import RemoteHostError
|
||||
import virsh_panel.libvirt_service as service_module
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def signed_host(tmp_path, monkeypatch):
|
||||
opened, managers = [], {}
|
||||
behavior = SimpleNamespace(fail_open=True, fail_after_write=False)
|
||||
|
||||
def open_session(uri, scope, *, timeout):
|
||||
opened.append(scope)
|
||||
if behavior.fail_open:
|
||||
raise service_module.libvirt.libvirtError("connection refused before open")
|
||||
connection = SimpleNamespace(networkLookupByName=lambda _name:
|
||||
SimpleNamespace(isActive=lambda: True))
|
||||
return SimpleNamespace(connection=connection, failure=None, close=lambda: None)
|
||||
|
||||
runtime = SimpleNamespace(open=open_session, queue_timeout=0.02)
|
||||
monkeypatch.setattr("virsh_panel.app.get_runtime", lambda: runtime)
|
||||
monkeypatch.setattr(service_module, "get_runtime", lambda: runtime)
|
||||
monkeypatch.setattr(MetricsCollector, "start", lambda _self: None)
|
||||
|
||||
def pool(service, conn, name, *_args):
|
||||
if behavior.fail_after_write:
|
||||
mark_remote_write_started()
|
||||
# A completed first session's write is still part of this request.
|
||||
behavior.fail_open = True
|
||||
with service._connect():
|
||||
pytest.fail("second connection should fail")
|
||||
return SimpleNamespace(name=lambda: name, listVolumes=lambda: [],
|
||||
info=lambda: (0, 100 * GIB, 0, 100 * GIB))
|
||||
|
||||
monkeypatch.setattr(VMService, "_active_pool", pool)
|
||||
monkeypatch.setattr(VMService, "_ensure_pool", pool)
|
||||
monkeypatch.setattr(VMService, "_refresh_directory_pool", lambda *_args: None)
|
||||
monkeypatch.setattr(VMService, "_same_pool_location", lambda *_args: False)
|
||||
store, logs = MemoryStore(), MemoryStore()
|
||||
config = {"service_secret": base64.b64encode(SECRET).decode(),
|
||||
"data_dir": str(tmp_path)}
|
||||
|
||||
def factory(host_id, user_isos, cloud_templates):
|
||||
directory = tmp_path / host_id
|
||||
directory.mkdir(mode=0o700, exist_ok=True)
|
||||
scoped = HostStore(store, host_id)
|
||||
manager = managers[host_id] = StagedManager()
|
||||
manager.host_id = host_id
|
||||
components = EngineComponents(
|
||||
SimpleNamespace(users=ExternalOwnership(scoped)), manager,
|
||||
DatabaseTemplateStore(scoped), scoped, logs, user_isos, cloud_templates)
|
||||
return components, create_app(Settings(data_dir=directory),
|
||||
engine_components=components)
|
||||
|
||||
registry = HostRegistry(config, store, logs, factory=factory)
|
||||
engine = EngineApplication(config, store, logs, registry=registry)
|
||||
with TestClient(engine, base_url="http://127.0.0.1") as client:
|
||||
yield SimpleNamespace(client=client, engine=engine, behavior=behavior,
|
||||
opened=opened, managers=managers)
|
||||
|
||||
|
||||
def test_signed_host_save_connection_failure_releases_claims_and_allows_retry(signed_host):
|
||||
host = signed_host
|
||||
path = "/internal/v1/hosts/7/host"
|
||||
response = call(host.client, "PUT", path, SUPER, HOST_BODY)
|
||||
assert response.status_code == 502, response.text
|
||||
assert response.json()["error_code"] == "libvirt_connection_failed"
|
||||
assert host.managers["7"].current() is None
|
||||
assert [row[0] for row in host.managers["7"].calls] == ["stage", "test", "abort"]
|
||||
assert host.engine.coordinator.snapshot() == {}
|
||||
# No orphan request/resource claim may block maintenance either.
|
||||
maintenance = host.engine.coordinator.claim(maintenance=True, kind="update")
|
||||
assert host.engine.coordinator.release(maintenance)
|
||||
|
||||
host.behavior.fail_open = False
|
||||
response = call(host.client, "PUT", path, SUPER, HOST_BODY)
|
||||
assert response.status_code == 200, response.text
|
||||
assert host.managers["7"].current()["host_id"] == "7"
|
||||
assert host.opened == ["7", "7", "7"]
|
||||
assert host.engine.coordinator.snapshot() == {}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("receipt_lost", [False, True])
|
||||
def test_signed_ssh_probe_failure_requires_durable_prestart_proof(
|
||||
signed_host, monkeypatch, receipt_lost):
|
||||
host = signed_host
|
||||
path = "/internal/v1/hosts/7/host"
|
||||
assert call(host.client, "GET", path, SUPER).status_code == 200
|
||||
manager = host.managers["7"]
|
||||
original_probe = manager.test
|
||||
|
||||
def refused(_staged):
|
||||
raise RemoteHostError("SSH probe connection refused", 502)
|
||||
|
||||
monkeypatch.setattr(manager, "test", refused)
|
||||
if receipt_lost:
|
||||
def cannot_record(*_args, **_kwargs):
|
||||
raise CoordinationError("coordination_unavailable", 503)
|
||||
monkeypatch.setattr(host.engine.coordinator, "update_metadata", cannot_record)
|
||||
response = call(host.client, "PUT", path, SUPER, HOST_BODY)
|
||||
assert response.status_code == (503 if receipt_lost else 502), response.text
|
||||
assert host.opened == []
|
||||
assert manager.current() is None
|
||||
if receipt_lost:
|
||||
assert [row["state"] for row in host.engine.coordinator.snapshot().values()] == ["uncertain"]
|
||||
with pytest.raises(CoordinationError):
|
||||
host.engine.coordinator.claim(maintenance=True, kind="update")
|
||||
return
|
||||
assert host.engine.coordinator.snapshot() == {}
|
||||
maintenance = host.engine.coordinator.claim(maintenance=True, kind="update")
|
||||
assert host.engine.coordinator.release(maintenance)
|
||||
monkeypatch.setattr(manager, "test", original_probe)
|
||||
host.behavior.fail_open = False
|
||||
assert call(host.client, "PUT", path, SUPER, HOST_BODY).status_code == 200
|
||||
assert host.engine.coordinator.snapshot() == {}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("method,suffix", [("PUT", "host"), ("POST", "host/test")])
|
||||
@pytest.mark.parametrize("receipt_lost", [False, True])
|
||||
def test_signed_stage_scan_failure_requires_durable_prestart_proof(
|
||||
signed_host, monkeypatch, method, suffix, receipt_lost):
|
||||
host = signed_host
|
||||
assert call(host.client, "GET", "/internal/v1/hosts/7/host", SUPER).status_code == 200
|
||||
manager = host.managers["7"]
|
||||
original_stage = manager.stage
|
||||
|
||||
def refused(*_args, **_kwargs):
|
||||
raise RemoteHostError("keyscan failed during staging", 502)
|
||||
|
||||
monkeypatch.setattr(manager, "stage", refused)
|
||||
if receipt_lost:
|
||||
def cannot_record(*_args, **_kwargs):
|
||||
raise CoordinationError("coordination_unavailable", 503)
|
||||
monkeypatch.setattr(host.engine.coordinator, "update_metadata", cannot_record)
|
||||
path = f"/internal/v1/hosts/7/{suffix}"
|
||||
response = call(host.client, method, path, SUPER, HOST_BODY)
|
||||
assert response.status_code == (503 if receipt_lost else 502), response.text
|
||||
assert host.opened == []
|
||||
assert manager.current() is None
|
||||
if receipt_lost:
|
||||
assert [row["state"] for row in host.engine.coordinator.snapshot().values()] == ["uncertain"]
|
||||
with pytest.raises(CoordinationError):
|
||||
host.engine.coordinator.claim(maintenance=True, kind="update")
|
||||
return
|
||||
assert host.engine.coordinator.snapshot() == {}
|
||||
maintenance = host.engine.coordinator.claim(maintenance=True, kind="update")
|
||||
assert host.engine.coordinator.release(maintenance)
|
||||
monkeypatch.setattr(manager, "stage", original_stage)
|
||||
host.behavior.fail_open = False
|
||||
assert call(host.client, method, path, SUPER, HOST_BODY).status_code == 200
|
||||
assert host.engine.coordinator.snapshot() == {}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("global_route", [False, True], ids=["host-scan", "discovery-scan"])
|
||||
@pytest.mark.parametrize("receipt_lost", [False, True])
|
||||
def test_signed_keyscan_failure_requires_durable_prestart_proof(
|
||||
signed_host, monkeypatch, global_route, receipt_lost):
|
||||
host = signed_host
|
||||
if global_route:
|
||||
manager = FakeRemoteManager()
|
||||
manager.close = lambda: None
|
||||
monkeypatch.setattr("virsh_panel.remote_host.RemoteHostManager", lambda *_args: manager)
|
||||
path = "/internal/v1/hosts/scan"
|
||||
else:
|
||||
assert call(host.client, "GET", "/internal/v1/hosts/7/host", SUPER).status_code == 200
|
||||
manager = host.managers["7"]
|
||||
path = "/internal/v1/hosts/7/host/scan"
|
||||
original_scan = manager.scan
|
||||
|
||||
def refused(*_args, **_kwargs):
|
||||
raise RemoteHostError("keyscan timed out", 502)
|
||||
|
||||
monkeypatch.setattr(manager, "scan", refused)
|
||||
if receipt_lost:
|
||||
def cannot_record(*_args, **_kwargs):
|
||||
raise CoordinationError("coordination_unavailable", 503)
|
||||
monkeypatch.setattr(host.engine.coordinator, "update_metadata", cannot_record)
|
||||
body = {"host": HOST_BODY["host"], "port": HOST_BODY["port"]}
|
||||
response = call(host.client, "POST", path, SUPER, body)
|
||||
# The global discovery route preserves its generic remote-error 503.
|
||||
assert response.status_code == (503 if receipt_lost or global_route else 502), response.text
|
||||
assert host.opened == []
|
||||
if receipt_lost:
|
||||
assert [row["state"] for row in host.engine.coordinator.snapshot().values()] == ["uncertain"]
|
||||
with pytest.raises(CoordinationError):
|
||||
host.engine.coordinator.claim(maintenance=True, kind="update")
|
||||
return
|
||||
assert host.engine.coordinator.snapshot() == {}
|
||||
maintenance = host.engine.coordinator.claim(maintenance=True, kind="update")
|
||||
assert host.engine.coordinator.release(maintenance)
|
||||
monkeypatch.setattr(manager, "scan", original_scan)
|
||||
assert call(host.client, "POST", path, SUPER, body).status_code == 200
|
||||
assert host.engine.coordinator.snapshot() == {}
|
||||
|
||||
|
||||
def test_signed_host_save_later_connection_failure_retains_prior_writes(signed_host):
|
||||
host = signed_host
|
||||
host.behavior.fail_open = False
|
||||
host.behavior.fail_after_write = True
|
||||
response = call(host.client, "PUT", "/internal/v1/hosts/7/host", SUPER, HOST_BODY)
|
||||
assert response.status_code == 502, response.text
|
||||
claims = host.engine.coordinator.snapshot().values()
|
||||
assert {row["kind"] for row in claims if row["state"] == "uncertain"} == {
|
||||
"request", "resource"}
|
||||
assert any("storage:7" in row["resources"] for row in claims)
|
||||
with pytest.raises(CoordinationError):
|
||||
host.engine.coordinator.claim(maintenance=True, kind="update")
|
||||
|
||||
|
||||
def test_signed_vm_connection_failure_does_not_keep_the_vm_locked(signed_host):
|
||||
host = signed_host
|
||||
host.behavior.fail_open = False
|
||||
assert call(host.client, "PUT", "/internal/v1/hosts/7/host", SUPER,
|
||||
HOST_BODY).status_code == 200
|
||||
host.behavior.fail_open = True
|
||||
response = call(host.client, "POST", f"/internal/v1/hosts/7/vms/{VM_ID}/actions",
|
||||
{**SUPER, "operation_id": "start-connection-one"}, {"action": "start"})
|
||||
assert response.status_code == 502, response.text
|
||||
assert response.json()["status"] == "failed"
|
||||
assert host.engine.coordinator.snapshot() == {}
|
||||
next_operation = host.engine.coordinator.claim(resources=(f"vm:7:{VM_ID}",))
|
||||
assert host.engine.coordinator.release(next_operation)
|
||||
|
||||
|
||||
def test_mixed_ssh_request_does_not_use_missing_rpc_marker_as_proof(signed_host, monkeypatch):
|
||||
host = signed_host
|
||||
calls = []
|
||||
|
||||
async def mixed_handler(scope, receive, send, body, delegated, **_kwargs):
|
||||
calls.append("SSH helper changed host")
|
||||
assert not current_execution().connection_failure_safe
|
||||
await json_response(send, 502, {"error_code": "libvirt_connection_failed"})
|
||||
|
||||
monkeypatch.setattr(host.engine, "_dispatch_regular", mixed_handler)
|
||||
response = call(host.client, "POST", "/internal/v1/vpcs/apply", SUPER, {})
|
||||
assert response.status_code == 502
|
||||
assert calls == ["SSH helper changed host"]
|
||||
assert [row["state"] for row in host.engine.coordinator.snapshot().values()] == ["uncertain"]
|
||||
|
||||
|
||||
def test_service_guard_cannot_relax_a_mixed_workflows_unknown_side_effects(monkeypatch):
|
||||
coordinator = DurableCoordinator(MemoryStore())
|
||||
parent = coordinator.claim()
|
||||
|
||||
def refused(*_args, **_kwargs):
|
||||
raise service_module.libvirt.libvirtError("connection refused")
|
||||
|
||||
monkeypatch.setattr(service_module, "get_runtime", lambda:
|
||||
SimpleNamespace(open=refused, queue_timeout=0.02))
|
||||
service = VMService("test", "images", "iso", "default", execution_scope="7")
|
||||
with execution_context(coordinator, parent):
|
||||
with pytest.raises(VMError):
|
||||
service.ensure_storage_pools()
|
||||
assert any(row["state"] == "uncertain" and row["resources"] == ["storage:7"]
|
||||
for row in coordinator.snapshot().values())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("code", ["libvirt_connection_failed", "libvirt_execution_timeout",
|
||||
"libvirt_execution_stopped", "libvirt_execution_uncertain"])
|
||||
def test_nested_connection_failures_never_release_earlier_session_writes(code):
|
||||
coordinator = DurableCoordinator(MemoryStore())
|
||||
parent = coordinator.claim(resources=(f"vm:7:{VM_ID}",))
|
||||
with execution_context(coordinator, parent, connection_failure_safe=True):
|
||||
with resource_guard(("storage:7",)):
|
||||
mark_remote_write_started()
|
||||
updated = coordinator.add_resources(parent, ("allocation:7",))
|
||||
update_current_claim(updated)
|
||||
assert not connection_failure_before_write("libvirt_connection_failed")
|
||||
with pytest.raises(VMError):
|
||||
with resource_guard(("storage:7",), connection_failure_safe=True):
|
||||
raise VMError("later session failed", 502, error_code=code)
|
||||
assert any(row["kind"] == "resource" and row["state"] == "uncertain"
|
||||
for row in coordinator.snapshot().values())
|
||||
|
||||
|
||||
def test_background_write_evidence_returns_to_parent_thread_context():
|
||||
coordinator = DurableCoordinator(MemoryStore())
|
||||
parent = coordinator.claim(resources=(f"vm:7:{VM_ID}",))
|
||||
with execution_context(coordinator, parent, connection_failure_safe=True):
|
||||
with ThreadPoolExecutor(max_workers=1) as executor:
|
||||
future = submit_durable(executor, mark_remote_write_started,
|
||||
outcome=lambda: {"status": "completed"},
|
||||
job_kind="snapshot_job", job_id="one")
|
||||
future.result(timeout=2)
|
||||
assert not connection_failure_before_write("libvirt_connection_failed")
|
||||
assert current_execution().remote_write_started.is_set()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("earlier_write", [False, True])
|
||||
def test_background_terminal_connection_failure_preserves_actual_execution_evidence(
|
||||
monkeypatch, earlier_write):
|
||||
store = MemoryStore()
|
||||
coordinator = DurableCoordinator(store)
|
||||
parent = coordinator.claim(resources=(f"vm:7:{VM_ID}",))
|
||||
|
||||
def refused(*_args, **_kwargs):
|
||||
raise service_module.libvirt.libvirtError("connection refused")
|
||||
|
||||
monkeypatch.setattr(service_module, "get_runtime", lambda:
|
||||
SimpleNamespace(open=refused, queue_timeout=0.02))
|
||||
service = VMService("test", "images", "iso", "default", execution_scope="7")
|
||||
|
||||
def job():
|
||||
if earlier_write:
|
||||
mark_remote_write_started()
|
||||
try:
|
||||
service.ensure_storage_pools()
|
||||
except VMError as exc:
|
||||
store.set("reinstall_job", "one", json.dumps({
|
||||
"status": "failed", "error_code": exc.error_code}))
|
||||
|
||||
with execution_context(coordinator, parent, connection_failure_safe=True):
|
||||
with ThreadPoolExecutor(max_workers=1) as executor:
|
||||
submit_durable(executor, job,
|
||||
outcome=lambda: json.loads(store.get("reinstall_job", "one")),
|
||||
job_kind="reinstall_job", job_id="one").result(timeout=2)
|
||||
coordinator.release(parent)
|
||||
claims = list(coordinator.snapshot().values())
|
||||
if earlier_write:
|
||||
assert {row["kind"] for row in claims if row["state"] == "uncertain"} == {
|
||||
"background", "resource"}
|
||||
else:
|
||||
assert claims == []
|
||||
|
||||
|
||||
def test_new_execution_does_not_inherit_another_requests_write_evidence():
|
||||
coordinator = DurableCoordinator(MemoryStore())
|
||||
first, second = coordinator.claim(), coordinator.claim()
|
||||
with execution_context(coordinator, first, connection_failure_safe=True):
|
||||
mark_remote_write_started()
|
||||
with execution_context(coordinator, second, connection_failure_safe=True):
|
||||
assert connection_failure_before_write("libvirt_connection_failed")
|
||||
assert not connection_failure_before_write("libvirt_connection_failed")
|
||||
|
||||
|
||||
def test_rpc_intent_hook_marks_parent_before_a_write(tmp_path):
|
||||
coordinator = DurableCoordinator(MemoryStore())
|
||||
parent = coordinator.claim()
|
||||
runtime = LibvirtRuntime(tmp_path / "intents")
|
||||
session = SimpleNamespace(scope="7", token="a" * 32, failure=None, intent=None)
|
||||
with execution_context(coordinator, parent, connection_failure_safe=True):
|
||||
runtime._before_write(session, "create")
|
||||
assert session.intent.is_file()
|
||||
assert current_execution().remote_write_started.is_set()
|
||||
@ -0,0 +1,73 @@
|
||||
"""Interrupted snapshots attempt bounded thaw immediately, without polling."""
|
||||
import threading
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
from test_engine_boundary import boundary, call, USER, VM_ID
|
||||
from virsh_panel.libvirt_service import VMError
|
||||
from virsh_panel.metrics import host_scope
|
||||
|
||||
|
||||
@pytest.mark.parametrize('manual', [False, True])
|
||||
def test_snapshot_runner_recovers_freeze_without_releasing_unknown_claim(boundary, manual):
|
||||
client, engine, _store, _logs, services = boundary
|
||||
call(client, 'GET', '/internal/v1/hosts/hk/vms')
|
||||
components = engine.registry.components['hk']
|
||||
components.state.users.assign_vm(
|
||||
host_scope(components.manager.profile), VM_ID, USER['user_id'])
|
||||
service = services['hk']
|
||||
service.vm.update(managed=True, os_source='cloud_template',
|
||||
rescue_mode=False, rescue_staged=False)
|
||||
observed = []
|
||||
thaw_entered = threading.Event()
|
||||
|
||||
def interrupted(vm_id, snapshot_key, *, consistency, progress=None, recovery_job_id=None):
|
||||
observed.append(('create', recovery_job_id, snapshot_key))
|
||||
progress('freezing')
|
||||
raise VMError('backup result unknown', 504, error_code='libvirt_execution_uncertain')
|
||||
|
||||
def thaw(vm_id, *, recovery_job_id=None, snapshot_key=None):
|
||||
observed.append(('thaw', recovery_job_id, snapshot_key))
|
||||
thaw_entered.set()
|
||||
if manual:
|
||||
raise VMError('unproven backup boundary', 409,
|
||||
error_code='snapshot_thaw_recovery_required')
|
||||
return {'uuid': vm_id, 'thawed': True, 'count': 1, 'power_state': 'running'}
|
||||
|
||||
service.create_live_snapshot = interrupted
|
||||
service.thaw_guest_filesystems = thaw
|
||||
actor = {**USER, 'operation_id': 'snapshot-emergency-thaw-test'}
|
||||
response = call(client, 'POST', f'/internal/v1/hosts/hk/vms/{VM_ID}/snapshots', actor,
|
||||
{'snapshot_key': 'e'*32, 'name': 'snapshot',
|
||||
'consistency_mode': 'filesystem'})
|
||||
assert response.status_code == 202, response.text
|
||||
# No operation/snapshot poll drives the emergency action.
|
||||
assert thaw_entered.wait(3)
|
||||
job_id = observed[0][1]
|
||||
for _ in range(100):
|
||||
saved = engine.registry.apps['hk'].state.snapshot_job_state(job_id)
|
||||
if saved.get('freeze_recovery'):
|
||||
break
|
||||
time.sleep(.01)
|
||||
assert observed == [('create', job_id, 'e'*32), ('thaw', job_id, 'e'*32)]
|
||||
assert saved['status'] == 'uncertain'
|
||||
assert saved['error_code'] == 'libvirt_execution_uncertain'
|
||||
if manual:
|
||||
assert saved['guest_freeze_state'] == 'attempted'
|
||||
assert saved['freeze_recovery'] == 'manual_required'
|
||||
assert saved['freeze_recovery_error_code'] == 'snapshot_thaw_recovery_required'
|
||||
assert '立即' in saved['error']
|
||||
else:
|
||||
assert saved['guest_freeze_state'] == 'recovered'
|
||||
assert saved['freeze_recovery'] == 'confirmed'
|
||||
# Polling must keep the original operation uncertain and non-refundable.
|
||||
operation = call(client, 'GET', '/internal/v1/operations/snapshot-emergency-thaw-test', actor)
|
||||
assert operation.json()['status'] == 'uncertain'
|
||||
assert operation.json()['safe_to_refund'] is False
|
||||
claims = engine.coordinator.snapshot()
|
||||
assert any(claim.get('operation_id') == actor['operation_id'] for claim in claims.values())
|
||||
blocked = call(client, 'POST', f'/internal/v1/hosts/hk/vms/{VM_ID}/actions',
|
||||
{**USER, 'operation_id': 'action-after-thaw'}, {'action': 'poweroff'})
|
||||
assert blocked.status_code == 503, blocked.text
|
||||
assert blocked.json()['error_code'] == 'coordination_uncertain'
|
||||
@ -0,0 +1,268 @@
|
||||
"""Exercise guarded freeze recovery through actual spawned RPC workers."""
|
||||
import importlib
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import types
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from virsh_panel.libvirt_rpc import ExecutionFailure, LibvirtRuntime
|
||||
|
||||
VM_ID = '00000000-0000-4000-8000-000000000001'
|
||||
JOB_ID = '00000000-0000-4000-8000-000000000002'
|
||||
TOKEN = 'a' * 32
|
||||
SNAPSHOT_KEY = 'b' * 32
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def scenario(tmp_path, monkeypatch):
|
||||
fixtures = Path(__file__).with_name('fixtures')
|
||||
monkeypatch.syspath_prepend(str(fixtures))
|
||||
monkeypatch.setenv('PYTHONPATH', os.pathsep.join((
|
||||
str(fixtures), str(fixtures.parents[1]), os.environ.get('PYTHONPATH', ''))))
|
||||
backend = importlib.import_module('snapshot_recovery_backend')
|
||||
if 'libvirt' not in sys.modules:
|
||||
monkeypatch.setitem(sys.modules, 'libvirt', backend)
|
||||
service_module = importlib.import_module('virsh_panel.libvirt_service')
|
||||
monkeypatch.setattr(service_module, 'libvirt', backend)
|
||||
volume = service_module.snapshot_volume_name(VM_ID, SNAPSHOT_KEY)
|
||||
state_path = tmp_path / 'remote.json'
|
||||
state = dict(vm_id=VM_ID, volume_name=volume, token=TOKEN,
|
||||
target_path='/images/' + volume, frozen=False, thaw_calls=0)
|
||||
state_path.write_text(json.dumps(state))
|
||||
runtimes = []
|
||||
|
||||
def runtime():
|
||||
item = LibvirtRuntime(tmp_path/'intents', rpc_timeout=.2,
|
||||
connect_timeout=3, backend_module='snapshot_recovery_backend')
|
||||
runtimes.append(item)
|
||||
return item
|
||||
|
||||
def update(**values):
|
||||
current = json.loads(state_path.read_text())
|
||||
current.update(values)
|
||||
state_path.write_text(json.dumps(current))
|
||||
|
||||
def begin(**values):
|
||||
update(**values)
|
||||
item = runtime()
|
||||
session = item.open(str(state_path), 'host-a', timeout=5)
|
||||
domain = session.connection.lookupByUUIDString(VM_ID)
|
||||
session.bind_snapshot_recovery(domain, job_id=JOB_ID, volume_name=volume,
|
||||
pool_name='images', target_path=state['target_path'],
|
||||
runtime_token=TOKEN)
|
||||
return item, session, domain
|
||||
|
||||
def recover(item=None, **kwargs):
|
||||
item = item or runtime()
|
||||
monkeypatch.setattr(service_module, 'get_runtime', lambda: item)
|
||||
service = service_module.VMService(str(state_path), 'images', 'iso', 'default',
|
||||
execution_scope='host-a')
|
||||
# A recovery must not attempt to acquire/release the uncertain VM claim.
|
||||
service._operation_guard = lambda *a, **k: pytest.fail('normal claim used for emergency thaw')
|
||||
return service.thaw_guest_filesystems(
|
||||
kwargs.pop('vm_id', VM_ID), recovery_job_id=kwargs.pop('job_id', JOB_ID),
|
||||
snapshot_key=kwargs.pop('snapshot_key', SNAPSHOT_KEY))
|
||||
|
||||
yield types.SimpleNamespace(runtime=runtime, begin=begin, recover=recover,
|
||||
update=update, state=lambda: json.loads(state_path.read_text()),
|
||||
path=state_path, intent_dir=tmp_path/'intents', module=service_module)
|
||||
for item in reversed(runtimes):
|
||||
item.close(timeout=1)
|
||||
|
||||
|
||||
def interrupted(scenario, **values):
|
||||
item, session, domain = scenario.begin(backup_hangs=True, **values)
|
||||
domain.fsFreeze(None, 0)
|
||||
with pytest.raises(ExecutionFailure):
|
||||
domain.backupBegin('<domainbackup/>', None, 0)
|
||||
session.close()
|
||||
assert scenario.state()['frozen'] is True
|
||||
return item, session
|
||||
|
||||
|
||||
def test_exact_active_backup_can_thaw_without_releasing_original_fence(scenario):
|
||||
interrupted(scenario)
|
||||
original = json.loads(next(scenario.intent_dir.rglob('*.json')).read_text())
|
||||
result = scenario.recover()
|
||||
assert result['thawed'] is True
|
||||
assert scenario.state()['frozen'] is False
|
||||
updated = json.loads(next(scenario.intent_dir.rglob('*.json')).read_text())
|
||||
assert {key: value for key, value in updated.items() if key != 'snapshot_recovery'} == {
|
||||
key: value for key, value in original.items() if key != 'snapshot_recovery'}
|
||||
assert updated['snapshot_recovery']['backup_state'] == original['snapshot_recovery']['backup_state']
|
||||
session = scenario.runtime().open(str(scenario.path), 'host-a')
|
||||
with pytest.raises(ExecutionFailure) as raised:
|
||||
session.connection.lookupByUUIDString(VM_ID).destroy()
|
||||
assert raised.value.error_code == 'libvirt_execution_blocked'
|
||||
session.close()
|
||||
assert scenario.state().get('destroyed') is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize('change', [
|
||||
{'backup_active': False}, {'backup_path': '/images/unrelated.qcow2'},
|
||||
{'token': 'c'*32}, {'runtime_id': 8}, {'job_operation': 99},
|
||||
])
|
||||
def test_missing_or_mismatched_remote_boundary_requires_manual_recovery(scenario, change):
|
||||
interrupted(scenario)
|
||||
scenario.update(**change)
|
||||
with pytest.raises(scenario.module.VMError) as raised:
|
||||
scenario.recover()
|
||||
assert raised.value.error_code == 'snapshot_thaw_recovery_required'
|
||||
assert scenario.state()['thaw_calls'] == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize('identity', [
|
||||
{'job_id': '00000000-0000-4000-8000-000000000009'},
|
||||
{'vm_id': '00000000-0000-4000-8000-000000000009'},
|
||||
{'snapshot_key': 'c'*32},
|
||||
])
|
||||
def test_wrong_job_vm_or_snapshot_never_thaws(scenario, identity):
|
||||
interrupted(scenario)
|
||||
with pytest.raises(scenario.module.VMError) as raised:
|
||||
scenario.recover(**identity)
|
||||
assert raised.value.error_code == 'snapshot_thaw_recovery_required'
|
||||
assert scenario.state()['thaw_calls'] == 0
|
||||
|
||||
|
||||
def test_active_original_worker_cannot_be_recovered(scenario):
|
||||
item, session, domain = scenario.begin()
|
||||
domain.fsFreeze(None, 0)
|
||||
with pytest.raises(scenario.module.VMError) as raised:
|
||||
scenario.recover(item)
|
||||
assert raised.value.error_code == 'snapshot_thaw_recovery_required'
|
||||
assert scenario.state()['thaw_calls'] == 0
|
||||
session._fail('libvirt_execution_uncertain')
|
||||
session.close()
|
||||
|
||||
|
||||
def test_unacknowledged_freeze_and_old_intent_never_authorize_thaw(scenario):
|
||||
item, session, domain = scenario.begin(freeze_hangs=True)
|
||||
with pytest.raises(ExecutionFailure):
|
||||
domain.fsFreeze(None, 0)
|
||||
session.close()
|
||||
with pytest.raises(scenario.module.VMError):
|
||||
scenario.recover()
|
||||
path = next(scenario.intent_dir.rglob('*.json'))
|
||||
record = json.loads(path.read_text())
|
||||
record.pop('snapshot_recovery')
|
||||
path.write_text(json.dumps(record))
|
||||
with pytest.raises(scenario.module.VMError) as raised:
|
||||
scenario.recover()
|
||||
assert raised.value.error_code == 'snapshot_thaw_recovery_required'
|
||||
assert scenario.state()['thaw_calls'] == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize('backup_ack', [False, True])
|
||||
def test_durable_ack_boundary_survives_worker_interruption(scenario, backup_ack):
|
||||
item, session, domain = scenario.begin()
|
||||
domain.fsFreeze(None, 0)
|
||||
if backup_ack:
|
||||
domain.backupBegin('<domainbackup/>', None, 0)
|
||||
scenario.update(backup_active=False)
|
||||
session._fail('libvirt_execution_uncertain')
|
||||
session.close()
|
||||
assert scenario.recover()['thawed'] is True
|
||||
|
||||
|
||||
def test_recovery_timeout_preserves_original_intent(scenario):
|
||||
interrupted(scenario)
|
||||
original = json.loads(next(scenario.intent_dir.rglob('*.json')).read_text())
|
||||
scenario.update(thaw_hangs=True)
|
||||
with pytest.raises(scenario.module.VMError) as raised:
|
||||
scenario.recover()
|
||||
assert raised.value.error_code == 'libvirt_execution_uncertain'
|
||||
assert raised.value.outcome_unknown is True
|
||||
updated = json.loads(next(scenario.intent_dir.rglob('*.json')).read_text())
|
||||
assert {key: value for key, value in updated.items() if key != 'snapshot_recovery'} == {
|
||||
key: value for key, value in original.items() if key != 'snapshot_recovery'}
|
||||
assert updated['snapshot_recovery']['backup_state'] == original['snapshot_recovery']['backup_state']
|
||||
assert scenario.state()['frozen'] is True
|
||||
|
||||
|
||||
def test_success_receipt_never_thaws_a_later_freeze(scenario):
|
||||
interrupted(scenario)
|
||||
assert scenario.recover()['thawed']
|
||||
scenario.update(frozen=True)
|
||||
assert scenario.recover()['thawed']
|
||||
assert scenario.state()['frozen'] is True
|
||||
assert scenario.state()['thaw_calls'] == 1
|
||||
|
||||
|
||||
def test_unknown_recovery_cannot_be_replayed(scenario):
|
||||
interrupted(scenario)
|
||||
scenario.update(thaw_hangs=True)
|
||||
with pytest.raises(scenario.module.VMError):
|
||||
scenario.recover()
|
||||
scenario.update(thaw_hangs=False)
|
||||
with pytest.raises(scenario.module.VMError) as raised:
|
||||
scenario.recover()
|
||||
assert raised.value.error_code == 'snapshot_thaw_recovery_required'
|
||||
assert scenario.state()['thaw_calls'] == 1
|
||||
|
||||
|
||||
def test_original_thaw_receipt_recovers_without_another_guest_command(scenario):
|
||||
item, session, domain = scenario.begin()
|
||||
domain.fsFreeze(None, 0)
|
||||
domain.backupBegin('<domainbackup/>', None, 0)
|
||||
domain.fsThaw(None, 0)
|
||||
session._fail('libvirt_execution_uncertain')
|
||||
session.close()
|
||||
scenario.update(frozen=True)
|
||||
assert scenario.recover()['thawed']
|
||||
assert scenario.state()['frozen'] is True
|
||||
assert scenario.state()['thaw_calls'] == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize('corruption', ['json', 'list', 'boolean_pid', 'missing_stage'])
|
||||
def test_invalid_durable_evidence_is_explicit_manual_recovery(scenario, corruption):
|
||||
interrupted(scenario)
|
||||
path = next(scenario.intent_dir.rglob('*.json'))
|
||||
record = json.loads(path.read_text())
|
||||
if corruption == 'json':
|
||||
path.write_text('{broken')
|
||||
elif corruption == 'list':
|
||||
path.write_text('[]')
|
||||
else:
|
||||
if corruption == 'boolean_pid':
|
||||
record['snapshot_recovery']['worker_pid'] = True
|
||||
else:
|
||||
record['snapshot_recovery'].pop('thaw_pending')
|
||||
path.write_text(json.dumps(record))
|
||||
with pytest.raises(scenario.module.VMError) as raised:
|
||||
scenario.recover()
|
||||
assert raised.value.error_code == 'snapshot_thaw_recovery_required'
|
||||
assert scenario.state()['thaw_calls'] == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize('behavior', ['thaw_invalid', 'thaw_type_error'])
|
||||
def test_failure_after_thaw_call_never_resets_pending(scenario, behavior):
|
||||
interrupted(scenario)
|
||||
scenario.update(**{behavior: True})
|
||||
with pytest.raises(scenario.module.VMError) as raised:
|
||||
scenario.recover()
|
||||
assert raised.value.error_code == 'libvirt_execution_uncertain'
|
||||
assert raised.value.outcome_unknown is True
|
||||
state = json.loads(next(scenario.intent_dir.rglob('*.json')).read_text())
|
||||
assert state['snapshot_recovery']['recovery_state'] == 'pending'
|
||||
scenario.update(**{behavior: False})
|
||||
with pytest.raises(scenario.module.VMError) as raised:
|
||||
scenario.recover()
|
||||
assert raised.value.error_code == 'snapshot_thaw_recovery_required'
|
||||
assert scenario.state()['thaw_calls'] == 1
|
||||
|
||||
|
||||
def test_failed_boundary_persistence_prevents_backup_dispatch(scenario, monkeypatch):
|
||||
item, session, domain = scenario.begin()
|
||||
domain.fsFreeze(None, 0)
|
||||
def failed_sync(path):
|
||||
raise OSError('fsync unavailable')
|
||||
monkeypatch.setattr(item, '_sync', failed_sync)
|
||||
with pytest.raises(OSError):
|
||||
domain.backupBegin('<domainbackup/>', None, 0)
|
||||
assert scenario.state().get('backup_active') is None
|
||||
assert session.failure is not None
|
||||
session.close()
|
||||
assert list(scenario.intent_dir.rglob('*.json'))
|
||||
Loading…
Reference in New Issue