From 3ca8cfa1316e8b53bb5d93f3130c4c25139bfdce Mon Sep 17 00:00:00 2001 From: admin Date: Tue, 13 Oct 2026 17:33:52 +0800 Subject: [PATCH] fix(engine): acknowledge libvirt writes before connection cleanup --- docs/ENGINE_COORDINATION.md | 2 + docs/LIBVIRT_EXECUTION.md | 24 +- docs/VALIDATION.md | 26 + .../test_connection_failure_admission.py | 11 +- engine/tests/test_libvirt_rpc.py | 4 +- engine/tests/test_libvirt_rpc_failures.py | 27 +- engine/tests/test_libvirt_service_rpc.py | 46 ++ .../tests/test_rpc_native_error_receipts.py | 91 ++++ engine/tests/test_rpc_write_receipts.py | 493 ++++++++++++++++++ .../tests/test_snapshot_thaw_recovery_rpc.py | 3 +- engine/virsh_panel/libvirt_rpc.py | 271 +++++++--- engine/virsh_panel/libvirt_service.py | 6 + 12 files changed, 906 insertions(+), 98 deletions(-) create mode 100644 engine/tests/test_rpc_native_error_receipts.py create mode 100644 engine/tests/test_rpc_write_receipts.py diff --git a/docs/ENGINE_COORDINATION.md b/docs/ENGINE_COORDINATION.md index adc34dc0..f311193a 100644 --- a/docs/ENGINE_COORDINATION.md +++ b/docs/ENGINE_COORDINATION.md @@ -30,6 +30,8 @@ 连接建立失败单独返回 `libvirt_connection_failed`。只有明确通过 libvirt 执行远端变更、且本次执行尚未发出任何写入 RPC 的调用,才释放本次请求与资源占用,允许连接恢复后重新提交。写入证据在发送 RPC 前记录,并由同一执行的嵌套资源占用、线程池任务和后续连接共同继承;第一条连接写过后,第二条连接失败不能被当成执行前失败。VPC/Fabric 等会运行 SSH 修改脚本的工作流不使用“尚无 libvirt 写入”作为无副作用证明。该判定只用于正在执行的请求,不能据此清除历史 uncertain 记录。 +RPC 另行逐次记录 pending 和成功回执:已确认写入后,读取或连接收尾超时不再留下宿主级写入阻断。回执只解除该 RPC 的待确认状态,不能重置上述工作流的累计写入证据,也不能替代多步骤任务完成或资源清理证明;快照冻结恢复仍保留独立凭据。详见 [libvirt 写入回执与恢复](LIBVIRT_EXECUTION.md#写入结果未知与恢复)。 + 宿主指纹扫描、暂存配置准备和 libvirt 校验前的只读 SSH 能力探测若失败,须先持久化本次请求的 `failed_before_start` 凭据,再释放占用;全局扫描入口和指定宿主入口遵循同一规则。凭据写入失败或结果不明仍保留保护;该例外不适用于暂存配置提交后的失败,也不适用于任意 SSH 修改命令。 ## 数据库协议 diff --git a/docs/LIBVIRT_EXECUTION.md b/docs/LIBVIRT_EXECUTION.md index a069482d..8c3fb354 100644 --- a/docs/LIBVIRT_EXECUTION.md +++ b/docs/LIBVIRT_EXECUTION.md @@ -24,23 +24,31 @@ 这些是连接及 RPC 期限,不把镜像下载、外部 SSH 命令或一个包含多个连接的业务工作流误称为同一个请求墙钟期限。现有下载/SSH/后台任务预算仍适用。环境变量必须是有限正数,容量必须是正整数。测试用的显式无 scope 原生适配器不应出现在生产工厂。 -超时或停机后,会话永久失败,关闭 socket,并向本地 worker 私有进程组发送 TERM,再在有界等待后发送 KILL,回收其 SSH 子进程。业务代码即使捕获底层异常,离开连接上下文时也会重新抛出统一错误,不能把超时伪装成成功或 404。 +业务 RPC 超时或停机后,会话永久失败,关闭 socket,并向本地 worker 私有进程组发送 TERM,再在有界等待后发送 KILL,回收其 SSH 子进程。业务代码即使捕获底层异常,离开连接上下文时也会重新抛出统一错误,不能把超时伪装成成功或 404。仅发生在连接收尾阶段、且没有未确认写入的错误单独记录为 `cleanup_failure`,不覆盖已经返回的命令或业务结果。 ## 写入结果未知与恢复 -**终止本地客户端不证明远端 libvirtd/QEMU 已停止写入。** 发出第一条变更 RPC 前,引擎在 `/libvirt-execution//.json` 写入并 fsync 一个意图文件。普通意图记录 scope、随机 session ID、首个方法及时间。文件系统一致性快照还记录下文限定恢复所需的任务、运行代、目标卷身份和阶段凭据;不保存连接 URI、凭据、完整 XML 或任意 RPC 参数。 +**终止本地客户端不证明远端 libvirtd/QEMU 已停止写入。** 每次变更 RPC 发送前,引擎在 `/libvirt-execution//.json` 持久写入版本 2 意图:scope、随机 session ID、递增 sequence、`pending_write` 的序号/方法、`last_write_ack` 及时间。文件系统一致性快照还记录下文限定恢复所需的任务、运行代、目标卷身份和阶段凭据;不保存连接 URI、凭据、完整 XML 或任意 RPC 参数。 -正常完成并安全关闭会话才删除意图。中断、异常退出、清理失败或关机时已有写入的会话保留意图;引擎重启仍能发现它。该宿主机后续 libvirt 写入会被拒绝,只读观察仍可执行。现有已发往远端的并发操作不能被宣称已撤销。 +成功响应完整解码后,普通写入通过删除本次意图并 fsync 目录确认回执,再返回调用者;下次写入必须重新持久化新的 pending。准备、发送、读取响应、解码及确认均在同一会话 I/O 锁内,读取不能确认写入,并发调用不能清除彼此的 pending。删除或 fsync 失败时恢复原文件;无法恢复则停止接纳写入。 + +因此,已经确认的写入之后,`info()` 等读取失败只报告观察失败;`close()` 超时单独记录清理错误,两者不再因为历史写入产生整个宿主机的阻断。多步骤业务是否完成仍由 VM/存储级任务及持久占用判断,不能把“没有 pending RPC”当作整个创建、删除或重装成功。`stream.finish()` / `abort()` 也经过写入回执边界,上传分块成功不代表最终提交已确认。 + +真正丢失写入响应、解码失败或回执无法持久化时,保留 pending 与宿主保护;后来的成功回执不能覆盖之前的未确认写入。已有版本 1 和其他旧未决文件仍按原规则阻断,不迁移、不自动清理。只读观察仍可执行,现有已发往远端的并发操作不能被宣称已撤销。 + +原生 `libvirtError` 与成功回执不同。收到错误时记录独立 `write_error_receipt`,维持现有业务在同连接内读取、回退及核验的能力,例如存储池 `build()` 报错后的 `create()`。后续成功 RPC 不清除此错误记录;只有原工作流正常收尾才按既有规则解除。后续读取或关闭超时仍保留保护,不把原生错误当成成功或无副作用证明。 | 错误码 | 含义 | | --- | --- | -| `libvirt_execution_timeout` | 尚未记录写入的会话超时或失去执行器 | -| `libvirt_execution_stopped` | 尚未记录写入时引擎停止接单 | -| `libvirt_execution_uncertain` | 会话已记录写入,结果未知 | +| `libvirt_execution_timeout` | 没有未确认写入的 RPC 超时或失去执行器;可能已有成功写入回执 | +| `libvirt_execution_stopped` | 没有未确认写入时引擎停止接单,或使用已关闭会话 | +| `libvirt_execution_uncertain` | 待确认写入、错误收尾或必要恢复阶段的结果未知 | | `libvirt_execution_blocked` | 同宿主存在未解决的写入意图,本次写入被阻止 | | `libvirt_connection_failed` | 原生 libvirt 连接未建立;只有本次执行尚无远端写入证据时才允许释放新占用 | -这组错误不会转换为 `libvirtError`。后两项跨重启保持 `uncertain`、`safe_to_refund=false`,不自动重放、释放 IP/归属或触发清理退款。快照、重装和救援任务保留原记录、必要凭据与任务占用;重装超时不会被当成新系统已提交。 +这组错误不会转换为 `libvirtError`。真正未知写入及其旧阻断记录跨重启保留,不自动重放、释放 IP/归属或触发清理退款;仅被拒绝的新请求仍适用[未派发失败规则](ENGINE_COORDINATION.md#执行前失败与准入拒绝2026-10-13)。快照、重装和救援任务保留原记录、必要凭据与任务占用;重装超时不会被当成新系统已提交。 + +依据:[virConnectClose](https://libvirt.org/html/libvirt-libvirt-host.html#virConnectClose) 是连接资源释放接口,不能用其失败否定此前 API 回执;[libvirt 错误接口](https://libvirt.org/html/libvirt-virterror.html) 返回原生错误信息,不提供面板业务事务回滚保证。上述序号、意图及回执是引擎自己的协议。 遇到写入保护时的运维流程: @@ -55,6 +63,8 @@ 普通写入保护不能阻止原快照任务对已冻结来宾的必要解冻。新的文件系统一致性快照在冻结前持久绑定宿主 scope、任务 ID、VM UUID、确定性快照卷名和路径、QEMU 运行 ID 及只存在于实时 XML 的 QGA 运行代令牌。冻结响应和备份启动阶段分别落盘;阶段证据无法持久化时停止后续操作。 +逐次 RPC 成功不会删除这份恢复记录:`pending_write` 可以清空,但已确认冻结仍需要解冻,恢复身份和阶段继续持久保留。原会话明确未冻结或已取得解冻回执,且没有其他未知结果时才可在收尾删除自身记录;真正未知阶段仍保留原凭据供受限恢复使用。 + 原会话失败并回收本地 worker 后,快照任务立即尝试受限恢复;服务启动核对和原任务核对也使用同一入口。只有原冻结已确认,且备份尚未发送、启动响应已确认,或启动响应未知但能读到本任务精确目标的活动备份时,才允许新连接执行一次限定的 `fsThaw`。恢复再次检查当前 VM 运行代和备份身份;其他活跃写入者、多个未决意图、缺少证据、目标不符或无法确认旧 worker 已退出都会拒绝自动解冻。旧 worker 消失本身不构成远端阶段证明。 该入口不能发送任意写操作;普通 `fsThaw` 仍受原写入保护约束。解冻发送前、确认后分别持久化凭据;已确认的恢复只读取原回执,不再次发送解冻,响应未知的恢复转人工核对。解冻确认只更新来宾冻结恢复状态,**不删除原意图、不解除 uncertain 占用、不把备份标成成功,也不允许重放原任务**。`fsThaw` 返回 0 表示当前没有文件系统需要解冻,可以确认恢复,但不能证明快照已完成。 diff --git a/docs/VALIDATION.md b/docs/VALIDATION.md index b59dba1a..8f2d336a 100644 --- a/docs/VALIDATION.md +++ b/docs/VALIDATION.md @@ -1,5 +1,31 @@ # 本版验证记录 +## 2026-10-13:成功写入后的查询与连接关闭不再封锁宿主机 + +- libvirt RPC 改为逐次持久化待确认写入和对应成功回执;正常回执确认后,后续 + `info()` 超时只报告观察失败,`close()` 超时记录独立清理错误,不再因历史写入 + 留下宿主级阻断。VMService 保留已确认的电源命令结果,清理错误写入脱敏日志。 + 工作流累计写入证据不清零,多步骤任务仍需各自完成与清理证明。 +- 真正丢失回执、响应格式错误、回执持久化失败及停机竞态仍保留保护;后续成功 + 不覆盖前一次未知写入。原生 libvirt 错误保留独立记录,允许原有存储池 + `build()` 失败后 `create()` 的回退核验;后续连接失败不会将它误确认为成功。 + 快照冻结/解冻凭据独立保留,流 `finish()` / `abort()` 纳入写入回执边界。 +- Python 21 个相关测试文件统一执行:**764 passed、81 subtests passed、3 failed**, + 耗时 65.65 秒。新增两套真实 spawn/socket 故障注入回归,覆盖 domain/QGA 写后 + 读取及关闭超时、同宿主其他 VM 继续写入、重启后行为、第二次写回执丢失、 + unlink/fsync 失败、并发 ACK 串行化、已关闭句柄、原生错误回退与旧记录保护。 + 同时覆盖 VMService 持久锁释放、快照恢复、未派发证明、资源锁及停机回归。 +- 三项失败在修改前 `e55b26f1fd31fc2304c56d8a8f7841cffdf1f871` 的独立 + 源码快照及相同 Python 3.12 依赖中再次复现,失败位置一致: + `test_cloud_template_import_job_and_create_dispatch`(策略快照旧断言)、 + `test_reinstall_cleanup_pending_is_not_reported_success`(错误字段旧断言)、 + `test_fabric_is_engine_global_and_super_admin_write_only`(Fabric readiness)。 + 未修改或放宽这些断言,不宣称整个测试集通过。 +- `git diff --check` 通过,独立复核未发现阻断项。无 schema 变更,不迁移或 + 自动清理旧任务、协调锁及版本 1 RPC 文件。未部署生产,未执行真实 Linux + libvirt/SSH/QGA/systemd/MariaDB 验收或大镜像上传性能测试。 + 协议与恢复边界见 [libvirt 执行说明](LIBVIRT_EXECUTION.md#写入结果未知与恢复)。 + ## 2026-10-13:未派发失败不再留下或扩散 VM 持久锁 - 状态/XML/创建前检查失败、RPC 排队超时以及被其他 VM 的存储占用拒绝, diff --git a/engine/tests/test_connection_failure_admission.py b/engine/tests/test_connection_failure_admission.py index 7a745b8a..d5d1aa8b 100644 --- a/engine/tests/test_connection_failure_admission.py +++ b/engine/tests/test_connection_failure_admission.py @@ -26,7 +26,7 @@ from virsh_panel.database import DatabaseTemplateStore from virsh_panel.durable_jobs import submit_durable from virsh_panel.engine_context import EngineComponents, ExternalOwnership, HostStore from virsh_panel.internal import EngineApplication, HostRegistry, json_response -from virsh_panel.libvirt_rpc import LibvirtRuntime +from virsh_panel.libvirt_rpc import LibvirtRuntime, RPCSession from virsh_panel.libvirt_service import GIB, VMError, VMService from virsh_panel.metrics import MetricsCollector from virsh_panel.remote_host import RemoteHostError @@ -366,8 +366,13 @@ def test_rpc_intent_hook_marks_parent_before_a_write(tmp_path): coordinator = DurableCoordinator(MemoryStore()) parent = coordinator.claim() runtime = LibvirtRuntime(tmp_path / "intents") - session = SimpleNamespace(scope="7", token="a" * 32, failure=None, intent=None) + session = RPCSession(runtime, "test:///default", "7", 5) with execution_context(coordinator, parent, connection_failure_safe=True): - runtime._before_write(session, "create") + sequence = runtime._before_write(session, "create") assert session.intent.is_file() assert current_execution().remote_write_started.is_set() + runtime._ack_write(session, sequence) + assert session.intent is None + # The RPC reply clears only its pending receipt, never the enclosing + # workflow's evidence that a remote write has already happened. + assert current_execution().remote_write_started.is_set() diff --git a/engine/tests/test_libvirt_rpc.py b/engine/tests/test_libvirt_rpc.py index b6edad62..8eca26a6 100644 --- a/engine/tests/test_libvirt_rpc.py +++ b/engine/tests/test_libvirt_rpc.py @@ -334,7 +334,9 @@ def test_connection_close_cannot_wait_forever(runtime_factory): started = time.monotonic() session.close() assert time.monotonic() - started < 3.0 - assert session.failure is not None + assert session.failure is None + assert session.cleanup_failure is not None + assert session.cleanup_failure.outcome_unknown is False assert not session.process.is_alive() diff --git a/engine/tests/test_libvirt_rpc_failures.py b/engine/tests/test_libvirt_rpc_failures.py index 743b22e3..2d8914bf 100644 --- a/engine/tests/test_libvirt_rpc_failures.py +++ b/engine/tests/test_libvirt_rpc_failures.py @@ -28,6 +28,9 @@ def writing_session(runtime): session = runtime.open("test:///default", "host-a", timeout=10.0) domain = session.connection.lookupByUUIDString(VM_ID) domain.destroy() + # The first RPC is acknowledged. These ordering tests start with the next + # write prepared, before its response is received and durably committed. + runtime._before_write(session, "destroy") assert session.intent is not None and session.intent.is_file() return session, domain @@ -99,6 +102,9 @@ def test_unlink_failure_keeps_fence_and_exposes_execution_failure(runtime, monke return unlink(path, *args, **kwargs) monkeypatch.setattr(Path, "unlink", fail_intent_unlink) + with pytest.raises(ExecutionFailure) as unacknowledged: + runtime._ack_write(session, session.pending_write["sequence"]) + assert unacknowledged.value.outcome_unknown is True session.close() assert session.failure.error_code == "libvirt_execution_uncertain" assert intent.read_bytes() == original @@ -128,6 +134,9 @@ def test_directory_sync_failure_restores_deleted_fence( return sync(directory) monkeypatch.setattr(runtime, "_sync", fail_sync) + with pytest.raises(ExecutionFailure) as unacknowledged: + runtime._ack_write(session, session.pending_write["sequence"]) + assert unacknowledged.value.outcome_unknown is True session.close() assert session.failure.error_code == "libvirt_execution_uncertain" assert session.failure.outcome_unknown is True @@ -172,15 +181,13 @@ def test_signal_failure_does_not_mask_rpc_failure( fault.setattr(session, "_signal", failed_signal) with pytest.raises(ExecutionFailure) as raised: session.connection.getHostname() - expected = "libvirt_execution_uncertain" if write_completed else "libvirt_execution_timeout" - assert raised.value.error_code == expected - assert raised.value.outcome_unknown is write_completed + assert raised.value.error_code == "libvirt_execution_timeout" + assert raised.value.outcome_unknown is False assert "Unable to signal libvirt worker" in caplog.text assert "injected sensitive signal details" not in caplog.text session.close() assert not session.process.is_alive() - if write_completed: - assert session.intent.is_file() + assert session.intent is None def test_shutdown_continues_interrupting_after_one_session_fails(runtime, monkeypatch): @@ -233,7 +240,7 @@ def test_runtime_close_reaps_other_groups_after_one_reap_fails(runtime, monkeypa @pytest.mark.parametrize("operation", ["call", "qga"]) -def test_intent_created_after_a_concurrent_failure_upgrades_the_outcome( +def test_concurrent_failure_before_admission_cannot_create_a_new_intent( runtime, monkeypatch, operation): session = runtime.open("test:///default", "host-a", timeout=10.0) domain = session.connection.lookupByUUIDString(VM_ID) @@ -249,8 +256,8 @@ def test_intent_created_after_a_concurrent_failure_upgrades_the_outcome( domain.destroy() else: session.qemu_agent_command(domain, '{"execute":"guest-exec"}', 1, 0) - assert raised.value.error_code == "libvirt_execution_uncertain" - assert raised.value.outcome_unknown is True - assert session.intent.is_file() + assert raised.value.error_code == "libvirt_execution_timeout" + assert raised.value.outcome_unknown is False + assert session.intent is None session.close() - assert session.intent.is_file() + assert not list(runtime.state_dir.rglob("*.json")) diff --git a/engine/tests/test_libvirt_service_rpc.py b/engine/tests/test_libvirt_service_rpc.py index 89805144..ae02985b 100644 --- a/engine/tests/test_libvirt_service_rpc.py +++ b/engine/tests/test_libvirt_service_rpc.py @@ -10,6 +10,7 @@ import types import pytest from test_libvirt_rpc import VM_ID, runtime_factory # noqa: F401 +from test_durable_coordination import coordinator # noqa: F401 @pytest.fixture @@ -87,3 +88,48 @@ def test_scoped_vmservice_cannot_swallow_a_poisoned_rpc_session(scoped_service, assert sessions[-1].pid != os.getpid() assert not sessions[-1].process.is_alive() assert bool(list(runtime.state_dir.rglob("*.json"))) is write + + +def test_acknowledged_power_action_survives_connection_close_timeout( + scoped_service, coordinator, monkeypatch, caplog): + from virsh_panel import coordination + + monkeypatch.setattr(coordination, "_default_coordinator", coordinator) + _module, _runtime, sessions, create = scoped_service + result = create("test:///close-hangs").action(VM_ID, "poweroff") + + assert result == {"uuid": VM_ID, "action": "poweroff", "command_accepted": True} + session = sessions[-1] + assert session.failure is None + assert session.cleanup_failure.error_code == "libvirt_execution_timeout" + assert session.cleanup_failure.outcome_unknown is False + assert not session.process.is_alive() + assert coordinator.snapshot() == {} + assert "libvirt connection cleanup failed" in caplog.text + assert "test:///close-hangs" not in caplog.text + + # Both durable VM admission and the host RPC admission remain usable. + assert create().action(VM_ID, "poweroff")["command_accepted"] is True + assert coordinator.snapshot() == {} + + +def test_missing_power_write_receipt_retains_both_protections( + scoped_service, coordinator, monkeypatch): + from virsh_panel import coordination + + monkeypatch.setattr(coordination, "_default_coordinator", coordinator) + module, runtime, sessions, create = scoped_service + + with pytest.raises(module.LibvirtExecutionError) as raised: + create("test:///write-hangs").action(VM_ID, "poweroff") + assert raised.value.error_code == "libvirt_execution_uncertain" + assert raised.value.outcome_unknown is True + assert sessions[-1].failure.outcome_unknown is True + claims = coordinator.snapshot() + assert len(claims) == 1 + assert next(iter(claims.values()))["state"] == "uncertain" + assert list(runtime.state_dir.rglob("*.json")) + + with pytest.raises(module.VMError) as blocked: + create().action(VM_ID, "poweroff") + assert blocked.value.error_code == "coordination_uncertain" diff --git a/engine/tests/test_rpc_native_error_receipts.py b/engine/tests/test_rpc_native_error_receipts.py new file mode 100644 index 00000000..30f1e65c --- /dev/null +++ b/engine/tests/test_rpc_native_error_receipts.py @@ -0,0 +1,91 @@ +"""Native error replies retain the existing same-session recovery contract.""" + +import os + +import pytest + +from test_libvirt_rpc import VM_ID, runtime_factory # noqa: F401 +from virsh_panel.libvirt_rpc import ExecutionFailure + + +@pytest.fixture +def native_error_runtime(runtime_factory, tmp_path, monkeypatch): + # Model the real storage-pool fallback: build can report an existing + # directory, after which create and readback must remain callable. + name = "libvirt_native_error_receipt_backend" + (tmp_path / (name + ".py")).write_text(''' +import libvirt_rpc_backend as base + +libvirtError = base.libvirtError + +class virStoragePool(base.virStoragePool): + def build(self, flags=0): + raise libvirtError("directory already exists", ( + 9, 18, "directory already exists", 2, None, None, None, 0, 0)) + + def create(self, flags=0): + return 0 + +class virConnect(base.virConnect): + def __init__(self, uri): + super().__init__(uri) + self.pool = virStoragePool(self) + +def open(uri): + return virConnect(uri) +''', encoding="utf-8") + monkeypatch.syspath_prepend(str(tmp_path)) + monkeypatch.setenv("PYTHONPATH", str(tmp_path) + os.pathsep + + os.environ.get("PYTHONPATH", "")) + return runtime_factory(backend_module=name) + + +def build_error(session): + pool = session.connection.storagePoolLookupByName("images") + with pytest.raises(Exception) as raised: + pool.build(0) + assert not isinstance(raised.value, ExecutionFailure) + assert raised.value.get_error_code() == 9 + assert session.failure is None + return pool + + +def test_native_error_still_allows_existing_pool_start_and_readback(native_error_runtime): + runtime = native_error_runtime + session = runtime.open("test:///default", "host-a", timeout=5) + pool = build_error(session) + assert pool.create(0) == 0 + assert pool.info()[0] == 2 + # The error receipt is not a success ACK. A later successful write must + # preserve its cleanup obligation until this healthy session closes. + assert session.intent.is_file() + session.close() + assert session.failure is None + assert not list(runtime.state_dir.rglob("*.json")) + + later = runtime.open("test:///default", "host-a", timeout=5) + assert later.connection.lookupByUUIDString(VM_ID).destroy() == 0 + later.close() + + +@pytest.mark.parametrize("failure", ["read", "close"]) +def test_native_error_obligation_survives_later_success_then_connection_failure( + native_error_runtime, failure): + runtime = native_error_runtime + session = runtime.open("test:///" + failure + "-hangs", "host-a", timeout=5) + assert build_error(session).create(0) == 0 + if failure == "read": + with pytest.raises(ExecutionFailure) as raised: + session.connection.getHostname() + assert raised.value.outcome_unknown is True + session.close() + assert session.failure.error_code == "libvirt_execution_uncertain" + assert session.failure.outcome_unknown is True + assert session.intent.is_file() + + observer = runtime.open("test:///default", "host-a", timeout=5) + assert observer.connection.getVersion() > 0 + with pytest.raises(ExecutionFailure) as blocked: + observer.connection.lookupByUUIDString(VM_ID).destroy() + assert blocked.value.error_code == "libvirt_execution_blocked" + observer.close() diff --git a/engine/tests/test_rpc_write_receipts.py b/engine/tests/test_rpc_write_receipts.py new file mode 100644 index 00000000..e8091de5 --- /dev/null +++ b/engine/tests/test_rpc_write_receipts.py @@ -0,0 +1,493 @@ +"""A later observation failure cannot erase or invent a libvirt write receipt.""" + +from __future__ import annotations + +import json +import os +import threading +import time +from pathlib import Path + +import pytest + +from virsh_panel import libvirt_rpc +from virsh_panel.libvirt_rpc import ExecutionFailure, LibvirtRuntime + + +VM_ID = "00000000-0000-4000-8000-000000000001" +OTHER_VM_ID = "00000000-0000-4000-8000-000000000002" +FIXTURES = Path(__file__).with_name("fixtures") + +# Keep the fault injection in an importable module: calls below cross the real +# multiprocessing/socket boundary rather than replacing the supervisor's RPC. +BACKEND = ''' +import json +import libvirt_rpc_backend as base +from libvirt_rpc_backend import libvirtError + + +class virDomain(base.virDomain): + def __init__(self, connection, uuid): + super().__init__(connection, uuid) + self.writes = 0 + + def destroy(self): + self.writes += 1 + if self.connection.uri == "test:///second-write-hangs" and self.writes == 2: + self.connection.hang() + if self.connection.uri.startswith("test:///write-error"): + raise libvirtError("write failed after entering native API") + return 0 + + def create(self): + self.writes += 1 + return 0 + + def info(self): + if self.connection.uri in {"test:///info-hangs", "test:///write-error-info-hangs"} and self.writes: + self.connection.hang() + return super().info() + + +class virStream(base.virStream): + def finish(self): + if self.connection.uri == "test:///stream-finish-hangs": + self.connection.hang() + return super().finish() + + def abort(self): + if self.connection.uri == "test:///stream-abort-hangs": + self.connection.hang() + return super().abort() + + +class virConnect(base.virConnect): + def __init__(self, uri): + super().__init__(uri) + self.domains = {} + + def lookupByUUIDString(self, uuid): + if uuid not in self.domains: + self.domains[uuid] = virDomain(self, uuid) + return self.domains[uuid] + + def newStream(self, flags=0): + return virStream(self) + + def close(self): + if self.uri == "test:///write-error-close-hangs": + self.hang() + return super().close() + + +def open(uri): + return virConnect(uri) + + +def qemuAgentCommand(domain, command, timeout, flags): + if json.loads(command).get("execute") == "guest-exec": + domain.writes += 1 + return base.qemuAgentCommand(domain, command, timeout, flags) +''' + + +@pytest.fixture +def receipt_runtime(tmp_path, monkeypatch): + backend_dir = tmp_path / "backend" + backend_dir.mkdir() + (backend_dir / "rpc_receipt_backend.py").write_text(BACKEND, encoding="utf-8") + monkeypatch.syspath_prepend(str(FIXTURES)) + monkeypatch.syspath_prepend(str(backend_dir)) + monkeypatch.setenv("PYTHONPATH", os.pathsep.join(filter(None, [ + str(backend_dir), str(FIXTURES), str(FIXTURES.parents[1]), + os.environ.get("PYTHONPATH", ""), + ]))) + runtimes = [] + + def create(**options): + runtime = LibvirtRuntime( + tmp_path / "rpc-state", rpc_timeout=options.pop("rpc_timeout", 0.35), + connect_timeout=3.0, backend_module="rpc_receipt_backend", **options, + ) + runtimes.append(runtime) + return runtime + + yield create + for runtime in reversed(runtimes): + runtime.close(timeout=2.0) + + +def assert_other_vm_writable(runtime): + other = runtime.open("test:///default", "host-a", timeout=5.0) + try: + assert other.connection.lookupByUUIDString(OTHER_VM_ID).destroy() == 0 + finally: + other.close() + + +def assert_other_vm_blocked(runtime): + other = runtime.open("test:///default", "host-a", timeout=5.0) + try: + domain = other.connection.lookupByUUIDString(OTHER_VM_ID) + assert domain.info()[0] == 1 # Fences restrict writes, not observations. + with pytest.raises(ExecutionFailure) as blocked: + domain.destroy() + assert blocked.value.error_code == "libvirt_execution_blocked" + finally: + other.close() + + +@pytest.mark.parametrize("mutation", ["domain", "qga"]) +def test_closed_handle_cannot_create_a_new_write_intent(receipt_runtime, mutation): + runtime = receipt_runtime() + session = runtime.open("test:///default", "host-a", timeout=5) + domain = session.connection.lookupByUUIDString(VM_ID) + domain.destroy() + session.close() + with pytest.raises(ExecutionFailure) as stopped: + if mutation == "domain": + domain.destroy() + else: + session.qemu_agent_command(domain, '{"execute":"guest-exec"}', 1, 0) + assert stopped.value.error_code == "libvirt_execution_stopped" + assert session.intent is None + assert not list(runtime.state_dir.rglob("*.json")) + assert_other_vm_writable(runtime) + + +@pytest.mark.parametrize("failure_phase", ["info", "close"]) +@pytest.mark.parametrize("mutation", ["domain", "qga"]) +def test_acknowledged_write_followed_by_read_or_close_timeout_has_no_host_fence( + receipt_runtime, failure_phase, mutation): + runtime = receipt_runtime() + session = runtime.open(f"test:///{failure_phase}-hangs", "host-a", timeout=5.0) + domain = session.connection.lookupByUUIDString(VM_ID) + if mutation == "domain": + assert domain.destroy() == 0 + else: + result = session.qemu_agent_command(domain, '{"execute":"guest-exec"}', 1, 0) + assert json.loads(result)["return"]["execute"] == "guest-exec" + + if failure_phase == "info": + with pytest.raises(ExecutionFailure) as timed_out: + domain.info() + assert timed_out.value.error_code == "libvirt_execution_timeout" + assert timed_out.value.outcome_unknown is False + session.close() + + failure = session.cleanup_failure if failure_phase == "close" else session.failure + assert failure is not None + assert failure.outcome_unknown is False + if failure_phase == "close": + assert session.failure is None # Cleanup cannot replace a command receipt. + assert not session.process.is_alive() + assert not list(runtime.state_dir.rglob("*.json")) + assert_other_vm_writable(runtime) + runtime.close(timeout=2.0) + assert_other_vm_writable(receipt_runtime()) + + +def test_lost_second_write_receipt_retains_fence_despite_first_success(receipt_runtime): + runtime = receipt_runtime() + session = runtime.open("test:///second-write-hangs", "host-a", timeout=5.0) + domain = session.connection.lookupByUUIDString(VM_ID) + assert domain.destroy() == 0 + with pytest.raises(ExecutionFailure) as lost: + domain.destroy() + assert lost.value.error_code == "libvirt_execution_uncertain" + assert lost.value.outcome_unknown is True + session.close() + fences = list(runtime.state_dir.rglob("*.json")) + assert len(fences) == 1 + evidence = json.loads(fences[0].read_text()) + assert evidence["pending_write"] == {"sequence": 2, "method": "destroy"} + assert evidence["last_write_ack"] == 1 + assert_other_vm_blocked(runtime) + runtime.close(timeout=2.0) + assert_other_vm_blocked(receipt_runtime()) + + +@pytest.mark.parametrize("failure_phase", ["info", "close"]) +def test_native_write_error_does_not_prove_a_rollback(receipt_runtime, failure_phase): + runtime = receipt_runtime() + session = runtime.open(f"test:///write-error-{failure_phase}-hangs", "host-a", timeout=5.0) + domain = session.connection.lookupByUUIDString(VM_ID) + with pytest.raises(Exception) as native: + domain.destroy() + assert not isinstance(native.value, ExecutionFailure) + assert session.failure is None # Native fallback/reconciliation can continue. + evidence = json.loads(session.intent.read_text()) + assert evidence["write_error_receipt"] == {"sequence": 1, "method": "destroy"} + assert evidence["pending_write"] is None + assert evidence["last_write_ack"] == 0 + if failure_phase == "info": + with pytest.raises(ExecutionFailure) as lost_observation: + domain.info() + assert lost_observation.value.outcome_unknown is True + session.close() + assert session.failure is not None + assert session.failure.outcome_unknown is True + assert len(list(runtime.state_dir.rglob("*.json"))) == 1 + assert_other_vm_blocked(runtime) + + +@pytest.mark.parametrize("read_times_out", [False, True]) +def test_successful_fallback_does_not_acknowledge_the_earlier_native_error( + receipt_runtime, read_times_out): + runtime = receipt_runtime() + uri = "test:///write-error-info-hangs" if read_times_out else "test:///write-error" + session = runtime.open(uri, "host-a", timeout=5.0) + domain = session.connection.lookupByUUIDString(VM_ID) + with pytest.raises(Exception) as native: + domain.destroy() + assert not isinstance(native.value, ExecutionFailure) + assert domain.create() == 0 + evidence = json.loads(session.intent.read_text()) + assert evidence["last_write_ack"] == 2 + assert evidence["write_error_receipt"] == {"sequence": 1, "method": "destroy"} + assert evidence["pending_write"] is None + if read_times_out: + with pytest.raises(ExecutionFailure) as lost_observation: + domain.info() + assert lost_observation.value.outcome_unknown is True + else: + assert domain.info()[0] == 1 + session.close() + if read_times_out: + assert_other_vm_blocked(runtime) + else: + assert session.failure is None + assert not list(runtime.state_dir.rglob("*.json")) + assert_other_vm_writable(runtime) + + +@pytest.mark.parametrize("completion", ["finish", "abort"]) +def test_stream_completion_lost_receipt_remains_a_pending_write( + receipt_runtime, completion): + runtime = receipt_runtime() + session = runtime.open(f"test:///stream-{completion}-hangs", "host-a", timeout=5.0) + connection = session.connection + volume = connection.storagePoolLookupByName("images").storageVolLookupByName("disk.qcow2") + stream = connection.newStream(0) + assert volume.upload(stream, 0, 3, 0) == 0 + assert stream.send(b"abc") == 3 + with pytest.raises(ExecutionFailure) as incomplete: + getattr(stream, completion)() + assert incomplete.value.outcome_unknown is True + session.close() + fences = list(runtime.state_dir.rglob("*.json")) + assert len(fences) == 1 + evidence = json.loads(fences[0].read_text()) + assert evidence["last_write_ack"] == 2 + assert evidence["pending_write"] == {"sequence": 3, "method": completion} + assert_other_vm_blocked(runtime) + + +def test_read_connection_cannot_clear_a_legacy_unconfirmed_write(receipt_runtime): + runtime = receipt_runtime() + directory = runtime._directory("host-a") + path = directory / "old-worker.json" + original = json.dumps({ + "version": 1, "scope": "host-a", "session": "old-worker", + "method": "destroy", "started_at": 1, + }).encode() + path.write_bytes(original) + observer = runtime.open("test:///default", "host-a", timeout=5.0) + assert observer.connection.lookupByUUIDString(VM_ID).info()[0] == 1 + observer.close() + assert_other_vm_blocked(runtime) + assert path.read_bytes() == original + + +def test_shutdown_after_acknowledged_write_does_not_quarantine_host(receipt_runtime): + runtime = receipt_runtime() + session = runtime.open("test:///default", "host-a", timeout=5.0) + assert session.connection.lookupByUUIDString(VM_ID).destroy() == 0 + runtime.begin_shutdown() + assert session.failure is not None + assert session.failure.outcome_unknown is False + assert runtime.close(timeout=2.0) is True + assert not list(runtime.state_dir.rglob("*.json")) + assert_other_vm_writable(receipt_runtime()) + + +def test_shutdown_during_next_write_preserves_its_unconfirmed_receipt( + receipt_runtime, tmp_path): + runtime = receipt_runtime(rpc_timeout=30.0) + session = runtime.open("test:///default", "host-a", timeout=60.0) + assert session.connection.lookupByUUIDString(VM_ID).destroy() == 0 + marker = tmp_path / "second-write-started" + errors = [] + + def write_until_stopped(): + try: + session.connection.hang(str(marker), True) + except BaseException as exc: + errors.append(exc) + + caller = threading.Thread(target=write_until_stopped, daemon=True) + caller.start() + deadline = time.monotonic() + 3.0 + while not marker.exists() and time.monotonic() < deadline: + time.sleep(0.01) + assert marker.exists(), "worker did not receive the next mutation" + runtime.begin_shutdown() + assert runtime.close(timeout=2.0) is True + caller.join(timeout=2.0) + assert not caller.is_alive() + assert len(errors) == 1 + assert isinstance(errors[0], ExecutionFailure) + assert errors[0].outcome_unknown is True + assert len(list(runtime.state_dir.rglob("*.json"))) == 1 + assert_other_vm_blocked(receipt_runtime()) + + +@pytest.mark.parametrize("failure_kind", ["unlink", "fsync"]) +def test_acknowledgement_durability_failure_keeps_the_write_fenced( + receipt_runtime, monkeypatch, failure_kind): + runtime = receipt_runtime() + session = runtime.open("test:///default", "host-a", timeout=5.0) + domain = session.connection.lookupByUUIDString(VM_ID) + receipt_received = threading.Event() + receive = libvirt_rpc._receive + sync = runtime._sync + unlink = Path.unlink + failures = [] + + def record_receipt(*args): + response = receive(*args) + receipt_received.set() + return response + + def fail_ack_sync(directory): + if failure_kind == "fsync" and receipt_received.is_set() and not failures: + failures.append(directory) + raise OSError("injected write receipt durability failure") + return sync(directory) + + def fail_ack_unlink(path, *args, **kwargs): + if (failure_kind == "unlink" and receipt_received.is_set() + and path == session.intent and not failures): + failures.append(path) + raise OSError("injected write receipt unlink failure") + return unlink(path, *args, **kwargs) + + with monkeypatch.context() as fault: + fault.setattr(libvirt_rpc, "_receive", record_receipt) + fault.setattr(runtime, "_sync", fail_ack_sync) + fault.setattr(Path, "unlink", fail_ack_unlink) + with pytest.raises(ExecutionFailure) as unsettled: + domain.destroy() + assert receipt_received.is_set() + assert len(failures) == 1 + assert unsettled.value.error_code == "libvirt_execution_uncertain" + assert unsettled.value.outcome_unknown is True + session.close() + assert len(list(runtime.state_dir.rglob("*.json"))) == 1 + assert_other_vm_blocked(runtime) + + +def test_shutdown_before_received_write_receipt_is_committed_cannot_downgrade_unknown( + receipt_runtime, monkeypatch): + runtime = receipt_runtime() + session = runtime.open("test:///default", "host-a", timeout=5.0) + domain = session.connection.lookupByUUIDString(VM_ID) + receive = libvirt_rpc._receive + + def stop_after_receive(*args): + response = receive(*args) + runtime.begin_shutdown() + return response + + with monkeypatch.context() as fault: + fault.setattr(libvirt_rpc, "_receive", stop_after_receive) + with pytest.raises(ExecutionFailure) as interrupted: + domain.destroy() + assert interrupted.value.error_code == "libvirt_execution_uncertain" + assert interrupted.value.outcome_unknown is True + assert runtime.close(timeout=2.0) is True + assert len(list(runtime.state_dir.rglob("*.json"))) == 1 + assert_other_vm_blocked(receipt_runtime()) + + +@pytest.mark.parametrize("response", [ + {"worker_error": True}, {"result": {"unsupported_value": "not decoded"}}, +]) +def test_malformed_write_response_cannot_be_committed_as_a_receipt( + receipt_runtime, monkeypatch, response): + runtime = receipt_runtime() + session = runtime.open("test:///default", "host-a", timeout=5.0) + domain = session.connection.lookupByUUIDString(VM_ID) + receive = libvirt_rpc._receive + + def corrupt_received_result(*args): + receive(*args) + return response + + with monkeypatch.context() as fault: + fault.setattr(libvirt_rpc, "_receive", corrupt_received_result) + with pytest.raises(ExecutionFailure) as malformed: + domain.destroy() + assert malformed.value.outcome_unknown is True + session.close() + assert len(list(runtime.state_dir.rglob("*.json"))) == 1 + assert_other_vm_blocked(runtime) + + +def test_next_write_waits_for_previous_receipt_to_be_committed( + receipt_runtime, monkeypatch): + runtime = receipt_runtime(rpc_timeout=3.0) + session = runtime.open("test:///default", "host-a", timeout=10.0) + domain = session.connection.lookupByUUIDString(VM_ID) + before_write = runtime._before_write + ack_write = runtime._ack_write + first_receipt = threading.Event() + commit_receipt = threading.Event() + second_started = threading.Event() + prepared = [] + results = [] + errors = [] + + def record_prepare(*args): + prepared.append(args[-1]) + return before_write(*args) + + def pause_first_receipt(current_session, sequence): + if sequence == 1: + first_receipt.set() + assert commit_receipt.wait(timeout=3.0) + return ack_write(current_session, sequence) + + def destroy(second=False): + if second: + second_started.set() + try: + results.append(domain.destroy()) + except BaseException as exc: + errors.append(exc) + + monkeypatch.setattr(runtime, "_before_write", record_prepare) + monkeypatch.setattr(runtime, "_ack_write", pause_first_receipt) + first = threading.Thread(target=destroy, daemon=True) + second = threading.Thread(target=destroy, args=(True,), daemon=True) + try: + first.start() + assert first_receipt.wait(timeout=3.0) + second.start() + assert second_started.wait(timeout=1.0) + time.sleep(0.05) + assert prepared == ["destroy"] + assert session.pending_write == {"sequence": 1, "method": "destroy"} + finally: + commit_receipt.set() + first.join(timeout=3.0) + if second.ident is not None: + second.join(timeout=3.0) + assert not first.is_alive() and not second.is_alive() + assert errors == [] + assert results == [0, 0] + assert prepared == ["destroy", "destroy"] + assert session.last_write_ack == 2 + assert session.pending_write is None + session.close() + assert_other_vm_writable(runtime) diff --git a/engine/tests/test_snapshot_thaw_recovery_rpc.py b/engine/tests/test_snapshot_thaw_recovery_rpc.py index e4dd70cb..9c46e78b 100644 --- a/engine/tests/test_snapshot_thaw_recovery_rpc.py +++ b/engine/tests/test_snapshot_thaw_recovery_rpc.py @@ -260,8 +260,9 @@ def test_failed_boundary_persistence_prevents_backup_dispatch(scenario, monkeypa def failed_sync(path): raise OSError('fsync unavailable') monkeypatch.setattr(item, '_sync', failed_sync) - with pytest.raises(OSError): + with pytest.raises(ExecutionFailure) as failed: domain.backupBegin('', None, 0) + assert failed.value.error_code == 'libvirt_execution_uncertain' assert scenario.state().get('backup_active') is None assert session.failure is not None session.close() diff --git a/engine/virsh_panel/libvirt_rpc.py b/engine/virsh_panel/libvirt_rpc.py index fc65d71f..7c79a8ff 100644 --- a/engine/virsh_panel/libvirt_rpc.py +++ b/engine/virsh_panel/libvirt_rpc.py @@ -49,7 +49,7 @@ READ_METHODS = frozenset({ "getMetadata", "metadata", "backupGetXMLDesc", "refresh", "snapshotLookupByName", "snapshotListNames", "listAllSnapshots", "agentSetResponseTimeout", - "newStream", "download", "recv", "finish", "abort", "close", + "newStream", "download", "recv", "close", }) @@ -321,37 +321,124 @@ class LibvirtRuntime: os.close(fd) def _before_write(self, session, method): - from .coordination import mark_admission_rejected, mark_remote_write_started + from .coordination import mark_remote_write_started with self._condition: - if self._closing: - raise ExecutionFailure("libvirt_execution_stopped") - if session.scope in self._snapshot_recoveries: - mark_admission_rejected() - raise ExecutionFailure("libvirt_execution_blocked", outcome_unknown=True) - directory = self._directory(session.scope) - active = {s.token for s in self._sessions if s.failure is None} - if any(path.stem not in active for path in directory.glob("*.json")): - mark_admission_rejected() - raise ExecutionFailure("libvirt_execution_blocked", outcome_unknown=True) - if session.intent is not None: - mark_remote_write_started() - return - path = directory / (session.token + ".json") - # Persist before sending the first write. Crash/lost receipt never - # becomes permission to replay, including after an engine restart. - data = json.dumps({"version": 1, "scope": session.scope, - "session": session.token, "method": method, - "started_at": int(time.time())}).encode() - fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL - | getattr(os, "O_NOFOLLOW", 0), 0o600) - with os.fdopen(fd, "wb") as stream: - stream.write(data) - stream.flush() - os.fsync(stream.fileno()) - self._sync(directory) - session.intent = path + self._write_admission(session) + if session.pending_write is not None: + raise session._fail("libvirt_execution_uncertain", outcome_unknown=True) + session.write_sequence += 1 + session.pending_write = {"sequence": session.write_sequence, "method": method} + try: + self._persist_write_record(session) + except Exception: + raise session._fail("libvirt_execution_uncertain", outcome_unknown=True) from None mark_remote_write_started() + return session.write_sequence + + def _write_admission(self, session): + from .coordination import mark_admission_rejected + + if session.failure is not None: + raise session._fail(session.failure.error_code, + outcome_unknown=session.failure.outcome_unknown) + if self._closing or session._closed: + raise ExecutionFailure("libvirt_execution_stopped") + active = {s.token for s in self._sessions if s.failure is None} + if (session.scope in self._snapshot_recoveries + or any(path.stem not in active + for path in self._directory(session.scope).glob("*.json"))): + mark_admission_rejected() + raise ExecutionFailure("libvirt_execution_blocked", outcome_unknown=True) + + def _persist_write_record(self, session): + record = {"version": 2, "scope": session.scope, "session": session.token, + "sequence": session.write_sequence, + "pending_write": session.pending_write, + "last_write_ack": session.last_write_ack, + "started_at": int(time.time())} + if session.write_error_receipt is not None: + record["write_error_receipt"] = session.write_error_receipt + if session.snapshot_evidence is not None: + record["snapshot_recovery"] = session.snapshot_evidence + if session.intent is not None: + self._replace_snapshot_record(session.intent, record) + return + session.intent = self._directory(session.scope) / (session.token + ".json") + fd = os.open(session.intent, os.O_WRONLY | os.O_CREAT | os.O_EXCL + | getattr(os, "O_NOFOLLOW", 0), 0o600) + with os.fdopen(fd, "wb") as stream: + stream.write(json.dumps(record).encode()) + stream.flush() + os.fsync(stream.fileno()) + self._sync(session.intent.parent) + + def _clear_intent(self, session): + """Durably acknowledge this receipt; restore the fence on fsync failure.""" + # Caller holds _condition through deletion, sync and restoration. + path, original = session.intent, None + try: + original = path.read_bytes() + path.unlink() + self._sync(path.parent) + except Exception: + try: + if original is not None and not path.exists(): + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL + | getattr(os, "O_NOFOLLOW", 0), 0o600) + with os.fdopen(fd, "wb") as stream: + stream.write(original) + stream.flush() + os.fsync(stream.fileno()) + self._sync(path.parent) + if not path.is_file(): + raise OSError("libvirt execution fence unavailable") + except Exception: + self._closing = True + self._condition.notify_all() + raise session._fail("libvirt_execution_uncertain", outcome_unknown=True) from None + session.intent = None + + def _ack_write(self, session, sequence): + with self._condition: + if (session.failure is not None or self._closing + or session.pending_write is None + or session.pending_write["sequence"] != sequence): + raise session._fail("libvirt_execution_uncertain", outcome_unknown=True) + if session.snapshot_evidence is None and session.write_error_receipt is None: + # Absence is persisted before returning the successful RPC. + # A later call creates a fresh pending record, even in this + # same connection. No old intent is inspected or removed. + self._clear_intent(session) + else: + # Freeze recovery outlives individual RPC acknowledgements. + # Keep its identity/stages independently of the pending call. + try: + record = json.loads(session.intent.read_text()) + record.update(pending_write=None, last_write_ack=sequence) + self._replace_snapshot_record(session.intent, record) + except Exception: + raise session._fail("libvirt_execution_uncertain", outcome_unknown=True) from None + session.last_write_ack = sequence + session.pending_write = None + + def _record_write_error(self, session): + """Keep a returned native error distinct from a success or lost reply. + + Existing callers may inspect/compensate a libvirt rejection in this + same connection. As before, only a clean end to that workflow removes + its obligation; a later success must not overwrite the earlier error. + """ + with self._condition: + try: + record = json.loads(session.intent.read_text()) + error_receipt = session.write_error_receipt or dict(session.pending_write) + record.update(pending_write=None, write_error_receipt=error_receipt) + self._replace_snapshot_record(session.intent, record) + except Exception: + raise session._fail("libvirt_execution_uncertain", outcome_unknown=True) from None + session.write_error_receipt = error_receipt + session.pending_write = None def _save_snapshot_evidence(self, session, evidence): """Persist a conservative phase before sending the corresponding RPC.""" @@ -515,12 +602,18 @@ class RPCSession: self.deadline = time.monotonic() + timeout self.token = uuid.uuid4().hex self.intent = None + self.pending_write = None + self.write_error_receipt = None + self.write_sequence = 0 + self.last_write_ack = 0 self.failure = None + self.cleanup_failure = None + self._cleaning_up = False self.process = None self.sock = None self.connection = None self._closed = False - self._io_lock = threading.Lock() + self._io_lock = threading.RLock() self._pgid = None self._signal_lock = threading.Lock() self._sent_signals = set() @@ -538,7 +631,12 @@ class RPCSession: # stop/error must never downgrade evidence of an unconfirmed write. with self.runtime._condition: previous = self.failure - if self.intent is not None: + if (self._cleaning_up and self.pending_write is None + and self.write_error_receipt is None + and previous is None and not outcome_unknown): + self.cleanup_failure = ExecutionFailure(code) + return self.cleanup_failure + if self.pending_write is not None or self.write_error_receipt is not None: code, outcome_unknown = "libvirt_execution_uncertain", True elif previous is not None and (previous.outcome_unknown or previous.error_code in { "libvirt_execution_uncertain", "libvirt_execution_blocked"}): @@ -599,13 +697,17 @@ class RPCSession: _send(self.sock, request, deadline) response = _receive(self.sock, deadline) except (OSError, EOFError, ValueError): - self._fail("libvirt_execution_timeout") + failure = self._fail("libvirt_execution_timeout") self.interrupt() - raise self.failure from None + raise failure from None if self.failure is not None: raise self._fail(self.failure.error_code, outcome_unknown=self.failure.outcome_unknown) + if not isinstance(response, dict): + raise self._fail("libvirt_execution_timeout") if "libvirt_error" in response: + if self.pending_write is not None: + self.runtime._record_write_error(self) backend = importlib.import_module(self.runtime.backend_module) error = backend.libvirtError(response["libvirt_error"]) if response.get("err") is not None: @@ -614,21 +716,32 @@ class RPCSession: if response.get("snapshot_recovery_required") is True: raise SnapshotRecoveryRequired() if "result" not in response: - self._fail("libvirt_execution_timeout") + failure = self._fail("libvirt_execution_timeout") self.interrupt() - raise self.failure - return _decode(response["result"], lambda value: RemoteObject(self, value)) + raise failure + try: + return _decode(response["result"], lambda value: RemoteObject(self, value)) + except (ValueError, TypeError, KeyError): + raise self._fail("libvirt_execution_timeout") from None def call(self, handle, method, args, kwargs): + # Serialize intent publication, dispatch, decoding and the matching + # acknowledgement. A concurrent caller must never clear our pending + # write or allow a read to consume its reply. + with self._io_lock: + return self._call(handle, method, args, kwargs) + + def _call(self, handle, method, args, kwargs): if self.failure is not None: raise self._fail(self.failure.error_code, outcome_unknown=self.failure.outcome_unknown) request = {"op": "call", "handle": handle, "method": method, "args": _encode(args, self._reference), "kwargs": _encode(kwargs, self._reference)} + sequence = None if method not in READ_METHODS: try: - self.runtime._before_write(self, method) + sequence = self.runtime._before_write(self, method) except ExecutionFailure as exc: raise self._fail( exc.error_code, outcome_unknown=exc.outcome_unknown) from None @@ -661,6 +774,8 @@ class RPCSession: evidence["thaw_ack"] = True evidence["thaw_pending"] = False self.runtime._save_snapshot_evidence(self, evidence) + if sequence is not None: + self.runtime._ack_write(self, sequence) return result def bind_snapshot_recovery(self, domain, *, job_id, volume_name, pool_name, @@ -679,9 +794,16 @@ class RPCSession: "freeze_ack": False, "backup_state": "not_sent", "thaw_ack": False, "thaw_pending": False, "recovery_state": "not_started", } - self.runtime._before_write(self, "fsFreeze") - self.runtime._save_snapshot_evidence(self, evidence) - self._snapshot_handle = domain._handle + with self._io_lock, self.runtime._condition: + self.runtime._write_admission(self) + if self.pending_write is not None: + raise SnapshotRecoveryRequired() + self.snapshot_evidence = evidence + try: + self.runtime._persist_write_record(self) + except Exception: + raise self._fail("libvirt_execution_uncertain", outcome_unknown=True) from None + self._snapshot_handle = domain._handle def recover_snapshot_thaw(self, *, job_id, vm_id, volume_name): path, record, evidence = self.runtime._snapshot_recovery_evidence( @@ -716,6 +838,10 @@ class RPCSession: self.runtime._snapshot_recoveries.discard(self.scope) def qemu_agent_command(self, domain, command, timeout, flags): + with self._io_lock: + return self._qemu_agent_command(domain, command, timeout, flags) + + def _qemu_agent_command(self, domain, command, timeout, flags): if self.failure is not None: raise self._fail(self.failure.error_code, outcome_unknown=self.failure.outcome_unknown) @@ -725,16 +851,20 @@ class RPCSession: command_name = json.loads(command).get("execute") except (TypeError, ValueError, AttributeError): command_name = None + request = {"op": "qga", "args": _encode( + (domain, command, timeout, flags), self._reference)} + sequence = None if command_name not in {"guest-info", "guest-ping", "guest-network-get-interfaces", "guest-exec-status", "guest-fsfreeze-status"}: try: - self.runtime._before_write(self, "qemuAgentCommand") + sequence = self.runtime._before_write(self, "qemuAgentCommand") except ExecutionFailure as exc: raise self._fail( exc.error_code, outcome_unknown=exc.outcome_unknown) from None - return self._request({"op": "qga", "args": _encode( - (domain, command, timeout, flags), self._reference)}, - timeout=min(self.runtime.rpc_timeout, max(1, timeout) + 2)) + result = self._request(request, timeout=min(self.runtime.rpc_timeout, max(1, timeout) + 2)) + if sequence is not None: + self.runtime._ack_write(self, sequence) + return result def interrupt(self): if self.sock is not None: @@ -786,10 +916,17 @@ class RPCSession: process.join(timeout=max(0, deadline - time.monotonic())) def close(self): + with self._io_lock: + self._close() + + def _close(self): if self._closed: return reaped = False + self._cleaning_up = True try: + if self.pending_write is not None: + self._fail("libvirt_execution_uncertain", outcome_unknown=True) if self.connection is not None and self.failure is None: try: self._request({"op": "call", "handle": self.connection._handle, @@ -810,41 +947,23 @@ class RPCSession: self._fail("libvirt_execution_timeout") if not reaped: self._fail("libvirt_execution_timeout") - # Keep deletion, durability verification and any restoration in - # the same admission critical section. No new write may pass - # through an unlink/fsync failure before its fence is restored. with self.runtime._condition: - if self.intent is not None and self.failure is None: - original = None - try: - original = self.intent.read_bytes() - self.intent.unlink() - self.runtime._sync(self.intent.parent) - except Exception: - self._fail("libvirt_execution_timeout") + if self.intent is not None and self.pending_write is None: + evidence = self.snapshot_evidence + if ((evidence is not None and (not evidence["freeze_ack"] or evidence["thaw_ack"]) + and (self.failure is None or not self.failure.outcome_unknown)) + or evidence is None and self.failure is None and self.cleanup_failure is None): try: - if original is not None: - try: - fd = os.open( - self.intent, os.O_WRONLY | os.O_CREAT | os.O_EXCL - | getattr(os, "O_NOFOLLOW", 0), 0o600) - except FileExistsError: - pass - else: - with os.fdopen(fd, "wb") as stream: - stream.write(original) - stream.flush() - os.fsync(stream.fileno()) - self.runtime._sync(self.intent.parent) - if not self.intent.is_file(): - raise OSError("libvirt execution fence unavailable") - except Exception: - # A failed filesystem cannot safely grant any new - # writes. Preserve existing files and stop admission - # until an operator repairs the state directory. - self.runtime._closing = True - self.runtime._condition.notify_all() + self.runtime._clear_intent(self) + self.write_error_receipt = None + except ExecutionFailure: + pass # _clear_intent has restored/fenced the receipt. + elif evidence is not None: + # A successful freeze is an outstanding recovery duty, + # independent of whether its RPC was acknowledged. + self._fail("libvirt_execution_uncertain", outcome_unknown=True) finally: + self._cleaning_up = False if reaped: self.runtime._release(self.scope, self) diff --git a/engine/virsh_panel/libvirt_service.py b/engine/virsh_panel/libvirt_service.py index b50b0af7..ee879028 100644 --- a/engine/virsh_panel/libvirt_service.py +++ b/engine/virsh_panel/libvirt_service.py @@ -916,6 +916,12 @@ class VMService: finally: if session is not None: session.close() + cleanup_failure = getattr(session, "cleanup_failure", None) + if cleanup_failure is not None: + # Closing the connection is housekeeping, not another + # receipt for an already acknowledged libvirt command. + LOG.warning("libvirt connection cleanup failed (pid=%s, code=%s)", + session.pid, cleanup_failure.error_code) # A low-level helper may catch/wrap an exception for rollback. # A poisoned session always wins; no false 404 or success. if session.failure is not None: