From 71fc613b5959831fe88be70fafc8beaaf0ba58ff Mon Sep 17 00:00:00 2001 From: admin Date: Mon, 12 Oct 2026 06:29:44 +0800 Subject: [PATCH] fix(engine): repair connection admission, snapshot thaw and VPC readback --- docs/ENGINE_COORDINATION.md | 4 + docs/LIBVIRT_EXECUTION.md | 19 +- docs/VALIDATION.md | 36 ++ docs/VPC_DATAPLANE_ACCEPTANCE.md | 8 +- .../fixtures/snapshot_recovery_backend.py | 124 ++++++ engine/tests/test_app.py | 5 +- .../test_connection_failure_admission.py | 373 ++++++++++++++++++ engine/tests/test_engine_boundary.py | 9 +- engine/tests/test_host_service_scope.py | 3 +- .../tests/test_snapshot_thaw_recovery_app.py | 73 ++++ .../tests/test_snapshot_thaw_recovery_rpc.py | 268 +++++++++++++ engine/tests/test_vpc_route_readback.py | 164 +++++++- engine/virsh_panel/app.py | 96 ++++- engine/virsh_panel/coordination.py | 50 ++- engine/virsh_panel/durable_jobs.py | 4 +- engine/virsh_panel/internal.py | 49 ++- engine/virsh_panel/libvirt_rpc.py | 255 +++++++++++- engine/virsh_panel/libvirt_service.py | 62 ++- engine/virsh_panel/vpc.py | 12 + engine/virsh_panel/vpc_dataplane.py | 51 ++- 20 files changed, 1596 insertions(+), 69 deletions(-) create mode 100644 engine/tests/fixtures/snapshot_recovery_backend.py create mode 100644 engine/tests/test_connection_failure_admission.py create mode 100644 engine/tests/test_snapshot_thaw_recovery_app.py create mode 100644 engine/tests/test_snapshot_thaw_recovery_rpc.py diff --git a/docs/ENGINE_COORDINATION.md b/docs/ENGINE_COORDINATION.md index c50c486e..93430e5a 100644 --- a/docs/ENGINE_COORDINATION.md +++ b/docs/ENGINE_COORDINATION.md @@ -28,6 +28,10 @@ 旧版本异常留下的未决协调记录不会因本次修复或重启自动解除。若升级或后续请求被其阻塞,必须按下文人工恢复流程逐条核验;修复连接构造本身不证明历史远端操作已经结束。 +连接建立失败单独返回 `libvirt_connection_failed`。只有明确通过 libvirt 执行远端变更、且本次执行尚未发出任何写入 RPC 的调用,才释放本次请求与资源占用,允许连接恢复后重新提交。写入证据在发送 RPC 前记录,并由同一执行的嵌套资源占用、线程池任务和后续连接共同继承;第一条连接写过后,第二条连接失败不能被当成执行前失败。VPC/Fabric 等会运行 SSH 修改脚本的工作流不使用“尚无 libvirt 写入”作为无副作用证明。该判定只用于正在执行的请求,不能据此清除历史 uncertain 记录。 + +宿主指纹扫描、暂存配置准备和 libvirt 校验前的只读 SSH 能力探测若失败,须先持久化本次请求的 `failed_before_start` 凭据,再释放占用;全局扫描入口和指定宿主入口遵循同一规则。凭据写入失败或结果不明仍保留保护;该例外不适用于暂存配置提交后的失败,也不适用于任意 SSH 修改命令。 + ## 数据库协议 `coordination.py` 使用现有加密 `engine_records` 表,`record_type=coordination`、`record_key=admission-v1`。版本 1 文档保存 revision、claims 和最多 100 条人工核验收据。通过现有 `DatabaseStore.compare_and_set` 的行锁事务完成短准入更新;远程调用期间不持有数据库事务。 diff --git a/docs/LIBVIRT_EXECUTION.md b/docs/LIBVIRT_EXECUTION.md index 89caa2ab..a069482d 100644 --- a/docs/LIBVIRT_EXECUTION.md +++ b/docs/LIBVIRT_EXECUTION.md @@ -28,7 +28,7 @@ ## 写入结果未知与恢复 -**终止本地客户端不证明远端 libvirtd/QEMU 已停止写入。** 发出第一条变更 RPC 前,引擎在 `/libvirt-execution//.json` 写入并 fsync 一个意图文件。文件仅记录 scope、随机 session ID、首个方法及时间,不含 URI、凭据、XML 或 RPC 参数。 +**终止本地客户端不证明远端 libvirtd/QEMU 已停止写入。** 发出第一条变更 RPC 前,引擎在 `/libvirt-execution//.json` 写入并 fsync 一个意图文件。普通意图记录 scope、随机 session ID、首个方法及时间。文件系统一致性快照还记录下文限定恢复所需的任务、运行代、目标卷身份和阶段凭据;不保存连接 URI、凭据、完整 XML 或任意 RPC 参数。 正常完成并安全关闭会话才删除意图。中断、异常退出、清理失败或关机时已有写入的会话保留意图;引擎重启仍能发现它。该宿主机后续 libvirt 写入会被拒绝,只读观察仍可执行。现有已发往远端的并发操作不能被宣称已撤销。 @@ -38,6 +38,7 @@ | `libvirt_execution_stopped` | 尚未记录写入时引擎停止接单 | | `libvirt_execution_uncertain` | 会话已记录写入,结果未知 | | `libvirt_execution_blocked` | 同宿主存在未解决的写入意图,本次写入被阻止 | +| `libvirt_connection_failed` | 原生 libvirt 连接未建立;只有本次执行尚无远端写入证据时才允许释放新占用 | 这组错误不会转换为 `libvirtError`。后两项跨重启保持 `uncertain`、`safe_to_refund=false`,不自动重放、释放 IP/归属或触发清理退款。快照、重装和救援任务保留原记录、必要凭据与任务占用;重装超时不会被当成新系统已提交。 @@ -50,6 +51,22 @@ 意图目录是持久安全状态,应与引擎数据一起备份,不属于可随意清理的临时缓存。它针对当前单引擎进程,不是多 worker 的分布式锁;不要据此增加 `--workers`。 +## 快照冻结后的限定解冻 + +普通写入保护不能阻止原快照任务对已冻结来宾的必要解冻。新的文件系统一致性快照在冻结前持久绑定宿主 scope、任务 ID、VM UUID、确定性快照卷名和路径、QEMU 运行 ID 及只存在于实时 XML 的 QGA 运行代令牌。冻结响应和备份启动阶段分别落盘;阶段证据无法持久化时停止后续操作。 + +原会话失败并回收本地 worker 后,快照任务立即尝试受限恢复;服务启动核对和原任务核对也使用同一入口。只有原冻结已确认,且备份尚未发送、启动响应已确认,或启动响应未知但能读到本任务精确目标的活动备份时,才允许新连接执行一次限定的 `fsThaw`。恢复再次检查当前 VM 运行代和备份身份;其他活跃写入者、多个未决意图、缺少证据、目标不符或无法确认旧 worker 已退出都会拒绝自动解冻。旧 worker 消失本身不构成远端阶段证明。 + +该入口不能发送任意写操作;普通 `fsThaw` 仍受原写入保护约束。解冻发送前、确认后分别持久化凭据;已确认的恢复只读取原回执,不再次发送解冻,响应未知的恢复转人工核对。解冻确认只更新来宾冻结恢复状态,**不删除原意图、不解除 uncertain 占用、不把备份标成成功,也不允许重放原任务**。`fsThaw` 返回 0 表示当前没有文件系统需要解冻,可以确认恢复,但不能证明快照已完成。 + +升级前记录缺少上述身份和阶段证据时,保留 `manual_required` 和 `snapshot_thaw_recovery_required` 提示,不猜测旧任务。此时应立即按以下步骤处理冻结来宾: + +1. 保存原任务、占用和意图,停止相关新任务。在可信宿主连接上核对原 VM UUID、运行代、活动 libvirt 备份及其目标路径;确认没有仍可能发送冻结请求的旧执行器。不能只按 VM 名称、当前 PID 或“没有活动 job”判断安全。 +2. 管理员确认原任务与当前来宾对应、冻结阶段已稳定后,在该宿主机执行 `virsh -c qemu:///system domfsthaw <原任务的完整-VM-UUID>`。这仅执行解冻,不自动 abort 备份、删除卷或解除引擎保护。随后核对 QGA 冻结状态与来宾业务恢复;QGA 状态不能单独替代任务和运行代核验。 +3. 继续按上一节核验备份结果、卷和持久任务;确认结果后才逐条归档意图并处理协调占用。无法证实阶段或运行代时,保留保护并人工排查,不自动重试冻结/备份。 + +依据:[libvirt backupBegin](https://libvirt.org/html/libvirt-libvirt-domain.html#virDomainBackupBegin)、[libvirt fsThaw](https://libvirt.org/html/libvirt-libvirt-domain.html#virDomainFSThaw) 和 [QEMU guest agent 冻结接口](https://www.qemu.org/docs/master/interop/qemu-ga-ref.html#command-guest-fsfreeze-status)。 + ## 关闭流程 引擎收到 lifespan shutdown 后停止新请求和任务接纳,立即中断 libvirt 执行器并启动回收。后台池、指标采集、Fabric/VPC 维护、共享导入 runtime、各宿主生命周期与默认 asyncio 执行器共享 **20 秒** deadline(内部 `shutdown_timeout_seconds` 可缩短用于测试)。不会为每个宿主机或每个池重新计时。 diff --git a/docs/VALIDATION.md b/docs/VALIDATION.md index 5295bdd7..7ebb46c4 100644 --- a/docs/VALIDATION.md +++ b/docs/VALIDATION.md @@ -782,6 +782,42 @@ HAVING live_subnets <> 1 OR deleting_subnets <> 0; 部署或重启服务;真实新增宿主流程仍需更新引擎后验收。旧未决协调记录保持原样, 不因修复自动清空或重放。 +## 2026-10-12:连接失败占用、快照解冻和停机 VPC 路由回归 + +- 连接建立失败现在使用 `libvirt_connection_failed`,明确尚无远端写入时释放本次 + 请求和资源占用;恢复连接后可重新保存宿主或提交新的 VM 操作。同执行的嵌套 + 资源和线程池任务共享写入证据,不能用后一次连接失败掩盖前一次写入。VPC/Fabric + 混合 SSH 流程不放宽。宿主指纹扫描、暂存准备和只读 SSH 探测失败还须持久化未开始 + 凭据,全局/指定宿主入口一致;凭据保存失败则保留保护。旧 uncertain 占用不会自动解除。 +- 新文件系统快照在冻结前绑定原任务、VM、实时运行代令牌和备份目标,并持久记录 + 冻结/启动/解冻边界。原 RPC 超时回收后立即尝试受限解冻;启动核对复用该入口。 + 解冻前的身份/备份核对与解冻后的异常分开,避免将已发送但结果未知的解冻当成 + 未执行而重放。解冻确认后原 libvirt 意图、协调占用和备份 uncertain 状态保持; + 旧记录或证据不足时显示立即人工核对提示。恢复步骤见 + [快照冻结后的限定解冻](LIBVIRT_EXECUTION.md#快照冻结后的限定解冻)。 +- 受管私网 bridge 的单个 `linkdown` 保留在路由回读中,可证明停机时配置存在, + 但不能证明 host 或对应成员 active/usable。WireGuard、guest 路由、未知/重复 + flags 继续拒绝。覆盖停机首次加入、开机、停止最后成员、停机修改重试和再次启动。 + 原始路由生成脚本的两个新增关键场景在修复前 `b82f3b8` 副本失败,修复后通过。 +- 新增签名请求/真实 VMService 协调回归 24 项;宿主身份相关 25 项继续通过。 + VPC 相关 7 个测试文件 252 项通过。快照恢复通过真实 spawn 子进程和持久模拟远端 + 覆盖冻结后超时、精确备份目标、错误任务/VM/运行代、旧 worker 存活、阶段落盘失败、 + 响应未知不可重放、解冻后异常及普通写仍被阻挡;另通过 HTTP 后台入口验证无轮询 + 即发起恢复,以及 `safe_to_refund=false` 和原占用仍保留。快照恢复专项组合 + 82 项通过,其中包含 24 项新增 RPC 恢复和 2 项新增 HTTP 后台入口测试。 +- 最终全量引擎回归:**2471 passed、19 skipped、10 failed、80 subtests passed**, + 本轮共增加 66 个测试场景。10 项失败与修复前 `cda57ce` 隔离基线逐项一致: + `test_app.py` 两项既有契约期望、Fabric helper 五项、`test_registration_http.py` + 三项公网地址池期望;没有新增失败,未修改这些业务逻辑或放宽断言。快照测试替身 + 补齐新增恢复任务参数,原运行中快照阻止断网的断言继续通过。 +- Node `CloudVpcUserPageBehaviorTest` 通过;隔离 PHP/WASM 的 + `CloudVpcUserPageContractTest`、`CloudVpcFailedReconcileTest` 和 + `CloudVpcMutationAdmissionTest` 通过。Python 修改文件 AST 与 `git diff --check` + 通过。此轮没有修改宿主机 libvirt 启动项、启动依赖或执行生产部署。 +- 使用 Python 3.12.14 和 pytest 9.1.1;没有连接真实 SSH/libvirt/QGA 宿主, + 未执行真实 Linux bridge/WireGuard 或 MariaDB 验收。部署后仍需在专用测试宿主验收 + 丢失 libvirt 响应、来宾解冻、停机 VPC 修改与再次启动;隔离测试不能代替这些检查。 + ## 在具备依赖的环境复测 不要使用生产数据库/宿主机运行额外真实集成测试;以下默认单元/隔离测试: diff --git a/docs/VPC_DATAPLANE_ACCEPTANCE.md b/docs/VPC_DATAPLANE_ACCEPTANCE.md index 3dd58154..0cad8d07 100644 --- a/docs/VPC_DATAPLANE_ACCEPTANCE.md +++ b/docs/VPC_DATAPLANE_ACCEPTANCE.md @@ -115,7 +115,13 @@ ACL-first helper 承担;带私网 VM 在 bridge 尚未恢复时的启动失败 路由验收同时使用裸 IPv4 与 `/32` 两种 host-route 输出,以及按 `dev` 过滤后省略 设备字段的输出;只有带设备过滤的查询可补回该设备,无过滤列表不得猜测设备。 -错误设备、重复路由、特殊路由类型、宽前缀、多路径和 `linkdown` 都不能通过回读。 +错误设备、重复路由、特殊路由类型、宽前缀、多路径和未知/重复 flags 都不能通过回读。 +停止最后一个成员后,明确受管的私网 bridge 可以因没有运行中的 tap 而出现单个 +`linkdown`;路由身份仍须完全匹配,回读保留 `routes[].flags=["linkdown"]`,只证明 +配置存在,不证明网络可用。WireGuard 路由及 guest 路由仍不接受 `linkdown`。 +验收需覆盖停机首次加入、启动、停止最后成员、停机重试修改和再次启动;停止时允许 +配置核验通过,但 host 和对应成员不能借陈旧 QGA 成功状态变成 active/usable,启动后 +必须同时取得无链路异常的路由和新的 guest 证据。 实际执行生成的 observe 路由片段,不能仅用 `sh -n` 检查嵌入的 AWK。 宿主路由归属查询使用 `ip -N` 的数字协议、scope 和 table,避免协议 `186` 被输出为 `bgp` 或本地自定义名称而误拒自己的路由;QGA 固定 `dev` 查询也须覆盖 JSON 省略设备 diff --git a/engine/tests/fixtures/snapshot_recovery_backend.py b/engine/tests/fixtures/snapshot_recovery_backend.py new file mode 100644 index 00000000..d8fdaba6 --- /dev/null +++ b/engine/tests/fixtures/snapshot_recovery_backend.py @@ -0,0 +1,124 @@ +"""Persistent fake remote state for the real emergency-thaw RPC boundary.""" +import json +import time +from pathlib import Path + +VIR_DOMAIN_XML_INACTIVE = 2 +VIR_DOMAIN_JOB_NONE = 0 +VIR_DOMAIN_JOB_UNBOUNDED = 2 +VIR_DOMAIN_JOB_OPERATION_BACKUP = 12 + + +class libvirtError(Exception): + pass + + +class virDomain: + def __init__(self, connection): + self.connection = connection + + def UUIDString(self): + return self.connection.read()['vm_id'] + + def ID(self): + return self.connection.read().get('runtime_id', 7) + + def isActive(self): + return self.connection.read().get('running', True) + + def XMLDesc(self, flags=0): + state = self.connection.read() + token = '' if flags else ( + '') + return f'{token}' + + def fsFreeze(self, mounts, flags): + self.connection.update(frozen=True) + if self.connection.read().get('freeze_hangs'): + time.sleep(10) + return 1 + + def backupBegin(self, xml, checkpoint, flags): + state = self.connection.read() + if not state.get('backup_missing'): + self.connection.update(backup_active=True) + if state.get('backup_hangs'): + time.sleep(10) + return 0 + + def fsThaw(self, mounts, flags): + state = self.connection.read() + self.connection.update(thaw_calls=state.get('thaw_calls', 0) + 1) + if state.get('thaw_hangs'): + time.sleep(10) + self.connection.update(frozen=False) + if state.get('thaw_invalid'): + return -1 + if state.get('thaw_type_error'): + raise TypeError('error after mutation') + return 1 + + def jobStats(self, flags): + state = self.connection.read() + return {'type': 2 if state.get('backup_active') else 0, + 'operation': state.get('job_operation', 12)} + + def backupGetXMLDesc(self, flags=0): + state = self.connection.read() + target = state.get('backup_path', state['target_path']) + return ('' + f'' + '') + + def destroy(self): + self.connection.update(destroyed=True) + return 0 + + +class virStorageVol: + def __init__(self, connection): + self.connection = connection + + def path(self): + return self.connection.read()['target_path'] + + +class virStoragePool: + def __init__(self, connection): + self.connection = connection + + def storageVolLookupByName(self, name): + if name != self.connection.read()['volume_name']: + raise libvirtError('wrong volume') + return virStorageVol(self.connection) + + +class virConnect: + def __init__(self, uri): + self.path = Path(uri) + + def read(self): + return json.loads(self.path.read_text()) + + def update(self, **values): + state = self.read() + state.update(values) + self.path.write_text(json.dumps(state)) + + def lookupByUUIDString(self, vm_id): + if vm_id != self.read()['vm_id']: + raise libvirtError('wrong domain') + return virDomain(self) + + def storagePoolLookupByName(self, name): + if name != 'images': + raise libvirtError('wrong pool') + return virStoragePool(self) + + def close(self): + return 0 + + +def open(uri): + return virConnect(uri) diff --git a/engine/tests/test_app.py b/engine/tests/test_app.py index 80cdde86..1243fd40 100644 --- a/engine/tests/test_app.py +++ b/engine/tests/test_app.py @@ -506,7 +506,8 @@ class FakeVMService: raise self.delete_vm_error return {"uuid": vm_id, "deleted": True, "deleted_disks": []} - def create_live_snapshot(self, vm_id, snapshot_key, *, consistency, progress=None): + def create_live_snapshot(self, vm_id, snapshot_key, *, consistency, progress=None, + recovery_job_id=None): self.calls.append(("create_live_snapshot", vm_id, snapshot_key, consistency)) if self.snapshot_create_error is not None: raise self.snapshot_create_error @@ -523,7 +524,7 @@ class FakeVMService: "volume_identity": "a" * 64, } - def thaw_guest_filesystems(self, vm_id): + def thaw_guest_filesystems(self, vm_id, *, recovery_job_id=None, snapshot_key=None): self.calls.append(("thaw_guest_filesystems", vm_id)) if self.snapshot_thaw_error is not None: raise self.snapshot_thaw_error diff --git a/engine/tests/test_connection_failure_admission.py b/engine/tests/test_connection_failure_admission.py new file mode 100644 index 00000000..7a745b8a --- /dev/null +++ b/engine/tests/test_connection_failure_admission.py @@ -0,0 +1,373 @@ +"""Connection admission failures must not become permanent mutation fences. + +Exercise the signed engine and real VMService/coordination boundaries. Only +SSH staging, remote objects and the RPC connection transport are simulated. +""" + +import base64 +import json +from concurrent.futures import ThreadPoolExecutor +from types import SimpleNamespace + +import pytest +from fastapi.testclient import TestClient + +from test_app import FakeRemoteManager, HOST_BODY, VM_ID +from test_engine_boundary import MemoryStore, SECRET, SUPER, call +from test_host_service_scope import StagedManager +from virsh_panel.app import create_app +from virsh_panel.config import Settings +from virsh_panel.coordination import ( + CoordinationError, DurableCoordinator, connection_failure_before_write, + current_execution, execution_context, mark_remote_write_started, + resource_guard, update_current_claim, +) +from virsh_panel.database import DatabaseTemplateStore +from virsh_panel.durable_jobs import submit_durable +from virsh_panel.engine_context import EngineComponents, ExternalOwnership, HostStore +from virsh_panel.internal import EngineApplication, HostRegistry, json_response +from virsh_panel.libvirt_rpc import LibvirtRuntime +from virsh_panel.libvirt_service import GIB, VMError, VMService +from virsh_panel.metrics import MetricsCollector +from virsh_panel.remote_host import RemoteHostError +import virsh_panel.libvirt_service as service_module + + +@pytest.fixture +def signed_host(tmp_path, monkeypatch): + opened, managers = [], {} + behavior = SimpleNamespace(fail_open=True, fail_after_write=False) + + def open_session(uri, scope, *, timeout): + opened.append(scope) + if behavior.fail_open: + raise service_module.libvirt.libvirtError("connection refused before open") + connection = SimpleNamespace(networkLookupByName=lambda _name: + SimpleNamespace(isActive=lambda: True)) + return SimpleNamespace(connection=connection, failure=None, close=lambda: None) + + runtime = SimpleNamespace(open=open_session, queue_timeout=0.02) + monkeypatch.setattr("virsh_panel.app.get_runtime", lambda: runtime) + monkeypatch.setattr(service_module, "get_runtime", lambda: runtime) + monkeypatch.setattr(MetricsCollector, "start", lambda _self: None) + + def pool(service, conn, name, *_args): + if behavior.fail_after_write: + mark_remote_write_started() + # A completed first session's write is still part of this request. + behavior.fail_open = True + with service._connect(): + pytest.fail("second connection should fail") + return SimpleNamespace(name=lambda: name, listVolumes=lambda: [], + info=lambda: (0, 100 * GIB, 0, 100 * GIB)) + + monkeypatch.setattr(VMService, "_active_pool", pool) + monkeypatch.setattr(VMService, "_ensure_pool", pool) + monkeypatch.setattr(VMService, "_refresh_directory_pool", lambda *_args: None) + monkeypatch.setattr(VMService, "_same_pool_location", lambda *_args: False) + store, logs = MemoryStore(), MemoryStore() + config = {"service_secret": base64.b64encode(SECRET).decode(), + "data_dir": str(tmp_path)} + + def factory(host_id, user_isos, cloud_templates): + directory = tmp_path / host_id + directory.mkdir(mode=0o700, exist_ok=True) + scoped = HostStore(store, host_id) + manager = managers[host_id] = StagedManager() + manager.host_id = host_id + components = EngineComponents( + SimpleNamespace(users=ExternalOwnership(scoped)), manager, + DatabaseTemplateStore(scoped), scoped, logs, user_isos, cloud_templates) + return components, create_app(Settings(data_dir=directory), + engine_components=components) + + registry = HostRegistry(config, store, logs, factory=factory) + engine = EngineApplication(config, store, logs, registry=registry) + with TestClient(engine, base_url="http://127.0.0.1") as client: + yield SimpleNamespace(client=client, engine=engine, behavior=behavior, + opened=opened, managers=managers) + + +def test_signed_host_save_connection_failure_releases_claims_and_allows_retry(signed_host): + host = signed_host + path = "/internal/v1/hosts/7/host" + response = call(host.client, "PUT", path, SUPER, HOST_BODY) + assert response.status_code == 502, response.text + assert response.json()["error_code"] == "libvirt_connection_failed" + assert host.managers["7"].current() is None + assert [row[0] for row in host.managers["7"].calls] == ["stage", "test", "abort"] + assert host.engine.coordinator.snapshot() == {} + # No orphan request/resource claim may block maintenance either. + maintenance = host.engine.coordinator.claim(maintenance=True, kind="update") + assert host.engine.coordinator.release(maintenance) + + host.behavior.fail_open = False + response = call(host.client, "PUT", path, SUPER, HOST_BODY) + assert response.status_code == 200, response.text + assert host.managers["7"].current()["host_id"] == "7" + assert host.opened == ["7", "7", "7"] + assert host.engine.coordinator.snapshot() == {} + + +@pytest.mark.parametrize("receipt_lost", [False, True]) +def test_signed_ssh_probe_failure_requires_durable_prestart_proof( + signed_host, monkeypatch, receipt_lost): + host = signed_host + path = "/internal/v1/hosts/7/host" + assert call(host.client, "GET", path, SUPER).status_code == 200 + manager = host.managers["7"] + original_probe = manager.test + + def refused(_staged): + raise RemoteHostError("SSH probe connection refused", 502) + + monkeypatch.setattr(manager, "test", refused) + if receipt_lost: + def cannot_record(*_args, **_kwargs): + raise CoordinationError("coordination_unavailable", 503) + monkeypatch.setattr(host.engine.coordinator, "update_metadata", cannot_record) + response = call(host.client, "PUT", path, SUPER, HOST_BODY) + assert response.status_code == (503 if receipt_lost else 502), response.text + assert host.opened == [] + assert manager.current() is None + if receipt_lost: + assert [row["state"] for row in host.engine.coordinator.snapshot().values()] == ["uncertain"] + with pytest.raises(CoordinationError): + host.engine.coordinator.claim(maintenance=True, kind="update") + return + assert host.engine.coordinator.snapshot() == {} + maintenance = host.engine.coordinator.claim(maintenance=True, kind="update") + assert host.engine.coordinator.release(maintenance) + monkeypatch.setattr(manager, "test", original_probe) + host.behavior.fail_open = False + assert call(host.client, "PUT", path, SUPER, HOST_BODY).status_code == 200 + assert host.engine.coordinator.snapshot() == {} + + +@pytest.mark.parametrize("method,suffix", [("PUT", "host"), ("POST", "host/test")]) +@pytest.mark.parametrize("receipt_lost", [False, True]) +def test_signed_stage_scan_failure_requires_durable_prestart_proof( + signed_host, monkeypatch, method, suffix, receipt_lost): + host = signed_host + assert call(host.client, "GET", "/internal/v1/hosts/7/host", SUPER).status_code == 200 + manager = host.managers["7"] + original_stage = manager.stage + + def refused(*_args, **_kwargs): + raise RemoteHostError("keyscan failed during staging", 502) + + monkeypatch.setattr(manager, "stage", refused) + if receipt_lost: + def cannot_record(*_args, **_kwargs): + raise CoordinationError("coordination_unavailable", 503) + monkeypatch.setattr(host.engine.coordinator, "update_metadata", cannot_record) + path = f"/internal/v1/hosts/7/{suffix}" + response = call(host.client, method, path, SUPER, HOST_BODY) + assert response.status_code == (503 if receipt_lost else 502), response.text + assert host.opened == [] + assert manager.current() is None + if receipt_lost: + assert [row["state"] for row in host.engine.coordinator.snapshot().values()] == ["uncertain"] + with pytest.raises(CoordinationError): + host.engine.coordinator.claim(maintenance=True, kind="update") + return + assert host.engine.coordinator.snapshot() == {} + maintenance = host.engine.coordinator.claim(maintenance=True, kind="update") + assert host.engine.coordinator.release(maintenance) + monkeypatch.setattr(manager, "stage", original_stage) + host.behavior.fail_open = False + assert call(host.client, method, path, SUPER, HOST_BODY).status_code == 200 + assert host.engine.coordinator.snapshot() == {} + + +@pytest.mark.parametrize("global_route", [False, True], ids=["host-scan", "discovery-scan"]) +@pytest.mark.parametrize("receipt_lost", [False, True]) +def test_signed_keyscan_failure_requires_durable_prestart_proof( + signed_host, monkeypatch, global_route, receipt_lost): + host = signed_host + if global_route: + manager = FakeRemoteManager() + manager.close = lambda: None + monkeypatch.setattr("virsh_panel.remote_host.RemoteHostManager", lambda *_args: manager) + path = "/internal/v1/hosts/scan" + else: + assert call(host.client, "GET", "/internal/v1/hosts/7/host", SUPER).status_code == 200 + manager = host.managers["7"] + path = "/internal/v1/hosts/7/host/scan" + original_scan = manager.scan + + def refused(*_args, **_kwargs): + raise RemoteHostError("keyscan timed out", 502) + + monkeypatch.setattr(manager, "scan", refused) + if receipt_lost: + def cannot_record(*_args, **_kwargs): + raise CoordinationError("coordination_unavailable", 503) + monkeypatch.setattr(host.engine.coordinator, "update_metadata", cannot_record) + body = {"host": HOST_BODY["host"], "port": HOST_BODY["port"]} + response = call(host.client, "POST", path, SUPER, body) + # The global discovery route preserves its generic remote-error 503. + assert response.status_code == (503 if receipt_lost or global_route else 502), response.text + assert host.opened == [] + if receipt_lost: + assert [row["state"] for row in host.engine.coordinator.snapshot().values()] == ["uncertain"] + with pytest.raises(CoordinationError): + host.engine.coordinator.claim(maintenance=True, kind="update") + return + assert host.engine.coordinator.snapshot() == {} + maintenance = host.engine.coordinator.claim(maintenance=True, kind="update") + assert host.engine.coordinator.release(maintenance) + monkeypatch.setattr(manager, "scan", original_scan) + assert call(host.client, "POST", path, SUPER, body).status_code == 200 + assert host.engine.coordinator.snapshot() == {} + + +def test_signed_host_save_later_connection_failure_retains_prior_writes(signed_host): + host = signed_host + host.behavior.fail_open = False + host.behavior.fail_after_write = True + response = call(host.client, "PUT", "/internal/v1/hosts/7/host", SUPER, HOST_BODY) + assert response.status_code == 502, response.text + claims = host.engine.coordinator.snapshot().values() + assert {row["kind"] for row in claims if row["state"] == "uncertain"} == { + "request", "resource"} + assert any("storage:7" in row["resources"] for row in claims) + with pytest.raises(CoordinationError): + host.engine.coordinator.claim(maintenance=True, kind="update") + + +def test_signed_vm_connection_failure_does_not_keep_the_vm_locked(signed_host): + host = signed_host + host.behavior.fail_open = False + assert call(host.client, "PUT", "/internal/v1/hosts/7/host", SUPER, + HOST_BODY).status_code == 200 + host.behavior.fail_open = True + response = call(host.client, "POST", f"/internal/v1/hosts/7/vms/{VM_ID}/actions", + {**SUPER, "operation_id": "start-connection-one"}, {"action": "start"}) + assert response.status_code == 502, response.text + assert response.json()["status"] == "failed" + assert host.engine.coordinator.snapshot() == {} + next_operation = host.engine.coordinator.claim(resources=(f"vm:7:{VM_ID}",)) + assert host.engine.coordinator.release(next_operation) + + +def test_mixed_ssh_request_does_not_use_missing_rpc_marker_as_proof(signed_host, monkeypatch): + host = signed_host + calls = [] + + async def mixed_handler(scope, receive, send, body, delegated, **_kwargs): + calls.append("SSH helper changed host") + assert not current_execution().connection_failure_safe + await json_response(send, 502, {"error_code": "libvirt_connection_failed"}) + + monkeypatch.setattr(host.engine, "_dispatch_regular", mixed_handler) + response = call(host.client, "POST", "/internal/v1/vpcs/apply", SUPER, {}) + assert response.status_code == 502 + assert calls == ["SSH helper changed host"] + assert [row["state"] for row in host.engine.coordinator.snapshot().values()] == ["uncertain"] + + +def test_service_guard_cannot_relax_a_mixed_workflows_unknown_side_effects(monkeypatch): + coordinator = DurableCoordinator(MemoryStore()) + parent = coordinator.claim() + + def refused(*_args, **_kwargs): + raise service_module.libvirt.libvirtError("connection refused") + + monkeypatch.setattr(service_module, "get_runtime", lambda: + SimpleNamespace(open=refused, queue_timeout=0.02)) + service = VMService("test", "images", "iso", "default", execution_scope="7") + with execution_context(coordinator, parent): + with pytest.raises(VMError): + service.ensure_storage_pools() + assert any(row["state"] == "uncertain" and row["resources"] == ["storage:7"] + for row in coordinator.snapshot().values()) + + +@pytest.mark.parametrize("code", ["libvirt_connection_failed", "libvirt_execution_timeout", + "libvirt_execution_stopped", "libvirt_execution_uncertain"]) +def test_nested_connection_failures_never_release_earlier_session_writes(code): + coordinator = DurableCoordinator(MemoryStore()) + parent = coordinator.claim(resources=(f"vm:7:{VM_ID}",)) + with execution_context(coordinator, parent, connection_failure_safe=True): + with resource_guard(("storage:7",)): + mark_remote_write_started() + updated = coordinator.add_resources(parent, ("allocation:7",)) + update_current_claim(updated) + assert not connection_failure_before_write("libvirt_connection_failed") + with pytest.raises(VMError): + with resource_guard(("storage:7",), connection_failure_safe=True): + raise VMError("later session failed", 502, error_code=code) + assert any(row["kind"] == "resource" and row["state"] == "uncertain" + for row in coordinator.snapshot().values()) + + +def test_background_write_evidence_returns_to_parent_thread_context(): + coordinator = DurableCoordinator(MemoryStore()) + parent = coordinator.claim(resources=(f"vm:7:{VM_ID}",)) + with execution_context(coordinator, parent, connection_failure_safe=True): + with ThreadPoolExecutor(max_workers=1) as executor: + future = submit_durable(executor, mark_remote_write_started, + outcome=lambda: {"status": "completed"}, + job_kind="snapshot_job", job_id="one") + future.result(timeout=2) + assert not connection_failure_before_write("libvirt_connection_failed") + assert current_execution().remote_write_started.is_set() + + +@pytest.mark.parametrize("earlier_write", [False, True]) +def test_background_terminal_connection_failure_preserves_actual_execution_evidence( + monkeypatch, earlier_write): + store = MemoryStore() + coordinator = DurableCoordinator(store) + parent = coordinator.claim(resources=(f"vm:7:{VM_ID}",)) + + def refused(*_args, **_kwargs): + raise service_module.libvirt.libvirtError("connection refused") + + monkeypatch.setattr(service_module, "get_runtime", lambda: + SimpleNamespace(open=refused, queue_timeout=0.02)) + service = VMService("test", "images", "iso", "default", execution_scope="7") + + def job(): + if earlier_write: + mark_remote_write_started() + try: + service.ensure_storage_pools() + except VMError as exc: + store.set("reinstall_job", "one", json.dumps({ + "status": "failed", "error_code": exc.error_code})) + + with execution_context(coordinator, parent, connection_failure_safe=True): + with ThreadPoolExecutor(max_workers=1) as executor: + submit_durable(executor, job, + outcome=lambda: json.loads(store.get("reinstall_job", "one")), + job_kind="reinstall_job", job_id="one").result(timeout=2) + coordinator.release(parent) + claims = list(coordinator.snapshot().values()) + if earlier_write: + assert {row["kind"] for row in claims if row["state"] == "uncertain"} == { + "background", "resource"} + else: + assert claims == [] + + +def test_new_execution_does_not_inherit_another_requests_write_evidence(): + coordinator = DurableCoordinator(MemoryStore()) + first, second = coordinator.claim(), coordinator.claim() + with execution_context(coordinator, first, connection_failure_safe=True): + mark_remote_write_started() + with execution_context(coordinator, second, connection_failure_safe=True): + assert connection_failure_before_write("libvirt_connection_failed") + assert not connection_failure_before_write("libvirt_connection_failed") + + +def test_rpc_intent_hook_marks_parent_before_a_write(tmp_path): + coordinator = DurableCoordinator(MemoryStore()) + parent = coordinator.claim() + runtime = LibvirtRuntime(tmp_path / "intents") + session = SimpleNamespace(scope="7", token="a" * 32, failure=None, intent=None) + with execution_context(coordinator, parent, connection_failure_safe=True): + runtime._before_write(session, "create") + assert session.intent.is_file() + assert current_execution().remote_write_started.is_set() diff --git a/engine/tests/test_engine_boundary.py b/engine/tests/test_engine_boundary.py index 65f87faa..f27cd66f 100644 --- a/engine/tests/test_engine_boundary.py +++ b/engine/tests/test_engine_boundary.py @@ -3981,7 +3981,7 @@ def test_network_suspend_cancels_same_vm_snapshot_still_queued_behind_host_job( blocker_release = threading.Event() def create_live_snapshot(vm_id, snapshot_key, *, consistency, - progress=None): + progress=None, recovery_job_id=None): if vm_id == VM_ID: blocker_started.set() assert blocker_release.wait(timeout=5) @@ -4092,12 +4092,13 @@ def test_network_suspend_rejects_snapshot_future_already_running(boundary): started = threading.Event() release = threading.Event() - def running_snapshot(vm_id, snapshot_key, *, consistency, progress=None): + def running_snapshot(vm_id, snapshot_key, *, consistency, progress=None, + recovery_job_id=None): started.set() assert release.wait(timeout=5) return original_create( vm_id, snapshot_key, consistency=consistency, - progress=progress) + progress=progress, recovery_job_id=recovery_job_id) service.create_live_snapshot = running_snapshot snapshot_actor = {**USER, "operation_id": "running-snapshot-guard"} @@ -4156,7 +4157,7 @@ def test_cancelled_snapshot_phase_one_save_failure_converges_on_poll(boundary): blocker_release = threading.Event() def create_live_snapshot(vm_id, snapshot_key, *, consistency, - progress=None): + progress=None, recovery_job_id=None): if vm_id == VM_ID: blocker_started.set() assert blocker_release.wait(timeout=5) diff --git a/engine/tests/test_host_service_scope.py b/engine/tests/test_host_service_scope.py index fcbc44e1..e5c825cf 100644 --- a/engine/tests/test_host_service_scope.py +++ b/engine/tests/test_host_service_scope.py @@ -65,7 +65,8 @@ def host_app(tmp_path, monkeypatch): monkeypatch.setattr(MetricsCollector, "start", lambda _self: None) @contextmanager - def resource_guard(resources, *, timeout): + def resource_guard(resources, *, timeout, connection_failure_safe): + assert connection_failure_safe is True locks.append(tuple(resources)) yield diff --git a/engine/tests/test_snapshot_thaw_recovery_app.py b/engine/tests/test_snapshot_thaw_recovery_app.py new file mode 100644 index 00000000..30931c66 --- /dev/null +++ b/engine/tests/test_snapshot_thaw_recovery_app.py @@ -0,0 +1,73 @@ +"""Interrupted snapshots attempt bounded thaw immediately, without polling.""" +import threading +import time + +import pytest + +from test_engine_boundary import boundary, call, USER, VM_ID +from virsh_panel.libvirt_service import VMError +from virsh_panel.metrics import host_scope + + +@pytest.mark.parametrize('manual', [False, True]) +def test_snapshot_runner_recovers_freeze_without_releasing_unknown_claim(boundary, manual): + client, engine, _store, _logs, services = boundary + call(client, 'GET', '/internal/v1/hosts/hk/vms') + components = engine.registry.components['hk'] + components.state.users.assign_vm( + host_scope(components.manager.profile), VM_ID, USER['user_id']) + service = services['hk'] + service.vm.update(managed=True, os_source='cloud_template', + rescue_mode=False, rescue_staged=False) + observed = [] + thaw_entered = threading.Event() + + def interrupted(vm_id, snapshot_key, *, consistency, progress=None, recovery_job_id=None): + observed.append(('create', recovery_job_id, snapshot_key)) + progress('freezing') + raise VMError('backup result unknown', 504, error_code='libvirt_execution_uncertain') + + def thaw(vm_id, *, recovery_job_id=None, snapshot_key=None): + observed.append(('thaw', recovery_job_id, snapshot_key)) + thaw_entered.set() + if manual: + raise VMError('unproven backup boundary', 409, + error_code='snapshot_thaw_recovery_required') + return {'uuid': vm_id, 'thawed': True, 'count': 1, 'power_state': 'running'} + + service.create_live_snapshot = interrupted + service.thaw_guest_filesystems = thaw + actor = {**USER, 'operation_id': 'snapshot-emergency-thaw-test'} + response = call(client, 'POST', f'/internal/v1/hosts/hk/vms/{VM_ID}/snapshots', actor, + {'snapshot_key': 'e'*32, 'name': 'snapshot', + 'consistency_mode': 'filesystem'}) + assert response.status_code == 202, response.text + # No operation/snapshot poll drives the emergency action. + assert thaw_entered.wait(3) + job_id = observed[0][1] + for _ in range(100): + saved = engine.registry.apps['hk'].state.snapshot_job_state(job_id) + if saved.get('freeze_recovery'): + break + time.sleep(.01) + assert observed == [('create', job_id, 'e'*32), ('thaw', job_id, 'e'*32)] + assert saved['status'] == 'uncertain' + assert saved['error_code'] == 'libvirt_execution_uncertain' + if manual: + assert saved['guest_freeze_state'] == 'attempted' + assert saved['freeze_recovery'] == 'manual_required' + assert saved['freeze_recovery_error_code'] == 'snapshot_thaw_recovery_required' + assert '立即' in saved['error'] + else: + assert saved['guest_freeze_state'] == 'recovered' + assert saved['freeze_recovery'] == 'confirmed' + # Polling must keep the original operation uncertain and non-refundable. + operation = call(client, 'GET', '/internal/v1/operations/snapshot-emergency-thaw-test', actor) + assert operation.json()['status'] == 'uncertain' + assert operation.json()['safe_to_refund'] is False + claims = engine.coordinator.snapshot() + assert any(claim.get('operation_id') == actor['operation_id'] for claim in claims.values()) + blocked = call(client, 'POST', f'/internal/v1/hosts/hk/vms/{VM_ID}/actions', + {**USER, 'operation_id': 'action-after-thaw'}, {'action': 'poweroff'}) + assert blocked.status_code == 503, blocked.text + assert blocked.json()['error_code'] == 'coordination_uncertain' diff --git a/engine/tests/test_snapshot_thaw_recovery_rpc.py b/engine/tests/test_snapshot_thaw_recovery_rpc.py new file mode 100644 index 00000000..e4dd70cb --- /dev/null +++ b/engine/tests/test_snapshot_thaw_recovery_rpc.py @@ -0,0 +1,268 @@ +"""Exercise guarded freeze recovery through actual spawned RPC workers.""" +import importlib +import json +import os +import sys +import types +from pathlib import Path + +import pytest + +from virsh_panel.libvirt_rpc import ExecutionFailure, LibvirtRuntime + +VM_ID = '00000000-0000-4000-8000-000000000001' +JOB_ID = '00000000-0000-4000-8000-000000000002' +TOKEN = 'a' * 32 +SNAPSHOT_KEY = 'b' * 32 + + +@pytest.fixture +def scenario(tmp_path, monkeypatch): + fixtures = Path(__file__).with_name('fixtures') + monkeypatch.syspath_prepend(str(fixtures)) + monkeypatch.setenv('PYTHONPATH', os.pathsep.join(( + str(fixtures), str(fixtures.parents[1]), os.environ.get('PYTHONPATH', '')))) + backend = importlib.import_module('snapshot_recovery_backend') + if 'libvirt' not in sys.modules: + monkeypatch.setitem(sys.modules, 'libvirt', backend) + service_module = importlib.import_module('virsh_panel.libvirt_service') + monkeypatch.setattr(service_module, 'libvirt', backend) + volume = service_module.snapshot_volume_name(VM_ID, SNAPSHOT_KEY) + state_path = tmp_path / 'remote.json' + state = dict(vm_id=VM_ID, volume_name=volume, token=TOKEN, + target_path='/images/' + volume, frozen=False, thaw_calls=0) + state_path.write_text(json.dumps(state)) + runtimes = [] + + def runtime(): + item = LibvirtRuntime(tmp_path/'intents', rpc_timeout=.2, + connect_timeout=3, backend_module='snapshot_recovery_backend') + runtimes.append(item) + return item + + def update(**values): + current = json.loads(state_path.read_text()) + current.update(values) + state_path.write_text(json.dumps(current)) + + def begin(**values): + update(**values) + item = runtime() + session = item.open(str(state_path), 'host-a', timeout=5) + domain = session.connection.lookupByUUIDString(VM_ID) + session.bind_snapshot_recovery(domain, job_id=JOB_ID, volume_name=volume, + pool_name='images', target_path=state['target_path'], + runtime_token=TOKEN) + return item, session, domain + + def recover(item=None, **kwargs): + item = item or runtime() + monkeypatch.setattr(service_module, 'get_runtime', lambda: item) + service = service_module.VMService(str(state_path), 'images', 'iso', 'default', + execution_scope='host-a') + # A recovery must not attempt to acquire/release the uncertain VM claim. + service._operation_guard = lambda *a, **k: pytest.fail('normal claim used for emergency thaw') + return service.thaw_guest_filesystems( + kwargs.pop('vm_id', VM_ID), recovery_job_id=kwargs.pop('job_id', JOB_ID), + snapshot_key=kwargs.pop('snapshot_key', SNAPSHOT_KEY)) + + yield types.SimpleNamespace(runtime=runtime, begin=begin, recover=recover, + update=update, state=lambda: json.loads(state_path.read_text()), + path=state_path, intent_dir=tmp_path/'intents', module=service_module) + for item in reversed(runtimes): + item.close(timeout=1) + + +def interrupted(scenario, **values): + item, session, domain = scenario.begin(backup_hangs=True, **values) + domain.fsFreeze(None, 0) + with pytest.raises(ExecutionFailure): + domain.backupBegin('', None, 0) + session.close() + assert scenario.state()['frozen'] is True + return item, session + + +def test_exact_active_backup_can_thaw_without_releasing_original_fence(scenario): + interrupted(scenario) + original = json.loads(next(scenario.intent_dir.rglob('*.json')).read_text()) + result = scenario.recover() + assert result['thawed'] is True + assert scenario.state()['frozen'] is False + updated = json.loads(next(scenario.intent_dir.rglob('*.json')).read_text()) + assert {key: value for key, value in updated.items() if key != 'snapshot_recovery'} == { + key: value for key, value in original.items() if key != 'snapshot_recovery'} + assert updated['snapshot_recovery']['backup_state'] == original['snapshot_recovery']['backup_state'] + session = scenario.runtime().open(str(scenario.path), 'host-a') + with pytest.raises(ExecutionFailure) as raised: + session.connection.lookupByUUIDString(VM_ID).destroy() + assert raised.value.error_code == 'libvirt_execution_blocked' + session.close() + assert scenario.state().get('destroyed') is None + + +@pytest.mark.parametrize('change', [ + {'backup_active': False}, {'backup_path': '/images/unrelated.qcow2'}, + {'token': 'c'*32}, {'runtime_id': 8}, {'job_operation': 99}, +]) +def test_missing_or_mismatched_remote_boundary_requires_manual_recovery(scenario, change): + interrupted(scenario) + scenario.update(**change) + with pytest.raises(scenario.module.VMError) as raised: + scenario.recover() + assert raised.value.error_code == 'snapshot_thaw_recovery_required' + assert scenario.state()['thaw_calls'] == 0 + + +@pytest.mark.parametrize('identity', [ + {'job_id': '00000000-0000-4000-8000-000000000009'}, + {'vm_id': '00000000-0000-4000-8000-000000000009'}, + {'snapshot_key': 'c'*32}, +]) +def test_wrong_job_vm_or_snapshot_never_thaws(scenario, identity): + interrupted(scenario) + with pytest.raises(scenario.module.VMError) as raised: + scenario.recover(**identity) + assert raised.value.error_code == 'snapshot_thaw_recovery_required' + assert scenario.state()['thaw_calls'] == 0 + + +def test_active_original_worker_cannot_be_recovered(scenario): + item, session, domain = scenario.begin() + domain.fsFreeze(None, 0) + with pytest.raises(scenario.module.VMError) as raised: + scenario.recover(item) + assert raised.value.error_code == 'snapshot_thaw_recovery_required' + assert scenario.state()['thaw_calls'] == 0 + session._fail('libvirt_execution_uncertain') + session.close() + + +def test_unacknowledged_freeze_and_old_intent_never_authorize_thaw(scenario): + item, session, domain = scenario.begin(freeze_hangs=True) + with pytest.raises(ExecutionFailure): + domain.fsFreeze(None, 0) + session.close() + with pytest.raises(scenario.module.VMError): + scenario.recover() + path = next(scenario.intent_dir.rglob('*.json')) + record = json.loads(path.read_text()) + record.pop('snapshot_recovery') + path.write_text(json.dumps(record)) + with pytest.raises(scenario.module.VMError) as raised: + scenario.recover() + assert raised.value.error_code == 'snapshot_thaw_recovery_required' + assert scenario.state()['thaw_calls'] == 0 + + +@pytest.mark.parametrize('backup_ack', [False, True]) +def test_durable_ack_boundary_survives_worker_interruption(scenario, backup_ack): + item, session, domain = scenario.begin() + domain.fsFreeze(None, 0) + if backup_ack: + domain.backupBegin('', None, 0) + scenario.update(backup_active=False) + session._fail('libvirt_execution_uncertain') + session.close() + assert scenario.recover()['thawed'] is True + + +def test_recovery_timeout_preserves_original_intent(scenario): + interrupted(scenario) + original = json.loads(next(scenario.intent_dir.rglob('*.json')).read_text()) + scenario.update(thaw_hangs=True) + with pytest.raises(scenario.module.VMError) as raised: + scenario.recover() + assert raised.value.error_code == 'libvirt_execution_uncertain' + assert raised.value.outcome_unknown is True + updated = json.loads(next(scenario.intent_dir.rglob('*.json')).read_text()) + assert {key: value for key, value in updated.items() if key != 'snapshot_recovery'} == { + key: value for key, value in original.items() if key != 'snapshot_recovery'} + assert updated['snapshot_recovery']['backup_state'] == original['snapshot_recovery']['backup_state'] + assert scenario.state()['frozen'] is True + + +def test_success_receipt_never_thaws_a_later_freeze(scenario): + interrupted(scenario) + assert scenario.recover()['thawed'] + scenario.update(frozen=True) + assert scenario.recover()['thawed'] + assert scenario.state()['frozen'] is True + assert scenario.state()['thaw_calls'] == 1 + + +def test_unknown_recovery_cannot_be_replayed(scenario): + interrupted(scenario) + scenario.update(thaw_hangs=True) + with pytest.raises(scenario.module.VMError): + scenario.recover() + scenario.update(thaw_hangs=False) + with pytest.raises(scenario.module.VMError) as raised: + scenario.recover() + assert raised.value.error_code == 'snapshot_thaw_recovery_required' + assert scenario.state()['thaw_calls'] == 1 + + +def test_original_thaw_receipt_recovers_without_another_guest_command(scenario): + item, session, domain = scenario.begin() + domain.fsFreeze(None, 0) + domain.backupBegin('', None, 0) + domain.fsThaw(None, 0) + session._fail('libvirt_execution_uncertain') + session.close() + scenario.update(frozen=True) + assert scenario.recover()['thawed'] + assert scenario.state()['frozen'] is True + assert scenario.state()['thaw_calls'] == 1 + + +@pytest.mark.parametrize('corruption', ['json', 'list', 'boolean_pid', 'missing_stage']) +def test_invalid_durable_evidence_is_explicit_manual_recovery(scenario, corruption): + interrupted(scenario) + path = next(scenario.intent_dir.rglob('*.json')) + record = json.loads(path.read_text()) + if corruption == 'json': + path.write_text('{broken') + elif corruption == 'list': + path.write_text('[]') + else: + if corruption == 'boolean_pid': + record['snapshot_recovery']['worker_pid'] = True + else: + record['snapshot_recovery'].pop('thaw_pending') + path.write_text(json.dumps(record)) + with pytest.raises(scenario.module.VMError) as raised: + scenario.recover() + assert raised.value.error_code == 'snapshot_thaw_recovery_required' + assert scenario.state()['thaw_calls'] == 0 + + +@pytest.mark.parametrize('behavior', ['thaw_invalid', 'thaw_type_error']) +def test_failure_after_thaw_call_never_resets_pending(scenario, behavior): + interrupted(scenario) + scenario.update(**{behavior: True}) + with pytest.raises(scenario.module.VMError) as raised: + scenario.recover() + assert raised.value.error_code == 'libvirt_execution_uncertain' + assert raised.value.outcome_unknown is True + state = json.loads(next(scenario.intent_dir.rglob('*.json')).read_text()) + assert state['snapshot_recovery']['recovery_state'] == 'pending' + scenario.update(**{behavior: False}) + with pytest.raises(scenario.module.VMError) as raised: + scenario.recover() + assert raised.value.error_code == 'snapshot_thaw_recovery_required' + assert scenario.state()['thaw_calls'] == 1 + + +def test_failed_boundary_persistence_prevents_backup_dispatch(scenario, monkeypatch): + item, session, domain = scenario.begin() + domain.fsFreeze(None, 0) + def failed_sync(path): + raise OSError('fsync unavailable') + monkeypatch.setattr(item, '_sync', failed_sync) + with pytest.raises(OSError): + domain.backupBegin('', None, 0) + assert scenario.state().get('backup_active') is None + assert session.failure is not None + session.close() + assert list(scenario.intent_dir.rglob('*.json')) diff --git a/engine/tests/test_vpc_route_readback.py b/engine/tests/test_vpc_route_readback.py index 72c2a099..3dc368fd 100644 --- a/engine/tests/test_vpc_route_readback.py +++ b/engine/tests/test_vpc_route_readback.py @@ -5,9 +5,15 @@ import subprocess import pytest -from test_vpc_dataplane import _plan +from test_vpc_dataplane import ( + _active_observation_markers, _guest_network_evidence, _plan, +) +from virsh_panel.vpc import VpcCoordinator from virsh_panel.vpc_dataplane import ( - _render_apply_script_v2, _routes, render_observe_script, + VPC_ROUTE_RECORDS_SH, VpcPrivateHostAdapter, _markers, + _render_apply_script_v2, _routes, patch_private_interface_xml, + private_guest_network_generation, private_guest_routes, + render_observe_script, ) @@ -97,7 +103,8 @@ def test_observe_confirms_all_local_remote_and_service_routes(tmp_path, explicit @pytest.mark.parametrize("change", [ "foreign_device", "duplicate_device", "missing_value", "duplicate_route", "wrong_table", "wrong_proto", "wrong_metric", "leading_zero", "invalid_ip", - "broad_prefix", "blackhole", "multipath", "linkdown", "duplicate_proto", "failed_command", + "broad_prefix", "blackhole", "multipath", "duplicate_linkdown", + "dead", "unresolved", "offload", "duplicate_proto", "failed_command", ]) def test_route_normalization_never_accepts_drift_or_unknown_output(tmp_path, phase, change): plan = _plan() @@ -118,7 +125,10 @@ def test_route_normalization_never_accepts_drift_or_unknown_output(tmp_path, pha "broad_prefix": first.replace(destination, destination + "/24", 1), "blackhole": "blackhole " + first, "multipath": first + f" nexthop dev {plan.resources.bridge} weight 1", - "linkdown": first + " linkdown", + "duplicate_linkdown": first + " linkdown linkdown", + "dead": first + " dead", + "unresolved": first + " unresolved", + "offload": first + " offload", "duplicate_proto": first + " proto 186", "failed_command": first, } @@ -137,3 +147,149 @@ def test_observe_does_not_infer_device_for_unfiltered_inventory(tmp_path): assert "__VC_VPC_ROUTE_EXTRA__=yes" in result.stdout assert not any(line.endswith("|owned") for line in result.stdout.splitlines() if line.startswith("__VC_VPC_ROUTE__=")) + + +@pytest.mark.parametrize("explicit_prefix", [False, True]) +@pytest.mark.parametrize("omit_device", [False, True]) +def test_apply_accepts_exact_bridge_routes_without_carrier( + tmp_path, explicit_prefix, omit_device): + inventory = _inventory(explicit_prefix=explicit_prefix, + omit_device=omit_device, bridge_only=True) + result = _run(tmp_path, "apply", inventory + " linkdown") + assert result.returncode == 0, result.stderr + + +@pytest.mark.parametrize("device", ["wg-fabric0", "br0"]) +def test_linkdown_is_only_admitted_for_the_explicit_private_bridge(tmp_path, device): + plan = _plan() + inventory = _inventory() + if device == "wg-fabric0": + inventory = "\n".join( + line + " linkdown" if f"dev {device} " in line else line + for line in inventory.splitlines()) + else: + inventory = inventory.replace(f"dev {plan.resources.bridge}", "dev br0") + inventory = inventory.replace("dev br0", "dev br0 linkdown") + result = _run(tmp_path, "observe", inventory) + assert "__VC_VPC_ROUTE_EXTRA__=yes" in result.stdout + assert all(route["state"] == "missing" + for route in _markers(result.stdout)["routes"]) + + +def test_route_parser_requires_explicit_carrier_exception(): + plan = _plan() + record = _inventory(bridge_only=True) + " linkdown" + result = subprocess.run( + ["/bin/sh", "-eu", "-c", VPC_ROUTE_RECORDS_SH + + '\nprintf "%s\\n" "$VC_ROUTES" | vc_vpc_route_records "$BRIDGE" main'], + capture_output=True, text=True, env=os.environ | { + "VC_ROUTES": record, "BRIDGE": plan.resources.bridge, + }) + assert result.returncode == 1 + assert result.stdout == "" + + +def _observe_lifecycle(tmp_path, *, running, linkdown, guest_verified=True): + """Run generated route parsing through the real host and VM projections. + + Only the remote ip inventory, unrelated host markers and domain evidence + are simulated. This is not a Linux-network-namespace or QGA integration. + """ + plan = _plan() + member = plan.local_members[0] + routes = private_guest_routes(plan.vpc, member) + inventory = "\n".join( + line + " linkdown" if linkdown and f"dev {plan.resources.bridge} " in line + else line for line in _inventory().splitlines()) + result = _run(tmp_path, "observe", inventory) + assert result.returncode == 0, result.stderr + route_prefixes = ("__VC_VPC_ROUTE__=", "__VC_VPC_ROUTE_EXTRA__=") + markers = [line for line in _active_observation_markers(plan) + if not line.startswith(route_prefixes)] + markers.extend(line for line in result.stdout.splitlines() + if line.startswith(route_prefixes)) + xml = patch_private_interface_xml( + "" + "" + "" + "", member=member, resources=plan.resources, + guest_network_generation=private_guest_network_generation(member, routes), + vpc_id=plan.vpc.vpc_id, desired_revision=plan.desired_revision, + desired_digest=plan.desired_digest) + guest = (_guest_network_evidence(member, routes) if running and guest_verified + else {"state": "unavailable" if running else "stopped", + "verified": False, + "reason_code": ("vpc_private_guest_agent_unavailable" if running + else "vpc_private_guest_stopped")}) + adapter = VpcPrivateHostAdapter(object(), {}) + adapter._deployment_contract = lambda: ("networkmanager", "", "", "") + adapter._run = lambda _script: "\n".join(markers) + "\n" + observed = adapter.observe(plan, xml_by_instance={member.instance_uuid: { + "inactive": xml, "live": xml if running else None, + "seed_verified": True, "guest_network": guest, + }}) + return plan, member, observed + + +def _instance_projection(plan, member, observed): + host = dict(observed, transport_verified=True, + private_dataplane_verified=observed["state"] == "active") + return VpcCoordinator._instance_private_state( + member.instance_uuid, required=True, vpc_id=plan.vpc.vpc_id, + member_id=member.member_id, desired_state="active", + private_ip_cidr=member.private_ip + "/32", private_mac=member.private_mac, + desired_revision=plan.desired_revision, desired_digest=plan.desired_digest, + host=host, member_nic=observed["member_nics"][0]) + + +def test_stop_last_member_retry_and_start_preserve_configuration_proof(tmp_path): + # Initial stopped attachment and stopping the last active VM both retain + # the same exact routes while the bridge has no active tap/carrier. + for running in (False, True, False, False, True): + bridge_inventory = _inventory(bridge_only=True) + if not running: + bridge_inventory += " linkdown" + applied = _run(tmp_path, "apply", bridge_inventory) + assert applied.returncode == 0, applied.stderr + plan, member, observed = _observe_lifecycle( + tmp_path, running=running, linkdown=not running) + assert observed["config_verified"] is True + assert observed["config_observed_revision"] == plan.desired_revision + assert observed["config_observed_digest"] == plan.desired_digest + private = _instance_projection(plan, member, observed) + assert private["readback"] == "confirmed" + assert private["state"] == ("active" if running else "stopped") + assert observed["state"] == ("active" if running else "applying") + bridge_route = next(route for route in observed["routes"] + if route["device"] == plan.resources.bridge) + assert bridge_route["flags"] == ([] if running else ["linkdown"]) + assert all(route["flags"] == [] for route in observed["routes"] + if route["device"] == "wg-fabric0") + + +def test_carrier_or_qga_loss_never_promotes_running_vm_to_usable(tmp_path): + for linkdown, guest_verified in ((True, True), (True, False), (False, False)): + plan, member, observed = _observe_lifecycle( + tmp_path, running=True, linkdown=linkdown, + guest_verified=guest_verified) + assert observed["config_verified"] is True + assert observed["state"] != "active" + assert observed["observed_revision"] == 0 + private = _instance_projection(plan, member, observed) + assert private["state"] == "uncertain" + assert private["readback"] != "confirmed" + + +def test_other_members_route_flag_does_not_erase_healthy_member_proof(tmp_path): + plan, member, observed = _observe_lifecycle( + tmp_path, running=True, linkdown=False) + # Host configuration and the member's own runtime proof remain exact even + # if a sibling's observation is unavailable. The projection is per VM. + observed["state"] = "applying" + own = next(route for route in observed["routes"] + if route["destination_cidr"] == member.private_ip + "/32") + observed["routes"].append(dict( + own, destination_cidr="10.42.0.12/32", flags=["linkdown"])) + private = _instance_projection(plan, member, observed) + assert private["state"] == "active" + assert private["readback"] == "confirmed" diff --git a/engine/virsh_panel/app.py b/engine/virsh_panel/app.py index 484445e0..cc0a8ba3 100644 --- a/engine/virsh_panel/app.py +++ b/engine/virsh_panel/app.py @@ -98,6 +98,7 @@ SNAPSHOT_PUBLIC_ERROR_CODES = { "live_snapshot_unsupported", "snapshot_pool_unsupported", "snapshot_result_uncertain", + "snapshot_thaw_recovery_required", } | JOB_UNCERTAIN_CODES REINSTALL_PUBLIC_ERROR_CODES = { REINSTALL_CLEANUP_PENDING_ERROR_CODE, @@ -1794,6 +1795,39 @@ def create_app( else: delete_job("user_iso_import_job", job_id) + def recover_snapshot_freeze(job_id: str, job: dict, service) -> dict: + """Attempt only thaw recovery, preserving the backup's unknown result.""" + if (job.get("action") != "create" or job.get("consistency") != "filesystem" + or job.get("guest_freeze_state") != "attempted"): + return job + try: + if job.get("job_id", job_id) != job_id: + raise VMError("快照恢复任务身份不一致", 409, + error_code="snapshot_thaw_recovery_required") + service.thaw_guest_filesystems( + job["vm_id"], recovery_job_id=job_id, + snapshot_key=job.get("snapshot_key")) + except Exception as exc: + job["freeze_recovery"] = ( + "manual_required" if getattr(exc, "error_code", None) + == "snapshot_thaw_recovery_required" else "failed") + job["freeze_recovery_error_code"] = "snapshot_thaw_recovery_required" + job["phase"] = "访客文件系统未确认解冻,需要立即人工核对" + job["error"] = ( + "快照结果仍未知;访客文件系统未确认解冻,请管理员立即核对原备份任务," + "按原虚拟机 UUID 执行紧急解冻流程。不要重放快照或清除写入保护。") + LOG.exception("Snapshot emergency thaw remains unresolved: job=%s", job_id) + else: + if job.get("freeze_recovery_error_code"): + job["phase"] = "访客文件系统已解冻,快照结果仍待核查" + job["error"] = "已确认原快照冻结解除;备份结果仍未知,不要重放快照或清除写入保护。" + job["guest_freeze_state"] = "recovered" + job["freeze_recovery"] = "confirmed" + job["freeze_recovered_at"] = int(time.time()) + job.pop("freeze_recovery_error_code", None) + save_job("snapshot_job", job_id, job) + return job + def restore_jobs() -> None: """Restore interrupted jobs without pretending mutations resumed.""" if data_store is None: @@ -1886,18 +1920,16 @@ def create_app( execution_scope=profile.get("host_id") or host_scope(profile), ) ) - recovery_service.thaw_guest_filesystems(job["vm_id"]) - except Exception: - job["freeze_recovery"] = "failed" + recover_snapshot_freeze(job_id, job, recovery_service) + except Exception as exc: + job["freeze_recovery"] = ( + "manual_required" if getattr(exc, "error_code", None) + == "snapshot_thaw_recovery_required" else "failed") LOG.exception( "Could not recover interrupted guest freeze: " "job=%s uuid=%s", job_id, job.get("vm_id"), ) - else: - job["guest_freeze_state"] = "recovered" - job["freeze_recovery"] = "confirmed" - job["freeze_recovered_at"] = int(time.time()) save_job(kind, job_id, job) if (kind == "reinstall_job" and job.get("status") in {"queued", "running"}): @@ -2449,6 +2481,19 @@ def create_app( def current_service() -> VMService: return new_service(current_profile()) + def host_connection_preflight(callback, *args, **kwargs): + # Scan/stage/test never mutate the remote host or activate a profile. + # Persist proof before returning a transient connection failure. Never + # use this wrapper for commit or work after libvirt validation. + try: + return callback(*args, **kwargs) + except RemoteHostError: + execution = current_execution() + if (execution is not None and execution.connection_failure_safe + and not execution.remote_write_started.is_set()): + mark_not_started() + raise + def validate_staged_libvirt( staged, body: HostBody, *, ensure_paths: bool = False, ) -> tuple[dict | None, str | None]: @@ -2899,7 +2944,7 @@ def create_app( @app.post("/api/v1/host/scan") def scan_host(body: HostScanBody, _session: Session = Depends(require_admin_write)): - return manager.scan(body.host, body.port) + return host_connection_preflight(manager.scan, body.host, body.port) @app.post("/api/v1/host/unlock") def unlock_host(body: UnlockBody, request: Request, _session: Session = Depends(require_admin_write)): @@ -2918,9 +2963,10 @@ def create_app( @app.post("/api/v1/host/test") def test_host(body: HostBody, _session: Session = Depends(require_admin_write)): - staged = manager.stage(body.profile(), confirmed_fingerprint=body.confirmed_fingerprint) + staged = host_connection_preflight( + manager.stage, body.profile(), confirmed_fingerprint=body.confirmed_fingerprint) try: - result = manager.test(staged) + result = host_connection_preflight(manager.test, staged) resources, resource_warning = validate_staged_libvirt(staged, body) return { **result, @@ -2969,10 +3015,11 @@ def create_app( "宿主机已有公网地址池;请先在公网地址池管理中处理网络接入,再修改默认网络", 409, ) - staged = manager.stage(body.profile(), confirmed_fingerprint=body.confirmed_fingerprint) + staged = host_connection_preflight( + manager.stage, body.profile(), confirmed_fingerprint=body.confirmed_fingerprint) committed = False try: - manager.test(staged) + host_connection_preflight(manager.test, staged) _resources, resource_warning = validate_staged_libvirt(staged, body, ensure_paths=True) profile = manager.commit(staged) committed = True @@ -4421,7 +4468,8 @@ def create_app( if action == "create": result = service.create_live_snapshot( vm_id, snapshot_key, - consistency=job["consistency"], progress=report_progress) + consistency=job["consistency"], progress=report_progress, + recovery_job_id=job_id) physical_committed = True verified_capacity = result.get("size_bytes") verified_identity = result.get("volume_identity") @@ -4569,6 +4617,14 @@ def create_app( save_job("snapshot_job", job_id, current) except DatabaseError: LOG.exception("Could not persist snapshot job failure: %s", job_id) + # VMService has already closed/reaped the original RPC session. + # Do not wait for restart or polling while guest filesystems may + # still be frozen. This never settles the backup or its claims. + if uncertain and current.get("guest_freeze_state") == "attempted": + try: + recover_snapshot_freeze(job_id, current, new_service(profile)) + except Exception: + LOG.exception("Could not persist emergency thaw result: job=%s", job_id) finally: with snapshot_lock: if snapshot_jobs[job_id].get("status") == "uncertain": @@ -4613,16 +4669,18 @@ def create_app( if (job.get("action") == "create" and job.get("consistency") == "filesystem" and job.get("guest_freeze_state") == "attempted"): - service.thaw_guest_filesystems(job["vm_id"]) with snapshot_lock: current = snapshot_jobs.get(job_id) if current is None or current.get("status") != "uncertain": return dict(current or job) - current["guest_freeze_state"] = "recovered" - current["freeze_recovery"] = "confirmed" - current["freeze_recovered_at"] = int(time.time()) - save_job("snapshot_job", job_id, current) - job = dict(current) + recover_snapshot_freeze(job_id, current, service) + job = dict(current) + if job.get("guest_freeze_state") == "attempted": + return job + if job.get("error_code") in EXECUTION_ERROR_CODES: + # Thawing says nothing about whether the unknown remote backup + # can still commit. Preserve its fence, claim and result. + return job if job["action"] == "create": if record is None: return job diff --git a/engine/virsh_panel/coordination.py b/engine/virsh_panel/coordination.py index 4a022a0d..f0bc195e 100644 --- a/engine/virsh_panel/coordination.py +++ b/engine/virsh_panel/coordination.py @@ -60,6 +60,11 @@ class Claim: class ExecutionContext: coordinator: DurableCoordinator claim: Claim + # Shared by nested claims and copied thread contexts for this execution. + # A later connection failure cannot erase an earlier session's writes. + remote_write_started: threading.Event = field(default_factory=threading.Event, + compare=False, repr=False) + connection_failure_safe: bool = False _CURRENT_EXECUTION: ContextVar[ExecutionContext | None] = ContextVar( @@ -375,12 +380,42 @@ def update_current_claim(claim: Claim) -> None: context = current_execution() if context is None or _identity(context.claim) != _identity(claim): raise ValueError("Only the current execution claim may be updated") - _CURRENT_EXECUTION.set(ExecutionContext(context.coordinator, claim)) + _CURRENT_EXECUTION.set(ExecutionContext( + context.coordinator, claim, context.remote_write_started, + context.connection_failure_safe)) + + +def mark_remote_write_started() -> None: + """Record conservative in-process proof before an RPC write is sent.""" + context = current_execution() + if context is not None: + context.remote_write_started.set() + + +def connection_failure_before_write(error_code: str) -> bool: + """Allow only explicitly libvirt-only scopes to use connection evidence. + + Other workflows can mutate hosts over SSH before their first libvirt call. + An absent RPC write marker never proves those workflows were side-effect + free. Persisted unknown claims are not released through this local proof. + """ + context = current_execution() + return bool(error_code == "libvirt_connection_failed" + and context is not None and context.connection_failure_safe + and not context.remote_write_started.is_set()) @contextmanager -def execution_context(coordinator: DurableCoordinator, claim: Claim): - token = _CURRENT_EXECUTION.set(ExecutionContext(coordinator, claim)) +def execution_context(coordinator: DurableCoordinator, claim: Claim, *, + connection_failure_safe: bool | None = None): + parent = current_execution() + same_execution = (parent is not None and parent.coordinator is coordinator + and parent.claim.execution_id == claim.execution_id) + evidence = parent.remote_write_started if same_execution else threading.Event() + safe = (parent.connection_failure_safe if same_execution else False) + if connection_failure_safe is not None: + safe = connection_failure_safe + token = _CURRENT_EXECUTION.set(ExecutionContext(coordinator, claim, evidence, safe)) try: yield finally: @@ -392,13 +427,17 @@ def _uncertain_exception(exc: BaseException) -> bool: if code in _UNKNOWN_CODES: return True if code in _SAFE_EXECUTION_CODES: + context = current_execution() + return bool(context is not None and context.remote_write_started.is_set()) + if connection_failure_before_write(code): return False status = getattr(exc, "status", None) return not isinstance(status, int) or status >= 500 @contextmanager -def resource_guard(resources, timeout: float = 5.0): +def resource_guard(resources, timeout: float = 5.0, *, + connection_failure_safe: bool | None = None): import math if (isinstance(timeout, bool) or not isinstance(timeout, (int, float)) @@ -423,7 +462,8 @@ def resource_guard(resources, timeout: float = 5.0): if exc.error_code != "coordination_busy" or remaining <= 0: raise time.sleep(min(0.02, remaining)) - with execution_context(coordinator, claim): + with execution_context(coordinator, claim, + connection_failure_safe=connection_failure_safe): try: yield claim except BaseException as exc: diff --git a/engine/virsh_panel/durable_jobs.py b/engine/virsh_panel/durable_jobs.py index f6f923cd..6c693ea0 100644 --- a/engine/virsh_panel/durable_jobs.py +++ b/engine/virsh_panel/durable_jobs.py @@ -10,7 +10,8 @@ import logging import threading from typing import Any, Callable, Mapping -from .coordination import CoordinationError, current_execution, execution_context +from .coordination import (CoordinationError, current_execution, execution_context, + connection_failure_before_write) LOG = logging.getLogger(__name__) @@ -24,6 +25,7 @@ def _terminal_outcome(record, coordinator, execution_id: str) -> bool: and record.get("status") in _TERMINAL and not record.get("cleanup_pending") and not str(code).startswith(("libvirt_execution_", "coordination_")) + and (code != "libvirt_connection_failed" or connection_failure_before_write(code)) and not any(claim["execution_id"] == execution_id and claim["state"] == "uncertain" for claim in coordinator.snapshot().values()) diff --git a/engine/virsh_panel/internal.py b/engine/virsh_panel/internal.py index db359121..26546a02 100644 --- a/engine/virsh_panel/internal.py +++ b/engine/virsh_panel/internal.py @@ -53,6 +53,7 @@ from .libvirt_rpc import EXECUTION_ERROR_CODES as LIBVIRT_EXECUTION_ERROR_CODES from .coordination import ( DurableCoordinator, CoordinationError, current_execution, execution_context, configure_default_coordinator, update_current_claim, _uncertain_exception, + connection_failure_before_write, ) from .shutdown import (shutdown_scope, remaining_shutdown_seconds, run_bounded_close, install_asyncio_executor, @@ -849,6 +850,15 @@ class EngineApplication: target and method == "POST" and path.endswith("/network-suspend") and delegated.get("can_manage")) + # These handlers reach remote mutations only through VMService's RPC + # boundary. Host staging/test is read-only on the host and commit is + # after libvirt validation. Do not extend this to VPC/Fabric: their SSH + # helpers can have written before their first libvirt connection. + libvirt_only = bool(operation_request) or bool(host_route and ( + method == "PUT" and resource == "host" + or method == "POST" and resource in {"host/test", "host/scan"})) or ( + method == "POST" and path == "/internal/v1/hosts/scan") + async def execute(): await asyncio.to_thread(self._fence_legacy_operations) await self._settle_update_claims() @@ -860,7 +870,8 @@ class EngineApplication: messages = [] async def capture(message): messages.append(message) - with execution_context(self.coordinator, claim): + with execution_context(self.coordinator, claim, + connection_failure_safe=libvirt_only): try: if await asyncio.to_thread(self.updater.active_job): raise EngineAuthError("面板正在更新,暂时不能执行虚拟化写操作", 409) @@ -900,8 +911,15 @@ class EngineApplication: raw = await asyncio.to_thread(self.store.get, "operation", op_id) if op_id else None record = json.loads(raw) if raw else {} current = (await asyncio.to_thread(self.coordinator.snapshot)).get(claim.token, {}) + response_error_code = None + if status >= 500: + with suppress(TypeError, ValueError, AttributeError): + response_error_code = json.loads(b"".join( + item.get("body", b"") for item in messages + if item["type"] == "http.response.body")).get("error_code") safe_failure = ( - current.get("metadata", {}).get("failed_before_start") is True + connection_failure_before_write(response_error_code) + or current.get("metadata", {}).get("failed_before_start") is True or record.get("status") == "failed" or record.get("mutation_started") is False or record.get("network_mutation_started") is False @@ -913,10 +931,17 @@ class EngineApplication: await asyncio.to_thread(self.coordinator.release, claim) except BaseException as exc: not_started = False + try: + current = (await asyncio.to_thread( + self.coordinator.snapshot)).get(claim.token, {}) + not_started = current.get("metadata", {}).get("failed_before_start") is True + except Exception: + pass if op_id: try: raw = await asyncio.to_thread(self.store.get, "operation", op_id) - not_started = raw is not None and json.loads(raw).get("mutation_started") is False + not_started = not_started or ( + raw is not None and json.loads(raw).get("mutation_started") is False) except Exception: pass if _uncertain_exception(exc) and not not_started and dispatch_started: @@ -999,7 +1024,8 @@ class EngineApplication: if not delegated["is_super_admin"] or not delegated["can_manage"]: raise EngineAuthError("仅超级管理员可以扫描宿主机", 403) from .app import HostScanBody - from .remote_host import RemoteHostManager + from .remote_host import RemoteHostError, RemoteHostManager + from .durable_jobs import mark_not_started from pydantic import ValidationError try: value = HostScanBody.model_validate_json(body) @@ -1008,7 +1034,14 @@ class EngineApplication: manager = await asyncio.to_thread(RemoteHostManager, Path(self.config["data_dir"]) / "discovery") try: - result = await asyncio.to_thread(manager.scan, value.host, value.port) + try: + result = await asyncio.to_thread(manager.scan, value.host, value.port) + except RemoteHostError: + context = current_execution() + if (context is not None and context.connection_failure_safe + and not context.remote_write_started.is_set()): + await asyncio.to_thread(mark_not_started) + raise await json_response(send, 200, result) finally: await asyncio.to_thread(manager.close) @@ -1231,9 +1264,11 @@ class EngineApplication: admission = ((await asyncio.to_thread(self.coordinator.snapshot)).get(context.claim.token, {}) if context else {}) failed_before_start = admission.get("metadata", {}).get("failed_before_start") is True + connection_failed = connection_failure_before_write(payload.get("error_code")) operation["status"] = ("pending" if status == 202 and payload.get("job_id") else "succeeded") if 200 <= status < 300 else ( "failed" if not create and ( - status < 500 or network_preflight_failed or failed_before_start) else "uncertain") + status < 500 or network_preflight_failed or failed_before_start + or connection_failed) else "uncertain") if operation["status"] == "succeeded": operation["result"] = payload elif operation["status"] == "pending": @@ -1292,7 +1327,7 @@ class EngineApplication: ) operation.pop("finished_at", None) elif (isinstance(preflight_status, int) - and preflight_status < 500): + and preflight_status < 500) or connection_failure_before_write(error_code): operation.update( status="failed", error=str(exc), diff --git a/engine/virsh_panel/libvirt_rpc.py b/engine/virsh_panel/libvirt_rpc.py index e028f8be..5cf26e43 100644 --- a/engine/virsh_panel/libvirt_rpc.py +++ b/engine/virsh_panel/libvirt_rpc.py @@ -15,12 +15,14 @@ import logging import math import multiprocessing import os +import re import signal import socket import struct import threading import time import uuid +import xml.etree.ElementTree as ET from pathlib import Path @@ -30,6 +32,8 @@ EXECUTION_ERROR_CODES = frozenset({ "libvirt_execution_stopped", "libvirt_execution_blocked", }) MAX_FRAME = 32 * 1024 * 1024 +SNAPSHOT_RECOVERY_ERROR = "snapshot_thaw_recovery_required" +_QGA_RUNTIME_TAG = "{urn:virsh-panel:qga-runtime:1}qga-runtime" READ_METHODS = frozenset({ "ID", "UUID", "UUIDString", "XMLDesc", "name", "path", "info", "getName", "getDomain", @@ -56,6 +60,78 @@ class ExecutionFailure(Exception): self.outcome_unknown = outcome_unknown +class SnapshotRecoveryRequired(Exception): + """The durable evidence does not authorize an emergency thaw.""" + + +def _snapshot_thaw_preflight(connection, evidence, backend): + """The only fence exception: thaw one proven, previously frozen runtime. + + No absence observation proves a timed-out backupBegin never arrived. + A pending begin therefore requires the exact active backup target. + """ + domain = connection.lookupByUUIDString(evidence["vm_id"]) + if not domain.isActive(): + return {"thawed": True, "count": 0, "power_state": "stopped"} + if domain.ID() != evidence["runtime_id"]: + raise SnapshotRecoveryRequired() + live = ET.fromstring(domain.XMLDesc(0)) + inactive = ET.fromstring(domain.XMLDesc(backend.VIR_DOMAIN_XML_INACTIVE)) + nodes = live.findall("./metadata/" + _QGA_RUNTIME_TAG) + if (len(nodes) != 1 or nodes[0].attrib != { + "domain-uuid": evidence["vm_id"], "token": evidence["runtime_token"]} + or inactive.find("./metadata/" + _QGA_RUNTIME_TAG) is not None): + raise SnapshotRecoveryRequired() + volume = connection.storagePoolLookupByName( + evidence["pool_name"]).storageVolLookupByName(evidence["volume_name"]) + if volume.path() != evidence["target_path"]: + raise SnapshotRecoveryRequired() + stats = domain.jobStats(0) + if not isinstance(stats, dict) or not isinstance(stats.get("type"), int): + raise SnapshotRecoveryRequired() + active_backup = stats["type"] != backend.VIR_DOMAIN_JOB_NONE + if active_backup: + if (evidence["backup_state"] == "not_sent" + or stats.get("operation") != backend.VIR_DOMAIN_JOB_OPERATION_BACKUP): + raise SnapshotRecoveryRequired() + backup = ET.fromstring(domain.backupGetXMLDesc(0)) + all_disks = backup.findall("./disks/disk") + disks = [disk for disk in all_disks if disk.get("backup", "yes") != "no"] + if (backup.tag != "domainbackup" or backup.get("mode", "push") != "push" + or len(disks) != 1 or disks[0].get("name") != "vda" + or disks[0].get("backup", "yes") != "yes" + or disks[0].get("type", "file") != "file" + or disks[0].find("target") is None + or disks[0].find("target").get("file") != evidence["target_path"] + or any(disk.find("target") is not None or disk.find("scratch") is not None + for disk in all_disks if disk.get("backup") == "no")): + raise SnapshotRecoveryRequired() + if evidence["backup_state"] == "pending" and not active_backup: + raise SnapshotRecoveryRequired() + # The last check is immediately before the sole permitted mutation. + if (not domain.isActive() or domain.ID() != evidence["runtime_id"] + or ET.fromstring(domain.XMLDesc(0)).find( + "./metadata/" + _QGA_RUNTIME_TAG).get("token") + != evidence["runtime_token"]): + raise SnapshotRecoveryRequired() + return domain + + +def _recover_snapshot_thaw(connection, evidence, backend): + try: + domain = _snapshot_thaw_preflight(connection, evidence, backend) + except (ET.ParseError, AttributeError, KeyError, TypeError, backend.libvirtError): + raise SnapshotRecoveryRequired() from None + if isinstance(domain, dict): + return domain + # From here onward any exception or invalid result is outcome-unknown. + # It must not reset the durable recovery attempt to "not_started". + count = domain.fsThaw(None, 0) + if isinstance(count, bool) or not isinstance(count, int) or count < 0: + raise RuntimeError("unconfirmed emergency thaw result") + return {"thawed": True, "count": count, "power_state": "running"} + + def _send(sock, value, deadline): payload = json.dumps(value, ensure_ascii=True, allow_nan=False).encode() if len(payload) > MAX_FRAME: @@ -158,11 +234,16 @@ def _worker(sock, uri, backend_module): if backend_module == "libvirt" else backend) result = qemu.qemuAgentCommand( *_decode(request["args"], resolve)) + elif request["op"] == "snapshot_thaw": + result = _recover_snapshot_thaw( + objects[request["handle"]], request["evidence"], backend) else: raise ValueError("invalid libvirt RPC operation") response = {"result": _encode(result, export)} except backend.libvirtError as exc: response = {"libvirt_error": str(exc), "err": getattr(exc, "err", None)} + except SnapshotRecoveryRequired: + response = {"snapshot_recovery_required": True} except Exception: # Never expose arbitrary exception text/arguments or credentials. response = {"worker_error": True} @@ -209,6 +290,7 @@ class LibvirtRuntime: self._sessions = set() self._reserved = {} self._closing = False + self._snapshot_recoveries = set() def _directory(self, scope): directory = self.state_dir / hashlib.sha256(scope.encode()).hexdigest() @@ -239,14 +321,19 @@ class LibvirtRuntime: os.close(fd) def _before_write(self, session, method): + from .coordination import mark_remote_write_started + with self._condition: if self._closing: raise ExecutionFailure("libvirt_execution_stopped") + if session.scope in self._snapshot_recoveries: + raise ExecutionFailure("libvirt_execution_blocked", outcome_unknown=True) directory = self._directory(session.scope) active = {s.token for s in self._sessions if s.failure is None} if any(path.stem not in active for path in directory.glob("*.json")): raise ExecutionFailure("libvirt_execution_blocked", outcome_unknown=True) if session.intent is not None: + mark_remote_write_started() return path = directory / (session.token + ".json") # Persist before sending the first write. Crash/lost receipt never @@ -262,6 +349,91 @@ class LibvirtRuntime: os.fsync(stream.fileno()) self._sync(directory) session.intent = path + mark_remote_write_started() + + def _save_snapshot_evidence(self, session, evidence): + """Persist a conservative phase before sending the corresponding RPC.""" + with self._condition: + if session.intent is None or session.failure is not None or self._closing: + raise SnapshotRecoveryRequired() + try: + original = json.loads(session.intent.read_text()) + original["snapshot_recovery"] = evidence + self._replace_snapshot_record(session.intent, original) + except Exception: + session._fail("libvirt_execution_uncertain", outcome_unknown=True) + raise + session.snapshot_evidence = evidence + + def _replace_snapshot_record(self, path, record): + temporary = path.with_suffix(".snapshot-tmp") + try: + with temporary.open("x", encoding="utf-8") as stream: + os.chmod(temporary, 0o600) + json.dump(record, stream) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + self._sync(path.parent) + finally: + temporary.unlink(missing_ok=True) + + def _snapshot_recovery_evidence(self, session, job_id, vm_id, volume_name): + """Authorize only a bound job whose old worker can send no more RPCs.""" + with self._condition: + if self._closing or session.scope in self._snapshot_recoveries: + raise SnapshotRecoveryRequired() + files = list(self._directory(session.scope).glob("*.json")) + # Another unresolved write may include a freeze we cannot identify. + if len(files) != 1 or files[0].stat().st_size > 16384: + raise SnapshotRecoveryRequired() + try: + record = json.loads(files[0].read_text()) + except (ValueError, OSError): + raise SnapshotRecoveryRequired() from None + if not isinstance(record, dict): + raise SnapshotRecoveryRequired() + evidence = record.get("snapshot_recovery") + if (record.get("scope") != session.scope + or not isinstance(evidence, dict) + or evidence.get("job_id") != job_id + or evidence.get("vm_id") != vm_id + or evidence.get("volume_name") != volume_name + or evidence.get("freeze_ack") is not True + or not isinstance(evidence.get("thaw_ack"), bool) + or evidence.get("thaw_pending") is not False + or evidence.get("recovery_state") not in {"not_started", "ack"} + or evidence.get("backup_state") not in {"not_sent", "pending", "ack"} + or not isinstance(evidence.get("worker_pid"), int) + or isinstance(evidence["worker_pid"], bool) + or evidence["worker_pid"] <= 0 + or not isinstance(evidence.get("runtime_id"), int) + or isinstance(evidence["runtime_id"], bool) + or evidence["runtime_id"] < 0 + or not isinstance(evidence.get("runtime_token"), str) + or re.fullmatch(r"[0-9a-f]{32}", evidence["runtime_token"]) is None + or not isinstance(evidence.get("pool_name"), str) + or not evidence["pool_name"] + or not isinstance(evidence.get("target_path"), str) + or not evidence["target_path"].startswith("/")): + raise SnapshotRecoveryRequired() + for other in self._sessions: + if other is session or other.scope != session.scope: + continue + if other.intent is not None and (other.process is None + or other.process.is_alive()): + raise SnapshotRecoveryRequired() + try: + os.kill(evidence["worker_pid"], 0) + except ProcessLookupError: + pass + except OSError: + raise SnapshotRecoveryRequired() from None + else: + # PID reuse errs towards manual recovery, never false admission. + raise SnapshotRecoveryRequired() + self._snapshot_recoveries.add(session.scope) + return files[0], record, evidence def open(self, uri, scope, *, timeout=180.0): if (not isinstance(scope, str) or not scope or len(scope) > 256 @@ -350,6 +522,8 @@ class RPCSession: self._pgid = None self._signal_lock = threading.Lock() self._sent_signals = set() + self.snapshot_evidence = None + self._snapshot_handle = None @property def pid(self): @@ -430,6 +604,8 @@ class RPCSession: if response.get("err") is not None: error.err = tuple(response["err"]) raise error + if response.get("snapshot_recovery_required") is True: + raise SnapshotRecoveryRequired() if "result" not in response: self._fail("libvirt_execution_timeout") self.interrupt() @@ -449,11 +625,88 @@ class RPCSession: except ExecutionFailure as exc: raise self._fail( exc.error_code, outcome_unknown=exc.outcome_unknown) from None + snapshot_call = self.snapshot_evidence is not None and handle == self._snapshot_handle + if snapshot_call and method in {"fsFreeze", "backupBegin", "fsThaw"}: + evidence = dict(self.snapshot_evidence) + if method == "fsFreeze": + # Never permit a second freeze under the same recovery identity. + if evidence["freeze_ack"]: + raise SnapshotRecoveryRequired() + elif method == "backupBegin": + if not evidence["freeze_ack"] or evidence["backup_state"] != "not_sent": + raise SnapshotRecoveryRequired() + evidence["backup_state"] = "pending" + else: + evidence["thaw_pending"] = True + self.runtime._save_snapshot_evidence(self, evidence) # Full disk cloning is one synchronous libvirt RPC. Its bound is the # storage operation budget, not the short per-query budget. - return self._request(request, timeout=( + result = self._request(request, timeout=( max(0, self.deadline - time.monotonic()) if method == "createXMLFrom" else None)) + if snapshot_call and method in {"fsFreeze", "backupBegin", "fsThaw"}: + evidence = dict(self.snapshot_evidence) + if method == "fsFreeze" and isinstance(result, int) and not isinstance(result, bool) and result > 0: + evidence["freeze_ack"] = True + elif method == "backupBegin" and result in (None, 0): + evidence["backup_state"] = "ack" + elif method == "fsThaw" and isinstance(result, int) and not isinstance(result, bool) and result >= 0: + evidence["thaw_ack"] = True + evidence["thaw_pending"] = False + self.runtime._save_snapshot_evidence(self, evidence) + return result + + def bind_snapshot_recovery(self, domain, *, job_id, volume_name, pool_name, + target_path, runtime_token): + if (not isinstance(job_id, str) or str(uuid.UUID(job_id)) != job_id + or not isinstance(runtime_token, str) + or re.fullmatch(r"[0-9a-f]{32}", runtime_token) is None + or self.snapshot_evidence is not None): + raise SnapshotRecoveryRequired() + self._reference(domain) + evidence = { + "job_id": job_id, "vm_id": domain.UUIDString(), + "volume_name": volume_name, "pool_name": pool_name, + "target_path": target_path, "runtime_id": domain.ID(), + "runtime_token": runtime_token, "worker_pid": self.pid, + "freeze_ack": False, "backup_state": "not_sent", "thaw_ack": False, + "thaw_pending": False, "recovery_state": "not_started", + } + self.runtime._before_write(self, "fsFreeze") + self.runtime._save_snapshot_evidence(self, evidence) + self._snapshot_handle = domain._handle + + def recover_snapshot_thaw(self, *, job_id, vm_id, volume_name): + path, record, evidence = self.runtime._snapshot_recovery_evidence( + self, job_id, vm_id, volume_name) + try: + if evidence["thaw_ack"]: + # A durable receipt never authorizes thawing a later freeze. + return {"thawed": True, "count": 0, "power_state": "unknown"} + evidence["recovery_state"] = "pending" + self.runtime._replace_snapshot_record(path, record) + # All other RPCs on this connection remain subject to the fence. + # The original intent survives even if this one operation succeeds. + try: + result = self._request({"op": "snapshot_thaw", + "handle": self.connection._handle, + "evidence": evidence}) + except SnapshotRecoveryRequired: + # The worker only returns this response before fsThaw begins. + evidence["recovery_state"] = "not_started" + self.runtime._replace_snapshot_record(path, record) + raise + evidence.update(thaw_ack=True, recovery_state="ack") + self.runtime._replace_snapshot_record(path, record) + return result + except (OSError, ExecutionFailure): + # The dedicated recovery session has no ordinary write intent of + # its own. Its persisted pending receipt still means fsThaw may + # have reached the guest, so a transport failure is outcome-unknown. + raise self._fail("libvirt_execution_uncertain", outcome_unknown=True) from None + finally: + with self.runtime._condition: + self.runtime._snapshot_recoveries.discard(self.scope) def qemu_agent_command(self, domain, command, timeout, flags): if self.failure is not None: diff --git a/engine/virsh_panel/libvirt_service.py b/engine/virsh_panel/libvirt_service.py index 45665e81..29077373 100644 --- a/engine/virsh_panel/libvirt_service.py +++ b/engine/virsh_panel/libvirt_service.py @@ -42,7 +42,9 @@ from virsh_panel.cloud_init import ( validate_guest_hostname, ) from virsh_panel.metrics import RawVMStats -from virsh_panel.libvirt_rpc import ExecutionFailure, RemoteObject, get_runtime +from virsh_panel.libvirt_rpc import ( + ExecutionFailure, RemoteObject, SnapshotRecoveryRequired, get_runtime, +) from virsh_panel.resource_locks import RESOURCE_LOCKS, ResourceLockTimeout from virsh_panel.network_roles import ( PUBLIC_LIBVIRT_ALIAS, @@ -871,7 +873,9 @@ class VMService: # local locks first could make an unrelated probe hold the # local VM lock while waiting for that same durable target. with resource_guard(resources, timeout=max( - 0.0, deadline - time.monotonic())): + 0.0, deadline - time.monotonic()), + connection_failure_safe=(current_execution() is None + or current_execution().connection_failure_safe)): with local_guard(): yield except CoordinationError as exc: @@ -892,10 +896,12 @@ class VMService: # unit adapters. Every application factory supplies a scope. conn = libvirt.open(self.uri) if conn is None: - raise VMError("无法连接虚拟化宿主机", 502) + raise VMError("无法连接虚拟化宿主机", 502, + error_code="libvirt_connection_failed") except libvirt.libvirtError as exc: LOG.warning("libvirt connection failed: %s", exc) - raise VMError("无法连接虚拟化宿主机", 502) from exc + raise VMError("无法连接虚拟化宿主机", 502, + error_code="libvirt_connection_failed") from exc except ExecutionFailure as exc: raise LibvirtExecutionError(exc) from None try: @@ -7644,6 +7650,7 @@ class VMService: *, consistency: str = "filesystem", progress: Callable[[str], None] | None = None, + recovery_job_id: str | None = None, ) -> dict[str, Any]: """Create an independent local qcow2 with libvirt push backup. @@ -7746,6 +7753,26 @@ class VMService: "宿主机不支持 QEMU Guest Agent 文件系统冻结", 409, error_code="snapshot_guest_agent_unavailable", ) + if isinstance(domain, RemoteObject): + # Bind recovery to this live QEMU generation and the + # durable panel job before a freeze can reach the host. + try: + runtime_token, _xml, _root = self._ensure_qga_runtime_token( + domain, vm_uuid, + inactive_xml=domain.XMLDesc(libvirt.VIR_DOMAIN_XML_INACTIVE), + live_xml=domain.XMLDesc(0), + ) + domain._session.bind_snapshot_recovery( + domain, job_id=recovery_job_id, + volume_name=volume_name, pool_name=pool.name(), + target_path=target_path, runtime_token=runtime_token, + ) + except (SnapshotRecoveryRequired, _GuestProbeUnavailable, + _GuestProbeUncertain, ValueError): + raise VMError( + "无法持久绑定快照解冻恢复身份,已停止冻结", 409, + error_code="snapshot_guest_agent_unavailable", + ) from None if progress is not None: progress("freezing") freeze_attempted = True @@ -7910,9 +7937,32 @@ class VMService: raise VMError("在线快照创建失败,未保留恢复点", 502) from exc raise - def thaw_guest_filesystems(self, vm_id: str) -> dict[str, Any]: - """Best-effort startup recovery for an interrupted QGA freeze window.""" + def thaw_guest_filesystems(self, vm_id: str, *, recovery_job_id: str | None = None, + snapshot_key: str | None = None) -> dict[str, Any]: + """Recover only the original job's proven freeze/backup boundary. + + This deliberately does not acquire/release the uncertain write claim: + one guarded thaw is a recovery action, not permission to replay a VM + mutation or report the backup complete. + """ vm_uuid = self._uuid(vm_id) + if self.execution_scope is not None: + try: + if (not isinstance(recovery_job_id, str) + or str(uuidlib.UUID(recovery_job_id)) != recovery_job_id): + raise SnapshotRecoveryRequired() + volume_name = snapshot_volume_name(vm_uuid, snapshot_key) + with self._connect(timeout=LIBVIRT_MUTATION_BUDGET) as conn: + result = conn._session.recover_snapshot_thaw( + job_id=recovery_job_id, vm_id=vm_uuid, + volume_name=volume_name) + return {"uuid": vm_uuid, **result} + except (SnapshotRecoveryRequired, ValueError, TypeError, OSError, + libvirt.libvirtError): + raise VMError( + "无法确认原冻结或备份边界,自动解冻已停止;请立即按原虚拟机 UUID 核对并人工解冻", + 409, error_code="snapshot_thaw_recovery_required", + ) from None with ( self._operation_guard(vm_uuid), self._connect(timeout=LIBVIRT_MUTATION_BUDGET) as conn, diff --git a/engine/virsh_panel/vpc.py b/engine/virsh_panel/vpc.py index a698df1b..80fa7477 100644 --- a/engine/virsh_panel/vpc.py +++ b/engine/virsh_panel/vpc.py @@ -3855,8 +3855,20 @@ class VpcCoordinator: guest_mac = None if isinstance(member_nic.get("guest_private_ip_cidr"), str): guest_ip = member_nic.get("guest_private_ip_cidr") + # Configuration may be exact while this member's bridge route has no + # carrier (for example immediately after the last VM stops). Do not + # promote a racing/stale QGA success to runtime readiness through the + # host_config_ok fallback. Other members' routes are independent here. + host_routes = host.get("routes", []) if isinstance(host, Mapping) else [] + member_route_ready = isinstance(host_routes, (list, tuple)) and not any( + isinstance(route, Mapping) + and route.get("destination_cidr") == private_ip_cidr + and route.get("flags") + for route in host_routes + ) active_nic_ok = bool( isinstance(member_nic, Mapping) + and member_route_ready and member_nic.get("xml_verified") is True and member_nic.get("seed_verified") is True and member_nic.get("restart_required") is False diff --git a/engine/virsh_panel/vpc_dataplane.py b/engine/virsh_panel/vpc_dataplane.py index aef2ce0b..f1b74627 100644 --- a/engine/virsh_panel/vpc_dataplane.py +++ b/engine/virsh_panel/vpc_dataplane.py @@ -55,10 +55,12 @@ _DIGEST = re.compile(r"[0-9a-f]{64}\Z") # iproute2 prints host routes without /32 and suppresses dev when its query # already selected that device. Infer only the explicitly supplied filter; -# unfiltered inventories must still contain a unique output device. +# unfiltered inventories must still contain a unique output device. A private +# bridge without a running tap legitimately has no carrier. Only that explicit +# device may retain linkdown as configuration evidence, never runtime readiness. VPC_ROUTE_RECORDS_SH = r''' vc_vpc_route_records() { - awk -v selected_device="${1-}" -v selected_table="${2-main}" ' + awk -v selected_device="${1-}" -v selected_table="${2-main}" -v private_bridge="${3-}" ' function ipv4(value, octets, pos) { if (split(value, octets, ".") != 4) return 0 for (pos=1; pos<=4; pos++) @@ -72,10 +74,11 @@ vc_vpc_route_records() { destination=destination "/32" device=selected_device; table=selected_table; via="-"; metric="0"; proto="" row_bad=0; dev_count=0; table_count=0; via_count=0; metric_count=0 - proto_count=0; scope_count=0; onlink_count=0 + proto_count=0; scope_count=0; onlink_count=0; linkdown_count=0 for (i=2; i<=NF; i++) { field=$i if (field=="onlink") {if (++onlink_count>1) row_bad=1; continue} + if (field=="linkdown") {if (++linkdown_count>1) row_bad=1; continue} if (i==NF) {row_bad=1; break} value=$(++i) if (field=="dev") { @@ -100,10 +103,11 @@ vc_vpc_route_records() { } if (table=="254") table="main" if (device=="" || proto_count!=1 || (onlink_count && via=="-") || + (linkdown_count && (private_bridge=="" || device!=private_bridge)) || (selected_device=="" && table!=selected_table) || ++seen[destination,table]>1) row_bad=1 if (row_bad) {bad=1; next} - printf "%s|%s|%s|%s|%s|%s\n", destination, device, table, via, metric, proto + printf "%s|%s|%s|%s|%s|%s|%s\n", destination, device, table, via, metric, proto, linkdown_count ? "linkdown" : "-" } END {exit bad ? 1 : 0} ' @@ -2476,8 +2480,8 @@ if [ -f "$TXN/state.before" ]; then fi sort -u "$TXN/bridge.routes.allowed" -o "$TXN/bridge.routes.allowed" bridge_routes=$($IP -o -4 -N route show table all dev "$BRIDGE" 2>/dev/null) || exit 71 -bridge_route_records=$(printf '%s\\n' "$bridge_routes" | vc_vpc_route_records "$BRIDGE" main) || exit 71 -while IFS='|' read -r destination device table via metric proto; do +bridge_route_records=$(printf '%s\\n' "$bridge_routes" | vc_vpc_route_records "$BRIDGE" main "$BRIDGE") || exit 71 +while IFS='|' read -r destination device table via metric proto flags; do [ -n "$destination" ] || continue printf '%s\\n' "$destination|$device|$table|$via|$metric|$proto" | grep -Fqx -f "$TXN/bridge.routes.allowed" || exit 71 done </dev/null) && - bridge_route_records=$(printf '%s\\n' "$bridge_routes" | vc_vpc_route_records "$BRIDGE" main) && + bridge_route_records=$(printf '%s\\n' "$bridge_routes" | vc_vpc_route_records "$BRIDGE" main "$BRIDGE") && printf '%s\\n' "$bridge_route_records" | awk -F'|' -v wanted_table="$ROUTE_TABLE" ' NF {{if ($3!=wanted_table) bad=1}} END {{exit bad ? 1 : 0}}'; then @@ -3863,8 +3866,8 @@ fi {bridge_observe} {arp_observe} if route_inventory=$("$IP" -o -4 -N route show table "$ROUTE_TABLE" 2>/dev/null) && - route_tuples=$(printf '%s\\n' "$route_inventory" | vc_vpc_route_records "" "$ROUTE_TABLE"); then - route_records=$(printf '%s\\n' "$route_tuples" | awk -F'|' 'NF {{print $1 "|" $2 "|" $4 "|" $5 "|" $6}}') + route_tuples=$(printf '%s\\n' "$route_inventory" | vc_vpc_route_records "" "$ROUTE_TABLE" "$BRIDGE"); then + route_records=$(printf '%s\\n' "$route_tuples" | awk -F'|' 'NF {{print $1 "|" $2 "|" $4 "|" $5 "|" $6 "|" $7}}') else route_records=invalid fi @@ -3880,7 +3883,7 @@ rule_scope_count=$(printf '%s\\n' "$rule_records" | awk -v expected_to={_q(plan. rule_extra=yes; [ "$rule_scope_count" = 2 ] && rule_extra=no printf '%s\\n' "__VC_VPC_RULE_EXTRA__=$rule_extra" {rule_checks} -route_extra=$(printf '%s\\n' "$route_records" | awk '{{ +route_extra=$(printf '%s\\n' "$route_records" | awk -F'|' '{{ if (length($0) == 0) next count++ {route_extra_awk} @@ -3947,12 +3950,16 @@ def _markers(output: str) -> dict[str, Any]: raise VpcDataplaneError("VPC 回读字段过长", 502, "vpc_readback_invalid") if key == "__VC_VPC_ROUTE__": p = value.split("|") - if len(p) == 6: + if len(p) in {6, 7}: result["routes"].append({ "destination_cidr": p[0], "device": p[1], "table": p[2], "via": None if p[3] == "-" else p[3], "metric": p[4], "owner": DATAPLANE_OWNER_MARKER, - "state": p[5], + "state": p[-1], + # Six-field receipts predate the carrier observation. + # Current scripts always retain the explicit flag field. + "flags": ([] if len(p) == 6 or p[5] == "-" + else [p[5]]), }) elif key == "__VC_VPC_PROXY_IP__": if re.fullmatch(r"[0-9]{1,3}(?:\.[0-9]{1,3}){3}", value): @@ -4472,6 +4479,14 @@ class VpcPrivateHostAdapter: x.get("via"), str(x.get("metric", 100))) for x in parsed.get("routes", []) if x.get("state") == "owned" } + route_flags_valid = all( + x.get("flags", []) == [] + or (x.get("flags") == ["linkdown"] + and x.get("device") == plan.resources.bridge) + for x in parsed.get("routes", []) + ) + route_links_ready = all(not x.get("flags") + for x in parsed.get("routes", [])) expected_rules = { (str(plan.resources.rule_priority), "-", plan.vpc.cidr, str(plan.resources.route_table)), @@ -4516,6 +4531,7 @@ class VpcPrivateHostAdapter: and parsed.get("fabric_nft") == "yes" and expected_routes == actual_routes and actual_routes == actual_route_state + and route_flags_valid and expected_rules == actual_rules and expected_wg_full == actual_wg_full and (plan.fabric_peer_keys is None @@ -4525,6 +4541,7 @@ class VpcPrivateHostAdapter: and all(x.get("config_verified") is True for x in members)) active = bool( host_config_verified + and route_links_ready and all( x["xml_verified"] and x.get("guest_network_verified") is True