From 5783ae618fd2f07d285a17d67cfba57b58932463 Mon Sep 17 00:00:00 2001 From: admin Date: Mon, 12 Oct 2026 14:33:23 +0800 Subject: [PATCH] fix(vpc): persist local neighbors and avoid healthy bridge resets --- docs/VALIDATION.md | 38 ++ docs/VPC_DATAPLANE_ACCEPTANCE.md | 65 +++ engine/tests/test_fabric.py | 3 +- .../tests/test_fabric_libvirt_independence.py | 18 +- engine/tests/test_fabric_local_neighbors.py | 211 ++++++++++ .../tests/test_fabric_nm_bridge_preserve.py | 5 +- engine/tests/test_vpc_apply_receipts.py | 379 ++++++++++++++++++ engine/tests/test_vpc_apply_reentry.py | 17 +- engine/tests/test_vpc_dataplane.py | 3 + .../test_vpc_local_neighbor_transaction.py | 282 +++++++++++++ engine/tests/test_vpc_nft_dataplane.py | 8 +- engine/tests/test_vpc_nm_bridge_preserve.py | 9 +- engine/tests/test_vpc_proxy_rollback.py | 14 +- engine/virsh_panel/fabric.py | 62 ++- engine/virsh_panel/shell_contracts.py | 108 +++++ engine/virsh_panel/vpc_dataplane.py | 361 ++++++++++++++++- 16 files changed, 1552 insertions(+), 31 deletions(-) create mode 100644 engine/tests/test_fabric_local_neighbors.py create mode 100644 engine/tests/test_vpc_apply_receipts.py create mode 100644 engine/tests/test_vpc_local_neighbor_transaction.py diff --git a/docs/VALIDATION.md b/docs/VALIDATION.md index aa93c101..531a1da7 100644 --- a/docs/VALIDATION.md +++ b/docs/VALIDATION.md @@ -1,5 +1,43 @@ # 本版验证记录 +## 2026-10-12:VPC 邻居持久化与双宿主互通 + +- 真机根因:无 IPv4 地址的私网 bridge 发出的 ARP 使用宿主公网源地址, + 来宾私网接口的严格 `rp_filter=1` 与该源地址的公网反向路由不一致,导致 + 邻居解析失败。手工 PERMANENT 邻居能恢复通信,但旧周期 apply 会重新 + down/up bridge 并清掉邻居,不能作为持久修复。 +- 本次将本地成员的精确私网 IP/MAC 纳入受管 `local_neighbor` 状态、事务 + 回滚、Fabric 恢复与回读;升级 helper 契约检查在网络写入前拒绝旧实现。 + 成功应用凭据及新鲜状态核对避免健康周期重复修改网络,XML、seed、QGA + 核验仍照常执行。机制与旧环境迁移步骤见 + [VPC 数据平面验收](VPC_DATAPLANE_ACCEPTANCE.md)。 +- 本地受影响的 12 套 VPC/Fabric 回归 **917 passed / 19 warnings**,耗时 + 295.50 秒;warnings 为现有 Element truthiness 弃用提示。覆盖邻居 + apply/restore/rollback、未知或冲突条目拒绝、旧 helper 门禁、重复协调、 + 回执失效与完整状态漂移。Python AST、12 个渲染脚本的 shell 语法及 + `git diff --check` 通过,独立复核未发现阻断问题。 +- 真机范围为 Debian/ifupdown 与 AlmaLinux/NetworkManager 两台宿主, + 每台各一台 VM。经用户确认,仅对 `10.0.0.1/32` 与 `10.0.0.2/32` + 配置双向 ingress/egress 全协议允许,面板及两台宿主均核实 revision 4。 + 两台 VM 的 QGA 仅增加 `virt_qemu_ga_t` 执行 `ifconfig_exec_t` 文件所需 + 权限,保持 SELinux Enforcing、`virt_qemu_ga_run_unconfined=off` 及 + 私网接口 `rp_filter=1`;该 SELinux 权限按文件类型生效,不限制命令参数。 + 两台宿主调用固定 QGA 路由探针均退出 0,读到真实私网路由。 +- 线上已有其他热修,先备份并核对源文件哈希,只部署本次三个 Engine 文件; + 未 reset/pull 覆盖原有改动。两台宿主在 Fabric 锁下完成受控 helper-only + 升级并保留旧文件;恢复 timer、Engine 与 VPC worker 后,正常后台协调 + 自动迁移两台旧 state,无需改数据库或重存 ACL。未修改 libvirt 启动依赖。 +- 2026-10-12 06:20–06:29 UTC 实测:两侧 state 均含准确的本地邻居记录, + 内核均为 PERMANENT,无活动事务 journal;各 190 秒 `ip monitor link` + 均无事件,跨多个正常协调周期未再出现 bridge 重置。Fabric timer 均 active。 + Engine 只读状态中 Fabric、VPC、两台 host 均 active,私网/配置/传输核验 + 成功且无错误;用户页面两个成员均为“已生效”。 +- 双向各 **180/180 ICMP**,丢包率 0%;同步就绪后双向各 **120/120 TCP** + 和 **120/120 UDP** 成功。首轮一侧先于对端监听开始统计,出现 10 次启动 + 阶段失败,延长就绪等待后完整重测通过;两个临时监听均已自动退出。 +- 本次未进行宿主重启、同宿主双 VM、拒绝规则或跨租户负向包测试;这些仍需 + 独立真机验收,不能由上述允许通信及本地回归推定通过。 + ## 2026-10-10:firewalld 支持退役 - 当前 Fabric/VPC 宿主机只支持 `native_nftables`。能力探测只在 `nft` 命令可用、 diff --git a/docs/VPC_DATAPLANE_ACCEPTANCE.md b/docs/VPC_DATAPLANE_ACCEPTANCE.md index 5be177f3..d4144347 100644 --- a/docs/VPC_DATAPLANE_ACCEPTANCE.md +++ b/docs/VPC_DATAPLANE_ACCEPTANCE.md @@ -31,6 +31,36 @@ `/etc/virt-cloud`、`/run/virt-cloud`、`/etc/nftables.d` 和 `sysctl.d` 不是符号链; - `br0`、管理接口、默认路由和未知 nft 规则的快照已经保存。 +## 旧 helper 和邻居状态的升级顺序 + +新增 `local_neighbor|IPv4|MAC` 前,所有相关宿主机上的 +`/usr/local/libexec/virt-cloud-wg-fabric` 必须已包含 +`# vpc-local-neighbor-contract-v1`。旧 helper 会拒绝新状态记录;仅升级面板代码 +不足以完成宿主机迁移。Fabric observer 会把缺少该标记的同摘要 helper 判为需要 +修复,但旧 state 未记录邻居、运行时却存在普通邻居时,完整 reconcile 仍可能因 +无法证明这些绑定而停止恢复。因此不能把普通公开 reconcile 当作所有旧环境的 +自动迁移入口,更不能通过忽略未知 MAC 或删除邻居绕过检查。 + +已有旧 state 的环境采用受控两阶段升级: + +1. 暂停面板 VPC worker、引擎后台协调和相关 Fabric timer,确认在途任务已结束。 + 有未确认操作、事务目录或持久锁时,先保留并核对原操作证据,不直接清理。 +2. 固定当前 Fabric desired 摘要、宿主机标识与 SSH 指纹,保存旧 helper 的 SHA256、 + owner/权限及相关配置快照。从已审阅版本按同一计划生成对应宿主机的新 helper。 +3. 在维护互斥和 Fabric 锁内,确认旧 helper 为 root 所有的普通文件、权限 `0700`, + owner、计划摘要和旧 SHA256 均精确匹配,且不存在活动网络事务。备份旧文件, + 验证新文件语法、契约标记及审阅时的新 SHA256,再仅原子替换 helper。此阶段 + 不执行组网、不改 unit、drop-in、libvirt 启动项或启动依赖,也不生成新密钥。 +4. 所有相关宿主机的 helper 更新并回读后,恢复原 Fabric timer、引擎与 VPC worker。 + Fabric readiness 要求 timer 为 active;持续暂停 timer 会关闭 VPC 的 Fabric gate, + 因而不能等待新 state 出现后才启动它。引擎自身会周期核对已生效 VPC, + 不需要为迁移重新保存相同 ACL 或手工修改数据库任务。通过原 VPC 操作推进当前 + desired revision/digest。由正常应用事务写入邻居绑定、恢复运行时 `PERMANENT` 项并 + 完整回读;未知绑定仍须停止处理。不得手写 state 或数据库成功状态代替迁移。 +5. 新 state 和运行时证据一致后,确认原有 worker/timer 调度持续正常,检查多个 + 后台周期中的桥、tap、路由和邻居稳定性,再执行同机、跨机流量验收。 + helper 发布成功本身不代表 VPC 或来宾网络已经可用。 + ## 成员接入和回读 对一个跨两台宿主机的 VPC,至少放置两个成员和一个 ACL,逐台检查: @@ -83,6 +113,11 @@ observe 还必须以计划携带的 `fabric_plan_digest` 重建并核验 Fabric table grammar、 owner/digest、精确 tenant set 和完整 stateless snapshot;Fabric revision 或任一租户 AllowedIPs 漂移时,即使本 VPC 三张表未变也不得报告 active。 + 每个本机成员还必须存在精确的 `private_ip → private_mac` 普通 IPv4 邻居项, + 状态为 `PERMANENT`;它与 proxy ARP 项是两类独立证据。绑定写入受管 state 的 + `local_neighbor` 记录,并在网桥 down/up、回滚和 Fabric 恢复后重新回读。 + 缺失、错误 MAC、未解析或动态邻居状态不能替代这项证明。旧 state 可读取以便 + 迁移,但必须经当前计划应用并验证后才能重新报告可用。 6. 从同机和跨机成员分别验证允许的 TCP/UDP/ICMP 流量、ACL deny、伪造源 MAC/IP、 IPv6、广播/多播和访问宿主机自身地址。验证结果同时记录 guest route、宿主机 route、WireGuard handshake 和 nft counter,不能只看页面的 `active`。 @@ -97,6 +132,20 @@ NetworkManager 原连接 UUID。隔离前先记录路由、proxy ARP 和 sysctl 空桥或冷启动仍按后述管理器流程恢复。验收重复核对与失败回滚时,必须比较操作前后 的 tap master、桥 ifindex、连接 UUID、路由和 proxy 项,不能只检查接口 UP。 +成功应用凭据只用于减少相同计划的重复写入。它在当前引擎进程内有界保存,绑定 +宿主机 SSH 配置代次、目标与指纹、完整应用脚本摘要和成功事务中取得的三张 nft +表摘要。后续每次尝试复用都必须重新读取宿主机配置、持久文件、Fabric/WireGuard、 +路由、规则、proxy ARP、普通永久邻居及完整 nft 摘要;全部一致才跳过再次应用和 +bridge down/up。SSH 超时、非零退出或无效凭据等已传播的读取异常会丢弃凭据并返回 +失败,不在该次请求中改写网络;回读不匹配、进程重启、缓存淘汰或计划/代码/SSH +代次变化则需要重新完成受保护的应用。部分旧观测分支仍将子命令失败汇总为 +不匹配,不能把这项优化描述为能够区分所有底层读取故障与真实配置漂移。 + +该凭据不是 `active`、`usable` 或来宾核验结果;复用路径仍返回 `applying/pending`, +后续 XML、seed、QGA 链路、地址和路由核验照常执行。验收既要证明多个协调周期不再 +无故扰动健康网桥,也要证明邻居或 nft 漂移会触发安全修复、读取超时不会派发写入, +且停机或 QGA 不可用的成员不会因已有应用凭据被标记为可用。 + AlmaLinux 10:执行 `nmcli connection reload`、`systemctl restart NetworkManager`、 panel-owned Fabric service/timer 修复测试和宿主机重启。不要用 stock `nftables.service` 的 start/reload/stop 作为验收步骤。每一步都要确认 private bridge @@ -176,6 +225,14 @@ VPC ACL 默认拒绝,未配置允许规则时,同机和跨机成员都不会 需要互通时,通过原 VPC ACL 接口保存明确的源、目标、方向和协议规则,并核对 双向请求与回复。不要临时插入无归属 nft accept 规则或停用防火墙来宣告恢复。 +ACL 已允许仍不通时,同时抓取 WireGuard 和私网 tap 上的 ICMP/ARP。无 IPv4 地址的 +私网桥可能选择宿主机公网地址作为 ARP 请求源;来宾启用严格 `rp_filter` 时,该源 +的反向路由经公网接口,ARP 请求可能被丢弃。此时转发规则命中 accept 也不能证明 +下一跳已解析。修复使用计划绑定的本机成员永久邻居,不改来宾全局或私网接口的 +反向路径校验。仅手工运行 `ip neigh replace ... nud permanent` 不构成持久修复: +网桥被置为 DOWN 后内核会删除这些项,应用与恢复流程必须重新建立并验证。 +验收应覆盖多个后台协调周期,并比较同机与跨机 TCP、UDP、ICMP,不能只记录一次 ping。 + “地址可见”和“来宾核验成功”也不同。当前 QGA 核验同时要求私网 MAC、链路状态、 `/32` 地址和精确 scope-link 路由。启用 `guest-exec`/`guest-exec-status` 后,SELinux 仍可能拒绝代理执行 `/usr/sbin/ip`;命令管道末尾的 `head` 成功会使 exitcode 为 0, @@ -183,6 +240,14 @@ VPC ACL 默认拒绝,未配置允许规则时,同机和跨机成员都不会 exitcode 标记成功,也不能默认关闭 SELinux 或让整个 QGA 不受限制。来宾执行权限 和 VPC ACL 是两个独立检查项;改变其访问范围前应由运维明确批准。 +若 AVC 已证实代理域为 `virt_qemu_ga_t`、`/usr/sbin/ip` 的标签为 +`ifconfig_exec_t`,可以在获准的来宾中安装只针对这两个类型的本地 SELinux 模块, +允许文件 `getattr open read map execute execute_no_trans`。安装前保存模块列表与 +原代理配置;安装后必须保持 `getenforce=Enforcing`、 +`virt_qemu_ga_run_unconfined=off`,再从宿主实际调用 QGA 固定路由探针,检查退出码、 +标准错误和完整路由内容。该类型许可不能限制命令参数,不应当作任意镜像通用策略 +或自动下发给未经批准的来宾;标签不同的发行版须重新依据 AVC 判断。 + ## 发布和规模边界 - 先应用包含 VPC evidence version 字段的数据库 schema,再发布 PHP/worker,最后发布 diff --git a/engine/tests/test_fabric.py b/engine/tests/test_fabric.py index f0a10166..edce6ce0 100644 --- a/engine/tests/test_fabric.py +++ b/engine/tests/test_fabric.py @@ -339,7 +339,8 @@ def test_observe_artifact_state_accepts_only_kind_specific_header( f"{first_line}\n" "# virt-cloud-wg-fabric-v1\n" f"# fabric-plan-digest={current.digest}\n" - "# libvirt-startup-independent-v1\n", + "# libvirt-startup-independent-v1\n" + "# vpc-local-neighbor-contract-v1\n", encoding="utf-8", ) artifact.chmod(int(mode, 8)) diff --git a/engine/tests/test_fabric_libvirt_independence.py b/engine/tests/test_fabric_libvirt_independence.py index ac5dc350..f0459e32 100644 --- a/engine/tests/test_fabric_libvirt_independence.py +++ b/engine/tests/test_fabric_libvirt_independence.py @@ -138,13 +138,15 @@ def test_new_scripts_neither_install_gate_nor_control_libvirt_units(mode): assert "nft_plan_owned" in apply -def startup_files(root, current, *, legacy=False, gate=True): +def startup_files(root, current, *, legacy=False, gate=True, legacy_neighbors=False): from virsh_panel.fabric import FABRIC_STARTUP_ISOLATION_MARKER root.mkdir(parents=True, exist_ok=True) for name, content, mode in (("helper", _BOOT_HELPER, 0o700), ("unit", _UNIT, 0o644)): content = content.replace("__VC_FABRIC_DIGEST__", current.digest) if legacy: content = content.replace(FABRIC_STARTUP_ISOLATION_MARKER + "\n", "") + if legacy_neighbors and name == "helper": + content = content.replace("# vpc-local-neighbor-contract-v1\n", "") target = root / name target.write_text(content) target.chmod(mode) @@ -193,7 +195,14 @@ def test_dangling_generator_still_prevents_exact_startup_readback(tmp_path): assert startup_readback(tmp_path, current) == ["foreign", "owned"] -def test_explicit_reconcile_upgrades_same_plan_then_skips_current_hosts(tmp_path, monkeypatch): +def test_same_plan_legacy_neighbor_helper_requires_upgrade_without_changing_unit(tmp_path): + current = plan() + startup_files(tmp_path, current, gate=False, legacy_neighbors=True) + assert startup_readback(tmp_path, current) == ["foreign", "owned"] + + +@pytest.mark.parametrize("upgrade", ["startup", "neighbors"]) +def test_explicit_reconcile_upgrades_same_plan_then_skips_current_hosts(tmp_path, monkeypatch, upgrade): import asyncio from virsh_panel.fabric import FabricCoordinator, FABRIC_STARTUP_ISOLATION_MARKER from test_fabric import AsyncCoordinatorAdapter, MemoryStore, Registry, _async_components, plan_payload @@ -215,6 +224,7 @@ def test_explicit_reconcile_upgrades_same_plan_then_skips_current_hosts(tmp_path def dispatch_apply(self, actual_plan, host, keys, operation_id=None): rendered = _apply_script(actual_plan, host, keys) assert FABRIC_STARTUP_ISOLATION_MARKER in rendered + assert "# vpc-local-neighbor-contract-v1" in rendered assert 'record_desired_absent gate_generator' in rendered startup_files(self.manager.startup_root, actual_plan, gate=False) return super().dispatch_apply(actual_plan, host, keys, operation_id) @@ -234,7 +244,9 @@ def test_explicit_reconcile_upgrades_same_plan_then_skips_current_hosts(tmp_path # Reproduce an upgrade: identical persisted successful plan and keys, # but both hosts still have the prior helper/unit/generator contract. for component in components.values(): - startup_files(component.manager.startup_root, current, legacy=True) + startup_files(component.manager.startup_root, current, + legacy=upgrade == "startup", gate=upgrade == "startup", + legacy_neighbors=upgrade == "neighbors") admitted = await coordinator.enqueue_reconcile(current.fabric_id) assert admitted["operation"]["id"] != initial["operation"]["id"] await wait() diff --git a/engine/tests/test_fabric_local_neighbors.py b/engine/tests/test_fabric_local_neighbors.py new file mode 100644 index 00000000..a442f1ba --- /dev/null +++ b/engine/tests/test_fabric_local_neighbors.py @@ -0,0 +1,211 @@ +"""Execute shared neighbor ownership checks and the durable Fabric parser.""" +from __future__ import annotations + +import json +import os +from pathlib import Path +import shlex +import subprocess +import sys + +import pytest + +from test_vpc_nft_dataplane import _plan +from virsh_panel.fabric import _BOOT_HELPER, _VPC_STATE_LOADER +from virsh_panel.shell_contracts import LOCAL_NEIGHBOR_SH +from virsh_panel.vpc_dataplane import _routes, _state + + +MAC = '02:00:00:00:00:10' +PAIR = '10.44.0.10|' + MAC +BRIDGE = 'vcv-0123456789a' + + +def _run(body, *, env=None, shell='/bin/sh'): + return subprocess.run([shell, '-s'], input='set -eu\n' + body, + text=True, capture_output=True, env=env) + + +@pytest.mark.parametrize('raw,expected', [ + ('', ''), + (f'10.44.0.10 lladdr {MAC} PERMANENT\n', PAIR + '|PERMANENT\n'), + (f'10.44.0.10 dev {BRIDGE} lladdr {MAC.upper()} STALE\n', PAIR + '|STALE\n'), + ('10.44.0.10 INCOMPLETE\n', '10.44.0.10|-|INCOMPLETE\n'), + ('10.44.0.10 FAILED\n', '10.44.0.10|-|FAILED\n'), + ('224.0.0.1 lladdr 01:00:5e:00:00:01 NOARP\n', ''), + (f'10.44.0.10 lladdr {MAC} PERMANENT router\n', None), + (f'10.44.0.10 lladdr {MAC} PERMANENT extern_learn\n', None), + ('10.44.0.10 proxy\n', None), + ('10.44.0.10 lladdr 03:00:00:00:00:10 PERMANENT\n', None), + ('10.44.0.10 lladdr 00:00:00:00:00:00 PERMANENT\n', None), + ('10.44.0.10 PERMANENT\n', None), + (f'10.44.0.10 lladdr {MAC} STALE REACHABLE\n', None), + (f'10.44.0.10 lladdr {MAC} STALE\n10.44.0.10 FAILED\n', None), + (f'10.44.0.999 lladdr {MAC} STALE\n', None), + (f'010.44.0.10 lladdr {MAC} STALE\n', None), +]) +def test_neighbor_records_are_strict_and_empty_is_valid(raw, expected): + script = LOCAL_NEIGHBOR_SH + '\n' + ''' +ip_fixture() { [ "$*" = "-o -4 neigh show dev ''' + BRIDGE + '''" ] || return 90 + printf '%s' ''' + shlex.quote(raw) + ''' +} +IP=ip_fixture +vc_local_neighbor_records ''' + BRIDGE + result = _run(script) + assert (result.returncode == 0) == (expected is not None), result.stderr + if expected is not None: + assert result.stdout == expected + + +@pytest.mark.parametrize('initial,success', [ + ([], True), + ([PAIR + '|STALE'], True), + (['10.44.0.10|-|FAILED'], True), + (['10.44.0.10|-|INCOMPLETE'], True), + ([PAIR + '|PERMANENT'], True), + (['10.44.0.10|02:00:00:00:00:99|PERMANENT'], False), + ([PAIR + '|PERMANENT', '10.44.0.99|02:00:00:00:00:99|STALE'], False), +]) +@pytest.mark.parametrize('shell', ['/bin/sh', '/bin/bash']) +def test_restore_missing_local_next_hop_without_overwriting_foreign(tmp_path, initial, success, shell): + state = tmp_path / 'neighbors.json' + state.write_text(json.dumps(initial)) + calls = tmp_path / 'calls' + calls.write_text('') + tool = tmp_path / 'ip' + tool.write_text('#!' + sys.executable + '\n' + r''' +import json, os, sys +from pathlib import Path +p=Path(os.environ['NEIGHBORS']); rows=json.loads(p.read_text()); args=sys.argv[1:] +if args == ['-o','-4','neigh','show','dev',os.environ['BRIDGE']]: + for row in rows: + ip,mac,nud=row.split('|') + print(ip + ('' if mac=='-' else ' lladdr '+mac) + ' ' + nud) +elif len(args)==10 and args[:3]==['-4','neigh','replace']: + assert args[4]=='lladdr' and args[6:]==['nud','permanent','dev',os.environ['BRIDGE']] + with Path(os.environ['CALLS']).open('a') as out: out.write(json.dumps(args)+'\n') + rows=[row for row in rows if row.split('|')[0]!=args[3]] + rows.append(args[3]+'|'+args[5]+'|PERMANENT'); p.write_text(json.dumps(rows)) +else: raise SystemExit(97) +''') + tool.chmod(0o700) + env = dict(os.environ, NEIGHBORS=str(state), CALLS=str(calls), BRIDGE=BRIDGE) + result = _run('IP=' + shlex.quote(str(tool)) + '\n' + LOCAL_NEIGHBOR_SH + + '\nvc_local_neighbor_ensure ' + BRIDGE + ' ' + shlex.quote(PAIR), + env=env, shell=shell) + assert (result.returncode == 0) == success, result.stderr + if success: + assert json.loads(state.read_text()) == [PAIR + '|PERMANENT'] + else: + assert json.loads(state.read_text()) == initial + assert calls.read_text() == '' + + +@pytest.mark.parametrize('mode', ['current', 'legacy', 'wrong_mac', 'multicast_mac', + 'duplicate', 'remote_ip', 'service_ip', 'outside_cidr', 'extra_field', 'partial']) +@pytest.mark.parametrize('shell', ['/bin/sh', '/bin/bash']) +def test_fabric_loader_binds_neighbors_only_to_local_member_direct_routes(tmp_path, mode, shell): + plan = _plan() + lines = [line for line in _state(plan, _routes(plan)).splitlines() + if not line.startswith('local_neighbor|')] + entry = 'local_neighbor|10.44.0.10|' + MAC + if mode == 'wrong_mac': + entry = entry.replace(MAC, '02:xx:00:00:00:10') + elif mode == 'multicast_mac': + entry = entry.replace(MAC, '03:00:00:00:00:10') + elif mode == 'remote_ip': + entry = entry.replace('10.44.0.10', '10.44.0.11') + elif mode in ('service_ip', 'outside_cidr'): + ip = '10.44.0.99' if mode == 'service_ip' else '10.45.0.10' + entry = entry.replace('10.44.0.10', ip) + lines.append(f'route|{ip}/32|{plan.resources.bridge}|{plan.resources.route_table}|10.44.0.10|100') + elif mode == 'extra_field': + entry += '|' + elif mode == 'partial': + lines.append(f'route|10.44.0.20/32|{plan.resources.bridge}|{plan.resources.route_table}|-|100') + if mode != 'legacy': + lines.append(entry) + if mode == 'duplicate': + lines.append(entry) + states = tmp_path / 'states' + states.mkdir(mode=0o700) + path = states / (plan.resources.token + '.state') + path.write_text('\n'.join(lines) + '\n') + path.chmod(0o600) + loader = (_VPC_STATE_LOADER + .replace('__VC_FABRIC_HOST_ID__', plan.host_id) + .replace('__VC_FABRIC_ID__', plan.vpc.fabric_id) + .replace('__VC_FABRIC_NETWORK__', '100.64.0.0/10') + .replace('__VC_VPC_STATE_OBSERVE__', 'yes')) + script = loader + '\n' + '\n'.join([ + 'VPC_STATE_DIR=' + shlex.quote(str(states)), + 'VPC_PEER_FILE=' + shlex.quote(str(tmp_path / 'peers')), + 'VPC_ROUTE_FILE=' + shlex.quote(str(tmp_path / 'routes')), + 'VPC_TENANT_FILE=' + shlex.quote(str(tmp_path / 'tenants')), + 'FIREWALL_MODE=native_nftables', + ]) + r''' +fixture_stat() { + case "$2" in + '%u:%a') if [ -d "$3" ]; then printf '0:700\n'; else printf '0:600\n'; fi ;; + '%s') wc -c < "$3" ;; + *) return 9 ;; + esac +} +STAT=fixture_stat +vpc_state_load +printf '%s\n' "$VPC_STATE_STATUS" +''' + result = _run(script, shell=shell) + assert (result.returncode == 0) == (mode in ('current', 'legacy')), result.stderr + if mode in ('current', 'legacy'): + assert result.stdout == 'owned\n' + + +def test_fabric_boot_proves_neighbors_before_quarantine_and_restores_before_open(): + emergency = _BOOT_HELPER[_BOOT_HELPER.index('vc_vpc_emergency_down() {'):] + assert emergency.index('vc_local_neighbor_preflight') < emergency.index('"$NMCLI" device set') + assert emergency.index('vc_local_neighbor_preflight') < emergency.index('vc_vpc_bridge_force_down') + recovery = _BOOT_HELPER[_BOOT_HELPER.index('recover_vpc_dataplanes() {'):] + restore = recovery.index('vc_local_neighbor_ensure "$vpc_bridge"') + assert recovery.index('vpc_expected_marker_unique') < restore + assert recovery.index('vc_ip_rule_ensure') < restore + assert restore < recovery.index('vc_vpc_nm_preserve_recover "$vpc_bridge"') + assert restore < recovery.index('vpc_local_neighbors_actual=$(vc_local_neighbor_records') + + +@pytest.mark.parametrize('shell', ['/bin/sh', '/bin/bash']) +@pytest.mark.parametrize('scenario,raw', [ + ('foreign_ip', '10.44.0.99 lladdr 02:00:00:00:00:99 STALE'), + ('foreign_mac', '10.44.0.10 lladdr 02:00:00:00:00:99 PERMANENT'), + ('legacy_failed', '10.44.0.10 FAILED'), + ('legacy_permanent', f'10.44.0.10 lladdr {MAC} PERMANENT'), +]) +def test_emergency_owned_bridge_stays_isolated_after_neighbor_drift(tmp_path, shell, scenario, raw): + from test_fabric_nm_bridge_preserve import _fixture, TOKEN, BRIDGE as owned_bridge + + body, env = _fixture(tmp_path, 'healthy') + tool = tmp_path / 'bin' / 'ip' + tool.write_text(tool.read_text().replace( + "elif args==['-o','-4','neigh','show','dev',b]:pass", + "elif args==['-o','-4','neigh','show','dev',b]:print(" + repr(raw) + ")")) + if not scenario.startswith('legacy_'): + state = tmp_path / 'state' / (TOKEN + '.state') + state.write_text(state.read_text() + 'local_neighbor|' + PAIR + '\n') + journal = tmp_path / 'evidence.txn' + journal.write_text('keep incomplete operation evidence\n') + # A failed preflight must not permit the later recovery stage, but the + # owner-proven bridge must still be quarantined. DOWN may clear neighbors; + # the helper performs no explicit neighbor replacement/deletion/flush. + body += 'vc_vpc_emergency_down\nprintf recovered\n' + result = _run(body, env=env, shell=shell) + assert result.returncode != 0, result.stderr + assert 'recovered' not in result.stdout + runtime = json.loads((tmp_path / 'runtime.json').read_text()) + assert not runtime['up'] + assert runtime['ports'] == ['vnet4'] and runtime['index'] == 19 + calls = [json.loads(line) for line in (tmp_path / 'calls').read_text().splitlines()] + assert ['ip', 'link', 'set', 'dev', owned_bridge, 'down'] in calls + assert not any(call[0] == 'ip' and 'neigh' in call and + any(verb in call for verb in ('replace', 'delete', 'del', 'flush')) + for call in calls) + assert journal.read_text() == 'keep incomplete operation evidence\n' diff --git a/engine/tests/test_fabric_nm_bridge_preserve.py b/engine/tests/test_fabric_nm_bridge_preserve.py index 839497fd..ef2288cf 100644 --- a/engine/tests/test_fabric_nm_bridge_preserve.py +++ b/engine/tests/test_fabric_nm_bridge_preserve.py @@ -10,7 +10,7 @@ from pathlib import Path import pytest from virsh_panel.fabric import _BOOT_HELPER -from virsh_panel.shell_contracts import BRIDGE_PORT_RECORDS_SH +from virsh_panel.shell_contracts import BRIDGE_PORT_RECORDS_SH, LOCAL_NEIGHBOR_SH VPC_ID='vpc-preserve-test' TOKEN=hashlib.sha256(VPC_ID.encode()).hexdigest()[:11] @@ -28,6 +28,7 @@ if name=='stat': elif name=='sha256sum':print(hashlib.sha256(sys.stdin.buffer.read()).hexdigest()) elif name=='ip': if args==['link','show','dev',b]:pass + elif args==['-o','-4','neigh','show','dev',b]:pass elif args==['-d','link','show','dev',b]:print('bridge forward_delay 1500 stp_state 0') elif args==['-o','link','show','dev',b]:print(str(s['index'])+': '+b+': mtu 1500') elif args==['-o','link','show','master',b]: @@ -73,7 +74,7 @@ def _fixture(tmp_path,scenario): (tmp_path/'calls').write_text('') start=_BOOT_HELPER.index('vc_bridge_port_names() {') end=_BOOT_HELPER.index('\nvc_vpc_emergency_down || exit 71',start) - helpers=BRIDGE_PORT_RECORDS_SH+'\n'+_BOOT_HELPER[start:end] + helpers=BRIDGE_PORT_RECORDS_SH+'\n'+LOCAL_NEIGHBOR_SH+'\n'+_BOOT_HELPER[start:end] helpers=helpers.replace('/run/virt-cloud/',str(tmp_path)+'/').replace('/etc/virt-cloud/vpc-dataplane',str(states)).replace('__VC_MAX_VPC_STATE_FILES__','256') env=dict(os.environ,ROOT=str(tmp_path),BRIDGE=BRIDGE,UUID=UUID,SCENARIO=scenario,VPC_ID=VPC_ID) body='set -eu\n'+'\n'.join(name+'='+shlex.quote(str(tools/tool)) for name,tool in [('IP','ip'),('NMCLI','nmcli'),('STAT','stat'),('SHA256','sha256sum')]) diff --git a/engine/tests/test_vpc_apply_receipts.py b/engine/tests/test_vpc_apply_receipts.py new file mode 100644 index 00000000..eeadbee5 --- /dev/null +++ b/engine/tests/test_vpc_apply_receipts.py @@ -0,0 +1,379 @@ +"""Successful apply receipts avoid disrupting an unchanged VPC bridge.""" +from __future__ import annotations + +from concurrent.futures import ThreadPoolExecutor +from dataclasses import replace +import hashlib +import os +from pathlib import Path +import re +import shlex +import subprocess +import sys + +import pytest + +from test_vpc_dataplane import _active_observation_markers, _plan +from virsh_panel import vpc_dataplane as dp + +SHELLS = [p for p in ('/bin/sh', '/bin/dash', '/bin/bash') if Path(p).exists()] +RECEIPT = ','.join(c * 64 for c in 'abc') + + +@pytest.fixture(autouse=True) +def empty_receipts(): + with dp._APPLY_RECEIPT_LOCK: + dp._APPLY_RECEIPTS.clear() + yield + with dp._APPLY_RECEIPT_LOCK: + dp._APPLY_RECEIPTS.clear() + + +@pytest.fixture +def plan(): + p = _plan() + return replace(p, fabric_peer_keys=frozenset(x.public_key for x in p.peers)) + + +class Adapter(dp.VpcPrivateHostAdapter): + def __init__(self, plan, **profile): + super().__init__(None, { + 'ssh_target': 'owned', 'ssh_config_file': '/generation-a/config', + 'host': '192.0.2.1', 'port': 22, 'fingerprint': 'host-key-a', **profile, + }) + self.plan = plan + self.calls = [] + self.observe_error = None + self.apply_error = None + self.observation = '\n'.join(_active_observation_markers(plan) + [ + '__VC_VPC_FIREWALL_MODE__=native_nftables', + '__VC_VPC_FIREWALL_STATE__=owned', + '__VC_VPC_STATE_FIREWALL_MODE__=native_nftables', + '__VC_VPC_WG_PEER_SET__=exact', + '__VC_VPC_HELPER_CONTRACT__=owned', + '__VC_VPC_BRIDGE_ADMIN__=up', + '__VC_VPC_NFT_FILE_EXACT__=yes', + '__VC_VPC_STATE_FILE_EXACT__=yes', + '__VC_VPC_PERSISTENCE_EXACT__=yes', + f'__VC_VPC_NFT_RECEIPT__={RECEIPT}', + ]) + self.applied = '\n'.join([ + '__VC_VPC_APPLIED__=yes', + f'__VC_VPC_OWNER__={dp.DATAPLANE_OWNER_MARKER}', + f'__VC_VPC_REVISION__={plan.desired_revision}', + f'__VC_VPC_DIGEST__={plan.desired_digest}', + f'__VC_VPC_NFT_RECEIPT__={RECEIPT}', + ]) + + def _deployment_contract(self): + return ('networkmanager_nmcli', '/usr/sbin/ifup', '/usr/sbin/ifdown', + '/usr/sbin/ifquery', 'native_nftables') + + def _run(self, script): + if '__VC_VPC_APPLIED__=yes' in script: + self.calls.append('apply') + if self.apply_error: + raise self.apply_error + return self.applied + if '__VC_VPC_REMOVED__=yes' in script: + self.calls.append('remove') + return '__VC_VPC_REMOVED__=yes' + self.calls.append('observe') + if self.observe_error: + raise self.observe_error + return self.observation + + +def test_success_is_reused_across_adapter_instances_without_activating_member(plan): + first = Adapter(plan) + assert first.apply(plan)['state'] == 'applying' + assert first.calls == ['apply'] + second = Adapter(plan) + result = second.apply(plan) + assert second.calls == ['observe'] + assert result == {'state': 'applying', 'readback': 'pending', 'rollback': 'none', + 'owner': dp.DATAPLANE_OWNER_MARKER} + public = second.observe(plan) + assert public['state'] != 'active' + assert public['config_verified'] is False + assert public['member_nics'][0]['xml_verified'] is False + assert second.calls == ['observe', 'observe'] + + +@pytest.mark.parametrize(('old', 'new'), [ + ('LOCAL_NEIGHBOR_STATE__=owned', 'LOCAL_NEIGHBOR_STATE__=missing'), + ('BRIDGE_ADMIN__=up', 'BRIDGE_ADMIN__=down'), + ('HELPER_CONTRACT__=owned', 'HELPER_CONTRACT__=missing'), + ('NFT_FILE_EXACT__=yes', 'NFT_FILE_EXACT__=no'), + ('STATE_FILE_EXACT__=yes', 'STATE_FILE_EXACT__=no'), + ('PERSISTENCE_EXACT__=yes', 'PERSISTENCE_EXACT__=no'), + ('FIREWALL_STATE__=owned', 'FIREWALL_STATE__=mismatch'), + ('PERSISTENCE__=yes', 'PERSISTENCE__=no'), + ('BRIDGE_BOUNDARY__=yes', 'BRIDGE_BOUNDARY__=no'), + ('FABRIC_NFT__=yes', 'FABRIC_NFT__=no'), + ('WG_PEER_SET__=exact', 'WG_PEER_SET__=extra'), + ('RP_FILTER__=2', 'RP_FILTER__=1'), + ('ROUTE_EXTRA__=no', 'ROUTE_EXTRA__=yes'), + ('RULE_EXTRA__=no', 'RULE_EXTRA__=yes'), + ('PROXY_STATE__=owned', 'PROXY_STATE__=missing'), + (f'NFT_RECEIPT__={RECEIPT}', 'NFT_RECEIPT__=' + ','.join(c * 64 for c in 'abd')), + ('NFT_RECEIPT__=', 'UNKNOWN_RECEIPT__='), + ('FIREWALL_MODE__=', 'UNKNOWN_FIREWALL_MODE__='), +]) +def test_every_host_gate_or_full_nft_drift_requires_protected_repair(plan, old, new): + adapter = Adapter(plan) + adapter.apply(plan) + assert old in adapter.observation + adapter.observation = adapter.observation.replace(old, new) + adapter.apply(plan) + assert adapter.calls == ['apply', 'observe', 'apply'] + + +def test_explicit_route_linkdown_never_skips(plan): + adapter = Adapter(plan) + adapter.apply(plan) + adapter.observation = re.sub( + r'(__VC_VPC_ROUTE__=[^\n]+)\|owned', r'\1|linkdown|owned', + adapter.observation) + adapter.apply(plan) + assert adapter.calls == ['apply', 'observe', 'apply'] + + +@pytest.mark.parametrize('error', [TimeoutError('read timed out'), + dp.VpcDataplaneError('read failed', 502, 'read')]) +def test_read_exception_invalidates_receipt_without_mutation(plan, error): + adapter = Adapter(plan) + adapter.apply(plan) + adapter.observe_error = error + with pytest.raises(type(error)): + adapter.apply(plan) + assert adapter.calls == ['apply', 'observe'] + assert not dp._APPLY_RECEIPTS + + +@pytest.mark.parametrize('change', [ + lambda s: s.replace('APPLIED__=yes', 'APPLIED__=no'), + lambda s: s.replace('OWNER__=virt-cloud-vpc-dataplane-v1', 'OWNER__=foreign'), + lambda s: s.replace('NFT_RECEIPT__=', 'MISSING_RECEIPT__='), + lambda s: s.replace(RECEIPT, 'malformed'), + lambda s: s + f'\n__VC_VPC_NFT_RECEIPT__={RECEIPT}', + lambda s: s + '\n__VC_VPC_APPLIED__=yes', + lambda s: s.replace('REVISION__=', 'MISSING_REVISION__='), + lambda s: s.replace('DIGEST__=', 'MISSING_DIGEST__='), +]) +def test_ambiguous_or_incomplete_apply_never_seeds_skip(plan, change): + adapter = Adapter(plan) + adapter.applied = change(adapter.applied) + try: + adapter.apply(plan) + except dp.VpcDataplaneError: + pass + assert not dp._APPLY_RECEIPTS + + +@pytest.mark.parametrize(('key', 'value'), [ + ('ssh_config_file', '/generation-b/config'), ('host', '192.0.2.2'), + ('port', 2222), ('fingerprint', 'other-key'), ('ssh_target', 'other-alias'), +]) +def test_new_host_identity_requires_successful_apply(plan, key, value): + Adapter(plan).apply(plan) + second = Adapter(plan, **{key: value}) + second.apply(plan) + assert second.calls == ['apply'] + + +def test_changed_plan_or_source_contract_does_not_use_old_receipt(plan, monkeypatch): + adapter = Adapter(plan) + adapter.apply(plan) + original = dp.render_apply_script + monkeypatch.setattr(dp, 'render_apply_script', lambda *a, **kw: original(*a, **kw) + '\n# new-source-contract\n') + adapter.apply(plan) + assert adapter.calls == ['apply', 'apply'] + + +def test_repair_failure_discards_old_success(plan): + adapter = Adapter(plan) + adapter.apply(plan) + adapter.observation = adapter.observation.replace('BRIDGE_ADMIN__=up', 'BRIDGE_ADMIN__=down') + adapter.apply_error = TimeoutError('uncertain write') + with pytest.raises(TimeoutError): + adapter.apply(plan) + assert not dp._APPLY_RECEIPTS + + +def test_remove_discards_success_receipt(plan): + adapter = Adapter(plan) + adapter.apply(plan) + empty = replace(plan, members=tuple(m for m in plan.members if m.host_id != plan.host_id)) + adapter.remove(empty) + assert not dp._APPLY_RECEIPTS + + +def test_cache_is_bounded_under_concurrent_adapters(monkeypatch): + monkeypatch.setattr(dp, '_APPLY_RECEIPT_LIMIT', 8) + with ThreadPoolExecutor(max_workers=8) as executor: + list(executor.map(lambda i: Adapter._save_receipt((str(i),), str(i), RECEIPT), range(100))) + assert len(dp._APPLY_RECEIPTS) == 8 + Adapter._save_receipt(('latest',), 'latest', RECEIPT) + assert len(dp._APPLY_RECEIPTS) == 8 + assert next(reversed(dp._APPLY_RECEIPTS)) == ('latest',) + + +NFT_MOCK = r''' +import os, sys +args=sys.argv[1:] +assert args[:3] == ['-s','-n','-y'], args +scenario=os.environ.get('SCENARIO','healthy') +if args[3:5] == ['list','tables']: + if scenario=='inventory_error': sys.exit(96) + if scenario=='malformed_inventory': print('permission denied'); sys.exit() + if scenario=='missing': sys.exit() + print('table '+args[5]+' owned') + if scenario=='duplicate': print('table '+args[5]+' owned') +else: + assert args[3:5]==['list','table'] and args[6]=='owned', args + if scenario=='read_error': sys.exit(97) + if scenario=='empty': sys.exit() + action='accept' if scenario=='verdict_drift' else 'drop' + print('table '+args[5]+' owned { chain forward { '+action+' comment "same-owner-digest"; } }') +''' + + +def executable(path, body): + path.write_text('#!' + sys.executable + '\n' + body) + path.chmod(0o700) + return str(path) + + +@pytest.mark.parametrize('shell', SHELLS) +@pytest.mark.parametrize('scenario', ['healthy', 'verdict_drift', 'inventory_error', + 'malformed_inventory', 'read_error', 'missing', 'duplicate', 'empty']) +def test_actual_nft_receipt_shell_rejects_read_errors_and_kept_comment_drift(tmp_path, shell, scenario): + nft = executable(tmp_path / 'nft', NFT_MOCK) + sha = executable(tmp_path / 'sha256sum', 'import hashlib,sys\nprint(hashlib.sha256(sys.stdin.buffer.read()).hexdigest()+" -")\n') + script = dp.VPC_APPLY_RECEIPT_SH + '\nNFT=$1; SHA256SUM=$2; TABLE=owned\nvc_vpc_nft_receipt\n' + result = subprocess.run([shell, '-eu', '-s', nft, sha], input=script, + text=True, capture_output=True, + env=os.environ | {'SCENARIO': scenario}) + if scenario in {'healthy', 'verdict_drift'}: + expected = ','.join(hashlib.sha256( + ('table '+family+' owned { chain forward { '+ + ('accept' if scenario=='verdict_drift' else 'drop')+ + ' comment "same-owner-digest"; } }\n').encode()).hexdigest() + for family in ('inet', 'bridge', 'arp')) + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == expected + elif scenario == 'missing': + assert result.returncode == 0 and result.stdout.strip() == 'missing' + else: + assert result.returncode != 0 + assert not result.stdout + + +@pytest.mark.parametrize('shell', SHELLS) +@pytest.mark.parametrize(('line', 'expected'), [ + ('20: vcv-test: mtu 1500 state UP', 'up'), + ('20: vcv-test: mtu 1500 state DOWN', 'up'), + ('20: vcv-test: mtu 1500 state DOWN', 'down'), + ('', None), ('malformed', None), +]) +def test_bridge_admin_uses_flag_not_carrier_state(tmp_path, shell, line, expected): + ip = executable(tmp_path / 'ip', 'import os\nprint(os.environ["LINK"])\n') + result = subprocess.run([shell, '-eu', '-s', ip], + input=dp.VPC_APPLY_RECEIPT_SH + '\nIP=$1; BRIDGE=vcv-test\nvc_vpc_bridge_admin_state\n', + text=True, capture_output=True, env=os.environ | {'LINK': line}) + if expected is None: + assert result.returncode != 0 + else: + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == expected + + +def test_receipt_is_anchored_in_protected_apply_and_persistence_exact(plan): + script = dp.render_apply_script(plan) + receipt = script.index('vc_apply_nft_receipt=$(vc_vpc_nft_receipt)') + assert script.rfind('vc_vpc_nft_matches_snapshot "$TXN/nft.desired" || exit 74', 0, receipt) > 0 + assert script.index('vc_vpc_nft_matches_snapshot "$TXN/nft.desired" || exit 74', receipt) < script.index('__VC_VPC_NFT_RECEIPT__=$vc_apply_nft_receipt') + assert script.index('__VC_VPC_NFT_RECEIPT__=$vc_apply_nft_receipt') < script.index('COMMITTED=yes', receipt) + persisted = dp.render_vpc_nft_rules(plan).rstrip() + '\n' + assert hashlib.sha256(persisted.encode()).hexdigest() in dp.render_observe_script(plan) + + +@pytest.mark.parametrize('key', ['host', 'port', 'ssh_target', 'ssh_config_file', 'fingerprint']) +def test_missing_host_identity_never_enters_shared_cache(plan, key): + adapter = Adapter(plan) + del adapter.profile[key] + adapter.apply(plan) + adapter.apply(plan) + assert adapter.calls == ['apply', 'apply'] + assert not dp._APPLY_RECEIPTS + + +@pytest.mark.parametrize('receipt', ['bad', RECEIPT + '\n__VC_VPC_NFT_RECEIPT__=' + RECEIPT]) +def test_malformed_observer_receipt_propagates_without_repair(plan, receipt): + adapter = Adapter(plan) + adapter.apply(plan) + adapter.observation = adapter.observation.replace(RECEIPT, receipt) + with pytest.raises(dp.VpcDataplaneError, match='收据无效'): + adapter.apply(plan) + assert adapter.calls == ['apply', 'observe'] + assert not dp._APPLY_RECEIPTS + + +def test_explicit_nft_absence_repairs_without_losing_public_observation(plan): + adapter = Adapter(plan) + adapter.apply(plan) + adapter.observation = adapter.observation.replace(RECEIPT, 'missing').replace( + 'ANTI_SPOOF__=yes', 'ANTI_SPOOF__=no') + assert adapter.observe(plan)['state'] != 'active' + adapter.apply(plan) + assert adapter.calls == ['apply', 'observe', 'observe', 'apply'] + + +@pytest.mark.parametrize('shell', SHELLS) +@pytest.mark.parametrize('mutation', ['none', 'nft', 'state', 'ifup', 'missing_state', 'helper']) +def test_rendered_host_receipt_requires_exact_persistent_bytes(tmp_path, shell, mutation, plan): + nft_file = tmp_path / 'vpc.nft' + state = tmp_path / 'vpc.state' + fragment = tmp_path / 'vpc.ifup' + nft_file.write_text(dp.render_vpc_nft_rules(plan).rstrip() + '\n') + state.write_text(dp._state(plan, dp._routes(plan))) + fragment.write_text( + f'# {dp.DATAPLANE_OWNER_MARKER} vpc={plan.vpc.vpc_id} ' + f'revision={plan.desired_revision} digest={plan.desired_digest}\n' + f'iface {plan.resources.bridge} inet manual\n' + ' bridge_ports none\n bridge-stp off\n') + if mutation in {'nft', 'state', 'ifup'}: + path = {'nft': nft_file, 'state': state, 'ifup': fragment}[mutation] + path.write_text(path.read_text() + '# retained-owner-but-drift\n') + if mutation == 'missing_state': + state.unlink() + sha = executable(tmp_path / 'sha256sum', + 'import hashlib,sys,pathlib\n' + 'raw=pathlib.Path(sys.argv[1]).read_bytes() if len(sys.argv)>1 else sys.stdin.buffer.read()\n' + 'print(hashlib.sha256(raw).hexdigest()+" -")\n') + stat = executable(tmp_path / 'stat', 'print("0:0:600")\n') + nft = executable(tmp_path / 'nft', NFT_MOCK) + ip = executable(tmp_path / 'ip', 'print("20: vcv-test: mtu 1500 state UP")\n') + rendered = dp.render_observe_script(plan) + fragment_script = rendered.split('# vc-apply-receipt-observe-begin', 1)[1].split( + '# vc-apply-receipt-observe-end', 1)[0] + settings = { + 'IP': ip, 'NFT': nft, 'SHA256SUM': sha, 'STAT': stat, + 'BRIDGE': 'vcv-test', 'TABLE': 'owned', 'NFT_FILE': str(nft_file), + 'STATE': str(state), 'IF_FRAGMENT': str(fragment), + 'nft_file_ok': 'yes', 'bridge_shape': 'yes', 'nm_owned': 'no', + 'ifup_owned': 'yes', 'nm_present': 'no', + } + script = dp.VPC_APPLY_RECEIPT_SH + '\n' + '\n'.join( + k + '=' + shlex.quote(v) for k, v in settings.items()) + '\n' + script += 'vc_vpc_neighbor_helper_ready() { return ' + ('1' if mutation == 'helper' else '0') + '; }\n' + result = subprocess.run([shell, '-eu'], input=script + fragment_script, + text=True, capture_output=True) + assert result.returncode == 0, result.stderr + markers = dp._markers(result.stdout) + assert markers['bridge_admin'] == 'up' + assert markers['nft_file_exact'] == ('no' if mutation == 'nft' else 'yes') + assert markers['state_file_exact'] == ('no' if mutation in {'state', 'missing_state'} else 'yes') + assert markers['persistence_exact'] == ('no' if mutation == 'ifup' else 'yes') + assert markers['helper_contract'] == ('missing' if mutation == 'helper' else 'owned') diff --git a/engine/tests/test_vpc_apply_reentry.py b/engine/tests/test_vpc_apply_reentry.py index 57932949..0813f816 100644 --- a/engine/tests/test_vpc_apply_reentry.py +++ b/engine/tests/test_vpc_apply_reentry.py @@ -6,13 +6,18 @@ import shlex import subprocess import pytest from test_vpc_nft_dataplane import _plan +from virsh_panel.shell_contracts import LOCAL_NEIGHBOR_SH from virsh_panel.vpc_dataplane import ( - VPC_APPLY_RUNTIME_SH, VPC_ROUTE_RECORDS_SH, render_apply_script, + VPC_APPLY_RUNTIME_SH, VPC_LOCAL_NEIGHBOR_TRANSACTION_SH, + VPC_ROUTE_RECORDS_SH, render_apply_script, ) MOCK = r''' fake_ip() { case "$*" in + # Ordinary guest-neighbor behavior has its own suite; keep this fixture + # empty while exercising the real neighbor baseline/rollback readers. + '-o -4 neigh show dev vcv-test') return 0 ;; '-o -4 -N route show table all') [ "$SCENARIO" != route_read_error ] || return 97 cat "$ROUTE_STATE"; return 0 ;; @@ -110,6 +115,7 @@ def env_and_files(tmp_path, scenario): 'route|10.44.0.11/32|wg-fabric0|16201|-|100\n' ) (txn/'state.before').write_text(state) + (txn/'neighbors.desired').write_text('') (txn/'owner.before').write_text('owner\n') (tmp_path/'state').write_text(state) (tmp_path/'owner').write_text('owner\n') @@ -137,7 +143,8 @@ def env_and_files(tmp_path, scenario): def source_helpers(): - return (VPC_ROUTE_RECORDS_SH+'\n'+VPC_APPLY_RUNTIME_SH).replace( + return ('\n'.join((VPC_ROUTE_RECORDS_SH, VPC_APPLY_RUNTIME_SH, + LOCAL_NEIGHBOR_SH, VPC_LOCAL_NEIGHBOR_TRANSACTION_SH))).replace( '/proc/sys/net/ipv4/conf/$BRIDGE/', '$PROC_ROOT/') @@ -167,11 +174,15 @@ def test_real_rollback_restores_prequarantine_runtime_after_manager_activation(t script=render_apply_script(_plan(),deployment_mode=mode) init=next(line for line in script.splitlines() if line.startswith('COMMITTED=no;')) rollback=script[script.index('rollback() {'):script.index('trap rollback EXIT HUP INT TERM')] - helper=script[script.index('vc_vpc_native_ifupdown_rollback_reopen() {'):script.index('# Never mutate even the bridge state')] + helper_start=script.index('vc_vpc_native_ifupdown_rollback_reopen() {') + # Include exactly the function, not the later helper-version admission + # gate. That independent pre-transaction gate has its own shell coverage. + helper=script[helper_start:script.index('\n}\n',helper_start)+3] body='set -eu\n'+MOCK+source_helpers()+'\n'+helper+'\n'+init+'\nMODE='+shlex.quote(mode)+r''' BRIDGE_TOPOLOGY_CAPTURED=yes; BRIDGE_WAS_UP=yes NM_PROFILE_WAS_PRESENT=yes; NM_WAS_ACTIVE=yes vc_vpc_runtime_baseline +vc_vpc_neighbors_baseline MUTATION_STARTED=yes; RUNTIME_QUARANTINED=yes if [ "$MODE" = networkmanager_nmcli ]; then NM_DEVICE_QUARANTINED=yes; fi fake_ip link set dev "$BRIDGE" down diff --git a/engine/tests/test_vpc_dataplane.py b/engine/tests/test_vpc_dataplane.py index 0a022f65..e125a747 100644 --- a/engine/tests/test_vpc_dataplane.py +++ b/engine/tests/test_vpc_dataplane.py @@ -889,6 +889,7 @@ def test_proxy_markers_are_bounded_and_typed(): "__VC_VPC_PROXY_ARP__=0\n" "__VC_VPC_PROXY_ARP_PVLAN__=1\n" "__VC_VPC_PROXY_STATE__=owned\n" + "__VC_VPC_LOCAL_NEIGHBOR_STATE__=owned\n" "__VC_VPC_PROXY_IP__=10.42.0.11\n" ) assert parsed["proxy_arp"] == "0" @@ -1244,6 +1245,7 @@ def _active_observation_markers(plan): "__VC_VPC_PROXY_ARP__=0", "__VC_VPC_PROXY_ARP_PVLAN__=1", "__VC_VPC_PROXY_STATE__=owned", + "__VC_VPC_LOCAL_NEIGHBOR_STATE__=owned", "__VC_VPC_RP_FILTER__=2", f"__VC_VPC_NFT_TABLE__={plan.resources.nft_table}", "__VC_VPC_STATE_OWNER__=owned", @@ -1410,6 +1412,7 @@ def test_observer_reports_pending_restart_instead_of_active(): "__VC_VPC_PROXY_ARP__=0", "__VC_VPC_PROXY_ARP_PVLAN__=1", "__VC_VPC_PROXY_STATE__=owned", + "__VC_VPC_LOCAL_NEIGHBOR_STATE__=owned", "__VC_VPC_RP_FILTER__=2", f"__VC_VPC_NFT_TABLE__={plan.resources.nft_table}", "__VC_VPC_STATE_OWNER__=owned", diff --git a/engine/tests/test_vpc_local_neighbor_transaction.py b/engine/tests/test_vpc_local_neighbor_transaction.py new file mode 100644 index 00000000..d358aaeb --- /dev/null +++ b/engine/tests/test_vpc_local_neighbor_transaction.py @@ -0,0 +1,282 @@ +"""Run production local-neighbor transactions against a stateful ip boundary.""" +from __future__ import annotations + +from dataclasses import replace +import json +import os +from pathlib import Path +import shlex +import subprocess +import sys + +import pytest + +from test_vpc_nft_dataplane import _plan +from virsh_panel.shell_contracts import LOCAL_NEIGHBOR_SH +from virsh_panel.vpc_dataplane import ( + VPC_LOCAL_NEIGHBOR_TRANSACTION_SH, + _routes, + _state, + render_apply_script, + render_observe_script, + render_remove_script, +) + +SHELLS = [path for path in ("/bin/sh", "/bin/dash", "/bin/bash") if Path(path).exists()] +FIRST = "10.44.0.10|02:00:00:00:00:10" +SECOND = "10.44.0.11|02:00:00:00:00:11" + +IP_MOCK = r''' +import json, os, pathlib, sys +args=sys.argv[1:] +path=pathlib.Path(os.environ['NEIGHBORS']) +state=json.loads(path.read_text()) +if args == ['-o','-4','neigh','show','dev','vcv-test']: + if os.environ.get('SCENARIO') == 'read_error': sys.exit(97) + for ip,(mac,nud) in sorted(state['vcv-test'].items()): + print(ip + (' lladdr '+mac if mac != '-' else '') + ' '+nud) + sys.exit(0) +if args == ['link','show','dev','vcv-test']: sys.exit(0) +if args == ['link','set','dev','vcv-test','down']: + with open(os.environ['ACTIONS'],'a') as log: log.write('down\n') + state['vcv-test']={} +else: + if args[:1] == ['-4']: args=args[1:] + if args[:2] not in (['neigh','replace'], ['neigh','del']): sys.exit(98) + action,ip=args[1:3] + assert args[-2:]==['dev','vcv-test'] + with open(os.environ['ACTIONS'],'a') as log: log.write(action+':'+ip+'\n') + if action == 'del': + if os.environ.get('SCENARIO') == 'delete_error': sys.exit(97) + if ip not in state['vcv-test']: sys.exit(2) + del state['vcv-test'][ip] + else: + assert args[3]=='lladdr' and args[5]=='nud' + if os.environ.get('SCENARIO') == 'write_error' and ip.endswith('.11'): sys.exit(97) + nud=args[6].upper() + if os.environ.get('SCENARIO') == 'readback_drift': nud='STALE' + state['vcv-test'][ip]=[args[4],nud] +path.write_text(json.dumps(state)) +''' + + +def boundary(tmp_path, current=None, *, before=FIRST, desired=FIRST, scenario=""): + txn = tmp_path / "txn" + txn.mkdir() + (txn / "state.before").write_text("".join( + f"local_neighbor|{pair}\n" for pair in before.splitlines() if pair)) + (txn / "neighbors.desired").write_text(desired + "\n" if desired else "") + state = tmp_path / "neighbors.json" + state.write_text(json.dumps({"vcv-test": current or {}, "other-bridge": { + "10.44.0.10": ["02:ff:00:00:00:01", "PERMANENT"]}})) + command = tmp_path / "ip" + command.write_text("#!" + sys.executable + "\n" + IP_MOCK) + command.chmod(0o700) + actions = tmp_path / "actions" + actions.write_text("") + env = dict(os.environ, TXN=str(txn), NEIGHBORS=str(state), ACTIONS=str(actions), + SCENARIO=scenario, IP=str(command)) + prefix = ("set -eu\nBRIDGE=vcv-test; CMP=cmp; BRIDGE_TOPOLOGY_CAPTURED=yes; " + "LOCAL_NEIGHBORS_CHANGED=no\n" + LOCAL_NEIGHBOR_SH + "\n" + + VPC_LOCAL_NEIGHBOR_TRANSACTION_SH + "\n") + return env, prefix, state, actions + + +def execute(shell, env, prefix, commands): + return subprocess.run([shell, "-s"], input=prefix + commands, text=True, + capture_output=True, env=env, timeout=15, check=False) + + +@pytest.mark.parametrize("shell", SHELLS) +@pytest.mark.parametrize("prior_state", ["PERMANENT", "STALE", "INCOMPLETE", "FAILED", "absent"]) +def test_apply_survives_down_and_repeated_convergence(tmp_path, shell, prior_state): + mac = "-" if prior_state in {"INCOMPLETE", "FAILED"} else FIRST.split("|")[1] + current = {} if prior_state == "absent" else {"10.44.0.10": [mac, prior_state]} + env, prefix, state, _ = boundary(tmp_path, current) + result = execute(shell, env, prefix, ''' +vc_vpc_neighbors_baseline +"$IP" link set dev "$BRIDGE" down +vc_vpc_neighbors_reconcile +# A later periodic apply must recreate permanent entries flushed by DOWN. +vc_vpc_neighbors_baseline +"$IP" link set dev "$BRIDGE" down +vc_vpc_neighbors_reconcile +''') + assert result.returncode == 0, result.stderr + actual = json.loads(state.read_text()) + assert actual["vcv-test"] == {"10.44.0.10": [FIRST.split("|")[1], "PERMANENT"]} + assert actual["other-bridge"] == {"10.44.0.10": ["02:ff:00:00:00:01", "PERMANENT"]} + + +@pytest.mark.parametrize("shell", SHELLS) +@pytest.mark.parametrize("current", [ + {"10.44.0.10": ["02:ff:00:00:00:10", "PERMANENT"]}, + {"10.44.0.99": ["02:00:00:00:00:99", "STALE"]}, + {"10.44.0.99": ["-", "FAILED"]}, +]) +def test_foreign_binding_rejected_before_mutation(tmp_path, shell, current): + env, prefix, state, actions = boundary(tmp_path, current) + original = state.read_bytes() + result = execute(shell, env, prefix, "vc_vpc_neighbors_baseline\nvc_vpc_neighbors_reconcile\n") + assert result.returncode != 0 + assert state.read_bytes() == original + assert actions.read_text() == "" + + +@pytest.mark.parametrize("shell", SHELLS) +@pytest.mark.parametrize("prior_state", ["PERMANENT", "REACHABLE", "NOARP"]) +def test_partial_apply_rollback_restores_only_sampled_bindings(tmp_path, shell, prior_state): + env, prefix, state, _ = boundary(tmp_path, + {"10.44.0.10": [FIRST.split("|")[1], prior_state]}, + desired=FIRST + "\n" + SECOND, scenario="write_error") + result = execute(shell, env, prefix, ''' +vc_vpc_neighbors_baseline +"$IP" link set dev "$BRIDGE" down +if vc_vpc_neighbors_reconcile; then exit 99; fi +vc_vpc_neighbors_rollback_guard +"$IP" link set dev "$BRIDGE" down +vc_vpc_neighbors_restore +''') + assert result.returncode == 0, result.stderr + assert json.loads(state.read_text())["vcv-test"] == { + "10.44.0.10": [FIRST.split("|")[1], prior_state if prior_state in {"PERMANENT", "NOARP"} else "STALE"]} + + +@pytest.mark.parametrize("shell", SHELLS) +@pytest.mark.parametrize("scenario", ["read_error", "readback_drift"]) +def test_read_or_readback_failure_never_succeeds(tmp_path, shell, scenario): + env, prefix, _, _ = boundary(tmp_path, scenario=scenario) + result = execute(shell, env, prefix, "vc_vpc_neighbors_baseline\nvc_vpc_neighbors_reconcile\n") + assert result.returncode != 0 + + +@pytest.mark.parametrize("shell", SHELLS) +@pytest.mark.parametrize("prior_state", ["PERMANENT", "STALE"]) +def test_legacy_state_adopts_only_authenticated_same_binding(tmp_path, shell, prior_state): + env, prefix, state, _ = boundary(tmp_path, + {"10.44.0.10": [FIRST.split("|")[1], prior_state]}, before="") + result = execute(shell, env, prefix, "vc_vpc_neighbors_baseline\nvc_vpc_neighbors_reconcile\n") + assert result.returncode == 0, result.stderr + assert json.loads(state.read_text())["vcv-test"]["10.44.0.10"][1] == "PERMANENT" + + +@pytest.mark.parametrize("shell", SHELLS) +def test_member_removal_deletes_old_binding_and_keeps_remaining(tmp_path, shell): + env, prefix, state, actions = boundary(tmp_path, { + "10.44.0.10": [FIRST.split("|")[1], "PERMANENT"], + "10.44.0.11": [SECOND.split("|")[1], "PERMANENT"]}, + before=FIRST + "\n" + SECOND, desired=SECOND) + result = execute(shell, env, prefix, "vc_vpc_neighbors_baseline\nvc_vpc_neighbors_reconcile\n") + assert result.returncode == 0, result.stderr + assert json.loads(state.read_text())["vcv-test"] == {"10.44.0.11": [SECOND.split("|")[1], "PERMANENT"]} + assert actions.read_text().splitlines() == ["del:10.44.0.10", "replace:10.44.0.11"] + + +@pytest.mark.parametrize("shell", SHELLS) +def test_empty_expected_does_not_accept_foreign_neighbor(tmp_path, shell): + env, prefix, _, actions = boundary(tmp_path, + {"10.44.0.10": [FIRST.split("|")[1], "PERMANENT"]}, before="", desired="") + result = execute(shell, env, prefix, "vc_vpc_neighbors_baseline\n") + assert result.returncode != 0 + assert actions.read_text() == "" + + +@pytest.mark.parametrize("shell", SHELLS) +def test_rollback_refuses_binding_changed_after_snapshot(tmp_path, shell): + env, prefix, state, actions = boundary(tmp_path, + {"10.44.0.10": [FIRST.split("|")[1], "PERMANENT"]}) + result = execute(shell, env, prefix, "vc_vpc_neighbors_baseline\n") + assert result.returncode == 0, result.stderr + current = json.loads(state.read_text()) + current["vcv-test"]["10.44.0.10"][0] = "02:ff:00:00:00:10" + state.write_text(json.dumps(current)) + original = state.read_bytes() + result = execute(shell, env, prefix, "vc_vpc_neighbors_restore\n") + assert result.returncode != 0 + assert state.read_bytes() == original + assert actions.read_text() == "" + + +@pytest.mark.parametrize("shell", SHELLS) +@pytest.mark.parametrize("case", ["new", "old", "wrong_digest", "wrong_mode", "symlink", "missing"]) +def test_helper_upgrade_gate_runs_before_network_mutation(tmp_path, shell, case): + env, prefix, state, actions = boundary(tmp_path) + digest = "ab" * 32 + helper = tmp_path / "virt-cloud-wg-fabric" + header = ("#!/bin/sh\n# virt-cloud-wg-fabric-v1\n# fabric-plan-digest=" + digest + + '\nPLAN_DIGEST="' + digest + '"\n# libvirt-startup-independent-v1\n') + if case != "old": header += "# vpc-local-neighbor-contract-v1\n" + if case == "wrong_digest": header = header.replace(digest, "cd" * 32) + if case != "missing": + helper.write_text(header) + helper.chmod(0o700) + if case == "symlink": + target = helper.with_suffix(".real") + helper.rename(target) + helper.symlink_to(target) + prefix = prefix.replace("vc_ln_helper=/usr/local/libexec/virt-cloud-wg-fabric", + "vc_ln_helper=" + shlex.quote(str(helper))) + prefix += "\nEXPECTED_DIGEST=" + digest + "\nSTAT=fake_stat\n" + prefix += "fake_stat() { printf '%s\\n' " + ("0:0:755" if case == "wrong_mode" else "0:0:700") + "; }\n" + script = render_apply_script(_plan()) + guard = "vc_vpc_neighbor_helper_ready || exit 71" + assert script.index(guard) < script.index('mkdir -m 700 "$TXN"') + assert script.index(guard) < script.index("MUTATION_STARTED=yes") + result = execute(shell, env, prefix, guard + '\n"$IP" link set dev "$BRIDGE" down\nprintf ready\n') + assert result.returncode == (0 if case == "new" else 71), result.stderr + assert result.stdout == ("ready" if case == "new" else "") + assert actions.read_text() == ("down\n" if case == "new" else "") + + +@pytest.mark.parametrize("shell", SHELLS) +@pytest.mark.parametrize("case", ["exact", "missing", "dynamic", "wrong_mac", "legacy", "reader_error"]) +def test_rendered_observer_requires_durable_and_permanent_binding(tmp_path, shell, case): + plan = _plan() + member = plan.local_members[0] + pair = f"{member.private_ip}|{member.private_mac}" + current = {} if case == "missing" else {member.private_ip: [ + "02:ff:00:00:00:10" if case == "wrong_mac" else member.private_mac, + "STALE" if case == "dynamic" else "PERMANENT"]} + env, prefix, _, _ = boundary(tmp_path, current, before="" if case == "legacy" else pair, + desired=pair, scenario="read_error" if case == "reader_error" else "") + env["STATE"] = str(tmp_path / "txn" / "state.before") + script = render_observe_script(plan) + fragment = script.split("# vc-local-neighbor-observe-begin\n", 1)[1].split( + "# vc-local-neighbor-observe-end", 1)[0] + result = execute(shell, env, prefix, fragment) + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "__VC_VPC_LOCAL_NEIGHBOR_STATE__=" + ("owned" if case == "exact" else "missing") + + +def test_local_neighbor_state_contains_only_local_vm_next_hops(): + plan = _plan() + state = _state(plan, _routes(plan)) + assert [line for line in state.splitlines() if line.startswith("local_neighbor|")] == [ + f"local_neighbor|{m.private_ip}|{m.private_mac}" for m in sorted(plan.local_members, key=lambda m: m.private_ip)] + assert all(f"local_neighbor|{m.private_ip}|" not in state for m in plan.remote_members) + script = render_apply_script(plan) + assert script.index("vc_vpc_neighbors_baseline || exit 71") < script.index("MUTATION_STARTED=yes\n RUNTIME_QUARANTINED=yes") + assert script.index("vc_vpc_neighbors_reconcile || exit 72") < script.index("ROUTES_CHANGED=yes\n") + assert script.index("vc_ln_final_actual=") < script.index("printf '%s\\n' ready") + + +@pytest.mark.parametrize("shell", SHELLS) +@pytest.mark.parametrize("case", ["healthy", "foreign", "delete_error"]) +def test_rendered_remove_cleanup_only_deletes_exact_old_members(tmp_path, shell, case): + env, prefix, state, actions = boundary(tmp_path, { + "10.44.0.10": ["02:ff:00:00:00:10" if case == "foreign" else FIRST.split("|")[1], "PERMANENT"]}, + scenario=case) + env["REMOVE_NEIGHBOR_EXPECTED"] = FIRST + plan = _plan() + script = render_remove_script(replace(plan, members=tuple(m for m in plan.members if m.host_id != plan.host_id))) + fragment = script.split("# vc-local-neighbor-remove-cleanup-begin\n", 1)[1].split( + "# vc-local-neighbor-remove-cleanup-end", 1)[0] + result = execute(shell, env, prefix, fragment) + if case == "healthy": + assert result.returncode == 0, result.stderr + assert json.loads(state.read_text())["vcv-test"] == {} + assert actions.read_text() == "del:10.44.0.10\n" + else: + assert result.returncode != 0 + assert json.loads(state.read_text())["vcv-test"] + if case == "foreign": assert actions.read_text() == "" diff --git a/engine/tests/test_vpc_nft_dataplane.py b/engine/tests/test_vpc_nft_dataplane.py index 56fb432b..1656da90 100644 --- a/engine/tests/test_vpc_nft_dataplane.py +++ b/engine/tests/test_vpc_nft_dataplane.py @@ -910,7 +910,7 @@ def test_vpc_bridge_topology_parser_propagates_malformed_output(): _plan(), deployment_mode="networkmanager_nmcli", firewall_mode=FIREWALL_MODE_NATIVE) start = script.index("vc_vpc_bridge_port_names() {") - end = script.index("# Never mutate even the bridge state", start) + end = script.index("\n}\n", start) + 3 helpers = BRIDGE_PORT_RECORDS_SH + "\n" + script[start:end] harness = "\n".join(( "set -eu", @@ -1021,8 +1021,7 @@ def test_native_ifupdown_rollback_reopens_only_exact_sampled_up_existing_bridge( firewall_mode=FIREWALL_MODE_NATIVE) helper_start = script.index( "vc_vpc_native_ifupdown_rollback_reopen() {") - helper_end = script.index( - "# Never mutate even the bridge state", helper_start) + helper_end = script.index("\n}\n", helper_start) + 3 helper = script[helper_start:helper_end] exact = helper.index( 'vc_vpc_nft_matches_snapshot "$TXN/nft.precommit.before"') @@ -1051,8 +1050,7 @@ def test_native_ifupdown_reopen_failure_reisolates_bridge(tmp_path): _plan(), deployment_mode="ifupdown", firewall_mode=FIREWALL_MODE_NATIVE) start = script.index("vc_vpc_native_ifupdown_rollback_reopen() {") - end = script.index( - "# Never mutate even the bridge state", start) + end = script.index("\n}\n", start) + 3 helper = script[start:end] link_state = tmp_path / "link" topology_calls = tmp_path / "topology-calls" diff --git a/engine/tests/test_vpc_nm_bridge_preserve.py b/engine/tests/test_vpc_nm_bridge_preserve.py index 6cbd4b0b..a9826822 100644 --- a/engine/tests/test_vpc_nm_bridge_preserve.py +++ b/engine/tests/test_vpc_nm_bridge_preserve.py @@ -3,7 +3,7 @@ import os import subprocess import pytest from test_vpc_nft_dataplane import _plan -from virsh_panel.shell_contracts import BRIDGE_PORT_RECORDS_SH +from virsh_panel.shell_contracts import BRIDGE_PORT_RECORDS_SH, LOCAL_NEIGHBOR_SH from virsh_panel.vpc_dataplane import NETWORKMANAGER_VPC_PROFILE_SH,render_apply_script MOCK=r''' @@ -13,6 +13,7 @@ NM_PRESERVE_ACTIVE_BRIDGE=yes NM_PRESERVED_PROFILE_UUID=$NM_PROFILE_UUID BRIDGE_TOPOLOGY_CAPTURED=yes; BRIDGE_IFINDEX_BEFORE=19; BRIDGE_PORTS_BEFORE=vnet4 fake_ip() { + if [ "$*" = '-o -4 neigh show dev vcv-test' ]; then return 0; fi if [ "$*" = '-o link show dev vcv-test' ]; then index=19 if [ "$SCENARIO" = index_drift ] && [ "$(cat "$LINK")" = down ]; then index=20; fi @@ -133,6 +134,7 @@ NM_PRESERVE_ACTIVE_BRIDGE=yes NM_PRESERVED_PROFILE_UUID=known-uuid vc_nm_preserve_bridge_ready() { return 0; } vc_vpc_runtime_baseline +vc_vpc_neighbors_baseline MUTATION_STARTED=yes; RUNTIME_QUARANTINED=yes; NM_DEVICE_QUARANTINED=yes fake_ip link set dev "$BRIDGE" down '''+rollback.replace('/proc/sys/net/ipv4/conf/$BRIDGE/','$PROC_ROOT/')+''' @@ -160,13 +162,14 @@ def test_remove_attached_bridge_quarantines_without_nm_deconfigure(tmp_path,shel start=script.index(' REMOVE_BRIDGE_TOPOLOGY_CAPTURED=yes') end=script.index(' if [ "$MODE" = networkmanager_nmcli ]; then',start) body=script[start:end] - harness='set -eu\n'+MOCK+r''' + harness='set -eu\n'+LOCAL_NEIGHBOR_SH+'\n'+MOCK+r''' REMOVE_BRIDGE_PORTS_BEFORE=vnet4 vc_nm_link_expect_down() { [ "$(cat "$LINK")" = down ]; } vc_remove_bridge_topology_matches() { return 0; } '''+body+'\nexit 99\n' (tmp_path/'link').write_text('up\n');(tmp_path/'actions').write_text('') - result=subprocess.run([shell,'-s'],input=harness,text=True,capture_output=True,env=dict(os.environ,SCENARIO='healthy',MODE=mode,LINK=str(tmp_path/'link'),ACTIONS=str(tmp_path/'actions'))) + (tmp_path/'state').write_text('') + result=subprocess.run([shell,'-s'],input=harness,text=True,capture_output=True,env=dict(os.environ,SCENARIO='healthy',MODE=mode,LINK=str(tmp_path/'link'),ACTIONS=str(tmp_path/'actions'),STATE_FILE=str(tmp_path/'state'))) assert result.returncode==71,result.stderr assert (tmp_path/'link').read_text()=='down\n' assert (tmp_path/'actions').read_text()=='down\n' diff --git a/engine/tests/test_vpc_proxy_rollback.py b/engine/tests/test_vpc_proxy_rollback.py index 0815e37f..e539eb29 100644 --- a/engine/tests/test_vpc_proxy_rollback.py +++ b/engine/tests/test_vpc_proxy_rollback.py @@ -3,9 +3,14 @@ import os import subprocess import pytest from test_vpc_nft_dataplane import _plan -from virsh_panel.vpc_dataplane import render_apply_script, VPC_APPLY_RUNTIME_SH +from virsh_panel.shell_contracts import LOCAL_NEIGHBOR_SH +from virsh_panel.vpc_dataplane import ( + render_apply_script, VPC_APPLY_RUNTIME_SH, VPC_LOCAL_NEIGHBOR_TRANSACTION_SH, +) MOCK = r''' fake_ip() { + # This suite isolates proxy ARP; ordinary guest-neighbor state is empty. + if [ "$*" = '-o -4 neigh show dev vcv-test' ]; then return 0; fi if [ "$*" = 'link show dev vcv-test' ]; then return 0; fi if [ "$*" = 'link set dev vcv-test down' ]; then if [ "$SCENARIO" != no_flush ]; then : > "$PROXY_STATE"; fi @@ -42,7 +47,7 @@ vc_vpc_bridge_link_state() { } vc_vpc_bridge_topology_matches() { [ "$SCENARIO" != foreign_tap ]; } IP=fake_ip; CMP=cmp; BRIDGE=vcv-test; MODE=ifupdown -ROLLBACK_FAILED=no; PROXY_ENTRIES_CHANGED=yes; RUNTIME_QUARANTINED=yes; BRIDGE_TOPOLOGY_CAPTURED=yes +ROLLBACK_FAILED=no; PROXY_ENTRIES_CHANGED=yes; RUNTIME_QUARANTINED=yes; BRIDGE_TOPOLOGY_CAPTURED=yes; LOCAL_NEIGHBORS_CHANGED=no ''' FOOTER = r''' printf 'rollback_failed=%s\n' "$ROLLBACK_FAILED" @@ -69,11 +74,14 @@ def test_proxy_rollback_verifies_quarantine_before_restoring(tmp_path, shell, sc current = tmp_path / 'current' current.write_text('10.44.0.99\n' if scenario == 'foreign_before' else '10.44.0.10\n10.44.0.11\n') (tmp_path / 'proxy.before').write_text('10.44.0.9\n') + (tmp_path / 'neighbors.desired').write_text('') deletes = tmp_path / 'deletes' deletes.write_text('') drift = 'ROLLBACK_FAILED=yes\n' if scenario == 'config_drift' else '' env = dict(os.environ, SCENARIO=scenario, TXN=str(tmp_path), PROXY_STATE=str(current), DELETE_LOG=str(deletes)) - result = subprocess.run([shell, '-s'], input='set -eu\n' + MOCK + quarantine + drift + proxy + FOOTER, text=True, capture_output=True, env=env, check=False) + helpers = LOCAL_NEIGHBOR_SH + '\n' + VPC_LOCAL_NEIGHBOR_TRANSACTION_SH + baseline = '\nvc_vpc_neighbors_baseline\n' + result = subprocess.run([shell, '-s'], input='set -eu\n' + MOCK + helpers + baseline + quarantine + drift + proxy + FOOTER, text=True, capture_output=True, env=env, check=False) assert result.returncode == 0, result.stderr values = dict(line.split('=', 1) for line in result.stdout.splitlines()) success = scenario in ('kernel_flush', 'no_flush', 'long_output', 'short_output') diff --git a/engine/virsh_panel/fabric.py b/engine/virsh_panel/fabric.py index 02e50ed2..1d5fe410 100644 --- a/engine/virsh_panel/fabric.py +++ b/engine/virsh_panel/fabric.py @@ -24,7 +24,7 @@ from typing import Any, Mapping from .remote_host import RemoteHostError from .durable_jobs import create_durable_task from .shell_contracts import ( - BRIDGE_PORT_RECORDS_SH, IP_RULE_CONVERGENCE_SH, + BRIDGE_PORT_RECORDS_SH, IP_RULE_CONVERGENCE_SH, LOCAL_NEIGHBOR_SH, VPC_SYSCTL_VALIDATION_SH, WG_ALLOWED_IPS_SH, ) from .shutdown import ManagedThreadPoolExecutor, close_executors @@ -274,6 +274,7 @@ VPC_STATE_STATUS=empty VPC_PEER_FILE="/run/virt-cloud/.virt-cloud-vpc-peers.$$" VPC_ROUTE_FILE="/run/virt-cloud/.virt-cloud-vpc-routes.$$" VPC_TENANT_FILE="/run/virt-cloud/.virt-cloud-vpc-tenant-ips.$$" +__VC_LOCAL_NEIGHBOR_CONTRACT__ vpc_state_fail() { VPC_STATE_STATUS=invalid if [ "$VPC_STATE_OBSERVE" = yes ]; then return 1; fi @@ -352,6 +353,30 @@ vpc_state_dir_secure() { [ -d "$VPC_STATE_DIR" ] && [ ! -L "$VPC_STATE_DIR" ] || return 1 [ "$($STAT -c '%u:%a' "$VPC_STATE_DIR" 2>/dev/null)" = 0:700 ] } +vpc_state_local_neighbors_valid() ( + # Legacy state has no bindings. It remains readable for migration, but + # cannot satisfy the VPC observer's new exact local-neighbor proof. + vc_state_neighbors=$(vc_local_neighbor_expected "$1") || exit 1 + [ -n "$vc_state_neighbors" ] || exit 0 + vc_state_direct=$(awk -F'|' -v bridge="$2" -v table="$3" ' + $1=="route" && $3==bridge && $4==table && $5=="-" { + if (NF!=6 || seen[$2]++) bad=1 + value=$2; sub(/\/32$/, "", value); print value + } + END {exit bad ? 1 : 0} + ' "$1") || exit 1 + vc_state_direct=$(printf '%s\n' "$vc_state_direct" | sed '/^$/d' | LC_ALL=C sort) || exit 1 + vc_state_neighbor_ips=$(printf '%s\n' "$vc_state_neighbors" | cut -d'|' -f1 | LC_ALL=C sort) || exit 1 + [ "$vc_state_direct" = "$vc_state_neighbor_ips" ] || exit 1 + while IFS='|' read -r vc_state_neighbor_ip vc_state_neighbor_mac; do + [ -n "$vc_state_neighbor_ip" ] || continue + vpc_state_cidr_contains "$vc_state_neighbor_ip/32" "$4" || exit 1 + vpc_state_in_fabric "$vc_state_neighbor_ip/32" && exit 1 + done <> "$VPC_TENANT_FILE" || vpc_state_fail || return 1 ;; + local_neighbor) + vpc_state_ipv4 "$vpc_a" && vpc_state_private "$vpc_a" && + [ -n "$vpc_b" ] && [ -z "$vpc_c$vpc_d" ] || + vpc_state_fail || return 1 ;; peer) vpc_state_host "$vpc_a" && vpc_state_key "$vpc_b" && [ -n "$vpc_c" ] && [ -z "$vpc_d" ] || vpc_state_fail || return 1 @@ -534,6 +563,8 @@ vpc_state_load() { fi [ "$vpc_state_firewall_mode" = "$FIREWALL_MODE" ] || vpc_state_fail || return 1 [ "$vpc_snapshot_count" = 0 ] || vpc_state_fail || return 1 + vpc_state_local_neighbors_valid "$vpc_state_file" "$vpc_state_bridge" \ + "$vpc_state_table" "$vpc_state_cidr_value" || vpc_state_fail || return 1 # Validate membership after the complete record has been parsed so a # malicious route cannot precede the cidr line and bypass the check. while IFS='|' read -r vpc_route_prefix vpc_route_dev vpc_route_tbl vpc_route_via vpc_route_metric vpc_route_extra; do @@ -691,7 +722,7 @@ END { vpc_nft_set_expected=$(LC_ALL=C sort "$VPC_TENANT_FILE") || return 1 [ "$vpc_nft_set_actual" = "$vpc_nft_set_expected" ] } -''' +'''.replace("__VC_LOCAL_NEIGHBOR_CONTRACT__", LOCAL_NEIGHBOR_SH) # Read back the actual tenant set from the Fabric quarantine table. The @@ -2624,6 +2655,7 @@ _BOOT_HELPER = r'''#!/bin/sh # fabric-plan-digest=__VC_FABRIC_DIGEST__ PLAN_DIGEST="__VC_FABRIC_DIGEST__" # libvirt-startup-independent-v1 +# vpc-local-neighbor-contract-v1 EXPECTED_DIGEST="$PLAN_DIGEST" EXPECTED_NFT_RULES_SHA256="__VC_FABRIC_NFT_RULES_SHA256__" DEPLOYMENT_MODE="__VC_FABRIC_MODE__" @@ -3207,6 +3239,17 @@ vc_vpc_emergency_down() { vc_vpc_emergency_failed=yes continue fi + # Preserve the original emergency isolation guarantee for an owned + # bridge even when its neighbors have drifted. DOWN can clear them; + # a failed preflight forbids recovery, not emergency quarantine. + vc_vpc_emergency_state="$vc_vpc_emergency_dir/$vc_vpc_emergency_token.state" + if [ ! -f "$vc_vpc_emergency_state" ] || + [ -L "$vc_vpc_emergency_state" ] || + [ "$($STAT -c '%u:%a' "$vc_vpc_emergency_state" 2>/dev/null)" != 0:600 ] || + ! vc_vpc_emergency_neighbors=$(vc_local_neighbor_expected "$vc_vpc_emergency_state") || + ! vc_local_neighbor_preflight "$vc_vpc_emergency_bridge" "$vc_vpc_emergency_neighbors"; then + vc_vpc_emergency_failed=yes + fi if [ "$DEPLOYMENT_MODE" = networkmanager_nmcli ]; then if [ -n "$vc_vpc_emergency_ports_before" ]; then if ! vc_vpc_nm_preserve_capture "$vc_vpc_emergency_bridge"; then @@ -4916,6 +4959,7 @@ recover_vpc_dataplanes() { vpc_sysctl_file="/etc/sysctl.d/99-virt-cloud-vpc-$vpc_token.conf" vc_vpc_sysctl_artifact_owned "$vpc_sysctl_file" "$vpc_id" \ "$vpc_revision" "$vpc_digest" "$vpc_bridge" || exit 71 + vpc_local_neighbors=$(vc_local_neighbor_expected "$vpc_state_file") || exit 71 vpc_bridge_ports_before_activation= vpc_nm_preserved=no if vc_vpc_nm_was_preserved "$vpc_bridge"; then @@ -4923,6 +4967,7 @@ recover_vpc_dataplanes() { vc_vpc_nm_preserve_matches "$vpc_bridge" - || exit 71 fi if "$IP" link show dev "$vpc_bridge" >/dev/null 2>&1; then + vc_local_neighbor_preflight "$vpc_bridge" "$vpc_local_neighbors" || exit 71 vpc_bridge_ports_before_activation=$(vpc_boot_bridge_port_names "$vpc_bridge") || exit 71 "$IP" -d link show dev "$vpc_bridge" 2>/dev/null | grep -Eq '(^|[[:space:]])bridge[[:space:]]+(forward_delay|stp_state|hello_time|ageing_time)([[:space:]]|$)' || exit 71 @@ -5079,11 +5124,20 @@ EOF vc_ip_rule_preflight "$((vpc_priority + 1))" "$IFACE" "$vpc_cidr" "$vpc_table" 71 72 - || exit $? vc_ip_rule_ensure "$vpc_priority" - "$vpc_cidr" "$vpc_table" 71 72 - - || exit $? vc_ip_rule_ensure "$((vpc_priority + 1))" "$IFACE" "$vpc_cidr" "$vpc_table" 71 72 - - || exit $? + # ACL/owner/state/manager and exact routes have now been verified. + # Restore only declared local guest MACs, never routed service /32s + # or remote peers. A missing legacy contract stays empty until VPC + # apply migrates it from the authoritative member document. + vc_local_neighbor_ensure "$vpc_bridge" "$vpc_local_neighbors" || exit 71 if [ "$vpc_nm_preserved" = yes ]; then # The original active profile, nft policy, routes and proxy entries # are now validated. Reopen the same bridge without NM deconfigure. vc_vpc_nm_preserve_recover "$vpc_bridge" "$vpc_nm_uuid" || exit 71 fi + vpc_local_neighbors_actual=$(vc_local_neighbor_records "$vpc_bridge") || exit 71 + vpc_local_neighbors_wanted=$(printf '%s\n' "$vpc_local_neighbors" | + awk 'NF {print $0 "|PERMANENT"}') || exit 71 + [ "$vpc_local_neighbors_actual" = "$vpc_local_neighbors_wanted" ] || exit 71 done } "$WG" set "$IFACE" private-key "$KEY" @@ -8096,6 +8150,10 @@ artifact_state() {{ grep -Fqx -- {_q(FABRIC_STARTUP_ISOLATION_MARKER)} "$artifact_path" || {{ printf '%s' foreign; return; }} fi + if [ "$artifact_kind" = helper ]; then + grep -Fqx -- '# vpc-local-neighbor-contract-v1' "$artifact_path" || + {{ printf '%s' foreign; return; }} + fi if [ "$artifact_kind" = helper ] && {{ [ -e "$GATE_GENERATOR" ] || [ -L "$GATE_GENERATOR" ]; }}; then printf '%s' foreign diff --git a/engine/virsh_panel/shell_contracts.py b/engine/virsh_panel/shell_contracts.py index 87565294..20b31898 100644 --- a/engine/virsh_panel/shell_contracts.py +++ b/engine/virsh_panel/shell_contracts.py @@ -1,6 +1,114 @@ """Shared shell fragments for remote host reconciliation scripts.""" +LOCAL_NEIGHBOR_SH = r''' +# The bridge has no IPv4 address from which the kernel can issue useful ARP +# requests. Its local guest next hops therefore use exact, durable bindings. +# These helpers never flush neighbors or overwrite another MAC's binding. +vc_local_neighbor_pairs() { + awk -F'|' ' + function ipv4(v,a,n,i) { + n=split(v,a,"."); if (n!=4) return 0 + for(i=1;i<=4;i++) if(a[i]!~/^[0-9]+$/ || a[i]>255 || + sprintf("%d",a[i])!=a[i]) return 0 + return a[1]==10 || (a[1]==172 && a[2]>=16 && a[2]<=31) || + (a[1]==192 && a[2]==168) + } + function mac(v,a,n,i) { + n=split(v,a,":"); if(n!=6 || v=="00:00:00:00:00:00") return 0 + for(i=1;i<=6;i++) if(length(a[i])!=2 || a[i]!~/^[0-9a-f]+$/) return 0 + return substr(a[1],2,1)~/^[02468ace]$/ + } + NF==0 {next} + {if(NF!=2 || !ipv4($1) || !mac($2) || ips[$1]++ || macs[$2]++) bad=1 + else print $1 "|" $2} + END {exit bad ? 1 : 0} + ' +} +vc_local_neighbor_expected() ( + [ "$#" -eq 1 ] && [ -f "$1" ] && [ ! -L "$1" ] || exit 1 + vc_ln_expected=$(awk -F'|' ' + $1=="local_neighbor" {if(NF!=3) bad=1; else print $2 "|" $3} + END {exit bad ? 1 : 0} + ' "$1") || exit 1 + vc_ln_expected=$(printf '%s\n' "$vc_ln_expected" | + vc_local_neighbor_pairs) || exit 1 + [ -z "$vc_ln_expected" ] || printf '%s\n' "$vc_ln_expected" | LC_ALL=C sort +) +vc_local_neighbor_records() ( + [ "$#" -eq 1 ] || exit 1 + case "$1" in ''|*[!A-Za-z0-9_.-]*) exit 1 ;; esac + [ "${#1}" -le 15 ] || exit 1 + vc_ln_raw=$("$IP" -o -4 neigh show dev "$1" 2>/dev/null) || exit 1 + vc_ln_records=$(printf '%s\n' "$vc_ln_raw" | awk -v bridge="$1" ' + function ipv4(v,a,n,i) { + n=split(v,a,"."); if(n!=4) return 0 + for(i=1;i<=4;i++) if(a[i]!~/^[0-9]+$/ || a[i]>255 || + sprintf("%d",a[i])!=a[i]) return 0 + return 1 + } + function mac(v,a,n,i) { + n=split(v,a,":"); if(n!=6 || v=="00:00:00:00:00:00") return 0 + for(i=1;i<=6;i++) if(length(a[i])!=2 || a[i]!~/^[0-9a-f]+$/) return 0 + return substr(a[1],2,1)~/^[02468ace]$/ + } + NF==0 {next} + { + if(!ipv4($1)) {bad=1; next} + split($1,octets,".") + if((octets[1]>=224 && octets[1]<=239) || $1=="255.255.255.255") next + ip=$1; address="-"; state=""; dev=0; lladdr=0 + for(i=2;i<=NF;i++) { + if($i=="dev" && i/dev/null)" = 0:0:700 ] || return 1 + [ "$(sed -n '2p' "$vc_ln_helper")" = '# virt-cloud-wg-fabric-v1' ] && + [ "$(sed -n '3p' "$vc_ln_helper")" = "# fabric-plan-digest=$EXPECTED_DIGEST" ] && + [ "$(sed -n '4p' "$vc_ln_helper")" = "PLAN_DIGEST=\"$EXPECTED_DIGEST\"" ] && + [ "$(sed -n '6p' "$vc_ln_helper")" = '# vpc-local-neighbor-contract-v1' ] +} +vc_vpc_neighbors_snapshot() { + vc_ln_snapshot=$(vc_local_neighbor_records "$BRIDGE") || return 1 + printf '%s\n' "$vc_ln_snapshot" | awk 'NF' > "$1" || return 1 + chmod 600 "$1" +} +vc_vpc_neighbors_subset() { + # Readers have already validated the grammar. A missing hardware address + # is repairable only for an explicitly owned, unresolved IP. + awk -F'|' 'FILENAME==ARGV[1] {expected[$1]=$2; next} + NF {if (!($1 in expected) || ($2!="-" && $2!=expected[$1])) bad=1} + END {exit bad ? 1 : 0}' "$2" "$1" +} +vc_vpc_neighbors_baseline() { + : > "$TXN/neighbors.expected.before" + if [ -f "$TXN/state.before" ]; then + vc_local_neighbor_expected "$TXN/state.before" > "$TXN/neighbors.expected.before" || return 1 + fi + # The desired authenticated plan supplies MACs during the one-time upgrade + # from states which predate local_neighbor records. Conflicting old/new + # identities are never silently replaced. + vc_ln_known=$(cat "$TXN/neighbors.expected.before" "$TXN/neighbors.desired") || return 1 + printf '%s\n' "$vc_ln_known" | sed '/^$/d' | + LC_ALL=C sort -u > "$TXN/neighbors.known" || return 1 + awk -F'|' 'NF {if (seen[$1]++) bad=1} END {exit bad ? 1 : 0}' \ + "$TXN/neighbors.known" || return 1 + : > "$TXN/neighbors.before" + if [ "$BRIDGE_TOPOLOGY_CAPTURED" = yes ]; then + vc_vpc_neighbors_snapshot "$TXN/neighbors.before" || return 1 + vc_vpc_neighbors_subset "$TXN/neighbors.before" "$TXN/neighbors.known" || return 1 + fi + chmod 600 "$TXN/neighbors.expected.before" "$TXN/neighbors.desired" \ + "$TXN/neighbors.known" "$TXN/neighbors.before" +} +vc_vpc_neighbors_reconcile() { + vc_vpc_neighbors_snapshot "$TXN/neighbors.activation" || return 1 + vc_vpc_neighbors_subset "$TXN/neighbors.activation" "$TXN/neighbors.known" || return 1 + LOCAL_NEIGHBORS_CHANGED=yes + while IFS='|' read -r vc_ln_ip vc_ln_mac vc_ln_state; do + [ -n "$vc_ln_ip" ] || continue + vc_ln_present=$(awk -F'|' -v ip="$vc_ln_ip" '$1==ip {found=1} END {print found ? "yes" : "no"}' "$TXN/neighbors.desired") || return 1 + if [ "$vc_ln_present" = no ]; then + "$IP" -4 neigh del "$vc_ln_ip" dev "$BRIDGE" >/dev/null 2>&1 || return 1 + fi + done < "$TXN/neighbors.activation" + vc_ln_desired=$(cat "$TXN/neighbors.desired") || return 1 + vc_local_neighbor_ensure "$BRIDGE" "$vc_ln_desired" +} +vc_vpc_neighbors_rollback_guard() { + vc_vpc_neighbors_snapshot "$TXN/neighbors.rollback.current" || return 1 + vc_vpc_neighbors_subset "$TXN/neighbors.rollback.current" "$TXN/neighbors.known" +} +vc_vpc_neighbors_restore() { + # Administrative DOWN and manager activation can clear permanent entries. + # Restore only after the original ACL and unchanged bridge are verified. + vc_vpc_neighbors_rollback_guard || return 1 + while IFS='|' read -r vc_ln_ip vc_ln_mac vc_ln_state; do + [ -n "$vc_ln_ip" ] || continue + "$IP" -4 neigh del "$vc_ln_ip" dev "$BRIDGE" >/dev/null 2>&1 || return 1 + done < "$TXN/neighbors.rollback.current" + : > "$TXN/neighbors.rollback.expected" + while IFS='|' read -r vc_ln_ip vc_ln_mac vc_ln_state; do + [ -n "$vc_ln_ip" ] && [ "$vc_ln_mac" != - ] || continue + # Dynamic timer state cannot be replayed. Restore the sampled binding as + # STALE so the kernel can validate it; permanent bindings stay permanent. + vc_ln_restore_state=stale + case "$vc_ln_state" in + PERMANENT) vc_ln_restore_state=permanent ;; + NOARP) vc_ln_restore_state=noarp ;; + esac + "$IP" -4 neigh replace "$vc_ln_ip" lladdr "$vc_ln_mac" nud "$vc_ln_restore_state" dev "$BRIDGE" >/dev/null 2>&1 || return 1 + printf '%s|%s\n' "$vc_ln_ip" "$vc_ln_mac" >> "$TXN/neighbors.rollback.expected" || return 1 + done < "$TXN/neighbors.before" + vc_vpc_neighbors_snapshot "$TXN/neighbors.rollback.restored" || return 1 + vc_vpc_neighbors_subset "$TXN/neighbors.rollback.restored" "$TXN/neighbors.rollback.expected" || return 1 + awk -F'|' 'NF {print $1 "|" $2}' "$TXN/neighbors.rollback.restored" | LC_ALL=C sort > "$TXN/neighbors.rollback.pairs" || return 1 + LC_ALL=C sort "$TXN/neighbors.rollback.expected" > "$TXN/neighbors.rollback.expected.sorted" || return 1 + "$CMP" -s "$TXN/neighbors.rollback.pairs" "$TXN/neighbors.rollback.expected.sorted" || return 1 + awk -F'|' 'FILENAME==ARGV[1] {if ($3=="PERMANENT" || $3=="NOARP") stable[$1]=$3; next} + NF && ($1 in stable) && $3!=stable[$1] {bad=1} + END {exit bad ? 1 : 0}' "$TXN/neighbors.before" "$TXN/neighbors.rollback.restored" +} +'''.strip() + VPC_NATIVE_TRANSACTION_GUARD_SH = r''' vc_vpc_firewall_boundary_clean() { [ -n "${NFT:-}" ] || return 1 @@ -472,6 +576,54 @@ vc_nft_table_matches_snapshot() { } '''.strip() +VPC_APPLY_RECEIPT_SH = r''' +vc_vpc_sha256() ( + [ -n "${SHA256SUM:-}" ] || return 1 + vc_receipt_hash=$("$SHA256SUM" "$1") || return 1 + vc_receipt_hash=${vc_receipt_hash%% *} + printf '%s\n' "$vc_receipt_hash" | grep -Eq '^[0-9a-f]{64}$' || return 1 + printf '%s\n' "$vc_receipt_hash" +) +vc_vpc_nft_receipt() ( + [ -n "${SHA256SUM:-}" ] || { printf '%s\n' unavailable; exit 0; } + vc_receipt_all= + for vc_receipt_family in inet bridge arp; do + # Failed reads are not absence. Only a successful strict inventory may + # report an absent table, which never matches a successful apply receipt. + vc_receipt_inventory=$("$NFT" -s -n -y list tables "$vc_receipt_family") || exit 1 + vc_receipt_present=$(printf '%s\n' "$vc_receipt_inventory" | + awk -v family="$vc_receipt_family" -v wanted="$TABLE" ' + NF==0 {next} + NF!=3 || $1!="table" || $2!=family || $3!~/^[A-Za-z0-9_.-]+$/ {bad=1} + $3==wanted {found++} + END {if (bad || found>1) exit 1; print found+0}') || exit 1 + if [ "$vc_receipt_present" = 0 ]; then + printf '%s\n' missing + exit 0 + fi + vc_receipt_text=$("$NFT" -s -n -y list table "$vc_receipt_family" "$TABLE") || exit 1 + [ -n "$vc_receipt_text" ] || exit 1 + vc_receipt_hash=$(printf '%s\n' "$vc_receipt_text" | "$SHA256SUM") || exit 1 + vc_receipt_hash=${vc_receipt_hash%% *} + printf '%s\n' "$vc_receipt_hash" | grep -Eq '^[0-9a-f]{64}$' || exit 1 + vc_receipt_all="${vc_receipt_all}${vc_receipt_all:+,}$vc_receipt_hash" + done + printf '%s\n' "$vc_receipt_all" +) +vc_vpc_bridge_admin_state() ( + vc_receipt_link=$("$IP" -o link show dev "$BRIDGE") || exit 1 + printf '%s\n' "$vc_receipt_link" | awk ' + NF { + rows++ + if ($1!~/^[0-9]+:$/ || $3!~/^<[^>]+>$/) {bad=1; next} + flags=$3; sub(/^$/,"",flags) + n=split(flags,items,","); for (i=1;i<=n;i++) if (items[i]=="UP") up++ + } + END {if (bad || rows!=1 || up>1) exit 1; print (up==1 ? "up" : "down")}' +) +'''.strip() + + NETWORKMANAGER_VPC_PROFILE_SH = r''' vc_nm_profile_disk_validate() { vc_nm_disk_file=$1 @@ -1519,6 +1671,8 @@ def _state( destination=route["destination_cidr"], device=route["device"], table=route["table"], via=route.get("via") or "-", metric=route.get("metric", 100))) + for member in sorted(plan.local_members, key=lambda item: item.private_ip): + lines.append(f"local_neighbor|{member.private_ip}|{member.private_mac}") for peer in plan.peers: ips = sorted(set( [m.private_ip + "/32" for m in plan.remote_members @@ -1577,6 +1731,7 @@ WG=/usr/bin/wg; [ -x "$WG" ] || WG=/usr/sbin/wg NFT=/usr/sbin/nft; [ -x "$NFT" ] || NFT=/usr/bin/nft FLOCK=/usr/bin/flock; [ -x "$FLOCK" ] || FLOCK=/usr/bin/flock {BRIDGE_PORT_RECORDS_SH} +{LOCAL_NEIGHBOR_SH} {IP_RULE_CONVERGENCE_SH} BASE=/etc/virt-cloud/vpc-dataplane BRIDGE={_q(r.bridge)}; TABLE={_q(r.nft_table)}; ROUTE_TABLE={r.route_table}; RULE_PRIORITY={r.rule_priority} @@ -1819,6 +1974,7 @@ SYSCTL=$(for p in /usr/sbin/sysctl /usr/bin/sysctl /sbin/sysctl /bin/sysctl; do BRIDGE_CMD=$(for p in /usr/sbin/bridge /usr/bin/bridge /sbin/bridge /bin/bridge; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done) SYSTEMCTL=$(for p in /usr/bin/systemctl /bin/systemctl /usr/sbin/systemctl /sbin/systemctl; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done) {BRIDGE_PORT_RECORDS_SH} +{LOCAL_NEIGHBOR_SH} {IP_RULE_CONVERGENCE_SH} {WG_ALLOWED_IPS_SH} {VPC_ROUTE_RECORDS_SH} @@ -1831,7 +1987,7 @@ IFUP={_q(ifup_path)}; IFDOWN={_q(ifdown_path)}; IFQUERY={_q(ifquery_path)}; NMCL NM_PROFILE_MARKER={_q(DATAPLANE_OWNER_MARKER + " vpc=" + plan.vpc.vpc_id)} EXPECTED_DIGEST={_q(plan.fabric_plan_digest)} NFT_RULE_MARKER={_q(nft_rule_marker)} -COMMITTED=no; MUTATION_STARTED=no; ROUTES_CHANGED=no; RUNTIME_QUARANTINED=no; BRIDGE_CREATED=no; BRIDGE_WAS_UP=no; BRIDGE_TOPOLOGY_CAPTURED=no; BRIDGE_IFINDEX_BEFORE=; BRIDGE_PORTS_BEFORE=; NM_PRESERVE_ACTIVE_BRIDGE=no; NM_PRESERVED_PROFILE_UUID=; NM_CREATED=no; NM_CREATED_UUID=; NM_PROFILE_WAS_PRESENT=no; NM_WAS_ACTIVE=no; NM_DEVICE_QUARANTINED=no; NM_ACTIVATION_ATTEMPTED=no; IF_FRAGMENT_CHANGED=no; SYSCTL_CHANGED=no; SYSCTL_TMP=; RP_FILTER_CHANGED=no; PROXY_ARP_CHANGED=no; PROXY_ARP_PVLAN_CHANGED=no; NFT_FILE_CHANGED=no; NFT_MAIN_CHANGED=no; NFT_FILE_BEFORE=no; NFT_MAIN_BEFORE=no; BRIDGE_NFT_BEFORE=no; ARP_NFT_BEFORE=no; FABRIC_NFT_BEFORE=no; NFT_APPLIED=no; FABRIC_NFT_APPLIED=no; PROXY_ENTRIES_CHANGED=no; PROXY_FILE_CHANGED=no; FIREWALL_HARD_QUARANTINED=no; ROLLBACK_FAILED=no +COMMITTED=no; LOCAL_NEIGHBORS_CHANGED=no; MUTATION_STARTED=no; ROUTES_CHANGED=no; RUNTIME_QUARANTINED=no; BRIDGE_CREATED=no; BRIDGE_WAS_UP=no; BRIDGE_TOPOLOGY_CAPTURED=no; BRIDGE_IFINDEX_BEFORE=; BRIDGE_PORTS_BEFORE=; NM_PRESERVE_ACTIVE_BRIDGE=no; NM_PRESERVED_PROFILE_UUID=; NM_CREATED=no; NM_CREATED_UUID=; NM_PROFILE_WAS_PRESENT=no; NM_WAS_ACTIVE=no; NM_DEVICE_QUARANTINED=no; NM_ACTIVATION_ATTEMPTED=no; IF_FRAGMENT_CHANGED=no; SYSCTL_CHANGED=no; SYSCTL_TMP=; RP_FILTER_CHANGED=no; PROXY_ARP_CHANGED=no; PROXY_ARP_PVLAN_CHANGED=no; NFT_FILE_CHANGED=no; NFT_MAIN_CHANGED=no; NFT_FILE_BEFORE=no; NFT_MAIN_BEFORE=no; BRIDGE_NFT_BEFORE=no; ARP_NFT_BEFORE=no; FABRIC_NFT_BEFORE=no; NFT_APPLIED=no; FABRIC_NFT_APPLIED=no; PROXY_ENTRIES_CHANGED=no; PROXY_FILE_CHANGED=no; FIREWALL_HARD_QUARANTINED=no; ROLLBACK_FAILED=no {NFT_LIVE_SNAPSHOT_SH} {_NMCLI_FILENAME_RESOLVER_SH} {NETWORKMANAGER_VPC_PROFILE_SH} @@ -1841,6 +1997,9 @@ COMMITTED=no; MUTATION_STARTED=no; ROUTES_CHANGED=no; RUNTIME_QUARANTINED=no; BR {_VPC_SYSCTL_VALIDATION_SH} {VPC_NATIVE_TRANSACTION_GUARD_SH} {VPC_APPLY_RUNTIME_SH} +{VPC_LOCAL_NEIGHBOR_TRANSACTION_SH} +{VPC_APPLY_RECEIPT_SH} +SHA256SUM=$(command -v sha256sum || true) nft_plan_owned() {{ nft_plan_text=$1 nft_plan_digest=$2 @@ -1954,6 +2113,9 @@ vc_vpc_native_ifupdown_rollback_reopen() {{ fi return 0 }} +# The old boot helper rejects the new state record. Require the coordinated +# helper upgrade before writing state or taking the bridge down. +vc_vpc_neighbor_helper_ready || exit 71 # Never mutate even the bridge state while an unmarked, legacy, or malformed # transaction exists. New native transactions carry an exact backend marker. vc_vpc_firewall_boundary_clean || exit 79 @@ -2174,6 +2336,9 @@ rollback() {{ ROLLBACK_FAILED=yes fi fi + if [ "$BRIDGE_TOPOLOGY_CAPTURED" = yes ] || [ "$LOCAL_NEIGHBORS_CHANGED" = yes ]; then + vc_vpc_neighbors_rollback_guard || ROLLBACK_FAILED=yes + fi # First rollback action: quarantine this already-proven panel bridge before # touching routes, peers, proxy state, sysctls or raw nft tables. This # closes the failure window where an ACL readback error could otherwise @@ -2500,7 +2665,7 @@ rollback() {{ if [ "$ROLLBACK_FAILED" = no ] && [ "$BRIDGE_CREATED" = no ] && [ "$BRIDGE_TOPOLOGY_CAPTURED" = yes ]; then vc_vpc_bridge_topology_matches && vc_vpc_runtime_restore && - vc_vpc_bridge_topology_matches || ROLLBACK_FAILED=yes + vc_vpc_neighbors_restore && vc_vpc_bridge_topology_matches || ROLLBACK_FAILED=yes fi if [ "$ROLLBACK_NEW_BRIDGE_RETAINED" = no ]; then if [ -f "$TXN/owner.before" ]; then cp -p "$TXN/owner.before" "$OWNER_FILE" || ROLLBACK_FAILED=yes; else rm -f "$OWNER_FILE" || ROLLBACK_FAILED=yes; fi @@ -2559,7 +2724,11 @@ cp -p "$NFT_MAIN" "$TXN/nft_main.before"; NFT_MAIN_BEFORE=yes if [ -f "$NFT_FILE" ]; then cp -p "$NFT_FILE" "$TXN/nft_file.before"; NFT_FILE_BEFORE=yes; fi if [ -f "$SYSCTL_FILE" ]; then cp -p "$SYSCTL_FILE" "$TXN/sysctl.before"; fi if [ -f "$PROXY_FILE" ]; then cp -p "$PROXY_FILE" "$TXN/proxy_file.before"; fi +cat > "$TXN/neighbors.desired" <<'__VC_LOCAL_NEIGHBORS__' +{chr(10).join(sorted(f"{m.private_ip}|{m.private_mac}" for m in plan.local_members))} +__VC_LOCAL_NEIGHBORS__ vc_vpc_runtime_baseline || exit 71 +vc_vpc_neighbors_baseline || exit 71 if [ "$BRIDGE_TOPOLOGY_CAPTURED" = yes ]; then # Kernel DOWN and NM unmanaged can flush proxy neighbours and routes. The # original ACL, owner, manager, topology and runtime are now journaled. @@ -2818,6 +2987,8 @@ cat <<'{proxy_marker}' | LC_ALL=C sort -u > "$TXN/proxy.desired" {proxy_lines} {proxy_marker} "$CMP" -s "$TXN/proxy.after" "$TXN/proxy.desired" || exit 72 +# Restore exact member bindings after the manager may have cleared them. +vc_vpc_neighbors_reconcile || exit 72 ROUTES_CHANGED=yes cat > "$TXN/routes.desired" <<'{route_marker}' {route_lines} @@ -2925,9 +3096,18 @@ vpc_state_nft_set_matches || exit 74 vc_nft_table_matches_snapshot inet virt_cloud_wg_fabric \ "$TXN/fabric.nft.desired" || exit 74 vc_vpc_nft_matches_snapshot "$TXN/nft.desired" || exit 74 +# Bind the optimization receipt to this successful atomic apply, not to a +# later observation that might already contain external rule drift. +vc_apply_nft_receipt=$(vc_vpc_nft_receipt) || exit 74 +vc_vpc_nft_matches_snapshot "$TXN/nft.desired" || exit 74 +vc_ln_final_expected=$(cat "$TXN/neighbors.desired") || exit 74 +vc_ln_final_actual=$(vc_local_neighbor_records "$BRIDGE") || exit 74 +vc_ln_final_wanted=$(printf '%s\\n' "$vc_ln_final_expected" | awk 'NF {{print $0 "|PERMANENT"}}') || exit 74 +[ "$vc_ln_final_actual" = "$vc_ln_final_wanted" ] || exit 74 rm -f "$VPC_PEER_FILE" "$VPC_ROUTE_FILE" "$VPC_TENANT_FILE" "$VPC_TENANT_FILE.nft" printf '%s\\n' ready > "$TXN/state"; chmod 600 "$TXN/state" printf '%s\\n' '__VC_VPC_APPLIED__=yes' +printf '%s\\n' "__VC_VPC_NFT_RECEIPT__=$vc_apply_nft_receipt" printf '%s\\n' '__VC_VPC_OWNER__={DATAPLANE_OWNER_MARKER}' printf '%s\\n' '__VC_VPC_REVISION__={plan.desired_revision}' printf '%s\\n' '__VC_VPC_DIGEST__={plan.desired_digest}' @@ -3003,6 +3183,7 @@ MKTEMP=$(for p in /usr/bin/mktemp /bin/mktemp /usr/sbin/mktemp /sbin/mktemp; do BRIDGE_CMD=$(for p in /usr/sbin/bridge /usr/bin/bridge /sbin/bridge /bin/bridge; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done) SYSTEMCTL=$(for p in /usr/bin/systemctl /bin/systemctl /usr/sbin/systemctl /sbin/systemctl; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done) {BRIDGE_PORT_RECORDS_SH} +{LOCAL_NEIGHBOR_SH} {IP_RULE_CONVERGENCE_SH} {WG_ALLOWED_IPS_SH} BRIDGE={_q(r.bridge)}; TABLE={_q(r.nft_table)}; ROUTE_TABLE={r.route_table}; WG_IFACE={_q(FABRIC_INTERFACE)} @@ -3065,6 +3246,7 @@ done if [ -e "$TXN" ] || [ -L "$TXN" ]; then vc_vpc_native_transaction_owned || exit 79 fi +REMOVE_NEIGHBOR_EXPECTED=; REMOVE_NEIGHBOR_BEFORE= REMOVE_BRIDGE_TOPOLOGY_CAPTURED=no; REMOVE_BRIDGE_PORTS_BEFORE= vc_remove_bridge_port_names() {{ vc_remove_ports_raw=$(vc_bridge_port_records "$BRIDGE" 2>/dev/null) || return 1 @@ -3134,6 +3316,11 @@ if [ -f "$STATE_FILE" ]; then grep -Eq '(^|[[:space:]])bridge[[:space:]]+(forward_delay|stp_state|hello_time|ageing_time)([[:space:]]|$)' || exit 71 REMOVE_BRIDGE_PORTS_BEFORE=$(vc_remove_bridge_port_names) || exit 71 REMOVE_BRIDGE_TOPOLOGY_CAPTURED=yes + # vc-local-neighbor-remove-preflight-begin + REMOVE_NEIGHBOR_EXPECTED=$(vc_local_neighbor_expected "$STATE_FILE") || exit 71 + vc_local_neighbor_preflight "$BRIDGE" "$REMOVE_NEIGHBOR_EXPECTED" || exit 71 + REMOVE_NEIGHBOR_BEFORE=$(vc_local_neighbor_records "$BRIDGE") || exit 71 + # vc-local-neighbor-remove-preflight-end if [ -n "$REMOVE_BRIDGE_PORTS_BEFORE" ]; then # Removal cannot own or detach a running VM tap. Quarantine the same # bridge without NM deconfigure, then refuse until its ports are gone. @@ -3476,6 +3663,8 @@ if [ "$RESUME" = no ]; then set_phase preparing if [ -f "$STATE_FILE" ]; then cp -p "$STATE_FILE" "$TXN/state.before"; fi : > "$TXN/routes.before"; : > "$TXN/rules.before"; : > "$TXN/proxy.before" + printf '%s\\n' "$REMOVE_NEIGHBOR_BEFORE" > "$TXN/neighbors.before" + chmod 600 "$TXN/neighbors.before" chmod 600 "$TXN/routes.before" "$TXN/rules.before" "$TXN/proxy.before" else # A resumed transaction must use the original snapshots. Creating an empty @@ -3552,6 +3741,20 @@ EOF fi verify_routes_removed || exit 72 set_phase routes_removed +# vc-local-neighbor-remove-cleanup-begin +if "$IP" link show dev "$BRIDGE" >/dev/null 2>&1; then + vc_local_neighbor_preflight "$BRIDGE" "$REMOVE_NEIGHBOR_EXPECTED" || exit 71 + remove_neighbor_current=$(vc_local_neighbor_records "$BRIDGE") || exit 71 + while IFS='|' read -r remove_neighbor_ip remove_neighbor_mac remove_neighbor_state; do + [ -n "$remove_neighbor_ip" ] || continue + "$IP" -4 neigh del "$remove_neighbor_ip" dev "$BRIDGE" || exit 72 + done <<__VC_REMOVE_LOCAL_NEIGHBORS__ +$remove_neighbor_current +__VC_REMOVE_LOCAL_NEIGHBORS__ + remove_neighbor_current=$(vc_local_neighbor_records "$BRIDGE") || exit 79 + [ -z "$remove_neighbor_current" ] || exit 79 +fi +# vc-local-neighbor-remove-cleanup-end if [ "$MODE" = networkmanager_nmcli ]; then if [ -n "$NMCLI" ] && "$NMCLI" connection show "$BRIDGE" >/dev/null 2>&1; then "$NMCLI" connection delete "$BRIDGE" >/dev/null 2>&1 || exit 72 @@ -3577,6 +3780,8 @@ if "$IP" link show dev "$BRIDGE" >/dev/null 2>&1; then [ -z "$bridge_routes6" ] || exit 79 proxy_left=$("$IP" neigh show proxy dev "$BRIDGE" 2>/dev/null | awk '$1 ~ /^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$/') || exit 71 [ -z "$proxy_left" ] || exit 79 + remove_neighbor_left=$(vc_local_neighbor_records "$BRIDGE") || exit 79 + [ -z "$remove_neighbor_left" ] || exit 79 "$IP" link del dev "$BRIDGE" type bridge || exit 72 fi if "$IP" link show dev "$BRIDGE" >/dev/null 2>&1; then exit 72; fi @@ -3762,6 +3967,13 @@ def render_observe_script( "__VC_FABRIC_ID__", plan.vpc.fabric_id).replace( "__VC_FABRIC_NETWORK__", "100.64.0.0/10").replace( "__VC_VPC_STATE_OBSERVE__", "yes") + expected_state_hash = hashlib.sha256(_state(plan, routes, firewall_mode).encode()).hexdigest() + expected_ifup = ( + f"# {DATAPLANE_OWNER_MARKER} vpc={plan.vpc.vpc_id} " + f"revision={plan.desired_revision} digest={plan.desired_digest}\n" + f"iface {r.bridge} inet manual\n" + " bridge_ports none\n bridge-stp off\n") + expected_ifup_hash = hashlib.sha256(expected_ifup.encode()).hexdigest() nft_marker = _nft_marker(plan) # These comments are emitted by render_vpc_nft_rules. Requiring the full # owned shape prevents a stale or foreign table being reported as active @@ -3848,6 +4060,7 @@ set -eu PATH=/usr/sbin:/usr/bin:/sbin:/bin IP=/usr/sbin/ip; [ -x "$IP" ] || IP=/usr/bin/ip {BRIDGE_PORT_RECORDS_SH} +{LOCAL_NEIGHBOR_SH} {IP_RULE_CONVERGENCE_SH} {WG_ALLOWED_IPS_SH} {VPC_ROUTE_RECORDS_SH} @@ -3872,6 +4085,9 @@ NFT_FILE={_q(nft_file)}; NFT_INCLUDE={_q(nft_include)}; LEGACY_MANIFEST={_q(LEGA NFT_MARKER={_q(nft_marker)} EXPECTED_DIGEST={_q(plan.fabric_plan_digest)} {NFT_LIVE_SNAPSHOT_SH} +{VPC_APPLY_RECEIPT_SH} +{VPC_LOCAL_NEIGHBOR_TRANSACTION_SH} +SHA256SUM=$(command -v sha256sum || true) {_NMCLI_FILENAME_RESOLVER_SH} {NETWORKMANAGER_VPC_PROFILE_SH} {_FIREWALL_MODE_GUARD_SH} @@ -4061,6 +4277,50 @@ if [ -f "$PROXY_FILE" ] && sed -n '1p' "$PROXY_FILE" | grep -Fqx '# virt-cloud-v if [ "$proxy_records" = "$expected_proxy" ]; then proxy_state=owned; fi fi printf '%s\\n' '__VC_VPC_PROXY_STATE__='"$proxy_state" +# vc-local-neighbor-observe-begin +local_neighbor_state=missing +local_neighbor_expected={_q(chr(10).join(sorted(f"{m.private_ip}|{m.private_mac}" for m in plan.local_members)))} +if local_neighbor_saved=$(vc_local_neighbor_expected "$STATE") && + local_neighbor_actual=$(vc_local_neighbor_records "$BRIDGE"); then + local_neighbor_wanted=$(printf '%s\\n' "$local_neighbor_expected" | awk 'NF {{print $0 "|PERMANENT"}}') + if [ "$local_neighbor_saved" = "$local_neighbor_expected" ] && + [ "$local_neighbor_actual" = "$local_neighbor_wanted" ]; then local_neighbor_state=owned; fi +fi +printf '%s\\n' "__VC_VPC_LOCAL_NEIGHBOR_STATE__=$local_neighbor_state" +# vc-local-neighbor-observe-end +# vc-apply-receipt-observe-begin +helper_contract=missing +if vc_vpc_neighbor_helper_ready; then helper_contract=owned; fi +printf '%s\\n' "__VC_VPC_HELPER_CONTRACT__=$helper_contract" +bridge_admin=missing +if [ "$bridge_shape" = yes ]; then + bridge_admin=$(vc_vpc_bridge_admin_state) || exit 74 +fi +printf '%s\\n' "__VC_VPC_BRIDGE_ADMIN__=$bridge_admin" +nft_file_exact=no +if [ "$nft_file_ok" = yes ] && [ -n "$SHA256SUM" ]; then + nft_file_hash=$(vc_vpc_sha256 "$NFT_FILE") || exit 74 + if [ "$nft_file_hash" = {_q(hashlib.sha256((render_vpc_nft_rules(plan).rstrip() + chr(10)).encode()).hexdigest())} ]; then nft_file_exact=yes; fi +fi +printf '%s\\n' "__VC_VPC_NFT_FILE_EXACT__=$nft_file_exact" +state_file_exact=no +if [ -n "$SHA256SUM" ] && [ -f "$STATE" ] && [ ! -L "$STATE" ] && + [ "$("$STAT" -c '%u:%g:%a' "$STATE" 2>/dev/null)" = 0:0:600 ]; then + state_file_hash=$(vc_vpc_sha256 "$STATE") || exit 74 + if [ "$state_file_hash" = {_q(expected_state_hash)} ]; then state_file_exact=yes; fi +fi +printf '%s\\n' "__VC_VPC_STATE_FILE_EXACT__=$state_file_exact" +persistence_exact=no +if [ "$nm_owned" = yes ] && [ "$ifup_owned" = no ]; then + persistence_exact=yes +elif [ "$ifup_owned" = yes ] && [ "$nm_present" = no ] && [ -n "$SHA256SUM" ]; then + ifup_file_hash=$(vc_vpc_sha256 "$IF_FRAGMENT") || exit 74 + if [ "$ifup_file_hash" = {_q(expected_ifup_hash)} ]; then persistence_exact=yes; fi +fi +printf '%s\\n' "__VC_VPC_PERSISTENCE_EXACT__=$persistence_exact" +vc_observe_nft_receipt=$(vc_vpc_nft_receipt) || exit 74 +printf '%s\\n' "__VC_VPC_NFT_RECEIPT__=$vc_observe_nft_receipt" +# vc-apply-receipt-observe-end if "$NFT" list table inet "$TABLE" >/dev/null 2>&1; then printf '%s\\n' '__VC_VPC_NFT_TABLE__='"$TABLE"; else printf '%s\\n' '__VC_VPC_NFT_TABLE__=missing'; fi if [ -f "$STATE" ] && grep -Fqx 'owner|{DATAPLANE_OWNER_MARKER}' "$STATE"; then printf '%s\\n' '__VC_VPC_STATE_OWNER__=owned' @@ -4590,6 +4850,33 @@ class VpcPrivateHostAdapter: return (mode, ifup_path, ifdown_path, ifquery_path, _firewall_mode(firewall_mode)) + def _receipt_scope(self, plan: VpcDataplanePlan) -> tuple[str, ...] | None: + current = self.profile + if current is None and callable(getattr(self.manager, "current", None)): + current = self.manager.current() + if not isinstance(current, Mapping) or any( + not isinstance(current.get(key), str) or not current[key] + for key in ("ssh_target", "ssh_config_file", "host", "fingerprint")): + return None + if type(current.get("port")) is not int or not 1 <= current["port"] <= 65535: + return None + return (plan.host_id, plan.vpc.vpc_id, + str(current.get("ssh_target", "")), + str(current.get("ssh_config_file", "")), + str(current.get("host", "")), str(current.get("port", "")), + str(current.get("fingerprint", ""))) + + @staticmethod + def _save_receipt(scope: tuple[str, ...] | None, fingerprint: str, receipt: Any) -> None: + if scope is None or not isinstance(receipt, str) or not re.fullmatch( + r"[0-9a-f]{64}(?:,[0-9a-f]{64}){2}", receipt): + return + with _APPLY_RECEIPT_LOCK: + _APPLY_RECEIPTS[scope] = (fingerprint, receipt) + _APPLY_RECEIPTS.move_to_end(scope) + while len(_APPLY_RECEIPTS) > _APPLY_RECEIPT_LIMIT: + _APPLY_RECEIPTS.popitem(last=False) + def apply(self, plan: VpcDataplanePlan) -> dict[str, Any]: mode, ifup_path, ifdown_path, ifquery_path, firewall_mode = self._render_contract() script = render_apply_script( @@ -4597,9 +4884,32 @@ class VpcPrivateHostAdapter: ifup_path=ifup_path, ifdown_path=ifdown_path, ifquery_path=ifquery_path, ) - markers = _markers(self._run(script)) + scope = self._receipt_scope(plan) + fingerprint = hashlib.sha256(script.encode("utf-8")).hexdigest() + with _APPLY_RECEIPT_LOCK: + previous = _APPLY_RECEIPTS.pop(scope, None) + if plan.local_members and previous is not None and previous[0] == fingerprint: + # Exceptions propagate; a failed read must never trigger writes. + # Pop first so timeouts/malformed receipts cannot keep stale trust. + _observed, host_ready, receipt = self._observe(plan) + if host_ready and receipt == previous[1]: + self._save_receipt(scope, fingerprint, receipt) + return {"state": "applying", "readback": "pending", "rollback": "none", "owner": DATAPLANE_OWNER_MARKER} + output = self._run(script) + markers = _markers(output) if markers.get("applied") != "yes": raise VpcDataplaneError("宿主机未确认 VPC 应用", 502, "vpc_apply_unconfirmed") + receipt_fields = { + "applied": "yes", "owner": DATAPLANE_OWNER_MARKER, + "revision": str(plan.desired_revision), "digest": plan.desired_digest, + } + receipt_keys = (*receipt_fields, "nft_receipt") + receipt_unique = all(sum( + line.startswith(f"__VC_VPC_{key.upper()}__=") + for line in output.splitlines()) == 1 for key in receipt_keys) + if (plan.local_members and receipt_unique + and all(markers.get(key) == value for key, value in receipt_fields.items())): + self._save_receipt(scope, fingerprint, markers.get("nft_receipt")) return {"state": "applying", "readback": "pending", "rollback": "none", "owner": DATAPLANE_OWNER_MARKER} def remove(self, plan: VpcDataplanePlan) -> dict[str, Any]: @@ -4612,6 +4922,9 @@ class VpcPrivateHostAdapter: tombstone. """ mode, ifup_path, ifdown_path, ifquery_path, firewall_mode = self._render_contract() + scope = self._receipt_scope(plan) + with _APPLY_RECEIPT_LOCK: + _APPLY_RECEIPTS.pop(scope, None) markers = _markers(self._run(render_remove_script( plan, deployment_mode=mode, firewall_mode=firewall_mode, ifup_path=ifup_path, @@ -4652,9 +4965,24 @@ class VpcPrivateHostAdapter: self, plan: VpcDataplanePlan, *, xml_by_instance: Mapping[str, Any] | None = None, ) -> dict[str, Any]: + observed, _host_ready, _receipt = self._observe( + plan, xml_by_instance=xml_by_instance) + return observed + + def _observe( + self, plan: VpcDataplanePlan, *, + xml_by_instance: Mapping[str, Any] | None = None, + ) -> tuple[dict[str, Any], bool, Any]: _mode, _ifup, _ifdown, _ifquery, firewall_mode = self._render_contract() - parsed = _markers(self._run(render_observe_script( - plan, firewall_mode=firewall_mode))) + output = self._run(render_observe_script(plan, firewall_mode=firewall_mode)) + parsed = _markers(output) + receipt_count = sum(line.startswith("__VC_VPC_NFT_RECEIPT__=") + for line in output.splitlines()) + receipt = parsed.get("nft_receipt") + if (receipt_count > 1 or (receipt_count == 1 + and receipt not in {"missing", "unavailable"} + and not re.fullmatch(r"[0-9a-f]{64}(?:,[0-9a-f]{64}){2}", receipt))): + raise VpcDataplaneError("VPC 回读收据无效", 502, "vpc_readback_invalid") expected_routes = { (x["destination_cidr"], x["device"], str(x["table"]), x.get("via"), str(x.get("metric", 100))) @@ -4732,7 +5060,7 @@ class VpcPrivateHostAdapter: } expected_peer_states = {p.host_id: "owned" for p in plan.peers} expected_bridge_nft = "yes" if plan.local_members else "none" - host_config_verified = (parsed.get("bridge") == plan.resources.bridge + host_dataplane_verified = (parsed.get("bridge") == plan.resources.bridge and parsed.get("bridge_owner") == DATAPLANE_OWNER_MARKER and (parsed.get("firewall_mode") == firewall_mode or (firewall_mode == FIREWALL_MODE_NATIVE @@ -4748,6 +5076,7 @@ class VpcPrivateHostAdapter: and parsed.get("proxy_arp") == "0" and parsed.get("proxy_arp_pvlan") == "1" and parsed.get("proxy_state") == "owned" + and parsed.get("local_neighbor_state") == "owned" and actual_proxy_ips == expected_proxy_ips and parsed.get("rp_filter") == "2" and parsed.get("nft_table") == plan.resources.nft_table @@ -4769,8 +5098,21 @@ class VpcPrivateHostAdapter: and (plan.fabric_peer_keys is None or parsed.get("wg_peer_set") == "exact") and expected_peer_states == parsed.get("wg_states", {}) - and parsed.get("arp_nft") == ("yes" if plan.local_members else "none") - and all(x.get("config_verified") is True for x in members)) + and parsed.get("arp_nft") == ("yes" if plan.local_members else "none")) + host_config_verified = host_dataplane_verified and all( + x.get("config_verified") is True for x in members) + host_ready = bool( + host_dataplane_verified and route_links_ready + and parsed.get("firewall_mode") == firewall_mode + and parsed.get("firewall_state") == "owned" + and parsed.get("state_firewall_mode") == firewall_mode + and plan.fabric_peer_keys is not None + and parsed.get("wg_peer_set") == "exact" + and parsed.get("helper_contract") == "owned" + and parsed.get("bridge_admin") == "up" + and parsed.get("nft_file_exact") == "yes" + and parsed.get("state_file_exact") == "yes" + and parsed.get("persistence_exact") == "yes") active = bool( host_config_verified and route_links_ready @@ -4800,7 +5142,7 @@ class VpcPrivateHostAdapter: proxy_arp_pvlan = (int(parsed["proxy_arp_pvlan"]) if str(parsed.get("proxy_arp_pvlan", "")).isdigit() else None) - return VpcDataplaneObservation( + observed = VpcDataplaneObservation( plan.vpc.vpc_id, plan.host_id, plan.desired_revision, plan.desired_revision if active else 0, plan.desired_digest, plan.desired_digest if active else None, @@ -4824,6 +5166,7 @@ class VpcPrivateHostAdapter: else guest_reason_code or "vpc_private_dataplane_readback_mismatch"), ).as_dict() + return observed, host_ready, parsed.get("nft_receipt") __all__ = ["DATAPLANE_OWNER_MARKER", "FABRIC_INTERFACE", "PRIVATE_INTERFACE", "FIREWALL_MODE_NATIVE", "VpcDataplaneError", "VpcFabricPeer", "VpcDataplaneResources", "VpcDataplanePlan", "VpcDataplaneObservation", "derive_private_mac", "private_guest_routes", "private_guest_network_generation", "resource_names", "parse_vpc_dataplane_plan", "render_vpc_nft_rules", "render_apply_script", "render_remove_script", "render_observe_script", "patch_private_interface_xml", "VpcPrivateHostAdapter"]