From 8172a3f3be6c93f3a74f710c53278f7cedbfa2be Mon Sep 17 00:00:00 2001 From: admin Date: Wed, 14 Oct 2026 16:02:03 +0800 Subject: [PATCH] fix: restore preserved VPC bridges before routes --- docs/VALIDATION.md | 32 ++ docs/VPC_DATAPLANE_ACCEPTANCE.md | 22 +- docs/知识库/Memory/CURRENT.md | 18 + engine/tests/test_fabric.py | 2 +- .../tests/test_fabric_libvirt_independence.py | 4 +- .../tests/test_fabric_nm_bridge_preserve.py | 201 ++++++++++- .../tests/test_fabric_preserved_isolation.py | 328 ++++++++++++++++++ engine/tests/test_vpc_apply_receipts.py | 2 +- .../test_vpc_local_neighbor_transaction.py | 2 +- engine/virsh_panel/fabric.py | 192 +++++++++- engine/virsh_panel/vpc_dataplane.py | 2 +- 11 files changed, 775 insertions(+), 30 deletions(-) create mode 100644 engine/tests/test_fabric_preserved_isolation.py diff --git a/docs/VALIDATION.md b/docs/VALIDATION.md index 54812c83..2fc874ab 100644 --- a/docs/VALIDATION.md +++ b/docs/VALIDATION.md @@ -1,5 +1,37 @@ # 本版验证记录 +## 2026-10-14:NM 保留私网桥的路由恢复顺序 + +- 现场显式 Fabric 核对 `cfb4e6913f764049bcbf507be55d8b40` 中,Debian 节点 + 已 `succeeded/observe/complete`,AlmaLinux 节点在 `final_readback` 失败。 + 同秒宿主日志为 `apply_check=convergence exit_status=79`,恢复 helper 为 + `helper_phase=vpc exit_status=72`,内核报 `Device for nexthop is not up.`。 + runner 将 79 泛化为 `journal_conflict`,不能据此断言旧事务损坏。 +- Alma 的新 v1 helper 已发布;旧 VPC 的桥保留两个运行 tap 和原 NM profile, + 但无行政 UP flag。代码先保存带 tap 桥身份并置 DOWN,恢复时跳过 NM 重新激活, + 却在恢复桥 UP 之前写本地 `/32` 路由。旧 `ss` 的 paused 标签不是本轮直接原因。 +- 修复顺序为:ACL/归属、sysctl、proxy、策略规则和永久邻居就绪,再恢复原桥的 + 行政 UP、复核 UUID/ifindex/tap,最后写路由并逐条精确回读目的、设备、table、 + via/onlink、proto 186 和 metric。允许 UP 但没有 carrier 的桥;任一步失败仍由 + 既有 cleanup 关闭被保留的桥,不重建桥、不调用 NM 反激活或拆除 tap。 +- 在策略规则变更和开桥之前额外读取 live inet forward 链,要求正确 hook/priority/policy,完整 CIDR + 四条边界规则、成员入口反伪造及出口隔离规则各一次,且在任何 accept/jump/goto/return 等 + 提前控制流前。整条链只接受生成器的匹配/计数/verdict 语法,未知改包或复制语句 + 同样拒绝;三个 IPv4 集合必须各一次、元素均是本 CIDR 内的主机地址。 + 缺失、篡改或读失败保持 DOWN,不以 owner comment 代替这些证明。 + 现场五条防回退规则已只读确认。本修复允许已授权流量逐步恢复,不承诺整个恢复 + 成功收尾前零流量;没有恢复退役 firewalld guard,也没有扩大网络权限。 +- helper 第七行升级为 `vpc-state-route-scope-contract-v2`,Fabric observer 和 + VPC apply 门禁同步升级,保证已安装的 v1 helper 经显式核对替换,普通 GET 不写。 +- 新行为回归在隔离旧 `68775a3` 上 **4 failed**,healthy/无 carrier × sh/bash + 均复现同文案和退出 72。修复后 13 个不重叠相关模块共 **1,340 passed**: + Fabric/邻居/state/独立启动/nft 五模块 522,VPC 回执/邻居事务及 Fabric 最终回读/ + 阶段日志/恢复顺序/路由六模块 493,保留桥恢复 61,live 隔离合同 264。 + 8 个 Python 文件 AST、两种管理器在 sh/dash/bash 下 15 项渲染语法检查和 + `git diff --check` 通过,独立复核通过;源码测试不等于真机恢复。 +- 本轮仅只读检查现场,尚未部署这次顺序修复或重试网络任务;后续先确认新 helper + 与 Fabric 节点完整回读,再通过原加入任务推进,不删除旧 VPC 或直接写成功状态。 + ## 2026-10-14:同宿主多 VPC 路由校验修复 - 现场新 VPC 的宿主回执为 `vpc_remote_command_failed`,成员网卡证据为空; diff --git a/docs/VPC_DATAPLANE_ACCEPTANCE.md b/docs/VPC_DATAPLANE_ACCEPTANCE.md index 58741a30..309b5a9a 100644 --- a/docs/VPC_DATAPLANE_ACCEPTANCE.md +++ b/docs/VPC_DATAPLANE_ACCEPTANCE.md @@ -70,7 +70,7 @@ 后置,以及跨到另一 VPC 网段的恶意 route 仍被拒绝。 宿主 `/usr/local/libexec/virt-cloud-wg-fabric` 还须包含第七行契约 -`# vpc-state-route-scope-contract-v1`。仅更新 Engine 不会替换已安装 helper; +`# vpc-state-route-scope-contract-v2`。仅更新 Engine 不会替换已安装 helper; 同计划旧 helper 会被 Fabric 观察判为需要升级,普通 GET 和周期观察不执行升级。 已有邻居契约、状态和在途操作均完整的环境,可通过原 Fabric 的显式“重新核对” 受控发布当前 helper;有旧邻居状态或未完成事务时,仍按上节先核实迁移证据。 @@ -80,6 +80,26 @@ helper 升级并通过完整回读后,正常关闭首次接入尚未完成的 任务执行“核对 / 重试”,再按保留的手动电源意图启动并验证 private0、`/32` 和来宾路由。不得重建加入任务或直接写库标记成功;本地回归不代替该现场验收。 +## NetworkManager 带运行网卡的桥恢复 + +对带运行 tap 的受管桥,Fabric 隔离只关闭桥的行政状态,保留 NM active profile、 +UUID、ifindex 和完整端口集合。恢复必须先证明 ACL、sysctl、proxy、策略规则和 +永久邻居就绪,再恢复行政 UP,随后安装 `/32` 路由并精确回读;Linux 拒绝向 +行政 DOWN 的设备安装该路由,不能把启用桥放在路由之后。 + +开桥前读取实际 inet forward 链,完整 CIDR 的四条防越界规则、成员入口反伪造与出口隔离规则 +必须各一次,位于任何提前放行或跳转之前,且 hook/priority/policy 正确;forward 链 +只接受生成器的匹配/计数/verdict 语法,未知改包或复制语句拒绝;三个 IPv4 集合 +必须各一次且主机元素均在本 CIDR 内。缺证据保持关闭。网络隔离依赖这些现有受管 ACL,允许授权流量逐步恢复。恢复任一步 +失败都须关闭保留桥,并确认原 UUID/ifindex/tap 未改变;不能手动开桥绕过检查。 +同时验证 UP 但 NO-CARRIER/operstate DOWN 的合法情况,不把没有运行端口误判为 +行政关闭。全程检查 br0、公网默认路由及未归属的规则保持原样。 + +若页面显示 `final_readback/journal_conflict`,先关联原 ID 的宿主日志。 +`apply_check=convergence`、`helper_phase=vpc` 和 nexthop device 报错属于恢复 +顺序证据,不能通过删除 journal 或重复创建 VPC 处理。helper v2 安装成功后, +仍需确认 Fabric active、原私网桥和成员路由,以及来宾地址和真实私网流量。 + ## 成员接入和回读 对一个跨两台宿主机的 VPC,至少放置两个成员和一个 ACL,逐台检查: diff --git a/docs/知识库/Memory/CURRENT.md b/docs/知识库/Memory/CURRENT.md index 4eeb3e77..2c13ac77 100644 --- a/docs/知识库/Memory/CURRENT.md +++ b/docs/知识库/Memory/CURRENT.md @@ -10,6 +10,24 @@ tags: # 当前项目交接:Virt Cloud +## 2026-10-14 Fabric 核对失败与保留桥恢复顺序 + +- **现场新事实**:`68775a3` 已部署,显式核对中 Debian 节点成功,Alma 节点在 + final_readback/convergence 失败;v1 helper 已安装,恢复 helper 的 VPC 阶段报 + nexthop device 未 UP、退出 72。旧 VPC 的带 tap 桥保留原 NM 身份但行政关闭, + 不是旧 `ss` paused 标签直接阻塞新 VPC。 +- **已修源码**:先校验 live ACL 的完整 CIDR 防回退规则,准备 sysctl/proxy、 + 策略规则和永久邻居,再以原身份恢复桥 UP 后写路由并精确回读;失败仍关闭桥。 + helper 契约升级 v2,普通观察不下发,原 Fabric 显式核对负责发布。 +- **验证边界**:旧实现 4 个行为组合均复现;13 个相关模块 1,340 项通过, + AST/渲染 Shell 语法/diff 及独立复核通过。 + 尚未部署本次顺序修复、未手动开桥或重试网络任务,不能记为现场已经恢复。 +- **下一步**:发布并回读 v2 helper,确认 Fabric active 及旧成员私网恢复,再推进 + 原加入任务;不删除旧 VPC、不清理 journal、不直接写成功状态。 +- 证据与验收见仓库 `docs/VALIDATION.md`、`docs/VPC_DATAPLANE_ACCEPTANCE.md`; + 关系沿用 [[Concepts/Virt Cloud WireGuard Fabric与VPC L3私网]] 与 + [[Concepts/Virt Cloud VPC永久邻居与重复协调]]。 + ## 2026-10-14 同宿主多 VPC 路由校验修复 - **当前源码**:在 `venix` 的 `54b55e2` 基础上完成本轮修复与回归,尚未部署。 diff --git a/engine/tests/test_fabric.py b/engine/tests/test_fabric.py index ce28c225..5570edac 100644 --- a/engine/tests/test_fabric.py +++ b/engine/tests/test_fabric.py @@ -341,7 +341,7 @@ def test_observe_artifact_state_accepts_only_kind_specific_header( f"# fabric-plan-digest={current.digest}\n" "# libvirt-startup-independent-v1\n" "# vpc-local-neighbor-contract-v1\n" - "# vpc-state-route-scope-contract-v1\n", + "# vpc-state-route-scope-contract-v2\n", encoding="utf-8", ) artifact.chmod(int(mode, 8)) diff --git a/engine/tests/test_fabric_libvirt_independence.py b/engine/tests/test_fabric_libvirt_independence.py index 18ca2893..1b25f57c 100644 --- a/engine/tests/test_fabric_libvirt_independence.py +++ b/engine/tests/test_fabric_libvirt_independence.py @@ -149,7 +149,7 @@ def startup_files(root, current, *, legacy=False, gate=True, if legacy_neighbors and name == "helper": content = content.replace("# vpc-local-neighbor-contract-v1\n", "") if legacy_routes and name == "helper": - content = content.replace("# vpc-state-route-scope-contract-v1\n", "") + content = content.replace("# vpc-state-route-scope-contract-v2\n", "") target = root / name target.write_text(content) target.chmod(mode) @@ -231,7 +231,7 @@ def test_explicit_reconcile_upgrades_same_plan_then_skips_current_hosts(tmp_path rendered = _apply_script(actual_plan, host, keys) assert FABRIC_STARTUP_ISOLATION_MARKER in rendered assert "# vpc-local-neighbor-contract-v1" in rendered - assert "# vpc-state-route-scope-contract-v1" in rendered + assert "# vpc-state-route-scope-contract-v2" in rendered assert 'record_desired_absent gate_generator' in rendered startup_files(self.manager.startup_root, actual_plan, gate=False) return super().dispatch_apply(actual_plan, host, keys, operation_id) diff --git a/engine/tests/test_fabric_nm_bridge_preserve.py b/engine/tests/test_fabric_nm_bridge_preserve.py index ef2288cf..bf0d1d72 100644 --- a/engine/tests/test_fabric_nm_bridge_preserve.py +++ b/engine/tests/test_fabric_nm_bridge_preserve.py @@ -5,12 +5,19 @@ import os import shlex import subprocess import sys +from dataclasses import replace from pathlib import Path import pytest -from virsh_panel.fabric import _BOOT_HELPER -from virsh_panel.shell_contracts import BRIDGE_PORT_RECORDS_SH, LOCAL_NEIGHBOR_SH +from test_vpc_nft_dataplane import _plan +from virsh_panel.fabric import _BOOT_HELPER, _VPC_STATE_LOADER +from virsh_panel.shell_contracts import ( + BRIDGE_PORT_RECORDS_SH, IP_RULE_CONVERGENCE_SH, LOCAL_NEIGHBOR_SH, +) +from virsh_panel.vpc_dataplane import ( + _routes, _state, render_vpc_nft_rules, resource_names, +) VPC_ID='vpc-preserve-test' TOKEN=hashlib.sha256(VPC_ID.encode()).hexdigest()[:11] @@ -28,9 +35,50 @@ if name=='stat': elif name=='sha256sum':print(hashlib.sha256(sys.stdin.buffer.read()).hexdigest()) elif name=='ip': if args==['link','show','dev',b]:pass - elif args==['-o','-4','neigh','show','dev',b]:pass + elif args==['-o','-4','neigh','show','dev',b]: + for address,mac in sorted(s.get('neighbors',{}).items()):print(address+' dev '+b+' lladdr '+mac+' PERMANENT') + elif args[:3]==['-4','neigh','replace']: + assert args[3:4] and args[4]=='lladdr' and args[6:]==['nud','permanent','dev',b],args + s.setdefault('neighbors',{})[args[3]]=args[5];save() + elif args[:3]==['neigh','replace','proxy']: + assert args[4:]==['dev',b],args + s.setdefault('proxies',[]).append(args[3]);save() + elif args[:5]==['-o','-4','-N','rule','show']: + for pref,row in sorted(s.get('rules',{}).items()): + if len(args)==5 or args[5:]==['pref',pref]:print(row) + elif args[:4]==['-4','rule','add','pref']: + pref=args[4];destination=args[args.index('to')+1];table=args[args.index('lookup')+1] + iif=' iif '+args[args.index('iif')+1] if 'iif' in args else '' + s.setdefault('rules',{})[pref]=pref+': from all to '+destination+iif+' lookup '+table;save() + elif args[:2]==['route','replace']: + device=args[args.index('dev')+1] + s.setdefault('route_attempts',[]).append({'args':args,'up':s['up']});save() + if (device==b and not s['up']) or scenario=='route_error_after_up': + print('Error: Device for nexthop is not up.',file=sys.stderr);raise SystemExit(2) + destination=args[2];table=args[args.index('table')+1] + # Match iproute2: host routes omit /32 and may retain linkdown without + # LOWER_UP, even though an administratively DOWN OIF rejects replacement. + row=destination.removesuffix('/32')+' dev '+device+' table '+table+' proto 186' + if 'via' in args:row+=' via '+args[args.index('via')+1]+' onlink' + else:row+=' scope link' + row+=' metric '+args[args.index('metric')+1] + if scenario=='up_no_carrier':row+=' linkdown' + if scenario=='route_wrong_device':row=row.replace('dev '+device,'dev foreign0') + elif scenario=='route_wrong_via' and 'via' in args:row=row.replace('via '+args[args.index('via')+1],'via 10.44.0.254') + elif scenario=='route_wrong_proto':row=row.replace('proto 186','proto 187') + elif scenario=='route_wrong_metric':row=row.replace('metric '+args[args.index('metric')+1],'metric 999') + elif scenario=='route_wrong_table':row=row.replace('table '+table,'table 999') + elif scenario=='route_extra_nhid':row+=' nhid 41' + elif scenario=='route_missing_onlink':row=row.replace(' onlink','') + if scenario=='route_missing':row='' + elif scenario=='route_duplicate':row+='\n'+row + s.setdefault('routes',{})[table+'|'+destination]=row;save() + elif args[:6]==['-N','-o','-4','route','show','table']: + assert args[7]=='exact',args + print(s.get('routes',{}).get(args[6]+'|'+args[8],'')) elif args==['-d','link','show','dev',b]:print('bridge forward_delay 1500 stp_state 0') - elif args==['-o','link','show','dev',b]:print(str(s['index'])+': '+b+': mtu 1500') + elif args==['-o','link','show','dev',b]: + print(str(s['index'])+': '+b+': mtu 1500 state '+('DOWN' if scenario=='up_no_carrier' or not s['up'] else 'UP')) elif args==['-o','link','show','master',b]: for p in s['ports']:print('4: '+p+': mtu 1500 master '+b) elif args==['link','set','dev',b,'down']: @@ -42,11 +90,16 @@ elif name=='ip': s['down_once']=True;save() elif args==['link','set','dev',b,'up']: s['up']=scenario!='up_no_effect' + s['activation_snapshot']={key:s.get(key) for key in ('neighbors','rules','proxies','isolation_reads')} if scenario=='index_after_up':s['index']+=1 if scenario=='ports_after_up':s['ports']=[] save() if scenario=='up_error':raise SystemExit(1) else:raise SystemExit(97) +elif name=='nft': + assert args==['-s','list','table','inet',os.environ['NFT_TABLE']],args + s['isolation_reads']=s.get('isolation_reads',0)+1;save() + print((root/'live.nft').read_text()) elif name=='nmcli': if args==['device','set',b,'managed','no']:s['ports']=[];s['managed']='no';s['uuid']='--';save() elif args[:2]==['connection','up'] or args==['device','set',b,'managed','yes']:raise SystemExit(98) @@ -65,7 +118,7 @@ elif name=='nmcli': def _fixture(tmp_path,scenario): tools=tmp_path/'bin';tools.mkdir() - for name in ('stat','sha256sum','ip','nmcli'): + for name in ('stat','sha256sum','ip','nmcli','nft'): p=tools/name;p.write_text('#!'+sys.executable+'\n'+TOOLS);p.chmod(0o700) states=tmp_path/'state';states.mkdir(mode=0o700) for suffix,content in [('owner','virt-cloud-vpc-dataplane-v1\n'),('state','vpc|'+VPC_ID+'\n')]: @@ -115,11 +168,147 @@ def test_fabric_preservation_safety_boundaries(tmp_path,shell,action): def test_fabric_boot_defers_preserved_activation_until_full_recovery(): recovery=_BOOT_HELPER[_BOOT_HELPER.index('recover_vpc_dataplanes() {'):] call=recovery.index('vc_vpc_nm_preserve_recover "$vpc_bridge" "$vpc_nm_uuid"') - for proof in ('nm_vpc_field connection.stable-id','vpc_expected_marker_unique','neigh replace proxy','vc_ip_rule_ensure'): + for proof in ('nm_vpc_field connection.stable-id','vpc_expected_marker_unique', + 'neigh replace proxy','vc_ip_rule_ensure', + 'vc_local_neighbor_ensure', 'vc_vpc_preserved_isolation_ready'): assert recovery.index(proof)> {shlex.quote(str(calls))}\n" + f"[ \"$*\" = '-s list table inet {plan.resources.nft_table}' ] || exit 90\n" + "printf '%s' " + shlex.quote(listing) + "\n" + f"exit {nft_status}\n") + nft.chmod(0o700) + start = _BOOT_HELPER.index("vc_vpc_preserved_isolation_ready() {") + function = _BOOT_HELPER[start:_BOOT_HELPER.index("\n}\n", start) + 3] + script = "\n".join(( + "set -eu", + "NFT=" + shlex.quote(str(nft)), + "AWK=awk", + function, + "vc_vpc_preserved_isolation_ready " + + " ".join(shlex.quote(value) for value in + (plan.resources.bridge, plan.vpc.cidr, + plan.resources.nft_table)) + " || exit 71", + "printf 'ready\\n'", + )) + result = subprocess.run([shell, "-s"], input=script, + text=True, capture_output=True) + assert calls.read_text().splitlines() == [ + f"-s list table inet {plan.resources.nft_table}"] + return result + + +@pytest.mark.parametrize("shell", SHELLS) +@pytest.mark.parametrize("style", ["rendered", "normalized"]) +def test_complete_live_vpc_isolation_allows_preserved_activation(tmp_path, shell, style): + listing = _listing() if style == "rendered" else _normalized_listing() + result = _check(tmp_path, shell, listing) + assert result.returncode == 0, result.stderr + assert result.stdout == "ready\n" + + +def _acl_plan(): + plan = _plan() + payload = plan.vpc.canonical() + payload["acls"] = [ + {"acl_id": "https", "direction": "ingress", "action": "allow", + "protocol": "tcp", "source_cidr": "10.44.0.0/24", + "destination_cidr": "10.44.0.10/32", "port_from": 443, + "port_to": 443, "priority": 1}, + {"acl_id": "udp-range", "direction": "egress", "action": "allow", + "protocol": "udp", "source_cidr": "10.44.0.10/32", + "destination_cidr": "10.44.0.11/32", "port_from": 10000, + "port_to": 10100, "priority": 2}, + {"acl_id": "icmp", "direction": "ingress", "action": "allow", + "protocol": "icmp", "source_cidr": "10.44.0.0/24", + "destination_cidr": "10.44.0.0/24", "priority": 3}, + {"acl_id": "deny-rest", "direction": "egress", "action": "deny", + "protocol": "any", "source_cidr": "10.44.0.0/24", + "destination_cidr": "10.44.0.0/24", "priority": 4}, + ] + return replace(plan, vpc=parse_vpc_plan(payload)) + + +@pytest.mark.parametrize("shell", SHELLS) +@pytest.mark.parametrize("style", ["rendered", "normalized"]) +@pytest.mark.parametrize("counters", [False, True]) +def test_generated_port_icmp_and_multiple_acls_remain_supported(tmp_path, shell, style, counters): + listing = _listing(_acl_plan()) + assert "tcp dport 443" in listing + assert "udp dport 10000-10100" in listing + assert "ip protocol icmp" in listing + if counters: + listing = re.sub(r'(accept|drop) comment ("[^"]*:acl:[^"]*")', + r'counter \1 comment \2', listing) + if style == "normalized": + listing = _normalized_listing(listing) + result = _check(tmp_path, shell, listing) + assert result.returncode == 0, result.stderr + assert result.stdout == "ready\n" + + +@pytest.mark.parametrize("shell", SHELLS) +@pytest.mark.parametrize("name", BOUNDED_SETS) +@pytest.mark.parametrize("mutation", ["missing", "duplicate"]) +def test_each_address_set_is_required_exactly_once(tmp_path, shell, name, mutation): + listing = _listing() + definition = _set_line(listing, name) + "\n" + listing = listing.replace(definition, "" if mutation == "missing" + else definition + definition) + result = _check(tmp_path, shell, listing) + assert result.returncode == 71, result.stderr + assert result.stdout == "" + + +@pytest.mark.parametrize("shell", SHELLS) +@pytest.mark.parametrize("name", BOUNDED_SETS) +@pytest.mark.parametrize("address", ["0.0.0.0/0", "203.0.113.7", "10.45.0.10/32"]) +def test_every_referenced_address_set_stays_within_the_vpc(tmp_path, shell, name, address): + listing = _listing() + definition = _set_line(listing, name) + altered = re.sub(r"elements = \{[^}]*\}", + "elements = { " + address + " }", definition) + assert altered != definition + result = _check(tmp_path, shell, listing.replace(definition, altered)) + assert result.returncode == 71, result.stderr + assert result.stdout == "" + + +@pytest.mark.parametrize("shell", SHELLS) +@pytest.mark.parametrize("body", [ + "type ipv4_addr; flags interval; elements = { 10.44.0.10-10.44.0.11 };", + "type ipv4_addr; flags interval; elements = { 10.44.0.0/24 };", + "type ipv4_addr; flags interval; elements = { 10.44.0.999 };", + "type ipv6_addr; elements = { 10.44.0.10 };", + "type ipv4_addr; timeout 60s; elements = { 10.44.0.10 };", + "type ipv4_addr; flags dynamic; elements = { 10.44.0.10 };", + "type ipv4_addr; type ipv4_addr; elements = { 10.44.0.10 };", + "elements = { 10.44.0.10 };", +]) +def test_address_set_parser_rejects_non_generated_metadata_and_elements(tmp_path, shell, body): + listing = _listing() + name = "vpc_dest_ips" + replacement = f" set {name} {{ {body} }}" + result = _check(tmp_path, shell, + listing.replace(_set_line(listing, name), replacement)) + assert result.returncode == 71, result.stderr + assert result.stdout == "" + + +@pytest.mark.parametrize("shell", SHELLS) +@pytest.mark.parametrize("semicolons", [False, True]) +@pytest.mark.parametrize("interval", [False, True]) +@pytest.mark.parametrize("empty", [False, True]) +def test_multiline_address_sets_accept_optional_interval_and_listing_semicolons( + tmp_path, shell, semicolons, interval, empty): + listing = _listing() + separator = ";" if semicolons else "" + for name in BOUNDED_SETS: + lines = [f"\tset {name} {{", "\t\ttype ipv4_addr" + separator] + if interval: + lines.append("\t\tflags interval" + separator) + if not empty: + lines.extend(("\t\telements = {", "\t\t\t10.44.0.10/32,", + "\t\t\t10.44.0.11", "\t\t}" + separator)) + lines.append("\t}") + listing = listing.replace(_set_line(listing, name), "\n".join(lines)) + result = _check(tmp_path, shell, listing) + assert result.returncode == 0, result.stderr + assert result.stdout == "ready\n" + + +@pytest.mark.parametrize("shell", SHELLS) +@pytest.mark.parametrize("position", ["before", "after"]) +@pytest.mark.parametrize("effect", [ + "ip daddr set 203.0.113.7", + 'dup to 203.0.113.7 device "br0"', + 'fwd to "br0"', + "meta mark set 0x1", + "ip daddr set ip daddr map @destination_map", + "ip saddr vmap @verdict_map", + "tcp dport set 4444", + "flow add @fastpath", +]) +def test_unmarked_side_effects_are_rejected_on_both_sides_of_guards(tmp_path, shell, position, effect): + listing = _listing() + anchor = _guard_line(listing, GUARDS[0] if position == "before" else GUARDS[-1]) + injected = " " + effect + replacement = (injected + "\n" + anchor if position == "before" + else anchor + "\n" + injected) + result = _check(tmp_path, shell, listing.replace(anchor, replacement)) + assert result.returncode == 71, result.stderr + assert result.stdout == "" + + +@pytest.mark.parametrize("shell", SHELLS) +@pytest.mark.parametrize("name", GUARDS) +@pytest.mark.parametrize("mutation", ["missing", "duplicate"]) +def test_each_boundary_rule_is_required_exactly_once(tmp_path, shell, name, mutation): + listing = _listing() + rule = _guard_line(listing, name) + "\n" + listing = listing.replace(rule, "" if mutation == "missing" else rule + rule) + result = _check(tmp_path, shell, listing) + assert result.returncode == 71, result.stderr + assert result.stdout == "" + + +@pytest.mark.parametrize("shell", SHELLS) +@pytest.mark.parametrize("mutation", [ + "bridge", "fabric", "cidr", "source_set", "destination_set", "verdict", +]) +def test_altered_boundary_cannot_authorize_activation(tmp_path, shell, mutation): + plan = _plan() + listing = _listing() + name = ("member-egress-isolation" if mutation in + {"source_set", "destination_set"} else GUARDS[2]) + rule = _guard_line(listing, name) + altered = { + "bridge": lambda: rule.replace(plan.resources.bridge, "vcv-foreign", 1), + "fabric": lambda: rule.replace('"wg-fabric0"', '"wg-other"', 1), + "cidr": lambda: rule.replace(plan.vpc.cidr, "10.45.0.0/24", 1), + "source_set": lambda: rule.replace("@local_member_ips", "@member_ips", 1), + "destination_set": lambda: rule.replace("@vpc_dest_ips", "@member_ips", 1), + "verdict": lambda: rule.replace(" drop comment", " accept comment", 1), + }[mutation]() + assert altered != rule + result = _check(tmp_path, shell, listing.replace(rule, altered)) + assert result.returncode == 71, result.stderr + assert result.stdout == "" + + +@pytest.mark.parametrize("shell", SHELLS) +@pytest.mark.parametrize("verdict", ["accept", "jump delegated", "goto delegated", "return"]) +@pytest.mark.parametrize("before", [GUARDS[0], GUARDS[-1]]) +def test_early_control_flow_cannot_skip_the_complete_boundary(tmp_path, shell, verdict, before): + listing = _listing() + rule = _guard_line(listing, before) + listing = listing.replace(rule, " " + verdict + "\n" + rule) + result = _check(tmp_path, shell, listing) + assert result.returncode == 71, result.stderr + assert result.stdout == "" + + +@pytest.mark.parametrize("shell", SHELLS) +@pytest.mark.parametrize("declaration", [ + "type filter hook input priority -190; policy accept;", + "type filter hook forward priority -191; policy accept;", + "type filter hook forward priority -190; policy drop;", + "type filter hook forward priority -190;", +]) +def test_forward_hook_priority_and_policy_are_part_of_the_proof(tmp_path, shell, declaration): + listing = _listing().replace( + "type filter hook forward priority -190; policy accept;", declaration) + result = _check(tmp_path, shell, listing) + assert result.returncode == 71, result.stderr + assert result.stdout == "" + + +@pytest.mark.parametrize("shell", SHELLS) +@pytest.mark.parametrize("mutation", ["chain", "table", "family", "empty", "read_error"]) +def test_unavailable_or_wrong_live_table_cannot_authorize_activation(tmp_path, shell, mutation): + plan = _plan() + listing = _listing() + if mutation == "chain": + listing = listing.replace("chain forward {", "chain unrelated {") + elif mutation == "table": + listing = listing.replace("table inet " + plan.resources.nft_table, + "table inet unrelated", 1) + elif mutation == "family": + listing = listing.replace("table inet ", "table bridge ", 1) + elif mutation == "empty": + listing = "" + result = _check(tmp_path, shell, listing, + nft_status=1 if mutation == "read_error" else 0) + assert result.returncode == 71, result.stderr + assert result.stdout == "" diff --git a/engine/tests/test_vpc_apply_receipts.py b/engine/tests/test_vpc_apply_receipts.py index b5c1a985..96196948 100644 --- a/engine/tests/test_vpc_apply_receipts.py +++ b/engine/tests/test_vpc_apply_receipts.py @@ -363,7 +363,7 @@ def test_rendered_host_receipt_requires_exact_persistent_bytes(tmp_path, shell, '# libvirt-startup-independent-v1\n' '# vpc-local-neighbor-contract-v1\n') if mutation != 'old_route_scope': - header += '# vpc-state-route-scope-contract-v1\n' + header += '# vpc-state-route-scope-contract-v2\n' helper.write_text(header) helper.chmod(0o700) stat = executable(tmp_path / 'stat', 'import sys\n' diff --git a/engine/tests/test_vpc_local_neighbor_transaction.py b/engine/tests/test_vpc_local_neighbor_transaction.py index f3677135..afa56a7e 100644 --- a/engine/tests/test_vpc_local_neighbor_transaction.py +++ b/engine/tests/test_vpc_local_neighbor_transaction.py @@ -209,7 +209,7 @@ def test_helper_upgrade_gate_runs_before_network_mutation(tmp_path, shell, case) '\nPLAN_DIGEST="' + digest + '"\n# libvirt-startup-independent-v1\n') if case != "old": header += "# vpc-local-neighbor-contract-v1\n" if case not in {"old", "old_route_scope"}: - header += "# vpc-state-route-scope-contract-v1\n" + header += "# vpc-state-route-scope-contract-v2\n" if case == "wrong_digest": header = header.replace(digest, "cd" * 32) if case != "missing": helper.write_text(header) diff --git a/engine/virsh_panel/fabric.py b/engine/virsh_panel/fabric.py index 30787972..41e7ba54 100644 --- a/engine/virsh_panel/fabric.py +++ b/engine/virsh_panel/fabric.py @@ -2659,7 +2659,7 @@ _BOOT_HELPER = r'''#!/bin/sh PLAN_DIGEST="__VC_FABRIC_DIGEST__" # libvirt-startup-independent-v1 # vpc-local-neighbor-contract-v1 -# vpc-state-route-scope-contract-v1 +# vpc-state-route-scope-contract-v2 EXPECTED_DIGEST="$PLAN_DIGEST" EXPECTED_NFT_RULES_SHA256="__VC_FABRIC_NFT_RULES_SHA256__" DEPLOYMENT_MODE="__VC_FABRIC_MODE__" @@ -3175,6 +3175,154 @@ vc_link_up_proven() { for (i=1;i<=n;i++) if (flags[i]=="UP") up++} END {exit (!bad && count==1 && up==1) ? 0 : 1}' } +vc_vpc_preserved_isolation_ready() { + vc_isolation_bridge=$1 + vc_isolation_cidr=$2 + vc_isolation_table=$3 + # A preserved bridge must be administratively UP before Linux accepts + # its /32 routes. Prove the existing VPC policy closes main-table fallback + # before reopening it; owner comments alone do not prove these rules. + vc_isolation_live=$("$NFT" -s list table inet "$vc_isolation_table" 2>/dev/null) || return 1 + [ "$(printf '%s' "$vc_isolation_live" | wc -c)" -le 262144 ] 2>/dev/null || return 1 + printf '%s\n' "$vc_isolation_live" | "$AWK" \ + -v bridge="$vc_isolation_bridge" -v cidr="$vc_isolation_cidr" \ + -v table="$vc_isolation_table" ' + function ipv4_number(value, parts, n, i, result) { + n=split(value, parts, "."); if (n!=4) return -1 + for (i=1;i<=4;i++) { + if (parts[i]!~/^(0|[1-9][0-9]*)$/ || parts[i]>255) return -1 + result=result*256+parts[i] + } + return result + } + function bounded_set(text, name, values, n, i, value, address) { + # Match the generated set grammar in both source and nft listing + # forms. Its elements must stay inside this VPC even if someone + # retained all rule comments while widening a live set. + gsub(/;/," ",text); gsub(/ +/," ",text) + sub("^set " name " \\{ *type ipv4_addr *", "", text) + sub(/^flags interval */, "", text) + if (text=="}") return 1 + if (text!~/^elements = \{.*\} *\}$/) return 0 + sub(/^elements = \{ */, "", text); sub(/ *\} *\}$/, "", text) + n=split(text,values,",") + for (i=1;i<=n;i++) { + value=values[i]; sub(/^ +/,"",value); sub(/ +$/,"",value) + sub(/\/32$/, "", value); address=ipv4_number(value) + if (address=cidr_end) return 0 + } + return 1 + } + function predicates_only(line, words, n, i, field, value) { + # Only the generated match/counter/verdict grammar is allowed. + # An unmarked rewrite/dup rule could otherwise preserve every + # owner comment yet change a packet before/after the CIDR guards. + n=split(line, words, " ") + if (words[n]!="accept" && words[n]!="drop") return 0 + for (i=1; in) return 0 + } + return i==n + } + BEGIN { + b="\"" bridge "\""; wg="\"wg-fabric0\"" + split(cidr,cidr_parts,"/") + cidr_start=ipv4_number(cidr_parts[1]); cidr_end=cidr_start+2^(32-cidr_parts[2]) + set_names["member_ips"]=1; set_names["local_member_ips"]=1; set_names["vpc_dest_ips"]=1 + rules["iifname " b " ip saddr != @local_member_ips drop"]=1 + rules["oifname " b " ip daddr " cidr " iifname != " b " iifname != " wg " drop"]=1 + rules["oifname " wg " ip daddr " cidr " iifname != " b " iifname != " wg " drop"]=1 + rules["iifname " b " ip daddr " cidr " oifname != " b " oifname != " wg " drop"]=1 + rules["iifname " wg " ip daddr " cidr " oifname != " b " oifname != " wg " drop"]=1 + rules["iifname " b " ip saddr @local_member_ips ip daddr != @vpc_dest_ips drop"]=1 + } + { + line=$0; gsub(/[[:space:]]+/, " ", line); sub(/^ /,"",line); sub(/ $/,"",line) + if (line=="table inet " table " {") table_seen++ + if (set_name=="" && $1=="set" && $2 in set_names) { + set_name=$2; set_text=""; set_depth=0; set_seen[set_name]++ + } + if (set_name!="") { + set_text=set_text (set_text=="" ? "" : " ") line + braces=line; set_depth+=gsub(/\{/,"",braces) + braces=line; set_depth-=gsub(/\}/,"",braces) + if (set_depth==0) { + if (!bounded_set(set_text,set_name)) bad=1 + set_name="" + } + next + } + if (line ~ /^chain forward \{( |$)/) { + chain_seen++; in_forward=1; sub(/^chain forward \{ */,"",line) + } + if (!in_forward || line=="") next + if (line=="}") {in_forward=0; closed++; next} + if (line=="type filter hook forward priority -190; policy accept;") {header++; next} + if (header!=1) bad=1 + sub(/ comment "[^"]*"$/, "", line) + if (!predicates_only(line)) bad=1 + if (line in rules) {if (++seen[line]!=1) bad=1; found++; next} + if (found<6 && line ~ /(^| )(accept|jump|goto|return|queue|vmap)( |;|$)/) bad=1 + } + END { + for (rule in rules) if (seen[rule]!=1) bad=1 + for (name in set_names) if (set_seen[name]!=1) bad=1 + exit (bad || set_name!="" || table_seen!=1 || chain_seen!=1 || header!=1 || closed!=1 || in_forward) ? 1 : 0 + }' +} +vc_vpc_restored_route_ready() { + vc_restored_routes=$("$IP" -N -o -4 route show table "$4" exact "$1" 2>/dev/null) || return 1 + printf '%s\n' "$vc_restored_routes" | "$AWK" \ + -v destination="$1" -v device="$2" -v via="$3" \ + -v wanted_table="$4" -v metric="$5" ' + __VC_FABRIC_ROUTE_VALIDATION_AWK__ + NF { + count++; actual_device=""; actual_via="-"; actual_metric=""; proto="" + if (canonical_host_prefix($1)!=destination) bad=1 + delete seen + for (i=2; i<=NF; i++) { + field=$i + if (field=="onlink") {if (via=="-" || seen[field]++) bad=1; continue} + if (field=="linkdown") {if (seen[field]++) bad=1; continue} + if (field!="dev" && field!="via" && field!="table" && + field!="proto" && field!="scope" && field!="metric") {bad=1; break} + if (++seen[field]!=1 || ++i>NF) {bad=1; break} + if (field=="dev") actual_device=$i + if (field=="via") actual_via=$i + if (field=="metric") actual_metric=$i + if (field=="proto") proto=$i + if (field=="table" && $i!=wanted_table) bad=1 + if (field=="scope" && $i!="link" && $i!="253") bad=1 + } + if (actual_device!=device || actual_via!=via || + actual_metric!=metric || proto!="186" || + (via!="-" && seen["onlink"]!=1)) bad=1 + } + END {exit (bad || count!=1) ? 1 : 0}' +} vc_vpc_nm_preserved_force_down() { while IFS='|' read -r vc_vpc_closed_bridge vc_vpc_closed_identity; do [ -n "$vc_vpc_closed_bridge" ] || continue @@ -5111,37 +5259,47 @@ EOF done </dev/null && [ "$vpc_metric" -le 65535 ] 2>/dev/null || exit 71 - if [ "$vpc_via" = - ]; then - "$IP" route replace "$vpc_destination" table "$vpc_route_table" dev "$vpc_device" proto 186 metric "$vpc_metric" || exit 72 - else - vpc_state_ipv4 "$vpc_via" || exit 71 - "$IP" route replace "$vpc_destination" table "$vpc_route_table" via "$vpc_via" dev "$vpc_device" onlink proto 186 metric "$vpc_metric" || exit 72 - fi - done < "$vpc_state_file" + if [ "$vpc_nm_preserved" = yes ]; then + # Prove fallback isolation before policy-rule changes can select + # an incompletely restored table, as well as before bridge UP. + vc_vpc_preserved_isolation_ready "$vpc_bridge" "$vpc_cidr" "$vpc_nft_name" || exit 71 + fi vc_ip_rule_scope_check "$vpc_cidr" "$vpc_table" "$vpc_priority" "$IFACE" managed 71 72 || exit $? vc_ip_rule_preflight "$vpc_priority" - "$vpc_cidr" "$vpc_table" 71 72 - || exit $? vc_ip_rule_preflight "$((vpc_priority + 1))" "$IFACE" "$vpc_cidr" "$vpc_table" 71 72 - || exit $? vc_ip_rule_ensure "$vpc_priority" - "$vpc_cidr" "$vpc_table" 71 72 - - || exit $? vc_ip_rule_ensure "$((vpc_priority + 1))" "$IFACE" "$vpc_cidr" "$vpc_table" 71 72 - - || exit $? - # ACL/owner/state/manager and exact routes have now been verified. + # ACL/owner/state/manager and route ownership are now verified. # Restore only declared local guest MACs, never routed service /32s # or remote peers. A missing legacy contract stays empty until VPC # apply migrates it from the authoritative member document. vc_local_neighbor_ensure "$vpc_bridge" "$vpc_local_neighbors" || exit 71 if [ "$vpc_nm_preserved" = yes ]; then - # The original active profile, nft policy, routes and proxy entries - # are now validated. Reopen the same bridge without NM deconfigure. + # ACL/sysctls, proxy entries, policy rules and local neighbors are + # ready and live CIDR fallback guards were proven before changing + # policy rules. Reopen without deconfiguring the NM profile/taps. vc_vpc_nm_preserve_recover "$vpc_bridge" "$vpc_nm_uuid" || exit 71 fi vpc_local_neighbors_actual=$(vc_local_neighbor_records "$vpc_bridge") || exit 71 vpc_local_neighbors_wanted=$(printf '%s\n' "$vpc_local_neighbors" | awk 'NF {print $0 "|PERMANENT"}') || exit 71 [ "$vpc_local_neighbors_actual" = "$vpc_local_neighbors_wanted" ] || exit 71 + # Linux rejects even a direct /32 route while its device is DOWN. + # ACLs remain in force during this bounded route restoration window; + # any failure re-quarantines every preserved bridge in cleanup(). + while IFS='|' read -r vpc_kind vpc_destination vpc_device vpc_route_table vpc_via vpc_metric; do + [ "$vpc_kind" = route ] || continue + [ "$vpc_route_table" = "$vpc_table" ] || exit 71 + [ "$vpc_device" = "$vpc_bridge" ] || [ "$vpc_device" = "$IFACE" ] || exit 71 + [ "$vpc_metric" -ge 1 ] 2>/dev/null && [ "$vpc_metric" -le 65535 ] 2>/dev/null || exit 71 + if [ "$vpc_via" = - ]; then + "$IP" route replace "$vpc_destination" table "$vpc_route_table" dev "$vpc_device" proto 186 metric "$vpc_metric" || exit 72 + else + vpc_state_ipv4 "$vpc_via" || exit 71 + "$IP" route replace "$vpc_destination" table "$vpc_route_table" via "$vpc_via" dev "$vpc_device" onlink proto 186 metric "$vpc_metric" || exit 72 + fi + vc_vpc_restored_route_ready "$vpc_destination" "$vpc_device" "$vpc_via" "$vpc_route_table" "$vpc_metric" || exit 72 + done < "$vpc_state_file" done } "$WG" set "$IFACE" private-key "$KEY" @@ -8157,7 +8315,7 @@ artifact_state() {{ if [ "$artifact_kind" = helper ]; then grep -Fqx -- '# vpc-local-neighbor-contract-v1' "$artifact_path" || {{ printf '%s' foreign; return; }} - grep -Fqx -- '# vpc-state-route-scope-contract-v1' "$artifact_path" || + grep -Fqx -- '# vpc-state-route-scope-contract-v2' "$artifact_path" || {{ printf '%s' foreign; return; }} fi if [ "$artifact_kind" = helper ] && diff --git a/engine/virsh_panel/vpc_dataplane.py b/engine/virsh_panel/vpc_dataplane.py index 025ebb31..add68581 100644 --- a/engine/virsh_panel/vpc_dataplane.py +++ b/engine/virsh_panel/vpc_dataplane.py @@ -243,7 +243,7 @@ vc_vpc_neighbor_helper_ready() { [ "$(sed -n '3p' "$vc_ln_helper")" = "# fabric-plan-digest=$EXPECTED_DIGEST" ] && [ "$(sed -n '4p' "$vc_ln_helper")" = "PLAN_DIGEST=\"$EXPECTED_DIGEST\"" ] && [ "$(sed -n '6p' "$vc_ln_helper")" = '# vpc-local-neighbor-contract-v1' ] && - [ "$(sed -n '7p' "$vc_ln_helper")" = '# vpc-state-route-scope-contract-v1' ] + [ "$(sed -n '7p' "$vc_ln_helper")" = '# vpc-state-route-scope-contract-v2' ] } vc_vpc_neighbors_snapshot() { vc_ln_snapshot=$(vc_local_neighbor_records "$BRIDGE") || return 1