From 7f34725d7ef3f09b1f302b6f5cebd464ab9a3ceb Mon Sep 17 00:00:00 2001 From: admin Date: Thu, 8 Oct 2026 12:16:09 +0800 Subject: [PATCH] fix: harden VPC lifecycle and guest readiness --- docs/INTEGRATION_PLAN.md | 54 +- docs/VALIDATION.md | 27 + docs/VPC_DATAPLANE_ACCEPTANCE.md | 51 +- engine/tests/test_cloud_init.py | 139 +- engine/tests/test_engine_boundary.py | 1 + engine/tests/test_libvirt_service.py | 1744 +++++++++++++++- engine/tests/test_vpc.py | 817 +++++++- engine/tests/test_vpc_dataplane.py | 416 ++++ engine/virsh_panel/cloud_init.py | 176 +- engine/virsh_panel/internal.py | 3 +- engine/virsh_panel/libvirt_service.py | 1752 ++++++++++++++++- engine/virsh_panel/vpc.py | 755 ++++++- engine/virsh_panel/vpc_dataplane.py | 299 ++- web/app/Models/CloudVpcOperation.php | 1 + .../Cloud/CloudOperationErrorCatalog.php | 55 +- web/app/Services/Cloud/CloudVpcService.php | 1271 ++++++++++-- .../Services/Cloud/CloudVpcWorkerService.php | 67 +- web/app/Services/Cloud/CloudWorkerService.php | 55 +- web/public/cloud/vpc.js | 7 +- web/sql/cloud.sql | 3 + .../CloudVpcControlPlaneContractTest.php | 78 +- web/tests/CloudVpcMemberReuseTest.php | 83 +- web/tests/CloudVpcRootDeletionTest.php | 1 + .../CloudVpcTombstoneConvergenceTest.php | 990 +++++++++- web/tests/CloudVpcUserPageContractTest.php | 3 +- 25 files changed, 8466 insertions(+), 382 deletions(-) diff --git a/docs/INTEGRATION_PLAN.md b/docs/INTEGRATION_PLAN.md index fa3057ce..ad6e0367 100644 --- a/docs/INTEGRATION_PLAN.md +++ b/docs/INTEGRATION_PLAN.md @@ -59,8 +59,9 @@ systemd-networkd 的 `.network/.netdev` 文件以及 ifupdown 的 interfaces 文 云初始化使用 `public0`/`private0` 作为 Netplan 配置键和产品角色;libvirt XML 使用符合设备契约的 `ua-public0`/`ua-private0` 用户 alias,两者不得混用。公网 -静态 IPv4 保留唯一默认路由;私网只接受 RFC1918 `/32`,关闭 DHCP/RA 且不带 -gateway、DNS 或默认路由。旧的单网卡域可按兼容规则视为公网;历史 +静态 IPv4 保留唯一默认路由;`private0` 地址只接受 RFC1918 `/32`,另且仅下发一条 +当前 VPC CIDR 的 scope-link 路由,关闭 DHCP/RA 且不带 gateway、DNS 或默认路由。 +旧的单网卡域可按兼容规则视为公网;历史 `netN`/`ua-net-N` 公网 alias 只有在其他网卡全部明确为私网时才可用于运行态 迁移,其他无角色多网卡域一律停止安全策略操作。面板的 `clean-traffic`/`vc-fw-*` nwfilter 只能绑定 `public0` 产品角色(libvirt alias @@ -161,8 +162,9 @@ PHP 只有在所有参与宿主机的观测 revision/digest 完全匹配时才 如果该服务未安装,API 仍会安全地保留 `queued` 状态,显式 `POST /api/v1/vpcs/{vpc_key}/reconcile` 可用于人工核对,但不能把 queued 当作 已经下发。引擎只证明 Fabric 已就绪、而本阶段没有私网数据面回读时,操作会进入 -`waiting_observed`、根同步状态为 `waiting`,专用 worker 不会每 30 秒重复请求;后续 -数据面上线后必须由显式 reconcile 重新推进,不能把 waiting 当成 active。 +`waiting_observed`、根同步状态为 `waiting`;专用 worker 会周期性执行幂等 +reconcile,也可由显式 reconcile 立即推进。重试必须保持 seed、XML 和宿主数据面 +幂等,不能把 waiting 当成 active。 VPC 私网数据面现在由引擎的独立 reconciler 执行。每台参与宿主机使用带 owner 标记的独立私网 bridge/tap,VM XML 使用不绑定 nwfilter 的 `ua-private0` alias @@ -173,7 +175,49 @@ VPC 私网数据面现在由引擎的独立 reconciler 执行。每台参与宿 bridge 和 ARP nft 表:ACL 按 priority 处理,bridge 入口校验 MAC/IP/ARP,IPv6、 广播、多播和未知目的地址默认隔离;代理 ARP 仅允许面板登记的精确 `/32`。 -云初始化 seed 同步写入 `public0`/`private0` 多网卡配置和 link-scope `/32` 路由。 +云初始化 seed 同步写入 `public0`/`private0` 多网卡配置和唯一的 scope-link VPC CIDR +路由。NoCloud 只在 boot 事件重新消费该网络配置:运行中的 VM 首次接入或其 MAC、私网 +IP、VPC CIDR +变化时,引擎不会改写 seed、持久 XML 或 live 网卡,而是保持旧配置并返回待重启;VM +完全停机后,下一次 reconcile 才原子写入 seed 和持久 XML,随后一次完整 start 才消费 +新网络配置。独立的 guest-network generation 只覆盖 MAC、私网 `/32` 地址和 VPC 前缀; +成员、自定义 route、ACL 等宿主侧变化不改变 generation,避免一次成员变更要求所有运行中 +成员重启。只有完整 stop/reconcile/start 后 live domain metadata 带有目标 +generation,成员才能通过 active/usable 门禁; +缺失或旧 generation 会保持 `waiting_observed` 并返回 +`vpc_private_guest_restart_required`。相同目标的重试不再轮换 NoCloud `instance-id`, +避免重放用户、密码和 sshd 等 per-instance 模块;缺少当前 `boot-v1` 网络更新策略的 +旧 seed 以 `vpc_seed_network_policy_migration_required` 失败关闭,必须重装或执行明确的 +离线迁移,不能通过轮换 instance-id 静默升级。 + +live generation 一致仍不等于来宾网络可用。每个运行成员还必须通过 QEMU Guest Agent +回读:`guest-info` 明确启用 `guest-exec`/`guest-exec-status`;链路探测固定使用 +`/usr/bin/timeout` 与 `/usr/bin/head` 读取 sysfs,路由探测只允许固定的 +`/bin/sh -c 'exec /usr/sbin/ip -j -4 route show table main dev "$1" | /usr/bin/head -c 131073'` +脚本,唯一可变参数是已校验的 guest 接口名。回读必须确认唯一接口的 MAC、精确 IPv4 +`/32`、admin/carrier/oper link-up,以及 main table 中恰好一条当前 VPC CIDR 的 +scope-link 路由,且没有 gateway、nexthop、`linkdown`、`dead` 或 `unresolved`。 +agent/channel/service、命令 allowlist 或固定二进制缺失时分别返回 +unsupported/unavailable/uncertain/deadline evidence,成员保持不可用。该 QGA 证据是 +租户可控来宾的 readiness 信号,不是 VPC 隔离的安全边界;隔离仍由宿主 bridge、route、 +WireGuard 和 nft 回读证明。显式 network suspend 是例外:精确 persistent/live XML 的 +`link state='down'` 与宿主配置可以证明隔离已生效,但该成员绝不能因此被标为 usable。 + +宿主配置提交和成员运行态分开收敛:当前 revision/digest 的 bridge、路由、ACL、seed +与 inactive/live XML 配置证据可完成控制面操作和 tombstone 回收;每个成员的 +active/usable 则只由自己的 current-generation QGA 证据推进。一台关机、待重启或 +agent 不可用的 VM 不得抹掉同 VPC 其他已验证成员的可用状态,但 VPC 聚合健康仍可显示 +degraded,周期 reconcile 会继续推进尚未就绪的成员。 + +历史 seed 中逐成员 `/32` route 只用于旧 generation;运行中成员保持 +`vpc_private_guest_restart_required`,完全停机后的 reconcile 才迁移为单条 VPC CIDR。 +4096 成员上限保护记录和回读大小,但普通周期 reconcile 在进入并发 QGA 阶段前仍会按宿主 +顺序执行每台 VM 的 seed/libvirt 核对;生命周期目标会优先,整宿主满规模延迟仍需真实压测, +不能把单次 HTTP 预算当作全量收敛时限。 + +本契约使用 `evidence_version=2`。滚动发布顺序必须是:先应用数据库 schema,再部署能 +持久化/校验 v2 的 PHP 与 worker,最后部署 engine;混合版本及历史缺失版本证据一律 +fail closed,不能回退接受仅有 XML 的旧 succeeded receipt。 宿主机 apply 在 Fabric 锁和 VPC 锁内执行,持久文件、NetworkManager/ifupdown 配置、路由、策略规则、WireGuard、nft、代理 ARP 和 sysctl 都有实际回读;apply、 实例接入和停机 libvirt 路径中途失败会按 ownership marker 恢复,响应不确定时保持 diff --git a/docs/VALIDATION.md b/docs/VALIDATION.md index 14cf8ad5..8babd3bd 100644 --- a/docs/VALIDATION.md +++ b/docs/VALIDATION.md @@ -1,5 +1,32 @@ # 本版验证记录 +## 2026-10-08:VPC 来宾网络启动代次门禁 + +- 私网 MAC、`/32` 地址和唯一的 VPC CIDR scope-link 路由共同生成独立 + guest-network generation;成员、自定义 route、ACL/Fabric 等纯宿主变化不改变该 + generation。 +- 运行中 VM 的新来宾网络不改写 NoCloud seed、inactive XML 或 live device;完全停机 + 后由 reconcile 写入 seed 与持久 XML,再启动消费。在完成 stop/reconcile/start 且 + live metadata 匹配之前,宿主保持 + `applying/pending_restart`,PHP 拒绝将缺失 `restart_required=false` 的证据视为 + active/usable。 +- 相同私网目标重试返回原 seed 字节,不重复轮换 NoCloud `instance-id`;写入 + 后会在 libvirt define 前重新下载 seed 并核对 MAC/IP/routes;缺少安全 boot 网络策略 + 的旧 seed 明确要求迁移,不通过新 instance-id 重放 credentials。 +- current generation 还要通过 QGA 地址、link 与 route 的精确回读;保留地址/路由但 + `NO-CARRIER`/`linkdown` 的接口不能通过。显式 link-down suspension 单独由 XML/config + 证明隔离,恢复 link-up 后才重新要求 QGA usable 证据。 +- `evidence_version=2` 使历史 XML-only succeeded 证据失效;宿主 config proof、每成员 + guest readiness 与 VPC 聚合健康分别收敛,tombstone 不再被无关成员的 QGA 状态阻塞。 +- 本地验证:Python compileall 通过;cloud-init、VPC 数据面、libvirt、协调器及删除回执 + 的定向回归 **394 passed**。完整 engine 回归为 **1071 passed、10 skipped、8 + failed**;8 个失败已在本轮修改前的 `6dc0d21` 干净快照复现,属于既有云模板 spec、 + 重装错误 envelope、Python 3.9 upload timeout/event-loop 与退役宿主盘点式公网池 fixture。 +- 隔离 PHP 8.5.10/WASM 下 5 个 VPC 契约/收敛测试全部通过,10 个修改 PHP 文件语法、 + `vpc.js` Node 语法和 `git diff --check` 通过。当前宿主没有原生 PHP CLI;真实 + libvirt/cloud-init 验收仍需在测试 VM 内核对地址、唯一 VPC CIDR scope-link 路由与跨 + VM 连通性。宿主策略路由、代理 ARP 和 WireGuard AllowedIPs 仍必须保持精确 `/32`。 + ## 2026-10-08:根 VPC 可恢复删除状态机 - 租户 API 新增根 VPC 删除:精确名称确认后,PHP 在事务中把根和子资源固定为 diff --git a/docs/VPC_DATAPLANE_ACCEPTANCE.md b/docs/VPC_DATAPLANE_ACCEPTANCE.md index caec44b8..3534ad2b 100644 --- a/docs/VPC_DATAPLANE_ACCEPTANCE.md +++ b/docs/VPC_DATAPLANE_ACCEPTANCE.md @@ -21,20 +21,41 @@ 对一个跨两台宿主机的 VPC,至少放置两个成员和一个 ACL,逐台检查: +镜像前置契约:安装并启用 `qemu-guest-agent`,domain 存在 agent channel, +`guest-info` 中 `guest-exec` 与 `guest-exec-status` 均为 enabled;镜像必须提供 +`/usr/bin/timeout`、`/usr/bin/head`、`/bin/sh` 和 `/usr/sbin/ip`。不要假定发行版默认允许 `guest-exec`:Alma/RHEL +系列镜像或加固策略可能在 QGA blocklist 中禁用它,必须逐镜像实际核对。任何一项缺失 +都应得到 `vpc_private_guest_probe_unsupported`(或 unavailable),成员不得 usable。 + +0. 运行中 VM 首次加入,或其 MAC、私网 IP、VPC CIDR 变化时,引擎不得改写 NoCloud seed、 + inactive XML,也不得对 live domain 执行私网网卡热插/热更新;旧配置必须原样保留。 + VM 完全停机后由下一次 reconcile 写入 seed 与 inactive XML,再执行完整 start。 + 在完成该 stop/reconcile/start 流程前,宿主状态必须是 `applying`、回读为 + `pending_restart`、错误码为 `vpc_private_guest_restart_required`,且 VPC + `usable=false`。普通 guest soft reboot 不能替代 libvirt stop/start。 + 1. 生成的 bridge 名称、owner/state/proxy 文件均为 root-only,bridge/tap 没有 地址、网关或 DNS;`private0` 产品角色对应的 libvirt alias 是 `ua-private0`,其 bridge、稳定 MAC、`/32` 和持久/实时 XML 都一致,私网接口 不绑定公网 `clean-traffic` nwfilter。 -2. 每个成员 `/32` 和每条自定义 `/32` route 都带 panel route protocol、正确 +2. QGA 回读必须找到唯一的 guest 接口名与 MAC,精确 IPv4 地址只有目标 `/32`;固定 + sysfs 探测必须同时确认接口 flags 含 `IFF_UP`、`operstate=up`、`carrier=1`。 + 接口 down 后即使地址和路由仍保留也必须返回 + `vpc_private_guest_link_mismatch`,不能通过 usable 门禁。显式 network suspend + 则以 persistent/live XML 都为 down 证明隔离,并保持成员 non-usable。 +3. 宿主机上的每个成员 `/32` 和每条自定义 `/32` route 都带 panel route protocol、正确 device/table、`via` 和 metric;不存在 VPC CIDR、默认路由或宽前缀的替代项。 -3. `wg show wg-fabric0 allowed-ips` 的每个 peer 与完整期望集合一致:overlay `/32` + 来宾 main table 的目标 route 必须只有一条 exact scope-link VPC CIDR,不能含 gateway、 + nexthop、`linkdown`、`dead` 或 `unresolved` flags;成员或自定义 route 变化不得改变 + 兄弟 VM 的 guest generation。历史逐成员 `/32` seed 只能在完全停机后迁移。 +4. `wg show wg-fabric0 allowed-ips` 的每个 peer 与完整期望集合一致:overlay `/32` 加精确 tenant/service `/32`,没有 `0.0.0.0/0`、宽前缀或其他 VPC 的未授权地址。 -4. nft inet/bridge/arp 三张表的 owner marker、chain priority、ACL comments、 +5. nft inet/bridge/arp 三张表的 owner marker、chain priority、ACL comments、 anti-spoof、IPv6/广播/未知目的 drop 和精确 proxy-ARP set 均能被 observe 回读。 bridge 上通用 `proxy_arp=0`、同桥下一跳所需的 `proxy_arp_pvlan=1`、 `rp_filter=2`,代理项集合与 route destination 集合 完全相同;其他地址的 ARP reply 必须被过滤。 -5. 从同机和跨机成员分别验证允许的 TCP/UDP/ICMP 流量、ACL deny、伪造源 MAC/IP、 +6. 从同机和跨机成员分别验证允许的 TCP/UDP/ICMP 流量、ACL deny、伪造源 MAC/IP、 IPv6、广播/多播和访问宿主机自身地址。验证结果同时记录 guest route、宿主机 route、WireGuard handshake 和 nft counter,不能只看页面的 `active`。 @@ -69,3 +90,25 @@ Debian 13:执行 `ifquery `、`ifup --force ` 跨宿主机最后还要单独验证 underlay 公网防火墙、NAT、MTU、SELinux 和 UDP 监听。 Fabric/VPC 的本地 nft 表不等于 underlay 已放行。 + +## 发布和规模边界 + +- 先应用包含 VPC evidence version 字段的数据库 schema,再发布 PHP/worker,最后发布 + engine。历史或混合版本 receipt 没有 `evidence_version=2` 时必须重新 reconcile,不能 + 继承旧 active/usable。 +- 用 1、32、256 个成员和至少两台宿主分别压测正常、QGA command disabled、QGA + timeout 三种场景。引擎的协作式探测预算到期后必须停止提交新 QGA/SSH 探测,并取消 + 未开始任务;它不是对已经开始的 libvirt 打开、XML、seed 或 define 操作的强制 HTTP + 截止。PHP HTTP timeout 必须大于该探测预算,operation lease 还要留出明确余量;超时只返回 + `vpc_private_guest_probe_deadline_exceeded` 并继续幂等 reconcile,不能 lease steal、 + 重复派发或遗留不断增长的 guest process/线程/SSH 连接。 +- 宿主 libvirt 必须为 upstream 4.2.0+,或由发行版明确回补 CVE-2018-1064 的 QGA 单响应 + 10 MiB 读取上限;Python 只解析 + 不超过 192 KiB 的响应,且默认最多并行 4 个来宾探测。`agentSetResponseTimeout` 恢复 + 默认值失败时必须返回 uncertain,不能在 domain-wide timeout 仍被修改时发布 confirmed。 +- 另用单宿主 4096 成员验证周期 reconcile:生命周期目标必须优先;进入 QGA 并发探测前 + 的逐 VM seed/libvirt 核对仍是已知串行边界,应记录首个目标完成时间和全量收敛时间, + 不能仅以请求超时作为容量结论。 +- 同一 VPC 放置 A、B 两个成员:A QGA exact、B stopped/unsupported 时,配置 operation + 应完成,A 的实例私网可用、B 单独显示未就绪,聚合健康可 degraded;随后只恢复 B 的 + agent/启动状态,不做新 mutation,周期 reconcile 必须最终将 B 和聚合健康推进为 active。 diff --git a/engine/tests/test_cloud_init.py b/engine/tests/test_cloud_init.py index 346a61a3..66b13c72 100644 --- a/engine/tests/test_cloud_init.py +++ b/engine/tests/test_cloud_init.py @@ -18,6 +18,7 @@ from virsh_panel.cloud_init import ( replace_seed_settings, validate_cloud_network, validate_guest_hostname, validate_private_network, _parse_network_config, + SeedNetworkPolicyMigrationRequired, ) @@ -251,7 +252,10 @@ def test_seed_iso_can_add_private_role_without_public_route_or_dns(): seed = build_seed_iso( str(uuid.uuid4()), "multi-nic-vm", "52:54:00:12:34:56", validated, private_mac="52:54:00:12:34:57", - private_network={"private_ip_cidr": "10.24.3.7/32"}, + private_network={ + "private_ip_cidr": "10.24.3.7/32", + "routes": ["10.24.0.0/16"], + }, ) network = _network_data(seed) assert set(network["ethernets"]) == {"public0", "private0"} @@ -261,6 +265,7 @@ def test_seed_iso_can_add_private_role_without_public_route_or_dns(): "dhcp6": False, "accept-ra": False, "addresses": ["10.24.3.7/32"], + "routes": [{"to": "10.24.0.0/16", "scope": "link"}], } assert read_seed_network(seed) == { "public_ip_cidr": "8.8.8.18/28", @@ -269,14 +274,14 @@ def test_seed_iso_can_add_private_role_without_public_route_or_dns(): "mac": "52:54:00:12:34:56", "private_ip_cidr": "10.24.3.7/32", "private_mac": "52:54:00:12:34:57", - "private_routes": [], + "private_routes": ["10.24.0.0/16"], "roles": ["public0", "private0"], } @pytest.mark.skipif(importlib.util.find_spec("pycdlib") is None, reason="pycdlib is not installed") -def test_private_seed_routes_are_exact_and_can_be_reconciled_or_removed(): +def test_private_seed_migrates_legacy_routes_to_stable_vpc_prefix(): validated = validate_cloud_network( "8.8.8.18/28", "8.8.8.17", ["1.1.1.1"], PUBLIC_KEY, ) @@ -285,36 +290,154 @@ def test_private_seed_routes_are_exact_and_can_be_reconciled_or_removed(): private_mac="52:54:00:12:34:57", private_network={ "private_ip_cidr": "10.24.3.7/32", - "routes": ["10.24.3.8/32"], + "routes": ["10.24.0.0/16"], }, ) - assert read_seed_network(seed)["private_routes"] == ["10.24.3.8/32"] + legacy_network = _network_data(seed) + legacy_network["ethernets"]["private0"]["routes"] = [ + {"to": "10.24.3.8/32", "scope": "link"}, + {"to": "10.24.3.10/32", "scope": "link"}, + ] + seed = _build_seed_iso(( + ("/USERDATA.;1", "user-data", "/user-data", + b"#cloud-config\n" + json.dumps( + _user_data(seed), ensure_ascii=True).encode("ascii") + b"\n"), + ("/METADATA.;1", "meta-data", "/meta-data", + json.dumps(_meta_data(seed), ensure_ascii=True).encode("ascii") + b"\n"), + ("/NETWORK.;1", "network-config", "/network-config", + json.dumps(legacy_network, ensure_ascii=True).encode("ascii") + b"\n"), + )) + assert read_seed_network(seed)["private_routes"] == [ + "10.24.3.10/32", "10.24.3.8/32", + ] changed = replace_seed_private_network( seed, private_mac="52:54:00:12:34:58", private_ip_cidr="10.24.3.9/32", - private_routes=["10.24.3.10/32", "10.24.3.11/32"], + private_routes=["10.24.0.0/16"], ) changed_network = read_seed_network(changed) + assert _meta_data(changed)["instance-id"] == _meta_data(seed)["instance-id"] + assert _user_data(changed) == _user_data(seed) assert changed_network["private_ip_cidr"] == "10.24.3.9/32" assert changed_network["private_mac"] == "52:54:00:12:34:58" - assert changed_network["private_routes"] == ["10.24.3.10/32", "10.24.3.11/32"] + assert changed_network["private_routes"] == ["10.24.0.0/16"] + unchanged = replace_seed_private_network( + changed, + private_mac="52:54:00:12:34:58", + private_ip_cidr="10.24.3.9/32", + private_routes=["10.24.0.0/16"], + ) + assert unchanged == changed removed = replace_seed_private_network( changed, private_mac=None, private_ip_cidr=None, ) removed_network = read_seed_network(removed) + assert _meta_data(removed)["instance-id"] == _meta_data(seed)["instance-id"] + assert _user_data(removed) == _user_data(seed) assert "private_ip_cidr" not in removed_network assert "private_routes" not in removed_network + for invalid_routes in ( + None, + [], + ["10.24.0.0/16", "10.24.3.10/32"], + ["10.24.0.0/16", "10.24.0.0/16"], + ["10.25.0.0/16"], + ["10.24.3.0/25"], + ): + with pytest.raises(ValueError): + replace_seed_private_network( + seed, + private_mac="52:54:00:12:34:58", + private_ip_cidr="10.24.3.9/32", + private_routes=invalid_routes, + ) + + +@pytest.mark.skipif(importlib.util.find_spec("pycdlib") is None, + reason="pycdlib is not installed") +def test_private_seed_rejects_public_connected_prefix_overlap(): + # The public address itself is global, but this deliberately broad + # connected prefix includes the RFC1918 VPC. private0 must not silently + # steal that connected public route. + validated = validate_cloud_network( + "8.8.8.18/5", "8.8.8.17", ["1.1.1.1"], PUBLIC_KEY, + ) + seed = build_seed_iso( + str(uuid.uuid4()), "overlap-vm", "52:54:00:12:34:56", validated, + ) with pytest.raises(ValueError): replace_seed_private_network( seed, private_mac="52:54:00:12:34:58", private_ip_cidr="10.24.3.9/32", - private_routes=["10.24.3.0/24"], + private_routes=["10.24.0.0/16"], + ) + + +def test_private_route_parser_rejects_mixed_legacy_and_stable_routes(): + config = { + "version": 2, + "ethernets": { + "public0": { + "match": {"macaddress": "52:54:00:12:34:56"}, + "dhcp4": False, "dhcp6": False, + "addresses": ["8.8.8.18/28"], + "routes": [{"to": "0.0.0.0/0", "via": "8.8.8.17"}], + "nameservers": {"addresses": ["1.1.1.1"]}, + }, + "private0": { + "match": {"macaddress": "52:54:00:12:34:57"}, + "dhcp4": False, "dhcp6": False, "accept-ra": False, + "addresses": ["10.24.3.7/32"], + "routes": [ + {"to": "10.24.0.0/16", "scope": "link"}, + {"to": "10.24.3.8/32", "scope": "link"}, + ], + }, + }, + } + with pytest.raises(ValueError, match="唯一 VPC"): + _parse_network_config(config) + + +@pytest.mark.skipif(importlib.util.find_spec("pycdlib") is None, + reason="pycdlib is not installed") +def test_private_seed_rejects_legacy_policy_without_rotating_instance_id(): + validated = validate_cloud_network( + "8.8.8.18/28", "8.8.8.17", ["1.1.1.1"], PUBLIC_KEY, + ) + seed = build_seed_iso( + str(uuid.uuid4()), "legacy-private-vm", "52:54:00:12:34:56", + validated, + ) + legacy_user_data = _user_data(seed) + legacy_user_data.pop("updates") + legacy_metadata = _meta_data(seed) + legacy_metadata.pop("virt-cloud-network-policy") + legacy_seed = _build_seed_iso(( + ("/USERDATA.;1", "user-data", "/user-data", + b"#cloud-config\n" + json.dumps( + legacy_user_data, ensure_ascii=True).encode("ascii") + b"\n"), + ("/METADATA.;1", "meta-data", "/meta-data", + json.dumps(legacy_metadata, ensure_ascii=True).encode("ascii") + + b"\n"), + ("/NETWORK.;1", "network-config", "/network-config", + json.dumps(_network_data(seed), ensure_ascii=True).encode("ascii") + + b"\n"), + )) + + with pytest.raises(SeedNetworkPolicyMigrationRequired): + replace_seed_private_network( + legacy_seed, + private_mac="52:54:00:12:34:57", + private_ip_cidr="10.24.3.7/32", ) + assert _meta_data(legacy_seed)["instance-id"] == ( + legacy_metadata["instance-id"]) @pytest.mark.skipif(importlib.util.find_spec("pycdlib") is None, diff --git a/engine/tests/test_engine_boundary.py b/engine/tests/test_engine_boundary.py index 9d0644f4..4ff3e350 100644 --- a/engine/tests/test_engine_boundary.py +++ b/engine/tests/test_engine_boundary.py @@ -412,6 +412,7 @@ def test_vpc_root_delete_endpoint_returns_exact_durable_receipt(boundary): assert receipt == { "vpc_id": "vpc-empty-delete", "operation_id": "vpc-empty-delete-op-0001", + "evidence_version": 2, "desired_revision": 3, "desired_digest": target["desired_digest"], "state": "deleted", diff --git a/engine/tests/test_libvirt_service.py b/engine/tests/test_libvirt_service.py index 0d77d6a7..1a7d5277 100644 --- a/engine/tests/test_libvirt_service.py +++ b/engine/tests/test_libvirt_service.py @@ -2,12 +2,15 @@ from __future__ import annotations +import base64 import hashlib import importlib.util +import ipaddress import json import pathlib import sys import tempfile +import time import types import unittest import uuid @@ -324,6 +327,29 @@ class FakeDomain: graphics.set("port", "5907" if self.running and flags == 0 else "-1") return ET.tostring(xml_root, encoding="unicode") + def setMetadata(self, metadata_type, metadata, key, uri, flags): + assert metadata_type == 1 + assert key == "qga-runtime" + assert uri == "urn:virsh-panel:qga-runtime:1" + assert flags == 1 + if not self.running or self.live_root is None: + raise FakeLibvirtError("domain is not running") + node = ET.fromstring(metadata) + metadata_node = self.live_root.find("metadata") + if metadata_node is None: + metadata_node = ET.SubElement(self.live_root, "metadata") + for old in list(metadata_node): + if old.tag == "{urn:virsh-panel:qga-runtime:1}qga-runtime": + metadata_node.remove(old) + metadata_node.append(copy.deepcopy(node)) + self.backend.metadata_updates.append((self._uuid, metadata, flags)) + if self.backend.stop_during_metadata_set: + self.running = False + self.live_root = None + if self.backend.fail_metadata_after_set: + raise FakeLibvirtError("metadata response lost") + return 0 + def autostart(self): return int(self.auto_start) @@ -371,6 +397,28 @@ class FakeDomain: def isPersistent(self): return self.persistent + def ID(self): + return self.backend.guest_domain_id if self.running else -1 + + def agentSetResponseTimeout(self, timeout, flags): + assert flags == 0 + self.backend.guest_agent_timeout_calls.append((timeout, flags)) + if (self.backend.fail_guest_agent_timeout_restore + and timeout == self.backend.guest_agent_default_timeout): + raise FakeLibvirtError("guest agent timeout restore failed") + if (self.backend.fail_guest_agent_timeout + and timeout != self.backend.guest_agent_default_timeout): + raise FakeLibvirtError("guest agent timeout unavailable") + return 0 + + def interfaceAddresses(self, source, flags): + assert source == self.backend.guest_agent_address_source + assert flags == 0 + self.backend.guest_interface_address_calls += 1 + if self.backend.fail_guest_interface_addresses: + raise FakeLibvirtError("guest agent interface query failed") + return copy.deepcopy(self.backend.guest_interfaces) + def create(self): if not self.persistent: raise FakeLibvirtError("cannot start a stopped transient domain") @@ -384,6 +432,7 @@ class FakeDomain: and not self.backend.bridge_active): raise FakeLibvirtError("bridge does not exist") self.live_root = copy.deepcopy(self.inactive_root) + self.backend.guest_exec_outputs.clear() self.running = True self.backend.domain_starts += 1 if self.backend.fail_domain_create_after_start: @@ -404,6 +453,7 @@ class FakeDomain: return self.running = False self.live_root = None + self.backend.guest_exec_outputs.clear() if not self.persistent: self.backend.domains.remove(self) @@ -417,6 +467,7 @@ class FakeDomain: self.backend.destroy_calls += 1 self.running = False self.live_root = None + self.backend.guest_exec_outputs.clear() if not self.persistent: self.backend.domains.remove(self) @@ -768,12 +819,99 @@ class FakeBackend: self.backup_terminal = self.job_completed self.opened = 0 self.closed = 0 + self.guest_domain_id = 7 + self.guest_agent_default_timeout = -1 + self.guest_agent_address_source = 1 + self.guest_agent_timeout_calls = [] + self.guest_interface_address_calls = 0 + self.guest_interfaces = {} + self.guest_links = [] + self.guest_routes = [] + self.guest_agent_commands = [] + self.guest_exec_outputs = {} + self.guest_exec_pid = 40 + self.fail_guest_agent_timeout = False + self.fail_guest_agent_timeout_restore = False + self.fail_guest_interface_addresses = False + self.fail_guest_agent_command = False + self.fail_guest_exec_status = False + self.guest_exec_status_exited = True + self.guest_exec_enabled = True + self.guest_exec_exitcode = 0 + self.metadata_updates = [] + self.fail_metadata_after_set = False + self.stop_during_metadata_set = False def open(self, uri): assert uri == "qemu+ssh://panel@hypervisor/system" self.opened += 1 return FakeConnection(self) + def qemu_agent_command(self, domain, payload, timeout, flags): + assert isinstance(domain, FakeDomain) + assert isinstance(timeout, int) and 1 <= timeout <= 5 + assert flags == 0 + if self.fail_guest_agent_command: + raise FakeLibvirtError("guest agent command unavailable") + request = json.loads(payload) + self.guest_agent_commands.append(copy.deepcopy(request)) + command = request.get("execute") + if command == "guest-info": + result = {"supported_commands": [ + {"name": "guest-exec", "enabled": self.guest_exec_enabled}, + {"name": "guest-exec-status", + "enabled": self.guest_exec_enabled}, + ]} + elif command == "guest-exec": + self.guest_exec_pid += 1 + arguments = request.get("arguments", {}).get("arg", []) + if any("/sys/class/net/" in value for value in arguments): + link = self.guest_links[0] if self.guest_links else {} + flags = { + str(value).upper() for value in link.get("flags", []) + } + carrier = ( + "1" if ("LOWER_UP" in flags + and "NO-CARRIER" not in flags) else "0") + numeric_flags = 0x1 if "UP" in flags else 0 + output = ( + f"{link.get('address', '')}\n" + f"{str(link.get('operstate', '')).lower()}\n" + f"{carrier}\n0x{numeric_flags:x}\n" + ).encode("ascii") + elif any("/usr/sbin/ip -j -4 route" in value + for value in arguments): + interface_name = arguments[-1] + rows = [copy.deepcopy(route) for route in self.guest_routes + if route.get("dev") == interface_name] + output = json.dumps( + rows, ensure_ascii=True, separators=(",", ":"), + ).encode("ascii")[:131073] + else: + output = b"" + self.guest_exec_outputs[self.guest_exec_pid] = copy.deepcopy(output) + result = {"pid": self.guest_exec_pid} + elif command == "guest-exec-status": + if self.fail_guest_exec_status: + raise FakeLibvirtError("guest agent status unavailable") + pid = request.get("arguments", {}).get("pid") + if pid not in self.guest_exec_outputs: + return json.dumps({"error": { + "class": "GenericError", + "desc": f"PID {pid} does not exist", + }}, separators=(",", ":")) + output = self.guest_exec_outputs[pid] + result = { + "exited": self.guest_exec_status_exited, + "exitcode": self.guest_exec_exitcode, + "out-data": base64.b64encode(output).decode("ascii"), + } + if self.guest_exec_status_exited: + self.guest_exec_outputs.pop(pid, None) + else: + return json.dumps({"error": {"class": "CommandNotFound"}}) + return json.dumps({"return": result}, separators=(",", ":")) + class VMServiceTests(unittest.TestCase): def setUp(self): @@ -788,8 +926,14 @@ class VMServiceTests(unittest.TestCase): fake_libvirt.VIR_DOMAIN_UNDEFINE_KEEP_NVRAM = 8 fake_libvirt.VIR_DOMAIN_UNDEFINE_CHECKPOINTS_METADATA = 16 fake_libvirt.VIR_DOMAIN_XML_INACTIVE = 1 + fake_libvirt.VIR_DOMAIN_METADATA_ELEMENT = 1 fake_libvirt.VIR_DOMAIN_AFFECT_LIVE = 1 fake_libvirt.VIR_DOMAIN_AFFECT_CONFIG = 2 + fake_libvirt.VIR_DOMAIN_INTERFACE_ADDRESSES_SRC_AGENT = ( + self.backend.guest_agent_address_source) + fake_libvirt.VIR_DOMAIN_AGENT_RESPONSE_TIMEOUT_DEFAULT = ( + self.backend.guest_agent_default_timeout) + fake_libvirt.VIR_IP_ADDR_TYPE_IPV4 = 0 fake_libvirt.VIR_DOMAIN_BACKUP_BEGIN_REUSE_EXTERNAL = ( self.backend.backup_reuse_external_flag) fake_libvirt.VIR_DOMAIN_JOB_NONE = self.backend.job_none @@ -803,10 +947,15 @@ class VMServiceTests(unittest.TestCase): fake_libvirt.VIR_DOMAIN_JOB_OPERATION_BACKUP = ( self.backend.job_operation_backup) fake_libvirt.open = self.backend.open + fake_libvirt_qemu = types.ModuleType("libvirt_qemu") + fake_libvirt_qemu.qemuAgentCommand = self.backend.qemu_agent_command module_spec = importlib.util.spec_from_file_location("service_under_test", SERVICE_PATH) self.assertIsNotNone(module_spec) module = importlib.util.module_from_spec(module_spec) - with patch.dict(sys.modules, {"libvirt": fake_libvirt}): + with patch.dict(sys.modules, { + "libvirt": fake_libvirt, + "libvirt_qemu": fake_libvirt_qemu, + }): module_spec.loader.exec_module(module) self.service_module = module self.service = module.VMService( @@ -2731,7 +2880,7 @@ class VMServiceTests(unittest.TestCase): replacement = replace_seed_private_network( bytes(seed.contents), private_mac=private_mac, private_ip_cidr=private_ip_cidr, - private_routes=["10.20.0.11/32"], + private_routes=["10.20.0.0/24"], ) seed.contents = bytearray(replacement) seed.capacity = len(replacement) @@ -2741,6 +2890,1382 @@ class VMServiceTests(unittest.TestCase): return (service, created, domain, seed, resources, private_mac, private_ip_cidr) + def private_guest_probe_domain(self, *, running=True): + vm_uuid = "33333333-3333-4333-8333-333333333333" + private_mac = "52:54:00:12:34:57" + private_ip_cidr = "10.20.0.10/32" + private_routes = ["10.20.0.0/24"] + domain = FakeDomain(self.backend, f""" + + guest-proof{vm_uuid} + 10485761 + + + + + + + + + + + + + """, running=running) + self.backend.domains.append(domain) + # The Python binding returns a mapping keyed by the guest interface + # name. Deliberately omit any duplicate name field in the value so a + # test cannot accidentally pass by reading the wrong shape. + self.backend.guest_interfaces = { + "ens7": { + "hwaddr": private_mac, + "addrs": [{ + "type": 0, "addr": "10.20.0.10", "prefix": 32, + }], + }, + } + self.backend.guest_links = [{ + "ifindex": 7, + "ifname": "ens7", + "flags": ["BROADCAST", "MULTICAST", "UP", "LOWER_UP"], + "address": private_mac, + "operstate": "UP", + }] + self.backend.guest_routes = [ + {"dst": route, "dev": "ens7", "scope": "link", + "protocol": "static"} + for route in private_routes + ] + return domain, private_mac, private_ip_cidr, private_routes + + def test_private_guest_probe_uses_binding_key_and_exact_qga_commands(self): + domain, private_mac, private_ip_cidr, private_routes = ( + self.private_guest_probe_domain()) + + pair = self.service.read_private_domain_pair( + domain.UUIDString(), + private_mac=private_mac, + private_ip_cidr=private_ip_cidr, + private_routes=private_routes, + ) + + evidence = pair["guest_network"] + self.assertTrue(evidence["verified"]) + self.assertEqual(evidence["state"], "confirmed") + self.assertEqual(evidence["source"], "qemu_guest_agent") + self.assertEqual(evidence["private_mac"], private_mac) + self.assertEqual(evidence["private_ip_cidr"], private_ip_cidr) + self.assertEqual(evidence["link_state"], "up") + self.assertEqual(evidence["routes"], sorted(private_routes)) + self.assertEqual( + self.backend.guest_agent_timeout_calls, + [(2, 0), (-1, 0), (2, 0), (-1, 0)], + ) + self.assertEqual( + [item["execute"] for item in self.backend.guest_agent_commands], + ["guest-info", "guest-exec", "guest-exec-status", + "guest-exec", "guest-exec-status"], + ) + link_execution = self.backend.guest_agent_commands[1]["arguments"] + self.assertEqual(link_execution["path"], "/usr/bin/timeout") + self.assertEqual( + link_execution["arg"], + ["--signal=KILL", "1", "/usr/bin/head", "-q", "-c", + "4097", "/sys/class/net/ens7/address", + "/sys/class/net/ens7/operstate", + "/sys/class/net/ens7/carrier", + "/sys/class/net/ens7/flags"], + ) + route_execution = self.backend.guest_agent_commands[3]["arguments"] + self.assertEqual(route_execution["path"], "/usr/bin/timeout") + self.assertEqual( + route_execution["arg"], + ["--signal=KILL", "1", "/bin/sh", "-c", + self.service_module.VPC_GUEST_ROUTE_SCRIPT, + "vpc-route-probe", "ens7"], + ) + self.assertTrue(link_execution["capture-output"]) + self.assertTrue(route_execution["capture-output"]) + + def test_private_guest_probe_rejects_down_link_with_retained_state(self): + domain, private_mac, private_ip_cidr, private_routes = ( + self.private_guest_probe_domain()) + self.backend.guest_links[0]["flags"] = [ + "BROADCAST", "MULTICAST", "UP", "NO-CARRIER", + ] + self.backend.guest_links[0]["operstate"] = "DOWN" + + evidence = self.service.read_private_domain_pair( + domain.UUIDString(), + private_mac=private_mac, + private_ip_cidr=private_ip_cidr, + private_routes=private_routes, + )["guest_network"] + + self.assertFalse(evidence["verified"]) + self.assertEqual(evidence["state"], "mismatch") + self.assertEqual( + evidence["reason_code"], + "vpc_private_guest_link_mismatch", + ) + + def test_private_guest_probe_rejects_linkdown_route_flag(self): + domain, private_mac, private_ip_cidr, private_routes = ( + self.private_guest_probe_domain()) + self.backend.guest_routes[0]["flags"] = ["linkdown"] + + evidence = self.service.read_private_domain_pair( + domain.UUIDString(), + private_mac=private_mac, + private_ip_cidr=private_ip_cidr, + private_routes=private_routes, + )["guest_network"] + + self.assertFalse(evidence["verified"]) + self.assertEqual(evidence["state"], "mismatch") + self.assertEqual( + evidence["reason_code"], + "vpc_private_guest_route_mismatch", + ) + + def test_private_guest_probe_rejects_missing_vpc_scope_route(self): + domain, private_mac, private_ip_cidr, private_routes = ( + self.private_guest_probe_domain()) + self.backend.guest_routes = [] + + evidence = self.service.read_private_domain_pair( + domain.UUIDString(), + private_mac=private_mac, + private_ip_cidr=private_ip_cidr, + private_routes=private_routes, + )["guest_network"] + + self.assertFalse(evidence["verified"]) + self.assertEqual(evidence["state"], "mismatch") + self.assertEqual( + evidence["reason_code"], + "vpc_private_guest_route_mismatch", + ) + + def test_private_guest_probe_rejects_address_mismatch(self): + domain, private_mac, private_ip_cidr, private_routes = ( + self.private_guest_probe_domain()) + self.backend.guest_interfaces["ens7"]["addrs"][0]["addr"] = ( + "10.20.0.99") + + evidence = self.service.read_private_domain_pair( + domain.UUIDString(), + private_mac=private_mac, + private_ip_cidr=private_ip_cidr, + private_routes=private_routes, + )["guest_network"] + + self.assertFalse(evidence["verified"]) + self.assertEqual(evidence["state"], "mismatch") + self.assertEqual( + evidence["reason_code"], + "vpc_private_guest_address_mismatch", + ) + self.assertEqual(self.backend.guest_agent_commands, []) + + def test_private_guest_probe_scopes_route_read_to_private_interface(self): + domain, private_mac, private_ip_cidr, private_routes = ( + self.private_guest_probe_domain()) + self.backend.guest_routes.append({ + "dst": private_routes[0], "dev": "ens3", "scope": "link", + "protocol": "static", + }) + + evidence = self.service.read_private_domain_pair( + domain.UUIDString(), + private_mac=private_mac, + private_ip_cidr=private_ip_cidr, + private_routes=private_routes, + )["guest_network"] + + self.assertTrue(evidence["verified"]) + self.assertEqual(evidence["state"], "confirmed") + + def test_private_guest_probe_restores_timeout_when_agent_is_unavailable(self): + domain, private_mac, private_ip_cidr, private_routes = ( + self.private_guest_probe_domain()) + self.backend.fail_guest_interface_addresses = True + + evidence = self.service.read_private_domain_pair( + domain.UUIDString(), + private_mac=private_mac, + private_ip_cidr=private_ip_cidr, + private_routes=private_routes, + )["guest_network"] + + self.assertFalse(evidence["verified"]) + self.assertEqual(evidence["state"], "unavailable") + self.assertEqual( + evidence["reason_code"], + "vpc_private_guest_agent_unavailable", + ) + self.assertEqual( + self.backend.guest_agent_timeout_calls, + [(2, 0), (-1, 0)], + ) + + def test_private_guest_probe_fails_closed_when_timeout_restore_fails(self): + domain, private_mac, private_ip_cidr, private_routes = ( + self.private_guest_probe_domain()) + self.backend.fail_guest_agent_timeout_restore = True + + evidence = self.service.read_private_domain_pair( + domain.UUIDString(), + private_mac=private_mac, + private_ip_cidr=private_ip_cidr, + private_routes=private_routes, + )["guest_network"] + + self.assertFalse(evidence["verified"]) + self.assertEqual(evidence["state"], "uncertain") + self.assertEqual( + evidence["reason_code"], + "vpc_private_guest_probe_uncertain", + ) + self.assertEqual( + self.backend.guest_agent_timeout_calls, + [(2, 0), (-1, 0)], + ) + + def test_private_guest_probe_reports_disabled_exec_as_unsupported(self): + domain, private_mac, private_ip_cidr, private_routes = ( + self.private_guest_probe_domain()) + self.backend.guest_exec_enabled = False + + evidence = self.service.read_private_domain_pair( + domain.UUIDString(), + private_mac=private_mac, + private_ip_cidr=private_ip_cidr, + private_routes=private_routes, + )["guest_network"] + + self.assertFalse(evidence["verified"]) + self.assertEqual(evidence["state"], "unsupported") + self.assertEqual( + evidence["reason_code"], + "vpc_private_guest_probe_unsupported", + ) + self.assertEqual( + [item["execute"] for item in self.backend.guest_agent_commands], + ["guest-info"], + ) + + def test_private_guest_probe_skips_qga_for_stale_live_generation(self): + domain, private_mac, private_ip_cidr, private_routes = ( + self.private_guest_probe_domain()) + managed = domain.live_root.find( + "./metadata/{urn:virsh-panel:1}managed") + managed.set("private-guest-network-generation", "old-generation") + + evidence = self.service.read_private_domain_pair( + domain.UUIDString(), + private_mac=private_mac, + private_ip_cidr=private_ip_cidr, + private_routes=private_routes, + )["guest_network"] + + self.assertFalse(evidence["verified"]) + self.assertEqual(evidence["state"], "pending_restart") + self.assertEqual( + evidence["reason_code"], + "vpc_private_guest_restart_required", + ) + self.assertEqual(self.backend.guest_agent_commands, []) + + def test_private_guest_probe_reports_stopped_without_contacting_agent(self): + domain, private_mac, private_ip_cidr, private_routes = ( + self.private_guest_probe_domain(running=False)) + + evidence = self.service.read_private_domain_pair( + domain.UUIDString(), + private_mac=private_mac, + private_ip_cidr=private_ip_cidr, + private_routes=private_routes, + )["guest_network"] + + self.assertFalse(evidence["verified"]) + self.assertEqual(evidence["state"], "stopped") + self.assertEqual( + evidence["reason_code"], + "vpc_private_guest_stopped", + ) + self.assertEqual(self.backend.guest_interface_address_calls, 0) + self.assertEqual(self.backend.guest_agent_commands, []) + + def test_private_guest_probe_deadline_fails_closed_before_qga(self): + domain, private_mac, private_ip_cidr, private_routes = ( + self.private_guest_probe_domain()) + + evidence = self.service.read_private_domain_pair( + domain.UUIDString(), + private_mac=private_mac, + private_ip_cidr=private_ip_cidr, + private_routes=private_routes, + guest_probe_deadline=time.monotonic() + 0.5, + )["guest_network"] + + self.assertFalse(evidence["verified"]) + self.assertEqual(evidence["state"], "deadline_exceeded") + self.assertEqual( + evidence["reason_code"], + "vpc_private_guest_probe_deadline_exceeded", + ) + self.assertEqual(self.backend.guest_interface_address_calls, 0) + self.assertEqual(self.backend.guest_agent_commands, []) + + def test_private_guest_probe_persists_pid_and_blocks_cross_request_status(self): + domain, private_mac, private_ip_cidr, private_routes = ( + self.private_guest_probe_domain()) + durable = {} + writes = [] + + def persist(value): + durable.clear() + durable.update(copy.deepcopy(value)) + writes.append(copy.deepcopy(value)) + + def clear(): + durable.clear() + + self.backend.fail_guest_exec_status = True + first_state = {} + first = self.service.read_private_domain_pair( + domain.UUIDString(), + private_mac=private_mac, + private_ip_cidr=private_ip_cidr, + private_routes=private_routes, + guest_exec_state=first_state, + guest_exec_persist=persist, + guest_exec_clear=clear, + )["guest_network"] + + self.assertFalse(first["verified"]) + self.assertEqual(first["state"], "unavailable") + self.assertEqual(durable.get("phase"), "pending") + self.assertEqual(durable.get("pid"), 41) + self.assertEqual(writes[0]["phase"], "launch_intent") + self.assertEqual(writes[1]["phase"], "pending") + self.assertEqual( + [item["execute"] for item in self.backend.guest_agent_commands] + .count("guest-exec"), + 1, + ) + + self.backend.fail_guest_exec_status = False + self.backend.guest_agent_commands.clear() + second_state = copy.deepcopy(durable) + second = self.service.read_private_domain_pair( + domain.UUIDString(), + private_mac=private_mac, + private_ip_cidr=private_ip_cidr, + private_routes=private_routes, + guest_exec_state=second_state, + guest_exec_persist=persist, + guest_exec_clear=clear, + )["guest_network"] + + self.assertFalse(second["verified"]) + self.assertEqual( + second["reason_code"], + "vpc_private_guest_probe_reap_required", + ) + commands = [ + item["execute"] for item in self.backend.guest_agent_commands + ] + self.assertEqual(commands, []) + self.assertEqual(durable.get("pid"), 41) + + # An observed stop proves that the old QEMU/QGA process ledger cannot + # survive. The later start gets a new LIVE token and may launch. + old_token = durable["qemu_runtime_token"] + domain.shutdown() + stopped = self.service.read_private_domain_pair( + domain.UUIDString(), private_mac=private_mac, + private_ip_cidr=private_ip_cidr, private_routes=private_routes, + guest_exec_state=copy.deepcopy(durable), + guest_exec_persist=persist, guest_exec_clear=clear, + )["guest_network"] + self.assertEqual(stopped["state"], "stopped") + self.assertEqual(durable, {}) + domain.create() + self.backend.guest_agent_commands.clear() + third = self.service.read_private_domain_pair( + domain.UUIDString(), private_mac=private_mac, + private_ip_cidr=private_ip_cidr, private_routes=private_routes, + guest_exec_state={}, + guest_exec_persist=persist, guest_exec_clear=clear, + )["guest_network"] + self.assertTrue(third["verified"]) + runtime_node = domain.live_root.find( + "./metadata/{urn:virsh-panel:qga-runtime:1}qga-runtime") + self.assertIsNotNone(runtime_node) + self.assertNotEqual(runtime_node.get("token"), old_token) + self.assertNotEqual( + self.backend.guest_agent_commands[0]["execute"], + "guest-exec-status", + ) + self.assertEqual(durable, {}) + + def test_private_guest_probe_launch_intent_blocks_until_stopped(self): + domain, private_mac, private_ip_cidr, private_routes = ( + self.private_guest_probe_domain()) + durable = {} + + def persist(value): + durable.clear() + durable.update(copy.deepcopy(value)) + + def clear(): + durable.clear() + + original = self.service_module.libvirt_qemu.qemuAgentCommand + + def lose_exec_reply(target, payload, timeout, flags): + if json.loads(payload).get("execute") == "guest-exec": + raise FakeLibvirtError("transport response lost") + return original(target, payload, timeout, flags) + + self.service_module.libvirt_qemu.qemuAgentCommand = lose_exec_reply + state = {} + first = self.service.read_private_domain_pair( + domain.UUIDString(), private_mac=private_mac, + private_ip_cidr=private_ip_cidr, private_routes=private_routes, + guest_exec_state=state, guest_exec_persist=persist, + guest_exec_clear=clear, + )["guest_network"] + self.assertFalse(first["verified"]) + self.assertEqual(durable.get("phase"), "launch_intent") + + self.service_module.libvirt_qemu.qemuAgentCommand = original + self.backend.guest_agent_commands.clear() + second_state = copy.deepcopy(durable) + second = self.service.read_private_domain_pair( + domain.UUIDString(), private_mac=private_mac, + private_ip_cidr=private_ip_cidr, private_routes=private_routes, + guest_exec_state=second_state, guest_exec_persist=persist, + guest_exec_clear=clear, + )["guest_network"] + self.assertEqual(second["state"], "uncertain") + self.assertEqual( + second["reason_code"], + "vpc_private_guest_probe_reap_required") + self.assertEqual(self.backend.guest_agent_commands, []) + + domain.running = False + stopped_state = copy.deepcopy(durable) + stopped = self.service.read_private_domain_pair( + domain.UUIDString(), private_mac=private_mac, + private_ip_cidr=private_ip_cidr, private_routes=private_routes, + guest_exec_state=stopped_state, guest_exec_persist=persist, + guest_exec_clear=clear, + )["guest_network"] + self.assertEqual(stopped["state"], "stopped") + self.assertEqual(durable, {}) + + def test_private_guest_probe_agent_restart_never_statuses_old_pid(self): + domain, private_mac, private_ip_cidr, private_routes = ( + self.private_guest_probe_domain()) + durable = {} + + def persist(value): + durable.clear() + durable.update(copy.deepcopy(value)) + + def clear(): + durable.clear() + + self.backend.fail_guest_exec_status = True + state = {} + self.service.read_private_domain_pair( + domain.UUIDString(), private_mac=private_mac, + private_ip_cidr=private_ip_cidr, private_routes=private_routes, + guest_exec_state=state, guest_exec_persist=persist, + guest_exec_clear=clear, + ) + self.assertEqual(durable.get("phase"), "pending") + + self.backend.fail_guest_exec_status = False + self.backend.guest_exec_outputs.clear() + self.backend.guest_agent_commands.clear() + recovered = self.service.read_private_domain_pair( + domain.UUIDString(), private_mac=private_mac, + private_ip_cidr=private_ip_cidr, private_routes=private_routes, + guest_exec_state=copy.deepcopy(durable), + guest_exec_persist=persist, guest_exec_clear=clear, + )["guest_network"] + self.assertFalse(recovered["verified"]) + self.assertEqual( + recovered["reason_code"], + "vpc_private_guest_probe_reap_required", + ) + self.assertEqual(self.backend.guest_agent_commands, []) + self.assertEqual(durable.get("phase"), "pending") + + def test_private_guest_probe_rejects_truncated_route_table(self): + domain, private_mac, private_ip_cidr, private_routes = ( + self.private_guest_probe_domain()) + self.backend.guest_routes.extend({ + "dst": f"10.{index // 256}.{index % 256}.1/32", + "dev": "ens7", "scope": "link", "protocol": "static", + } for index in range(4096)) + + evidence = self.service.read_private_domain_pair( + domain.UUIDString(), private_mac=private_mac, + private_ip_cidr=private_ip_cidr, private_routes=private_routes, + )["guest_network"] + + self.assertFalse(evidence["verified"]) + self.assertEqual(evidence["state"], "uncertain") + self.assertEqual( + evidence["reason_code"], "vpc_private_guest_probe_uncertain") + + def test_private_guest_probe_suspended_skips_guest_exec(self): + domain, private_mac, private_ip_cidr, private_routes = ( + self.private_guest_probe_domain()) + ET.SubElement( + domain.live_root.find("./devices/interface[2]"), + "link", {"state": "down"}, + ) + + evidence = self.service.read_private_domain_pair( + domain.UUIDString(), private_mac=private_mac, + private_ip_cidr=private_ip_cidr, private_routes=private_routes, + )["guest_network"] + + self.assertFalse(evidence["verified"]) + self.assertEqual(evidence["state"], "suspended") + self.assertEqual( + evidence["reason_code"], "vpc_private_network_suspended") + self.assertEqual(self.backend.guest_agent_commands, []) + + def test_private_guest_probe_runtime_token_response_loss_uses_readback(self): + domain, private_mac, private_ip_cidr, private_routes = ( + self.private_guest_probe_domain()) + self.backend.fail_metadata_after_set = True + + pair = self.service.read_private_domain_pair( + domain.UUIDString(), private_mac=private_mac, + private_ip_cidr=private_ip_cidr, private_routes=private_routes, + ) + + self.assertTrue(pair["guest_network"]["verified"]) + self.assertEqual(len(self.backend.metadata_updates), 1) + runtime = domain.live_root.find( + "./metadata/{urn:virsh-panel:qga-runtime:1}qga-runtime") + self.assertIsNotNone(runtime) + self.assertEqual(runtime.get("domain-uuid"), domain.UUIDString()) + self.assertRegex(runtime.get("token"), r"^[0-9a-f]{32}$") + self.assertNotIn("qga-runtime", pair["live"]) + self.assertNotIn("qga-runtime", pair["inactive"]) + self.assertIsNone(domain.inactive_root.find( + "./metadata/{urn:virsh-panel:qga-runtime:1}qga-runtime")) + + def test_private_guest_probe_runtime_token_stop_race_fails_closed(self): + domain, private_mac, private_ip_cidr, private_routes = ( + self.private_guest_probe_domain()) + self.backend.stop_during_metadata_set = True + + evidence = self.service.read_private_domain_pair( + domain.UUIDString(), private_mac=private_mac, + private_ip_cidr=private_ip_cidr, private_routes=private_routes, + )["guest_network"] + + self.assertFalse(evidence["verified"]) + self.assertEqual(evidence["state"], "unavailable") + self.assertEqual(self.backend.guest_agent_commands, []) + self.assertIsNone(domain.inactive_root.find( + "./metadata/{urn:virsh-panel:qga-runtime:1}qga-runtime")) + + def test_private_guest_probe_runtime_token_is_bound_to_domain_uuid(self): + domain, private_mac, private_ip_cidr, private_routes = ( + self.private_guest_probe_domain()) + metadata = domain.live_root.find("metadata") + ET.SubElement( + metadata, + "{urn:virsh-panel:qga-runtime:1}qga-runtime", + {"domain-uuid": "44444444-4444-4444-8444-444444444444", + "token": "a" * 32}, + ) + + evidence = self.service.read_private_domain_pair( + domain.UUIDString(), private_mac=private_mac, + private_ip_cidr=private_ip_cidr, private_routes=private_routes, + )["guest_network"] + + self.assertFalse(evidence["verified"]) + self.assertEqual(evidence["state"], "unavailable") + self.assertEqual(self.backend.guest_agent_commands, []) + + def test_private_guest_probe_guest_reboot_keeps_old_pid_blocked(self): + domain, private_mac, private_ip_cidr, private_routes = ( + self.private_guest_probe_domain()) + durable = {} + + def persist(value): + durable.clear() + durable.update(copy.deepcopy(value)) + + def clear(): + durable.clear() + + self.backend.fail_guest_exec_status = True + self.service.read_private_domain_pair( + domain.UUIDString(), private_mac=private_mac, + private_ip_cidr=private_ip_cidr, private_routes=private_routes, + guest_exec_state={}, guest_exec_persist=persist, + guest_exec_clear=clear, + ) + token = durable["qemu_runtime_token"] + domain.guest_reboot() + self.backend.guest_exec_outputs.clear() + self.backend.fail_guest_exec_status = False + self.backend.guest_agent_commands.clear() + + evidence = self.service.read_private_domain_pair( + domain.UUIDString(), private_mac=private_mac, + private_ip_cidr=private_ip_cidr, private_routes=private_routes, + guest_exec_state=copy.deepcopy(durable), + guest_exec_persist=persist, guest_exec_clear=clear, + )["guest_network"] + + self.assertEqual( + evidence["reason_code"], + "vpc_private_guest_probe_reap_required", + ) + self.assertEqual(durable["qemu_runtime_token"], token) + self.assertEqual(self.backend.guest_agent_commands, []) + + def test_private_guest_probe_changed_live_token_cannot_clear_ledger(self): + domain, private_mac, private_ip_cidr, private_routes = ( + self.private_guest_probe_domain()) + durable = {} + clear_calls = 0 + + def persist(value): + durable.clear() + durable.update(copy.deepcopy(value)) + + def clear(): + nonlocal clear_calls + clear_calls += 1 + durable.clear() + + self.backend.fail_guest_exec_status = True + self.service.read_private_domain_pair( + domain.UUIDString(), private_mac=private_mac, + private_ip_cidr=private_ip_cidr, private_routes=private_routes, + guest_exec_state={}, guest_exec_persist=persist, + guest_exec_clear=clear, + ) + old_token = durable["qemu_runtime_token"] + runtime = domain.live_root.find( + "./metadata/{urn:virsh-panel:qga-runtime:1}qga-runtime") + replacement = "f" * 32 if old_token != "f" * 32 else "e" * 32 + runtime.set("token", replacement) + self.backend.fail_guest_exec_status = False + self.backend.guest_agent_commands.clear() + + evidence = self.service.read_private_domain_pair( + domain.UUIDString(), private_mac=private_mac, + private_ip_cidr=private_ip_cidr, private_routes=private_routes, + guest_exec_state=copy.deepcopy(durable), + guest_exec_persist=persist, guest_exec_clear=clear, + )["guest_network"] + + self.assertEqual( + evidence["reason_code"], + "vpc_private_guest_probe_reap_required", + ) + self.assertEqual(clear_calls, 0) + self.assertEqual(durable["qemu_runtime_token"], old_token) + self.assertEqual(durable.get("phase"), "pending") + self.assertEqual(self.backend.guest_agent_commands, []) + + def test_private_guest_probe_missing_live_token_cannot_clear_ledger(self): + domain, private_mac, private_ip_cidr, private_routes = ( + self.private_guest_probe_domain()) + durable = {} + + def persist(value): + durable.clear() + durable.update(copy.deepcopy(value)) + + self.backend.fail_guest_exec_status = True + self.service.read_private_domain_pair( + domain.UUIDString(), private_mac=private_mac, + private_ip_cidr=private_ip_cidr, private_routes=private_routes, + guest_exec_state={}, guest_exec_persist=persist, + guest_exec_clear=lambda: durable.clear(), + ) + metadata = domain.live_root.find("metadata") + runtime = metadata.find( + "{urn:virsh-panel:qga-runtime:1}qga-runtime") + metadata.remove(runtime) + update_count = len(self.backend.metadata_updates) + self.backend.fail_guest_exec_status = False + self.backend.guest_agent_commands.clear() + + evidence = self.service.read_private_domain_pair( + domain.UUIDString(), private_mac=private_mac, + private_ip_cidr=private_ip_cidr, private_routes=private_routes, + guest_exec_state=copy.deepcopy(durable), + guest_exec_persist=persist, + guest_exec_clear=lambda: durable.clear(), + )["guest_network"] + + self.assertEqual( + evidence["reason_code"], + "vpc_private_guest_probe_reap_required", + ) + self.assertEqual(len(self.backend.metadata_updates), update_count) + self.assertEqual(self.backend.guest_agent_commands, []) + self.assertEqual(durable.get("phase"), "pending") + + def test_running_private_network_waits_for_stop_before_staging(self): + from virsh_panel.vpc import VpcMember + + service = self.cloud_service() + spec = self.cloud_spec() + created = self.create_cloud_vm(service, spec) + domain = self.backend.domains[0] + managed_before = domain.inactive_root.find( + f"./metadata/{{{self.service_module.PANEL_NS}}}managed") + seed_name = managed_before.get("seed-volume") + member = VpcMember( + "member-stage-test", created["uuid"], "host-a", + "10.20.0.10", "subnet-a", "52:54:00:12:34:57", + ) + resources = types.SimpleNamespace(bridge="vcbr-private") + seed = self.backend.pools["images"].volumes[seed_name] + initial_inactive = domain.XMLDesc(1) + initial_live = domain.XMLDesc(0) + initial_seed = bytes(seed.contents) + initial_defines = len(self.backend.define_xmls) + private_seed = b"SEED-WITH-PRIVATE-NETWORK" + updated_private_seed = b"SEED-WITH-UPDATED-PRIVATE-ROUTES" + + def read_network(contents): + result = { + "mac": domain.inactive_root.find( + "./devices/interface/mac").get("address"), + "public_ip_cidr": spec["public_ip_cidr"], + } + if contents in {private_seed, updated_private_seed}: + result.update({ + "private_mac": member.private_mac, + "private_ip_cidr": f"{member.private_ip}/32", + "private_routes": (["10.20.0.0/24"] + if contents == private_seed else + ["10.20.0.0/23"]), + "roles": ["public0", "private0"], + }) + return result + + def replace_network(contents, **changes): + desired = (updated_private_seed + if changes.get("private_routes") == [ + "10.20.0.0/23"] + else private_seed) + return contents if contents == desired else desired + + patches = ( + patch.object(self.service_module, "read_seed_network", + side_effect=read_network), + patch.object(self.service_module, "replace_seed_private_network", + side_effect=replace_network), + ) + with patches[0], patches[1]: + staged = service.ensure_private_interface( + created["uuid"], member=member, resources=resources, + vpc_id="vpc-stage-test", desired_revision=1, + desired_digest="a" * 64, + private_routes=["10.20.0.0/24"], + ) + + self.assertEqual(staged["readback"], "pending_restart") + self.assertTrue(staged["restart_required"]) + self.assertFalse(staged["seed_updated"]) + self.assertEqual( + ET.canonicalize(staged["xml"], rewrite_prefixes=True), + ET.canonicalize(initial_inactive, rewrite_prefixes=True), + ) + self.assertEqual( + ET.canonicalize(staged["live"], rewrite_prefixes=True), + ET.canonicalize(initial_live, rewrite_prefixes=True), + ) + self.assertEqual(bytes(seed.contents), initial_seed) + self.assertEqual( + ET.canonicalize(domain.XMLDesc(1), rewrite_prefixes=True), + ET.canonicalize(initial_inactive, rewrite_prefixes=True), + ) + self.assertEqual(len(self.backend.define_xmls), initial_defines) + live_before = ET.fromstring(initial_live) + live_after_stage = ET.fromstring(domain.XMLDesc(0)) + self.assertIsNone(live_before.find( + "./devices/interface/alias[@name='ua-private0']")) + self.assertIsNone(live_after_stage.find( + "./devices/interface/alias[@name='ua-private0']")) + self.assertIsNone(live_after_stage.find( + f"./metadata/{{{self.service_module.PANEL_NS}}}managed" + ).get("private-guest-network-generation")) + self.assertEqual(self.backend.device_updates, []) + + retried = service.ensure_private_interface( + created["uuid"], member=member, resources=resources, + vpc_id="vpc-stage-test", desired_revision=1, + desired_digest="a" * 64, + private_routes=["10.20.0.0/24"], + ) + self.assertEqual(retried["readback"], "pending_restart") + self.assertFalse(retried["seed_updated"]) + self.assertEqual(bytes(seed.contents), initial_seed) + self.assertEqual( + ET.canonicalize(domain.XMLDesc(1), rewrite_prefixes=True), + ET.canonicalize(initial_inactive, rewrite_prefixes=True), + ) + self.assertEqual(len(self.backend.define_xmls), initial_defines) + + domain.shutdown() + stopped_stage = service.ensure_private_interface( + created["uuid"], member=member, resources=resources, + vpc_id="vpc-stage-test", desired_revision=1, + desired_digest="a" * 64, + private_routes=["10.20.0.0/24"], + ) + self.assertEqual(stopped_stage["readback"], "confirmed") + self.assertTrue(stopped_stage["seed_updated"]) + self.assertFalse(stopped_stage["restart_required"]) + self.assertIsNone(stopped_stage["live"]) + inactive = ET.fromstring(domain.XMLDesc(1)) + managed = inactive.find( + f"./metadata/{{{self.service_module.PANEL_NS}}}managed") + generation = managed.get("private-guest-network-generation") + self.assertRegex(generation, r"^[0-9a-f]{64}$") + self.assertIsNotNone( + inactive.find("./devices/interface/alias[@name='ua-private0']")) + self.assertEqual(bytes(seed.contents), private_seed) + self.assertEqual(len(self.backend.define_xmls), + initial_defines + 1) + + stopped_guest = service.read_private_domain_pair( + created["uuid"], private_mac=member.private_mac, + private_ip_cidr=f"{member.private_ip}/32", + private_routes=["10.20.0.0/24"], + )["guest_network"] + self.assertFalse(stopped_guest["verified"]) + self.assertEqual(stopped_guest["state"], "stopped") + + self.backend.guest_interfaces = { + "ens7": { + "hwaddr": member.private_mac, + "addrs": [{ + "type": 0, + "addr": member.private_ip, + "prefix": 32, + }], + }, + } + self.backend.guest_links = [{ + "ifindex": 7, + "ifname": "ens7", + "flags": ["BROADCAST", "MULTICAST", "UP", "LOWER_UP"], + "address": member.private_mac, + "operstate": "UP", + }] + self.backend.guest_routes = [{ + "dst": "10.20.0.0/24", + "dev": "ens7", + "scope": "link", + "protocol": "static", + }] + domain.create() + confirmed = service.ensure_private_interface( + created["uuid"], member=member, resources=resources, + vpc_id="vpc-stage-test", desired_revision=1, + desired_digest="a" * 64, + private_routes=["10.20.0.0/24"], + ) + self.assertEqual(confirmed["readback"], "confirmed") + self.assertFalse(confirmed["restart_required"]) + live = ET.fromstring(domain.XMLDesc(0)) + live_managed = live.find( + f"./metadata/{{{self.service_module.PANEL_NS}}}managed") + self.assertEqual( + live_managed.get("private-guest-network-generation"), + generation) + guest_ready = service.read_private_domain_pair( + created["uuid"], private_mac=member.private_mac, + private_ip_cidr=f"{member.private_ip}/32", + private_routes=["10.20.0.0/24"], + )["guest_network"] + self.assertTrue(guest_ready["verified"]) + self.assertEqual(guest_ready["state"], "confirmed") + + device_updates = len(self.backend.device_updates) + acl_only = service.ensure_private_interface( + created["uuid"], member=member, resources=resources, + vpc_id="vpc-stage-test", desired_revision=2, + desired_digest="b" * 64, + private_routes=["10.20.0.0/24"], + ) + self.assertEqual(acl_only["readback"], "confirmed") + self.assertFalse(acl_only["seed_updated"]) + self.assertFalse(acl_only["restart_required"]) + self.assertEqual(len(self.backend.device_updates), device_updates) + + route_change_inactive = domain.XMLDesc(1) + route_change_live = domain.XMLDesc(0) + route_change_seed = bytes(seed.contents) + route_change_defines = len(self.backend.define_xmls) + route_change = service.ensure_private_interface( + created["uuid"], member=member, resources=resources, + vpc_id="vpc-stage-test", desired_revision=3, + desired_digest="c" * 64, + private_routes=["10.20.0.0/23"], + ) + self.assertEqual(route_change["readback"], "pending_restart") + self.assertFalse(route_change["seed_updated"]) + self.assertTrue(route_change["restart_required"]) + self.assertEqual( + ET.canonicalize(route_change["xml"], rewrite_prefixes=True), + ET.canonicalize(route_change_inactive, rewrite_prefixes=True), + ) + self.assertEqual( + ET.canonicalize(route_change["live"], rewrite_prefixes=True), + ET.canonicalize(route_change_live, rewrite_prefixes=True), + ) + self.assertEqual(bytes(seed.contents), route_change_seed) + self.assertEqual( + ET.canonicalize(domain.XMLDesc(1), rewrite_prefixes=True), + ET.canonicalize(route_change_inactive, rewrite_prefixes=True), + ) + self.assertEqual(len(self.backend.define_xmls), + route_change_defines) + self.assertEqual(len(self.backend.device_updates), device_updates) + + domain.shutdown() + route_staged = service.ensure_private_interface( + created["uuid"], member=member, resources=resources, + vpc_id="vpc-stage-test", desired_revision=3, + desired_digest="c" * 64, + private_routes=["10.20.0.0/23"], + ) + self.assertEqual(route_staged["readback"], "confirmed") + self.assertTrue(route_staged["seed_updated"]) + self.assertEqual(bytes(seed.contents), updated_private_seed) + self.assertEqual(len(self.backend.define_xmls), + route_change_defines + 1) + + domain.create() + self.backend.guest_routes[0]["dst"] = "10.20.0.0/23" + updated_guest = service.read_private_domain_pair( + created["uuid"], private_mac=member.private_mac, + private_ip_cidr=f"{member.private_ip}/32", + private_routes=["10.20.0.0/23"], + )["guest_network"] + self.assertTrue(updated_guest["verified"]) + self.assertEqual(updated_guest["state"], "confirmed") + + def test_private_seed_upload_is_read_back_before_domain_define(self): + from virsh_panel.vpc import VpcMember + + (service, created, domain, seed, resources, + private_mac, _private_ip_cidr) = self.cloud_vm_with_private_artifacts( + include_private_nic=True) + member = VpcMember( + "member-delete-test", created["uuid"], "host-a", + "10.20.0.10", "subnet-a", private_mac, + ) + old_xml = domain.XMLDesc(1) + old_seed = bytes(seed.contents) + define_count = len(self.backend.define_xmls) + original_download = service._download_seed_volume + downloads = 0 + + def corrupt_first_uploaded_readback(connection, volume): + nonlocal downloads + downloads += 1 + if downloads == 2: + return b"not-a-seed" + return original_download(connection, volume) + + with patch.object( + service, "_download_seed_volume", + side_effect=corrupt_first_uploaded_readback): + with self.assertRaises(self.error_type) as mismatch: + service.ensure_private_interface( + created["uuid"], member=member, resources=resources, + vpc_id="vpc-delete-test", desired_revision=2, + desired_digest="b" * 64, + private_routes=["10.20.0.0/23"], + ) + + self.assertEqual( + mismatch.exception.error_code, + "vpc_seed_readback_mismatch", + ) + self.assertEqual(downloads, 2) + self.assertEqual(len(self.backend.define_xmls), define_count) + self.assertEqual( + ET.canonicalize(domain.XMLDesc(1), rewrite_prefixes=True), + ET.canonicalize(old_xml, rewrite_prefixes=True), + ) + self.assertEqual(bytes(seed.contents), old_seed) + + def test_private_seed_rollback_failure_is_uncertain(self): + from virsh_panel.vpc import VpcMember + + (service, created, _domain, seed, resources, + private_mac, _private_ip_cidr) = ( + self.cloud_vm_with_private_artifacts(include_private_nic=True)) + member = VpcMember( + "member-delete-test", created["uuid"], "host-a", + "10.20.0.10", "subnet-a", private_mac, + ) + old_seed = bytes(seed.contents) + original_download = service._download_seed_volume + original_upload = service._upload_volume + downloads = 0 + uploads = 0 + + def corrupt_uploaded_readback(connection, volume): + nonlocal downloads + downloads += 1 + if downloads == 2: + return b"not-a-seed" + return original_download(connection, volume) + + def fail_restore_upload(connection, volume, chunks, size): + nonlocal uploads + uploads += 1 + if uploads == 2: + raise FakeLibvirtError("rollback seed upload failed") + return original_upload(connection, volume, chunks, size) + + with patch.object( + service, "_download_seed_volume", + side_effect=corrupt_uploaded_readback), patch.object( + service, "_upload_volume", + side_effect=fail_restore_upload): + with self.assertRaises(self.error_type) as uncertain: + service.ensure_private_interface( + created["uuid"], member=member, resources=resources, + vpc_id="vpc-delete-test", desired_revision=2, + desired_digest="b" * 64, + private_routes=["10.20.0.0/23"], + ) + + self.assertEqual( + uncertain.exception.error_code, + "vpc_private_result_uncertain", + ) + self.assertEqual(uploads, 2) + self.assertNotEqual(bytes(seed.contents), old_seed) + + def test_private_xml_rollback_failure_is_uncertain(self): + from virsh_panel.vpc import VpcMember + + (service, created, domain, seed, resources, + private_mac, _private_ip_cidr) = ( + self.cloud_vm_with_private_artifacts(include_private_nic=True)) + member = VpcMember( + "member-delete-test", created["uuid"], "host-a", + "10.20.0.10", "subnet-a", private_mac, + ) + old_xml = domain.XMLDesc(1) + old_seed = bytes(seed.contents) + original_define = FakeConnection.defineXML + defines = 0 + + def fail_rollback_define(connection, xml): + nonlocal defines + defines += 1 + if defines == 1: + return original_define(connection, xml) + raise FakeLibvirtError("rollback define failed") + + readback_error = self.error_type( + "forced readback mismatch", 502, + error_code="vpc_private_xml_readback_mismatch", + ) + with patch.object( + FakeConnection, "defineXML", + new=fail_rollback_define), patch.object( + service, "read_private_domain_pair", + side_effect=readback_error): + with self.assertRaises(self.error_type) as uncertain: + service.ensure_private_interface( + created["uuid"], member=member, resources=resources, + vpc_id="vpc-delete-test", desired_revision=2, + desired_digest="b" * 64, + private_routes=["10.20.0.0/23"], + ) + + self.assertEqual( + uncertain.exception.error_code, + "vpc_private_result_uncertain", + ) + self.assertEqual(defines, 2) + self.assertNotEqual(domain.XMLDesc(1), old_xml) + self.assertNotEqual(bytes(seed.contents), old_seed) + + def test_private_xml_rollback_none_is_uncertain(self): + from virsh_panel.vpc import VpcMember + + (service, created, domain, seed, resources, + private_mac, _private_ip_cidr) = ( + self.cloud_vm_with_private_artifacts(include_private_nic=True)) + member = VpcMember( + "member-delete-test", created["uuid"], "host-a", + "10.20.0.10", "subnet-a", private_mac, + ) + old_xml = domain.XMLDesc(1) + old_seed = bytes(seed.contents) + original_define = FakeConnection.defineXML + original_upload = service._upload_volume + defines = 0 + uploads = 0 + + def none_rollback_define(connection, xml): + nonlocal defines + defines += 1 + if defines == 1: + return original_define(connection, xml) + return None + + def track_upload(connection, volume, chunks, size): + nonlocal uploads + uploads += 1 + return original_upload(connection, volume, chunks, size) + + readback_error = self.error_type( + "forced readback mismatch", 502, + error_code="vpc_private_xml_readback_mismatch", + ) + with patch.object( + FakeConnection, "defineXML", + new=none_rollback_define), patch.object( + service, "_upload_volume", + side_effect=track_upload), patch.object( + service, "read_private_domain_pair", + side_effect=readback_error): + with self.assertRaises(self.error_type) as uncertain: + service.ensure_private_interface( + created["uuid"], member=member, resources=resources, + vpc_id="vpc-delete-test", desired_revision=2, + desired_digest="b" * 64, + private_routes=["10.20.0.0/23"], + ) + + self.assertEqual( + uncertain.exception.error_code, + "vpc_private_result_uncertain", + ) + self.assertEqual(defines, 2) + # A failed XML inverse must stop the rollback chain before the seed + # is restored, otherwise the stopped definition and next-boot media + # would describe different network generations. + self.assertEqual(uploads, 1) + self.assertNotEqual(domain.XMLDesc(1), old_xml) + self.assertNotEqual(bytes(seed.contents), old_seed) + + def test_stopped_private_apply_start_after_seed_fails_uncertain(self): + from virsh_panel.vpc import VpcMember + + (service, created, domain, seed, resources, + private_mac, _private_ip_cidr) = self.cloud_vm_with_private_artifacts( + include_private_nic=True) + member = VpcMember( + "member-delete-test", created["uuid"], "host-a", + "10.20.0.10", "subnet-a", private_mac, + ) + original_upload = service._upload_volume + uploads = 0 + + def upload_then_start(connection, volume, chunks, size): + nonlocal uploads + original_upload(connection, volume, chunks, size) + uploads += 1 + if uploads == 1: + domain.live_root = copy.deepcopy(domain.inactive_root) + domain.running = True + + define_count = len(self.backend.define_xmls) + with patch.object(service, "_upload_volume", + side_effect=upload_then_start): + with self.assertRaises(self.error_type) as uncertain: + service.ensure_private_interface( + created["uuid"], member=member, resources=resources, + vpc_id="vpc-delete-test", desired_revision=2, + desired_digest="b" * 64, + private_routes=["10.20.0.0/23"], + ) + + self.assertEqual( + uncertain.exception.error_code, "vpc_private_result_uncertain") + self.assertTrue(domain.running) + self.assertEqual(len(self.backend.define_xmls), define_count) + # The seed may have committed before the start race, but rollback must + # not rewrite media now attached to a running guest. + self.assertNotEqual(bytes(seed.contents), b"") + + def test_stopped_private_apply_upload_error_after_start_skips_rollback(self): + from virsh_panel.vpc import VpcMember + + (service, created, domain, _seed, resources, + private_mac, _private_ip_cidr) = self.cloud_vm_with_private_artifacts( + include_private_nic=True) + member = VpcMember( + "member-delete-test", created["uuid"], "host-a", + "10.20.0.10", "subnet-a", private_mac, + ) + original_upload = service._upload_volume + uploads = 0 + + def upload_then_start_and_fail(connection, volume, chunks, size): + nonlocal uploads + uploads += 1 + original_upload(connection, volume, chunks, size) + domain.live_root = copy.deepcopy(domain.inactive_root) + domain.running = True + raise FakeLibvirtError("seed upload response lost after start") + + define_count = len(self.backend.define_xmls) + with patch.object(service, "_upload_volume", + side_effect=upload_then_start_and_fail): + with self.assertRaises(self.error_type) as uncertain: + service.ensure_private_interface( + created["uuid"], member=member, resources=resources, + vpc_id="vpc-delete-test", desired_revision=2, + desired_digest="b" * 64, + private_routes=["10.20.0.0/23"], + ) + + self.assertEqual( + uncertain.exception.error_code, "vpc_private_result_uncertain") + self.assertTrue(domain.running) + self.assertEqual(uploads, 1) + self.assertEqual(len(self.backend.define_xmls), define_count) + + def test_stopped_private_apply_rollback_probe_failure_is_uncertain(self): + from virsh_panel.vpc import VpcMember + + (service, created, domain, _seed, resources, + private_mac, _private_ip_cidr) = self.cloud_vm_with_private_artifacts( + include_private_nic=True) + member = VpcMember( + "member-delete-test", created["uuid"], "host-a", + "10.20.0.10", "subnet-a", private_mac, + ) + original_is_active = domain.isActive + original_upload = service._upload_volume + fail_probe = False + uploads = 0 + + def is_active_or_fail(): + if fail_probe: + raise FakeLibvirtError("domain state unavailable") + return original_is_active() + + def upload_then_lose_state(connection, volume, chunks, size): + nonlocal fail_probe, uploads + uploads += 1 + original_upload(connection, volume, chunks, size) + fail_probe = True + raise FakeLibvirtError("seed upload response lost") + + define_count = len(self.backend.define_xmls) + with patch.object(domain, "isActive", side_effect=is_active_or_fail), \ + patch.object(service, "_upload_volume", + side_effect=upload_then_lose_state): + with self.assertRaises(self.error_type) as uncertain: + service.ensure_private_interface( + created["uuid"], member=member, resources=resources, + vpc_id="vpc-delete-test", desired_revision=2, + desired_digest="b" * 64, + private_routes=["10.20.0.0/23"], + ) + + self.assertEqual( + uncertain.exception.error_code, "vpc_private_result_uncertain") + self.assertEqual(uploads, 1) + self.assertEqual(len(self.backend.define_xmls), define_count) + + def test_stopped_private_apply_start_after_define_fails_uncertain(self): + from virsh_panel.vpc import VpcMember + + (service, created, domain, _seed, resources, + private_mac, _private_ip_cidr) = self.cloud_vm_with_private_artifacts( + include_private_nic=True) + member = VpcMember( + "member-delete-test", created["uuid"], "host-a", + "10.20.0.10", "subnet-a", private_mac, + ) + original_define = FakeConnection.defineXML + started = False + + def define_then_start(connection, xml): + nonlocal started + result = original_define(connection, xml) + if not started: + started = True + domain.live_root = copy.deepcopy(domain.inactive_root) + domain.running = True + return result + + with patch.object(FakeConnection, "defineXML", + new=define_then_start): + with self.assertRaises(self.error_type) as uncertain: + service.ensure_private_interface( + created["uuid"], member=member, resources=resources, + vpc_id="vpc-delete-test", desired_revision=2, + desired_digest="b" * 64, + private_routes=["10.20.0.0/24"], + ) + + self.assertEqual( + uncertain.exception.error_code, "vpc_private_result_uncertain") + self.assertTrue(domain.running) + self.assertTrue(started) + + def test_stopped_private_apply_start_after_readback_fails_uncertain(self): + from virsh_panel.vpc import VpcMember + + (service, created, domain, _seed, resources, + private_mac, _private_ip_cidr) = self.cloud_vm_with_private_artifacts( + include_private_nic=True) + member = VpcMember( + "member-delete-test", created["uuid"], "host-a", + "10.20.0.10", "subnet-a", private_mac, + ) + original_read = service.read_private_domain_pair + + def read_then_start(*args, **kwargs): + result = original_read(*args, **kwargs) + domain.live_root = copy.deepcopy(domain.inactive_root) + domain.running = True + return result + + with patch.object(service, "read_private_domain_pair", + side_effect=read_then_start): + with self.assertRaises(self.error_type) as uncertain: + service.ensure_private_interface( + created["uuid"], member=member, resources=resources, + vpc_id="vpc-delete-test", desired_revision=2, + desired_digest="b" * 64, + private_routes=["10.20.0.0/24"], + ) + + self.assertEqual( + uncertain.exception.error_code, "vpc_private_result_uncertain") + self.assertTrue(domain.running) + def test_remove_private_interface_repairs_absent_nic_stale_metadata_and_seed(self): from virsh_panel.cloud_init import read_seed_network @@ -2771,6 +4296,83 @@ class VMServiceTests(unittest.TestCase): self.assertNotIn("private_mac", observed_seed) self.assertNotIn("private_routes", observed_seed) + def test_remove_private_interface_start_during_seed_write_is_uncertain(self): + (service, created, domain, seed, resources, + private_mac, private_ip_cidr) = self.cloud_vm_with_private_artifacts( + include_private_nic=True) + original_upload = service._upload_volume + uploads = 0 + + def upload_then_start(connection, volume, chunks, size): + nonlocal uploads + original_upload(connection, volume, chunks, size) + uploads += 1 + if uploads == 1: + domain.live_root = copy.deepcopy(domain.inactive_root) + domain.running = True + + with patch.object(service, "_upload_volume", + side_effect=upload_then_start): + with self.assertRaises(self.error_type) as uncertain: + service.remove_private_interface( + created["uuid"], vpc_id="vpc-delete-test", + resources=resources, desired_revision=2, + desired_digest="b" * 64, + expected_member_id="member-delete-test", + expected_private_mac=private_mac, + expected_private_ip_cidr=private_ip_cidr, + ) + + self.assertEqual( + uncertain.exception.error_code, + "vpc_private_remove_uncertain", + ) + self.assertTrue(domain.running) + self.assertEqual(uploads, 1) + # Do not report removal or roll back persistent/seed state while the + # guest may already be consuming the crossed configuration. + self.assertIsNone(self.service._private_interface_from_root( + domain.inactive_root)) + self.assertGreater(len(seed.contents), 0) + + def test_remove_upload_error_after_start_skips_rollback_mutations(self): + (service, created, domain, _seed, resources, + private_mac, private_ip_cidr) = self.cloud_vm_with_private_artifacts( + include_private_nic=True) + original_upload = service._upload_volume + uploads = 0 + + def upload_then_start_and_fail(connection, volume, chunks, size): + nonlocal uploads + uploads += 1 + original_upload(connection, volume, chunks, size) + domain.live_root = copy.deepcopy(domain.inactive_root) + domain.running = True + raise FakeLibvirtError("seed upload response lost after start") + + define_count = len(self.backend.define_xmls) + with patch.object(service, "_upload_volume", + side_effect=upload_then_start_and_fail): + with self.assertRaises(self.error_type) as uncertain: + service.remove_private_interface( + created["uuid"], vpc_id="vpc-delete-test", + resources=resources, desired_revision=2, + desired_digest="b" * 64, + expected_member_id="member-delete-test", + expected_private_mac=private_mac, + expected_private_ip_cidr=private_ip_cidr, + ) + + self.assertEqual( + uncertain.exception.error_code, + "vpc_private_remove_uncertain", + ) + self.assertTrue(domain.running) + self.assertEqual(uploads, 1) + # The one forward define committed before the upload. A rollback must + # not issue a second define after the guest has started. + self.assertEqual(len(self.backend.define_xmls), define_count + 1) + def test_remove_private_interface_retries_define_committed_seed_stale(self): from virsh_panel.cloud_init import read_seed_network @@ -2842,6 +4444,55 @@ class VMServiceTests(unittest.TestCase): private_ip_cidr, ) + def test_remove_legacy_seed_policy_is_stable_conflict_without_mutation(self): + from virsh_panel.cloud_init import _build_seed_iso, _read_seed_files + + (service, created, domain, seed, resources, + private_mac, private_ip_cidr) = self.cloud_vm_with_private_artifacts( + include_private_nic=True) + files = _read_seed_files(bytes(seed.contents)) + prefix = b"#cloud-config\n" + user_data = json.loads(files["user-data"][len(prefix):]) + user_data.pop("updates", None) + metadata = json.loads(files["meta-data"]) + metadata.pop("virt-cloud-network-policy", None) + legacy_seed = _build_seed_iso(( + ("/USERDATA.;1", "user-data", "/user-data", + prefix + json.dumps(user_data, ensure_ascii=True).encode("ascii") + + b"\n"), + ("/METADATA.;1", "meta-data", "/meta-data", + json.dumps(metadata, ensure_ascii=True).encode("ascii") + b"\n"), + ("/NETWORK.;1", "network-config", "/network-config", + files["network-config"]), + )) + seed.contents = bytearray(legacy_seed) + seed.capacity = len(legacy_seed) + old_xml = domain.XMLDesc(1) + define_count = len(self.backend.define_xmls) + + with patch.object( + service, "_upload_volume", + wraps=service._upload_volume) as upload: + with self.assertRaises(self.error_type) as migration: + service.remove_private_interface( + created["uuid"], vpc_id="vpc-delete-test", + resources=resources, desired_revision=2, + desired_digest="b" * 64, + expected_member_id="member-delete-test", + expected_private_mac=private_mac, + expected_private_ip_cidr=private_ip_cidr, + ) + + self.assertEqual(migration.exception.status, 409) + self.assertEqual( + migration.exception.error_code, + "vpc_seed_network_policy_migration_required", + ) + self.assertEqual(domain.XMLDesc(1), old_xml) + self.assertEqual(bytes(seed.contents), legacy_seed) + self.assertEqual(len(self.backend.define_xmls), define_count) + upload.assert_not_called() + def test_remove_private_interface_rejects_nic_owner_or_mac_mismatch(self): (service, created, domain, seed, resources, private_mac, private_ip_cidr) = self.cloud_vm_with_private_artifacts( @@ -2996,6 +4647,95 @@ class VMServiceTests(unittest.TestCase): self.assertEqual(domain.XMLDesc(1), old_xml) self.assertEqual(bytes(seed.contents), old_seed) + def test_remove_private_interface_rollback_define_none_is_uncertain(self): + (service, created, domain, seed, resources, + private_mac, private_ip_cidr) = self.cloud_vm_with_private_artifacts( + include_private_nic=True) + old_xml = domain.XMLDesc(1) + old_seed = bytes(seed.contents) + original_define = FakeConnection.defineXML + defines = 0 + uploads = 0 + + def none_rollback_define(connection, xml): + nonlocal defines + defines += 1 + if defines == 1: + return original_define(connection, xml) + return None + + def fail_forward_upload(_connection, volume, _chunks, _size): + nonlocal uploads + uploads += 1 + volume.contents = bytearray(b"partial-seed-write") + raise FakeLibvirtError("seed upload failed") + + with patch.object( + FakeConnection, "defineXML", + new=none_rollback_define), patch.object( + service, "_upload_volume", + side_effect=fail_forward_upload): + with self.assertRaises(self.error_type) as uncertain: + service.remove_private_interface( + created["uuid"], vpc_id="vpc-delete-test", + resources=resources, desired_revision=2, + desired_digest="b" * 64, + expected_member_id="member-delete-test", + expected_private_mac=private_mac, + expected_private_ip_cidr=private_ip_cidr, + ) + + self.assertEqual( + uncertain.exception.error_code, + "vpc_private_remove_uncertain", + ) + self.assertEqual(defines, 2) + self.assertEqual(uploads, 1) + self.assertNotEqual(domain.XMLDesc(1), old_xml) + self.assertNotEqual(bytes(seed.contents), old_seed) + + def test_remove_private_interface_rollback_seed_failure_is_uncertain(self): + (service, created, domain, seed, resources, + private_mac, private_ip_cidr) = self.cloud_vm_with_private_artifacts( + include_private_nic=True) + old_xml = domain.XMLDesc(1) + old_seed = bytes(seed.contents) + original_upload = service._upload_volume + uploads = 0 + + def fail_forward_and_restore_upload( + connection, volume, chunks, size, + ): + nonlocal uploads + uploads += 1 + if uploads == 1: + volume.contents = bytearray(b"partial-seed-write") + raise FakeLibvirtError("seed upload failed") + if uploads == 2: + raise FakeLibvirtError("rollback seed upload failed") + return original_upload(connection, volume, chunks, size) + + with patch.object( + service, "_upload_volume", + side_effect=fail_forward_and_restore_upload): + with self.assertRaises(self.error_type) as uncertain: + service.remove_private_interface( + created["uuid"], vpc_id="vpc-delete-test", + resources=resources, desired_revision=2, + desired_digest="b" * 64, + expected_member_id="member-delete-test", + expected_private_mac=private_mac, + expected_private_ip_cidr=private_ip_cidr, + ) + + self.assertEqual( + uncertain.exception.error_code, + "vpc_private_remove_uncertain", + ) + self.assertEqual(uploads, 2) + self.assertEqual(domain.XMLDesc(1), old_xml) + self.assertNotEqual(bytes(seed.contents), old_seed) + def test_create_list_resources_and_console(self): resources = self.service.resources() self.assertEqual(resources["networks"], ["default"]) diff --git a/engine/tests/test_vpc.py b/engine/tests/test_vpc.py index 3a310022..b6561fb2 100644 --- a/engine/tests/test_vpc.py +++ b/engine/tests/test_vpc.py @@ -2,8 +2,10 @@ from __future__ import annotations import asyncio +from concurrent.futures import ThreadPoolExecutor import json import threading +import time from contextlib import asynccontextmanager from pathlib import Path from types import SimpleNamespace @@ -13,6 +15,7 @@ import pytest from virsh_panel.fabric import FabricError, FabricNode, FabricPlan from virsh_panel.vpc import ( VPC_DELETE_RECEIPT_KIND, + VPC_GUEST_EXEC_KIND, VPC_SCHEMA_VERSION, VpcCoordinator, VpcError, @@ -304,6 +307,77 @@ def test_coordinator_persists_desired_children_and_blocks_without_fabric(): assert set(store.list("vpc_host")) == {"vpc-hk-1.host-a"} +def test_guest_exec_ledger_is_immediate_bounded_private_and_restart_safe(): + store = MemoryStore() + plan = parse_vpc_plan(payload()) + member = plan.members[0] + coordinator = VpcCoordinator(store, FakeFabric()) + state, persist, clear = coordinator._guest_exec_context(plan, member) + assert state == {} + core = { + "version": 1, + "phase": "pending", + "domain_uuid": member.instance_uuid, + "domain_runtime_id": 7, + "qemu_runtime_token": "a" * 32, + "guest_network_generation": "generation-1", + "probe_kind": "link", + "pid": 41, + } + + persist(core) + + key = f"{plan.vpc_id}.{member.instance_uuid}" + encoded = store.get(VPC_GUEST_EXEC_KIND, key) + assert encoded is not None + assert len(encoded.encode("utf-8")) <= 1024 + assert json.loads(encoded)["pid"] == 41 + restarted = VpcCoordinator(store, FakeFabric()) + loaded, _persist_again, clear_again = restarted._guest_exec_context( + plan, member) + assert loaded == core + + asyncio.run(coordinator.apply(payload())) + public = json.dumps(coordinator.get_status(plan.vpc_id), sort_keys=True) + assert '"pid"' not in public + assert "qemu_runtime_token" not in public + assert "a" * 32 not in public + + clear_again() + assert store.get(VPC_GUEST_EXEC_KIND, key) is None + persist(core) + coordinator._purge_deleted_vpc(plan.vpc_id) + assert store.get(VPC_GUEST_EXEC_KIND, key) is None + + +def test_guest_exec_ledger_rejects_oversize_or_unknown_fields(): + store = MemoryStore() + plan = parse_vpc_plan(payload()) + member = plan.members[0] + key = f"{plan.vpc_id}.{member.instance_uuid}" + store.set(VPC_GUEST_EXEC_KIND, key, "x" * 1025) + with pytest.raises(VpcError, match="账本损坏"): + VpcCoordinator(store, FakeFabric())._guest_exec_context(plan, member) + + bad = { + "vpc_id": plan.vpc_id, + "host_id": member.host_id, + "member_id": member.member_id, + "instance_uuid": member.instance_uuid, + "version": 1, + "phase": "launch_intent", + "domain_uuid": member.instance_uuid, + "domain_runtime_id": 7, + "qemu_runtime_token": "a" * 32, + "guest_network_generation": "generation-1", + "probe_kind": "link", + "unexpected": True, + } + store.set(VPC_GUEST_EXEC_KIND, key, json.dumps(bad)) + with pytest.raises(VpcError, match="账本损坏"): + VpcCoordinator(store, FakeFabric())._guest_exec_context(plan, member) + + def test_observed_projection_keeps_fabric_gate_for_data_plane_deferral(): store = MemoryStore() result = asyncio.run(VpcCoordinator(store, FakeFabric()).apply(payload())) @@ -942,11 +1016,32 @@ def test_instance_reconcile_returns_host_verified_private_state_and_preserves_li state="active", observed_revision=plan.desired_revision, observed_digest=plan.desired_digest, transport_verified=True, private_dataplane_verified=True, + config_verified=True, + config_observed_revision=plan.desired_revision, + config_observed_digest=plan.desired_digest, member_nics=[{ "member_id": "member-a", "instance_uuid": INSTANCE_A, "interface": "private0", "private_ip_cidr": "10.20.0.10/32", "private_mac": derive_private_mac("vpc-hk-1", "member-a"), "xml_verified": True, + "config_verified": True, + "seed_verified": True, + "restart_required": False, + "domain_running": True, + "guest_network_verified": requested != "down", + "guest_network_readback": ( + "confirmed" if requested != "down" else "mismatch"), + "guest_reason_code": ( + None if requested != "down" + else "vpc_private_guest_link_mismatch"), + "guest_private_mac": (derive_private_mac( + "vpc-hk-1", "member-a") + if requested != "down" else None), + "guest_private_ip_cidr": ( + "10.20.0.10/32" if requested != "down" else None), + "guest_link_state": ( + "up" if requested != "down" else None), + "guest_routes": [], "link_state": "down" if requested == "down" else "up", }], ) @@ -970,6 +1065,59 @@ def test_instance_reconcile_returns_host_verified_private_state_and_preserves_li assert state["link_state"] == "up" +def test_instance_reconcile_invalidates_stale_proof_when_fabric_is_inactive(): + store = MemoryStore() + coordinator = VpcCoordinator( + store, FakeFabric(hosts=("host-a",))) + asyncio.run(coordinator.apply(payload())) + + async def confirm(plan, record, gate, previous_plan=None, + private_link_states=None): + record["hosts"][0].update( + state="active", observed_revision=plan.desired_revision, + observed_digest=plan.desired_digest, transport_verified=True, + private_dataplane_verified=True, config_verified=True, + config_observed_revision=plan.desired_revision, + config_observed_digest=plan.desired_digest, + member_nics=[{ + "member_id": "member-a", "instance_uuid": INSTANCE_A, + "interface": "private0", + "private_ip_cidr": "10.20.0.10/32", + "private_mac": derive_private_mac( + "vpc-hk-1", "member-a"), + "xml_verified": True, "config_verified": True, + "seed_verified": True, "restart_required": False, + "domain_running": True, "guest_network_verified": True, + "guest_network_readback": "confirmed", + "guest_reason_code": None, + "guest_private_mac": derive_private_mac( + "vpc-hk-1", "member-a"), + "guest_private_ip_cidr": "10.20.0.10/32", + "guest_link_state": "up", + "guest_routes": [], "link_state": "up", + }], + ) + + coordinator._reconcile_dataplane = confirm + active = asyncio.run(coordinator.reconcile_instance( + INSTANCE_A, host_id="host-a", + expected={"vpc_id": "vpc-hk-1", "member_id": "member-a"}, + )) + assert active["readback"] == "confirmed" + + coordinator.fabric = FakeFabric( + state="degraded", gate=False, hosts=("host-a",)) + coordinator._reconcile_dataplane = ( + VpcCoordinator._reconcile_dataplane.__get__( + coordinator, VpcCoordinator)) + stale = asyncio.run(coordinator.reconcile_instance( + INSTANCE_A, host_id="host-a", + expected={"vpc_id": "vpc-hk-1", "member_id": "member-a"}, + )) + assert stale["state"] == "uncertain" + assert stale["readback"] == "missing" + + def test_removed_member_reconcile_uses_durable_retiring_tombstone(): store = MemoryStore() coordinator = VpcCoordinator(store, FakeFabric(hosts=("host-a",))) @@ -983,13 +1131,7 @@ def test_removed_member_reconcile_uses_durable_retiring_tombstone(): async def fake_reconcile(plan, record, gate, previous_plan=None, private_link_states=None): - host = record["hosts"][0] - host.update( - state="active", observed_revision=plan.desired_revision, - observed_digest=plan.desired_digest, transport_verified=True, - private_dataplane_verified=True, member_nics=[], - retiring_members=[], - ) + confirm_delete_hosts(plan, record) coordinator._reconcile_dataplane = fake_reconcile state = asyncio.run(coordinator.reconcile_instance( @@ -1036,10 +1178,23 @@ def test_instance_reconcile_rejects_host_mac_drift(): state="active", observed_revision=plan.desired_revision, observed_digest=plan.desired_digest, transport_verified=True, private_dataplane_verified=True, + config_verified=True, + config_observed_revision=plan.desired_revision, + config_observed_digest=plan.desired_digest, member_nics=[{ "member_id": "member-a", "instance_uuid": INSTANCE_A, "interface": "private0", "private_ip_cidr": "10.20.0.10/32", "private_mac": "02:00:00:00:00:01", "xml_verified": True, + "config_verified": True, "seed_verified": True, + "restart_required": False, + "domain_running": True, + "guest_network_verified": True, + "guest_network_readback": "confirmed", + "guest_reason_code": None, + "guest_private_mac": "02:00:00:00:00:01", + "guest_private_ip_cidr": "10.20.0.10/32", + "guest_link_state": "up", + "guest_routes": [], "link_state": "up", }], ) @@ -1086,13 +1241,7 @@ def test_removed_member_reconcile_locks_retiring_vm_identity(): async def fake_unlocked(plan, record, gate, previous_plan=None, private_link_states=None): - host = record["hosts"][0] - host.update( - state="active", observed_revision=plan.desired_revision, - observed_digest=plan.desired_digest, transport_verified=True, - private_dataplane_verified=True, member_nics=[], - retiring_members=[], - ) + confirm_delete_hosts(plan, record) coordinator._reconcile_dataplane_unlocked = fake_unlocked state = asyncio.run(coordinator.reconcile_instance( @@ -1221,6 +1370,7 @@ def test_root_delete_accepts_revision_jump_and_superseded_unknown_host(): assert result == { "vpc_id": "vpc-hk-1", "operation_id": "vpc-delete-revision-jump-0001", + "evidence_version": 2, "desired_revision": 4, "desired_digest": target["desired_digest"], "state": "deleted", @@ -1500,6 +1650,643 @@ def test_delete_pending_projection_does_not_probe_confirmed_offline_host(): record["hosts"][0], plan.desired_revision, plan.desired_digest) +def test_member_tombstone_closes_from_config_proof_when_sibling_not_ready(): + target = payload( + desired_revision=2, operation_id="vpc-op-config-proof-0002", + hosts=[{"host_id": "host-a", "members": [{ + "member_id": "member-b", "instance_uuid": INSTANCE_B, + "private_ip_cidr": "10.20.0.11/32", "subnet_id": "subnet-a", + }]}], + ) + plan = parse_vpc_plan(target) + fabric_plan = FabricPlan( + "fabric-hk", "hk", "100.64.0.0/24", 7, 51820, + (FabricNode( + "host-a", "100.64.0.1", "192.0.2.1:51820", 51820),), + "f" * 64, + ) + + class RuntimeFabric(FakeFabric): + @staticmethod + def _plan_from_record(_record): + return fabric_plan + + async def _components(self, _plan): + return { + "host-a": ( + SimpleNamespace(manager=object()), + {"host_id": "host-a"}, + ), + } + + coordinator = VpcCoordinator( + MemoryStore(), RuntimeFabric(hosts=("host-a",)), + registry=object(), + ) + coordinator._mark_lifecycle_conflicts = lambda *args, **kwargs: False + coordinator._validate_dataplane_resources = lambda _plan: None + coordinator._fabric_record = lambda _plan: { + "public_keys": {"host-a": "a" * 44}, + } + coordinator._fabric_peers_for_plan = lambda _plan, _record: [] + coordinator._run_host_dataplane = lambda *args, **kwargs: { + "state": "applying", "transport_verified": True, + "private_dataplane_verified": False, + "config_verified": True, + "config_observed_revision": plan.desired_revision, + "config_observed_digest": plan.desired_digest, + "observed_revision": 0, "observed_digest": None, + "member_nics": [{ + "member_id": "member-b", "instance_uuid": INSTANCE_B, + "config_verified": True, "xml_verified": True, + "domain_running": False, "guest_network_verified": False, + "restart_required": True, + }], + "reason_code": "vpc_private_guest_restart_required", + } + record = { + "vpc_id": plan.vpc_id, + "desired_revision": plan.desired_revision, + "desired_digest": plan.desired_digest, + "hosts": [{ + "host_id": "host-a", "member_ids": ["member-b"], + "retiring_members": [retiring_member_a()], + "desired_revision": plan.desired_revision, + "desired_digest": plan.desired_digest, + }], + } + + asyncio.run(coordinator._reconcile_dataplane_unlocked( + plan, record, {"active": True, "available": True}, + previous_plan=plan, + )) + + assert record["hosts"][0]["state"] == "applying" + assert record["hosts"][0]["retiring_members"] == [] + + +def test_reconcile_skips_exact_host_so_later_host_gets_next_budget(): + target = payload( + hosts=[ + {"host_id": "host-a", "members": [{ + "member_id": "member-a", "instance_uuid": INSTANCE_A, + "private_ip_cidr": "10.20.0.10/32", + "subnet_id": "subnet-a", + }]}, + {"host_id": "host-b", "members": [{ + "member_id": "member-b", "instance_uuid": INSTANCE_B, + "private_ip_cidr": "10.20.0.11/32", + "subnet_id": "subnet-a", + }]}, + ], + ) + plan = parse_vpc_plan(target) + gate = {"active": True, "available": True} + coordinator = VpcCoordinator( + MemoryStore(), FakeFabric(hosts=("host-a", "host-b")), + registry=object(), + ) + record = coordinator._desired_record( + plan, gate, state="applying") + calls = [] + + def exact(host): + member = next( + item for item in plan.members + if item.host_id == host["host_id"]) + private_mac = derive_private_mac(plan.vpc_id, member.member_id) + host.update({ + "state": "active", + "transport_verified": True, + "private_dataplane_verified": True, + "config_verified": True, + "observed_revision": plan.desired_revision, + "observed_digest": plan.desired_digest, + "config_observed_revision": plan.desired_revision, + "config_observed_digest": plan.desired_digest, + "retiring_members": [], + "member_nics": [{ + "member_id": member.member_id, + "instance_uuid": member.instance_uuid, + "private_ip_cidr": f"{member.private_ip}/32", + "private_mac": private_mac, + "config_verified": True, + "xml_verified": True, + "seed_verified": True, + "domain_running": True, + "guest_network_verified": True, + "restart_required": False, + }], + }) + + async def consume_one_host(_plan, execution, _gate, _previous, **kwargs): + host_ids = [host["host_id"] for host in execution["hosts"]] + calls.append(host_ids) + if len(calls) == 1: + exact(execution["hosts"][0]) + execution["hosts"][1].update({ + "state": "uncertain", + "transport_verified": False, + "private_dataplane_verified": False, + "last_error_code": + "vpc_private_guest_probe_deadline_exceeded", + }) + else: + assert host_ids == ["host-b"] + exact(execution["hosts"][0]) + + coordinator._reconcile_dataplane = consume_one_host + asyncio.run(coordinator._reconcile_pending_hosts( + plan, record, gate, plan)) + asyncio.run(coordinator._reconcile_pending_hosts( + plan, record, gate, plan)) + + assert calls == [["host-a", "host-b"], ["host-b"]] + assert all(host["state"] == "active" for host in record["hosts"]) + + +def test_reconcile_reprobes_oldest_active_host_and_revokes_stale_guest_proof(): + target = payload(hosts=[ + {"host_id": "host-a", "members": [{ + "member_id": "member-a", "instance_uuid": INSTANCE_A, + "private_ip_cidr": "10.20.0.10/32", + "subnet_id": "subnet-a", + }]}, + {"host_id": "host-b", "members": [{ + "member_id": "member-b", "instance_uuid": INSTANCE_B, + "private_ip_cidr": "10.20.0.11/32", + "subnet_id": "subnet-a", + }]}, + ]) + plan = parse_vpc_plan(target) + gate = {"active": True, "available": True} + coordinator = VpcCoordinator( + MemoryStore(), FakeFabric(hosts=("host-a", "host-b")), + registry=object(), + ) + record = coordinator._desired_record(plan, gate, state="applying") + attempted_at = {"host-a": 200, "host-b": 100} + for host in record["hosts"]: + member = next( + item for item in plan.members + if item.host_id == host["host_id"]) + private_mac = derive_private_mac(plan.vpc_id, member.member_id) + host.update({ + "state": "active", + "transport_verified": True, + "private_dataplane_verified": True, + "config_verified": True, + "observed_revision": plan.desired_revision, + "observed_digest": plan.desired_digest, + "config_observed_revision": plan.desired_revision, + "config_observed_digest": plan.desired_digest, + "retiring_members": [], + "probe_attempted_at_ns": attempted_at[host["host_id"]], + "member_nics": [{ + "member_id": member.member_id, + "instance_uuid": member.instance_uuid, + "private_ip_cidr": f"{member.private_ip}/32", + "private_mac": private_mac, + "config_verified": True, + "xml_verified": True, + "seed_verified": True, + "domain_running": True, + "guest_network_verified": True, + "restart_required": False, + }], + }) + + calls = [] + + async def lose_qga_proof(_plan, execution, _gate, _previous, **kwargs): + calls.append([host["host_id"] for host in execution["hosts"]]) + host = execution["hosts"][0] + host.update({ + "state": "applying", + "private_dataplane_verified": False, + "observed_revision": 0, + "observed_digest": None, + "last_error_code": "vpc_private_guest_agent_unavailable", + }) + host["member_nics"][0].update({ + "guest_network_verified": False, + "guest_network_readback": "unavailable", + "guest_reason_code": "vpc_private_guest_agent_unavailable", + }) + + coordinator._reconcile_dataplane = lose_qga_proof + asyncio.run(coordinator._reconcile_pending_hosts( + plan, record, gate, plan)) + + assert calls == [["host-b"]] + hosts = {host["host_id"]: host for host in record["hosts"]} + assert hosts["host-a"]["state"] == "active" + assert hosts["host-b"]["state"] == "applying" + assert hosts["host-b"]["private_dataplane_verified"] is False + assert hosts["host-b"]["member_nics"][0][ + "guest_network_verified"] is False + assert record["state"] == "applying" + assert record["next_stage_allowed"] is False + assert record["private_dataplane_verified"] is False + + +def test_reconcile_rotates_pending_host_after_it_consumes_budget(): + target = payload(hosts=[ + {"host_id": "host-a", "members": [{ + "member_id": "member-a", "instance_uuid": INSTANCE_A, + "private_ip_cidr": "10.20.0.10/32", + "subnet_id": "subnet-a", + }]}, + {"host_id": "host-b", "members": [{ + "member_id": "member-b", "instance_uuid": INSTANCE_B, + "private_ip_cidr": "10.20.0.11/32", + "subnet_id": "subnet-a", + }]}, + ]) + plan = parse_vpc_plan(target) + gate = {"active": True, "available": True} + coordinator = VpcCoordinator( + MemoryStore(), FakeFabric(hosts=("host-a", "host-b")), + registry=object(), + ) + record = coordinator._desired_record( + plan, gate, state="applying") + calls = [] + + async def consume_budget(_plan, execution, _gate, _previous, **kwargs): + calls.append([host["host_id"] for host in execution["hosts"]]) + consumer = execution["hosts"][0] + consumer["probe_attempted_at_ns"] = 100 + len(calls) + consumer.update({ + "state": "uncertain", + "transport_verified": False, + "private_dataplane_verified": False, + "last_error_code": + "vpc_private_guest_probe_deadline_exceeded", + }) + + coordinator._reconcile_dataplane = consume_budget + asyncio.run(coordinator._reconcile_pending_hosts( + plan, record, gate, plan)) + asyncio.run(coordinator._reconcile_pending_hosts( + plan, record, gate, plan)) + + assert calls == [["host-a", "host-b"], ["host-b", "host-a"]] + + +def test_instance_lifecycle_target_precedes_unrelated_pending_host(): + target = payload(hosts=[ + {"host_id": "host-a", "members": [{ + "member_id": "member-a", "instance_uuid": INSTANCE_A, + "private_ip_cidr": "10.20.0.10/32", + "subnet_id": "subnet-a", + }]}, + {"host_id": "host-b", "members": [{ + "member_id": "member-b", "instance_uuid": INSTANCE_B, + "private_ip_cidr": "10.20.0.11/32", + "subnet_id": "subnet-a", + }]}, + ]) + plan = parse_vpc_plan(target) + gate = {"active": True, "available": True} + coordinator = VpcCoordinator( + MemoryStore(), FakeFabric(hosts=("host-a", "host-b")), + registry=object(), + ) + record = coordinator._desired_record(plan, gate, state="applying") + record["hosts"][1]["probe_attempted_at_ns"] = 999 + calls = [] + + async def capture_order(_plan, execution, _gate, _previous, **kwargs): + calls.append([host["host_id"] for host in execution["hosts"]]) + + coordinator._reconcile_dataplane = capture_order + asyncio.run(coordinator._reconcile_pending_hosts( + plan, record, gate, plan, + private_link_states={INSTANCE_B: "down"}, + )) + + assert calls == [["host-b", "host-a"]] + + +def test_guest_probe_pool_is_rolling_bounded_and_deadline_fail_closed( + monkeypatch): + import sys + import types + + try: + import libvirt # noqa: F401 + except ModuleNotFoundError: + stub = types.ModuleType("libvirt") + stub.libvirtError = type("libvirtError", (Exception,), {}) + monkeypatch.setitem(sys.modules, "libvirt", stub) + from virsh_panel.libvirt_service import VMService + from virsh_panel.vpc_dataplane import VpcPrivateHostAdapter + + members = [{ + "member_id": f"member-{index}", + "instance_uuid": + f"00000000-0000-4000-8000-{index + 1:012x}", + "private_ip_cidr": f"10.20.0.{index + 10}/32", + "subnet_id": "subnet-a", + } for index in range(10)] + plan = parse_vpc_plan(payload(hosts=[{ + "host_id": "host-a", "members": members, + }])) + monkeypatch.setattr("virsh_panel.vpc.VPC_GUEST_PROBE_WORKERS", 2) + monkeypatch.setattr(VMService, "__init__", lambda self, *args, **kwargs: None) + + def configure(self, instance_uuid, **kwargs): + return { + "xml": f"inactive:{instance_uuid}", + "live": f"live:{instance_uuid}", + "restart_required": False, + } + + monkeypatch.setattr(VMService, "ensure_private_interface", configure) + lock = threading.Lock() + active = 0 + maximum = 0 + started = [] + + def probe(self, instance_uuid, **kwargs): + nonlocal active, maximum + assert isinstance(kwargs.get("guest_probe_deadline"), float) + with lock: + active += 1 + maximum = max(maximum, active) + started.append(instance_uuid) + time.sleep(0.1) + with lock: + active -= 1 + return { + "inactive": f"inactive:{instance_uuid}", + "live": f"live:{instance_uuid}", + "guest_network": { + "state": "confirmed", "verified": True, + "source": "qemu_guest_agent", + }, + } + + monkeypatch.setattr(VMService, "read_private_domain_pair", probe) + monkeypatch.setattr( + VpcPrivateHostAdapter, "__init__", + lambda self, manager, profile, **kwargs: None, + ) + monkeypatch.setattr( + VpcPrivateHostAdapter, "apply", lambda self, dataplane: {}) + captured = {} + + def observe(self, dataplane, *, xml_by_instance=None): + captured.update(xml_by_instance or {}) + return { + "state": "applying", + "config_verified": True, + "config_observed_revision": plan.desired_revision, + "config_observed_digest": plan.desired_digest, + "member_nics": [], + "reason_code": "vpc_private_guest_probe_deadline_exceeded", + } + + monkeypatch.setattr(VpcPrivateHostAdapter, "observe", observe) + coordinator = VpcCoordinator(MemoryStore(), FakeFabric()) + result = coordinator._run_host_dataplane( + plan, "host-a", SimpleNamespace(manager=object()), + {"libvirt_uri": "qemu:///system", "storage_pool": "default", + "iso_pool": "iso", "default_network": "default"}, + [], None, fabric_public_keys={}, + request_deadline=time.monotonic() + 1.05, + ) + + assert maximum == 2 + assert len(started) == 2 + assert active == 0 + assert len(captured) == 10 + deadline_rows = [ + pair["guest_network"] for pair in captured.values() + if pair["guest_network"].get("reason_code") + == "vpc_private_guest_probe_deadline_exceeded" + ] + assert len(deadline_rows) == 8 + assert result["config_verified"] is True + assert result["private_dataplane_verified"] is False + + +def test_guest_probe_default_pool_bounds_libvirt_reply_amplification(): + from virsh_panel.vpc import VPC_GUEST_PROBE_WORKERS + + # virDomainQemuAgentCommand materializes libvirt's bounded raw reply before + # Python can enforce its 192 KiB parse limit. Keep the default narrow even + # though individual tests may monkeypatch it lower for deterministic timing. + assert VPC_GUEST_PROBE_WORKERS == 4 + + +def test_guest_probe_limit_is_process_wide_across_vpc_requests(monkeypatch): + import sys + import types + + try: + import libvirt # noqa: F401 + except ModuleNotFoundError: + stub = types.ModuleType("libvirt") + stub.libvirtError = type("libvirtError", (Exception,), {}) + monkeypatch.setitem(sys.modules, "libvirt", stub) + from virsh_panel.libvirt_service import VMService + from virsh_panel.vpc_dataplane import VpcPrivateHostAdapter + + def make_plan(label, second_octet, first_uuid): + members = [{ + "member_id": f"member-{label}-{index}", + "instance_uuid": + f"00000000-0000-4000-8000-{first_uuid + index:012x}", + "private_ip_cidr": + f"10.{second_octet}.0.{index + 10}/32", + "subnet_id": f"subnet-{label}", + } for index in range(4)] + return parse_vpc_plan(payload( + vpc_id=f"vpc-hk-{label}", + cidr=f"10.{second_octet}.0.0/24", + operation_id=f"vpc-op-{label}-00000001", + subnets=[{ + "subnet_id": f"subnet-{label}", + "cidr": f"10.{second_octet}.0.0/25", + }], + hosts=[{"host_id": "host-a", "members": members}], + )) + + plans = (make_plan("a", 20, 1), make_plan("b", 21, 101)) + monkeypatch.setattr(VMService, "__init__", lambda self, *args, **kwargs: None) + monkeypatch.setattr( + VMService, "ensure_private_interface", + lambda self, instance_uuid, **kwargs: { + "xml": f"inactive:{instance_uuid}", + "live": f"live:{instance_uuid}", + "restart_required": False, + }, + ) + lock = threading.Lock() + release_full_wave = threading.Event() + active = 0 + maximum = 0 + started = 0 + + def probe(self, instance_uuid, **kwargs): + nonlocal active, maximum, started + with lock: + active += 1 + started += 1 + maximum = max(maximum, active) + if active >= 8: + release_full_wave.set() + # Without the process gate, both four-worker request pools enter + # together. With it, a bounded wave of four leaves before the next. + release_full_wave.wait(0.2) + with lock: + active -= 1 + return { + "inactive": f"inactive:{instance_uuid}", + "live": f"live:{instance_uuid}", + "guest_network": { + "state": "confirmed", "verified": True, + "source": "qemu_guest_agent", + }, + } + + monkeypatch.setattr(VMService, "read_private_domain_pair", probe) + monkeypatch.setattr( + VpcPrivateHostAdapter, "__init__", + lambda self, manager, profile, **kwargs: None, + ) + monkeypatch.setattr( + VpcPrivateHostAdapter, "apply", lambda self, dataplane: {}) + + def observe(self, dataplane, *, xml_by_instance=None): + return { + "state": "active", + "config_verified": True, + "config_observed_revision": dataplane.vpc.desired_revision, + "config_observed_digest": dataplane.vpc.desired_digest, + "member_nics": [], + } + + monkeypatch.setattr(VpcPrivateHostAdapter, "observe", observe) + + def run(plan): + return VpcCoordinator( + MemoryStore(), FakeFabric(), + )._run_host_dataplane( + plan, "host-a", SimpleNamespace(manager=object()), + {"libvirt_uri": "qemu:///system", "storage_pool": "default", + "iso_pool": "iso", "default_network": "default"}, + [], None, fabric_public_keys={}, + request_deadline=time.monotonic() + 3, + ) + + with ThreadPoolExecutor(max_workers=2) as requests: + results = [future.result() for future in ( + requests.submit(run, plans[0]), + requests.submit(run, plans[1]), + )] + + assert started == 8 + assert maximum == 4 + assert all(result["config_verified"] is True for result in results) + + +def test_request_probe_deadline_stops_next_host_and_keeps_config_proof( + monkeypatch): + target = payload(hosts=[ + {"host_id": "host-a", "members": [{ + "member_id": "member-a", "instance_uuid": INSTANCE_A, + "private_ip_cidr": "10.20.0.10/32", + "subnet_id": "subnet-a", + }]}, + {"host_id": "host-b", "members": [{ + "member_id": "member-b", "instance_uuid": INSTANCE_B, + "private_ip_cidr": "10.20.0.11/32", + "subnet_id": "subnet-a", + }]}, + ]) + plan = parse_vpc_plan(target) + fabric_plan = FabricPlan( + "fabric-hk", "hk", "100.64.0.0/24", 7, 51820, + ( + FabricNode( + "host-a", "100.64.0.1", "192.0.2.1:51820", 51820), + FabricNode( + "host-b", "100.64.0.2", "192.0.2.2:51820", 51820), + ), + "f" * 64, + ) + + class DeadlineFabric(FakeFabric): + @staticmethod + def _plan_from_record(_record): + return fabric_plan + + async def _components(self, _plan): + return { + host_id: ( + SimpleNamespace(manager=object()), + {"host_id": host_id}, + ) + for host_id in ("host-a", "host-b") + } + + coordinator = VpcCoordinator( + MemoryStore(), DeadlineFabric(hosts=("host-a", "host-b")), + registry=object(), + ) + coordinator._mark_lifecycle_conflicts = lambda *args, **kwargs: False + coordinator._validate_dataplane_resources = lambda _plan: None + coordinator._fabric_record = lambda _plan: { + "public_keys": {"host-a": "a" * 44, "host-b": "b" * 44}, + } + coordinator._fabric_peers_for_plan = lambda _plan, _record: [] + monkeypatch.setattr("virsh_panel.vpc.VPC_REQUEST_DEADLINE_SECONDS", 0.05) + calls = [] + + def slow_host(*args, **kwargs): + host_id = args[1] + calls.append(host_id) + time.sleep(0.08) + return { + "state": "active", + "transport_verified": True, + "private_dataplane_verified": True, + "config_verified": True, + "config_observed_revision": plan.desired_revision, + "config_observed_digest": plan.desired_digest, + "member_nics": [{ + "member_id": "member-a", + "guest_network_verified": True, + }], + "reason_code": None, + } + + coordinator._run_host_dataplane = slow_host + gate = {"active": True, "available": True} + record = coordinator._desired_record( + plan, gate, state="applying") + + asyncio.run(coordinator._reconcile_dataplane_unlocked( + plan, record, gate, previous_plan=plan)) + + assert calls == ["host-a"] + first, second = record["hosts"] + assert first["transport_verified"] is True + assert first["probe_attempted_at_ns"] > 0 + assert first["config_verified"] is True + assert first["config_observed_revision"] == plan.desired_revision + assert first["private_dataplane_verified"] is False + assert first["member_nics"][0]["guest_network_verified"] is False + assert first["last_error_code"] \ + == "vpc_private_guest_probe_deadline_exceeded" + assert second["transport_verified"] is False + assert second["last_error_code"] \ + == "vpc_private_guest_probe_deadline_exceeded" + + def test_root_delete_surfaces_only_allowlisted_deterministic_host_errors(): deterministic_codes = ( "vpc_private_remove_requires_stopped", @@ -1712,7 +2499,7 @@ def test_delete_manifest_removes_unknown_member_nic_before_host_resources( monkeypatch.setattr(VMService, "remove_private_interface", remove_nic) monkeypatch.setattr( VpcPrivateHostAdapter, "__init__", - lambda self, manager, profile: None, + lambda self, manager, profile, **kwargs: None, ) def remove_host(self, dataplane): diff --git a/engine/tests/test_vpc_dataplane.py b/engine/tests/test_vpc_dataplane.py index 32dbde96..fb153547 100644 --- a/engine/tests/test_vpc_dataplane.py +++ b/engine/tests/test_vpc_dataplane.py @@ -10,16 +10,25 @@ import os import re import subprocess import tempfile +import xml.etree.ElementTree as ET from dataclasses import replace from pathlib import Path +import pytest + from virsh_panel.shell_contracts import IP_RULE_CONVERGENCE_SH from virsh_panel.vpc import VPC_SCHEMA_VERSION from virsh_panel.vpc_dataplane import ( VpcDataplaneError, + VpcPrivateHostAdapter, + _allowed, + _member_nics, _markers, + _routes, parse_vpc_dataplane_plan, patch_private_interface_xml, + private_guest_network_generation, + private_guest_routes, render_apply_script, render_observe_script, render_remove_script, @@ -188,6 +197,20 @@ def _plan(): }, "host-a") +def _guest_network_evidence(member, routes): + return { + "state": "confirmed", + "verified": True, + "source": "qemu_guest_agent", + "private_mac": member.private_mac, + "private_ip_cidr": f"{member.private_ip}/32", + "link_state": "up", + "routes": sorted(routes), + "reason_code": None, + "observed_at": 1_700_000_000, + } + + def test_rendered_data_plane_has_exact_routes_proxy_arp_and_atomic_nft_batch(): plan = _plan() apply = render_apply_script(plan) @@ -212,6 +235,11 @@ def test_rendered_data_plane_has_exact_routes_proxy_arp_and_atomic_nft_batch(): assert "anti-spoof-bridge-to-foreign" in nft assert "anti-spoof-fabric-to-foreign" in nft assert 'oifname "wg-fabric0" ip daddr @vpc_dest_ips drop' in nft + assert ( + "arp operation request arp daddr ip != @local_dest_ips drop" + in nft + ) + assert "bridge-unknown-arp-target-drop" in nft def test_rendered_scripts_are_posix_shell(): @@ -613,6 +641,8 @@ def test_private_xml_patch_migrates_pre_fix_aliases_and_leaves_private_filter_fr """ patched = patch_private_interface_xml( xml, member=member, resources=plan.resources, + guest_network_generation=private_guest_network_generation( + member, private_guest_routes(plan.vpc, member)), vpc_id=plan.vpc.vpc_id, desired_revision=plan.desired_revision, desired_digest=plan.desired_digest, ) @@ -636,6 +666,8 @@ def test_private_xml_patch_promotes_legacy_public_aliases_idempotently(): """ patched = patch_private_interface_xml( legacy, member=member, resources=plan.resources, + guest_network_generation=private_guest_network_generation( + member, private_guest_routes(plan.vpc, member)), vpc_id=plan.vpc.vpc_id, desired_revision=plan.desired_revision, desired_digest=plan.desired_digest, ) @@ -644,6 +676,8 @@ def test_private_xml_patch_promotes_legacy_public_aliases_idempotently(): # A second reconciliation must still identify the original NIC as public. reparsed = patch_private_interface_xml( patched, member=member, resources=plan.resources, + guest_network_generation=private_guest_network_generation( + member, private_guest_routes(plan.vpc, member)), vpc_id=plan.vpc.vpc_id, desired_revision=plan.desired_revision, desired_digest=plan.desired_digest, ) @@ -665,6 +699,9 @@ def test_private_xml_patch_rejects_domain_wide_alias_collision(): try: patch_private_interface_xml( xml, member=plan.members[0], resources=plan.resources, + guest_network_generation=private_guest_network_generation( + plan.members[0], + private_guest_routes(plan.vpc, plan.members[0])), vpc_id=plan.vpc.vpc_id, desired_revision=plan.desired_revision, desired_digest=plan.desired_digest, ) @@ -673,3 +710,382 @@ def test_private_xml_patch_rejects_domain_wide_alias_collision(): assert exc.error_code == "vpc_private_xml_role_invalid" else: raise AssertionError("domain-wide duplicate libvirt alias was accepted") + + +def test_guest_network_generation_is_stable_across_member_and_route_churn(): + plan = _plan() + member = plan.local_members[0] + routes = private_guest_routes(plan.vpc, member) + assert routes == ("10.42.0.0/24",) + expected = private_guest_network_generation(member, routes) + assert private_guest_routes( + replace(plan.vpc, routes=()), member) == routes + assert private_guest_routes( + replace(plan.vpc, members=(member,)), member) == routes + wider = private_guest_routes( + replace(plan.vpc, cidr="10.42.0.0/16"), member) + assert private_guest_network_generation(member, wider) != expected + for invalid in ((), ("10.42.0.11/32",), + ("10.42.0.0/24", "10.42.0.11/32")): + with pytest.raises(VpcDataplaneError): + private_guest_network_generation(member, invalid) + + +def test_running_member_requires_booted_guest_network_generation(): + plan = _plan() + member = plan.local_members[0] + routes = private_guest_routes(plan.vpc, member) + generation = private_guest_network_generation(member, routes) + original = """ + + + """ + persistent = patch_private_interface_xml( + original, member=member, resources=plan.resources, + guest_network_generation=generation, + vpc_id=plan.vpc.vpc_id, desired_revision=plan.desired_revision, + desired_digest=plan.desired_digest, + ) + stale_live_root = ET.fromstring(persistent) + stale_managed = stale_live_root.find( + "./metadata/{urn:virsh-panel:1}managed") + stale_managed.attrib.pop("private-guest-network-generation") + stale_live = ET.tostring(stale_live_root, encoding="unicode") + + pending = _member_nics(plan, { + member.instance_uuid: { + "inactive": persistent, + "live": stale_live, + }, + })[0] + # Device identity is present in both definitions, but the stale live + # generation means this running guest still needs a full restart before + # a missing guest-agent proof can be treated as potentially curable. + # Legacy v1 observations did not carry physical seed proof. Even with + # matching XML device identity they must remain non-usable. + assert pending["xml_verified"] is False + assert pending["config_verified"] is True + assert pending["guest_network_verified"] is False + assert pending["restart_required"] is True + + booted = _member_nics(plan, { + member.instance_uuid: { + "inactive": persistent, + "live": persistent, + "seed_verified": True, + "guest_network": _guest_network_evidence(member, routes), + }, + })[0] + assert booted["xml_verified"] is True + assert booted["config_verified"] is True + assert booted["domain_running"] is True + assert booted["guest_network_verified"] is True + assert booted["guest_private_ip_cidr"] == f"{member.private_ip}/32" + assert booted["guest_routes"] == sorted(routes) + assert booted["restart_required"] is False + + +def test_booted_generation_fails_closed_when_guest_evidence_is_unavailable(): + plan = _plan() + member = plan.local_members[0] + routes = private_guest_routes(plan.vpc, member) + persistent = patch_private_interface_xml( + """ + + + """, + member=member, resources=plan.resources, + guest_network_generation=private_guest_network_generation( + member, routes), + vpc_id=plan.vpc.vpc_id, desired_revision=plan.desired_revision, + desired_digest=plan.desired_digest, + ) + + observed = _member_nics(plan, { + member.instance_uuid: { + "inactive": persistent, + "live": persistent, + "seed_verified": True, + "guest_network": { + "state": "unavailable", "verified": False, + "reason_code": "vpc_private_guest_agent_unavailable", + }, + }, + })[0] + + assert observed["config_verified"] is True + assert observed["xml_verified"] is True + assert observed["domain_running"] is True + assert observed["guest_network_verified"] is False + assert observed["guest_network_readback"] == "unavailable" + assert observed["guest_reason_code"] == ( + "vpc_private_guest_agent_unavailable") + # The generation is current, so a restart would not cure an unavailable + # agent. The host remains non-active for the guest-specific reason. + assert observed["restart_required"] is False + + +def test_booted_generation_rejects_guest_address_or_route_mismatch(): + plan = _plan() + member = plan.local_members[0] + routes = private_guest_routes(plan.vpc, member) + persistent = patch_private_interface_xml( + """ + + + """, + member=member, resources=plan.resources, + guest_network_generation=private_guest_network_generation( + member, routes), + vpc_id=plan.vpc.vpc_id, desired_revision=plan.desired_revision, + desired_digest=plan.desired_digest, + ) + mismatch = _guest_network_evidence(member, routes) + mismatch.update({ + "state": "mismatch", "verified": False, + "private_ip_cidr": "10.42.0.99/32", + "routes": [], + "reason_code": "vpc_private_guest_address_mismatch", + }) + + observed = _member_nics(plan, { + member.instance_uuid: { + "inactive": persistent, + "live": persistent, + "seed_verified": True, + "guest_network": mismatch, + }, + })[0] + + assert observed["config_verified"] is True + assert observed["xml_verified"] is True + assert observed["guest_network_verified"] is False + assert observed["guest_private_ip_cidr"] == "10.42.0.99/32" + assert observed["guest_routes"] == [] + assert observed["guest_reason_code"] == ( + "vpc_private_guest_address_mismatch") + assert observed["restart_required"] is False + + +def test_stopped_member_preserves_config_proof_but_is_not_runtime_verified(): + plan = _plan() + member = plan.local_members[0] + routes = private_guest_routes(plan.vpc, member) + persistent = patch_private_interface_xml( + """ + + + """, + member=member, resources=plan.resources, + guest_network_generation=private_guest_network_generation( + member, routes), + vpc_id=plan.vpc.vpc_id, desired_revision=plan.desired_revision, + desired_digest=plan.desired_digest, + ) + + observed = _member_nics(plan, { + member.instance_uuid: { + "inactive": persistent, + "live": None, + "seed_verified": True, + "guest_network": { + "state": "stopped", "verified": False, + "reason_code": "vpc_private_guest_stopped", + }, + }, + })[0] + + assert observed["config_verified"] is True + assert observed["domain_running"] is False + assert observed["xml_verified"] is True + assert observed["guest_network_verified"] is False + assert observed["restart_required"] is False + assert observed["guest_reason_code"] == ( + "vpc_private_guest_stopped") + + +def _active_observation_markers(plan): + markers = [ + f"__VC_VPC_BRIDGE__={plan.resources.bridge}", + "__VC_VPC_BRIDGE_OWNER__=virt-cloud-vpc-dataplane-v1", + "__VC_VPC_PERSISTENCE__=yes", + "__VC_VPC_BRIDGE_BOUNDARY__=yes", + "__VC_VPC_PROXY_ARP__=0", + "__VC_VPC_PROXY_ARP_PVLAN__=1", + "__VC_VPC_PROXY_STATE__=owned", + "__VC_VPC_RP_FILTER__=2", + f"__VC_VPC_NFT_TABLE__={plan.resources.nft_table}", + "__VC_VPC_STATE_OWNER__=owned", + f"__VC_VPC_STATE_REVISION__={plan.desired_revision}", + f"__VC_VPC_STATE_DIGEST__={plan.desired_digest}", + f"__VC_VPC_ACL_REVISION__={plan.desired_revision}", + f"__VC_VPC_ACL_DIGEST__={plan.desired_digest}", + "__VC_VPC_ANTI_SPOOF__=yes", + "__VC_VPC_BRIDGE_NFT__=yes", + "__VC_VPC_ROUTE_EXTRA__=no", + "__VC_VPC_RULE_EXTRA__=no", + "__VC_VPC_ARP_NFT__=yes", + ] + for route in _routes(plan): + markers.append( + "__VC_VPC_ROUTE__=" + f"{route['destination_cidr']}|{route['device']}|" + f"{route['table']}|{route.get('via') or '-'}|" + f"{route.get('metric', 100)}|owned") + markers.append( + "__VC_VPC_PROXY_IP__=" + f"{route['destination_cidr'].removesuffix('/32')}") + markers.extend(( + "__VC_VPC_RULE__=" + f"{plan.resources.rule_priority}|-|{plan.vpc.cidr}|" + f"{plan.resources.route_table}|owned", + "__VC_VPC_RULE__=" + f"{plan.resources.rule_priority + 1}|wg-fabric0|{plan.vpc.cidr}|" + f"{plan.resources.route_table}|owned", + )) + for peer in plan.peers: + markers.extend(( + f"__VC_VPC_WG_ALLOWED__={peer.host_id}|{peer.public_key}|" + f"{_allowed(plan, peer)}", + f"__VC_VPC_WG_STATE__={peer.host_id}|owned", + )) + return markers + + +def test_observer_does_not_activate_unavailable_or_mismatched_guest(): + plan = _plan() + member = plan.local_members[0] + routes = private_guest_routes(plan.vpc, member) + persistent = patch_private_interface_xml( + """ + + + """, + member=member, resources=plan.resources, + guest_network_generation=private_guest_network_generation( + member, routes), + vpc_id=plan.vpc.vpc_id, desired_revision=plan.desired_revision, + desired_digest=plan.desired_digest, + ) + adapter = VpcPrivateHostAdapter(object(), {}) + adapter._deployment_contract = lambda: ("networkmanager", "", "", "") + adapter._run = lambda _script: "\n".join( + _active_observation_markers(plan)) + "\n" + + for state, reason in ( + ("unavailable", "vpc_private_guest_agent_unavailable"), + ("mismatch", "vpc_private_guest_address_mismatch"), + ): + observed = adapter.observe(plan, xml_by_instance={ + member.instance_uuid: { + "inactive": persistent, + "live": persistent, + "seed_verified": True, + "guest_network": { + "state": state, "verified": False, + "reason_code": reason, + }, + }, + }) + assert observed["config_verified"] is True + assert observed["state"] == "applying" + assert observed["observed_revision"] == 0 + assert observed["observed_digest"] is None + assert observed["readback"] == "pending_guest" + assert observed["reason_code"] == reason + assert observed["member_nics"][0]["xml_verified"] is True + assert observed["member_nics"][0]["guest_network_verified"] is False + + confirmed = adapter.observe(plan, xml_by_instance={ + member.instance_uuid: { + "inactive": persistent, + "live": persistent, + "seed_verified": True, + "guest_network": _guest_network_evidence(member, routes), + }, + }) + assert confirmed["state"] == "active" + assert confirmed["observed_revision"] == plan.desired_revision + assert confirmed["observed_digest"] == plan.desired_digest + assert confirmed["readback"] == "confirmed" + + +def test_observer_reports_pending_restart_instead_of_active(): + plan = _plan() + member = plan.local_members[0] + generation = private_guest_network_generation( + member, private_guest_routes(plan.vpc, member)) + original = """ + + + """ + persistent = patch_private_interface_xml( + original, member=member, resources=plan.resources, + guest_network_generation=generation, + vpc_id=plan.vpc.vpc_id, desired_revision=plan.desired_revision, + desired_digest=plan.desired_digest, + ) + stale_live_root = ET.fromstring(persistent) + stale_live_root.find( + "./metadata/{urn:virsh-panel:1}managed" + ).attrib.pop("private-guest-network-generation") + stale_live = ET.tostring(stale_live_root, encoding="unicode") + + markers = [ + f"__VC_VPC_BRIDGE__={plan.resources.bridge}", + "__VC_VPC_BRIDGE_OWNER__=virt-cloud-vpc-dataplane-v1", + "__VC_VPC_PERSISTENCE__=yes", + "__VC_VPC_BRIDGE_BOUNDARY__=yes", + "__VC_VPC_PROXY_ARP__=0", + "__VC_VPC_PROXY_ARP_PVLAN__=1", + "__VC_VPC_PROXY_STATE__=owned", + "__VC_VPC_RP_FILTER__=2", + f"__VC_VPC_NFT_TABLE__={plan.resources.nft_table}", + "__VC_VPC_STATE_OWNER__=owned", + f"__VC_VPC_STATE_REVISION__={plan.desired_revision}", + f"__VC_VPC_STATE_DIGEST__={plan.desired_digest}", + f"__VC_VPC_ACL_REVISION__={plan.desired_revision}", + f"__VC_VPC_ACL_DIGEST__={plan.desired_digest}", + "__VC_VPC_ANTI_SPOOF__=yes", + "__VC_VPC_BRIDGE_NFT__=yes", + "__VC_VPC_ROUTE_EXTRA__=no", + "__VC_VPC_RULE_EXTRA__=no", + "__VC_VPC_ARP_NFT__=yes", + ] + for route in _routes(plan): + markers.append( + "__VC_VPC_ROUTE__=" + f"{route['destination_cidr']}|{route['device']}|" + f"{route['table']}|{route.get('via') or '-'}|" + f"{route.get('metric', 100)}|owned") + markers.append( + "__VC_VPC_PROXY_IP__=" + f"{route['destination_cidr'].removesuffix('/32')}") + markers.extend(( + "__VC_VPC_RULE__=" + f"{plan.resources.rule_priority}|-|{plan.vpc.cidr}|" + f"{plan.resources.route_table}|owned", + "__VC_VPC_RULE__=" + f"{plan.resources.rule_priority + 1}|wg-fabric0|{plan.vpc.cidr}|" + f"{plan.resources.route_table}|owned", + )) + for peer in plan.peers: + markers.extend(( + f"__VC_VPC_WG_ALLOWED__={peer.host_id}|{peer.public_key}|" + f"{_allowed(plan, peer)}", + f"__VC_VPC_WG_STATE__={peer.host_id}|owned", + )) + + adapter = VpcPrivateHostAdapter(object(), {}) + adapter._deployment_contract = lambda: ("networkmanager", "", "", "") + adapter._run = lambda _script: "\n".join(markers) + "\n" + observed = adapter.observe(plan, xml_by_instance={ + member.instance_uuid: { + "inactive": persistent, + "live": stale_live, + }, + }) + assert observed["state"] == "applying" + assert observed["readback"] == "pending_restart" + assert observed["reason_code"] == "vpc_private_guest_restart_required" + assert observed["member_nics"][0]["restart_required"] is True diff --git a/engine/virsh_panel/cloud_init.py b/engine/virsh_panel/cloud_init.py index d340faf0..0b2c7049 100644 --- a/engine/virsh_panel/cloud_init.py +++ b/engine/virsh_panel/cloud_init.py @@ -53,6 +53,15 @@ ROOT_PASSWORD_ALPHABET = ( ROOT_PASSWORD_UPPERCASE + ROOT_PASSWORD_LOWERCASE + ROOT_PASSWORD_DIGITS + ROOT_PASSWORD_SYMBOLS ) +_RFC1918_NETWORKS = ( + ipaddress.IPv4Network("10.0.0.0/8"), + ipaddress.IPv4Network("172.16.0.0/12"), + ipaddress.IPv4Network("192.168.0.0/16"), +) + + +class SeedNetworkPolicyMigrationRequired(ValueError): + """A legacy seed cannot be made reboot-safe without replaying user-data.""" def validate_guest_hostname(value: Any) -> str: @@ -288,10 +297,9 @@ def validate_cloud_network( def validate_private_network(private_ip_cidr: Any) -> dict[str, str]: """Validate one L3 VPC guest address. - VPC members are routed as host/guest ``/32`` endpoints. They do not get - a default route or a provider gateway from this function; those routes are - owned by the host dataplane and are added only after the VPC ACL/readback - barrier succeeds. + Each guest address and the host route to that address use ``/32``. Guests + do not get a default route or provider gateway here; the separate seed + renderer adds exactly one on-link VPC CIDR route. """ if (not isinstance(private_ip_cidr, str) or not private_ip_cidr @@ -309,30 +317,33 @@ def validate_private_network(private_ip_cidr: Any) -> dict[str, str]: # ``is_private`` also includes loopback/link-local ranges, which are not # valid tenant addresses. Explicitly keep the product contract to RFC1918 # space so a host-local or metadata address cannot be routed as a member. - rfc1918 = ( - ipaddress.IPv4Network("10.0.0.0/8"), - ipaddress.IPv4Network("172.16.0.0/12"), - ipaddress.IPv4Network("192.168.0.0/16"), - ) - if (not any(address in network for network in rfc1918) + if (not any(address in network for network in _RFC1918_NETWORKS) or address.is_loopback or address.is_link_local or address.is_reserved or address.is_multicast): raise ValueError("私网 IP 必须是 RFC1918 IPv4 地址") return {"private_ip_cidr": str(interface)} -def _validate_private_routes(value: Any) -> list[str]: - """Validate the guest's exact on-link VPC routes. - - The host owns inter-host forwarding, while the guest still needs a route - for each remote /32. Keep this list deliberately narrow: no gateways, - default routes, or broad prefixes are accepted on ``private0``. +def _validate_private_routes( + value: Any, + *, + allow_legacy: bool = True, + private_ip_cidr: str | None = None, + public_ip_cidr: str | None = None, +) -> list[str]: + """Validate one stable VPC prefix or one legacy exact-/32 route set. + + New seeds use exactly one RFC1918 /16../24 link-scope prefix. This keeps + the guest contract stable when members or service /32s change; exact host + routes, proxy neighbours and nftables destination sets remain the routing + authority. The parser still accepts an all-/32 legacy set so a stopped VM + can migrate, but mixed or ambiguous representations are rejected. """ if value is None: - return [] + value = [] if not isinstance(value, (list, tuple)) or len(value) > 4096: raise ValueError("私网路由列表无效") - result: set[str] = set() + result: list[ipaddress.IPv4Network] = [] for item in value: if isinstance(item, str): destination = item @@ -340,26 +351,50 @@ def _validate_private_routes(value: Any) -> list[str]: if item.get("to") in {"default", "0.0.0.0/0", "::/0"}: # Keep the legacy diagnostic used by seed validation callers: # private0 is never allowed to acquire a default route. - raise ValueError("私网网卡必须是无默认路由的静态 /32") + raise ValueError("私网地址必须是静态 /32 且无默认路由") if set(item) != {"to", "scope"} or item.get("scope") != "link": - raise ValueError("私网路由必须是 link scope 的 /32") + raise ValueError("私网路由必须使用 link scope") destination = item.get("to") else: raise ValueError("私网路由格式无效") if destination in {"default", "0.0.0.0/0", "::/0"}: - raise ValueError("私网网卡必须是无默认路由的静态 /32") + raise ValueError("私网地址必须是静态 /32 且无默认路由") try: network = ipaddress.ip_network(destination, strict=True) except (TypeError, ValueError) as exc: - raise ValueError("私网路由必须是 IPv4 /32") from exc + raise ValueError("私网路由必须是 IPv4 网络") from exc if (not isinstance(network, ipaddress.IPv4Network) - or network.prefixlen != 32 - or not any(network.subnet_of(ipaddress.IPv4Network(block)) - for block in ("10.0.0.0/8", "172.16.0.0/12", - "192.168.0.0/16"))): - raise ValueError("私网路由必须是 RFC1918 IPv4 /32") - result.add(str(network)) - return sorted(result) + or not any(network.subnet_of(block) + for block in _RFC1918_NETWORKS)): + raise ValueError("私网路由必须位于 RFC1918 IPv4 空间") + result.append(network) + + if len(set(result)) != len(result): + raise ValueError("私网路由不能重复") + legacy = all(network.prefixlen == 32 for network in result) + stable = bool( + len(result) == 1 and 16 <= result[0].prefixlen <= 24 + ) + if not stable and not (allow_legacy and legacy): + raise ValueError( + "私网路由必须是唯一 VPC /16 到 /24 前缀," + "或完全由历史 /32 路由组成" + ) + + if private_ip_cidr is not None: + private_ip = ipaddress.ip_interface( + validate_private_network(private_ip_cidr)["private_ip_cidr"] + ).ip + if stable and private_ip not in result[0]: + raise ValueError("私网实例地址必须位于 VPC 路由前缀内") + if public_ip_cidr is not None: + try: + public_network = ipaddress.ip_interface(public_ip_cidr).network + except (TypeError, ValueError) as exc: + raise ValueError("公网直连前缀无效") from exc + if any(network.overlaps(public_network) for network in result): + raise ValueError("VPC 前缀不能与公网网卡直连前缀重叠") + return sorted(str(network) for network in result) def _validate_network_mac(value: Any, label: str) -> str: @@ -446,7 +481,7 @@ def _parse_network_config(config: Any) -> dict[str, Any]: or len(private["addresses"]) != 1 or not isinstance(private_match, dict) or set(private_match) != {"macaddress"}): - raise ValueError("原 cloud-init 私网网卡必须是无默认路由的静态 /32") + raise ValueError("原 cloud-init 私网地址必须是静态 /32 且无默认路由") private_mac = _validate_network_mac(private_mac, "私网") private_network = validate_private_network(private["addresses"][0]) if private_mac == public_mac: @@ -454,7 +489,11 @@ def _parse_network_config(config: Any) -> dict[str, Any]: result.update({ **private_network, "private_mac": private_mac, - "private_routes": _validate_private_routes(private.get("routes")), + "private_routes": _validate_private_routes( + private.get("routes"), + private_ip_cidr=private_network["private_ip_cidr"], + public_ip_cidr=public_network["public_ip_cidr"], + ), "roles": [PUBLIC_INTERFACE_ROLE, PRIVATE_INTERFACE_ROLE], }) return result @@ -595,7 +634,11 @@ def build_seed_iso( ) private_routes = _validate_private_routes( private_network.get("routes", []) - if isinstance(private_network, dict) else []) + if isinstance(private_network, dict) else [], + allow_legacy=False, + private_ip_cidr=private["private_ip_cidr"], + public_ip_cidr=network["public_ip_cidr"], + ) if normalized_private_mac == normalized_mac: raise ValueError("公网和私网网卡不能使用同一个 MAC") network_config["ethernets"][PRIVATE_INTERFACE_ROLE] = { @@ -853,31 +896,66 @@ def replace_seed_private_network( _parse_network_config(config) user_data = _enable_network_updates(files["user-data"]) metadata = _decode_seed_metadata(files["meta-data"]) + if user_data != files["user-data"]: + # Changing instance-id would make NoCloud replay credential and + # sshd per-instance modules. Keeping the id may leave legacy + # cloud-init versions on their cached network policy. Refuse this + # ambiguous migration; a reinstall or explicit offline migration + # must establish boot-v1 first. + raise SeedNetworkPolicyMigrationRequired( + "原 cloud-init 配置盘缺少安全的启动网络更新策略") ethernets = config.get("ethernets") if not isinstance(ethernets, dict): raise ValueError("原 cloud-init 网卡配置无效") if private_mac is None: - ethernets.pop(PRIVATE_INTERFACE_ROLE, None) + network_unchanged = PRIVATE_INTERFACE_ROLE not in parsed["roles"] + if not network_unchanged: + ethernets.pop(PRIVATE_INTERFACE_ROLE, None) else: normalized_mac = _validate_network_mac(private_mac, "私网") private = validate_private_network(private_ip_cidr) - routes = _validate_private_routes(private_routes) + routes = _validate_private_routes( + private_routes, + allow_legacy=False, + private_ip_cidr=private["private_ip_cidr"], + public_ip_cidr=parsed["public_ip_cidr"], + ) if normalized_mac == parsed["mac"]: raise ValueError("公网和私网网卡不能使用同一个 MAC") - ethernets[PRIVATE_INTERFACE_ROLE] = { - "match": {"macaddress": normalized_mac}, - "dhcp4": False, - "dhcp6": False, - "accept-ra": False, - "addresses": [private["private_ip_cidr"]], - **({"routes": [ - {"to": destination, "scope": "link"} - for destination in routes - ]} if routes else {}), - } - # Rotate the datasource identity so both add and remove operations are - # consumed on the next boot. Credentials remain opaque bytes. - metadata["instance-id"] = f"virt-cloud-private-{uuid.uuid4().hex}" + network_unchanged = bool( + PRIVATE_INTERFACE_ROLE in parsed["roles"] + and parsed.get("private_mac") == normalized_mac + and parsed.get("private_ip_cidr") + == private["private_ip_cidr"] + and parsed.get("private_routes", []) == routes + ) + if not network_unchanged: + ethernets[PRIVATE_INTERFACE_ROLE] = { + "match": {"macaddress": normalized_mac}, + "dhcp4": False, + "dhcp6": False, + "accept-ra": False, + "addresses": [private["private_ip_cidr"]], + **({"routes": [ + {"to": destination, "scope": "link"} + for destination in routes + ]} if routes else {}), + } + policy_unchanged = bool( + user_data == files["user-data"] + and metadata.get(_NETWORK_POLICY_KEY) == _NETWORK_POLICY_VALUE + ) + # A waiting-observed VPC is retried periodically. Returning the exact + # original ISO for an identical target prevents every retry from + # inventing a new NoCloud instance and re-running per-instance modules + # at the next boot. + if network_unchanged and policy_unchanged: + return seed + # Never rotate the NoCloud instance identity for a network-only + # change. A new instance-id would make cloud-init run every + # per-instance module again, including the credentials and sshd + # policy stored in this seed. The explicit boot event above is the + # only mechanism used to consume the refreshed network-config. metadata[_NETWORK_POLICY_KEY] = _NETWORK_POLICY_VALUE network_data = json.dumps(config, ensure_ascii=True).encode("ascii") + b"\n" return _build_seed_iso(( @@ -886,6 +964,8 @@ def replace_seed_private_network( _encode_seed_metadata(metadata)), ("/NETWORK.;1", "network-config", "/network-config", network_data), )) + except SeedNetworkPolicyMigrationRequired: + raise except (KeyError, TypeError, ValueError, json.JSONDecodeError) as exc: raise ValueError("原 cloud-init 网络配置格式无效") from exc diff --git a/engine/virsh_panel/internal.py b/engine/virsh_panel/internal.py index 69ae6f12..8d59c6aa 100644 --- a/engine/virsh_panel/internal.py +++ b/engine/virsh_panel/internal.py @@ -818,7 +818,8 @@ class EngineApplication: raise EngineAuthError("私网核对链路状态无效", 422) expected = { key: value[key] for key in ( - "required", "vpc_id", "member_id", "private_ip_cidr", + "lifecycle_kind", "required", "vpc_id", "member_id", + "private_ip_cidr", "private_mac", "desired_revision", "desired_digest", "desired_state") if key in value } diff --git a/engine/virsh_panel/libvirt_service.py b/engine/virsh_panel/libvirt_service.py index 678d13b8..9d84cd83 100644 --- a/engine/virsh_panel/libvirt_service.py +++ b/engine/virsh_panel/libvirt_service.py @@ -7,6 +7,8 @@ that host; libvirt-python does not turn an interactive SSH prompt into an API. from __future__ import annotations +import base64 +import binascii import hashlib import hmac import ipaddress @@ -22,11 +24,17 @@ import xml.etree.ElementTree as ET from contextlib import contextmanager from datetime import datetime, timezone from pathlib import Path -from typing import Any, Callable, Iterator, Sequence +from typing import Any, Callable, Iterator, Mapping, Sequence import libvirt +try: + import libvirt_qemu +except ImportError: # pragma: no cover - available with libvirt on Linux hosts + libvirt_qemu = None + from virsh_panel.cloud_init import ( + SeedNetworkPolicyMigrationRequired, _validate_guest_auth, _validate_ssh_public_key, build_seed_iso, default_guest_hostname, read_seed_network, replace_seed_network, replace_seed_private_network, replace_seed_settings, @@ -58,6 +66,8 @@ from virsh_panel.user_isos import ( LOG = logging.getLogger(__name__) GIB = 1024**3 PANEL_NS = "urn:virsh-panel:1" +QGA_RUNTIME_NS = "urn:virsh-panel:qga-runtime:1" +QGA_RUNTIME_KEY = "qga-runtime" CLEAN_TRAFFIC_FILTER = "clean-traffic" FIREWALL_FILTER_PREFIX = "vc-fw-" FIREWALL_FILTER_COMPILER_VERSION = 4 @@ -105,6 +115,15 @@ SERVICE_SUSPEND_SHUTDOWN_POLL_INTERVAL = 0.5 POWER_ACTION_TIMEOUT = 60 POWER_ACTION_POLL_INTERVAL = 0.5 POWER_ACTION_RESULT_UNCERTAIN_ERROR_CODE = "power_action_result_uncertain" +VPC_GUEST_AGENT_TIMEOUT = 2 +VPC_GUEST_AGENT_RESPONSE_MAX = 192 * 1024 +VPC_GUEST_LINK_OUTPUT_MAX = 16 * 1024 +VPC_GUEST_ROUTE_OUTPUT_MAX = 128 * 1024 +VPC_GUEST_ROUTE_MAX = 4096 +VPC_GUEST_ROUTE_SCRIPT = ( + 'exec /usr/sbin/ip -j -4 route show table main dev "$1" ' + '| /usr/bin/head -c 131073' +) SNAPSHOT_JOB_POLL_INTERVAL = 0.5 SNAPSHOT_JOB_TIMEOUT = 4 * 60 * 60 SNAPSHOT_ABORT_TIMEOUT = 30 @@ -135,6 +154,209 @@ class VMError(Exception): self.error_code = error_code +class _GuestProbeUnavailable(Exception): + """The bounded guest-agent observation could not be completed.""" + + +class _GuestProbeUnsupported(_GuestProbeUnavailable): + """The guest image/agent does not expose the required read-only probe.""" + + +class _GuestProbeUncertain(_GuestProbeUnavailable): + """The guest returned malformed, truncated, or time-unstable evidence.""" + + +class _GuestProbeDeadlineExceeded(_GuestProbeUnavailable): + """The request-wide guest observation budget has been exhausted.""" + + +class _GuestExecNotFound(_GuestProbeUnavailable): + """QGA positively reports that one previously returned PID is absent.""" + + +class _GuestExecRejected(_GuestProbeUnavailable): + """QGA rejected guest-exec before returning a child PID.""" + + +class _GuestExecUnsupported(_GuestProbeUnsupported): + """QGA explicitly rejected guest-exec as disabled or unsupported.""" + + +class _GuestExecReapRequired(_GuestProbeUncertain): + """A prior request may still own an unreaped QGA process record.""" + + +class _GuestAddressMismatch(Exception): + """The private address is not the exact desired guest state.""" + + +class _GuestLinkMismatch(Exception): + """The private interface is not administratively and operationally up.""" + + +class _GuestRouteMismatch(Exception): + """The private route set is not the exact desired guest state.""" + + +class _GuestExecLedger: + """One bounded durable launch/PID record for a VM's QGA probe. + + The caller owns persistence. ``launch_intent`` is committed before the + external request, and ``pending`` is committed immediately after QGA + returns a PID. An ambiguous launch intent cannot be retried safely: QGA + has no command which enumerates guest-exec records, so it remains closed + until an observed inactive domain proves that the old agent process no + longer exists. A PID loaded from storage is never queried: guest reboot or + qemu-ga restart can reuse a PID while the QEMU process and libvirt domain + ID remain unchanged, and a changed live metadata token is not sufficient + proof to recover that cross-request capability. + """ + + VERSION = 1 + PHASES = frozenset({"launch_intent", "pending"}) + PROBES = frozenset({"link", "routes"}) + + def __init__( + self, + state: dict[str, Any] | None, + *, + domain_uuid: str, + domain_runtime_id: int, + qemu_runtime_token: str, + guest_network_generation: str, + persist: Callable[[Mapping[str, Any]], None] | None = None, + clear: Callable[[], None] | None = None, + ) -> None: + if state is None: + state = {} + if not isinstance(state, dict): + raise _GuestProbeUncertain("invalid guest exec ledger") + if (not isinstance(domain_uuid, str) + or len(domain_uuid) != 36 + or isinstance(domain_runtime_id, bool) + or not isinstance(domain_runtime_id, int) + or domain_runtime_id < 0 + or domain_runtime_id > 2**31 - 1 + or not isinstance(qemu_runtime_token, str) + or re.fullmatch(r"[0-9a-f]{32}", qemu_runtime_token) is None + or not isinstance(guest_network_generation, str) + or not 1 <= len(guest_network_generation) <= 128 + or any(ord(char) < 0x20 or ord(char) > 0x7e + for char in guest_network_generation)): + raise _GuestProbeUncertain("invalid guest exec identity") + self.state = state + self.domain_uuid = domain_uuid + self.domain_runtime_id = domain_runtime_id + self.qemu_runtime_token = qemu_runtime_token + self.guest_network_generation = guest_network_generation + self.persist = persist + self.clear_callback = clear + # This capability is deliberately process-local and invocation-local. + # It is granted only after this object launches and durably records a + # PID; reconstructing an object from storage never grants it. + self._owned_pid: int | None = None + self._validate(self.state) + if self.state and ( + self.state.get("domain_uuid") != self.domain_uuid + or self.state.get("qemu_runtime_token") + != self.qemu_runtime_token): + raise _GuestProbeUncertain("guest exec identity mismatch") + + @classmethod + def _validate(cls, value: dict[str, Any]) -> None: + if not value: + return + phase = value.get("phase") + expected = { + "version", "phase", "domain_uuid", "domain_runtime_id", + "qemu_runtime_token", "guest_network_generation", "probe_kind", + } + if phase == "pending": + expected.add("pid") + if (set(value) != expected + or value.get("version") != cls.VERSION + or phase not in cls.PHASES + or not isinstance(value.get("domain_uuid"), str) + or len(value["domain_uuid"]) != 36 + or isinstance(value.get("domain_runtime_id"), bool) + or not isinstance(value.get("domain_runtime_id"), int) + or not 0 <= value["domain_runtime_id"] <= 2**31 - 1 + or not isinstance(value.get("qemu_runtime_token"), str) + or re.fullmatch( + r"[0-9a-f]{32}", value["qemu_runtime_token"]) is None + or not isinstance(value.get("guest_network_generation"), str) + or not 1 <= len(value["guest_network_generation"]) <= 128 + or any(ord(char) < 0x20 or ord(char) > 0x7e + for char in value["guest_network_generation"]) + or value.get("probe_kind") not in cls.PROBES): + raise _GuestProbeUncertain("invalid guest exec ledger") + if phase == "pending" and ( + isinstance(value.get("pid"), bool) + or not isinstance(value.get("pid"), int) + or not 1 <= value["pid"] <= 2**63 - 1): + raise _GuestProbeUncertain("invalid guest exec ledger") + + def snapshot(self) -> dict[str, Any]: + self._validate(self.state) + return dict(self.state) + + def begin(self, probe_kind: str) -> None: + if self.state: + raise _GuestProbeUncertain("guest exec cleanup is pending") + candidate = { + "version": self.VERSION, + "phase": "launch_intent", + "domain_uuid": self.domain_uuid, + "domain_runtime_id": self.domain_runtime_id, + "qemu_runtime_token": self.qemu_runtime_token, + "guest_network_generation": self.guest_network_generation, + "probe_kind": probe_kind, + } + self.state.update(candidate) + if self.persist is not None: + try: + self.persist(dict(candidate)) + except Exception as exc: + raise _GuestProbeUnavailable( + "guest exec ledger persistence failed") from exc + + def set_pid(self, pid: int) -> None: + current = self.snapshot() + if (current.get("phase") != "launch_intent" + or isinstance(pid, bool) or not isinstance(pid, int) + or not 1 <= pid <= 2**63 - 1): + raise _GuestProbeUncertain("invalid guest exec pid") + candidate = dict(current) + candidate.update({"phase": "pending", "pid": pid}) + self.state.clear() + self.state.update(candidate) + self._owned_pid = pid + if self.persist is not None: + try: + self.persist(dict(candidate)) + except Exception as exc: + raise _GuestProbeUnavailable( + "guest exec ledger persistence failed") from exc + + def clear(self) -> None: + if not self.state: + return + if self.clear_callback is not None: + try: + self.clear_callback() + except Exception as exc: + # Retain the local state too. A caller may otherwise launch a + # replacement command while the durable row still names the + # prior command whose cleanup result was not committed. + raise _GuestProbeUnavailable( + "guest exec ledger cleanup failed") from exc + self.state.clear() + self._owned_pid = None + + def owns_pid(self, pid: int) -> bool: + return self._owned_pid == pid + + class VMSnapshotUncertainError(VMError): """The snapshot operation crossed a destructive/ambiguous boundary.""" @@ -1502,8 +1724,850 @@ class VMService: error_code="vpc_vm_not_persistent") return domain.XMLDesc(libvirt.VIR_DOMAIN_XML_INACTIVE) - def read_private_domain_pair(self, uuid: str) -> dict[str, str | None]: - """Read persistent and live XML for an exact private NIC proof.""" + @staticmethod + def _qga_runtime_token_from_root( + root: ET.Element, vm_uuid: str, + ) -> str | None: + """Read one exact VM-bound token from the live-only namespace.""" + nodes = root.findall( + f"./metadata/{{{QGA_RUNTIME_NS}}}{QGA_RUNTIME_KEY}") + if not nodes: + return None + if len(nodes) != 1: + raise _GuestProbeUncertain("ambiguous QEMU runtime token") + node = nodes[0] + token = node.get("token") + if (set(node.attrib) != {"domain-uuid", "token"} + or node.get("domain-uuid") != vm_uuid + or not isinstance(token, str) + or re.fullmatch(r"[0-9a-f]{32}", token) is None + or list(node) + or (node.text is not None and node.text.strip())): + raise _GuestProbeUncertain("invalid QEMU runtime token") + return token + + @classmethod + def _without_qga_runtime_metadata(cls, xml: str | None) -> str | None: + """Remove the internal capability token from returned XML evidence.""" + if xml is None: + return None + try: + root = ET.fromstring(xml) + except ET.ParseError: + return xml + metadata = root.find("metadata") + if metadata is None: + return xml + changed = False + for node in list(metadata): + if node.tag == f"{{{QGA_RUNTIME_NS}}}{QGA_RUNTIME_KEY}": + metadata.remove(node) + changed = True + if not changed: + return xml + if len(metadata) == 0 and not (metadata.text or "").strip(): + root.remove(metadata) + return ET.tostring(root, encoding="unicode") + + @classmethod + def _ensure_qga_runtime_token( + cls, + domain: Any, + vm_uuid: str, + *, + inactive_xml: str, + live_xml: str, + ) -> tuple[str, str, ET.Element]: + """Return a read-back LIVE-only QEMU token and current live XML. + + The token proves only QEMU runtime replacement. It does not prove a + qemu-ga/guest boot identity and therefore never authorizes querying a + PID recovered from another request. + """ + try: + inactive_root = ET.fromstring(inactive_xml) + live_root = ET.fromstring(live_xml) + except ET.ParseError as exc: + raise _GuestProbeUncertain("invalid domain XML") from exc + if cls._qga_runtime_token_from_root(inactive_root, vm_uuid) is not None: + raise _GuestProbeUncertain( + "QEMU runtime token leaked into persistent XML") + token = cls._qga_runtime_token_from_root(live_root, vm_uuid) + if token is not None: + return token, live_xml, live_root + + setter = getattr(domain, "setMetadata", None) + if not callable(setter) or not domain.isActive(): + raise _GuestProbeUnavailable("QEMU runtime token unavailable") + candidate = secrets.token_hex(16) + token_xml = ( + f"<{QGA_RUNTIME_KEY} xmlns='{QGA_RUNTIME_NS}' " + f"domain-uuid='{vm_uuid}' token='{candidate}'/>") + set_error: Exception | None = None + try: + setter( + libvirt.VIR_DOMAIN_METADATA_ELEMENT, + token_xml, + QGA_RUNTIME_KEY, + QGA_RUNTIME_NS, + libvirt.VIR_DOMAIN_AFFECT_LIVE, + ) + except Exception as exc: + # setMetadata may commit and then lose its response. Only a + # fresh, exact live XML readback of our candidate resolves it. + set_error = exc + try: + if not domain.isActive(): + raise _GuestProbeUnavailable( + "domain stopped while setting QEMU runtime token") + refreshed = domain.XMLDesc(0) + refreshed_root = ET.fromstring(refreshed) + observed = cls._qga_runtime_token_from_root( + refreshed_root, vm_uuid) + if observed != candidate or not domain.isActive(): + raise _GuestProbeUnavailable( + "QEMU runtime token readback mismatch") + persistent_after = ET.fromstring( + domain.XMLDesc(libvirt.VIR_DOMAIN_XML_INACTIVE)) + if (cls._qga_runtime_token_from_root( + persistent_after, vm_uuid) is not None): + raise _GuestProbeUncertain( + "QEMU runtime token leaked into persistent XML") + except (ET.ParseError, _GuestProbeUnavailable, + _GuestProbeUncertain): + raise + except Exception as exc: + raise _GuestProbeUnavailable( + "QEMU runtime token readback failed") from exc + if set_error is not None: + LOG.info( + "Recovered QEMU runtime metadata response loss for %s", + vm_uuid, + ) + return candidate, refreshed, refreshed_root + + @staticmethod + def _qga_command( + domain: Any, + command: str, + *, + arguments: dict[str, Any] | None = None, + timeout: int = VPC_GUEST_AGENT_TIMEOUT, + deadline: float | None = None, + ) -> Any: + """Run one bounded, read-only QGA request and return its result. + + Only callers in the private-network observer use this helper. They + pass fixed command names and never interpolate a shell command. Raw + guest replies are deliberately not logged because a tenant controls + the guest agent and could place arbitrary data in them. + """ + if libvirt_qemu is None: + raise _GuestProbeUnavailable( + "libvirt qemu guest-agent bindings unavailable") + if (not isinstance(command, str) or not command + or not isinstance(timeout, int) or timeout < 1 or timeout > 5): + raise _GuestProbeUncertain("invalid guest-agent request") + if deadline is not None: + if (not isinstance(deadline, (int, float)) + or isinstance(deadline, bool)): + raise _GuestProbeUncertain("invalid guest-agent deadline") + remaining = deadline - time.monotonic() + # libvirt accepts whole-second QGA timeouts. Do not begin another + # call unless one bounded second still fits in the request-wide + # budget, and round down so the call cannot consume the next + # member's reserved time. + if remaining < 1: + raise _GuestProbeDeadlineExceeded( + "guest-agent request deadline exceeded") + timeout = min(timeout, int(remaining)) + request: dict[str, Any] = {"execute": command} + if arguments is not None: + if not isinstance(arguments, dict): + raise _GuestProbeUncertain("invalid guest-agent arguments") + request["arguments"] = arguments + raw = libvirt_qemu.qemuAgentCommand( + domain, + json.dumps(request, ensure_ascii=True, separators=(",", ":")), + timeout, + 0, + ) + if (not isinstance(raw, str) or not raw + or len(raw) > VPC_GUEST_AGENT_RESPONSE_MAX): + raise _GuestProbeUncertain("invalid guest-agent response") + try: + response = json.loads(raw) + except (TypeError, ValueError, json.JSONDecodeError) as exc: + raise _GuestProbeUncertain( + "invalid guest-agent response") from exc + if not isinstance(response, dict): + raise _GuestProbeUncertain("invalid guest-agent response") + if "error" in response: + error = response.get("error") + error_class = error.get("class") if isinstance(error, dict) else None + error_desc = error.get("desc") if isinstance(error, dict) else None + status_pid = ( + arguments.get("pid") if command == "guest-exec-status" + and isinstance(arguments, dict) else None + ) + if (command == "guest-exec-status" + and error_class == "GenericError" + and isinstance(status_pid, int) + and not isinstance(status_pid, bool) + and isinstance(error_desc, str) + and len(error_desc) <= 64 + and error_desc == f"PID {status_pid} does not exist"): + raise _GuestExecNotFound("guest exec record is absent") + if error_class in { + "CommandDisabled", "CommandNotFound", "NotSupported"}: + if command == "guest-exec": + raise _GuestExecUnsupported( + "guest exec command is unavailable") + raise _GuestProbeUnsupported("guest-agent command disabled") + if command == "guest-exec": + # A structured QGA error is a rejection, not an ambiguous + # transport failure: no GuestExec/PID was returned. The + # caller may therefore remove its pre-launch intent safely. + raise _GuestExecRejected("guest exec command was rejected") + raise _GuestProbeUnavailable("guest-agent command failed") + if "return" not in response: + raise _GuestProbeUncertain("invalid guest-agent response") + return response["return"] + + @classmethod + def _qga_assert_current_runtime( + cls, domain: Any, ledger: _GuestExecLedger, + ) -> None: + """Re-read the VM-bound LIVE token immediately before a spawn.""" + try: + if not domain.isActive(): + raise _GuestProbeUnavailable("domain stopped before probe") + root = ET.fromstring(domain.XMLDesc(0)) + token = cls._qga_runtime_token_from_root( + root, ledger.domain_uuid) + runtime_id = domain.ID() + if (not domain.isActive() + or token != ledger.qemu_runtime_token + or runtime_id != ledger.domain_runtime_id): + raise _GuestProbeUnavailable( + "QEMU runtime changed before guest exec") + except (_GuestProbeUnavailable, _GuestProbeUncertain): + raise + except Exception as exc: + raise _GuestProbeUnavailable( + "QEMU runtime readback failed") from exc + + @classmethod + def _qga_wait_guest_exec( + cls, + domain: Any, + ledger: _GuestExecLedger, + pid: int, + *, + timeout: int, + deadline: float | None = None, + ) -> dict[str, Any]: + """Wait for and reap one exact ledger PID, retaining it on ambiguity.""" + if not ledger.owns_pid(pid): + raise _GuestExecReapRequired( + "guest exec belongs to an earlier request") + command_deadline = time.monotonic() + timeout + if deadline is not None: + command_deadline = min(command_deadline, deadline) + status: Any = None + while time.monotonic() < command_deadline: + remaining = max( + 1, min(timeout, int(command_deadline - time.monotonic()))) + status = cls._qga_command( + domain, + "guest-exec-status", + arguments={"pid": pid}, + timeout=remaining, + deadline=deadline, + ) + if (not isinstance(status, dict) + or not isinstance(status.get("exited"), bool)): + raise _GuestProbeUncertain("invalid guest exec status") + if status["exited"] is True: + # QGA's official contract says this status read reaps the + # exited child and its captured metadata. Delete the durable + # row only after that positive response. + ledger.clear() + return status + sleep_for = min( + 0.05, max(0.0, command_deadline - time.monotonic())) + if sleep_for: + time.sleep(sleep_for) + raise _GuestProbeUncertain("guest exec did not finish") + + @classmethod + def _qga_reap_existing_guest_exec( + cls, + domain: Any, + ledger: _GuestExecLedger, + *, + timeout: int, + deadline: float | None = None, + ) -> None: + current = ledger.snapshot() + if not current: + return + # Never query a PID reconstructed from durable storage. qemu-ga is a + # guest process: it may restart (or the guest may reboot) without a + # QEMU/libvirt runtime change, and its host PID can then collide with + # an unrelated guest-exec record. The bounded row is a stop marker, + # not a capability to reap across requests. + raise _GuestExecReapRequired( + "guest exec cleanup requires a confirmed stopped domain") + + @classmethod + def _qga_best_effort_reap( + cls, + domain: Any, + ledger: _GuestExecLedger, + *, + deadline: float | None, + ) -> None: + current = ledger.snapshot() + if current.get("phase") != "pending": + return + pid = current.get("pid") + if not ledger.owns_pid(pid): + return + try: + status = cls._qga_command( + domain, "guest-exec-status", arguments={"pid": pid}, + timeout=1, deadline=deadline) + if isinstance(status, dict) and status.get("exited") is True: + ledger.clear() + except _GuestExecNotFound: + try: + ledger.clear() + except _GuestProbeUnavailable: + pass + except _GuestProbeUnavailable: + pass + + @classmethod + def _qga_exec_bytes( + cls, + domain: Any, + ledger: _GuestExecLedger, + *, + probe_kind: str, + path: str, + arguments: Sequence[str], + output_max: int, + timeout: int, + deadline: float | None = None, + ) -> bytes: + """Run one fixed command with a durable two-phase QGA ledger.""" + arguments_valid = ( + isinstance(arguments, (list, tuple)) + and 1 <= len(arguments) <= 16 + and all(isinstance(value, str) + and 1 <= len(value) <= 256 + and all(0x20 <= ord(char) <= 0x7e for char in value) + for value in arguments) + ) + fixed_command = False + if arguments_valid and probe_kind == "link" \ + and path == "/usr/bin/head" and len(arguments) == 7: + match = re.fullmatch( + r"/sys/class/net/([A-Za-z0-9][A-Za-z0-9_.:@-]{0,14})/address", + arguments[3], + ) + if match is not None: + base = f"/sys/class/net/{match.group(1)}" + fixed_command = tuple(arguments) == ( + "-q", "-c", "4097", f"{base}/address", + f"{base}/operstate", f"{base}/carrier", + f"{base}/flags", + ) + elif arguments_valid and probe_kind == "routes" \ + and path == "/bin/sh" and len(arguments) == 4: + fixed_command = ( + tuple(arguments[:3]) == ( + "-c", VPC_GUEST_ROUTE_SCRIPT, "vpc-route-probe") + and re.fullmatch( + r"[A-Za-z0-9][A-Za-z0-9_.:@-]{0,14}", + arguments[3], + ) is not None + ) + if (probe_kind not in _GuestExecLedger.PROBES + or not fixed_command + or not arguments_valid + or isinstance(output_max, bool) + or not isinstance(output_max, int) + or not 1 <= output_max <= VPC_GUEST_ROUTE_OUTPUT_MAX + 1 + ): + raise _GuestProbeUncertain("invalid guest probe command") + cls._qga_reap_existing_guest_exec( + domain, ledger, timeout=timeout, deadline=deadline) + cls._qga_assert_current_runtime(domain, ledger) + ledger.begin(probe_kind) + try: + try: + started = cls._qga_command( + domain, + "guest-exec", + arguments={ + # GNU timeout bounds the child even if the host-side + # request is lost. Every command also pipes/reads + # through ``head`` so QGA cannot first fill its much + # larger per-stream capture buffer. + "path": "/usr/bin/timeout", + "arg": [ + "--signal=KILL", "1", path, *arguments, + ], + "capture-output": True, + }, + timeout=timeout, + deadline=deadline, + ) + except (_GuestExecRejected, _GuestExecUnsupported): + # A structured command rejection proves that no child/PID was + # created, so the pre-launch intent can be removed. + ledger.clear() + raise + if (not isinstance(started, dict) + or isinstance(started.get("pid"), bool) + or not isinstance(started.get("pid"), int) + or not 1 <= started["pid"] <= 2**63 - 1): + # The launch may have happened. Retain launch_intent and do + # not guess a PID or issue a replacement command. + raise _GuestProbeUncertain("invalid guest exec result") + ledger.set_pid(started["pid"]) + try: + status = cls._qga_wait_guest_exec( + domain, ledger, started["pid"], timeout=timeout, + deadline=deadline) + except _GuestExecNotFound as exc: + ledger.clear() + raise _GuestProbeUncertain( + "new guest exec record disappeared") from exc + finally: + # A status transport/deadline failure must not drop the PID. One + # final bounded read is attempted only while the request budget + # still permits it; otherwise the durable row is the next pass's + # mandatory first action. + cls._qga_best_effort_reap( + domain, ledger, deadline=deadline) + if (not isinstance(status, dict) or status.get("exited") is not True + or isinstance(status.get("exitcode"), bool) + or status.get("signal") is not None + or status.get("err-data") not in (None, "") + or status.get("out-truncated") is True + or status.get("err-truncated") is True): + raise _GuestProbeUncertain("guest probe failed") + if status.get("exitcode") in {126, 127}: + raise _GuestProbeUnsupported("guest probe dependencies missing") + if status.get("exitcode") != 0: + raise _GuestProbeUnavailable("guest probe failed") + encoded = status.get("out-data") + encoded_max = ((output_max + 2) // 3) * 4 + if not isinstance(encoded, str) or len(encoded) > encoded_max: + raise _GuestProbeUncertain("invalid guest probe output") + try: + payload = base64.b64decode(encoded, validate=True) + except (ValueError, binascii.Error) as exc: + raise _GuestProbeUncertain("invalid guest probe output") from exc + if not payload or len(payload) > output_max: + raise _GuestProbeUncertain("invalid guest probe output") + return payload + + @classmethod + def _qga_link_state( + cls, + domain: Any, + ledger: _GuestExecLedger, + interface_name: str, + *, + timeout: int, + deadline: float | None = None, + ) -> tuple[str, str, str, str]: + base = f"/sys/class/net/{interface_name}" + payload = cls._qga_exec_bytes( + domain, ledger, probe_kind="link", path="/usr/bin/head", + arguments=( + "-q", "-c", "4097", f"{base}/address", + f"{base}/operstate", f"{base}/carrier", f"{base}/flags", + ), + output_max=VPC_GUEST_LINK_OUTPUT_MAX, + timeout=timeout, deadline=deadline, + ) + try: + text = payload.decode("ascii") + except UnicodeError as exc: + raise _GuestProbeUncertain("invalid guest link output") from exc + if not text.endswith("\n"): + raise _GuestProbeUncertain("truncated guest link output") + rows = text.splitlines() + if len(rows) != 4 or any(not 1 <= len(row) <= 128 for row in rows): + raise _GuestProbeUncertain("invalid guest link output") + return tuple(rows) # type: ignore[return-value] + + @classmethod + def _qga_route_rows( + cls, + domain: Any, + ledger: _GuestExecLedger, + interface_name: str, + *, + timeout: int, + deadline: float | None = None, + ) -> list[dict[str, Any]]: + payload = cls._qga_exec_bytes( + domain, ledger, probe_kind="routes", path="/bin/sh", + arguments=( + "-c", VPC_GUEST_ROUTE_SCRIPT, "vpc-route-probe", + interface_name, + ), + output_max=VPC_GUEST_ROUTE_OUTPUT_MAX + 1, + timeout=timeout, deadline=deadline, + ) + if len(payload) > VPC_GUEST_ROUTE_OUTPUT_MAX: + raise _GuestProbeUncertain("guest route output exceeds limit") + try: + text = payload.decode("utf-8") + except UnicodeError as exc: + raise _GuestProbeUncertain("invalid guest route output") from exc + try: + rows = json.loads(text) + except (TypeError, ValueError, json.JSONDecodeError) as exc: + raise _GuestProbeUncertain("invalid guest route output") from exc + if (not isinstance(rows, list) + or len(rows) > VPC_GUEST_ROUTE_MAX + or any(not isinstance(row, dict) for row in rows)): + raise _GuestProbeUncertain("invalid guest route output") + return rows + + @classmethod + def _qga_private_routes( + cls, + domain: Any, + ledger: _GuestExecLedger, + interface_name: str, + expected_mac: str, + expected_routes: Sequence[str], + *, + timeout: int, + deadline: float | None = None, + ) -> tuple[str, ...]: + """Prove link state and read exact routes through fixed QGA commands.""" + if (not isinstance(interface_name, str) + or re.fullmatch( + r"[A-Za-z0-9][A-Za-z0-9_.:@-]{0,14}", + interface_name, + ) is None): + raise _GuestProbeUncertain("invalid guest interface name") + info = cls._qga_command( + domain, "guest-info", timeout=timeout, deadline=deadline) + supported = info.get("supported_commands") if isinstance(info, dict) else None + if not isinstance(supported, list) or len(supported) > 1024: + raise _GuestProbeUncertain( + "guest-agent command catalog unavailable") + enabled = { + item.get("name") for item in supported + if isinstance(item, dict) and item.get("enabled") is True + and isinstance(item.get("name"), str) + } + if not {"guest-exec", "guest-exec-status"}.issubset(enabled): + raise _GuestProbeUnsupported("guest route probe is unsupported") + link_mac, operstate, carrier, link_flags = cls._qga_link_state( + domain, ledger, interface_name, + timeout=timeout, deadline=deadline) + try: + numeric_link_flags = int(link_flags, 16) + except ValueError as exc: + raise _GuestProbeUncertain( + "invalid private guest link flags") from exc + if (link_mac.lower() != expected_mac + or operstate != "up" + or carrier != "1" + or numeric_link_flags & 0x1 != 0x1): + raise _GuestLinkMismatch("private guest link is not usable") + rows = cls._qga_route_rows( + domain, ledger, interface_name, + timeout=timeout, deadline=deadline) + expected = set(expected_routes) + routes: set[str] = set() + for row in rows: + if (len(row) > 32 + or any(not isinstance(key, str) or len(key) > 64 + for key in row) + or row.get("dev") != interface_name + or not isinstance(row.get("dst"), str) + or len(row["dst"]) > 64): + raise _GuestProbeUncertain("invalid private guest route") + try: + network = ipaddress.ip_network(row["dst"], strict=True) + except (TypeError, ValueError) as exc: + raise _GuestProbeUncertain( + "invalid private guest route") from exc + if not isinstance(network, ipaddress.IPv4Network): + raise _GuestProbeUncertain("invalid private guest route") + normalized = str(network) + route_type = row.get("type", "unicast") + table = row.get("table", "main") + flags = row.get("flags", []) + if (not isinstance(flags, list) or len(flags) > 32 + or any(not isinstance(flag, str) or len(flag) > 64 + for flag in flags)): + raise _GuestProbeUncertain("invalid private guest route") + if (normalized not in expected + or row.get("scope") != "link" + or route_type != "unicast" + or table not in {"main", 254} + or "gateway" in row or "via" in row + or "nexthops" in row or "nhid" in row + or {str(flag).lower() for flag in flags}.intersection({ + "dead", "linkdown", "unresolved", + })): + raise _GuestRouteMismatch( + "private route is not the active scope-link VPC CIDR") + if normalized in routes: + raise _GuestRouteMismatch("duplicate private guest route") + routes.add(normalized) + return tuple(sorted(routes)) + + @classmethod + def _guest_private_network_evidence( + cls, + domain: Any, + ledger: _GuestExecLedger, + *, + private_mac: str, + private_ip_cidr: str, + private_routes: Sequence[str], + timeout: int, + deadline: float | None = None, + ) -> dict[str, Any]: + """Prove the guest kernel owns the requested private address/routes.""" + observed_at = int(time.time()) + unavailable = { + "state": "unavailable", + "verified": False, + "reason_code": "vpc_private_guest_agent_unavailable", + "observed_at": observed_at, + } + try: + expected_mac = private_mac.lower() + if re.fullmatch(r"[0-9a-f]{2}(?::[0-9a-f]{2}){5}", expected_mac) is None: + raise _GuestProbeUncertain("invalid private MAC") + expected_ip = ipaddress.ip_interface(private_ip_cidr) + if (not isinstance(expected_ip, ipaddress.IPv4Interface) + or expected_ip.network.prefixlen != 32): + raise _GuestProbeUncertain("invalid private IP") + expected_routes = tuple(sorted(set(private_routes))) + if (len(expected_routes) != 1 + or len(expected_routes) != len(tuple(private_routes))): + raise _GuestProbeUncertain( + "invalid expected private routes") + try: + expected_route = ipaddress.ip_network( + expected_routes[0], strict=True) + except ValueError as exc: + raise _GuestProbeUncertain( + "invalid expected private route") from exc + if (not isinstance(expected_route, ipaddress.IPv4Network) + or not 16 <= expected_route.prefixlen <= 24 + or str(expected_route) != expected_routes[0]): + raise _GuestProbeUncertain( + "invalid expected private route") + + set_timeout = getattr(domain, "agentSetResponseTimeout", None) + get_addresses = getattr(domain, "interfaceAddresses", None) + if not callable(set_timeout) or not callable(get_addresses): + return unavailable + def read_interfaces() -> Any: + try: + effective_timeout = timeout + if deadline is not None: + remaining = deadline - time.monotonic() + if remaining < 1: + raise _GuestProbeDeadlineExceeded( + "guest-agent request deadline exceeded") + effective_timeout = min(timeout, int(remaining)) + set_timeout(effective_timeout, 0) + return get_addresses( + libvirt.VIR_DOMAIN_INTERFACE_ADDRESSES_SRC_AGENT, 0) + finally: + try: + set_timeout( + getattr( + libvirt, + "VIR_DOMAIN_AGENT_RESPONSE_TIMEOUT_DEFAULT", + -1, + ), + 0, + ) + except Exception as exc: + LOG.warning( + "Could not restore guest-agent timeout for %s", + domain.UUIDString()) + # agentSetResponseTimeout is domain-wide. A positive + # readback is not safe to publish while a probe-owned + # timeout may still affect unrelated QGA operations. + raise _GuestProbeUncertain( + "guest-agent timeout restore failed") from exc + + interfaces = read_interfaces() + if not isinstance(interfaces, dict) or len(interfaces) > 256: + raise _GuestProbeUncertain( + "invalid guest interface response") + matches = [] + for key, value in interfaces.items(): + if not isinstance(key, str) or not isinstance(value, dict): + raise _GuestProbeUncertain( + "invalid guest interface response") + hardware = value.get("hwaddr") + if isinstance(hardware, str) and hardware.lower() == expected_mac: + matches.append((key, value)) + if len(matches) != 1: + raise _GuestAddressMismatch( + "private guest interface is missing or ambiguous") + interface_name, interface = matches[0] + addresses = interface.get("addrs") + if (not isinstance(interface_name, str) + or not isinstance(addresses, list) + or len(addresses) > 128): + raise _GuestProbeUncertain( + "invalid private guest interface") + ipv4_addresses: set[str] = set() + ipv4_type = getattr(libvirt, "VIR_IP_ADDR_TYPE_IPV4", 0) + for address in addresses: + if not isinstance(address, dict): + raise _GuestProbeUncertain( + "invalid guest address response") + if address.get("type") != ipv4_type: + continue + value = address.get("addr") + prefix = address.get("prefix") + if (not isinstance(value, str) or isinstance(prefix, bool) + or not isinstance(prefix, int)): + raise _GuestProbeUncertain( + "invalid guest IPv4 response") + try: + ipv4_addresses.add(str(ipaddress.ip_interface( + f"{value}/{prefix}"))) + except ValueError as exc: + raise _GuestProbeUncertain( + "invalid guest IPv4 response") from exc + if ipv4_addresses != {str(expected_ip)}: + raise _GuestAddressMismatch( + "private guest IPv4 address mismatch") + observed_routes = cls._qga_private_routes( + domain, ledger, interface_name, expected_mac, expected_routes, + timeout=timeout, + deadline=deadline, + ) + if observed_routes != expected_routes: + raise _GuestRouteMismatch("private guest route mismatch") + interfaces_after = read_interfaces() + if (not isinstance(interfaces_after, dict) + or interface_name not in interfaces_after + or not isinstance(interfaces_after[interface_name], dict) + or interfaces_after[interface_name].get("hwaddr", "").lower() + != expected_mac): + raise _GuestProbeUncertain( + "private guest interface changed during probe") + after_addresses = interfaces_after[interface_name].get("addrs") + if not isinstance(after_addresses, list): + raise _GuestProbeUncertain( + "private guest interface changed during probe") + after_ipv4: set[str] = set() + for address in after_addresses: + if (not isinstance(address, dict) + or address.get("type") != ipv4_type): + continue + value = address.get("addr") + prefix = address.get("prefix") + if (not isinstance(value, str) or isinstance(prefix, bool) + or not isinstance(prefix, int)): + raise _GuestProbeUncertain( + "invalid guest IPv4 response") + try: + after_ipv4.add(str(ipaddress.ip_interface( + f"{value}/{prefix}"))) + except ValueError as exc: + raise _GuestProbeUncertain( + "invalid guest IPv4 response") from exc + if after_ipv4 != {str(expected_ip)}: + raise _GuestProbeUncertain( + "private guest interface changed during probe") + return { + "state": "confirmed", + "verified": True, + "source": "qemu_guest_agent", + "private_mac": expected_mac, + "private_ip_cidr": str(expected_ip), + "link_state": "up", + "routes": list(observed_routes), + "reason_code": None, + "observed_at": observed_at, + } + except _GuestAddressMismatch: + return { + "state": "mismatch", + "verified": False, + "reason_code": "vpc_private_guest_address_mismatch", + "observed_at": observed_at, + } + except _GuestLinkMismatch: + return { + "state": "mismatch", + "verified": False, + "reason_code": "vpc_private_guest_link_mismatch", + "observed_at": observed_at, + } + except _GuestRouteMismatch: + return { + "state": "mismatch", + "verified": False, + "reason_code": "vpc_private_guest_route_mismatch", + "observed_at": observed_at, + } + except _GuestProbeUnsupported: + return { + "state": "unsupported", + "verified": False, + "reason_code": "vpc_private_guest_probe_unsupported", + "observed_at": observed_at, + } + except _GuestProbeDeadlineExceeded: + return { + "state": "deadline_exceeded", + "verified": False, + "reason_code": "vpc_private_guest_probe_deadline_exceeded", + "observed_at": observed_at, + } + except _GuestProbeUncertain: + return { + "state": "uncertain", + "verified": False, + "reason_code": "vpc_private_guest_probe_uncertain", + "observed_at": observed_at, + } + except _GuestProbeUnavailable: + return unavailable + except Exception: + return unavailable + + def read_private_domain_pair( + self, + uuid: str, + *, + private_mac: str | None = None, + private_ip_cidr: str | None = None, + private_routes: Sequence[str] | None = None, + guest_probe_timeout: int = VPC_GUEST_AGENT_TIMEOUT, + guest_probe_deadline: float | None = None, + guest_exec_state: dict[str, Any] | None = None, + guest_exec_persist: Callable[[Mapping[str, Any]], None] | None = None, + guest_exec_clear: Callable[[], None] | None = None, + ) -> dict[str, Any]: + """Read XML and, when requested, exact guest-kernel network state.""" vm_uuid = self._uuid(uuid) with self._connect() as conn: domain = self._domain(conn, vm_uuid) @@ -1511,8 +2575,256 @@ class VMService: raise VMError("VPC 私网网卡只能接入持久化虚拟机", 409, error_code="vpc_vm_not_persistent") inactive = domain.XMLDesc(libvirt.VIR_DOMAIN_XML_INACTIVE) - live = domain.XMLDesc(0) if domain.isActive() else None - return {"inactive": inactive, "live": live} + running = bool(domain.isActive()) + live = domain.XMLDesc(0) if running else None + result: dict[str, Any] = { + "inactive": inactive, + "live": self._without_qga_runtime_metadata(live), + } + requested = (private_mac, private_ip_cidr, private_routes) + if any(value is not None for value in requested): + if (private_mac is None or private_ip_cidr is None + or private_routes is None): + raise VMError("私网来宾回读目标不完整", 500, + error_code="vpc_private_guest_probe_invalid") + qemu_runtime_token = "0" * 32 + try: + inactive_root = ET.fromstring(inactive) + live_root = ET.fromstring(live or "") if running else None + _GuestExecLedger._validate( + guest_exec_state if guest_exec_state is not None + else {}) + if running: + if self._qga_runtime_token_from_root( + inactive_root, vm_uuid) is not None: + raise _GuestProbeUncertain( + "QEMU runtime token is persistent") + current_token = self._qga_runtime_token_from_root( + live_root, vm_uuid) + if (guest_exec_state + and current_token != guest_exec_state.get( + "qemu_runtime_token")): + # Missing or changed metadata on an otherwise + # still-running domain does not prove that the old + # qemu-ga PID namespace is gone. The guest agent + # (or the guest itself) can restart independently + # of QEMU and reuse a PID, so neither mint a token + # nor clear the durable stop marker here. + result["guest_network"] = { + "state": "uncertain", "verified": False, + "reason_code": + "vpc_private_guest_probe_reap_required", + "observed_at": int(time.time()), + } + return result + if current_token is None: + (qemu_runtime_token, live, live_root) = ( + self._ensure_qga_runtime_token( + domain, + vm_uuid, + inactive_xml=inactive, + live_xml=live or "", + ) + ) + result["live"] = ( + self._without_qga_runtime_metadata(live)) + else: + qemu_runtime_token = current_token + inactive_managed = inactive_root.find( + f"./metadata/{{{PANEL_NS}}}managed") + live_managed = ( + live_root.find(f"./metadata/{{{PANEL_NS}}}managed") + if live_root is not None else None) + inactive_generation = ( + inactive_managed.get( + "private-guest-network-generation") + if inactive_managed is not None else None) + live_generation = ( + live_managed.get( + "private-guest-network-generation") + if live_managed is not None else None) + live_private = ( + self._private_interface_from_root(live_root) + if live_root is not None else None) + live_mac = (live_private.find("mac") + if live_private is not None else None) + except (ET.ParseError, ValueError, + _GuestProbeUnavailable, _GuestProbeUncertain): + live_private = None + live_mac = None + inactive_generation = None + live_generation = None + result["guest_network"] = { + "state": "unavailable", "verified": False, + "reason_code": + "vpc_private_guest_agent_unavailable", + "observed_at": int(time.time()), + } + return result + ledger_generation = next((value for value in ( + live_generation, inactive_generation, + ) if isinstance(value, str) and value + and len(value) <= 128 + and all(0x20 <= ord(char) <= 0x7e + for char in value)), "unknown") + try: + if not running and guest_exec_state: + # No QEMU exists, so the stored token is used only to + # validate and delete the exact durable row. + qemu_runtime_token = guest_exec_state[ + "qemu_runtime_token"] + ledger = _GuestExecLedger( + guest_exec_state, + domain_uuid=vm_uuid, + domain_runtime_id=domain.ID() if running else 0, + qemu_runtime_token=qemu_runtime_token, + guest_network_generation=ledger_generation, + persist=guest_exec_persist, + clear=guest_exec_clear, + ) + except _GuestProbeUncertain: + result["guest_network"] = { + "state": "uncertain", "verified": False, + "reason_code": "vpc_private_guest_probe_uncertain", + "observed_at": int(time.time()), + } + return result + except _GuestProbeUnavailable: + result["guest_network"] = { + "state": "unavailable", "verified": False, + "reason_code": + "vpc_private_guest_agent_unavailable", + "observed_at": int(time.time()), + } + return result + if not running: + try: + # An inactive QEMU process cannot retain QGA child + # metadata. This is the only safe way to clear an + # ambiguous launch_intent for which no PID exists. + ledger.clear() + except _GuestProbeUnavailable: + result["guest_network"] = { + "state": "unavailable", "verified": False, + "reason_code": + "vpc_private_guest_agent_unavailable", + "observed_at": int(time.time()), + } + return result + result["guest_network"] = { + "state": "stopped", "verified": False, + "reason_code": "vpc_private_guest_stopped", + "observed_at": int(time.time()), + } + elif (guest_probe_deadline is not None + and guest_probe_deadline - time.monotonic() < 1): + result["guest_network"] = { + "state": "deadline_exceeded", "verified": False, + "reason_code": + "vpc_private_guest_probe_deadline_exceeded", + "observed_at": int(time.time()), + } + elif guest_probe_timeout < 1: + result["guest_network"] = { + "state": "unavailable", "verified": False, + "reason_code": "vpc_private_guest_agent_unavailable", + "observed_at": int(time.time()), + } + else: + try: + cls_timeout = min( + VPC_GUEST_AGENT_TIMEOUT, guest_probe_timeout) + self._qga_reap_existing_guest_exec( + domain, ledger, timeout=cls_timeout, + deadline=guest_probe_deadline) + except _GuestProbeUnsupported: + result["guest_network"] = { + "state": "unsupported", "verified": False, + "reason_code": + "vpc_private_guest_probe_unsupported", + "observed_at": int(time.time()), + } + return result + except _GuestProbeDeadlineExceeded: + result["guest_network"] = { + "state": "deadline_exceeded", "verified": False, + "reason_code": + "vpc_private_guest_probe_deadline_exceeded", + "observed_at": int(time.time()), + } + return result + except _GuestExecReapRequired: + result["guest_network"] = { + "state": "uncertain", "verified": False, + "reason_code": + "vpc_private_guest_probe_reap_required", + "observed_at": int(time.time()), + } + return result + except _GuestProbeUncertain: + result["guest_network"] = { + "state": "uncertain", "verified": False, + "reason_code": + "vpc_private_guest_probe_uncertain", + "observed_at": int(time.time()), + } + return result + except _GuestProbeUnavailable: + result["guest_network"] = { + "state": "unavailable", "verified": False, + "reason_code": + "vpc_private_guest_agent_unavailable", + "observed_at": int(time.time()), + } + return result + if (live_private is None or live_mac is None + or live_mac.get("address", "").lower() + != private_mac.lower() + or not inactive_generation + or live_generation != inactive_generation): + result["guest_network"] = { + "state": "pending_restart", "verified": False, + "reason_code": "vpc_private_guest_restart_required", + "observed_at": int(time.time()), + } + return result + live_link = live_private.find("link") + if (live_link is not None + and live_link.get("state", "up").lower() + == "down"): + # Policy suspension is proven by the exact current + # generation/MAC plus live libvirt link-down. It is + # intentionally non-usable and must not run a guest + # link-up command merely to confirm isolation. + result["guest_network"] = { + "state": "suspended", "verified": False, + "source": "libvirt_live_xml", + "private_mac": private_mac.lower(), + "private_ip_cidr": private_ip_cidr, + "link_state": "down", "routes": [], + "reason_code": + "vpc_private_network_suspended", + "observed_at": int(time.time()), + } + return result + boot_id = domain.ID() + evidence = self._guest_private_network_evidence( + domain, ledger, + private_mac=private_mac, + private_ip_cidr=private_ip_cidr, + private_routes=private_routes, + timeout=min(VPC_GUEST_AGENT_TIMEOUT, + guest_probe_timeout), + deadline=guest_probe_deadline, + ) + if (not domain.isActive() or domain.ID() != boot_id): + evidence = { + "state": "unavailable", "verified": False, + "reason_code": "vpc_private_guest_agent_unavailable", + "observed_at": int(time.time()), + } + result["guest_network"] = evidence + return result @staticmethod def _private_interface_from_root(root: ET.Element) -> ET.Element | None: @@ -1522,6 +2834,18 @@ class VMService: raise VMError("虚拟机网卡角色不明确,已停止私网接入", 409, error_code="vpc_private_xml_role_invalid") from exc + @staticmethod + def _assert_private_rollback_stopped( + domain: Any, *, message: str, error_code: str, + ) -> None: + """Refuse an inverse private-network mutation without fresh stop proof.""" + try: + active = bool(domain.isActive()) + except Exception as exc: + raise VMError(message, 502, error_code=error_code) from exc + if active: + raise VMError(message, 502, error_code=error_code) + def ensure_private_interface( self, vm_id: str, @@ -1536,14 +2860,20 @@ class VMService: ) -> dict[str, Any]: """Attach one panel-owned ``private0`` NIC and refresh its seed. - The operation is intentionally a single libvirt/seed transaction. A - running domain uses ``attach/updateDeviceFlags`` for live and config - state; a stopped domain is updated through ``defineXML``. On any - confirmed failure the original XML and seed are restored. If a live + The operation is intentionally a single libvirt/seed transaction. + Guest-network changes are left untouched while a domain is running; + a later reconcile may stage the seed and persistent XML only after it + observes a full stop. Existing booted generations may use + ``attach/updateDeviceFlags`` for link-only reconciliation. On any + confirmed failure the original XML and seed are restored. If a live libvirt response is lost, the method raises an ``uncertain`` error and leaves reconciliation to the VPC readback lane instead of guessing. """ - from .vpc_dataplane import VpcDataplaneError, patch_private_interface_xml + from virsh_panel.vpc_dataplane import ( + VpcDataplaneError, + patch_private_interface_xml, + private_guest_network_generation, + ) vm_uuid = self._uuid(vm_id) if not isinstance(vpc_id, str) or not vpc_id: @@ -1558,6 +2888,13 @@ class VMService: if link_state not in {None, "preserve", "up", "down"}: raise VMError("私网网卡链路状态无效", 422, error_code="vpc_private_link_state_invalid") + requested_private_routes = tuple(private_routes or ()) + try: + guest_network_generation = private_guest_network_generation( + member, requested_private_routes) + except VpcDataplaneError as exc: + raise VMError( + str(exc), exc.status, error_code=exc.error_code) from exc with _CREATE_LOCK, self._connect() as conn: domain = self._domain(conn, vm_uuid) @@ -1600,27 +2937,71 @@ class VMService: try: new_xml = patch_private_interface_xml( old_inactive_xml, member=member, resources=resources, + guest_network_generation=guest_network_generation, vpc_id=vpc_id, desired_revision=desired_revision, desired_digest=desired_digest, link_state=effective_link_state) except VpcDataplaneError as exc: - raise VMError(str(exc), exc.status, exc.error_code) from exc + raise VMError( + str(exc), exc.status, error_code=exc.error_code) from exc new_root = ET.fromstring(new_xml) target_private = self._private_interface_from_root(new_root) if target_private is None: raise VMError("无法生成 private0 网卡", 502, error_code="vpc_private_xml_invalid") + def private_device_state(interface: ET.Element | None) -> tuple[Any, ...] | None: + if interface is None: + return None + source = interface.find("source") + mac = interface.find("mac") + model = interface.find("model") + link = interface.find("link") + alias = interface.find("alias") + return ( + interface.get("type"), + tuple(sorted(source.attrib.items())) + if source is not None else (), + mac.get("address", "").lower() + if mac is not None else None, + model.get("type") if model is not None else None, + link.get("state", "up").lower() + if link is not None else "up", + alias.get("name") if alias is not None else None, + tuple(sorted( + ET.tostring(item, encoding="unicode") + for item in interface.findall("filterref"))), + ) + was_running = bool(domain.isActive()) - live_mutator = None + old_live_xml = None + old_live_root = None + live_private_before = None + live_generation_current = False if was_running: - live_mutator = (getattr(domain, "attachDeviceFlags", None) - if private_before is None - else getattr(domain, "updateDeviceFlags", None)) - if not callable(live_mutator): + try: + old_live_xml = domain.XMLDesc(0) + old_live_root = ET.fromstring(old_live_xml) + live_private_before = self._private_interface_from_root( + old_live_root) + live_managed = old_live_root.find( + f"./metadata/{{{PANEL_NS}}}managed") + live_generation_current = bool( + live_managed is not None + and live_managed.get("private-vpc-id") == vpc_id + and live_managed.get("private-member-id") + == member.member_id + and live_managed.get("private-ip-cidr") + == f"{member.private_ip}/32" + and live_managed.get( + "private-guest-network-generation") + == guest_network_generation + ) + except (ET.ParseError, ValueError) as exc: raise VMError( - "运行中的虚拟机不支持安全添加或更新 private0", 409, - error_code="vpc_private_live_unsupported") + "无法核对运行中虚拟机的私网 generation", 502, + error_code="vpc_private_live_readback_mismatch", + ) from exc seed = self._owned_seed_volume(conn, domain) if seed is None: @@ -1652,47 +3033,129 @@ class VMService: replacement_seed = replace_seed_private_network( old_seed, private_mac=private_mac.get("address"), private_ip_cidr=private_ip, - private_routes=list(private_routes or ())) + private_routes=list(requested_private_routes)) seed_check = read_seed_network(replacement_seed) - expected_seed_routes = sorted(set(private_routes or ())) - if sorted(seed_check.get("private_routes", [])) != expected_seed_routes: - raise VMError("私网 cloud-init 路由回读不一致", 502, - error_code="vpc_seed_route_readback_mismatch") + expected_seed_routes = sorted(set(requested_private_routes)) + if (seed_check.get("private_mac") + != private_mac.get("address", "").lower() + or seed_check.get("private_ip_cidr") != private_ip + or sorted(seed_check.get("private_routes", [])) + != expected_seed_routes): + raise VMError("私网 cloud-init 配置回读不一致", 502, + error_code="vpc_seed_readback_mismatch") + except SeedNetworkPolicyMigrationRequired as exc: + raise VMError( + "旧 cloud-init 配置盘需要先完成安全网络策略迁移", 409, + error_code="vpc_seed_network_policy_migration_required", + ) from exc except (ImportError, ValueError) as exc: raise VMError("无法生成私网 cloud-init 配置", 409, error_code="vpc_seed_update_failed") from exc + seed_changed = replacement_seed != old_seed + # NoCloud cannot consume a changed seed while the guest is + # running. More importantly, overwriting the seed and persistent + # definition in place creates a mixed state: the current guest is + # still using the old generation while the next-boot artifacts + # already describe the new one. Leave both artifacts untouched + # until a later reconcile observes a fully stopped domain. + if was_running and seed_changed: + return { + "xml": old_inactive_xml, + "live": old_live_xml, + "seed_updated": False, + "seed_verified": False, + "restart_required": True, + "readback": "pending_restart", + } + normalized_old_xml = ET.tostring(old_root, encoding="unicode") + config_changed = normalized_old_xml != new_xml + restart_required = bool( + was_running and not live_generation_current) + live_update_required = bool( + was_running and not restart_required + and private_device_state(live_private_before) + != private_device_state(target_private)) + live_mutator = None + if live_update_required: + live_mutator = ( + getattr(domain, "attachDeviceFlags", None) + if live_private_before is None + else getattr(domain, "updateDeviceFlags", None) + ) + if not callable(live_mutator): + raise VMError( + "运行中的虚拟机不支持安全添加或更新 private0", 409, + error_code="vpc_private_live_unsupported") + switched_config = False - switched_live = False seed_touched = False config_attempted = False live_attempted = False result_uncertain = False committed = False try: - current_capacity = int(seed.info()[1]) - if len(replacement_seed) > current_capacity: - seed.resize(len(replacement_seed), 0) - # Mark the seed dirty before the stream starts. A transport - # error can occur after a partial write, so setting this only - # after _upload_volume returns would skip the rollback. - seed_touched = True - self._upload_volume(conn, seed, iter((replacement_seed,)), - len(replacement_seed)) + if seed_changed: + if not was_running and domain.isActive(): + result_uncertain = True + raise VMError( + "虚拟机在私网配置期间启动,结果需要重新回读", 502, + error_code="vpc_private_result_uncertain") + current_capacity = int(seed.info()[1]) + if len(replacement_seed) > current_capacity: + seed.resize(len(replacement_seed), 0) + # Mark the seed dirty before the stream starts. A + # transport error can occur after a partial write, so + # setting this only after _upload_volume returns would + # skip the rollback. + seed_touched = True + self._upload_volume( + conn, seed, iter((replacement_seed,)), + len(replacement_seed)) + # Confirm the physical seed write before crossing the + # libvirt definition boundary. A short or misplaced + # upload is still safely reversible here because neither + # persistent nor live domain XML has been touched. + try: + uploaded_seed = read_seed_network( + self._download_seed_volume(conn, seed)) + except (ImportError, ValueError) as exc: + raise VMError( + "私网 cloud-init 写入回读失败", 502, + error_code="vpc_seed_readback_mismatch", + ) from exc + if (uploaded_seed.get("private_mac") + != private_mac.get("address", "").lower() + or uploaded_seed.get("private_ip_cidr") + != private_ip + or sorted(uploaded_seed.get( + "private_routes", [])) + != expected_seed_routes): + raise VMError( + "私网 cloud-init 写入回读不一致", 502, + error_code="vpc_seed_readback_mismatch", + ) # Device APIs cannot update domain-level ownership metadata. - # Always persist the complete XML first, then alter only the - # live device when QEMU is running. - config_attempted = True - defined = conn.defineXML(new_xml) - if defined is None: - raise VMError("宿主机未能保存 private0 配置", 502, - error_code="vpc_private_define_failed") - switched_config = True - if was_running: + # Persist the complete XML, but never present a new guest + # network generation to a running guest. NoCloud consumes it + # only at boot, so a live NIC update would be a false proof of + # address and route readiness. + if config_changed: + if not was_running and domain.isActive(): + result_uncertain = True + raise VMError( + "虚拟机在私网配置期间启动,结果需要重新回读", 502, + error_code="vpc_private_result_uncertain") + config_attempted = True + defined = conn.defineXML(new_xml) + if defined is None: + raise VMError("宿主机未能保存 private0 配置", 502, + error_code="vpc_private_define_failed") + switched_config = True + if live_update_required: target_xml = ET.tostring(target_private, encoding="unicode") live_attempted = True live_mutator(target_xml, libvirt.VIR_DOMAIN_AFFECT_LIVE) - switched_live = True pair = self.read_private_domain_pair(vm_uuid) inactive_after = ET.fromstring(pair["inactive"] or "") @@ -1703,10 +3166,32 @@ class VMService: or observed_mac.get("address", "").lower() != private_mac.get("address", "").lower() or observed_meta is None or observed_meta.get("private-vpc-id") != vpc_id + or observed_meta.get( + "private-guest-network-generation") + != guest_network_generation or observed_meta.get("private-network-revision") != str(desired_revision) or observed_meta.get("private-network-digest") != desired_digest): raise VMError("private0 配置回读不一致", 502, error_code="vpc_private_xml_readback_mismatch") + try: + observed_seed = read_seed_network( + self._download_seed_volume(conn, seed)) + except (ImportError, ValueError) as exc: + raise VMError("私网 cloud-init 配置回读失败", 502, + error_code="vpc_seed_readback_mismatch") from exc + if (observed_seed.get("private_mac") + != private_mac.get("address", "").lower() + or observed_seed.get("private_ip_cidr") != private_ip + or sorted(observed_seed.get("private_routes", [])) + != expected_seed_routes): + raise VMError("私网 cloud-init 配置回读不一致", 502, + error_code="vpc_seed_readback_mismatch") + if not was_running and domain.isActive(): + result_uncertain = True + raise VMError( + "虚拟机在私网配置回读期间启动,结果需要重新回读", 502, + error_code="vpc_private_result_uncertain") + restart_required_after = restart_required if was_running: live_after = ET.fromstring(pair["live"] or "") live_private = self._private_interface_from_root(live_after) @@ -1714,17 +3199,36 @@ class VMService: if live_private is not None else None) live_mac = (live_private.find("mac") if live_private is not None else None) - if (live_private is None or live_source is None + live_meta = live_after.find( + f"./metadata/{{{PANEL_NS}}}managed") + live_generation_verified = bool( + live_meta is not None + and live_meta.get("private-vpc-id") == vpc_id + and live_meta.get("private-member-id") + == member.member_id + and live_meta.get("private-ip-cidr") == private_ip + and live_meta.get( + "private-guest-network-generation") + == guest_network_generation + ) + restart_required_after = bool( + restart_required or not live_generation_verified) + if (live_generation_verified + and (live_private is None or live_source is None or live_source.get("bridge") != resources.bridge or live_mac is None or live_mac.get("address", "").lower() != private_mac.get("address", "").lower() - or live_private.findall("filterref")): + or live_private.findall("filterref"))): raise VMError("private0 实时配置回读不一致", 502, error_code="vpc_private_live_readback_mismatch") committed = True return {"xml": pair["inactive"], "live": pair["live"], - "seed_updated": True, "readback": "confirmed"} + "seed_updated": seed_changed, + "seed_verified": True, + "restart_required": restart_required_after, + "readback": ("pending_restart" + if restart_required_after else "confirmed")} except VMError as exc: # Once a persistent definition or live device mutation has # been attempted on a running domain, even a typed readback @@ -1762,18 +3266,59 @@ class VMService: not result_uncertain and (not was_running or not config_attempted and not live_attempted) ) - if seed_touched and not committed and (rollback_xml_confirmed or rollback_seed_confirmed): - try: - # If defineXML committed, restore the original - # persistent definition before restoring the seed. - if switched_config and rollback_xml_confirmed: - conn.defineXML(old_inactive_xml) - if rollback_seed_confirmed: + if (not committed + and ((switched_config and rollback_xml_confirmed) + or (seed_touched and rollback_seed_confirmed))): + # If defineXML committed, restore the original persistent + # definition before restoring the seed. Re-check the live + # state immediately before every inverse mutation: an + # external start may race the forward-path checks, and a + # failed state probe is not proof that rollback is safe. + if switched_config and rollback_xml_confirmed: + self._assert_private_rollback_stopped( + domain, + message=( + "虚拟机在私网回滚前启动,结果需要重新回读" + ), + error_code="vpc_private_result_uncertain", + ) + try: + restored_domain = conn.defineXML( + old_inactive_xml) + if restored_domain is None: + raise VMError( + "宿主机未确认原 private0 配置恢复", 502, + error_code=( + "vpc_private_result_uncertain" + ), + ) + except Exception as rollback_exc: + LOG.exception( + "Could not restore VPC domain XML for %s", + vm_uuid, + ) + raise VMError( + "私网配置回滚失败,结果需要重新回读", 502, + error_code="vpc_private_result_uncertain", + ) from rollback_exc + if seed_touched and rollback_seed_confirmed: + self._assert_private_rollback_stopped( + domain, + message=( + "虚拟机在私网回滚前启动,结果需要重新回读" + ), + error_code="vpc_private_result_uncertain", + ) + try: if len(old_seed) > int(seed.info()[1]): seed.resize(len(old_seed), 0) self._upload_volume(conn, seed, iter((old_seed,)), len(old_seed)) - except Exception: - LOG.exception("Could not restore VPC seed for %s", vm_uuid) + except Exception as rollback_exc: + LOG.exception("Could not restore VPC seed for %s", vm_uuid) + raise VMError( + "私网配置盘回滚失败,结果需要重新回读", 502, + error_code="vpc_private_result_uncertain", + ) from rollback_exc def remove_private_interface( self, vm_id: str, *, vpc_id: str, resources: Any, @@ -1879,6 +3424,10 @@ class VMService: # This is a positive absence proof. An unowned CD-ROM is # deliberately not inspected or rewritten and therefore # cannot be treated as proof of seed convergence. + if domain.isActive(): + raise VMError( + "虚拟机在私网移除核验期间启动", 502, + error_code="vpc_private_remove_uncertain") return {"removed": True, "already_absent": True, "xml": old_xml, "readback": "confirmed"} raise VMError("找不到 cloud-init 配置盘,无法移除私网", 409, @@ -1916,19 +3465,31 @@ class VMService: for key in tuple(managed.attrib): if key.startswith("private-"): managed.attrib.pop(key, None) - replacement = ( - replace_seed_private_network( - old_seed, private_mac=None, private_ip_cidr=None) - if seed_has_private else old_seed - ) + try: + replacement = ( + replace_seed_private_network( + old_seed, private_mac=None, private_ip_cidr=None) + if seed_has_private else old_seed + ) + except SeedNetworkPolicyMigrationRequired as exc: + raise VMError( + "旧 cloud-init 配置盘需要先完成安全网络策略迁移", 409, + error_code="vpc_seed_network_policy_migration_required", + ) from exc define_attempted = False defined = False seed_attempted = False committed = False + runtime_changed = False already_absent = not xml_changed and not seed_has_private try: defined_domain = domain if xml_changed: + if domain.isActive(): + runtime_changed = True + raise VMError( + "虚拟机在私网移除期间启动", 502, + error_code="vpc_private_remove_uncertain") define_attempted = True defined_domain = conn.defineXML( ET.tostring(root, encoding="unicode")) @@ -1937,6 +3498,11 @@ class VMService: error_code="vpc_private_remove_failed") defined = True if seed_has_private: + if domain.isActive(): + runtime_changed = True + raise VMError( + "虚拟机在私网移除期间启动", 502, + error_code="vpc_private_remove_uncertain") seed_attempted = True if len(replacement) > int(seed.info()[1]): seed.resize(len(replacement), 0) @@ -1961,6 +3527,11 @@ class VMService: or "private0" in observed_seed.get("roles", [])): raise VMError("私网配置盘移除回读不一致", 502, error_code="vpc_seed_remove_readback_mismatch") + if domain.isActive(): + runtime_changed = True + raise VMError( + "虚拟机在私网移除回读期间启动", 502, + error_code="vpc_private_remove_uncertain") committed = True return {"removed": True, "already_absent": already_absent, "xml": observed_xml, "readback": "confirmed"} @@ -1975,20 +3546,59 @@ class VMService: raise VMError("宿主机未能移除 private0", 502, error_code="vpc_private_remove_failed") from exc finally: - if not committed and (defined or seed_attempted): - try: - # The domain is stopped, so restoring the persistent - # XML and seed is a confirmed, safe inverse operation. - conn.defineXML(old_xml) - if seed_attempted: + if (not committed and not runtime_changed + and (defined or seed_attempted)): + # A stopped preflight is stale by rollback time. Verify it + # again immediately before each inverse mutation; an + # active guest (or failed state read) makes the result + # uncertain and forbids both XML and seed restoration. + if defined: + self._assert_private_rollback_stopped( + domain, + message=( + "虚拟机在私网移除回滚前启动,结果需要重新回读" + ), + error_code="vpc_private_remove_uncertain", + ) + try: + restored_domain = conn.defineXML(old_xml) + if restored_domain is None: + raise VMError( + "宿主机未确认原 private0 配置恢复", 502, + error_code=( + "vpc_private_remove_uncertain" + ), + ) + except Exception as rollback_exc: + LOG.exception( + "Could not roll back VPC private XML for %s", + vm_uuid, + ) + raise VMError( + "私网移除回滚失败,请等待宿主机核对", 502, + error_code="vpc_private_remove_uncertain", + ) from rollback_exc + if seed_attempted: + self._assert_private_rollback_stopped( + domain, + message=( + "虚拟机在私网移除回滚前启动,结果需要重新回读" + ), + error_code="vpc_private_remove_uncertain", + ) + try: if len(old_seed) > int(seed.info()[1]): seed.resize(len(old_seed), 0) self._upload_volume(conn, seed, iter((old_seed,)), len(old_seed)) - except Exception as rollback_exc: - LOG.exception("Could not roll back VPC private removal for %s", vm_uuid) - raise VMError( - "私网移除回滚失败,请等待宿主机核对", 502, - error_code="vpc_private_remove_rollback_failed") from rollback_exc + except Exception as rollback_exc: + LOG.exception( + "Could not roll back VPC private seed for %s", + vm_uuid, + ) + raise VMError( + "私网移除回滚失败,请等待宿主机核对", 502, + error_code="vpc_private_remove_uncertain", + ) from rollback_exc def get_vm_state(self, uuid: str) -> dict[str, Any]: """Fetch one domain's power state without loading its XML metadata.""" diff --git a/engine/virsh_panel/vpc.py b/engine/virsh_panel/vpc.py index 1120d651..986071d1 100644 --- a/engine/virsh_panel/vpc.py +++ b/engine/virsh_panel/vpc.py @@ -10,11 +10,13 @@ to the later data-plane phase. from __future__ import annotations import asyncio +from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait from contextlib import AsyncExitStack, asynccontextmanager import hashlib import ipaddress import json import re +import threading import time from dataclasses import dataclass, replace from typing import Any, Mapping, Sequence @@ -31,6 +33,27 @@ VPC_MAX_ROUTES = 1024 VPC_MAX_ACLS = 2048 VPC_MAX_READBACK = 4096 VPC_MAX_READBACK_VALUE = 512 +VPC_EVIDENCE_VERSION = 2 +# libvirt rejects an incomplete QGA response after 10 MiB, but its API must +# materialize the complete bounded reply before Python can apply the much +# smaller per-command gate below. Keep the rolling pool deliberately narrow +# so several hostile or broken guests cannot multiply that upstream buffer +# into an avoidable host-memory spike. +VPC_GUEST_PROBE_WORKERS = 4 +# Per-request rolling pools must also share one process-wide admission gate. +# Without it, concurrent VPC requests each admit four libvirt QGA replies and +# multiply the upstream response-buffer bound. BoundedSemaphore also turns an +# accidental over-release into an immediate programming error. +_VPC_GUEST_PROBE_SLOTS = threading.BoundedSemaphore(VPC_GUEST_PROBE_WORKERS) +VPC_GUEST_EXEC_KIND = "vpc_guest_exec" +VPC_GUEST_EXEC_MAX_BYTES = 1024 +# PHP gives this endpoint 60 seconds and holds the operation lease for 120 +# seconds. This monotonic budget stops admission of further hosts/QGA calls at +# 40 seconds, leaving 20 seconds for response/transport and another 60 seconds +# before a second worker may reclaim the operation. A libvirt/seed mutation +# already in flight is not safely preemptible; shutdown waits for bounded QGA +# calls instead of leaving background work after the response. +VPC_REQUEST_DEADLINE_SECONDS = 40.0 VPC_STATES = frozenset({"applying", "active", "degraded", "uncertain"}) VPC_DELETE_RECEIPT_KIND = "vpc_deleted" VPC_DELETE_DETERMINISTIC_HOST_ERRORS = frozenset({ @@ -46,6 +69,17 @@ _IDENTIFIER = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,63}\Z") _HOST_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]{0,63}\Z") _DIGEST = re.compile(r"[0-9a-f]{64}\Z") _OPERATION_ID = re.compile(r"[A-Za-z0-9_-]{8,80}\Z") +_PRIVATE_LIFECYCLE_KIND = re.compile( + r"(?:create|reinstall|network_change|instance_settings|rescue|" + r"snapshot_create|snapshot_restore|snapshot_delete|delete|expiry_delete|" + r"action|service_suspend|service_resume|network_suspend|network_resume|" + r"firewall_apply|" + r"action_(?:start|shutdown|reboot|poweroff|reset)_policy_" + r"(?:active|blocked|unready)|" + r"service_suspend_policy_(?:active|blocked|unready)|" + r"service_resume_(?:start|stopped|unbound)_policy_" + r"(?:active|blocked|unready))\Z" +) # Compatibility markers used by the asynchronous reinstall worker. A same # operation private reconcile may pass the lifecycle barrier only after the # replacement domain has crossed its irreversible commit boundary. @@ -222,6 +256,46 @@ def _safe_scalar(value: Any) -> bool: and all(ord(char) >= 32 and ord(char) != 127 for char in value)) +def _guest_exec_core(value: Any) -> dict[str, Any]: + """Validate one bounded internal QGA launch/PID ledger payload.""" + if not isinstance(value, Mapping): + raise VpcError("VPC 来宾执行账本损坏", 500, + "vpc_record_corrupt") + phase = value.get("phase") + fields = { + "version", "phase", "domain_uuid", "domain_runtime_id", + "qemu_runtime_token", "guest_network_generation", "probe_kind", + } + if phase == "pending": + fields.add("pid") + generation = value.get("guest_network_generation") + if (set(value) != fields + or value.get("version") != 1 + or phase not in {"launch_intent", "pending"} + or not isinstance(value.get("domain_uuid"), str) + or _UUID.fullmatch(value["domain_uuid"]) is None + or isinstance(value.get("domain_runtime_id"), bool) + or not isinstance(value.get("domain_runtime_id"), int) + or not 0 <= value["domain_runtime_id"] <= 2**31 - 1 + or not isinstance(value.get("qemu_runtime_token"), str) + or re.fullmatch( + r"[0-9a-f]{32}", value["qemu_runtime_token"]) is None + or not isinstance(generation, str) + or not 1 <= len(generation) <= 128 + or any(ord(char) < 0x20 or ord(char) > 0x7e + for char in generation) + or value.get("probe_kind") not in {"link", "routes"}): + raise VpcError("VPC 来宾执行账本损坏", 500, + "vpc_record_corrupt") + if phase == "pending" and ( + isinstance(value.get("pid"), bool) + or not isinstance(value.get("pid"), int) + or not 1 <= value["pid"] <= 2**63 - 1): + raise VpcError("VPC 来宾执行账本损坏", 500, + "vpc_record_corrupt") + return dict(value) + + def _canonical_cidr(network: ipaddress.IPv4Network) -> str: return str(network) @@ -948,6 +1022,77 @@ class VpcCoordinator: self.store.set(kind, key, json.dumps(value, ensure_ascii=True, separators=(",", ":"))) + def _guest_exec_context( + self, plan: VpcPlan, member: VpcMember, + ) -> tuple[ + dict[str, Any], + Any, + Any, + ]: + """Load one private durable QGA ledger and build exact callbacks. + + The record is deliberately separate from ``vpc_desired``: a PID must + be committed immediately after ``guest-exec`` returns, while rewriting + a multi-thousand-member desired document for every probe would create + quadratic database traffic. Existing VPC/instance locks serialize + these callbacks for the same member. + """ + key = self._key(plan.vpc_id, member.instance_uuid) + raw = self.store.get(VPC_GUEST_EXEC_KIND, key) + state: dict[str, Any] = {} + base = { + "vpc_id": plan.vpc_id, + "host_id": member.host_id, + "member_id": member.member_id, + "instance_uuid": member.instance_uuid, + } + if raw is not None: + if (not isinstance(raw, str) + or len(raw.encode("utf-8")) > VPC_GUEST_EXEC_MAX_BYTES): + raise VpcError("VPC 来宾执行账本损坏", 500, + "vpc_record_corrupt") + try: + value = json.loads(raw) + except (TypeError, ValueError) as exc: + raise VpcError("VPC 来宾执行账本损坏", 500, + "vpc_record_corrupt") from exc + if (not isinstance(value, Mapping) + or set(value) != set(base) | { + "version", "phase", "domain_uuid", + "domain_runtime_id", "qemu_runtime_token", + "guest_network_generation", + "probe_kind", + } | ({"pid"} if value.get("phase") == "pending" else set()) + or any(value.get(field) != expected + for field, expected in base.items())): + raise VpcError("VPC 来宾执行账本损坏", 500, + "vpc_record_corrupt") + state = _guest_exec_core({ + field: value[field] for field in value if field not in base + }) + if state.get("domain_uuid") != member.instance_uuid: + raise VpcError("VPC 来宾执行账本损坏", 500, + "vpc_record_corrupt") + + def persist(core: Mapping[str, Any]) -> None: + normalized = _guest_exec_core(core) + if normalized.get("domain_uuid") != member.instance_uuid: + raise VpcError("VPC 来宾执行账本损坏", 500, + "vpc_record_corrupt") + record = dict(base) + record.update(normalized) + encoded = json.dumps( + record, ensure_ascii=True, separators=(",", ":")) + if len(encoded.encode("utf-8")) > VPC_GUEST_EXEC_MAX_BYTES: + raise VpcError("VPC 来宾执行账本过大", 500, + "vpc_record_corrupt") + self.store.set(VPC_GUEST_EXEC_KIND, key, encoded) + + def clear() -> None: + self.store.delete(VPC_GUEST_EXEC_KIND, key) + + return state, persist, clear + @staticmethod def _confirmed_removal( host: Mapping[str, Any], revision: int, digest: str, @@ -992,6 +1137,71 @@ class VpcCoordinator: and host.get("rollback") in {"none", "ok"} ) + @staticmethod + def _confirmed_active_host(host: Mapping[str, Any], plan: VpcPlan) -> bool: + """Whether this exact host revision already has complete live proof. + + A request-wide probe budget is shared by every host. Retrying an + already exact host on every reconcile would let an early slow host + starve later hosts forever. Preserve only the strong same-revision + proof; any tombstone, member-set change, or incomplete guest evidence + puts the host back in the execution closure. + """ + host_id = host.get("host_id") + if not isinstance(host_id, str): + return False + expected_members = tuple( + member for member in plan.members if member.host_id == host_id) + expected_ids = {member.member_id for member in expected_members} + raw_ids = host.get("member_ids") + member_nics = host.get("member_nics") + if (not isinstance(raw_ids, list) + or any(not isinstance(value, str) for value in raw_ids) + or set(raw_ids) != expected_ids + or len(raw_ids) != len(expected_ids) + or not isinstance(member_nics, list) + or len(member_nics) != len(expected_members)): + return False + expected_by_id = { + member.member_id: member for member in expected_members + } + proven_ids: set[str] = set() + for item in member_nics: + if not isinstance(item, Mapping): + continue + member_id = item.get("member_id") + member = expected_by_id.get(member_id) + if (member is None + or item.get("instance_uuid") != member.instance_uuid + or item.get("private_ip_cidr") + != f"{member.private_ip}/32" + or item.get("private_mac") + != _effective_private_mac( + plan.vpc_id, member.member_id, + member.private_mac) + or item.get("config_verified") is not True + or item.get("xml_verified") is not True + or item.get("domain_running") is not True + or item.get("guest_network_verified") is not True + or item.get("restart_required") is not False): + continue + proven_ids.add(member_id) + return bool( + host.get("retiring") is not True + and host.get("retiring_members", []) == [] + and host.get("state") == "active" + and host.get("transport_verified") is True + and host.get("private_dataplane_verified") is True + and host.get("config_verified") is True + and host.get("desired_revision") == plan.desired_revision + and host.get("observed_revision") == plan.desired_revision + and host.get("config_observed_revision") == plan.desired_revision + and host.get("desired_digest") == plan.desired_digest + and host.get("observed_digest") == plan.desired_digest + and host.get("config_observed_digest") == plan.desired_digest + and proven_ids == expected_ids + ) + @classmethod def _terminal_removal(cls, host: Mapping[str, Any]) -> bool: """Validate a self-contained historical host absence proof.""" @@ -1200,6 +1410,7 @@ class VpcCoordinator: "desired_digest", "state", "deleted", "readback", "removed_host_ids", "completed_at", )} + compact["evidence_version"] = int(receipt.get("evidence_version", 0)) if receipt != compact: self._save(VPC_DELETE_RECEIPT_KIND, vpc_id, compact) return compact @@ -1207,7 +1418,8 @@ class VpcCoordinator: @staticmethod def _public_delete_receipt(receipt: Mapping[str, Any]) -> dict[str, Any]: return {key: receipt[key] for key in ( - "vpc_id", "operation_id", "desired_revision", "desired_digest", + "vpc_id", "operation_id", "evidence_version", + "desired_revision", "desired_digest", "state", "deleted", "readback", "removed_host_ids", "completed_at", )} @@ -1235,7 +1447,7 @@ class VpcCoordinator: def _purge_deleted_vpc(self, vpc_id: str) -> None: prefix = vpc_id + "." for kind in ("vpc_subnet", "vpc_member", "vpc_route", "vpc_acl", - "vpc_host"): + "vpc_host", VPC_GUEST_EXEC_KIND): for key in tuple(self.store.list(kind)): if key.startswith(prefix): self.store.delete(kind, key) @@ -1451,13 +1663,9 @@ class VpcCoordinator: @staticmethod def _private_routes(plan: VpcPlan, member: VpcMember) -> list[str]: - destinations = { - f"{other.private_ip}/32" for other in plan.members - if other.private_ip != member.private_ip - } - destinations.update(route.destination_cidr for route in plan.routes) - destinations.discard(f"{member.private_ip}/32") - return sorted(destinations) + from .vpc_dataplane import private_guest_routes + + return list(private_guest_routes(plan, member)) @staticmethod def _observation_error(exc: Exception) -> str: @@ -1473,6 +1681,7 @@ class VpcCoordinator: retiring_members: Sequence[Mapping[str, Any]] | None = None, fabric_public_keys: Mapping[str, str] | None = None, private_link_states: Mapping[str, str] | None = None, + request_deadline: float | None = None, ) -> dict[str, Any]: """Apply and read back one host synchronously behind the VPC lock.""" from .libvirt_service import VMService @@ -1515,12 +1724,43 @@ class VpcCoordinator: dataplane = VpcDataplanePlan( plan.vpc, host_id, tuple(plan.members), tuple(), resource_names(plan.vpc_id)) - adapter = VpcPrivateHostAdapter(component.manager, profile) + if request_deadline is None: + request_deadline = ( + time.monotonic() + VPC_REQUEST_DEADLINE_SECONDS) + adapter = VpcPrivateHostAdapter( + component.manager, profile, request_deadline=request_deadline) service = VMService( profile["libvirt_uri"], profile["storage_pool"], profile["iso_pool"], profile["default_network"], rescue_iso_pool=profile.get("rescue_iso_pool", ""), ) + guest_exec_contexts: dict[str, tuple[dict[str, Any], Any, Any]] = {} + + def guest_exec_context( + member: VpcMember, + ) -> tuple[dict[str, Any], Any, Any]: + context = guest_exec_contexts.get(member.instance_uuid) + if context is None: + context = self._guest_exec_context(plan, member) + guest_exec_contexts[member.instance_uuid] = context + return context + + def require_request_budget() -> None: + if time.monotonic() >= request_deadline: + raise VpcError( + "VPC 来宾网络核验超过请求预算", 502, + "vpc_private_guest_probe_deadline_exceeded") + + # A lifecycle-triggered reconcile must reach its target before a + # large host's shared probe budget expires. Mapping membership is the + # priority signal even for ``preserve``; ordinary periodic passes have + # no entries and retain canonical member order. + priority_instances = set(private_link_states or {}) + local_members = tuple(sorted( + dataplane.local_members, + key=lambda item: item.instance_uuid not in priority_instances, + )) + old_members: list[VpcMember] = [] if previous_plan is not None: old_members.extend( @@ -1551,6 +1791,9 @@ class VpcCoordinator: continue if old_member.member_id in current_ids: continue + require_request_budget() + ledger_state, _persist_ledger, clear_ledger = ( + guest_exec_context(old_member)) service.remove_private_interface( old_member.instance_uuid, vpc_id=cleanup_plan.vpc_id, @@ -1563,11 +1806,17 @@ class VpcCoordinator: old_member.private_mac), expected_private_ip_cidr=f"{old_member.private_ip}/32", ) + # remove_private_interface positively requires a stopped domain. + # Its old QGA process therefore cannot retain either a launch + # intent or PID record. Delete the ledger only after that proof. + clear_ledger() + ledger_state.clear() # A host with no local members is a removal target. Run the # manager-aware teardown and require its positive absence marker; # returning a normal empty observation would leave stale bridge, # route, nft, and peer state behind. - if not dataplane.local_members: + if not local_members: + require_request_budget() removed = adapter.remove(dataplane) removed.update({ "vpc_id": plan.vpc_id, @@ -1581,9 +1830,15 @@ class VpcCoordinator: "rollback": removed.get("rollback", "none"), }) return removed + require_request_budget() adapter.apply(dataplane) - for member in dataplane.local_members: - service.ensure_private_interface( + pairs: dict[str, dict[str, Any]] = {} + probe_members: list[VpcMember] = [] + for member in local_members: + require_request_budget() + ledger_state, _persist_ledger, _clear_ledger = ( + guest_exec_context(member)) + configured = service.ensure_private_interface( member.instance_uuid, member=member, resources=dataplane.resources, vpc_id=plan.vpc_id, desired_revision=plan.desired_revision, @@ -1592,9 +1847,141 @@ class VpcCoordinator: link_state=(private_link_states or {}).get( member.instance_uuid, "preserve"), ) - xml_by_instance: dict[str, Mapping[str, str | None]] = {} - for member in dataplane.local_members: - pair = service.read_private_domain_pair(member.instance_uuid) + pair = { + "inactive": configured.get("xml"), + "live": configured.get("live"), + "seed_verified": configured.get("seed_verified") is True, + } + if configured.get("live") is None: + pair["guest_network"] = { + "state": "stopped", "verified": False, + "reason_code": "vpc_private_guest_stopped", + "observed_at": int(time.time()), + } + if ledger_state: + probe_members.append(member) + elif configured.get("restart_required") is True: + pair["guest_network"] = { + "state": "pending_restart", "verified": False, + "reason_code": "vpc_private_guest_restart_required", + "observed_at": int(time.time()), + } + if ledger_state: + probe_members.append(member) + else: + # This placeholder is deliberately fail-closed. A completed + # QGA future replaces it; an unsubmitted/cancelled future + # remains explicit deadline evidence while preserving the + # persistent/live XML already read back by the transaction. + pair["guest_network"] = { + "state": "deadline_exceeded", "verified": False, + "reason_code": + "vpc_private_guest_probe_deadline_exceeded", + "observed_at": int(time.time()), + } + probe_members.append(member) + pairs[member.instance_uuid] = pair + + def read_member(member: VpcMember) -> tuple[VpcMember, dict[str, Any]]: + ledger_state, persist_ledger, clear_ledger = ( + guest_exec_context(member)) + configured = pairs.get(member.instance_uuid, {}) + + def deadline_pair() -> dict[str, Any]: + return { + "inactive": configured.get("inactive"), + "live": configured.get("live"), + "seed_verified": bool(configured.get("seed_verified")), + "guest_network": { + "state": "deadline_exceeded", "verified": False, + "reason_code": + "vpc_private_guest_probe_deadline_exceeded", + "observed_at": int(time.time()), + }, + } + + remaining = max(0.0, request_deadline - time.monotonic()) + acquired = _VPC_GUEST_PROBE_SLOTS.acquire(timeout=remaining) + if not acquired: + return member, deadline_pair() + try: + # QGA rounds its command timeout to whole seconds. Do not + # enter libvirt after waiting for a slot if less than one + # useful command second remains in the shared request budget. + if request_deadline - time.monotonic() < 1: + return member, deadline_pair() + pair = service.read_private_domain_pair( + member.instance_uuid, + private_mac=member.private_mac, + private_ip_cidr=f"{member.private_ip}/32", + private_routes=self._private_routes(plan, member), + guest_probe_deadline=request_deadline, + guest_exec_state=ledger_state, + guest_exec_persist=persist_ledger, + guest_exec_clear=clear_ledger, + ) + pair["seed_verified"] = bool( + configured.get("seed_verified")) + return member, pair + finally: + _VPC_GUEST_PROBE_SLOTS.release() + + # Keep at most one worker-width of QGA calls submitted. This avoids a + # 4,096-future queue, while the process-wide semaphore prevents pools + # belonging to concurrent VPC requests from multiplying that width. + # Both gates share the request deadline, and every unstarted member is + # explicit fail-closed evidence. Shutdown waits for bounded calls, so + # no probe thread survives this request. + if probe_members: + workers = min(VPC_GUEST_PROBE_WORKERS, len(probe_members)) + executor = ThreadPoolExecutor( + max_workers=workers, + thread_name_prefix="vpc-guest-probe") + futures: dict[Any, VpcMember] = {} + next_member = 0 + probe_error: BaseException | None = None + + def submit_available() -> None: + nonlocal next_member + while (len(futures) < workers + and next_member < len(probe_members) + and request_deadline - time.monotonic() >= 1): + member = probe_members[next_member] + next_member += 1 + futures[executor.submit(read_member, member)] = member + + try: + submit_available() + while futures: + remaining = request_deadline - time.monotonic() + if remaining <= 0: + break + done, _pending = wait( + tuple(futures), timeout=remaining, + return_when=FIRST_COMPLETED) + if not done: + break + for future in done: + futures.pop(future, None) + try: + member, pair = future.result() + except BaseException as exc: + probe_error = exc + break + pairs[member.instance_uuid] = pair + if probe_error is not None: + break + submit_available() + finally: + for future in futures: + future.cancel() + executor.shutdown(wait=True, cancel_futures=True) + if probe_error is not None: + raise probe_error + + xml_by_instance: dict[str, Mapping[str, Any]] = {} + for member in local_members: + pair = pairs[member.instance_uuid] inactive = pair.get("inactive") if not isinstance(inactive, str): raise VpcError("私网网卡持久配置回读缺失", 502, @@ -1605,6 +1992,8 @@ class VpcCoordinator: "vpc_private_live_readback_missing") xml_by_instance[member.instance_uuid] = { "inactive": inactive, "live": live, + "guest_network": pair.get("guest_network"), + "seed_verified": pair.get("seed_verified") is True, } observed = adapter.observe(dataplane, xml_by_instance=xml_by_instance) observed.update({ @@ -1882,6 +2271,7 @@ class VpcCoordinator: operation_id: str | None = None, ) -> None: """Apply/read back every participating host; stay closed on failure.""" + request_deadline = time.monotonic() + VPC_REQUEST_DEADLINE_SECONDS retiring_ids = { item.get("host_id") for item in record.get("hosts", []) if isinstance(item, Mapping) @@ -1957,6 +2347,17 @@ class VpcCoordinator: if not isinstance(host, dict): continue host_id = host.get("host_id") + if time.monotonic() >= request_deadline: + host.update({ + "state": "uncertain", + "transport_verified": False, + "private_dataplane_verified": False, + "observed_revision": 0, + "observed_digest": None, + "last_error_code": + "vpc_private_guest_probe_deadline_exceeded", + }) + continue if not isinstance(host_id, str) or host_id not in components: host.update({"state": "uncertain", "transport_verified": False, @@ -1965,11 +2366,37 @@ class VpcCoordinator: continue component, profile = components[host_id] try: + # Persist a fair retry order. If this host consumes the shared + # budget, never-probed/older hosts sort ahead of it on the + # next reconcile even when this runtime proof stayed pending. + host["probe_attempted_at_ns"] = time.time_ns() observed = await asyncio.to_thread( self._run_host_dataplane, plan, host_id, component, profile, all_peers, previous_plan, host.get("retiring_members", []), fabric_public_keys, - private_link_states) + private_link_states, + request_deadline=request_deadline) + if time.monotonic() >= request_deadline: + member_nics = observed.get("member_nics") + if isinstance(member_nics, list): + for member_nic in member_nics: + if not isinstance(member_nic, dict): + continue + member_nic.update({ + "guest_network_verified": False, + "guest_network_readback": + "deadline_exceeded", + "guest_reason_code": + "vpc_private_guest_probe_deadline_exceeded", + }) + observed.update({ + "state": "applying", + "private_dataplane_verified": False, + "observed_revision": 0, + "observed_digest": None, + "reason_code": + "vpc_private_guest_probe_deadline_exceeded", + }) host.update(observed) host["observed_revision"] = ( plan.desired_revision @@ -1982,10 +2409,15 @@ class VpcCoordinator: if host.get("retiring") is True: host["removed"] = observed.get("removed") is True if (host.get("retiring_members") - and observed.get("state") == "active"): + and observed.get("config_verified") is True + and observed.get("config_observed_revision") + == plan.desired_revision + and observed.get("config_observed_digest") + == plan.desired_digest): # The host adapter has positively removed every member - # listed in this tombstone. Drop only the consumed rows; - # a failed/uncertain pass keeps them retryable. + # listed in this tombstone. Guest runtime readiness for a + # different member is deliberately irrelevant to this + # persistent removal proof. host["retiring_members"] = [] except Exception as exc: host.update({"state": "uncertain", @@ -2072,6 +2504,9 @@ class VpcCoordinator: # tenant VPC. Only the future host adapter may set this # after private0/route/nft/member readback. "private_dataplane_verified": False, + "config_verified": False, + "config_observed_revision": 0, + "config_observed_digest": None, "removed": bool(old.get("removed", False)) if isinstance(old, Mapping) else False, "bridge": old.get("bridge") if isinstance(old, Mapping) else None, "bridge_owner": old.get("bridge_owner") if isinstance(old, Mapping) else None, @@ -2153,6 +2588,9 @@ class VpcCoordinator: "state": "deleting", "transport_verified": bool(gate.get("active")), "private_dataplane_verified": False, + "config_verified": False, + "config_observed_revision": 0, + "config_observed_digest": None, "retiring": True, "retiring_members": old_members, "bridge": None, "bridge_owner": None, "nft_table": None, @@ -2164,6 +2602,7 @@ class VpcCoordinator: }) return { "schema_version": VPC_SCHEMA_VERSION, + "evidence_version": VPC_EVIDENCE_VERSION, "vpc_id": plan.vpc_id, "region": plan.region, "fabric_id": plan.fabric_id, "fabric_revision": plan.fabric_revision, @@ -2199,6 +2638,7 @@ class VpcCoordinator: result: dict[str, Any] = {} for key in ( "vpc_id", "region", "fabric_id", "fabric_revision", "cidr", + "evidence_version", "desired_revision", "desired_digest", "observed_revision", "observed_digest", "state", "next_stage_allowed", "private_dataplane_verified", "operation_id", "updated_at"): @@ -2239,6 +2679,8 @@ class VpcCoordinator: "desired_digest", "bridge", "bridge_owner", "nft_table", "observed_revision", "observed_digest", "state", "transport_verified", "private_dataplane_verified", + "config_verified", "config_observed_revision", + "config_observed_digest", "last_error_code", "removed", "retiring", "anti_spoof", "rp_filter", "acl_revision", "rollback", "proxy_arp", "proxy_arp_pvlan", "proxy_ips", "member_nics", "routes", "wg_allowed_ips") if key in raw} readback = raw.get("readback") @@ -2264,6 +2706,8 @@ class VpcCoordinator: item: dict[str, Any] = {} for key in ("state", "observed_revision", "observed_digest", "transport_verified", "private_dataplane_verified", + "config_verified", "config_observed_revision", + "config_observed_digest", "last_error_code", "vpc_id", "host_id", "desired_revision", "desired_digest", "bridge", "bridge_owner", "nft_table", "anti_spoof", @@ -2296,6 +2740,7 @@ class VpcCoordinator: ) observed_hosts[host_id] = item result["observed"] = { + "evidence_version": value.get("evidence_version", 0), "revision": value.get("observed_revision", 0), "digest": value.get("observed_digest"), "state": value.get("state"), @@ -2424,6 +2869,7 @@ class VpcCoordinator: @classmethod def _finish(cls, record: dict[str, Any], gate: Mapping[str, Any]) -> None: + record["evidence_version"] = VPC_EVIDENCE_VERSION record["fabric_gate"] = dict(gate) # ``recover`` marks a durable record as needing one fresh # control-plane pass. Once the Fabric gate is available again, reset @@ -2446,6 +2892,9 @@ class VpcCoordinator: item["state"] = "applying" item["transport_verified"] = True item["private_dataplane_verified"] = False + item["config_verified"] = False + item["config_observed_revision"] = 0 + item["config_observed_digest"] = None hosts = record.get("hosts") desired_revision = record.get("desired_revision") desired_digest = record.get("desired_digest") @@ -2493,6 +2942,24 @@ class VpcCoordinator: record["state"] = "degraded" if gate.get("available") else "uncertain" record["next_stage_allowed"] = False record["private_dataplane_verified"] = False + for item in hosts if isinstance(hosts, list) else []: + if not isinstance(item, dict): + continue + if (isinstance(desired_revision, int) + and isinstance(desired_digest, str) + and cls._confirmed_removal( + item, desired_revision, desired_digest)): + continue + # Fabric reachability is a prerequisite for both shared host + # configuration and member runtime usability. Never let a + # prior active observation survive an inactive current gate. + item["transport_verified"] = False + item["private_dataplane_verified"] = False + item["config_verified"] = False + item["observed_revision"] = 0 + item["observed_digest"] = None + item["config_observed_revision"] = 0 + item["config_observed_digest"] = None elif hosts and any(item.get("state") == "uncertain" for item in hosts if isinstance(item, Mapping)): record["state"] = "uncertain" record["next_stage_allowed"] = False @@ -2514,24 +2981,73 @@ class VpcCoordinator: extra_identities: Sequence[tuple[str, str]] | None = None, operation_id: str | None = None, ) -> None: - """Retry only hosts without an exact durable absence proof.""" + """Retry only hosts without an exact durable runtime/removal proof.""" self._finish(record, gate) durable_hosts = self._host_closure(record) member_host_ids = {member.host_id for member in plan.members} + targeted_host_ids = { + member.host_id for member in plan.members + if (private_link_states is not None + and member.instance_uuid in private_link_states) + } + if extra_identities is not None: + targeted_host_ids.update( + host_id for host_id, _instance_uuid in extra_identities) pending_host_ids = { host_id for host_id, host in durable_hosts.items() - if (host_id in member_host_ids - or not self._confirmed_removal( - host, plan.desired_revision, plan.desired_digest)) + if ((host_id in member_host_ids + and (host_id in targeted_host_ids + or not self._confirmed_active_host(host, plan))) + or (host_id not in member_host_ids + and not self._confirmed_removal( + host, plan.desired_revision, plan.desired_digest))) } if not pending_host_ids: - return + def attempted_at(host: Mapping[str, Any]) -> int: + value = host.get("probe_attempted_at_ns", 0) + if (isinstance(value, bool) + or not isinstance(value, int) or value < 0): + return 0 + return value + + # Exact guest runtime evidence is not timeless: QGA, its route, + # or the guest address may disappear after the durable host row + # became active. Pending and lifecycle-targeted work remains the + # priority; only an otherwise idle periodic pass spends its + # bounded budget revalidating one oldest active member host. + oldest_active = min( + ( + (attempted_at(host), host_id) + for host_id, host in durable_hosts.items() + if (host_id in member_host_ids + and self._confirmed_active_host(host, plan)) + ), + default=None, + ) + if oldest_active is None: + return + pending_host_ids.add(oldest_active[1]) execution_record = dict(record) - execution_record["hosts"] = [ + execution_hosts = [ dict(host) for host in record.get("hosts", []) if (isinstance(host, Mapping) and host.get("host_id") in pending_host_ids) ] + def attempt_order(host: Mapping[str, Any]) -> tuple[int, int, str]: + attempted_at = host.get("probe_attempted_at_ns", 0) + if (isinstance(attempted_at, bool) + or not isinstance(attempted_at, int) + or attempted_at < 0): + attempted_at = 0 + host_id = str(host.get("host_id", "")) + # A lifecycle request (suspend/resume/delete) must not sit behind + # unrelated periodic guest probes. Fair rotation applies within + # the targeted and ordinary groups independently. + return (0 if host_id in targeted_host_ids else 1, + attempted_at, host_id) + + execution_hosts.sort(key=attempt_order) + execution_record["hosts"] = execution_hosts reconcile_kwargs: dict[str, Any] = {} if private_link_states is not None: reconcile_kwargs["private_link_states"] = private_link_states @@ -2588,6 +3104,7 @@ class VpcCoordinator: if previous is None and not plan.host_ids: receipt = { "schema_version": VPC_SCHEMA_VERSION, + "evidence_version": VPC_EVIDENCE_VERSION, "vpc_id": vpc_id, "operation_id": plan.operation_id, "desired_revision": plan.desired_revision, @@ -2858,6 +3375,7 @@ class VpcCoordinator: receipt = { "schema_version": VPC_SCHEMA_VERSION, + "evidence_version": VPC_EVIDENCE_VERSION, "vpc_id": vpc_id, "operation_id": plan.operation_id, "desired_revision": plan.desired_revision, @@ -3058,6 +3576,8 @@ class VpcCoordinator: raise VpcError("VPC readback 无效") typed_fields = ( "vpc_id", "host_id", "desired_revision", "desired_digest", + "config_verified", "config_observed_revision", + "config_observed_digest", "bridge", "bridge_owner", "nft_table", "anti_spoof", "rp_filter", "proxy_arp", "proxy_arp_pvlan", "proxy_ips", "acl_revision", "rollback", "removed", "member_nics", "routes", "wg_allowed_ips", @@ -3108,13 +3628,30 @@ class VpcCoordinator: host.get("observed_revision", 0) if isinstance(host, Mapping) else 0) observed_digest = ( host.get("observed_digest") if isinstance(host, Mapping) else None) - host_ok = bool( + config_observed_revision = ( + host.get("config_observed_revision", 0) + if isinstance(host, Mapping) else 0) + config_observed_digest = ( + host.get("config_observed_digest") + if isinstance(host, Mapping) else None) + host_runtime_ok = bool( isinstance(host, Mapping) and host.get("state") == "active" and host.get("private_dataplane_verified") is True and observed_revision == desired_revision and observed_digest == desired_digest ) + host_config_ok = bool( + isinstance(host, Mapping) + and host.get("transport_verified") is True + and host.get("config_verified") is True + and config_observed_revision == desired_revision + and config_observed_digest == desired_digest + ) + # Host configuration is shared, while guest readiness is per member. + # A stopped or unobservable sibling must not erase a healthy member's + # independently confirmed runtime state. + host_ok = host_runtime_ok or host_config_ok desired_mac: str | None = None if (required and desired_state != "removed" and isinstance(vpc_id, str) and isinstance(member_id, str)): @@ -3128,22 +3665,59 @@ class VpcCoordinator: # guessed address. desired_mac = None observed_mac: str | None = None + guest_mac: str | None = None + guest_ip: str | None = None if isinstance(member_nic, Mapping): try: observed_mac = _mac(member_nic.get("private_mac")) except VpcError: observed_mac = None - nic_ok = bool( + try: + guest_mac = _mac(member_nic.get("guest_private_mac")) + except VpcError: + guest_mac = None + if isinstance(member_nic.get("guest_private_ip_cidr"), str): + guest_ip = member_nic.get("guest_private_ip_cidr") + active_nic_ok = bool( isinstance(member_nic, Mapping) and member_nic.get("xml_verified") is True + and member_nic.get("seed_verified") is True + and member_nic.get("restart_required") is False + and member_nic.get("domain_running") is True + and member_nic.get("guest_network_verified") is True + and member_nic.get("guest_link_state") == "up" and desired_mac is not None and observed_mac == desired_mac + and guest_mac == desired_mac + and guest_ip == private_ip_cidr ) link_state = ( member_nic.get("link_state") if isinstance(member_nic, Mapping) else "absent") if link_state not in {"up", "down"}: link_state = "absent" + suspended_nic_ok = bool( + isinstance(member_nic, Mapping) + and member_nic.get("xml_verified") is True + and member_nic.get("config_verified") is True + and member_nic.get("seed_verified") is True + and member_nic.get("restart_required") is False + and member_nic.get("domain_running") is True + and link_state == "down" + and desired_mac is not None + and observed_mac == desired_mac + ) + stopped_nic_ok = bool( + isinstance(member_nic, Mapping) + and member_nic.get("xml_verified") is True + and member_nic.get("config_verified") is True + and member_nic.get("seed_verified") is True + and member_nic.get("restart_required") is False + and member_nic.get("domain_running") is False + and member_nic.get("private_ip_cidr") == private_ip_cidr + and desired_mac is not None + and observed_mac == desired_mac + ) if not required: state = "disabled" if not isinstance(member_nic, Mapping) else "uncertain" return { @@ -3158,6 +3732,12 @@ class VpcCoordinator: "reason_code": None if state == "disabled" else reason_code or "vpc_private_observation_mismatch", "observed_at": int(time.time()), + "xml_verified": False, "config_verified": False, + "seed_verified": False, "domain_running": False, + "restart_required": False, + "guest_network_verified": False, + "guest_network_readback": "missing", + "guest_link_state": None, } removed = desired_state == "removed" if removed: @@ -3169,17 +3749,29 @@ class VpcCoordinator: mac = None readback = "confirmed" if state == "removed" else "missing" else: - shape_ok = nic_ok - state = ("suspended" if link_state == "down" else "active") \ - if host_ok and shape_ok else "uncertain" + # An intentional live link-down is an isolation state, not a + # usable-network claim. Exact persistent/live XML plus the host + # config proof is sufficient to confirm suspension; requiring a + # guest link-up probe here would make network_suspend impossible. + shape_ok = (stopped_nic_ok if stopped_nic_ok + else suspended_nic_ok if link_state == "down" + else active_nic_ok) + state = ( + "stopped" if stopped_nic_ok + else "suspended" if link_state == "down" else "active" + ) if host_ok and shape_ok else "uncertain" interface = "private0" if shape_ok else None link = link_state - ip = private_ip_cidr if shape_ok else None + ip = (private_ip_cidr + if stopped_nic_ok or suspended_nic_ok else guest_ip) \ + if shape_ok else None # Return the value actually present in the bounded host # observation. Equality with the derived/configured target above # is part of ``shape_ok``; this prevents a non-empty but stale # desired MAC from masking host XML drift. - mac = observed_mac if shape_ok else None + mac = (observed_mac + if stopped_nic_ok or suspended_nic_ok else guest_mac) \ + if shape_ok else None readback = "confirmed" if host_ok and shape_ok else "missing" return { "version": 1, "required": True, "vm_uuid": instance_uuid, @@ -3187,15 +3779,50 @@ class VpcCoordinator: "interface": interface, "state": state, "desired_state": desired_state, "desired_revision": desired_revision, - "observed_revision": observed_revision if host_ok else 0, + "observed_revision": ( + observed_revision if host_runtime_ok + else config_observed_revision if host_config_ok else 0), "desired_digest": desired_digest, - "observed_digest": observed_digest if host_ok else None, + "observed_digest": ( + observed_digest if host_runtime_ok + else config_observed_digest if host_config_ok else None), "readback": readback, "link_state": link, "private_ip_cidr": ip, "private_mac": mac, - "reason_code": (None if readback == "confirmed" else reason_code + "reason_code": ( + "vpc_private_guest_stopped" + if state == "stopped" else + None if readback == "confirmed" else reason_code or ("vpc_private_teardown_uncertain" if removed else "vpc_private_observation_mismatch")), "observed_at": int(time.time()), + "xml_verified": ( + member_nic.get("xml_verified") is True + if isinstance(member_nic, Mapping) else False), + "config_verified": ( + member_nic.get("config_verified") is True + if isinstance(member_nic, Mapping) else False), + "seed_verified": ( + member_nic.get("seed_verified") is True + if isinstance(member_nic, Mapping) else False), + "domain_running": ( + member_nic.get("domain_running") is True + if isinstance(member_nic, Mapping) else False), + "restart_required": ( + member_nic.get("restart_required") is True + if isinstance(member_nic, Mapping) else False), + "guest_network_verified": ( + member_nic.get("guest_network_verified") is True + if isinstance(member_nic, Mapping) else False), + "guest_network_readback": ( + str(member_nic.get("guest_network_readback"))[:32] + if isinstance(member_nic, Mapping) + and isinstance(member_nic.get("guest_network_readback"), str) + else "missing"), + "guest_link_state": ( + member_nic.get("guest_link_state") + if isinstance(member_nic, Mapping) + and member_nic.get("guest_link_state") in {"up", "down"} + else None), } async def reconcile_instance( @@ -3220,9 +3847,31 @@ class VpcCoordinator: expected_vpc = expected.get("vpc_id") expected_member = expected.get("member_id") expected_state = expected.get("desired_state") + lifecycle_kind = expected.get("lifecycle_kind") expected_revision = expected.get("desired_revision") expected_digest = expected.get("desired_digest") expected_operation_id = expected.get("operation_id") + if lifecycle_kind is None: + lifecycle_kind = "legacy" + elif (not isinstance(lifecycle_kind, str) + or _PRIVATE_LIFECYCLE_KIND.fullmatch(lifecycle_kind) is None): + raise VpcError("私网生命周期类型无效", 422, + "vpc_private_lifecycle_kind_invalid") + lifecycle_link_invalid = ( + (lifecycle_kind in {"delete", "expiry_delete"} + and link_state != "absent") + or (lifecycle_kind == "network_suspend" + and link_state not in {"down", "preserve"}) + or (lifecycle_kind == "network_resume" + and link_state not in {"up", "preserve"}) + or (lifecycle_kind not in { + "delete", "expiry_delete", "network_suspend", + "network_resume"} + and link_state != "preserve") + ) + if lifecycle_kind != "legacy" and lifecycle_link_invalid: + raise VpcError("私网生命周期策略与链路目标不一致", 422, + "vpc_private_lifecycle_policy_invalid") if expected_vpc is not None and not isinstance(expected_vpc, str): raise VpcError("私网 VPC 标识无效", 422) if expected_member is not None and not isinstance(expected_member, str): @@ -3416,8 +4065,11 @@ class VpcCoordinator: desired_revision=plan.desired_revision, desired_digest=plan.desired_digest, host=None, member_nic=None, reason_code="vpc_private_expectation_mismatch") - link_map = {} if link_state in {"preserve", "absent"} else { - instance_uuid: link_state} + # Keep the target UUID in the map even for preserve. The + # host lane uses membership to prioritize this lifecycle VM + # ahead of a large periodic member batch without changing its + # link policy. + link_map = {instance_uuid: link_state} gate = self._fabric_gate(plan) reconcile_kwargs: dict[str, Any] = { "private_link_states": link_map, @@ -3452,7 +4104,7 @@ class VpcCoordinator: and item.get("instance_uuid") == instance_uuid): nic = item break - return self._instance_private_state( + result = self._instance_private_state( instance_uuid, required=True, vpc_id=plan.vpc_id, member_id=(desired.member_id if desired is not None else target_row.get("member_id") if target_row else expected_member), @@ -3466,6 +4118,31 @@ class VpcCoordinator: reason_code=(host_row.get("last_error_code") if isinstance(host_row, Mapping) else None), ) + if result.get("readback") == "confirmed": + required_link: str | None = None + if link_state == "down": + required_link = "down" + elif link_state == "up": + required_link = "up" + elif link_state == "absent": + required_link = "absent" + elif lifecycle_kind.endswith("_policy_active"): + required_link = "up" + elif lifecycle_kind.endswith("_policy_blocked"): + required_link = "down" + policy_unready = lifecycle_kind.endswith("_policy_unready") + if (policy_unready + or required_link is not None + and result.get("link_state") != required_link): + result.update({ + "state": "uncertain", "readback": "mismatch", + "interface": None, "private_ip_cidr": None, + "private_mac": None, + "reason_code": + "vpc_private_lifecycle_policy_mismatch", + "observed_at": int(time.time()), + }) + return result async def reconcile(self, vpc_id: str) -> dict[str, Any]: _identifier(vpc_id, "VPC") diff --git a/engine/virsh_panel/vpc_dataplane.py b/engine/virsh_panel/vpc_dataplane.py index a3c68eee..b983d66c 100644 --- a/engine/virsh_panel/vpc_dataplane.py +++ b/engine/virsh_panel/vpc_dataplane.py @@ -13,6 +13,7 @@ import hashlib import ipaddress import re import shlex +import time import xml.etree.ElementTree as ET from dataclasses import dataclass from typing import Any, Mapping, Sequence @@ -135,6 +136,9 @@ class VpcDataplaneObservation: proxy_arp: int | None proxy_arp_pvlan: int | None proxy_ips: tuple[str, ...] + config_verified: bool + config_observed_revision: int + config_observed_digest: str | None readback: str rollback: str reason_code: str | None = None @@ -157,6 +161,9 @@ class VpcDataplaneObservation: "proxy_arp": self.proxy_arp, "proxy_arp_pvlan": self.proxy_arp_pvlan, "proxy_ips": list(self.proxy_ips), + "config_verified": self.config_verified, + "config_observed_revision": self.config_observed_revision, + "config_observed_digest": self.config_observed_digest, "readback": self.readback, "rollback": self.rollback, "reason_code": self.reason_code, "removed": self.removed, @@ -203,6 +210,77 @@ def derive_private_mac(vpc_id: str, member_id: str) -> str: return ":".join(f"{x:02x}" for x in bytes([first]) + digest[1:6]) +def private_guest_routes(plan: VpcPlan, member: VpcMember) -> tuple[str, ...]: + """Return the stable on-link VPC prefix installed inside one guest. + + Host ACL, nftables and Fabric changes deliberately do not participate in + this projection. The host keeps exact member/service /32 routes, proxy + neighbours and destination sets; the guest needs only one link-scope VPC + prefix. Member and service-route churn therefore cannot force every + running sibling to consume a new boot-only NoCloud generation. + """ + try: + network = ipaddress.ip_network(plan.cidr, strict=True) + private_ip = ipaddress.ip_address(member.private_ip) + except (TypeError, ValueError) as exc: + raise VpcDataplaneError("VPC 来宾私网路由无效") from exc + if (not isinstance(network, ipaddress.IPv4Network) + or not 16 <= network.prefixlen <= 24 + or not isinstance(private_ip, ipaddress.IPv4Address) + or private_ip not in network + or not any(network.subnet_of(block) for block in ( + ipaddress.IPv4Network("10.0.0.0/8"), + ipaddress.IPv4Network("172.16.0.0/12"), + ipaddress.IPv4Network("192.168.0.0/16"), + ))): + raise VpcDataplaneError("VPC 来宾私网路由无效") + return (str(network),) + + +def private_guest_network_generation( + member: VpcMember, + private_routes: Sequence[str], +) -> str: + """Hash only the private network state that a guest consumes at boot.""" + private_mac = _mac(member.private_mac) + try: + private_ip = ipaddress.ip_interface(f"{member.private_ip}/32") + except (TypeError, ValueError) as exc: + raise VpcDataplaneError("VPC 成员私网 IP 无效") from exc + if not isinstance(private_ip, ipaddress.IPv4Interface): + raise VpcDataplaneError("VPC 成员私网 IP 无效") + routes: list[ipaddress.IPv4Network] = [] + if (not isinstance(private_routes, Sequence) + or isinstance(private_routes, (str, bytes)) + or len(private_routes) != 1): + raise VpcDataplaneError("VPC 来宾私网路由无效") + for value in private_routes: + if not isinstance(value, str): + raise VpcDataplaneError("VPC 来宾私网路由无效") + try: + route = ipaddress.ip_network(value, strict=True) + except ValueError as exc: + raise VpcDataplaneError("VPC 来宾私网路由无效") from exc + if (not isinstance(route, ipaddress.IPv4Network) + or not 16 <= route.prefixlen <= 24 + or private_ip.ip not in route + or not any(route.subnet_of(block) for block in ( + ipaddress.IPv4Network("10.0.0.0/8"), + ipaddress.IPv4Network("172.16.0.0/12"), + ipaddress.IPv4Network("192.168.0.0/16"), + ))): + raise VpcDataplaneError( + "VPC 来宾私网路由必须是包含成员地址的 IPv4 /16 到 /24") + routes.append(route) + payload = "\n".join(( + "virt-cloud-private-guest-network-v2", + private_mac, + str(private_ip), + str(routes[0]), + )).encode("ascii") + return hashlib.sha256(payload).hexdigest() + + def resource_names(vpc_id: str) -> VpcDataplaneResources: vpc_id = _id(vpc_id, "VPC") digest = hashlib.sha256(vpc_id.encode()).hexdigest() @@ -551,7 +629,8 @@ def render_vpc_nft_rules(plan: VpcDataplanePlan) -> str: # VPC destination set; restricting it to local routes would drop every # same-VPC cross-host flow before the inet routing policy can run. local_destination_cidrs = set(destination_cidrs) - # Use explicit proxy-neighbour entries for the guest's routed /32 model. + # Use explicit proxy-neighbour entries for the host's exact /32 routes to + # guest addresses. # The generic kernel proxy responder stays disabled, so a nftables reload # or a short ordering window cannot make the host answer for arbitrary # addresses. proxy_arp_pvlan is enabled only for the same-bridge @@ -672,6 +751,11 @@ def render_vpc_nft_rules(plan: VpcDataplanePlan) -> str: f' meta ibrname "{r.bridge}" ether type arp arp saddr ether . arp saddr ip != @local_member_sources drop comment "{marker}:bridge-anti-spoof-arp"', f' meta ibrname "{r.bridge}" ether type ip ip saddr != @local_member_ips drop comment "{marker}:bridge-anti-spoof-ip"', f' meta ibrname "{r.bridge}" ether type arp arp saddr ip != @local_member_ips drop comment "{marker}:bridge-anti-spoof-arp-ip"', + # A stable VPC-prefix route lets a guest ask for every address in + # that prefix. Only exact member/service destinations may reach + # another tap or the host proxy responder; unknown ARP targets are + # dropped before they become a bridge-wide scan/flood primitive. + f' meta ibrname "{r.bridge}" ether type arp arp operation request arp daddr ip != @local_dest_ips drop comment "{marker}:bridge-unknown-arp-target-drop"', # This is an L3 /32 fabric. Do not let broadcast or multicast # discovery turn the private bridge back into a shared L2 LAN. f' meta ibrname "{r.bridge}" ip daddr 224.0.0.0/4 drop comment "{marker}:bridge-multicast-drop"', @@ -2601,6 +2685,7 @@ def _markers(output: str) -> dict[str, Any]: def patch_private_interface_xml(xml: str | bytes, *, member: VpcMember, resources: VpcDataplaneResources, + guest_network_generation: str, link_state: str = "up", vpc_id: str | None = None, desired_revision: int | None = None, @@ -2608,6 +2693,9 @@ def patch_private_interface_xml(xml: str | bytes, *, member: VpcMember, """Add/validate a private0 NIC on an explicit panel-owned bridge.""" if link_state not in {"up", "down"}: raise VpcDataplaneError("私网网卡 link 状态无效") + if (not isinstance(guest_network_generation, str) + or _DIGEST.fullmatch(guest_network_generation) is None): + raise VpcDataplaneError("来宾私网 generation 无效") private_mac = _mac(member.private_mac) try: root = ET.fromstring(xml) @@ -2680,6 +2768,8 @@ def patch_private_interface_xml(xml: str | bytes, *, member: VpcMember, managed.set("private-network-bridge", resources.bridge) managed.set("private-network-verified", "true") managed.set("private-dataplane-owner", DATAPLANE_OWNER_MARKER) + managed.set("private-guest-network-generation", + guest_network_generation) if vpc_id is not None: managed.set("private-vpc-id", vpc_id) if desired_revision is not None: @@ -2694,7 +2784,11 @@ def _member_nics( plan: VpcDataplanePlan, xml_by_instance: Mapping[str, Any] | None, ) -> tuple[dict[str, Any], ...]: - def xml_details(xml: str, member: VpcMember, *, metadata: bool) -> tuple[bool, str | None]: + def xml_details( + xml: str, + member: VpcMember, + expected_generation: str, + ) -> tuple[bool, bool, bool, str | None]: try: root = ET.fromstring(xml) private = select_private_interface( @@ -2715,57 +2809,151 @@ def _member_nics( link = private.find("link") if private is not None else None link_state = link.get("state", "up").lower() if link is not None else "up" if link_state not in {"up", "down"}: - return False, None - if not metadata: - return ok, link_state + return False, False, False, None managed = root.find(f"./metadata/{{{PANEL_NS}}}managed") - return bool( - ok and managed is not None + guest_generation_ok = bool( + managed is not None and managed.get("private-vpc-id") == plan.vpc.vpc_id and managed.get("private-member-id") == member.member_id and managed.get("private-ip-cidr") == f"{member.private_ip}/32" + and managed.get("private-network-bridge") + == plan.resources.bridge + and managed.get("private-dataplane-owner") + == DATAPLANE_OWNER_MARKER + and managed.get("private-guest-network-generation") + == expected_generation + ) + desired_metadata_ok = bool( + guest_generation_ok and managed.get("private-network-revision") == str(plan.desired_revision) and managed.get("private-network-digest") == plan.desired_digest - ), link_state + ) + return ok, guest_generation_ok, desired_metadata_ok, link_state except (ET.ParseError, ValueError): - return False, None + return False, False, False, None result = [] for member in plan.local_members: verified = False + config_verified = False + seed_verified = False + restart_required = False + guest_verified = False + guest_readback = "missing" + guest_reason_code = "vpc_private_guest_agent_unavailable" + guest_private_mac: str | None = None + guest_private_ip_cidr: str | None = None + guest_link_state: str | None = None + guest_routes: list[str] = [] + domain_running = False link_state: str | None = None + expected_routes = private_guest_routes(plan.vpc, member) + expected_generation = private_guest_network_generation( + member, expected_routes) if xml_by_instance and member.instance_uuid in xml_by_instance: value = xml_by_instance[member.instance_uuid] if isinstance(value, Mapping): + seed_verified = value.get("seed_verified") is True inactive = value.get("inactive") live = value.get("live") inactive_ok = False - live_ok = True + live_device_ok = False + live_generation_ok = False inactive_link: str | None = None live_link: str | None = None if isinstance(inactive, str): - inactive_ok, inactive_link = xml_details( - inactive, member, metadata=True) - if live is not None: + (inactive_device_ok, inactive_generation_ok, + inactive_metadata_ok, inactive_link) = xml_details( + inactive, member, expected_generation) + inactive_ok = bool( + inactive_device_ok and inactive_generation_ok + and inactive_metadata_ok) + config_verified = inactive_ok + domain_running = live is not None + if domain_running: if isinstance(live, str): - live_ok, live_link = xml_details( - live, member, metadata=False) - else: - live_ok = False - # A running domain is only healthy when the persistent and - # live role agree. This makes a stale config-only link state - # visible to the lifecycle gate instead of silently choosing - # whichever XML happened to be read first. - verified = inactive_ok and live_ok and ( - live is None or inactive_link == live_link) - link_state = live_link if live is not None else inactive_link + (live_device_ok, live_generation_ok, + _live_metadata_ok, live_link) = xml_details( + live, member, expected_generation) + guest = value.get("guest_network") + if isinstance(guest, Mapping): + guest_readback = str(guest.get("state", "missing"))[:32] + reason = guest.get("reason_code") + if isinstance(reason, str) and re.fullmatch( + r"[a-z0-9_]{1,64}", reason): + guest_reason_code = reason + observed_routes = guest.get("routes") + if (isinstance(observed_routes, list) + and len(observed_routes) <= 4096 + and all(isinstance(route, str) + for route in observed_routes)): + guest_routes = sorted(set(observed_routes)) + observed_mac = guest.get("private_mac") + observed_ip = guest.get("private_ip_cidr") + guest_private_mac = ( + observed_mac.lower() + if isinstance(observed_mac, str) else None) + guest_private_ip_cidr = ( + observed_ip if isinstance(observed_ip, str) else None) + guest_link_state = ( + guest.get("link_state") + if guest.get("link_state") in {"up", "down"} + else None) + guest_verified = bool( + guest.get("verified") is True + and guest_readback == "confirmed" + and guest.get("source") == "qemu_guest_agent" + and guest_link_state == "up" + and guest_private_mac == member.private_mac + and guest_private_ip_cidr == f"{member.private_ip}/32" + and guest_routes == list(expected_routes) + ) + if guest_verified: + guest_reason_code = None + # Persistent and live libvirt device identity remains a + # prerequisite, but only a bounded guest-agent observation can + # prove that the address and routes exist inside the guest. + stopped_exact = bool( + inactive_ok and seed_verified and not domain_running) + verified = bool( + stopped_exact + or (inactive_ok and seed_verified and domain_running + and live_device_ok + and inactive_link == live_link)) + restart_required = bool( + not stopped_exact and ( + not domain_running + or not inactive_ok + or not seed_verified + or not live_generation_ok)) + if stopped_exact: + guest_readback = "stopped" + guest_reason_code = "vpc_private_guest_stopped" + if restart_required: + guest_reason_code = "vpc_private_guest_restart_required" + link_state = live_link if domain_running else inactive_link elif isinstance(value, str): - verified, link_state = xml_details(value, member, metadata=True) + (device_ok, generation_ok, metadata_ok, + link_state) = xml_details( + value, member, expected_generation) + verified = device_ok and generation_ok and metadata_ok + config_verified = verified result.append({"member_id": member.member_id, "instance_uuid": member.instance_uuid, "interface": PRIVATE_INTERFACE, "private_ip_cidr": f"{member.private_ip}/32", "private_mac": member.private_mac, "xml_verified": verified, + "config_verified": config_verified, + "seed_verified": seed_verified, + "restart_required": restart_required, + "domain_running": domain_running, + "guest_network_verified": guest_verified, + "guest_network_readback": guest_readback, + "guest_reason_code": guest_reason_code, + "guest_private_mac": guest_private_mac, + "guest_private_ip_cidr": guest_private_ip_cidr, + "guest_link_state": guest_link_state, + "guest_routes": guest_routes, "link_state": link_state}) return tuple(result) @@ -2773,9 +2961,16 @@ def _member_nics( class VpcPrivateHostAdapter: """Execute generated scripts through a pinned remote host manager.""" - def __init__(self, manager: Any, profile: Mapping[str, Any] | None = None): + def __init__( + self, + manager: Any, + profile: Mapping[str, Any] | None = None, + *, + request_deadline: float | None = None, + ): self.manager = manager self.profile = dict(profile) if isinstance(profile, Mapping) else None + self.request_deadline = request_deadline def _run(self, script: str) -> str: current = self.profile if self.profile is not None else self.manager.current() @@ -2783,9 +2978,18 @@ class VpcPrivateHostAdapter: raise VpcDataplaneError("宿主机尚未配置", 409, "vpc_host_not_configured") if current.get("key_locked"): raise VpcDataplaneError("宿主机 SSH 私钥仍锁定", 409, "vpc_host_key_locked") + timeout = 35 + if self.request_deadline is not None: + remaining = self.request_deadline - time.monotonic() + if remaining < 1: + raise VpcDataplaneError( + "VPC 来宾网络核验超过请求预算", 502, + "vpc_private_guest_probe_deadline_exceeded") + timeout = min(timeout, int(remaining)) argv = [getattr(self.manager, "ssh_binary", "ssh"), "-F", current["ssh_config_file"], "-T", "-o", "ConnectTimeout=10", current["ssh_target"], "/bin/sh -s"] try: - result = self.manager._exec(argv, timeout=35, input_text=script) + result = self.manager._exec( + argv, timeout=timeout, input_text=script) except Exception as exc: raise VpcDataplaneError("宿主机 VPC SSH 请求失败", 502, "vpc_host_connection_failed") from exc if result.returncode != 0: @@ -2866,6 +3070,9 @@ class VpcPrivateHostAdapter: "observed_revision": plan.desired_revision, "desired_digest": plan.desired_digest, "observed_digest": plan.desired_digest, + "config_verified": True, + "config_observed_revision": plan.desired_revision, + "config_observed_digest": plan.desired_digest, "state": "active", "removed": True, "bridge": None, @@ -2960,7 +3167,7 @@ class VpcPrivateHostAdapter: } expected_peer_states = {p.host_id: "owned" for p in plan.peers} expected_bridge_nft = "yes" if plan.local_members else "none" - active = (parsed.get("bridge") == plan.resources.bridge + host_config_verified = (parsed.get("bridge") == plan.resources.bridge and parsed.get("bridge_owner") == DATAPLANE_OWNER_MARKER and parsed.get("persistence") == "yes" and parsed.get("bridge_boundary") == "yes" @@ -2987,7 +3194,22 @@ class VpcPrivateHostAdapter: or parsed.get("wg_peer_set") == "exact") and expected_peer_states == parsed.get("wg_states", {}) and parsed.get("arp_nft") == ("yes" if plan.local_members else "none") - and all(x["xml_verified"] for x in members)) + and all(x.get("config_verified") is True for x in members)) + active = bool( + host_config_verified + and all( + x["xml_verified"] + and x.get("guest_network_verified") is True + for x in members + ) + ) + restart_required = any( + x.get("restart_required") is True for x in members) + guest_reason_code = next(( + x.get("guest_reason_code") for x in members + if x.get("guest_network_verified") is not True + and isinstance(x.get("guest_reason_code"), str) + ), None) projected_wg = tuple( dict(item, tenant_allowed_ips=sorted( expected_wg[item.get("peer_host_id")][1] @@ -3005,15 +3227,26 @@ class VpcPrivateHostAdapter: plan.vpc.vpc_id, plan.host_id, plan.desired_revision, plan.desired_revision if active else 0, plan.desired_digest, plan.desired_digest if active else None, - "active" if active else "degraded", plan.resources.bridge, + "active" if active else ( + "applying" if restart_required or guest_reason_code else "degraded"), + plan.resources.bridge, parsed.get("bridge_owner"), members, tuple(parsed.get("routes", [])), projected_wg, parsed.get("nft_table"), plan.desired_revision if active else None, parsed.get("anti_spoof") == "yes", rp_filter, proxy_arp, proxy_arp_pvlan, - tuple(sorted(actual_proxy_ips)), "confirmed" if active else "mismatch", - "none", None if active else "vpc_private_dataplane_readback_mismatch", + tuple(sorted(actual_proxy_ips)), + host_config_verified, + plan.desired_revision if host_config_verified else 0, + plan.desired_digest if host_config_verified else None, + "confirmed" if active else ( + "pending_restart" if restart_required + else "pending_guest" if guest_reason_code else "mismatch"), + "none", None if active else ( + "vpc_private_guest_restart_required" if restart_required + else guest_reason_code + or "vpc_private_dataplane_readback_mismatch"), ).as_dict() -__all__ = ["DATAPLANE_OWNER_MARKER", "FABRIC_INTERFACE", "PRIVATE_INTERFACE", "VpcDataplaneError", "VpcFabricPeer", "VpcDataplaneResources", "VpcDataplanePlan", "VpcDataplaneObservation", "derive_private_mac", "resource_names", "parse_vpc_dataplane_plan", "render_vpc_nft_rules", "render_apply_script", "render_remove_script", "render_observe_script", "patch_private_interface_xml", "VpcPrivateHostAdapter"] +__all__ = ["DATAPLANE_OWNER_MARKER", "FABRIC_INTERFACE", "PRIVATE_INTERFACE", "VpcDataplaneError", "VpcFabricPeer", "VpcDataplaneResources", "VpcDataplanePlan", "VpcDataplaneObservation", "derive_private_mac", "private_guest_routes", "private_guest_network_generation", "resource_names", "parse_vpc_dataplane_plan", "render_vpc_nft_rules", "render_apply_script", "render_remove_script", "render_observe_script", "patch_private_interface_xml", "VpcPrivateHostAdapter"] diff --git a/web/app/Models/CloudVpcOperation.php b/web/app/Models/CloudVpcOperation.php index a46949d6..90637d65 100644 --- a/web/app/Models/CloudVpcOperation.php +++ b/web/app/Models/CloudVpcOperation.php @@ -13,6 +13,7 @@ final class CloudVpcOperation extends Model 'vpc_id' => 'int', 'user_id' => 'int', 'desired_revision' => 'int', + 'evidence_version' => 'int', 'created_at' => 'int', 'updated_at' => 'int', 'finished_at' => 'int', diff --git a/web/app/Services/Cloud/CloudOperationErrorCatalog.php b/web/app/Services/Cloud/CloudOperationErrorCatalog.php index 7c692b13..3114a90a 100644 --- a/web/app/Services/Cloud/CloudOperationErrorCatalog.php +++ b/web/app/Services/Cloud/CloudOperationErrorCatalog.php @@ -124,6 +124,21 @@ final class CloudOperationErrorCatalog 'vpc_private_observation_missing' => '实例私网状态尚未完成回读,请稍后刷新,切勿重复提交', 'vpc_private_observation_mismatch' => '实例私网状态与当前 VPC 目标不一致,正在核对', 'vpc_private_dataplane_unverified' => 'VPC 私网数据面尚未完成核验,请稍后刷新', + 'vpc_private_guest_restart_required' => '私网配置尚未写入当前实例;请完全关机,等待系统核对写入后再开机', + 'vpc_private_guest_agent_unavailable' => 'QEMU Guest Agent 暂不可用,无法确认来宾私网地址和路由', + 'vpc_private_guest_probe_unsupported' => '来宾缺少受支持的网络探测能力,无法确认私网地址和路由', + 'vpc_private_guest_probe_uncertain' => '来宾私网探测结果不完整,正在重新核验', + 'vpc_private_guest_probe_deadline_exceeded' => '来宾私网探测超时,将继续后台核验', + 'vpc_private_guest_probe_reap_required' => '来宾私网探测清理状态未确认;请完全关机再开机后继续核验', + 'vpc_private_guest_address_mismatch' => '来宾私网地址与当前 VPC 目标不一致,正在核对', + 'vpc_private_guest_route_mismatch' => '来宾私网路由与当前 VPC 目标不一致,正在核对', + 'vpc_private_guest_link_mismatch' => '来宾私网网卡未处于可用状态,正在核对', + 'vpc_private_network_suspended' => '实例网络已按策略暂停,私网当前不可用', + 'vpc_private_guest_stopped' => '实例已关机;私网配置已核对,但运行态网络不可用', + 'vpc_private_guest_shutdown_pending' => '实例正在完成正常关机;私网运行态暂不可用', + 'vpc_private_guest_readback_unavailable' => 'QEMU Guest Agent 暂不可用,无法确认来宾私网地址和路由', + 'vpc_private_guest_network_mismatch' => '来宾私网地址或路由与当前 VPC 目标不一致,正在核对', + 'vpc_observation_contract_outdated' => 'VPC 观测证据版本过旧,正在重新核验来宾网络', 'vpc_private_teardown_uncertain' => '实例私网清理状态尚未确认,请勿重复提交', 'vpc_private_expectation_invalid' => '实例私网目标记录无效,请联系管理员核对', 'vpc_private_expectation_unavailable' => '实例私网目标暂时无法读取,请稍后核对', @@ -328,6 +343,18 @@ final class CloudOperationErrorCatalog 'vpc_private_observation_missing', 'vpc_private_observation_mismatch', 'vpc_private_teardown_uncertain', 'vpc_private_expectation_invalid', 'vpc_private_expectation_unavailable', 'vpc_private_dataplane_unverified', + 'vpc_private_guest_restart_required', + 'vpc_private_guest_agent_unavailable', + 'vpc_private_guest_probe_unsupported', 'vpc_private_guest_probe_uncertain', + 'vpc_private_guest_probe_deadline_exceeded', + 'vpc_private_guest_probe_reap_required', + 'vpc_private_guest_address_mismatch', 'vpc_private_guest_route_mismatch', + 'vpc_private_guest_link_mismatch', + 'vpc_private_network_suspended', + 'vpc_private_guest_stopped', + 'vpc_private_guest_shutdown_pending', + 'vpc_private_guest_readback_unavailable', + 'vpc_private_guest_network_mismatch', 'vpc_observation_contract_outdated', ], true)) { return $engineCode; } @@ -588,7 +615,7 @@ final class CloudOperationErrorCatalog $result[$key] = is_string($item) && $item !== '' ? self::normalize($item) : null; } - foreach (['state' => ['disabled', 'active', 'suspended', 'removed', 'degraded', 'uncertain'], + foreach (['state' => ['disabled', 'active', 'suspended', 'stopping', 'stopped', 'removed', 'degraded', 'uncertain'], 'desired_state' => ['disabled', 'active', 'removed'], 'readback' => ['confirmed', 'missing', 'mismatch'], 'link_state' => ['up', 'down', 'absent']] as $key => $allowed) { @@ -605,6 +632,20 @@ final class CloudOperationErrorCatalog $result['desired_revision'] = max(0, (int) ($value['desired_revision'] ?? 0)); $result['observed_revision'] = max(0, (int) ($value['observed_revision'] ?? 0)); $result['observed_at'] = max(0, (int) ($value['observed_at'] ?? 0)); + foreach (['xml_verified', 'config_verified', 'seed_verified', 'domain_running', + 'guest_network_verified'] as $key) { + $result[$key] = ($value[$key] ?? false) === true; + } + $result['restart_required'] = is_bool($value['restart_required'] ?? null) + ? (bool) $value['restart_required'] : null; + $result['guest_network_readback'] = is_string( + $value['guest_network_readback'] ?? null + ) && in_array($value['guest_network_readback'], [ + 'confirmed', 'missing', 'unsupported', 'unavailable', + ], true) ? $value['guest_network_readback'] : 'missing'; + $result['guest_link_state'] = is_string($value['guest_link_state'] ?? null) + && in_array($value['guest_link_state'], ['up', 'down'], true) + ? $value['guest_link_state'] : null; foreach (['desired_digest', 'observed_digest'] as $key) { if ($result[$key] !== null && preg_match('/^[a-f0-9]{64}$/Di', $result[$key]) !== 1) { @@ -643,6 +684,18 @@ final class CloudOperationErrorCatalog 'private_source_unverified', 'vpc_private_observation_missing', 'vpc_private_observation_mismatch', 'vpc_private_teardown_uncertain', 'vpc_private_dataplane_unverified', + 'vpc_private_guest_restart_required', + 'vpc_private_guest_agent_unavailable', + 'vpc_private_guest_probe_unsupported', 'vpc_private_guest_probe_uncertain', + 'vpc_private_guest_probe_deadline_exceeded', + 'vpc_private_guest_probe_reap_required', + 'vpc_private_guest_address_mismatch', 'vpc_private_guest_route_mismatch', + 'vpc_private_guest_link_mismatch', + 'vpc_private_network_suspended', + 'vpc_private_guest_stopped', + 'vpc_private_guest_shutdown_pending', + 'vpc_private_guest_readback_unavailable', + 'vpc_private_guest_network_mismatch', 'vpc_observation_contract_outdated', 'vpc_private_expectation_invalid', 'vpc_private_expectation_unavailable', ], true) ? $reason : null; return $result; diff --git a/web/app/Services/Cloud/CloudVpcService.php b/web/app/Services/Cloud/CloudVpcService.php index c259db1c..f68f85b8 100644 --- a/web/app/Services/Cloud/CloudVpcService.php +++ b/web/app/Services/Cloud/CloudVpcService.php @@ -35,6 +35,38 @@ final class CloudVpcService public const MAX_ROUTES = 256; public const MAX_ACL_RULES = 256; + /** + * Version of the engine observation contract that proves guest-visible + * private addressing and routes, rather than only host/libvirt state. + */ + public const OBSERVATION_EVIDENCE_VERSION = 2; + + /** PHP transport budget must exceed the engine-wide VPC probe budget. */ + public const ENGINE_TIMEOUT_SECONDS = 60.0; + public const OPERATION_LEASE_SECONDS = 120; + + /** Engine reason codes that may cross the host/operation/root boundary. */ + private const DEFERRED_EVIDENCE_ERRORS = [ + 'vpc_private_guest_restart_required', + 'vpc_private_guest_agent_unavailable', + 'vpc_private_guest_probe_unsupported', + 'vpc_private_guest_probe_uncertain', + 'vpc_private_guest_probe_deadline_exceeded', + 'vpc_private_guest_probe_reap_required', + 'vpc_private_guest_address_mismatch', + 'vpc_private_guest_route_mismatch', + 'vpc_private_guest_link_mismatch', + // Compatibility with the short-lived pre-contract preview. + 'vpc_private_guest_readback_unavailable', + 'vpc_private_guest_network_mismatch', + 'vpc_observation_contract_outdated', + // Intentional lifecycle state, safe for host/root diagnostics but not + // treated as a guest-probe success. + 'vpc_private_network_suspended', + 'vpc_private_guest_stopped', + 'vpc_private_guest_shutdown_pending', + ]; + /** Maximum exact /32 destinations present in one host readback. */ private const MAX_OBSERVED_DESTINATIONS = self::MAX_MEMBERS + self::MAX_ROUTES; @@ -975,8 +1007,25 @@ final class CloudVpcService if ($locked === null) { throw new CloudError('VPC 不存在', 404); } + $runtimeRetryableSync = in_array( + (string) $locked->sync_state, ['waiting', 'uncertain'], true + ) || ((string) $locked->sync_state === 'applied' + && in_array((string) ($locked->last_error_code ?? ''), [ + 'vpc_private_guest_shutdown_pending', + 'vpc_private_guest_stopped', + ], true)); + $runtimeObservationPending = $allowDeferredRetry + && (string) $locked->state === 'degraded' + && $runtimeRetryableSync + && (int) $locked->desired_revision === (int) $locked->observed_revision + && self::digestMatches( + (string) $locked->desired_digest, + (string) $locked->observed_digest + ); $operation = CloudVpcOperation::where('vpc_id', $locked->id) - ->where(function ($query) use ($allowDeferredRetry): void { + ->where(function ($query) use ( + $allowDeferredRetry, $runtimeObservationPending, $locked + ): void { $query->whereIn('status', $allowDeferredRetry ? ['queued', 'running', 'uncertain', 'waiting_observed'] : ['queued', 'running', 'uncertain']); @@ -988,6 +1037,32 @@ final class CloudVpcService $failed->where('status', 'failed') ->where('kind', 'vpc_delete'); }); + $query->orWhere(function ($legacy) use ($locked): void { + $legacy->where('status', 'succeeded') + ->where('kind', '!=', 'vpc_delete') + ->where('evidence_version', '<', + self::OBSERVATION_EVIDENCE_VERSION) + ->where('operation_id', + (string) $locked->last_operation_id) + ->where('desired_revision', + (int) $locked->desired_revision); + }); + if ($runtimeObservationPending) { + // Member removal may already be durably committed + // while a surviving VM still lacks exact guest + // evidence. Re-run only this root's current, + // idempotent target until runtime proof catches up. + $query->orWhere(function ($runtime) use ($locked): void { + $runtime->where('status', 'succeeded') + ->where('kind', '!=', 'vpc_delete') + ->where('evidence_version', '>=', + self::OBSERVATION_EVIDENCE_VERSION) + ->where('operation_id', + (string) $locked->last_operation_id) + ->where('desired_revision', + (int) $locked->desired_revision); + }); + } } }) ->orderByDesc('desired_revision')->lockForUpdate()->first(); @@ -1012,8 +1087,11 @@ final class CloudVpcService return null; } $now = time(); - if ((string) $operation->status === 'running' - && (int) $operation->lease_until > $now) { + // Runtime-only refreshes intentionally keep a completed operation + // terminal while borrowing its lease. The lease, rather than the + // status string, is therefore the concurrency boundary for both + // ordinary applies and background observation refreshes. + if ((int) $operation->lease_until > $now) { return null; } $payload = json_decode((string) $operation->payload_json, true); @@ -1036,23 +1114,30 @@ final class CloudVpcService $locked->save(); return null; } - $operation->status = 'running'; + $runtimeProofRefresh = (string) $operation->status === 'succeeded' + && (int) ($operation->evidence_version ?? 0) + >= self::OBSERVATION_EVIDENCE_VERSION; + if (!$runtimeProofRefresh) { + $operation->status = 'running'; + $operation->finished_at = null; + } $operation->lease_token = self::randomKey(); - $operation->lease_until = $now + 120; - $operation->finished_at = null; + $operation->lease_until = $now + self::OPERATION_LEASE_SECONDS; $operation->attempts = (int) $operation->attempts + 1; $operation->updated_at = $now; $operation->save(); - return [$operation, $payload]; + return [$operation, $payload, $runtimeProofRefresh]; }, 3); if ($claim === null) { return self::show($vpcKey, $actor); } - [$operation, $payload] = $claim; + [$operation, $payload, $runtimeProofRefresh] = $claim; $leaseToken = (string) $operation->lease_token; try { if ((string) $operation->kind === 'vpc_delete') { - $result = (new CloudEngineClient())->vpc( + $result = (new CloudEngineClient( + null, self::ENGINE_TIMEOUT_SECONDS + ))->vpc( 'POST', '/internal/v1/vpcs/' . $vpcKey . '/delete', $actor, $payload, (string) $operation->operation_id ); @@ -1079,7 +1164,9 @@ final class CloudVpcService ); return self::show($vpcKey, $actor); } - $result = (new CloudEngineClient())->vpc('POST', '/internal/v1/vpcs/apply', + $result = (new CloudEngineClient( + null, self::ENGINE_TIMEOUT_SECONDS + ))->vpc('POST', '/internal/v1/vpcs/apply', $actor, $payload, (string) $operation->operation_id); $expectedRevision = (int) ($payload['desired_revision'] ?? 0); $expectedDigest = (string) ($payload['desired_digest'] ?? ''); @@ -1136,7 +1223,10 @@ final class CloudVpcService $status = $error->httpStatus >= 500 ? 'uncertain' : 'failed'; $now = time(); $connection = (new CloudVpc())->getConnection(); - $connection->transaction(function () use ($operation, $leaseToken, $vpc, $status, $error, $now): void { + $connection->transaction(function () use ( + $operation, $leaseToken, $vpc, $status, $error, $now, + $runtimeProofRefresh + ): void { // Keep the lock order identical to every mutator: VPC root // first, then its operation row. This also prevents an old // lease from overwriting a newer desired revision. @@ -1145,11 +1235,21 @@ final class CloudVpcService return; } $row = CloudVpcOperation::where('operation_id', (string) $operation->operation_id) - ->where('status', 'running') + ->where('status', $runtimeProofRefresh ? 'succeeded' : 'running') ->where('lease_token', $leaseToken)->lockForUpdate()->first(); if ($row === null) { return; } + if ($runtimeProofRefresh) { + // A runtime-only poll must not erase the last successful + // configuration proof when transport is temporarily + // unavailable. Release the lease and retry later. + $row->lease_token = null; + $row->lease_until = 0; + $row->updated_at = $now; + $row->save(); + return; + } $row->status = $status; $row->error_code = self::safeErrorCode($error->getMessage()) ?? ($status === 'uncertain' ? 'vpc_result_uncertain' : 'vpc_apply_failed'); @@ -1373,6 +1473,7 @@ final class CloudVpcService ]; $operation->observed_json = json_encode($safeReceipt, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES); + $operation->evidence_version = self::OBSERVATION_EVIDENCE_VERSION; $operation->status = 'succeeded'; $operation->error_code = null; $operation->lease_token = null; @@ -1421,7 +1522,14 @@ final class CloudVpcService ->where('vpc_id', $vpc->id); if ($leaseToken !== null) { $operationQuery->where('lease_token', $leaseToken) - ->where('status', 'running'); + ->where(function ($status): void { + $status->where('status', 'running') + ->orWhere(function ($runtime): void { + $runtime->where('status', 'succeeded') + ->where('evidence_version', '>=', + self::OBSERVATION_EVIDENCE_VERSION); + }); + }); } $operation = $operationQuery->lockForUpdate()->first(); if ($operation === null) { @@ -1435,16 +1543,39 @@ final class CloudVpcService $hostKey = (string) ($item['host_id'] ?? $key); $hosts[$hostKey] = $item; } + $evidenceVersion = max(0, (int) ($observed['evidence_version'] + ?? $observed['observation_version'] ?? 0)); + $evidenceContractCurrent = + $evidenceVersion === self::OBSERVATION_EVIDENCE_VERSION; + // An older engine can report an otherwise exact host/libvirt + // observation without proving that the running guest consumed its + // address and routes. Preserve the receipt for audit, but close + // every activation gate and keep the same operation retryable. + if (!$evidenceContractCurrent) { + $observed['evidence_version'] = $evidenceVersion; + $observed['next_stage_allowed'] = false; + $observed['private_dataplane_verified'] = false; + foreach ($hosts as $hostKey => $item) { + $item['private_dataplane_verified'] = false; + $item['last_error_code'] = 'vpc_observation_contract_outdated'; + $hosts[$hostKey] = $item; + } + $observed['hosts'] = $hosts; + } // Validate typed tenant dataplane evidence before evaluating the // all-host gate. Fabric transport alone can only leave this // independent VPC lane waiting/uncertain. $expectedMembersByHost = []; $allExpectedMembers = []; $expectedDestinations = []; + $expectedVpcCidr = null; $desiredRoutes = []; $desiredContractValid = true; try { $desired = self::desiredDocument($vpc); + $expectedVpcCidr = self::cidr( + $desired['cidr'] ?? null, 16, 24, 'VPC CIDR' + )['cidr']; // The host adapter installs only member /32 routes and the // explicitly configured VPC routes. The VPC/subnet CIDRs // themselves are control-plane address ranges, not an @@ -1474,46 +1605,202 @@ final class CloudVpcService $expectedMembersByHost = []; $allExpectedMembers = []; $expectedDestinations = []; + $expectedVpcCidr = null; $desiredRoutes = []; } $expectedDestinations = array_values(array_unique(array_filter( $expectedDestinations, static fn ($value): bool => is_string($value) && $value !== '' ))); + $shutdownPendingMembers = []; + if ($allExpectedMembers !== []) { + $shutdownPendingMembers = array_fill_keys( + CloudVpcMember::where('vpc_id', $vpc->id) + ->where('state', 'stopping') + ->whereIn('member_key', array_keys($allExpectedMembers)) + ->pluck('member_key')->map( + static fn ($value): string => (string) $value + )->all(), + true + ); + } + $memberGuestEvidence = []; foreach ($hosts as $hostKey => $item) { - if (($item['private_dataplane_verified'] ?? false) !== true) { - continue; - } - $errorCode = $desiredContractValid + $localExpectedMembers = + $expectedMembersByHost[(string) $hostKey] ?? []; + $configError = $evidenceContractCurrent && $desiredContractValid ? self::validateHostDataplaneEvidence( $item, (string) $hostKey, - $expectedMembersByHost[(string) $hostKey] ?? [], + $localExpectedMembers, array_values($allExpectedMembers), $expectedDestinations, + (string) $expectedVpcCidr, $desiredRoutes, $vpcKey, $revision, - $digest + $digest, + false ) - : 'vpc_private_readback_mismatch'; - if ($errorCode !== null) { - $item['private_dataplane_verified'] = false; - $item['last_error_code'] = $errorCode; - $hosts[$hostKey] = $item; - $observedHosts = is_array($observed['hosts'] ?? null) - ? $observed['hosts'] : []; - $observedHosts[$hostKey] = $item; - $observed['hosts'] = $observedHosts; - $observed['private_dataplane_verified'] = false; - } elseif (($item['removed'] ?? false) === true) { + : ($evidenceContractCurrent + ? 'vpc_private_readback_mismatch' + : 'vpc_observation_contract_outdated'); + $item['config_evidence_verified'] = $configError === null; + if ($configError !== null) { + $item['last_error_code'] = $configError; + } + if (($item['private_dataplane_verified'] ?? false) === true) { + $errorCode = $evidenceContractCurrent && $desiredContractValid + ? self::validateHostDataplaneEvidence( + $item, + (string) $hostKey, + $localExpectedMembers, + array_values($allExpectedMembers), + $expectedDestinations, + (string) $expectedVpcCidr, + $desiredRoutes, + $vpcKey, + $revision, + $digest + ) + : ($evidenceContractCurrent + ? 'vpc_private_readback_mismatch' + : 'vpc_observation_contract_outdated'); + if ($errorCode !== null) { + $item['private_dataplane_verified'] = false; + $item['last_error_code'] = $errorCode; + $observed['private_dataplane_verified'] = false; + } elseif (($item['removed'] ?? false) === true) { + $item['removal_verified'] = true; + } + } elseif ($configError === null + && !is_string($item['last_error_code'] ?? null)) { + $item['last_error_code'] = 'vpc_private_guest_probe_uncertain'; + } + if (($item['removed'] ?? false) === true && $configError === null) { $item['removal_verified'] = true; - $hosts[$hostKey] = $item; - $observedHosts = is_array($observed['hosts'] ?? null) - ? $observed['hosts'] : []; - $observedHosts[$hostKey] = $item; - $observed['hosts'] = $observedHosts; } + // Host configuration can be exact while only a subset of its + // running guests has consumed that configuration. Preserve + // one proof result per member; aggregate host/root readiness + // is evaluated separately below. + if ($configError === null + && ($item['transport_verified'] ?? false) === true + && ($item['removed'] ?? false) !== true + && is_array($item['member_nics'] ?? null)) { + foreach ($item['member_nics'] as $nic) { + if (!is_array($nic)) { + continue; + } + $memberId = (string) ($nic['member_id'] ?? ''); + $expectedMember = $localExpectedMembers[$memberId] ?? null; + if (!is_array($expectedMember)) { + continue; + } + $memberError = self::validateMemberNicEvidence( + $nic, + $expectedMember, + (string) $expectedVpcCidr, + true + ); + $intentionalSuspended = ($nic['xml_verified'] ?? false) === true + && ($nic['config_verified'] ?? false) === true + && ($nic['seed_verified'] ?? false) === true + && ($nic['domain_running'] ?? false) === true + && ($nic['restart_required'] ?? null) === false + && ($nic['link_state'] ?? null) === 'down'; + $intentionalStopped = ($nic['xml_verified'] ?? false) === true + && ($nic['config_verified'] ?? false) === true + && ($nic['seed_verified'] ?? false) === true + && ($nic['domain_running'] ?? null) === false + && ($nic['restart_required'] ?? null) === false + && in_array(($nic['link_state'] ?? null), + ['up', 'down'], true); + $intentionalStopping = isset( + $shutdownPendingMembers[$memberId] + ) && !$intentionalStopped + && ($nic['xml_verified'] ?? false) === true + && ($nic['config_verified'] ?? false) === true + && ($nic['seed_verified'] ?? false) === true + && ($nic['restart_required'] ?? null) === false + && in_array(($nic['link_state'] ?? null), + ['up', 'down'], true); + // Retain the exact shape used to classify this member. + // In particular, a generic probe/link failure must not + // be able to masquerade as an intentional suspension. + $memberGuestEvidence[$memberId] = [ + 'state' => $intentionalStopped ? 'stopped' + : ($intentionalStopping ? 'stopping' + : ($intentionalSuspended ? 'suspended' + : ($memberError === null ? 'active' : 'degraded'))), + 'error_code' => $intentionalStopped + ? 'vpc_private_guest_stopped' + : ($intentionalStopping + ? 'vpc_private_guest_shutdown_pending' + : ($intentionalSuspended + ? 'vpc_private_network_suspended' + : ($memberError === null ? null + : (self::deferredEvidenceError($memberError) + ?? 'vpc_private_observation_mismatch')))), + 'xml_verified' => ($nic['xml_verified'] ?? false) === true, + 'config_verified' => + ($nic['config_verified'] ?? false) === true, + 'seed_verified' => + ($nic['seed_verified'] ?? false) === true, + 'domain_running' => + ($nic['domain_running'] ?? false) === true, + 'restart_required' => $nic['restart_required'] ?? null, + 'link_state' => is_string($nic['link_state'] ?? null) + ? (string) $nic['link_state'] : null, + 'guest_network_verified' => + ($nic['guest_network_verified'] ?? false) === true, + 'guest_network_readback' => is_string( + $nic['guest_network_readback'] ?? null + ) ? (string) $nic['guest_network_readback'] : null, + 'guest_link_state' => is_string( + $nic['guest_link_state'] ?? null + ) ? (string) $nic['guest_link_state'] : null, + 'guest_exact' => $memberError === null, + ]; + } + $localSuspended = false; + $localStopping = false; + $localStopped = false; + $localUnresolved = false; + foreach ($localExpectedMembers as $localMemberId => $_localMember) { + $localRuntime = $memberGuestEvidence[ + (string) $localMemberId + ] ?? null; + $localState = is_array($localRuntime) + ? (string) ($localRuntime['state'] ?? '') : ''; + if ($localState === 'suspended') { + $localSuspended = true; + } elseif ($localState === 'stopping') { + $localStopping = true; + } elseif ($localState === 'stopped') { + $localStopped = true; + } elseif ($localState !== 'active') { + $localUnresolved = true; + } + } + if (!$localUnresolved) { + if ($localStopped) { + $item['last_error_code'] = + 'vpc_private_guest_stopped'; + } elseif ($localStopping) { + $item['last_error_code'] = + 'vpc_private_guest_shutdown_pending'; + } elseif ($localSuspended) { + $item['last_error_code'] = + 'vpc_private_network_suspended'; + } + } + } + $hosts[$hostKey] = $item; + $observedHosts = is_array($observed['hosts'] ?? null) + ? $observed['hosts'] : []; + $observedHosts[$hostKey] = $item; + $observed['hosts'] = $observedHosts; } $deployments = CloudVpcHostDeployment::where('vpc_id', $vpc->id) ->lockForUpdate()->get()->keyBy(static fn ($row): int => (int) $row->host_id); @@ -1532,7 +1819,8 @@ final class CloudVpcService ?? $observed['observed_digest'] ?? ''); $privateDataplaneVerified = ($observed['private_dataplane_verified'] ?? false) === true; - $globalActive = ($observed['state'] ?? null) === 'active' + $globalActive = $evidenceContractCurrent + && ($observed['state'] ?? null) === 'active' && ($observed['next_stage_allowed'] ?? false) === true && $privateDataplaneVerified && $observedRevision === $revision @@ -1604,9 +1892,10 @@ final class CloudVpcService // In particular, a retiring host must not make a still-live sibling // inexact merely because both appear in the same operation. $allRequiredHostsExact = $required->count() > 0; + $allRequiredHostsConfigExact = $required->count() > 0; $allRemovalProof = true; $hasRemovalTarget = false; - $currentActiveExactHostIds = []; + $currentConfigExactHostIds = []; $onlyKnownDeletedHostEvidence = true; $exactRemovalEvidence = static function ($item) use ($revision, $digest): bool { return is_array($item) @@ -1628,6 +1917,7 @@ final class CloudVpcService $knownDeleted = $knownDeletedDeployments->get((int) $hostKey); if ($knownDeleted === null || !$exactRemovalEvidence($item)) { $allRequiredHostsExact = false; + $allRequiredHostsConfigExact = false; $onlyKnownDeletedHostEvidence = false; continue; } @@ -1653,6 +1943,7 @@ final class CloudVpcService } if (!is_array($item)) { $allRequiredHostsExact = false; + $allRequiredHostsConfigExact = false; if ($deleting) { // A missing deleting host is failed removal proof, // not an absent target. Without this assignment a @@ -1675,12 +1966,22 @@ final class CloudVpcService } $itemRevision = (int) ($item['observed_revision'] ?? $item['revision'] ?? 0); $itemDigest = (string) ($item['observed_digest'] ?? $item['digest'] ?? ''); - $active = $globalActive && ($item['state'] ?? null) === 'active' + $itemError = self::deferredEvidenceError( + $item['last_error_code'] ?? null + ); + $active = ($item['state'] ?? null) === 'active' && ($item['transport_verified'] ?? false) === true && ($item['private_dataplane_verified'] ?? false) === true && ($item['removed'] ?? false) !== true && $itemRevision <= $revision && $itemRevision === $revision && hash_equals($digest, $itemDigest); + $configActive = !$deleting + && ($item['config_evidence_verified'] ?? false) === true + && ($item['transport_verified'] ?? false) === true + && ($item['removed'] ?? false) !== true + && (int) ($item['config_observed_revision'] ?? 0) === $revision + && hash_equals($digest, + (string) ($item['config_observed_digest'] ?? '')); $removalActive = $deleting && ($item['removed'] ?? false) === true && ($item['removal_verified'] ?? false) === true @@ -1697,14 +1998,20 @@ final class CloudVpcService if (($deleting && !$removalActive) || (!$deleting && !$active)) { $allRequiredHostsExact = false; } - if (!$deleting && $active) { - $currentActiveExactHostIds[(int) $hostId] = true; + if (($deleting && !$removalActive) + || (!$deleting && !$configActive)) { + $allRequiredHostsConfigExact = false; + } + if (!$deleting && $configActive) { + $currentConfigExactHostIds[(int) $hostId] = true; } $hostEvidence[(int) $hostId] = [ 'active' => $active, + 'config_active' => $configActive, 'removal_active' => $removalActive, 'revision' => $itemRevision, 'digest' => $itemDigest, + 'last_error_code' => $itemError, ]; } // Persist each exact removal proof monotonically even when another @@ -1721,8 +2028,11 @@ final class CloudVpcService $deleting = (string) $deployment->state === 'deleting'; $itemRevision = (int) ($evidence['revision'] ?? 0); $itemDigest = (string) ($evidence['digest'] ?? ''); - $deployment->observed_revision = $itemRevision; - $deployment->observed_digest = $itemDigest !== '' ? $itemDigest : null; + $configActive = ($evidence['config_active'] ?? false) === true; + $deployment->observed_revision = $configActive + ? $revision : $itemRevision; + $deployment->observed_digest = $configActive + ? $digest : ($itemDigest !== '' ? $itemDigest : null); // An empty-host target is only a desired control-plane // document. Without a data-plane removal readback, preserve // the deleting state even when the Fabric digest is exact. @@ -1730,8 +2040,11 @@ final class CloudVpcService $deployment->state = 'deleted'; $deployment->last_error_code = null; } elseif (!$deleting) { + $hostError = is_string($evidence['last_error_code'] ?? null) + ? (string) $evidence['last_error_code'] : null; $deployment->state = $active ? 'active' : 'degraded'; - $deployment->last_error_code = $active ? null : 'vpc_host_observation_mismatch'; + $deployment->last_error_code = $active ? null + : ($hostError ?? 'vpc_host_observation_mismatch'); } else { $deployment->last_error_code = $removalActive ? 'vpc_observation_incomplete' @@ -1755,6 +2068,7 @@ final class CloudVpcService ); if (!$allKnownDeletedProofValid) { $allRequiredHostsExact = false; + $allRequiredHostsConfigExact = false; } // A terminal no-members/no-participating-hosts response is a // positive control-plane proof only when this revision has no @@ -1763,7 +2077,7 @@ final class CloudVpcService // ACL and an unreferenced subnet tombstone may converge without // pretending that a private NIC, address or member was removed. // Keep the proof in the operation audit. - $noDataPlaneTerminalProof = $terminalGate + $noDataPlaneTerminalProof = $evidenceContractCurrent && $terminalGate && $required->count() === 0 && $liveRequired->count() === 0 && ($hosts === [] || $onlyKnownDeletedHostEvidence) @@ -1771,6 +2085,7 @@ final class CloudVpcService && !$pendingHostTombstone; $terminalSubnetReleased = 0; $storedMemberRemovalReleased = 0; + $configMemberRemovalReleased = 0; if ($noDataPlaneTerminalProof) { $observed['no_data_plane_terminal_proof'] = [ 'revision' => $revision, @@ -1823,7 +2138,7 @@ final class CloudVpcService // Close their tombstones only after every required live host was // individually matched above; a top-level active envelope with a // missing or mismatched host must remain fail-closed. - if ($allRequiredHostsExact && $allRemovalProof) { + if ($allRequiredHostsConfigExact && $allRemovalProof) { $observedAt = time(); foreach ([CloudVpcRoute::class, CloudVpcAclRule::class] as $childModel) { $childModel::where('vpc_id', $vpc->id) @@ -1841,8 +2156,8 @@ final class CloudVpcService // teardown. Physically release member/address identity rows // so their global unique keys can be reused; the operation's // bounded observed_json remains the durable audit record. - self::releaseMemberTombstones( - $vpc, $revision, $digest, true, $currentActiveExactHostIds + $configMemberRemovalReleased = self::releaseMemberTombstones( + $vpc, $revision, $digest, true, $currentConfigExactHostIds ); self::releaseSubnetTombstones($vpc, $revision, $digest); } @@ -1862,8 +2177,20 @@ final class CloudVpcService break; } } - $allActive = $allRequiredHostsExact && $allRemovalProof + $allMemberRuntimeActive = $allExpectedMembers !== []; + foreach ($allExpectedMembers as $memberId => $_expectedMember) { + if ((string) ($memberGuestEvidence[(string) $memberId]['state'] ?? '') + !== 'active') { + $allMemberRuntimeActive = false; + break; + } + } + $allActive = $globalActive && $allRequiredHostsExact && $allRemovalProof && $liveRequired->count() > 0 + && $allMemberRuntimeActive + && !$pendingChildTombstone; + $configurationComplete = $allRequiredHostsConfigExact + && $allRemovalProof && $liveRequired->count() > 0 && !$pendingChildTombstone; $removalOnlyComplete = $hasRemovalTarget && $allRemovalProof && $allRequiredHostsExact && $liveRequired->count() === 0 @@ -1874,8 +2201,68 @@ final class CloudVpcService $storedMemberRemovalComplete = $noDataPlaneTerminalProof && (string) $operation->kind === 'member_detach' && $storedMemberRemovalReleased > 0 && !$pendingChildTombstone; - $operationSucceeded = $allActive || $removalOnlyComplete - || $terminalSubnetComplete || $storedMemberRemovalComplete; + $configMemberRemovalComplete = + (string) $operation->kind === 'member_detach' + && $configMemberRemovalReleased > 0 + && $allRequiredHostsConfigExact && $allRemovalProof + && !$pendingChildTombstone; + // Once the detach mutation has advanced the root to this exact + // revision, later runtime-only retries must not reopen or repeat + // the already released member/IP tombstone. They keep the same + // operation terminal while retrying guest evidence for siblings. + $memberDetachPreviouslyCommitted = + (string) $operation->kind === 'member_detach' + && (int) $vpc->observed_revision === $revision + && self::digestMatches($digest, (string) $vpc->observed_digest) + && !$pendingChildTombstone; + $operationPreviouslySucceeded = + (string) $operation->status === 'succeeded' + && (int) ($operation->evidence_version ?? 0) + >= self::OBSERVATION_EVIDENCE_VERSION + && (int) $vpc->observed_revision === $revision + && self::digestMatches($digest, (string) $vpc->observed_digest); + $operationSucceeded = $configurationComplete || $allActive + || $removalOnlyComplete + || $terminalSubnetComplete || $storedMemberRemovalComplete + || $configMemberRemovalComplete || $memberDetachPreviouslyCommitted + || $operationPreviouslySucceeded; + // A reviewed guest reason is diagnostic evidence even when the + // outer engine envelope chose `degraded` instead of `applying`. + // Derive it solely from the per-host closed set; mixed reasons + // intentionally collapse to the generic control-plane code. + $deferredError = self::singleDeferredEvidenceError($hostEvidence); + $hasUnresolvedMemberEvidence = false; + $hasSuspendedMemberEvidence = false; + $hasStoppingMemberEvidence = false; + $hasStoppedMemberEvidence = false; + foreach ($allExpectedMembers as $memberId => $_expectedMember) { + $runtimeEvidence = $memberGuestEvidence[(string) $memberId] ?? null; + $runtimeState = is_array($runtimeEvidence) + ? (string) ($runtimeEvidence['state'] ?? '') : ''; + if (!in_array( + $runtimeState, ['active', 'suspended', 'stopping', 'stopped'], true + )) { + $hasUnresolvedMemberEvidence = true; + } elseif ($runtimeState === 'suspended') { + $hasSuspendedMemberEvidence = true; + } elseif ($runtimeState === 'stopping') { + $hasStoppingMemberEvidence = true; + } elseif ($runtimeState === 'stopped') { + $hasStoppedMemberEvidence = true; + } + } + $runtimeObservationPending = $operationSucceeded + && $configurationComplete + && $liveRequired->count() > 0 && !$allActive + && ($hasUnresolvedMemberEvidence || $hasStoppingMemberEvidence + || $hasStoppedMemberEvidence); + $intentionalRuntimeSuspended = $operationSucceeded + && $configurationComplete && !$allActive + && !$hasUnresolvedMemberEvidence && !$hasStoppingMemberEvidence + && !$hasStoppedMemberEvidence + && $hasSuspendedMemberEvidence; + $configurationObservationUncertain = $operationPreviouslySucceeded + && !$configurationComplete && $liveRequired->count() > 0; // A no-members Fabric gate is terminal only after the database has // no unresolved host deployment. While a deleting host still // needs its SSH/libvirt proof, keep the operation retryable instead @@ -1887,22 +2274,14 @@ final class CloudVpcService // revision was accepted by every participating host. Mirror that // gate onto live child rows. Deletion tombstones intentionally // remain untouched until a typed removal readback proves absence. - if ($allActive) { + if ($configurationComplete || $allActive) { $observedAt = time(); - // Persist a proof marker on each member only after the full - // host-scoped private data-plane gate has passed. Older rows - // without this marker remain fail-closed in instance - // projections until a fresh observation is recorded. - CloudVpcMember::where('vpc_id', $vpc->id) - ->whereNotIn('state', ['deleted']) - ->update([ - 'last_operation_id' => $operationId, - 'updated_at' => $observedAt, - ]); + // Subnets, routes and ACLs are host configuration. Members + // and leases are advanced separately from their own exact + // guest proof below; one stopped VM cannot revoke a ready + // sibling's runtime evidence. foreach ([ CloudVpcSubnet::class, - CloudVpcPrivateIp::class, - CloudVpcMember::class, CloudVpcRoute::class, CloudVpcAclRule::class, ] as $childModel) { @@ -1917,6 +2296,15 @@ final class CloudVpcService 'updated_at' => $observedAt, ]); } + self::persistMemberGuestEvidence( + $vpc, + $revision, + $digest, + $operationId, + $allExpectedMembers, + $memberGuestEvidence, + $observedAt + ); } elseif ($terminalSubnetComplete) { // No host exists to observe an unchanged address range. The // terminal no-data-plane proof is nevertheless exact for the @@ -1935,15 +2323,18 @@ final class CloudVpcService } $operation->observed_json = json_encode(self::boundedObserved($observed), JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES); + $operation->evidence_version = $evidenceContractCurrent + ? self::OBSERVATION_EVIDENCE_VERSION : 0; $operation->status = $operationSucceeded ? 'succeeded' : ($dataPlaneDeferred ? 'waiting_observed' : ($terminalFailure ? 'failed' : 'uncertain')); $operation->error_code = $operationSucceeded ? null - : ($dataPlaneDeferred ? 'vpc_data_plane_deferred' + : ($dataPlaneDeferred ? ($deferredError ?? 'vpc_data_plane_deferred') : ($terminalFailure ? $gateError : 'vpc_observation_incomplete')); $operation->updated_at = time(); $operation->finished_at = ($operationSucceeded || $dataPlaneDeferred || $terminalFailure) ? time() : null; + $operation->lease_token = null; $operation->lease_until = 0; $operation->save(); $vpc->observed_revision = $operationSucceeded ? $revision : (int) $vpc->observed_revision; @@ -1953,10 +2344,16 @@ final class CloudVpcService // revision and finish the operation, but stays degraded/unusable // until a live member host passes the Fabric gate. $vpc->state = $operationSucceeded && $allActive ? 'active' : 'degraded'; - $vpc->sync_state = $operationSucceeded ? 'applied' + $vpc->sync_state = $operationSucceeded + ? ($configurationObservationUncertain ? 'uncertain' + : ($runtimeObservationPending ? 'waiting' : 'applied')) : ($dataPlaneDeferred ? 'waiting' : ($terminalFailure ? 'failed' : 'uncertain')); - $vpc->last_error_code = $operationSucceeded ? null - : ($dataPlaneDeferred ? 'vpc_data_plane_deferred' + $vpc->last_error_code = $operationSucceeded + ? (($runtimeObservationPending || $configurationObservationUncertain) + ? ($deferredError ?? 'vpc_data_plane_deferred') + : ($intentionalRuntimeSuspended + ? 'vpc_private_network_suspended' : null)) + : ($dataPlaneDeferred ? ($deferredError ?? 'vpc_data_plane_deferred') : ($terminalFailure ? $gateError : 'vpc_observation_incomplete')); $vpc->last_success_at = $operationSucceeded ? time() : (int) $vpc->last_success_at; $vpc->last_failure_at = $operationSucceeded ? (int) $vpc->last_failure_at : time(); @@ -2501,9 +2898,14 @@ final class CloudVpcService ? CloudVpcOperation::where('vpc_id', (int) $vpc->id) ->where('operation_id', $rootOperationId) ->where('status', 'succeeded') - ->first(['operation_id', 'desired_revision', 'created_at', 'finished_at']) + ->first([ + 'operation_id', 'desired_revision', 'evidence_version', + 'created_at', 'finished_at', + ]) : null; $operationFresh = $operation !== null + && (int) $operation->evidence_version + >= self::OBSERVATION_EVIDENCE_VERSION && (int) $operation->desired_revision === (int) $base['desired_revision'] && (int) ($operation->finished_at ?? 0) > 0 && (int) ($operation->finished_at ?? 0) @@ -2515,41 +2917,59 @@ final class CloudVpcService && (int) ($privateIp->updated_at ?? 0) > 0 && (int) ($member->updated_at ?? 0) >= (int) ($operation->created_at ?? 0) && (int) ($privateIp->updated_at ?? 0) >= (int) ($operation->created_at ?? 0); - $rootExact = (string) $vpc->state === 'active' - && (string) $vpc->sync_state === 'applied' + // Root state is the aggregate runtime view. A different member + // may still be stopped or suspended while this member has exact + // guest proof, so bind to the current configuration revision + // without requiring every sibling to be runtime-active. + $rootExact = in_array((string) $vpc->state, ['active', 'degraded'], true) + && in_array((string) $vpc->sync_state, ['applied', 'waiting'], true) && $rootDesiredRevision === (int) $base['desired_revision'] && $rootObservedRevision === $rootDesiredRevision && $rootDesiredDigest !== null && $rootObservedDigest !== null && self::digestMatches($rootDesiredDigest, $rootObservedDigest) && self::digestMatches((string) $base['desired_digest'], $rootDesiredDigest); - $memberExact = (string) $member->state === 'active' - && $observedRevision === (int) $base['desired_revision'] + $memberRevisionExact = $observedRevision === (int) $base['desired_revision'] && $observedDigest !== null && self::digestMatches((string) $base['desired_digest'], $observedDigest) && (int) $member->desired_revision === (int) $base['desired_revision'] && self::digestMatches((string) $base['desired_digest'], (string) $member->desired_digest); - $privateExact = (string) $privateIp->state === 'active' - && $privateDesiredRevision === (int) $base['desired_revision'] + $privateRevisionExact = $privateDesiredRevision === (int) $base['desired_revision'] && $privateObservedRevision === $privateDesiredRevision && $privateDesiredDigest !== null && $privateObservedDigest !== null && self::digestMatches((string) $base['desired_digest'], $privateDesiredDigest) && self::digestMatches($privateDesiredDigest, $privateObservedDigest); + $activeExact = (string) $member->state === 'active' + && (string) $privateIp->state === 'active'; + $suspendedExact = (string) $member->state === 'suspended' + && (string) $privateIp->state === 'suspended'; + $stoppingExact = (string) $member->state === 'stopping' + && (string) $privateIp->state === 'stopping'; + $stoppedExact = (string) $member->state === 'stopped' + && (string) $privateIp->state === 'stopped'; $confirmed = $proofMarker && $operationFresh && $rootExact - && $memberExact && $privateExact; + && $memberRevisionExact && $privateRevisionExact + && ($activeExact || $suspendedExact || $stoppingExact + || $stoppedExact); $base['observed_revision'] = $observedRevision; $base['observed_digest'] = $observedDigest; // Keep the stable desired identity in the projection even before // a host readback is confirmed. The observation validator below // still requires the engine to echo the exact same MAC. $base['private_mac'] = $expectation['private_mac'] ?? null; - $base['state'] = $confirmed ? 'active' + $base['state'] = $confirmed + ? ($stoppedExact ? 'stopped' + : ($stoppingExact ? 'stopping' + : ($suspendedExact ? 'suspended' : 'active'))) : ((string) $member->state === 'degraded' || (string) $privateIp->state === 'degraded' ? 'degraded' : 'uncertain'); $base['readback'] = $confirmed ? 'confirmed' : 'missing'; - $base['reason_code'] = $confirmed ? null + $base['reason_code'] = $confirmed + ? ($stoppedExact ? 'vpc_private_guest_stopped' + : ($stoppingExact ? 'vpc_private_guest_shutdown_pending' + : ($suspendedExact ? 'vpc_private_network_suspended' : null))) : (self::safeErrorCode((string) ($privateIp->last_error_code ?? $member->last_error_code ?? '')) ?? 'vpc_private_observation_missing'); @@ -2559,7 +2979,27 @@ final class CloudVpcService (int) ($vpc->last_success_at ?? 0), ], static fn (int $value): bool => $value > 0)); $base['observed_at'] = $freshness === [] ? 0 : min($freshness); - if ($confirmed) { + if ($confirmed && $suspendedExact) { + $base['link_state'] = 'down'; + } elseif ($confirmed && ($stoppingExact || $stoppedExact)) { + // A stopping/stopped domain has no usable runtime link, but + // its persistent XML may deliberately retain either policy value. + // The instance policy ledger is the durable source for that + // configured value until the next start supplies fresh QGA + // evidence. + $networkStatus = (string) (CloudInstance::where('id', $instanceId) + ->value('network_status') ?? ''); + if (in_array($networkStatus, [ + 'blocking', 'blocking_expired', + 'blocked_traffic', 'blocked_expired', + ], true)) { + $base['link_state'] = 'down'; + } elseif ($networkStatus === 'active') { + $base['link_state'] = 'up'; + } else { + $base['link_state'] = 'absent'; + } + } elseif ($confirmed) { // The member ledger proves the VPC revision, while the // instance network policy records whether the last gated // operation intentionally disconnected all NICs. Pending or @@ -2612,6 +3052,12 @@ final class CloudVpcService 'observed_digest' => null, 'readback' => 'missing', 'link_state' => 'absent', 'private_ip_cidr' => null, 'private_mac' => $expectation['private_mac'] ?? null, + 'xml_verified' => false, 'config_verified' => false, + 'seed_verified' => false, + 'domain_running' => false, 'restart_required' => null, + 'guest_network_verified' => false, + 'guest_network_readback' => 'missing', + 'guest_link_state' => null, 'reason_code' => 'vpc_private_observation_missing', 'observed_at' => time(), ]; @@ -2637,7 +3083,12 @@ final class CloudVpcService foreach (['version', 'required', 'vm_uuid', 'vpc_id', 'member_id', 'interface', 'state', 'desired_revision', 'observed_revision', 'desired_digest', 'observed_digest', 'readback', 'link_state', - 'private_ip_cidr', 'private_mac', 'reason_code', 'observed_at'] as $key) { + 'private_ip_cidr', 'private_mac', 'xml_verified', + 'config_verified', 'seed_verified', 'domain_running', + 'restart_required', + 'guest_network_verified', 'guest_network_readback', + 'guest_link_state', + 'reason_code', 'observed_at'] as $key) { if (array_key_exists($key, $observation)) { $state[$key] = $observation[$key]; } @@ -2667,6 +3118,23 @@ final class CloudVpcService ? strtolower(trim($observation['state'])) : 'uncertain'; $state['readback'] = is_string($observation['readback'] ?? null) ? strtolower(trim($observation['readback'])) : 'missing'; + $state['xml_verified'] = ($observation['xml_verified'] ?? false) === true; + $state['config_verified'] = + ($observation['config_verified'] ?? false) === true; + $state['seed_verified'] = + ($observation['seed_verified'] ?? false) === true; + $state['domain_running'] = + ($observation['domain_running'] ?? false) === true; + $state['restart_required'] = $observation['restart_required'] ?? null; + $state['guest_network_verified'] = + ($observation['guest_network_verified'] ?? false) === true; + $state['guest_network_readback'] = is_string( + $observation['guest_network_readback'] ?? null + ) ? strtolower(trim((string) $observation['guest_network_readback'])) + : 'missing'; + $state['guest_link_state'] = is_string( + $observation['guest_link_state'] ?? null + ) ? strtolower(trim((string) $observation['guest_link_state'])) : null; if (!$expectedRequired) { // When a structured observation is available, a legacy VM is @@ -2686,16 +3154,35 @@ final class CloudVpcService 'error_code' => $ok ? null : $state['reason_code']]; } + $policyActive = str_ends_with($kind, '_policy_active'); + $policyBlocked = str_ends_with($kind, '_policy_blocked'); + $policyUnready = str_ends_with($kind, '_policy_unready'); + $gracefulShutdownTarget = str_starts_with( + $kind, 'action_shutdown_policy_' + ); + $stoppedTarget = str_starts_with($kind, 'action_poweroff_policy_') + || str_starts_with($kind, 'service_suspend_policy_') + || str_starts_with($kind, 'service_resume_stopped_policy_'); + $runningTarget = str_starts_with($kind, 'action_start_policy_') + || str_starts_with($kind, 'action_reboot_policy_') + || str_starts_with($kind, 'action_reset_policy_') + || str_starts_with($kind, 'service_resume_start_policy_'); + $unboundResume = str_starts_with( + $kind, 'service_resume_unbound_policy_' + ); + $preserveTarget = $kind === 'policy_skipped'; $linkExpected = $expectedState === 'removed' ? 'absent' : match ($kind) { 'network_suspend' => 'down', - // service_resume restores the VM power/autostart target. A - // quota-blocked VM may still intentionally keep every NIC down; - // network_resume is the separate operation that proves link-up. 'network_resume' => 'up', default => null, }; + if ($expectedState !== 'removed' + && ($gracefulShutdownTarget || $stoppedTarget || $runningTarget)) { + $linkExpected = $policyActive ? 'up' + : ($policyBlocked ? 'down' : null); + } $identity = is_string($expectation['vpc_id'] ?? null) && is_string($expectation['member_id'] ?? null) && $observedVpcId !== null && $observedMemberId !== null @@ -2728,17 +3215,81 @@ final class CloudVpcService && is_string($state['private_mac'] ?? null) && hash_equals(strtolower((string) $expectation['private_mac']), strtolower((string) $state['private_mac']))); - $desired = $expectedState === 'removed' - ? in_array($state['state'], ['disabled', 'removed'], true) - : in_array($state['state'], ['active', 'suspended'], true); + $stoppedEvidence = $state['state'] === 'stopped'; + if ($expectedState === 'removed') { + $desired = in_array($state['state'], ['disabled', 'removed'], true); + } elseif ($policyUnready || $unboundResume) { + $desired = false; + } elseif ($stoppedTarget) { + $desired = $stoppedEvidence; + } elseif ($gracefulShutdownTarget) { + $desired = $stoppedEvidence + || ($policyActive && $state['state'] === 'active') + || ($policyBlocked && $state['state'] === 'suspended'); + } elseif ($runningTarget) { + $desired = $policyActive + ? $state['state'] === 'active' + : ($policyBlocked && $state['state'] === 'suspended'); + } elseif ($preserveTarget) { + $desired = in_array( + $state['state'], ['active', 'suspended', 'stopped'], true + ); + } elseif ($kind === 'network_resume') { + $desired = in_array($state['state'], ['active', 'stopped'], true); + } elseif ($kind === 'network_suspend') { + $desired = in_array($state['state'], ['suspended', 'stopped'], true); + } else { + $desired = in_array($state['state'], ['active', 'suspended'], true); + } $link = $linkExpected === null || $state['link_state'] === $linkExpected; - $ok = $shape && $identity && $revision && $digest && $ip && $mac && $desired && $link - && $state['readback'] === 'confirmed'; + $stoppedConfigExact = !$stoppedEvidence || ( + $state['xml_verified'] === true + && $state['config_verified'] === true + && $state['seed_verified'] === true + && $state['domain_running'] === false + && $state['restart_required'] === false + && in_array($state['link_state'], ['up', 'down'], true) + ); + // Every active result crosses the same guest-visible readiness + // boundary. Do not let a rolling/preview engine make a generic + // lifecycle job successful with the older XML-only observation + // shape merely because that job is not a start/resume operation. + $freshRunningProofRequired = $state['state'] === 'active'; + $freshRunningProof = !$freshRunningProofRequired || ( + $state['xml_verified'] === true + && $state['config_verified'] === true + && $state['seed_verified'] === true + && $state['domain_running'] === true + && $state['restart_required'] === false + && $state['guest_network_verified'] === true + && $state['guest_network_readback'] === 'confirmed' + && $state['guest_link_state'] === 'up' + ); + $intentionalSuspendedProof = $state['state'] !== 'suspended' + || ( + $state['xml_verified'] === true + && $state['config_verified'] === true + && $state['seed_verified'] === true + && $state['domain_running'] === true + && $state['restart_required'] === false + && $state['link_state'] === 'down' + ); + $ok = $shape && $identity && $revision && $digest && $ip && $mac + && $desired && $link && $stoppedConfigExact && $freshRunningProof + && $intentionalSuspendedProof && $state['readback'] === 'confirmed'; if (!$ok) { $state['state'] = $state['state'] === 'degraded' ? 'degraded' : 'uncertain'; $state['readback'] = $state['readback'] === 'mismatch' ? 'mismatch' : 'missing'; $state['reason_code'] = $expectedState === 'removed' ? 'vpc_private_teardown_uncertain' : 'vpc_private_observation_mismatch'; + } elseif ($stoppedEvidence) { + $state['reason_code'] = 'vpc_private_guest_stopped'; + } elseif ($gracefulShutdownTarget) { + // ACPI shutdown success confirms request acceptance, not immediate + // domain inactivity. Persist a non-usable transition and let the + // VPC/runtime observers replace it with exact stopped evidence. + $state['state'] = 'stopping'; + $state['reason_code'] = 'vpc_private_guest_shutdown_pending'; } return ['ok' => $ok, 'state' => $state, 'error_code' => $ok ? null : $state['reason_code']]; @@ -2844,11 +3395,21 @@ final class CloudVpcService } $member->observed_revision = max(0, (int) ($state['observed_revision'] ?? 0)); $member->observed_digest = self::safeDigest($state['observed_digest'] ?? null); + $runtimeSuspended = $ok && $desired !== 'removed' + && (string) ($state['state'] ?? '') === 'suspended' + && (string) ($state['link_state'] ?? '') === 'down'; + $runtimeStopped = $ok && $desired !== 'removed' + && (string) ($state['state'] ?? '') === 'stopped'; + $runtimeStopping = $ok && $desired !== 'removed' + && (string) ($state['state'] ?? '') === 'stopping'; // Mirror the member outcome into its lease row. A private-IP // state left at `deleting` or `active` after readback would make // the next expectation disagree with the member ledger. $privateState = $ok - ? ($desired === 'removed' ? 'deleted' : 'active') + ? ($desired === 'removed' ? 'deleted' + : ($runtimeStopped ? 'stopped' + : ($runtimeStopping ? 'stopping' + : ($runtimeSuspended ? 'suspended' : 'active')))) : ($desired === 'removed' ? 'deleting' : 'uncertain'); if ($ok && $desired === 'removed') { $member->state = 'deleted'; @@ -2860,8 +3421,13 @@ final class CloudVpcService $private->updated_at = time(); $private->save(); } elseif ($ok) { - $member->state = 'active'; - $member->last_error_code = null; + $member->state = $runtimeStopped ? 'stopped' + : ($runtimeStopping ? 'stopping' + : ($runtimeSuspended ? 'suspended' : 'active')); + $member->last_error_code = $runtimeStopped + ? 'vpc_private_guest_stopped' + : ($runtimeStopping ? 'vpc_private_guest_shutdown_pending' + : ($runtimeSuspended ? 'vpc_private_network_suspended' : null)); } else { $member->state = $desired === 'removed' ? 'deleting' : 'uncertain'; $member->last_error_code = is_string($validation['error_code'] ?? null) @@ -2875,10 +3441,105 @@ final class CloudVpcService $private->last_error_code = $member->last_error_code; $private->updated_at = time(); $private->save(); + self::refreshVpcRuntimeAggregate($vpc, (int) $member->host_id); return true; }, 3); } + /** Refresh host/root runtime state without changing exact config proof. */ + private static function refreshVpcRuntimeAggregate(CloudVpc $vpc, int $hostId): void + { + $now = time(); + $host = CloudVpcHostDeployment::where('vpc_id', $vpc->id) + ->where('host_id', $hostId)->lockForUpdate()->first(); + if ($host !== null && (string) $host->state !== 'deleting' + && (string) $host->state !== 'deleted') { + $hostMembers = CloudVpcMember::where('vpc_id', $vpc->id) + ->where('host_id', $hostId) + ->whereNotIn('state', ['deleting', 'deleted'])->get(); + $hostReady = !$hostMembers->isEmpty() + && $hostMembers->every(static fn ($row): bool => + (string) $row->state === 'active'); + $hostError = null; + foreach ($hostMembers as $hostMember) { + if ((string) $hostMember->state === 'active') { + continue; + } + $hostError = self::safeErrorCode( + (string) ($hostMember->last_error_code ?? '') + ) ?? 'vpc_private_observation_missing'; + break; + } + $host->state = $hostReady ? 'active' : 'degraded'; + $host->last_error_code = $hostReady ? null : $hostError; + $host->updated_at = $now; + if ($hostReady) { + $host->last_success_at = $now; + } else { + $host->last_failure_at = $now; + } + $host->save(); + } + + $members = CloudVpcMember::where('vpc_id', $vpc->id) + ->whereNotIn('state', ['deleting', 'deleted'])->get(); + $memberIds = $members->pluck('id')->map( + static fn ($id): int => (int) $id + )->values()->all(); + $leases = CloudVpcPrivateIp::where('vpc_id', $vpc->id) + ->whereIn('member_id', $memberIds)->get()->keyBy('member_id'); + $runtimeReady = !$members->isEmpty(); + $rootError = null; + foreach ($members as $member) { + $lease = $leases->get((int) $member->id); + if ((string) $member->state === 'active' + && $lease !== null && (string) $lease->state === 'active') { + continue; + } + $runtimeReady = false; + $runtimeError = self::safeErrorCode( + (string) ($member->last_error_code ?? '') + ); + if ($runtimeError === null && $lease !== null) { + $runtimeError = self::safeErrorCode( + (string) ($lease->last_error_code ?? '') + ); + } + $rootError ??= self::safeErrorCode( + (string) ($runtimeError ?? '') + ) ?? 'vpc_private_observation_missing'; + } + $hostsReady = !CloudVpcHostDeployment::where('vpc_id', $vpc->id) + ->whereNotIn('state', ['active', 'deleted'])->exists(); + $runtimeReady = $runtimeReady && $hostsReady; + $configExact = (int) $vpc->desired_revision === (int) $vpc->observed_revision + && self::digestMatches( + (string) $vpc->desired_digest, + (string) $vpc->observed_digest + ); + if ($configExact && (string) $vpc->state !== 'deleting' + && (string) $vpc->state !== 'deleted') { + $vpc->state = $runtimeReady ? 'active' : 'degraded'; + if ($runtimeReady) { + $vpc->sync_state = 'applied'; + $vpc->last_error_code = null; + $vpc->last_success_at = $now; + } else { + // An intentional lifecycle link-down does not invalidate the + // shared host configuration receipt. Preserve an applied + // config state; other runtime gaps retain their waiting lane. + if ((string) $vpc->sync_state === 'applied') { + $vpc->sync_state = 'applied'; + } + $vpc->last_error_code = $rootError + ?? 'vpc_private_observation_missing'; + $vpc->last_failure_at = $now; + } + $vpc->updated_at = $now; + $vpc->save(); + } + } + private static function safeDigest($value): ?string { return is_string($value) && preg_match('/^[a-f0-9]{64}$/Di', $value) === 1 @@ -3194,6 +3855,8 @@ final class CloudVpcService $revision = (int) $operation->desired_revision; if ((string) $operation->status !== 'succeeded' || ($operation->error_code ?? null) !== null + || (int) ($operation->evidence_version ?? 0) + < self::OBSERVATION_EVIDENCE_VERSION || $revision < 1 || $createdAt < 1 || $finishedAt < $createdAt) { return null; } @@ -3228,6 +3891,17 @@ final class CloudVpcService || !is_array($desiredRoutes) || !array_is_list($desiredRoutes)) { return null; } + try { + $expectedVpcCidr = self::cidr( + $payload['cidr'] ?? null, 16, 24, 'VPC CIDR' + )['cidr']; + } catch (\Throwable) { + return null; + } + if (!is_string($payload['cidr'] ?? null) + || !hash_equals($expectedVpcCidr, (string) $payload['cidr'])) { + return null; + } $expectedMembersByHost = []; $allMembers = []; $memberIds = []; @@ -3340,7 +4014,8 @@ final class CloudVpcService && self::validateHostDataplaneEvidence( $item, $hostId, $localMembers, array_values($allMembers), $expectedDestinations, - $desiredRoutes, (string) $vpc->vpc_key, + $expectedVpcCidr, $desiredRoutes, + (string) $vpc->vpc_key, $revision, $digest ) === null; if ($exact) { @@ -3370,7 +4045,7 @@ final class CloudVpcService ) || self::validateHostDataplaneEvidence( $item, $hostId, [], array_values($allMembers), - $expectedDestinations, $desiredRoutes, + $expectedDestinations, $expectedVpcCidr, $desiredRoutes, (string) $vpc->vpc_key, $revision, $digest ) !== null) { return null; @@ -3442,7 +4117,9 @@ final class CloudVpcService /** * Release member and lease tombstones after the complete host set has - * supplied exact libvirt/data-plane readback for this desired revision. + * supplied exact persistent libvirt/host configuration readback for this + * desired revision. Guest readiness is a separate usability gate and must + * not keep a proven-absent historical NIC/address identity allocated. * * Keeping terminal rows would continue to occupy the global instance and * private-address unique keys. Validate the one-to-one ledger under the @@ -3451,7 +4128,7 @@ final class CloudVpcService */ private static function releaseMemberTombstones( CloudVpc $vpc, int $revision, string $digest, bool $includeDeleting, - array $currentActiveExactHostIds + array $currentConfigExactHostIds ): int { $states = $includeDeleting ? ['deleting', 'deleted'] : ['deleted']; $members = CloudVpcMember::where('vpc_id', $vpc->id) @@ -3481,15 +4158,11 @@ final class CloudVpcService $private = $privateById[$privateId] ?? null; $memberState = (string) $member->state; $owner = $owners->get((int) $member->host_id); - $ownerActiveCurrent = isset( - $currentActiveExactHostIds[(int) $member->host_id] + $ownerConfigCurrent = isset( + $currentConfigExactHostIds[(int) $member->host_id] ) && $owner !== null - && (string) $owner->state === 'active' && (int) $owner->desired_revision === $revision - && (int) $owner->observed_revision === $revision - && self::digestMatches($digest, (string) $owner->desired_digest) - && self::digestMatches($digest, (string) $owner->observed_digest) - && ($owner->last_error_code ?? null) === null; + && self::digestMatches($digest, (string) $owner->desired_digest); $ownerRemoved = $owner !== null && self::hostDeploymentRemovalProven($owner); $deletedPairProven = $memberState !== 'deleted' || ( @@ -3511,7 +4184,7 @@ final class CloudVpcService || (int) $private->subnet_id !== (int) $member->subnet_id || (string) $private->private_ip !== (string) $member->private_ip || !$deletedPairProven - || (!$ownerActiveCurrent && !$ownerRemoved)) { + || (!$ownerConfigCurrent && !$ownerRemoved)) { throw new CloudError('VPC 待释放成员与私网地址账本不一致', 409); } $usedPrivateIds[$privateId] = true; @@ -3571,6 +4244,114 @@ final class CloudVpcService return count($subnetIds); } + /** + * Persist runtime readiness independently for each live member/lease. + * Host configuration is already exact when this method is called; only a + * matching guest proof may advance one pair to active for this revision. + */ + private static function persistMemberGuestEvidence( + CloudVpc $vpc, + int $revision, + string $digest, + string $operationId, + array $expectedMembers, + array $memberGuestEvidence, + int $observedAt + ): void { + $members = CloudVpcMember::where('vpc_id', $vpc->id) + ->whereNotIn('state', ['deleting', 'deleted']) + ->orderBy('id')->lockForUpdate()->get() + ->keyBy(static fn ($row): string => (string) $row->member_key); + if ($members->count() !== count($expectedMembers)) { + throw new CloudError('VPC 成员来宾证据账本不一致', 409); + } + $privateIds = $members->pluck('private_ip_id')->map( + static fn ($id): int => (int) $id + )->filter(static fn (int $id): bool => $id > 0)->values()->all(); + $privateIps = CloudVpcPrivateIp::where('vpc_id', $vpc->id) + ->whereIn('id', $privateIds)->orderBy('id')->lockForUpdate()->get()->keyBy('id'); + foreach ($expectedMembers as $memberId => $expected) { + $member = $members->get((string) $memberId); + $private = $member === null ? null + : $privateIps->get((int) $member->private_ip_id); + $expectedIp = is_array($expected) + ? preg_replace('/\/32$/D', '', (string) ($expected['private_ip_cidr'] ?? '')) + : null; + if ($member === null || $private === null || !is_array($expected) + || $expectedIp === null || $expectedIp === '' + || (int) $member->host_id !== (int) ($expected['host_id'] ?? 0) + || (string) $member->private_ip !== $expectedIp + || (int) $private->member_id !== (int) $member->id + || (int) $private->instance_id !== (int) $member->instance_id + || (int) $private->subnet_id !== (int) $member->subnet_id + || (string) $private->private_ip !== (string) $member->private_ip + || (int) $member->desired_revision !== $revision + || (int) $private->desired_revision !== $revision + || !self::digestMatches($digest, (string) $member->desired_digest) + || !self::digestMatches($digest, (string) $private->desired_digest)) { + throw new CloudError('VPC 成员来宾证据账本不一致', 409); + } + $runtimeEvidence = $memberGuestEvidence[(string) $memberId] ?? null; + $runtimeState = is_array($runtimeEvidence) + ? (string) ($runtimeEvidence['state'] ?? 'degraded') : 'degraded'; + $errorCode = is_array($runtimeEvidence) + ? ($runtimeEvidence['error_code'] ?? null) + : 'vpc_private_observation_missing'; + $configShapeExact = is_array($runtimeEvidence) + && ($runtimeEvidence['xml_verified'] ?? false) === true + && ($runtimeEvidence['config_verified'] ?? false) === true + && ($runtimeEvidence['seed_verified'] ?? false) === true + && ($runtimeEvidence['restart_required'] ?? null) === false; + $activeExact = $runtimeState === 'active' && $configShapeExact + && ($runtimeEvidence['domain_running'] ?? false) === true + && ($runtimeEvidence['guest_exact'] ?? false) === true + && ($runtimeEvidence['guest_network_verified'] ?? false) === true + && ($runtimeEvidence['guest_network_readback'] ?? null) === 'confirmed' + && ($runtimeEvidence['link_state'] ?? null) === 'up' + && ($runtimeEvidence['guest_link_state'] ?? null) === 'up'; + $suspendedExact = $runtimeState === 'suspended' && $configShapeExact + && ($runtimeEvidence['domain_running'] ?? false) === true + && ($runtimeEvidence['link_state'] ?? null) === 'down'; + $stoppingExact = $runtimeState === 'stopping' && $configShapeExact + && ($runtimeEvidence['domain_running'] ?? false) === true + && in_array(($runtimeEvidence['link_state'] ?? null), + ['up', 'down'], true); + $stoppedExact = $runtimeState === 'stopped' && $configShapeExact + && ($runtimeEvidence['domain_running'] ?? null) === false + && in_array(($runtimeEvidence['link_state'] ?? null), + ['up', 'down'], true); + $exact = $activeExact || $suspendedExact || $stoppingExact + || $stoppedExact; + if ($exact) { + $member->observed_revision = $revision; + $member->observed_digest = $digest; + $member->state = $stoppedExact ? 'stopped' + : ($stoppingExact ? 'stopping' + : ($suspendedExact ? 'suspended' : 'active')); + $member->last_operation_id = $operationId; + $member->last_error_code = $stoppedExact + ? 'vpc_private_guest_stopped' + : ($stoppingExact ? 'vpc_private_guest_shutdown_pending' + : ($suspendedExact ? 'vpc_private_network_suspended' : null)); + $private->observed_revision = $revision; + $private->observed_digest = $digest; + $private->state = $member->state; + $private->last_error_code = $member->last_error_code; + } else { + $safeError = self::deferredEvidenceError($errorCode) + ?? 'vpc_private_observation_mismatch'; + $member->state = 'degraded'; + $member->last_error_code = $safeError; + $private->state = 'degraded'; + $private->last_error_code = $safeError; + } + $member->updated_at = $observedAt; + $member->save(); + $private->updated_at = $observedAt; + $private->save(); + } + } + private static function touchDesiredLocked(CloudVpc $vpc, string $kind, string $requestKey, string $requestHash): array { @@ -3673,6 +4454,7 @@ final class CloudVpcService $operation->idempotency_key = $requestKey; $operation->request_hash = $requestHash; $operation->status = 'queued'; + $operation->evidence_version = 0; $operation->payload_json = json_encode($document, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE); $operation->created_at = $operation->updated_at = time(); @@ -3797,13 +4579,48 @@ final class CloudVpcService // instance that has left the active lifecycle state. This closes the // window where a delete_failed/uncertain VM could leave an otherwise // matching root digest looking healthy to the tenant UI. - $memberInstances = CloudVpcMember::where('vpc_id', $vpc->id) - ->whereNotIn('state', ['deleted'])->pluck('instance_id')->unique()->values()->all(); + $runtimeMembers = CloudVpcMember::where('vpc_id', $vpc->id) + ->whereNotIn('state', ['deleted'])->get([ + 'id', 'instance_id', 'private_ip_id', 'state', + ]); + $memberInstances = $runtimeMembers->pluck('instance_id') + ->map(static fn ($id): int => (int) $id) + ->unique()->values()->all(); if ($memberInstances !== []) { $instanceStates = CloudInstance::whereIn('id', $memberInstances) - ->pluck('status', 'id'); - foreach ($memberInstances as $instanceId) { - if ((string) ($instanceStates[(int) $instanceId] ?? '') !== 'active') { + ->get(['id', 'host_id', 'uuid', 'status', 'network_status']) + ->keyBy('id'); + $runtimeStates = CloudInstanceRuntimeService::forInstances( + $instanceStates->all() + ); + $privateIds = $runtimeMembers->pluck('private_ip_id') + ->map(static fn ($id): int => (int) $id) + ->filter(static fn (int $id): bool => $id > 0)->values()->all(); + $privateStates = CloudVpcPrivateIp::where('vpc_id', $vpc->id) + ->whereIn('id', $privateIds)->pluck('state', 'id'); + foreach ($runtimeMembers as $runtimeMember) { + $runtime = $runtimeStates[ + (string) ((int) $runtimeMember->instance_id) + ] ?? null; + if ((string) $runtimeMember->state !== 'active' + || (string) ($privateStates[(int) $runtimeMember->private_ip_id] ?? '') + !== 'active' + || (string) ($instanceStates->get( + (int) $runtimeMember->instance_id + )?->status ?? '') !== 'active' + || (string) ($instanceStates->get( + (int) $runtimeMember->instance_id + )?->network_status ?? '') !== 'active') { + $hostsReady = false; + break; + } + // Only a fresh, identity-bound running observation can support + // tenant usability. A graceful-shutdown transition, stopped, + // paused, crashed, unknown or stale cache is immediately + // non-usable even if the last VPC/QGA receipt was active. + if (!is_array($runtime) + || ($runtime['state_fresh'] ?? false) !== true + || (string) ($runtime['power_state'] ?? '') !== 'running') { $hostsReady = false; break; } @@ -3820,6 +4637,16 @@ final class CloudVpcService } } $rootReady = self::digestMatches($desiredDigest, $observedDigest); + $currentOperationReady = false; + $lastOperationId = (string) ($vpc->last_operation_id ?? ''); + if (preg_match(self::KEY_PATTERN, $lastOperationId) === 1) { + $currentOperationReady = CloudVpcOperation::where('vpc_id', $vpc->id) + ->where('operation_id', $lastOperationId) + ->where('desired_revision', (int) $vpc->desired_revision) + ->where('status', 'succeeded') + ->where('evidence_version', '>=', self::OBSERVATION_EVIDENCE_VERSION) + ->exists(); + } $result = [ 'vpc_key' => (string) $vpc->vpc_key, 'name' => (string) $vpc->name, 'region_id' => (int) $vpc->region_id, 'cidr' => (string) $vpc->cidr, @@ -3830,7 +4657,7 @@ final class CloudVpcService 'usable' => (string) $vpc->state === 'active' && (string) $vpc->sync_state === 'applied' && (int) $vpc->desired_revision === (int) $vpc->observed_revision - && $rootReady && $hostsReady, + && $rootReady && $hostsReady && $currentOperationReady, 'last_operation_id' => $vpc->last_operation_id, 'last_error_code' => self::safeErrorCode((string) ($vpc->last_error_code ?? '')), 'created_at' => (int) $vpc->created_at, 'updated_at' => (int) $vpc->updated_at, @@ -3900,6 +4727,7 @@ final class CloudVpcService { return ['operation_id' => (string) $row->operation_id, 'kind' => (string) $row->kind, 'desired_revision' => (int) $row->desired_revision, 'status' => (string) $row->status, + 'evidence_version' => (int) ($row->evidence_version ?? 0), 'error_code' => self::safeErrorCode((string) ($row->error_code ?? '')), 'created_at' => (int) $row->created_at, 'updated_at' => (int) $row->updated_at, 'finished_at' => $row->finished_at === null ? null : (int) $row->finished_at]; @@ -4300,6 +5128,121 @@ final class CloudVpcService return 10000 + (hexdec(substr($digest, 12, 8)) % 10000); } + /** + * Propagate one actionable guest-readback reason without allowing an + * arbitrary host string, or one host's reason, to mask a different host. + */ + private static function singleDeferredEvidenceError(array $hostEvidence): ?string + { + $codes = []; + foreach ($hostEvidence as $evidence) { + if (!is_array($evidence)) { + continue; + } + $code = self::deferredEvidenceError( + $evidence['last_error_code'] ?? null + ); + if (($evidence['active'] ?? false) === true && $code === null) { + continue; + } + if ($code === null) { + return null; + } + $codes[$code] = true; + } + return count($codes) === 1 ? (string) array_key_first($codes) : null; + } + + /** Accept only reviewed reasons; syntactically valid engine text is not enough. */ + private static function deferredEvidenceError($value): ?string + { + return is_string($value) + && in_array($value, self::DEFERRED_EVIDENCE_ERRORS, true) + ? $value : null; + } + + /** Validate one member NIC without coupling it to sibling guest state. */ + private static function validateMemberNicEvidence( + array $nic, + array $expected, + string $expectedVpcCidr, + bool $requireGuestEvidence + ): ?string { + $memberId = (string) ($expected['member_id'] ?? ''); + $observedMac = strtolower((string) ($nic['private_mac'] ?? '')); + if ($memberId === '' + || (string) ($nic['member_id'] ?? '') !== $memberId + || (string) ($nic['interface'] ?? '') !== self::PRIVATE_INTERFACE + || strtolower((string) ($nic['instance_uuid'] ?? '')) + !== strtolower((string) ($expected['instance_uuid'] ?? '')) + || (string) ($nic['private_ip_cidr'] ?? '') + !== (string) ($expected['private_ip_cidr'] ?? '') + || $observedMac !== strtolower((string) ($expected['private_mac'] ?? '')) + || ($nic['config_verified'] ?? false) !== true + || ($nic['seed_verified'] ?? false) !== true) { + return 'vpc_private_nic_readback_mismatch'; + } + if (($nic['xml_verified'] ?? false) !== true) { + return 'vpc_private_nic_readback_mismatch'; + } + if (!$requireGuestEvidence) { + return null; + } + if (($nic['restart_required'] ?? null) === true + || ($nic['domain_running'] ?? null) === false) { + return 'vpc_private_guest_restart_required'; + } + if (($nic['restart_required'] ?? null) !== false + || ($nic['domain_running'] ?? null) !== true) { + return 'vpc_private_guest_agent_unavailable'; + } + if (($nic['guest_network_verified'] ?? false) !== true) { + return self::deferredEvidenceError( + $nic['guest_reason_code'] ?? null + ) ?? 'vpc_private_guest_probe_uncertain'; + } + try { + $canonicalVpcCidr = self::cidr( + $expectedVpcCidr, 16, 24, 'VPC CIDR' + )['cidr']; + } catch (\Throwable) { + return 'vpc_private_guest_route_mismatch'; + } + if (!hash_equals($canonicalVpcCidr, $expectedVpcCidr)) { + return 'vpc_private_guest_route_mismatch'; + } + $guestRoutes = $nic['guest_routes'] ?? null; + if (!is_array($guestRoutes) || count($guestRoutes) !== 1) { + return 'vpc_private_guest_route_mismatch'; + } + $normalizedGuestRoutes = []; + foreach ($guestRoutes as $guestRoute) { + if (!is_string($guestRoute) || strlen($guestRoute) > 64) { + return 'vpc_private_guest_route_mismatch'; + } + $normalizedGuestRoutes[] = $guestRoute; + } + if (count(array_unique($normalizedGuestRoutes)) + !== count($normalizedGuestRoutes)) { + return 'vpc_private_guest_route_mismatch'; + } + if ((string) ($nic['guest_network_readback'] ?? '') !== 'confirmed') { + return 'vpc_private_guest_probe_uncertain'; + } + if (($nic['link_state'] ?? null) !== 'up' + || ($nic['guest_link_state'] ?? null) !== 'up') { + return 'vpc_private_guest_link_mismatch'; + } + if (strtolower((string) ($nic['guest_private_mac'] ?? '')) + !== strtolower((string) ($expected['private_mac'] ?? '')) + || (string) ($nic['guest_private_ip_cidr'] ?? '') + !== (string) ($expected['private_ip_cidr'] ?? '')) { + return 'vpc_private_guest_address_mismatch'; + } + return $normalizedGuestRoutes === [$expectedVpcCidr] + ? null : 'vpc_private_guest_route_mismatch'; + } + /** * Validate typed host-side private dataplane evidence. Raw command output * is deliberately ignored: only bounded identity, route, ACL and XML @@ -4314,10 +5257,12 @@ final class CloudVpcService array $expectedMembers, array $allMembers, array $expectedDestinations, + string $expectedVpcCidr, array $desiredRoutes, string $vpcKey, int $revision, - string $digest + string $digest, + bool $requireGuestEvidence = true ): ?string { // These fields are identity evidence, not optional status hints. A // compact/legacy host projection must therefore stay closed until a @@ -4331,10 +5276,7 @@ final class CloudVpcService || (string) $item['vpc_id'] !== $vpcKey || !self::digestMatches($digest, (string) $item['desired_digest']) || (string) $item['host_id'] !== $hostId - || (string) ($item['state'] ?? '') !== 'active' - || (int) $item['desired_revision'] !== $revision - || (int) $item['observed_revision'] !== $revision - || !self::digestMatches($digest, (string) $item['observed_digest'])) { + || (int) $item['desired_revision'] !== $revision) { return 'vpc_private_readback_mismatch'; } // A host removal is a distinct typed proof. It must not be treated @@ -4354,6 +5296,9 @@ final class CloudVpcService } if ($expectedMembers !== [] || $readbackState !== 'confirmed' + || (string) ($item['state'] ?? '') !== 'active' + || (int) $item['observed_revision'] !== $revision + || !self::digestMatches($digest, (string) $item['observed_digest']) || ($item['bridge'] ?? null) !== null || ($item['bridge_owner'] ?? null) !== null || ($item['nft_table'] ?? null) !== null @@ -4362,6 +5307,27 @@ final class CloudVpcService } return null; } + if (($item['config_verified'] ?? false) !== true + || (int) ($item['config_observed_revision'] ?? 0) !== $revision + || !self::digestMatches( + $digest, (string) ($item['config_observed_digest'] ?? '') + )) { + return 'vpc_private_dataplane_unverified'; + } + if ($requireGuestEvidence + && ((string) ($item['state'] ?? '') !== 'active' + || (int) $item['observed_revision'] !== $revision + || !self::digestMatches( + $digest, (string) ($item['observed_digest'] ?? '') + ))) { + return 'vpc_private_readback_mismatch'; + } + if (!$requireGuestEvidence + && !in_array((string) ($item['state'] ?? ''), [ + 'applying', 'active', 'degraded', + ], true)) { + return 'vpc_private_readback_mismatch'; + } $bridge = self::privateBridgeForVpc($vpcKey); $nftTable = self::privateNftTableForVpc($vpcKey); $routeTable = self::privateRouteTableForVpc($vpcKey); @@ -4384,11 +5350,12 @@ final class CloudVpcService : (is_array($readback) ? strtolower((string) ($readback['state'] ?? $readback['status'] ?? '')) : ''); - if ($readbackState !== 'confirmed') { + if ($requireGuestEvidence && $readbackState !== 'confirmed') { return 'vpc_private_readback_missing'; } - // The bridge uses routed guest /32s. Generic proxy ARP stays off; + // The host bridge routes each guest address as an exact /32. Generic + // proxy ARP stays off; // only exact panel-owned proxy-neighbour entries (with pvlan enabled // for same-bridge next hops) may answer. A missing or broadened // proxy set keeps the host degraded. @@ -4436,15 +5403,14 @@ final class CloudVpcService return 'vpc_private_nic_readback_mismatch'; } $expected = $expectedMembers[$memberId]; - $observedMac = strtolower((string) ($nic['private_mac'] ?? '')); - if ((string) ($nic['interface'] ?? '') !== self::PRIVATE_INTERFACE - || strtolower((string) ($nic['instance_uuid'] ?? '')) - !== strtolower((string) $expected['instance_uuid']) - || (string) ($nic['private_ip_cidr'] ?? '') - !== (string) $expected['private_ip_cidr'] - || $observedMac !== strtolower((string) $expected['private_mac']) - || ($nic['xml_verified'] ?? false) !== true) { - return 'vpc_private_nic_readback_mismatch'; + $memberError = self::validateMemberNicEvidence( + $nic, + $expected, + $expectedVpcCidr, + $requireGuestEvidence + ); + if ($memberError !== null) { + return $memberError; } $seen[$memberId] = true; } @@ -4686,6 +5652,12 @@ final class CloudVpcService 'private_dataplane_verified' => ( ($host['private_dataplane_verified'] ?? false) === true ), + 'config_verified' => ($host['config_verified'] ?? false) === true, + 'config_observed_revision' => max(0, + (int) ($host['config_observed_revision'] ?? 0)), + 'config_observed_digest' => is_string( + $host['config_observed_digest'] ?? null + ) ? substr((string) $host['config_observed_digest'], 0, 64) : null, 'removed' => ($host['removed'] ?? false) === true, 'retiring' => ($host['retiring'] ?? false) === true, 'next_stage_allowed' => ($host['next_stage_allowed'] ?? false) === true, @@ -4725,6 +5697,17 @@ final class CloudVpcService if (!is_array($nic)) { continue; } + $guestRoutes = []; + foreach (array_slice( + is_array($nic['guest_routes'] ?? null) + ? $nic['guest_routes'] : [], + 0, + self::MAX_OBSERVED_DESTINATIONS + ) as $guestRoute) { + if (is_string($guestRoute)) { + $guestRoutes[] = substr($guestRoute, 0, 64); + } + } $bounded['member_nics'][] = [ 'member_id' => is_string($nic['member_id'] ?? null) ? substr($nic['member_id'], 0, 64) : null, @@ -4737,6 +5720,40 @@ final class CloudVpcService 'private_mac' => is_string($nic['private_mac'] ?? null) ? substr(strtolower($nic['private_mac']), 0, 32) : null, 'xml_verified' => ($nic['xml_verified'] ?? false) === true, + 'config_verified' => + ($nic['config_verified'] ?? false) === true, + 'seed_verified' => + ($nic['seed_verified'] ?? false) === true, + // Missing on an older engine response is unsafe: it + // cannot prove that this boot-only NoCloud generation + // was loaded by the running domain. + 'restart_required' => + ($nic['restart_required'] ?? true) !== false, + 'domain_running' => ($nic['domain_running'] ?? false) === true, + 'guest_network_verified' => + ($nic['guest_network_verified'] ?? false) === true, + 'guest_network_readback' => is_string( + $nic['guest_network_readback'] ?? null + ) ? substr($nic['guest_network_readback'], 0, 32) : 'missing', + 'guest_link_state' => is_string( + $nic['guest_link_state'] ?? null + ) && in_array($nic['guest_link_state'], ['up', 'down'], true) + ? $nic['guest_link_state'] : null, + 'guest_reason_code' => is_string( + $nic['guest_reason_code'] ?? null + ) && preg_match('/^[a-z0-9_]{1,64}$/D', + $nic['guest_reason_code']) === 1 + ? $nic['guest_reason_code'] : null, + 'guest_private_mac' => is_string( + $nic['guest_private_mac'] ?? null + ) ? substr(strtolower($nic['guest_private_mac']), 0, 32) : null, + 'guest_private_ip_cidr' => is_string( + $nic['guest_private_ip_cidr'] ?? null + ) ? substr($nic['guest_private_ip_cidr'], 0, 64) : null, + 'link_state' => is_string($nic['link_state'] ?? null) + && in_array($nic['link_state'], ['up', 'down'], true) + ? $nic['link_state'] : null, + 'guest_routes' => $guestRoutes, ]; } } @@ -4787,7 +5804,11 @@ final class CloudVpcService } $hosts[(string) $key] = $bounded; } - $result = ['hosts' => $hosts]; + $result = [ + 'evidence_version' => max(0, (int) ($value['evidence_version'] + ?? $value['observation_version'] ?? 0)), + 'hosts' => $hosts, + ]; if ($terminalProof !== null) { $result['no_data_plane_terminal_proof'] = $terminalProof; } diff --git a/web/app/Services/Cloud/CloudVpcWorkerService.php b/web/app/Services/Cloud/CloudVpcWorkerService.php index 27a28cd5..e6a5544f 100644 --- a/web/app/Services/Cloud/CloudVpcWorkerService.php +++ b/web/app/Services/Cloud/CloudVpcWorkerService.php @@ -38,13 +38,78 @@ final class CloudVpcWorkerService })->orWhere(function ($running) use ($now): void { $running->where('status', 'running') ->where('lease_until', '<=', $now); + })->orWhere(function ($legacy) use ($now): void { + // A rolling upgrade must not keep a previously + // `succeeded` host/libvirt-only receipt usable. + // Re-run only the root's current last operation; + // historical audit rows are never replayed. + $legacy->where('status', 'succeeded') + ->where('kind', '!=', 'vpc_delete') + ->where('evidence_version', '<', + CloudVpcService::OBSERVATION_EVIDENCE_VERSION) + ->where('lease_until', '<=', $now) + ->where('updated_at', '<=', $now - 2) + ->whereExists(static function ($current): void { + $current->selectRaw('1') + ->from('cloud_vpcs as current_vpc') + ->whereColumn('current_vpc.id', + 'cloud_vpc_operations.vpc_id') + ->whereColumn('current_vpc.last_operation_id', + 'cloud_vpc_operations.operation_id') + ->whereColumn('current_vpc.desired_revision', + 'cloud_vpc_operations.desired_revision') + ->where('current_vpc.state', '!=', 'deleted'); + }); + })->orWhere(function ($runtime) use ($now): void { + // Configuration may already be committed while one + // or more live guests are not yet observable. Keep + // the mutation terminal, but idempotently re-apply + // only the current root target until runtime proof + // is exact. + $runtime->where('status', 'succeeded') + ->where('kind', '!=', 'vpc_delete') + ->where('evidence_version', '>=', + CloudVpcService::OBSERVATION_EVIDENCE_VERSION) + ->where('lease_until', '<=', $now) + ->where('updated_at', '<=', $now - 30) + ->whereExists(static function ($current): void { + $current->selectRaw('1') + ->from('cloud_vpcs as current_vpc') + ->whereColumn('current_vpc.id', + 'cloud_vpc_operations.vpc_id') + ->whereColumn('current_vpc.last_operation_id', + 'cloud_vpc_operations.operation_id') + ->whereColumn('current_vpc.desired_revision', + 'cloud_vpc_operations.desired_revision') + ->whereColumn('current_vpc.observed_revision', + 'current_vpc.desired_revision') + ->whereColumn('current_vpc.observed_digest', + 'current_vpc.desired_digest') + ->where('current_vpc.state', 'degraded') + ->where(static function ($runtimeState): void { + $runtimeState->whereIn( + 'current_vpc.sync_state', + ['waiting', 'uncertain'] + )->orWhere(static function ($stopped): void { + $stopped->where( + 'current_vpc.sync_state', 'applied' + )->whereIn( + 'current_vpc.last_error_code', [ + 'vpc_private_guest_shutdown_pending', + 'vpc_private_guest_stopped', + ] + ); + }); + }); + }); }); }) ->orderBy('id')->lockForUpdate()->first(); if ($operation === null) { return null; } - $allowDeferredRetry = (string) $operation->status === 'waiting_observed'; + $allowDeferredRetry = (string) $operation->status === 'waiting_observed' + || (string) $operation->status === 'succeeded'; $vpc = CloudVpc::where('id', (int) $operation->vpc_id)->first(); if ($vpc === null) { $operation->status = 'failed'; diff --git a/web/app/Services/Cloud/CloudWorkerService.php b/web/app/Services/Cloud/CloudWorkerService.php index 898c2f41..2ddb92fc 100644 --- a/web/app/Services/Cloud/CloudWorkerService.php +++ b/web/app/Services/Cloud/CloudWorkerService.php @@ -1190,6 +1190,21 @@ final class CloudWorkerService $status = is_string($operation['status'] ?? null) ? strtolower((string) $operation['status']) : 'uncertain'; $kind = (string) $job->kind; + $lifecycleKind = $kind; + $payloadBody = is_array($payload['body'] ?? null) + ? $payload['body'] : []; + $powerAction = null; + if ($kind === 'action') { + // The generic job kind is not enough to decide whether a stopped + // private-state proof is an expected postcondition. Bind that + // decision to the validated encrypted request payload; never infer + // it from an engine result or from the VM's current state. + $action = is_string($payloadBody['action'] ?? null) + ? (string) $payloadBody['action'] : ''; + if (CloudOperationCatalog::isPowerAction($action)) { + $powerAction = $action; + } + } $postCommitReinstall = $kind === 'reinstall' && $status === 'failed' && self::reinstallNeedsReview($operation); $terminalSuccess = in_array($status, ['succeeded', 'completed', 'done'], true); @@ -1208,6 +1223,33 @@ final class CloudWorkerService } $instance = $job->instance_id === null ? null : CloudInstance::find($job->instance_id); + // Power operations preserve the independently managed network-policy + // target. Bind private readback to the trusted database state instead + // of treating every successful start/reboot as permission to raise a + // quota- or expiry-blocked link. Transitional/failed policy states are + // deliberately unprovable until their own policy job converges. + $networkPolicy = match ((string) ($instance?->network_status ?? '')) { + 'active' => 'active', + 'blocked_traffic', 'blocked_expired' => 'blocked', + default => 'unready', + }; + if ($powerAction !== null) { + $lifecycleKind = 'action_' . $powerAction + . '_policy_' . $networkPolicy; + } elseif ($kind === 'service_suspend') { + $lifecycleKind = 'service_suspend_policy_' . $networkPolicy; + } elseif ($kind === 'service_resume') { + // The resume endpoint defaults false, but the private-state gate + // accepts a stopped postcondition only when that target was + // explicitly captured in the encrypted job payload. A missing or + // non-boolean target remains fail-closed. + $resumeTarget = array_key_exists('start_if_suspended', $payloadBody) + && is_bool($payloadBody['start_if_suspended']) + ? ($payloadBody['start_if_suspended'] ? 'start' : 'stopped') + : 'unbound'; + $lifecycleKind = 'service_resume_' . $resumeTarget + . '_policy_' . $networkPolicy; + } $currentExpectation = CloudVpcService::instancePrivateExpectation( $instance === null ? null : (int) $instance->id ); @@ -1342,7 +1384,7 @@ final class CloudWorkerService 'observed_at' => time(), ], 'error_code' => 'vpc_private_observation_mismatch'] : CloudVpcService::validatePrivateObservation( - $expectation, $observation, $kind + $expectation, $observation, $lifecycleKind ); // A normal private-state GET is intentionally an observation only. A // lifecycle mutation can have replaced the domain XML (or removed a @@ -1356,15 +1398,18 @@ final class CloudWorkerService // Only the PHP policy short-circuit carries this private marker. // An engine result may also expose `skipped` after it has already // proved a target state; that must retain network_suspend/down or - // network_resume/up semantics rather than weakening the gate. + // network_resume/up semantics rather than weakening the gate. A + // skipped target uses a dedicated preserve validator that accepts + // stopped only with exact persistent configuration proof. $privateReconcileSkipped = ($operation['_private_policy_skipped'] ?? false) === true; $reconciled = $this->reconcilePrivateState( - $job, $uuid, $kind, $status, $expectation, $actor, + $job, $uuid, $lifecycleKind, $status, $expectation, $actor, $privateReconcileSkipped ); $validation = CloudVpcService::validatePrivateObservation( $expectation, $reconciled, - $privateReconcileSkipped ? 'action' : $kind, $uuid + $privateReconcileSkipped ? 'policy_skipped' : $lifecycleKind, + $uuid ); } elseif (($expectation['required'] ?? false) === true && $uuid === null && !$expectationStale) { @@ -1372,7 +1417,7 @@ final class CloudWorkerService // an embedded operation payload. Keep it uncertain until the // worker can address the host-side reconcile endpoint explicitly. $validation = CloudVpcService::validatePrivateObservation( - $expectation, null, $kind + $expectation, null, $lifecycleKind ); } if ($instance !== null && ($expectation['required'] ?? false) === true diff --git a/web/public/cloud/vpc.js b/web/public/cloud/vpc.js index 30f2ae72..4fbe5739 100644 --- a/web/public/cloud/vpc.js +++ b/web/public/cloud/vpc.js @@ -10,7 +10,8 @@ const state = { const STATUS_LABELS = Object.freeze({ applying: '应用中', waiting: '等待宿主机回读', uncertain: '结果待核对', - degraded: '部分失败', active: '已生效', deleting: '清理中', unknown: '状态未知', + degraded: '部分失败', active: '已生效', suspended: '网络已暂停', + deleting: '清理中', unknown: '状态未知', }); function element(tag, className = '', textContent = '') { @@ -120,8 +121,8 @@ function rootStatus(root) { function childStatus(value) { const raw = String(value || '').toLowerCase(); - return ['active', 'applying', 'waiting', 'uncertain', 'degraded', 'deleting', 'deleted'] - .includes(raw) ? raw : 'unknown'; + return ['active', 'suspended', 'applying', 'waiting', 'uncertain', 'degraded', + 'deleting', 'deleted'].includes(raw) ? raw : 'unknown'; } function statusPill(status, extra = '') { diff --git a/web/sql/cloud.sql b/web/sql/cloud.sql index 76ac1896..272342c6 100644 --- a/web/sql/cloud.sql +++ b/web/sql/cloud.sql @@ -517,6 +517,7 @@ CREATE TABLE IF NOT EXISTS cloud_vpc_operations ( status VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT 'queued', payload_json LONGTEXT NOT NULL, observed_json LONGTEXT DEFAULT NULL, + evidence_version SMALLINT UNSIGNED NOT NULL DEFAULT 0, error_code VARCHAR(64) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL, created_at BIGINT UNSIGNED NOT NULL, updated_at BIGINT UNSIGNED NOT NULL, @@ -538,6 +539,8 @@ ALTER TABLE cloud_vpc_operations ADD COLUMN IF NOT EXISTS lease_until BIGINT UNSIGNED NOT NULL DEFAULT 0 AFTER lease_token; ALTER TABLE cloud_vpc_operations ADD COLUMN IF NOT EXISTS attempts INT UNSIGNED NOT NULL DEFAULT 0 AFTER lease_until; +ALTER TABLE cloud_vpc_operations ADD COLUMN IF NOT EXISTS + evidence_version SMALLINT UNSIGNED NOT NULL DEFAULT 0 AFTER observed_json; ALTER TABLE cloud_vpc_operations ADD UNIQUE INDEX IF NOT EXISTS cloud_vpc_operation_idempotency (vpc_id, idempotency_key); diff --git a/web/tests/CloudVpcControlPlaneContractTest.php b/web/tests/CloudVpcControlPlaneContractTest.php index f406cfa1..b712f84d 100644 --- a/web/tests/CloudVpcControlPlaneContractTest.php +++ b/web/tests/CloudVpcControlPlaneContractTest.php @@ -25,6 +25,11 @@ $jobService = $read('app/Services/Cloud/CloudJobService.php'); $orderService = $read('app/Services/Cloud/CloudOrderService.php'); $workerService = $read('app/Services/Cloud/CloudWorkerService.php'); $vpcWorkerService = $read('app/Services/Cloud/CloudVpcWorkerService.php'); +$runtimeService = $read('app/Services/Cloud/CloudInstanceRuntimeService.php'); +$operationErrorCatalog = $read('app/Services/Cloud/CloudOperationErrorCatalog.php'); +$engineVpcService = (string) file_get_contents( + dirname($root) . '/engine/virsh_panel/vpc.py' +); foreach ([ 'cloud_region_fabrics', @@ -43,7 +48,8 @@ $assert(str_contains($schema, 'UNIQUE KEY cloud_vpc_member_instance_global (inst && str_contains($schema, 'source_int BIGINT UNSIGNED NOT NULL') && str_contains($schema, 'destination_broadcast_int BIGINT UNSIGNED NOT NULL') && str_contains($schema, 'desired_digest CHAR(64)') - && str_contains($schema, 'observed_digest CHAR(64)'), + && str_contains($schema, 'observed_digest CHAR(64)') + && str_contains($schema, 'evidence_version SMALLINT UNSIGNED NOT NULL DEFAULT 0'), 'VPC schema lost the single-NIC, idempotency or revision digest guards.'); $assert(str_contains($service, 'lockForUpdate()') @@ -53,10 +59,29 @@ $assert(str_contains($service, 'lockForUpdate()') && str_contains($service, 'CloudVpcPrivateIp') && str_contains($service, 'lockedPrivateIpForMember') && str_contains($service, 'CloudVpcHostDeployment') - && str_contains($service, 'CloudEngineClient())->vpc') + && str_contains($service, 'new CloudEngineClient(') + && str_contains($service, '))->vpc(') && str_contains($service, 'canonicalJson') && str_contains($service, 'next_stage_allowed'), 'VPC service lost independent locking, desired/observed revisions or Fabric gate.'); +$assert(str_contains($service, 'public const ENGINE_TIMEOUT_SECONDS = 60.0;') + && str_contains($service, 'public const OPERATION_LEASE_SECONDS = 120;') + && str_contains($engineVpcService, 'VPC_REQUEST_DEADLINE_SECONDS = 40.0') + && str_contains($service, 'self::ENGINE_TIMEOUT_SECONDS') + && str_contains($service, 'self::OPERATION_LEASE_SECONDS'), + 'VPC PHP timeout/lease no longer contain the engine request-wide probe budget.'); +preg_match('/VPC_REQUEST_DEADLINE_SECONDS\s*=\s*([0-9.]+)/', + $engineVpcService, $engineBudgetMatch); +$engineBudget = isset($engineBudgetMatch[1]) ? (float) $engineBudgetMatch[1] : 0.0; +preg_match('/ENGINE_TIMEOUT_SECONDS\s*=\s*([0-9.]+)/', + $service, $phpTimeoutMatch); +preg_match('/OPERATION_LEASE_SECONDS\s*=\s*([0-9]+)/', + $service, $phpLeaseMatch); +$phpTimeout = isset($phpTimeoutMatch[1]) ? (float) $phpTimeoutMatch[1] : 0.0; +$phpLease = isset($phpLeaseMatch[1]) ? (int) $phpLeaseMatch[1] : 0; +$assert($engineBudget > 0.0 && $phpTimeout > $engineBudget + && $phpLease >= $phpTimeout + 30.0, + 'VPC PHP timeout/lease numeric ordering no longer bounds the engine probe.'); $assert(str_contains($service, 'CloudRegionFabric') && str_contains($service, 'fabric_region') && str_contains($adminResource, 'saveRegionFabric') @@ -185,6 +210,23 @@ $assert(str_contains($service, 'privateMacForMember') && str_contains($service, "'private_mac' => \$privateMac") && str_contains($service, 'validateHostDataplaneEvidence') && str_contains($service, 'member_nics') + && str_contains($service, 'restart_required') + && str_contains($service, 'guest_network_verified') + && str_contains($service, '($nic[\'link_state\'] ?? null) !== \'up\'') + && str_contains($service, 'vpc_private_guest_link_mismatch') + && str_contains($service, 'vpc_private_guest_probe_deadline_exceeded') + && str_contains($service, 'vpc_private_guest_probe_reap_required') + && str_contains($service, 'guest_private_ip_cidr') + && str_contains($service, 'guest_routes') + && str_contains($service, 'config_observed_digest') + && str_contains($service, 'OBSERVATION_EVIDENCE_VERSION') + && str_contains($service, "?->network_status ?? '') !== 'active'") + && str_contains($service, 'CloudInstanceRuntimeService::forInstances') + && str_contains($service, "['state_fresh'] ?? false") + && str_contains($service, "['power_state'] ?? '') !== 'running'") + && str_contains($service, "'state' => \$intentionalStopped ? 'stopped'") + && str_contains($service, "'vpc_private_guest_stopped'") + && str_contains($service, "'vpc_private_guest_shutdown_pending'") && str_contains($service, 'wg_allowed_ips') && str_contains($service, 'privateRouteTableForVpc') && str_contains($service, 'expectedRouteSet') @@ -194,10 +236,28 @@ $assert(str_contains($service, 'privateMacForMember') && str_contains($service, 'FABRIC_INTERFACE') && str_contains($service, 'PRIVATE_DATAPLANE_OWNER'), 'VPC desired DTO or host dataplane readback contract lost stable NIC identity/ACL gates.'); +$assert(str_contains($runtimeService, "'shutdown' => 'shutdown'") + && str_contains($runtimeService, "'poweroff' => 'stopped'") + && str_contains($runtimeService, "'start', 'reboot', 'reset' => 'running'"), + 'Runtime cache lost bounded power-action transition semantics.'); +$assert(str_contains( + $operationErrorCatalog, + '私网配置尚未写入当前实例;请完全关机,等待系统核对写入后再开机' + ) && str_contains( + $operationErrorCatalog, + '来宾私网探测清理状态未确认;请完全关机再开机后继续核验' + ) && !str_contains($operationErrorCatalog, '私网地址或路由已暂存'), + 'VPC restart guidance must not claim the pending seed/XML was already staged.'); $assert(str_contains($vpcWorkerService, "'waiting_observed'") && str_contains($vpcWorkerService, 'allowDeferredRetry') + && str_contains($vpcWorkerService, "where('status', 'succeeded')") + && str_contains($vpcWorkerService, 'current_vpc.last_operation_id') + && str_contains($vpcWorkerService, "where('kind', '!=', 'vpc_delete')") + && str_contains($vpcWorkerService, 'current_vpc.sync_state') + && str_contains($vpcWorkerService, 'vpc_private_guest_shutdown_pending') + && str_contains($vpcWorkerService, 'OBSERVATION_EVIDENCE_VERSION') && str_contains($vpcWorkerService, 'CloudVpcService::reconcile($vpcKey, $actor, $allowDeferredRetry)'), - 'VPC worker must retry waiting_observed through its independent apply lane.'); + 'VPC worker must retry waiting or legacy current evidence through its independent apply lane.'); $assert(str_contains($routes, "'/vpcs'") && str_contains($routes, "'/vpcs/{vpc_key:[0-9a-f]{32}}/reconcile'") @@ -257,10 +317,20 @@ $assert(str_contains($workerService, "'_private_policy_skipped' => true") && str_contains($workerService, 'Only the PHP policy short-circuit carries this private marker') && str_contains($workerService, "'link_state' => self::privateReconcileLinkState"), 'A superseded PHP policy must preserve the private link without weakening real engine target proof.'); +$assert(str_contains($workerService, "'blocked_traffic', 'blocked_expired' => 'blocked'") + && str_contains($workerService, "'_policy_' . \$networkPolicy") + && str_contains($workerService, "'start_if_suspended'") + && str_contains($workerService, "? 'start' : 'stopped'") + && str_contains($service, "service_resume_stopped_policy_") + && str_contains($service, "service_resume_start_policy_") + && str_contains($service, "action_reboot_policy_") + && str_contains($service, '$policyUnready || $unboundResume'), + 'Power/service private readback is no longer bound to network policy and payload target.'); $assert(str_contains($service, '?string $expectedVmUuid = null') && str_contains($service, '$observedVmUuid') && str_contains($service, 'hash_equals(strtolower($expectedVmUuid)') - && str_contains($workerService, "? 'action' : \$kind, \$uuid"), + && str_contains($workerService, "? 'policy_skipped' : \$lifecycleKind") + && str_contains($service, "\$preserveTarget = \$kind === 'policy_skipped'"), 'Private observation validation must enforce the lifecycle VM UUID binding.'); $assert(str_contains($workerService, "'private_mac', 'desired_revision'") && str_contains($service, "'private_mac', 'desired_revision'"), diff --git a/web/tests/CloudVpcMemberReuseTest.php b/web/tests/CloudVpcMemberReuseTest.php index 12e845b3..5bdba2ec 100644 --- a/web/tests/CloudVpcMemberReuseTest.php +++ b/web/tests/CloudVpcMemberReuseTest.php @@ -80,6 +80,20 @@ $schema->create('cloud_instances', static function (Blueprint $table): void { $table->unsignedBigInteger('created_at'); $table->unsignedBigInteger('updated_at'); }); +$schema->create('cloud_instance_runtime', static function (Blueprint $table): void { + $table->unsignedInteger('instance_id')->primary(); + $table->unsignedInteger('host_id'); + $table->string('uuid', 36); + $table->string('power_state', 16)->default('unknown'); + $table->string('sync_state', 16)->default('never'); + $table->unsignedBigInteger('last_success_at')->default(0); + $table->unsignedBigInteger('last_attempt_at')->default(0); + $table->unsignedBigInteger('revision')->default(0); + $table->string('last_error_code', 64)->default(''); + $table->unsignedBigInteger('created_at'); + $table->unsignedBigInteger('updated_at'); + $table->unique(['host_id', 'uuid']); +}); $schema->create('cloud_vpcs', static function (Blueprint $table): void { $table->increments('id'); $table->string('vpc_key', 32)->unique(); @@ -234,6 +248,7 @@ $schema->create('cloud_vpc_operations', static function (Blueprint $table): void $table->string('status', 24); $table->text('payload_json'); $table->text('observed_json')->nullable(); + $table->unsignedSmallInteger('evidence_version')->default(0); $table->string('error_code', 64)->nullable(); $table->unsignedBigInteger('created_at'); $table->unsignedBigInteger('updated_at'); @@ -296,7 +311,7 @@ $routeTableMethod = new ReflectionMethod(CloudVpcService::class, 'privateRouteTa $privateMacMethod = new ReflectionMethod(CloudVpcService::class, 'privateMacForMember'); $activeHostEvidence = static function ( string $vpcKey, int $hostId, int $revision, string $digest, - array $members = [], array $desiredRoutes = [] + string $vpcCidr, array $members = [], array $desiredRoutes = [] ) use ($bridgeMethod, $nftMethod, $routeTableMethod): array { $bridge = $bridgeMethod->invoke(null, $vpcKey); $routeTable = $routeTableMethod->invoke(null, $vpcKey); @@ -316,7 +331,16 @@ $activeHostEvidence = static function ( 'interface' => 'private0', 'private_ip_cidr' => (string) $member['private_ip_cidr'], 'private_mac' => (string) $member['private_mac'], - 'xml_verified' => true, + 'xml_verified' => true, 'config_verified' => true, + 'seed_verified' => true, + 'restart_required' => false, + 'domain_running' => true, 'guest_network_verified' => true, + 'guest_network_readback' => 'confirmed', + 'guest_reason_code' => null, + 'guest_private_mac' => (string) $member['private_mac'], + 'guest_private_ip_cidr' => (string) $member['private_ip_cidr'], + 'link_state' => 'up', 'guest_link_state' => 'up', + 'guest_routes' => [$vpcCidr], ]; } else { $remoteIpsByHost[(string) $member['host_id']][] = @@ -367,6 +391,8 @@ $activeHostEvidence = static function ( 'observed_revision' => $revision, 'observed_digest' => $digest, 'state' => 'active', 'transport_verified' => true, 'private_dataplane_verified' => true, 'removed' => false, + 'config_verified' => true, 'config_observed_revision' => $revision, + 'config_observed_digest' => $digest, 'bridge' => $bridge, 'bridge_owner' => 'virt-cloud-vpc-dataplane-v1', 'nft_table' => $nftMethod->invoke(null, $vpcKey), @@ -391,6 +417,13 @@ $seedInstance = static function (int $id, int $hostId, int $userId = 9) use ( 'service_status' => 'exempt', 'network_status' => 'active', 'expires_at' => null, 'created_at' => $now, 'updated_at' => $now, ]); + Capsule::table('cloud_instance_runtime')->insert([ + 'instance_id' => $id, 'host_id' => $hostId, 'uuid' => $uuidFor($id), + 'power_state' => 'running', 'sync_state' => 'ok', + 'last_success_at' => $now, 'last_attempt_at' => $now, + 'revision' => 1, 'last_error_code' => '', + 'created_at' => $now, 'updated_at' => $now, + ]); }; $seedVpc = static function ( int $id, string $cidr, string $subnetCidr, int $hostId, @@ -437,10 +470,12 @@ $seedVpc = static function ( $payload['desired_digest'] = $digest; $payload['operation_id'] = $operationId; $hostEvidence = $activeHostEvidence( - $vpcKey, $hostId, 1, $digest, $memberRows, $desiredRoutes + $vpcKey, $hostId, 1, $digest, $networkIp . '/24', + $memberRows, $desiredRoutes ); if ($proofMode === 'deleted') { $observed = [ + 'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION, 'revision' => 0, 'digest' => null, 'state' => 'degraded', 'next_stage_allowed' => false, 'private_dataplane_verified' => false, 'fabric_gate' => ['active' => false, 'error_code' => 'vpc_no_members'], @@ -455,6 +490,7 @@ $seedVpc = static function ( unset($hostEvidence['nft_table']); } $observed = [ + 'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION, 'revision' => 1, 'digest' => $digest, 'state' => 'active', 'next_stage_allowed' => true, 'private_dataplane_verified' => $proofMode === 'active', @@ -508,6 +544,7 @@ $seedVpc = static function ( 'observed_json' => json_encode( $observed, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES ), + 'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION, 'error_code' => null, 'created_at' => $now, 'updated_at' => $now, 'finished_at' => $now, 'lease_token' => null, 'lease_until' => 0, 'attempts' => 1, @@ -635,10 +672,11 @@ $installCurrentProof = static function ( foreach ($observedHostIds as $hostId) { $observedHosts[(string) $hostId] = $activeHostEvidence( (string) $vpc['vpc_key'], (int) $hostId, - $revision, $digest, $memberRows, [] + $revision, $digest, (string) $root->cidr, $memberRows, [] ); } $observed = [ + 'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION, 'revision' => $revision, 'digest' => $digest, 'state' => 'active', 'next_stage_allowed' => true, 'private_dataplane_verified' => true, 'fabric_gate' => ['active' => true], 'hosts' => $observedHosts, @@ -656,6 +694,7 @@ $installCurrentProof = static function ( 'observed_json' => json_encode( $observed, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES ), + 'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION, 'error_code' => null, 'created_at' => $currentTime, 'updated_at' => $currentTime, 'finished_at' => $currentTime, 'lease_token' => null, 'lease_until' => 0, 'attempts' => 1, @@ -733,6 +772,7 @@ $installRemovalProof = static function ( 'proxy_ips' => [], ]; $observed = [ + 'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION, 'revision' => $revision, 'digest' => $digest, 'state' => 'degraded', 'next_stage_allowed' => true, 'private_dataplane_verified' => true, 'fabric_gate' => ['active' => true], @@ -751,6 +791,7 @@ $installRemovalProof = static function ( 'observed_json' => json_encode( $observed, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES ), + 'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION, 'error_code' => null, 'created_at' => $proofTime, 'updated_at' => $proofTime, 'finished_at' => $proofTime, 'lease_token' => null, 'lease_until' => 0, 'attempts' => 1, @@ -816,14 +857,44 @@ $boundedHosts = []; for ($index = 1; $index <= 257; $index++) { $boundedHosts[(string) $index] = [ 'host_id' => (string) $index, + 'config_verified' => true, 'config_observed_revision' => 9, + 'config_observed_digest' => str_repeat('a', 64), + 'member_nics' => $index === 1 ? [[ + 'member_id' => str_repeat('b', 32), + 'instance_uuid' => 'bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb', + 'interface' => 'private0', 'private_ip_cidr' => '10.200.0.1/32', + 'private_mac' => '02:00:00:00:00:01', + 'xml_verified' => true, 'config_verified' => true, + 'seed_verified' => true, + 'restart_required' => false, 'domain_running' => true, + 'guest_network_verified' => true, + 'guest_network_readback' => 'confirmed', + 'guest_reason_code' => null, + 'guest_private_mac' => '02:00:00:00:00:01', + 'guest_private_ip_cidr' => '10.200.0.1/32', + 'link_state' => 'up', 'guest_link_state' => 'up', + 'guest_routes' => ['10.200.0.0/24'], + ]] : [], 'routes' => $index === 1 ? $boundedRoutes : [], 'wg_allowed_ips' => $index === 1 ? $boundedPeers : [], ]; } -$boundedObserved = $boundedObservedMethod->invoke(null, ['hosts' => $boundedHosts]); +$boundedObserved = $boundedObservedMethod->invoke(null, [ + 'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION, + 'hosts' => $boundedHosts, +]); $assert(count($boundedObserved['hosts'] ?? []) === 257 && count($boundedObserved['hosts']['1']['routes'] ?? []) === 257 - && count($boundedObserved['hosts']['1']['wg_allowed_ips'] ?? []) === 257, + && count($boundedObserved['hosts']['1']['wg_allowed_ips'] ?? []) === 257 + && ($boundedObserved['hosts']['1']['config_verified'] ?? false) === true + && ($boundedObserved['hosts']['1']['member_nics'][0]['guest_network_verified'] + ?? false) === true + && ($boundedObserved['hosts']['1']['member_nics'][0]['link_state'] ?? null) + === 'up' + && ($boundedObserved['hosts']['1']['member_nics'][0]['guest_routes'] ?? []) + === ['10.200.0.0/24'] + && ($boundedObserved['evidence_version'] ?? 0) + === CloudVpcService::OBSERVATION_EVIDENCE_VERSION, 'Observed proof persistence still truncates hosts, routes or peers at 256.'); // A fully proven historical pair must release both uniqueness keys before a diff --git a/web/tests/CloudVpcRootDeletionTest.php b/web/tests/CloudVpcRootDeletionTest.php index db178940..becca213 100644 --- a/web/tests/CloudVpcRootDeletionTest.php +++ b/web/tests/CloudVpcRootDeletionTest.php @@ -157,6 +157,7 @@ $schema->create('cloud_vpc_operations', static function (Blueprint $table): void $table->string('status', 16); $table->text('payload_json'); $table->text('observed_json')->nullable(); + $table->unsignedSmallInteger('evidence_version')->default(0); $table->string('error_code', 64)->nullable(); $table->string('lease_token', 32)->nullable(); $table->unsignedBigInteger('lease_until')->default(0); diff --git a/web/tests/CloudVpcTombstoneConvergenceTest.php b/web/tests/CloudVpcTombstoneConvergenceTest.php index f21848d5..4dc7c8d2 100644 --- a/web/tests/CloudVpcTombstoneConvergenceTest.php +++ b/web/tests/CloudVpcTombstoneConvergenceTest.php @@ -75,10 +75,25 @@ $schema->create('cloud_vpc_subnets', static function (Blueprint $table): void { }); $schema->create('cloud_instances', static function (Blueprint $table): void { $table->increments('id'); + $table->unsignedInteger('host_id'); $table->string('uuid', 36); $table->string('status', 24); $table->string('network_status', 24)->default('active'); }); +$schema->create('cloud_instance_runtime', static function (Blueprint $table): void { + $table->unsignedInteger('instance_id')->primary(); + $table->unsignedInteger('host_id'); + $table->string('uuid', 36); + $table->string('power_state', 16)->default('unknown'); + $table->string('sync_state', 16)->default('never'); + $table->unsignedBigInteger('last_success_at')->default(0); + $table->unsignedBigInteger('last_attempt_at')->default(0); + $table->unsignedBigInteger('revision')->default(0); + $table->string('last_error_code', 64)->default(''); + $table->unsignedBigInteger('created_at'); + $table->unsignedBigInteger('updated_at'); + $table->unique(['host_id', 'uuid']); +}); $schema->create('cloud_vpc_members', static function (Blueprint $table): void { $table->increments('id'); $table->string('member_key', 32); @@ -165,6 +180,7 @@ $schema->create('cloud_vpc_operations', static function (Blueprint $table): void $table->string('status', 24); $table->text('payload_json'); $table->text('observed_json')->nullable(); + $table->unsignedSmallInteger('evidence_version')->default(0); $table->string('error_code', 64)->nullable(); $table->string('lease_token', 32)->nullable(); $table->unsignedBigInteger('lease_until')->default(0); @@ -181,6 +197,13 @@ $bridgeMethod = new ReflectionMethod(CloudVpcService::class, 'privateBridgeForVp $nftMethod = new ReflectionMethod(CloudVpcService::class, 'privateNftTableForVpc'); $routeTableMethod = new ReflectionMethod(CloudVpcService::class, 'privateRouteTableForVpc'); $publicRootMethod = new ReflectionMethod(CloudVpcService::class, 'publicRoot'); +$validateHostMethod = new ReflectionMethod( + CloudVpcService::class, 'validateHostDataplaneEvidence' +); +$assert(CloudVpcService::ENGINE_TIMEOUT_SECONDS > 40.0 + && CloudVpcService::OPERATION_LEASE_SECONDS + >= CloudVpcService::ENGINE_TIMEOUT_SECONDS + 30.0, + 'VPC timeout/lease do not contain the engine request-wide probe budget.'); $seedVpc = static function ( int $id, string $vpcKey, int $revision, string $digest, @@ -233,9 +256,17 @@ $seedMember = static function ( $packed = unpack('Nvalue', inet_pton($ip)); $ipInt = (int) sprintf('%u', (int) $packed['value']); Capsule::table('cloud_instances')->insert([ - 'id' => $instanceId, 'uuid' => $uuid, 'status' => 'active', + 'id' => $instanceId, 'host_id' => $hostId, + 'uuid' => $uuid, 'status' => 'active', 'network_status' => 'active', ]); + Capsule::table('cloud_instance_runtime')->insert([ + 'instance_id' => $instanceId, 'host_id' => $hostId, 'uuid' => $uuid, + 'power_state' => 'running', 'sync_state' => 'ok', + 'last_success_at' => $now, 'last_attempt_at' => $now, + 'revision' => 1, 'last_error_code' => '', + 'created_at' => $now, 'updated_at' => $now, + ]); Capsule::table('cloud_vpc_members')->insert([ 'id' => $memberId, 'member_key' => $memberKey, 'vpc_id' => $vpcId, 'subnet_id' => $subnetId, 'private_ip_id' => $leaseId, @@ -273,7 +304,7 @@ $seedHost = static function ( ]); }; $activeHost = static function ( - string $vpcKey, int $hostId, int $revision, string $digest, + string $vpcKey, string $vpcCidr, int $hostId, int $revision, string $digest, string $memberKey, string $uuid, string $ip ) use ($privateMacMethod, $bridgeMethod, $nftMethod, $routeTableMethod): array { $bridge = $bridgeMethod->invoke(null, $vpcKey); @@ -283,6 +314,8 @@ $activeHost = static function ( 'observed_revision' => $revision, 'observed_digest' => $digest, 'state' => 'active', 'transport_verified' => true, 'private_dataplane_verified' => true, 'removed' => false, + 'config_verified' => true, 'config_observed_revision' => $revision, + 'config_observed_digest' => $digest, 'bridge' => $bridge, 'bridge_owner' => 'virt-cloud-vpc-dataplane-v1', 'nft_table' => $nftMethod->invoke(null, $vpcKey), 'anti_spoof' => true, 'rp_filter' => 2, 'proxy_arp' => 0, @@ -293,7 +326,16 @@ $activeHost = static function ( 'member_id' => $memberKey, 'instance_uuid' => $uuid, 'interface' => 'private0', 'private_ip_cidr' => $ip . '/32', 'private_mac' => $privateMacMethod->invoke(null, $vpcKey, $memberKey), - 'xml_verified' => true, 'link_state' => 'up', + 'xml_verified' => true, 'config_verified' => true, + 'seed_verified' => true, + 'restart_required' => false, 'domain_running' => true, + 'guest_network_verified' => true, + 'guest_network_readback' => 'confirmed', + 'guest_reason_code' => null, + 'guest_private_mac' => $privateMacMethod->invoke(null, $vpcKey, $memberKey), + 'guest_private_ip_cidr' => $ip . '/32', + 'guest_routes' => [$vpcCidr], + 'link_state' => 'up', 'guest_link_state' => 'up', ]], 'routes' => [[ 'destination_cidr' => $ip . '/32', 'device' => $bridge, @@ -322,12 +364,86 @@ $removedHost = static function ( }; $observed = static function (int $revision, string $digest, array $hosts): array { return [ + 'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION, 'revision' => $revision, 'digest' => $digest, 'state' => 'active', 'next_stage_allowed' => true, 'private_dataplane_verified' => true, 'fabric_gate' => ['active' => true], 'hosts' => $hosts, ]; }; +$validatorVpcKey = str_repeat('a', 32); +$validatorDigest = str_repeat('d', 64); +$validatorMemberKey = str_repeat('b', 32); +$validatorMember = [ + 'member_id' => $validatorMemberKey, + 'instance_uuid' => 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa', + 'host_id' => '901', 'private_ip_cidr' => '10.90.0.10/32', + 'private_mac' => $privateMacMethod->invoke( + null, $validatorVpcKey, $validatorMemberKey + ), +]; +$validatorHost = $activeHost( + $validatorVpcKey, '10.90.0.0/24', 901, 2, $validatorDigest, + $validatorMemberKey, + $validatorMember['instance_uuid'], '10.90.0.10' +); +$validatorArgs = [ + '901', [$validatorMemberKey => $validatorMember], [$validatorMember], + ['10.90.0.10/32'], '10.90.0.0/24', [], + $validatorVpcKey, 2, $validatorDigest, +]; +$assert($validateHostMethod->invoke(null, $validatorHost, ...$validatorArgs) === null, + 'An exact guest address/route proof did not open the typed host gate.'); +$validatorRouteMismatch = $validatorHost; +$validatorRouteMismatch['member_nics'][0]['guest_routes'] = ['10.91.0.0/24']; +$assert($validateHostMethod->invoke( + null, $validatorRouteMismatch, ...$validatorArgs +) === 'vpc_private_guest_route_mismatch', + 'A mismatched guest route opened the typed host gate.'); +$validatorMissingGuest = $validatorHost; +unset($validatorMissingGuest['member_nics'][0]['domain_running']); +$assert($validateHostMethod->invoke( + null, $validatorMissingGuest, ...$validatorArgs +) === 'vpc_private_guest_agent_unavailable', + 'Missing guest runtime evidence did not fail closed.'); +$validatorDeadline = $validatorHost; +$validatorDeadline['member_nics'][0]['guest_network_verified'] = false; +$validatorDeadline['member_nics'][0]['guest_network_readback'] = 'unavailable'; +$validatorDeadline['member_nics'][0]['guest_reason_code'] = + 'vpc_private_guest_probe_deadline_exceeded'; +$assert($validateHostMethod->invoke( + null, $validatorDeadline, ...$validatorArgs +) === 'vpc_private_guest_probe_deadline_exceeded', + 'The bounded guest-probe deadline reason was not safely propagated.'); +$validatorReap = $validatorHost; +$validatorReap['member_nics'][0]['guest_network_verified'] = false; +$validatorReap['member_nics'][0]['guest_network_readback'] = 'unavailable'; +$validatorReap['member_nics'][0]['guest_reason_code'] = + 'vpc_private_guest_probe_reap_required'; +$assert($validateHostMethod->invoke( + null, $validatorReap, ...$validatorArgs +) === 'vpc_private_guest_probe_reap_required', + 'Guest-probe reap-required was not retained as a safe retryable reason.'); +$validatorLinkDown = $validatorHost; +$validatorLinkDown['member_nics'][0]['link_state'] = 'down'; +$assert($validateHostMethod->invoke( + null, $validatorLinkDown, ...$validatorArgs +) === 'vpc_private_guest_link_mismatch', + 'A down guest private link opened the typed host gate.'); +$validatorLinkMissing = $validatorHost; +unset($validatorLinkMissing['member_nics'][0]['link_state']); +$assert($validateHostMethod->invoke( + null, $validatorLinkMissing, ...$validatorArgs +) === 'vpc_private_guest_link_mismatch', + 'Missing guest private link evidence did not fail closed.'); +$validatorMissingXml = $validatorHost; +$validatorMissingXml['member_nics'][0]['xml_verified'] = false; +$configValidatorArgs = [...$validatorArgs, false]; +$assert($validateHostMethod->invoke( + null, $validatorMissingXml, ...$configValidatorArgs +) === 'vpc_private_nic_readback_mismatch', + 'Configuration-only convergence accepted a private NIC without exact XML.'); + // Same-host partial detach: exact current member_nics proves the old private0 // was consumed before the host became active. $vpcKey1 = str_repeat('1', 32); @@ -342,7 +458,8 @@ $liveUuid1 = '11111111-1111-4111-8111-111111111112'; $seedMember(102, 202, 1, 101, $liveKey1, 1002, $liveUuid1, 1, '10.21.0.11', 'active', 2, $digest1); $seedHost(1, 1, 'applying', 2, $digest1, $operation1); -$host1 = $activeHost($vpcKey1, 1, 2, $digest1, $liveKey1, $liveUuid1, '10.21.0.11'); +$host1 = $activeHost($vpcKey1, '10.21.0.0/24', 1, 2, $digest1, + $liveKey1, $liveUuid1, '10.21.0.11'); CloudVpcService::recordObserved($vpcKey1, 2, $digest1, $observed(2, $digest1, ['1' => $host1]), $operation1); $assert(Capsule::table('cloud_vpc_members')->where('id', 101)->count() === 0 @@ -389,7 +506,7 @@ $seedHost(2, 10, 'deleting', 4, $digest2, $operation2); $seedHost(2, 20, 'applying', 4, $digest2, $operation2); $hosts2 = [ '10' => $removedHost($vpcKey2, 10, 4, $digest2), - '20' => $activeHost($vpcKey2, 20, 4, $digest2, + '20' => $activeHost($vpcKey2, '10.22.0.0/24', 20, 4, $digest2, $liveKey2, $liveUuid2, '10.22.0.11'), ]; CloudVpcService::recordObserved($vpcKey2, 4, $digest2, @@ -417,7 +534,8 @@ $liveUuid3 = '33333333-3333-4333-8333-333333333333'; $seedMember(105, 205, 3, 103, $liveKey3, 3001, $liveUuid3, 30, '10.23.0.10', 'active', 6, $digest3); $seedHost(3, 30, 'applying', 6, $digest3, $operation3); -$host3 = $activeHost($vpcKey3, 30, 6, $digest3, $liveKey3, $liveUuid3, '10.23.0.10'); +$host3 = $activeHost($vpcKey3, '10.23.0.0/24', 30, 6, $digest3, + $liveKey3, $liveUuid3, '10.23.0.10'); CloudVpcService::recordObserved($vpcKey3, 6, $digest3, $observed(6, $digest3, ['30' => $host3]), $operation3); $assert(Capsule::table('cloud_vpc_subnets')->where('id', 104)->count() === 0 @@ -436,6 +554,7 @@ $operation4 = str_repeat('d', 32); $seedVpc(4, $vpcKey4, 8, $digest4, $operation4, 'subnet_remove'); $seedSubnet(105, 4, str_repeat('e', 32), '10.24.0.0/25', 'deleting', 8, $digest4); CloudVpcService::recordObserved($vpcKey4, 8, $digest4, [ + 'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION, 'revision' => 0, 'digest' => null, 'state' => 'degraded', 'next_stage_allowed' => false, 'private_dataplane_verified' => false, 'fabric_gate' => ['active' => false, 'error_code' => 'vpc_no_members'], @@ -467,7 +586,7 @@ $seedMember(107, 207, 5, 106, $liveKey5, 5002, $liveUuid5, 60, '10.25.0.11', 'active', 10, $digest5); $seedHost(5, 50, 'deleting', 10, $digest5, $operation5); $seedHost(5, 60, 'applying', 10, $digest5, $operation5); -$host5 = $activeHost($vpcKey5, 60, 10, $digest5, +$host5 = $activeHost($vpcKey5, '10.25.0.0/24', 60, 10, $digest5, $liveKey5, $liveUuid5, '10.25.0.11'); CloudVpcService::recordObserved($vpcKey5, 10, $digest5, $observed(10, $digest5, ['60' => $host5]), $operation5); @@ -514,6 +633,7 @@ $seedMember(109, 209, 7, 108, str_repeat('5', 32), 7001, 'deleting', 14, $digest7); try { CloudVpcService::recordObserved($vpcKey7, 14, $digest7, [ + 'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION, 'revision' => 0, 'digest' => null, 'state' => 'degraded', 'next_stage_allowed' => false, 'private_dataplane_verified' => false, 'fabric_gate' => ['active' => false, 'error_code' => 'vpc_no_members'], @@ -547,7 +667,7 @@ $liveUuid8 = '88888888-8888-4888-8888-888888888882'; $seedMember(111, 211, 8, 109, $liveKey8, 8002, $liveUuid8, 81, '10.28.0.11', 'active', 16, $digest8); $seedHost(8, 81, 'applying', 16, $digest8, $operation8); -$host8 = $activeHost($vpcKey8, 81, 16, $digest8, +$host8 = $activeHost($vpcKey8, '10.28.0.0/24', 81, 16, $digest8, $liveKey8, $liveUuid8, '10.28.0.11'); try { CloudVpcService::recordObserved($vpcKey8, 16, $digest8, @@ -597,7 +717,7 @@ $assert(Capsule::table('cloud_vpc_members')->where('id', 112) && Capsule::table('cloud_vpc_operations')->where('operation_id', $operation9) ->value('status') !== 'succeeded', 'The incomplete first retry lost A proof or released its identities early.'); -$host9 = $activeHost($vpcKey9, 91, 18, $digest9, +$host9 = $activeHost($vpcKey9, '10.29.0.0/24', 91, 18, $digest9, $liveKey9, $liveUuid9, '10.29.0.11'); CloudVpcService::recordObserved($vpcKey9, 18, $digest9, $observed(18, $digest9, ['91' => $host9]), $operation9); @@ -630,6 +750,7 @@ Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 10) 'last_error_code' => null, 'last_success_at' => $now, ]); CloudVpcService::recordObserved($vpcKey10, 20, $digest10, [ + 'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION, 'revision' => 0, 'digest' => null, 'state' => 'degraded', 'next_stage_allowed' => false, 'private_dataplane_verified' => false, 'fabric_gate' => ['active' => false, 'error_code' => 'vpc_no_members'], @@ -667,6 +788,7 @@ Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 11) 'last_error_code' => null, 'last_success_at' => $now, ]); CloudVpcService::recordObserved($vpcKey11, 22, $digest11, [ + 'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION, 'revision' => 0, 'digest' => null, 'state' => 'degraded', 'next_stage_allowed' => false, 'private_dataplane_verified' => false, 'fabric_gate' => ['active' => false, 'error_code' => 'vpc_no_members'], @@ -693,6 +815,7 @@ $seedMember(116, 216, 12, 113, str_repeat('2', 31) . '1', 12001, 'deleting', 24, $digest12); $seedHost(12, 120, 'deleting', 24, $digest12, $operation12); CloudVpcService::recordObserved($vpcKey12, 24, $digest12, [ + 'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION, 'revision' => 0, 'digest' => null, 'state' => 'degraded', 'next_stage_allowed' => false, 'private_dataplane_verified' => false, 'fabric_gate' => ['active' => false, 'error_code' => 'vpc_no_members'], @@ -710,6 +833,7 @@ $assert(Capsule::table('cloud_vpc_operations')->where('operation_id', $operation ->value('state') === 'deleting', 'A transient no-members response froze an unresolved host/member tombstone.'); CloudVpcService::recordObserved($vpcKey12, 24, $digest12, [ + 'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION, 'revision' => 0, 'digest' => null, 'state' => 'degraded', 'next_stage_allowed' => false, 'private_dataplane_verified' => false, 'fabric_gate' => ['active' => false, 'error_code' => 'vpc_no_members'], @@ -744,7 +868,7 @@ $seedMember(118, 218, 13, 114, $liveKey13, 13002, $liveUuid13, 130, '10.33.0.11', 'active', 26, $digest13); $seedHost(13, 130, 'applying', 26, $digest13, $operation13); $seedHost(13, 131, 'deleted', 26, $digest13, $operation13); -$host13 = $activeHost($vpcKey13, 130, 26, $digest13, +$host13 = $activeHost($vpcKey13, '10.33.0.0/24', 130, 26, $digest13, $liveKey13, $liveUuid13, '10.33.0.11'); CloudVpcService::recordObserved($vpcKey13, 26, $digest13, $observed(26, $digest13, ['130' => $host13]), $operation13); @@ -770,4 +894,850 @@ $assert(Capsule::table('cloud_vpc_members')->where('id', 117)->count() === 0 ->value('status') === 'succeeded', 'Refreshing the stale historical removal proof did not unblock convergence.'); +// A running host/libvirt configuration is not usable until the guest itself +// proves the exact address and routes. Preserve one actionable restart reason +// through the host, member and root projections while configuration operation +// completion remains terminal. +$vpcKey14 = str_repeat('e', 32); +$digest14 = str_repeat('7', 64); +$operation14 = str_repeat('e', 31) . '0'; +$memberKey14 = str_repeat('6', 31) . '1'; +$uuid14 = 'eeeeeeee-eeee-4eee-8eee-eeeeeeeeeee1'; +$seedVpc(14, $vpcKey14, 28, $digest14, $operation14, 'member_attach'); +$seedSubnet(115, 14, str_repeat('7', 31) . '1', '10.34.0.0/25', + 'active', 28, $digest14); +$seedMember(119, 219, 14, 115, $memberKey14, 14001, $uuid14, + 140, '10.34.0.10', 'applying', 28, $digest14); +$seedHost(14, 140, 'applying', 28, $digest14, $operation14); +$host14 = $activeHost( + $vpcKey14, '10.34.0.0/24', 140, 28, $digest14, + $memberKey14, $uuid14, '10.34.0.10' +); +$host14['state'] = 'applying'; +$host14['private_dataplane_verified'] = false; +$host14['last_error_code'] = 'vpc_private_guest_restart_required'; +$host14['member_nics'][0]['restart_required'] = true; +$host14['member_nics'][0]['guest_network_verified'] = false; +$host14['member_nics'][0]['guest_network_readback'] = 'pending_restart'; +$host14['member_nics'][0]['guest_reason_code'] = + 'vpc_private_guest_restart_required'; +$deferred14 = $observed(28, $digest14, ['140' => $host14]); +$deferred14['state'] = 'applying'; +$deferred14['next_stage_allowed'] = false; +$deferred14['private_dataplane_verified'] = false; +CloudVpcService::recordObserved( + $vpcKey14, 28, $digest14, $deferred14, $operation14 +); +$rootProjection14 = $publicRootMethod->invoke(null, CloudVpc::find(14)); +$assert(Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 14) + ->where('host_id', 140)->value('last_error_code') + === 'vpc_private_guest_restart_required' + && Capsule::table('cloud_vpc_operations')->where('operation_id', $operation14) + ->value('status') === 'succeeded' + && Capsule::table('cloud_vpc_operations')->where('operation_id', $operation14) + ->value('error_code') === null + && Capsule::table('cloud_vpc_members')->where('id', 119) + ->value('state') === 'degraded' + && Capsule::table('cloud_vpc_members')->where('id', 119) + ->value('last_error_code') === 'vpc_private_guest_restart_required' + && Capsule::table('cloud_vpcs')->where('id', 14) + ->value('last_error_code') === 'vpc_private_guest_restart_required' + && ($rootProjection14['usable'] ?? true) === false, + 'A config-complete guest restart was not retained as non-usable runtime state.'); + +// Each deployment must retain its own host item. A stale loop variable used +// to copy the final host's error onto every prior host. Mixed typed reasons +// remain per-host and collapse to the generic operation/root code. +$vpcKey15 = str_repeat('f', 32); +$digest15 = str_repeat('8', 64); +$operation15 = str_repeat('f', 31) . '0'; +$memberKey15a = str_repeat('8', 31) . '1'; +$memberKey15b = str_repeat('9', 31) . '1'; +$uuid15a = 'ffffffff-ffff-4fff-8fff-fffffffffff1'; +$uuid15b = 'ffffffff-ffff-4fff-8fff-fffffffffff2'; +$seedVpc(15, $vpcKey15, 30, $digest15, $operation15, 'member_attach'); +$seedSubnet(116, 15, str_repeat('a', 31) . '1', '10.35.0.0/25', + 'active', 30, $digest15); +$seedMember(120, 220, 15, 116, $memberKey15a, 15001, $uuid15a, + 150, '10.35.0.10', 'applying', 30, $digest15); +$seedMember(121, 221, 15, 116, $memberKey15b, 15002, $uuid15b, + 151, '10.35.0.11', 'applying', 30, $digest15); +$seedHost(15, 150, 'applying', 30, $digest15, $operation15); +$seedHost(15, 151, 'applying', 30, $digest15, $operation15); +$host15a = $activeHost( + $vpcKey15, '10.35.0.0/24', 150, 30, $digest15, + $memberKey15a, $uuid15a, '10.35.0.10' +); +$host15b = $activeHost( + $vpcKey15, '10.35.0.0/24', 151, 30, $digest15, + $memberKey15b, $uuid15b, '10.35.0.11' +); +$routeTable15 = $routeTableMethod->invoke(null, $vpcKey15); +$peerKey15 = base64_encode(str_repeat("\x15", 32)); +$host15a['proxy_ips'] = ['10.35.0.10', '10.35.0.11']; +$host15b['proxy_ips'] = ['10.35.0.10', '10.35.0.11']; +$host15a['member_nics'][0]['guest_routes'] = ['10.35.0.0/24']; +$host15b['member_nics'][0]['guest_routes'] = ['10.35.0.0/24']; +$host15a['routes'][] = [ + 'destination_cidr' => '10.35.0.11/32', 'device' => 'wg-fabric0', + 'table' => $routeTable15, 'owner' => 'virt-cloud-vpc-dataplane-v1', + 'via' => null, 'metric' => 100, +]; +$host15b['routes'][] = [ + 'destination_cidr' => '10.35.0.10/32', 'device' => 'wg-fabric0', + 'table' => $routeTable15, 'owner' => 'virt-cloud-vpc-dataplane-v1', + 'via' => null, 'metric' => 100, +]; +$host15a['wg_allowed_ips'] = [[ + 'peer_host_id' => '151', 'public_key' => $peerKey15, + 'tenant_allowed_ips' => ['10.35.0.11/32'], +]]; +$host15b['wg_allowed_ips'] = [[ + 'peer_host_id' => '150', 'public_key' => $peerKey15, + 'tenant_allowed_ips' => ['10.35.0.10/32'], +]]; +$host15a['state'] = $host15b['state'] = 'applying'; +$host15a['private_dataplane_verified'] = false; +$host15b['private_dataplane_verified'] = false; +$host15a['member_nics'][0]['guest_network_verified'] = false; +$host15b['member_nics'][0]['guest_network_verified'] = false; +$host15a['last_error_code'] = 'vpc_private_guest_restart_required'; +$host15b['last_error_code'] = 'vpc_private_guest_agent_unavailable'; +$deferred15 = $observed(30, $digest15, [ + '150' => $host15a, '151' => $host15b, +]); +$deferred15['state'] = 'applying'; +$deferred15['next_stage_allowed'] = false; +$deferred15['private_dataplane_verified'] = false; +CloudVpcService::recordObserved( + $vpcKey15, 30, $digest15, $deferred15, $operation15 +); +$assert(Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 15) + ->where('host_id', 150)->value('last_error_code') + === 'vpc_private_guest_restart_required' + && Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 15) + ->where('host_id', 151)->value('last_error_code') + === 'vpc_private_guest_agent_unavailable' + && Capsule::table('cloud_vpc_operations')->where('operation_id', $operation15) + ->value('status') === 'succeeded' + && Capsule::table('cloud_vpc_operations')->where('operation_id', $operation15) + ->value('error_code') === null + && Capsule::table('cloud_vpcs')->where('id', 15) + ->value('last_error_code') === 'vpc_data_plane_deferred', + 'Multi-host guest errors were cross-contaminated or unsafely collapsed.'); + +// A pre-upgrade succeeded receipt has no guest-evidence contract version. It +// must fail closed immediately; the worker contract separately reclaims only +// this current last_operation_id for fresh observation. +$vpcKey16 = str_repeat('0', 32); +$digest16 = str_repeat('9', 64); +$operation16 = str_repeat('0', 31) . '1'; +$memberKey16 = str_repeat('b', 31) . '1'; +$uuid16 = '00000000-0000-4000-8000-000000000001'; +$seedVpc(16, $vpcKey16, 32, $digest16, $operation16, 'member_attach'); +$seedSubnet(117, 16, str_repeat('c', 31) . '1', '10.36.0.0/25', + 'active', 32, $digest16); +$seedMember(122, 222, 16, 117, $memberKey16, 16001, $uuid16, + 160, '10.36.0.10', 'active', 32, $digest16); +$seedHost(16, 160, 'active', 32, $digest16, $operation16); +Capsule::table('cloud_vpc_operations')->where('operation_id', $operation16)->update([ + 'status' => 'succeeded', 'evidence_version' => 0, 'finished_at' => $now, +]); +Capsule::table('cloud_vpcs')->where('id', 16)->update([ + 'state' => 'active', 'sync_state' => 'applied', 'observed_revision' => 32, + 'observed_digest' => $digest16, 'last_success_at' => $now, +]); +Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 16) + ->where('host_id', 160)->update([ + 'state' => 'active', 'observed_revision' => 32, + 'observed_digest' => $digest16, 'last_success_at' => $now, + ]); +$rootProjection16 = $publicRootMethod->invoke(null, CloudVpc::find(16)); +$assert(($rootProjection16['usable'] ?? true) === false, + 'A legacy succeeded observation without guest evidence remained usable.'); + +// Retiring member A on a host with live sibling B must converge from exact +// current configuration even when B's guest probe is unsupported. The proof +// releases only A/IP; B, the host and the root remain fail-closed for runtime +// usability until B supplies exact guest evidence. +$vpcKey17 = str_repeat('1', 31) . '7'; +$digest17 = str_repeat('a', 63) . '7'; +$operation17 = str_repeat('2', 31) . '7'; +$retiringKey17 = str_repeat('3', 31) . '7'; +$liveKey17 = str_repeat('4', 31) . '7'; +$liveUuid17 = '17171717-1717-4717-8717-171717171717'; +$seedVpc(17, $vpcKey17, 34, $digest17, $operation17, 'member_detach'); +$seedSubnet(118, 17, str_repeat('5', 31) . '7', '10.37.0.0/25', + 'active', 34, $digest17); +$seedMember(123, 223, 17, 118, $retiringKey17, 17001, + '17171717-1717-4717-8717-171717171701', 170, '10.37.0.10', + 'deleting', 34, $digest17); +$seedMember(124, 224, 17, 118, $liveKey17, 17002, $liveUuid17, + 170, '10.37.0.11', 'applying', 34, $digest17); +$seedHost(17, 170, 'applying', 34, $digest17, $operation17); +$detachRuntimePayload17 = json_encode([ + 'schema_version' => 1, 'vpc_id' => $vpcKey17, + 'desired_revision' => 34, 'desired_digest' => $digest17, + 'operation_id' => $operation17, + 'members' => [[ + 'member_id' => $liveKey17, 'instance_uuid' => $liveUuid17, + 'host_id' => '170', 'private_ip_cidr' => '10.37.0.11/32', + ]], +], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES); +Capsule::table('cloud_vpc_operations')->where('operation_id', $operation17) + ->update(['payload_json' => $detachRuntimePayload17]); +$host17 = $activeHost( + $vpcKey17, '10.37.0.0/24', 170, 34, $digest17, + $liveKey17, $liveUuid17, '10.37.0.11' +); +$host17['state'] = 'applying'; +$host17['private_dataplane_verified'] = false; +$host17['readback'] = 'pending_guest'; +$host17['last_error_code'] = 'vpc_private_guest_probe_unsupported'; +$host17['member_nics'][0]['guest_network_verified'] = false; +$host17['member_nics'][0]['guest_network_readback'] = 'unsupported'; +$host17['member_nics'][0]['guest_reason_code'] = + 'vpc_private_guest_probe_unsupported'; +$deferred17 = $observed(34, $digest17, ['170' => $host17]); +$deferred17['state'] = 'applying'; +$deferred17['next_stage_allowed'] = false; +$deferred17['private_dataplane_verified'] = false; +CloudVpcService::recordObserved( + $vpcKey17, 34, $digest17, $deferred17, $operation17 +); +$rootProjection17 = $publicRootMethod->invoke(null, CloudVpc::find(17)); +$assert(Capsule::table('cloud_vpc_members')->where('id', 123)->count() === 0 + && Capsule::table('cloud_vpc_private_ips')->where('id', 223)->count() === 0 + && Capsule::table('cloud_vpc_members')->where('id', 124)->count() === 1 + && Capsule::table('cloud_vpc_private_ips')->where('id', 224)->count() === 1 + && Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 17) + ->where('host_id', 170)->value('state') === 'degraded' + && Capsule::table('cloud_vpc_operations')->where('operation_id', $operation17) + ->value('status') === 'succeeded' + && Capsule::table('cloud_vpc_operations')->where('operation_id', $operation17) + ->value('error_code') === null + && Capsule::table('cloud_vpcs')->where('id', 17) + ->value('state') === 'degraded' + && Capsule::table('cloud_vpcs')->where('id', 17) + ->value('sync_state') === 'waiting' + && Capsule::table('cloud_vpcs')->where('id', 17) + ->value('last_error_code') === 'vpc_private_guest_probe_unsupported' + && ($rootProjection17['usable'] ?? true) === false, + 'Config-exact detach did not release only A while B guest readiness stayed closed.'); + +// No new mutation is required once B becomes observable. The worker reuses +// the current, idempotent detach operation and the exact guest receipt opens +// only the runtime gate; the already released A/IP rows stay absent. +$readyHost17 = $activeHost( + $vpcKey17, '10.37.0.0/24', 170, 34, $digest17, + $liveKey17, $liveUuid17, '10.37.0.11' +); +CloudVpcService::recordObserved( + $vpcKey17, 34, $digest17, + $observed(34, $digest17, ['170' => $readyHost17]), + $operation17 +); +$readyRootProjection17 = $publicRootMethod->invoke(null, CloudVpc::find(17)); +$assert(Capsule::table('cloud_vpc_members')->where('id', 123)->count() === 0 + && Capsule::table('cloud_vpc_private_ips')->where('id', 223)->count() === 0 + && Capsule::table('cloud_vpc_operations')->where('vpc_id', 17) + ->where('operation_id', $operation17)->count() === 1 + && Capsule::table('cloud_vpc_operations')->where('operation_id', $operation17) + ->value('payload_json') === $detachRuntimePayload17 + && !str_contains($detachRuntimePayload17, $retiringKey17) + && Capsule::table('cloud_vpc_operations')->where('operation_id', $operation17) + ->value('status') === 'succeeded' + && Capsule::table('cloud_vpcs')->where('id', 17) + ->value('state') === 'active' + && Capsule::table('cloud_vpcs')->where('id', 17) + ->value('sync_state') === 'applied' + && Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 17) + ->where('host_id', 170)->value('state') === 'active' + && ($readyRootProjection17['usable'] ?? false) === true, + 'A current detach operation could not finish sibling runtime reconciliation.'); + +// Runtime evidence is member-scoped. A running/exact A remains usable when B +// is stopped after the same host configuration has completed. The operation +// is terminal for configuration, while the same ID remains the worker's +// idempotent runtime-reconcile target for B. +$vpcKey19 = str_repeat('9', 31) . 'a'; +$digest19 = str_repeat('c', 63) . '9'; +$operation19 = str_repeat('a', 31) . '9'; +$memberKey19a = str_repeat('b', 31) . '9'; +$memberKey19b = str_repeat('c', 31) . '8'; +$uuid19a = '19191919-1919-4919-8919-191919191901'; +$uuid19b = '19191919-1919-4919-8919-191919191902'; +$seedVpc(19, $vpcKey19, 38, $digest19, $operation19, 'member_attach'); +$seedSubnet(120, 19, str_repeat('d', 31) . '9', '10.39.0.0/25', + 'applying', 38, $digest19); +$seedMember(126, 226, 19, 120, $memberKey19a, 19001, $uuid19a, + 190, '10.39.0.10', 'applying', 38, $digest19); +$seedMember(127, 227, 19, 120, $memberKey19b, 19002, $uuid19b, + 190, '10.39.0.11', 'applying', 38, $digest19); +$seedHost(19, 190, 'applying', 38, $digest19, $operation19); +$host19 = $activeHost( + $vpcKey19, '10.39.0.0/24', 190, 38, $digest19, + $memberKey19a, $uuid19a, '10.39.0.10' +); +$bridge19 = $bridgeMethod->invoke(null, $vpcKey19); +$mac19b = $privateMacMethod->invoke(null, $vpcKey19, $memberKey19b); +$host19['state'] = 'applying'; +$host19['private_dataplane_verified'] = false; +$host19['readback'] = 'pending_guest'; +$host19['last_error_code'] = 'vpc_private_guest_restart_required'; +$host19['proxy_ips'][] = '10.39.0.11'; +$host19['member_nics'][0]['guest_routes'] = ['10.39.0.0/24']; +$host19['member_nics'][] = [ + 'member_id' => $memberKey19b, 'instance_uuid' => $uuid19b, + 'interface' => 'private0', 'private_ip_cidr' => '10.39.0.11/32', + 'private_mac' => $mac19b, 'xml_verified' => true, + 'config_verified' => true, 'seed_verified' => true, + 'restart_required' => true, + 'domain_running' => false, 'guest_network_verified' => false, + 'guest_network_readback' => 'stopped', + 'guest_reason_code' => 'vpc_private_guest_restart_required', + 'guest_private_mac' => null, 'guest_private_ip_cidr' => null, + 'guest_routes' => [], 'link_state' => 'up', 'guest_link_state' => null, +]; +$host19['routes'][] = [ + 'destination_cidr' => '10.39.0.11/32', 'device' => $bridge19, + 'table' => $routeTableMethod->invoke(null, $vpcKey19), + 'owner' => 'virt-cloud-vpc-dataplane-v1', 'via' => null, 'metric' => 100, +]; +$partial19 = $observed(38, $digest19, ['190' => $host19]); +$partial19['state'] = 'applying'; +$partial19['next_stage_allowed'] = false; +$partial19['private_dataplane_verified'] = false; +CloudVpcService::recordObserved( + $vpcKey19, 38, $digest19, $partial19, $operation19 +); +$projection19a = CloudVpcService::instancePrivateProjection(19001); +$projection19b = CloudVpcService::instancePrivateProjection(19002); +$rootProjection19 = $publicRootMethod->invoke(null, CloudVpc::find(19)); +$assert(Capsule::table('cloud_vpc_operations')->where('operation_id', $operation19) + ->value('status') === 'succeeded' + && Capsule::table('cloud_vpc_members')->where('id', 126) + ->value('state') === 'active' + && Capsule::table('cloud_vpc_private_ips')->where('id', 226) + ->value('state') === 'active' + && Capsule::table('cloud_vpc_members')->where('id', 127) + ->value('state') === 'degraded' + && Capsule::table('cloud_vpc_members')->where('id', 127) + ->value('last_error_code') === 'vpc_private_guest_restart_required' + && ($projection19a['state'] ?? null) === 'active' + && ($projection19a['readback'] ?? null) === 'confirmed' + && ($projection19b['reason_code'] ?? null) + === 'vpc_private_guest_restart_required' + && ($rootProjection19['usable'] ?? true) === false, + 'A stopped sibling blocked an independently proven running member.'); + +$ready19 = $host19; +$ready19['state'] = 'active'; +$ready19['private_dataplane_verified'] = true; +$ready19['readback'] = 'confirmed'; +$ready19['last_error_code'] = null; +$ready19['member_nics'][1] = [ + 'member_id' => $memberKey19b, 'instance_uuid' => $uuid19b, + 'interface' => 'private0', 'private_ip_cidr' => '10.39.0.11/32', + 'private_mac' => $mac19b, 'xml_verified' => true, + 'config_verified' => true, 'seed_verified' => true, + 'restart_required' => false, + 'domain_running' => true, 'guest_network_verified' => true, + 'guest_network_readback' => 'confirmed', 'guest_reason_code' => null, + 'guest_private_mac' => $mac19b, + 'guest_private_ip_cidr' => '10.39.0.11/32', + 'guest_routes' => ['10.39.0.0/24'], + 'link_state' => 'up', 'guest_link_state' => 'up', +]; +CloudVpcService::recordObserved( + $vpcKey19, 38, $digest19, + $observed(38, $digest19, ['190' => $ready19]), + $operation19 +); +$readyProjection19b = CloudVpcService::instancePrivateProjection(19002); +$readyRootProjection19 = $publicRootMethod->invoke(null, CloudVpc::find(19)); +$assert(Capsule::table('cloud_vpc_operations')->where('operation_id', $operation19) + ->value('operation_id') === $operation19 + && Capsule::table('cloud_vpc_members')->where('id', 126) + ->value('state') === 'active' + && Capsule::table('cloud_vpc_members')->where('id', 127) + ->value('state') === 'active' + && ($readyProjection19b['state'] ?? null) === 'active' + && ($readyProjection19b['readback'] ?? null) === 'confirmed' + && ($readyRootProjection19['usable'] ?? false) === true, + 'Starting B did not converge only its runtime gate on the same operation.'); + +// Runtime usability also depends on the instance network-policy state. The +// VPC/member proof is still exact while quota/expiry enforcement is applying, +// but the public root must fail closed immediately in that window. +Capsule::table('cloud_instances')->where('id', 19001)->update([ + 'network_status' => 'blocking', +]); +$blockedRoot19 = $publicRootMethod->invoke(null, CloudVpc::find(19)); +$assert(($blockedRoot19['usable'] ?? true) === false, + 'A blocking instance network policy left an exact VPC publicly usable.'); +Capsule::table('cloud_instances')->where('id', 19001)->update([ + 'network_status' => 'active', +]); +$restoredPolicyRoot19 = $publicRootMethod->invoke(null, CloudVpc::find(19)); +$assert(($restoredPolicyRoot19['usable'] ?? false) === true, + 'An active instance network policy did not restore the unchanged VPC proof.'); + +// A graceful shutdown can be accepted while the VM is still running. Once +// the ordinary runtime heartbeat observes the later shutoff, public usability +// must close even though no VPC mutation or new VPC operation occurred. +Capsule::table('cloud_instance_runtime')->where('instance_id', 19002)->update([ + 'power_state' => 'stopped', 'sync_state' => 'ok', + 'last_success_at' => time(), 'updated_at' => time(), +]); +$heartbeatStoppedRoot19 = $publicRootMethod->invoke(null, CloudVpc::find(19)); +$assert(($heartbeatStoppedRoot19['usable'] ?? true) === false, + 'A fresh stopped runtime heartbeat left an old VPC proof publicly usable.'); +Capsule::table('cloud_instance_runtime')->where('instance_id', 19002)->update([ + 'power_state' => 'running', 'sync_state' => 'ok', + 'last_success_at' => time(), 'updated_at' => time(), +]); +$heartbeatRunningRoot19 = $publicRootMethod->invoke(null, CloudVpc::find(19)); +$assert(($heartbeatRunningRoot19['usable'] ?? false) === true, + 'A fresh running heartbeat did not restore the unchanged exact VPC proof.'); + +$expectation19b = CloudVpcService::instancePrivateExpectation(19002); +$stoppedObservation19b = [ + 'version' => 1, 'required' => true, 'vm_uuid' => $uuid19b, + 'vpc_id' => $vpcKey19, 'member_id' => $memberKey19b, + 'interface' => 'private0', 'state' => 'stopped', + 'desired_revision' => 38, 'observed_revision' => 38, + 'desired_digest' => $digest19, 'observed_digest' => $digest19, + 'readback' => 'confirmed', 'link_state' => 'up', + 'private_ip_cidr' => '10.39.0.11/32', 'private_mac' => $mac19b, + 'xml_verified' => true, 'config_verified' => true, + 'seed_verified' => true, + 'domain_running' => false, 'restart_required' => false, + 'guest_network_verified' => false, + 'guest_network_readback' => 'stopped', 'guest_link_state' => null, + 'reason_code' => null, 'observed_at' => time(), +]; +$shutdownRunningObservation19b = $stoppedObservation19b; +$shutdownRunningObservation19b['state'] = 'active'; +$shutdownRunningObservation19b['domain_running'] = true; +$shutdownRunningObservation19b['guest_network_verified'] = true; +$shutdownRunningObservation19b['guest_network_readback'] = 'confirmed'; +$shutdownRunningObservation19b['guest_link_state'] = 'up'; +$shutdownAccepted19b = CloudVpcService::validatePrivateObservation( + $expectation19b, $shutdownRunningObservation19b, + 'action_shutdown_policy_active', $uuid19b +); +$assert(($shutdownAccepted19b['ok'] ?? false) === true + && ($shutdownAccepted19b['state']['state'] ?? null) === 'stopping' + && CloudVpcService::recordPrivateObservation( + 19002, $shutdownAccepted19b, str_repeat('0', 31) . '9' + ) === true, + 'Accepted graceful shutdown did not enter a non-usable transition.'); +Capsule::table('cloud_instance_runtime')->where('instance_id', 19002)->update([ + 'power_state' => 'shutdown', 'sync_state' => 'ok', + 'last_success_at' => time(), 'updated_at' => time(), +]); +$shutdownPendingProjection19b = CloudVpcService::instancePrivateProjection(19002); +$shutdownPendingRoot19 = $publicRootMethod->invoke(null, CloudVpc::find(19)); +$assert(Capsule::table('cloud_vpc_members')->where('id', 127) + ->value('state') === 'stopping' + && Capsule::table('cloud_vpc_private_ips')->where('id', 227) + ->value('state') === 'stopping' + && ($shutdownPendingProjection19b['state'] ?? null) === 'stopping' + && ($shutdownPendingRoot19['usable'] ?? true) === false, + 'Graceful shutdown acceptance left member, lease or root runtime usable.'); +$periodicShutdownPending19 = $ready19; +$periodicShutdownPending19['state'] = 'active'; +$periodicShutdownPending19['private_dataplane_verified'] = true; +CloudVpcService::recordObserved( + $vpcKey19, 38, $digest19, + $observed(38, $digest19, ['190' => $periodicShutdownPending19]), + $operation19 +); +$assert(Capsule::table('cloud_vpc_members')->where('id', 127) + ->value('state') === 'stopping' + && Capsule::table('cloud_vpcs')->where('id', 19) + ->value('sync_state') === 'waiting' + && Capsule::table('cloud_vpcs')->where('id', 19) + ->value('last_error_code') === 'vpc_private_guest_shutdown_pending', + 'A periodic running/QGA receipt reopened a graceful-shutdown transition.'); +$shutdownValidation19b = CloudVpcService::validatePrivateObservation( + $expectation19b, $stoppedObservation19b, + 'action_shutdown_policy_active', $uuid19b +); +$poweroffValidation19b = CloudVpcService::validatePrivateObservation( + $expectation19b, $stoppedObservation19b, + 'action_poweroff_policy_active', $uuid19b +); +$assert(($shutdownValidation19b['ok'] ?? false) === true + && ($poweroffValidation19b['ok'] ?? false) === true + && CloudVpcService::recordPrivateObservation( + 19002, $shutdownValidation19b, str_repeat('1', 31) . '9' + ) === true, + 'Shutdown with a preserved configured-up link was not stored as stopped.'); +Capsule::table('cloud_instance_runtime')->where('instance_id', 19002)->update([ + 'power_state' => 'stopped', 'sync_state' => 'ok', + 'last_success_at' => time(), 'updated_at' => time(), +]); +$stoppedProjection19b = CloudVpcService::instancePrivateProjection(19002); +$stoppedRoot19 = $publicRootMethod->invoke(null, CloudVpc::find(19)); +$assert(Capsule::table('cloud_vpc_members')->where('id', 127) + ->value('state') === 'stopped' + && Capsule::table('cloud_vpc_private_ips')->where('id', 227) + ->value('state') === 'stopped' + && ($stoppedProjection19b['state'] ?? null) === 'stopped' + && ($stoppedProjection19b['link_state'] ?? null) === 'up' + && ($stoppedRoot19['usable'] ?? true) === false, + 'A confirmed stopped guest remained active or publicly usable.'); + +// Periodic VPC observation must preserve a lifecycle-confirmed stopped state; +// the strict guest probe is intentionally absent for an inactive domain. +$periodicStopped19 = $ready19; +$periodicStopped19['state'] = 'applying'; +$periodicStopped19['private_dataplane_verified'] = false; +$periodicStopped19['last_error_code'] = 'vpc_private_guest_stopped'; +$periodicStopped19['member_nics'][1]['domain_running'] = false; +$periodicStopped19['member_nics'][1]['restart_required'] = false; +$periodicStopped19['member_nics'][1]['guest_network_verified'] = false; +$periodicStopped19['member_nics'][1]['guest_network_readback'] = 'stopped'; +$periodicStopped19['member_nics'][1]['guest_reason_code'] = + 'vpc_private_guest_stopped'; +$periodicStopped19['member_nics'][1]['guest_private_mac'] = null; +$periodicStopped19['member_nics'][1]['guest_private_ip_cidr'] = null; +$periodicStopped19['member_nics'][1]['guest_routes'] = []; +$periodicStoppedObservation19 = $observed(38, $digest19, [ + '190' => $periodicStopped19, +]); +$periodicStoppedObservation19['state'] = 'applying'; +$periodicStoppedObservation19['next_stage_allowed'] = false; +$periodicStoppedObservation19['private_dataplane_verified'] = false; +CloudVpcService::recordObserved( + $vpcKey19, 38, $digest19, $periodicStoppedObservation19, $operation19 +); +$assert(Capsule::table('cloud_vpc_members')->where('id', 127) + ->value('state') === 'stopped' + && Capsule::table('cloud_vpc_private_ips')->where('id', 227) + ->value('state') === 'stopped' + && Capsule::table('cloud_vpcs')->where('id', 19) + ->value('last_error_code') === 'vpc_private_guest_stopped', + 'Periodic reconciliation overwrote exact stopped evidence with degraded.'); + +// Service policy preserves both the stopped power target and the independent +// network policy. A tenant-stopped guest must remain stopped across expiry and +// renewal when start_if_suspended=false. +$serviceSuspendStopped19b = CloudVpcService::validatePrivateObservation( + $expectation19b, $stoppedObservation19b, + 'service_suspend_policy_active', $uuid19b +); +$serviceResumeStopped19b = CloudVpcService::validatePrivateObservation( + $expectation19b, $stoppedObservation19b, + 'service_resume_stopped_policy_active', $uuid19b +); +$unboundServiceResume19b = CloudVpcService::validatePrivateObservation( + $expectation19b, $stoppedObservation19b, + 'service_resume_unbound_policy_active', $uuid19b +); +$assert(($serviceSuspendStopped19b['ok'] ?? false) === true + && ($serviceResumeStopped19b['ok'] ?? false) === true + && ($unboundServiceResume19b['ok'] ?? true) === false, + 'Stopped expiry/resume policy was not bound to its explicit payload target.'); + +// Link policy may change while the domain stays stopped. Both operations are +// exact configuration transitions, but neither makes runtime usable. +$stoppedDownObservation19b = $stoppedObservation19b; +$stoppedDownObservation19b['link_state'] = 'down'; +$stoppedSuspendValidation19b = CloudVpcService::validatePrivateObservation( + $expectation19b, $stoppedDownObservation19b, + 'network_suspend', $uuid19b +); +$blockedPoweroffValidation19b = CloudVpcService::validatePrivateObservation( + $expectation19b, $stoppedDownObservation19b, + 'action_poweroff_policy_blocked', $uuid19b +); +$blockedServiceSuspend19b = CloudVpcService::validatePrivateObservation( + $expectation19b, $stoppedDownObservation19b, + 'service_suspend_policy_blocked', $uuid19b +); +$blockedServiceResumeStopped19b = CloudVpcService::validatePrivateObservation( + $expectation19b, $stoppedDownObservation19b, + 'service_resume_stopped_policy_blocked', $uuid19b +); +Capsule::table('cloud_instances')->where('id', 19002)->update([ + 'network_status' => 'blocked_traffic', +]); +$assert(($stoppedSuspendValidation19b['ok'] ?? false) === true + && ($blockedPoweroffValidation19b['ok'] ?? false) === true + && ($blockedServiceSuspend19b['ok'] ?? false) === true + && ($blockedServiceResumeStopped19b['ok'] ?? false) === true + && CloudVpcService::recordPrivateObservation( + 19002, $stoppedSuspendValidation19b, str_repeat('2', 31) . '9' + ) === true + && (CloudVpcService::instancePrivateProjection(19002)['state'] ?? null) + === 'stopped' + && (CloudVpcService::instancePrivateProjection(19002)['link_state'] ?? null) + === 'down', + 'Stopped network suspend did not persist configured-down without activation.'); +$stoppedResumeValidation19b = CloudVpcService::validatePrivateObservation( + $expectation19b, $stoppedObservation19b, + 'network_resume', $uuid19b +); +Capsule::table('cloud_instances')->where('id', 19002)->update([ + 'network_status' => 'active', +]); +$assert(($stoppedResumeValidation19b['ok'] ?? false) === true + && CloudVpcService::recordPrivateObservation( + 19002, $stoppedResumeValidation19b, str_repeat('3', 31) . '9' + ) === true + && (CloudVpcService::instancePrivateProjection(19002)['state'] ?? null) + === 'stopped' + && (CloudVpcService::instancePrivateProjection(19002)['link_state'] ?? null) + === 'up', + 'Stopped network resume became usable or lost configured-up evidence.'); + +// Starting an active-policy VM needs a new, guest-visible QGA proof. The same +// config-only stopped receipt is insufficient. +$activeObservation19b = $stoppedObservation19b; +$activeObservation19b['state'] = 'active'; +$activeObservation19b['domain_running'] = true; +$activeObservation19b['guest_network_verified'] = true; +$activeObservation19b['guest_network_readback'] = 'confirmed'; +$activeObservation19b['guest_link_state'] = 'up'; +$startValidation19b = CloudVpcService::validatePrivateObservation( + $expectation19b, $activeObservation19b, + 'action_start_policy_active', $uuid19b +); +$missingStartGuest19b = $activeObservation19b; +$missingStartGuest19b['guest_network_verified'] = false; +$missingStartValidation19b = CloudVpcService::validatePrivateObservation( + $expectation19b, $missingStartGuest19b, + 'action_start_policy_active', $uuid19b +); +$genericActiveValidation19b = CloudVpcService::validatePrivateObservation( + $expectation19b, $activeObservation19b, + 'snapshot_restore', $uuid19b +); +$legacyActiveObservation19b = $activeObservation19b; +unset($legacyActiveObservation19b['seed_verified']); +unset($legacyActiveObservation19b['restart_required']); +unset($legacyActiveObservation19b['guest_network_verified']); +unset($legacyActiveObservation19b['guest_network_readback']); +unset($legacyActiveObservation19b['guest_link_state']); +$genericMissingGuest19b = CloudVpcService::validatePrivateObservation( + $expectation19b, $legacyActiveObservation19b, + 'snapshot_restore', $uuid19b +); +$assert(($startValidation19b['ok'] ?? false) === true + && ($missingStartValidation19b['ok'] ?? true) === false + && ($genericActiveValidation19b['ok'] ?? false) === true + && ($genericMissingGuest19b['ok'] ?? true) === false + && CloudVpcService::recordPrivateObservation( + 19002, $startValidation19b, str_repeat('4', 31) . '9' + ) === true, + 'An active lifecycle result did not require fresh guest-visible private readiness.'); +Capsule::table('cloud_instance_runtime')->where('instance_id', 19002)->update([ + 'power_state' => 'running', 'sync_state' => 'ok', + 'last_success_at' => time(), 'updated_at' => time(), +]); +$startedRoot19 = $publicRootMethod->invoke(null, CloudVpc::find(19)); +$assert(($startedRoot19['usable'] ?? false) === true, + 'Fresh QGA start did not restore VPC usability after stopped state.'); + +$suspendedObservation19b = [ + 'version' => 1, 'required' => true, 'vm_uuid' => $uuid19b, + 'vpc_id' => $vpcKey19, 'member_id' => $memberKey19b, + 'interface' => 'private0', 'state' => 'suspended', + 'desired_revision' => 38, 'observed_revision' => 38, + 'desired_digest' => $digest19, 'observed_digest' => $digest19, + 'readback' => 'confirmed', 'link_state' => 'down', + 'private_ip_cidr' => '10.39.0.11/32', 'private_mac' => $mac19b, + 'xml_verified' => true, 'config_verified' => true, + 'seed_verified' => true, + 'domain_running' => true, 'restart_required' => false, + 'guest_network_verified' => false, + 'guest_network_readback' => 'suspended', 'guest_link_state' => null, + 'reason_code' => null, 'observed_at' => time(), +]; +$suspendValidation19b = CloudVpcService::validatePrivateObservation( + $expectation19b, $suspendedObservation19b, 'network_suspend', $uuid19b +); +$assert(($suspendValidation19b['ok'] ?? false) === true + && CloudVpcService::recordPrivateObservation( + 19002, $suspendValidation19b, str_repeat('d', 31) . '9' + ) === true, + 'An exact lifecycle link-down observation was not persisted.'); +Capsule::table('cloud_instances')->where('id', 19002)->update([ + 'network_status' => 'blocked_traffic', +]); +$suspendedProjection19b = CloudVpcService::instancePrivateProjection(19002); +$suspendedRoot19 = $publicRootMethod->invoke(null, CloudVpc::find(19)); +$assert(Capsule::table('cloud_vpc_members')->where('id', 127) + ->value('state') === 'suspended' + && Capsule::table('cloud_vpc_private_ips')->where('id', 227) + ->value('state') === 'suspended' + && Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 19) + ->where('host_id', 190)->value('state') === 'degraded' + && Capsule::table('cloud_vpcs')->where('id', 19) + ->value('state') === 'degraded' + && ($suspendedProjection19b['state'] ?? null) === 'suspended' + && ($suspendedProjection19b['link_state'] ?? null) === 'down' + && ($suspendedRoot19['usable'] ?? true) === false, + 'Lifecycle suspension left member, host or VPC runtime falsely usable.'); + +$periodicSuspended19 = $ready19; +$periodicSuspended19['state'] = 'applying'; +$periodicSuspended19['private_dataplane_verified'] = false; +$periodicSuspended19['last_error_code'] = 'vpc_private_guest_link_mismatch'; +$periodicSuspended19['member_nics'][1]['link_state'] = 'down'; +$periodicSuspended19['member_nics'][1]['guest_network_verified'] = false; +$periodicSuspended19['member_nics'][1]['guest_network_readback'] = 'suspended'; +$periodicSuspended19['member_nics'][1]['guest_reason_code'] = + 'vpc_private_guest_link_mismatch'; +$periodicObserved19 = $observed(38, $digest19, [ + '190' => $periodicSuspended19, +]); +$periodicObserved19['state'] = 'applying'; +$periodicObserved19['next_stage_allowed'] = false; +$periodicObserved19['private_dataplane_verified'] = false; +CloudVpcService::recordObserved( + $vpcKey19, 38, $digest19, $periodicObserved19, $operation19 +); +$periodicRoot19 = $publicRootMethod->invoke(null, CloudVpc::find(19)); +$assert(Capsule::table('cloud_vpc_members')->where('id', 127) + ->value('state') === 'suspended' + && Capsule::table('cloud_vpc_private_ips')->where('id', 227) + ->value('state') === 'suspended' + && Capsule::table('cloud_vpc_operations')->where('operation_id', $operation19) + ->value('status') === 'succeeded' + && Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 19) + ->where('host_id', 190)->value('last_error_code') + === 'vpc_private_network_suspended' + && Capsule::table('cloud_vpcs')->where('id', 19) + ->value('sync_state') === 'applied' + && Capsule::table('cloud_vpcs')->where('id', 19) + ->value('last_error_code') === 'vpc_private_network_suspended' + && ($periodicRoot19['usable'] ?? true) === false, + 'Periodic VPC observation overwrote intentional suspended runtime state.'); + +// User power actions and service resume preserve a blocked network target. +// A running domain is represented as suspended/config-exact, and an active/up +// receipt is rejected so neither path can bypass quota/expiry enforcement. +$blockedReboot19b = CloudVpcService::validatePrivateObservation( + $expectation19b, $suspendedObservation19b, + 'action_reboot_policy_blocked', $uuid19b +); +$blockedStart19b = CloudVpcService::validatePrivateObservation( + $expectation19b, $suspendedObservation19b, + 'action_start_policy_blocked', $uuid19b +); +$blockedServiceStart19b = CloudVpcService::validatePrivateObservation( + $expectation19b, $suspendedObservation19b, + 'service_resume_start_policy_blocked', $uuid19b +); +$blockedButUp19b = $activeObservation19b; +$blockedButUpValidation19b = CloudVpcService::validatePrivateObservation( + $expectation19b, $blockedButUp19b, + 'action_reboot_policy_blocked', $uuid19b +); +$transitioningPower19b = CloudVpcService::validatePrivateObservation( + $expectation19b, $suspendedObservation19b, + 'action_reboot_policy_unready', $uuid19b +); +$assert(($blockedReboot19b['ok'] ?? false) === true + && ($blockedStart19b['ok'] ?? false) === true + && ($blockedServiceStart19b['ok'] ?? false) === true + && ($blockedButUpValidation19b['ok'] ?? true) === false + && ($transitioningPower19b['ok'] ?? true) === false + && CloudVpcService::recordPrivateObservation( + 19002, $blockedReboot19b, str_repeat('5', 31) . '9' + ) === true + && Capsule::table('cloud_vpc_members')->where('id', 127) + ->value('state') === 'suspended' + && Capsule::table('cloud_instances')->where('id', 19002) + ->value('network_status') === 'blocked_traffic', + 'A blocked power/service action raised private0 or failed to converge safely.'); + +$resumedObservation19b = $suspendedObservation19b; +$resumedObservation19b['state'] = 'active'; +$resumedObservation19b['link_state'] = 'up'; +$resumedObservation19b['guest_network_verified'] = true; +$resumedObservation19b['guest_network_readback'] = 'confirmed'; +$resumedObservation19b['guest_link_state'] = 'up'; +$resumeValidation19b = CloudVpcService::validatePrivateObservation( + $expectation19b, $resumedObservation19b, 'network_resume', $uuid19b +); +$assert(($resumeValidation19b['ok'] ?? false) === true + && CloudVpcService::recordPrivateObservation( + 19002, $resumeValidation19b, str_repeat('e', 31) . '9' + ) === true, + 'An exact lifecycle link-up observation was not persisted.'); +Capsule::table('cloud_instances')->where('id', 19002)->update([ + 'network_status' => 'active', +]); +$resumedProjection19b = CloudVpcService::instancePrivateProjection(19002); +$resumedRoot19 = $publicRootMethod->invoke(null, CloudVpc::find(19)); +$assert(Capsule::table('cloud_vpc_members')->where('id', 127) + ->value('state') === 'active' + && Capsule::table('cloud_vpc_private_ips')->where('id', 227) + ->value('state') === 'active' + && Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 19) + ->where('host_id', 190)->value('state') === 'active' + && Capsule::table('cloud_vpcs')->where('id', 19) + ->value('state') === 'active' + && ($resumedProjection19b['state'] ?? null) === 'active' + && ($resumedProjection19b['link_state'] ?? null) === 'up' + && ($resumedRoot19['usable'] ?? false) === true, + 'Lifecycle resume became usable before exact link-up or failed to recover.'); + +// A running last member cannot be detached safely. Keep its tombstone and the +// operation retryable; after the VM is stopped, the exact removal receipt must +// converge the same operation ID instead of requiring a second detach request. +$vpcKey18 = str_repeat('6', 31) . '8'; +$digest18 = str_repeat('b', 63) . '8'; +$operation18 = str_repeat('7', 31) . '8'; +$memberKey18 = str_repeat('8', 31) . '8'; +$seedVpc(18, $vpcKey18, 36, $digest18, $operation18, 'member_detach'); +$seedSubnet(119, 18, str_repeat('9', 31) . '8', '10.38.0.0/25', + 'active', 36, $digest18); +$seedMember(125, 225, 18, 119, $memberKey18, 18001, + '18181818-1818-4818-8818-181818181818', 180, '10.38.0.10', + 'deleting', 36, $digest18); +$seedHost(18, 180, 'deleting', 36, $digest18, $operation18); +CloudVpcService::recordObserved($vpcKey18, 36, $digest18, [ + 'evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION, + 'revision' => 0, 'digest' => null, 'state' => 'degraded', + 'next_stage_allowed' => false, 'private_dataplane_verified' => false, + 'fabric_gate' => ['active' => true], + 'hosts' => ['180' => [ + 'vpc_id' => $vpcKey18, 'host_id' => '180', + 'desired_revision' => 36, 'desired_digest' => $digest18, + 'observed_revision' => 0, 'observed_digest' => null, + 'state' => 'uncertain', 'transport_verified' => false, + 'private_dataplane_verified' => false, 'removed' => false, + 'last_error_code' => 'vpc_private_remove_requires_stopped', + ]], +], $operation18); +$assert(Capsule::table('cloud_vpc_operations')->where('operation_id', $operation18) + ->value('status') === 'uncertain' + && Capsule::table('cloud_vpc_members')->where('id', 125) + ->value('state') === 'deleting' + && Capsule::table('cloud_vpc_private_ips')->where('id', 225) + ->value('state') === 'deleting', + 'A running-member detach became terminal or released its tombstone early.'); +CloudVpcService::recordObserved( + $vpcKey18, 36, $digest18, + $observed(36, $digest18, [ + '180' => $removedHost($vpcKey18, 180, 36, $digest18), + ]), + $operation18 +); +$assert(Capsule::table('cloud_vpc_operations')->where('operation_id', $operation18) + ->value('status') === 'succeeded' + && Capsule::table('cloud_vpc_operations')->where('operation_id', $operation18) + ->value('operation_id') === $operation18 + && Capsule::table('cloud_vpc_members')->where('id', 125)->count() === 0 + && Capsule::table('cloud_vpc_private_ips')->where('id', 225)->count() === 0 + && Capsule::table('cloud_vpc_host_deployments')->where('vpc_id', 18) + ->where('host_id', 180)->value('state') === 'deleted', + 'Stopping the VM did not converge the original detach operation.'); + echo "VPC member/IP/subnet tombstone convergence contracts passed.\n"; diff --git a/web/tests/CloudVpcUserPageContractTest.php b/web/tests/CloudVpcUserPageContractTest.php index a1324916..7030bd73 100644 --- a/web/tests/CloudVpcUserPageContractTest.php +++ b/web/tests/CloudVpcUserPageContractTest.php @@ -126,7 +126,8 @@ $assert(str_contains($script, 'X-CSRF-Token') && str_contains($script, 'rule_key: row.dataset.aclKey'), 'VPC mutations must use same-origin CSRF protection and separate idempotency/entity keys.'); -foreach (['applying', 'degraded', 'uncertain', 'waiting', 'active', 'deleting'] as $status) { +foreach (['applying', 'degraded', 'uncertain', 'waiting', 'active', 'suspended', + 'deleting'] as $status) { $assert(str_contains($script, $status), 'VPC UI lost state projection for ' . $status . '.'); }