diff --git a/docs/FABRIC_HOST_ACCEPTANCE.md b/docs/FABRIC_HOST_ACCEPTANCE.md index 26542ad0..d83ace4a 100644 --- a/docs/FABRIC_HOST_ACCEPTANCE.md +++ b/docs/FABRIC_HOST_ACCEPTANCE.md @@ -156,7 +156,8 @@ ACL-first helper。后一分支不得 restart 主 Fabric unit 或任何 libvirt 保留时)和 `journalctl -u virt-cloud-fabric-apply-.service --no-pager`。 transient unit 使用 `--collect`,因此终态后 `systemctl status` 可以变成 not-found, 但 journal 仍必须可按完整 unit 名回查。journal 只得出现 - operation ID、固定 state/checkpoint/reason 与 systemd 生命周期;不得出现私钥、完整脚本、 + operation ID、固定 state/checkpoint/reason、固定 helper_phase/apply_check/退出码 + 与 systemd 生命周期;不得出现私钥、完整脚本、 peer 全量配置、SSH 路径或原始 stderr。 journal 恢复必须按实际进度显示固定子检查点 `journal_artifact_restore`、 `journal_nft_restore`、`journal_runtime_restore` 或 `journal_finalize`;任何未知值都只能投影为 @@ -164,6 +165,10 @@ ACL-first helper。后一分支不得 restart 主 Fabric unit 或任何 libvirt `journal_dispatch` 表示已验证恢复 helper 身份并即将调用;执行前检查分别显示 `preflight_sysctl/tools/artifacts/firewall/vpc/underlay/routes/persistence`(每项为独立 `preflight_` 检查点)。旧 `preflight` 的未知命令失败不得再统一误标为资源归属冲突。 + async runner 必须继续丢弃原始 stdout/stderr,只通过独立描述符记录净化后的 + `helper_state=failed helper_phase=... exit_status=...`。apply 在回滚开始前记录 + `apply_state=failed apply_check=... exit_status=...`,用于区别首次失败与回滚/服务 + 的次生错误;未知子项只能为 `unknown`,不能输出命令、配置或外部错误文本。 3. `/run/virt-cloud/fabric-operations/` 必须为 root:root `0700`,manifest/status 必须为 `0600`,apply/update/runner 必须为 `0700`;任一符号链接、多余文件、owner/mode/hash 漂移都必须 fail-closed。达到上限时只能精确删除最旧的已验证终态目录,不得删除 @@ -187,6 +192,14 @@ ACL-first helper。后一分支不得 restart 主 Fabric unit 或任何 libvirt ## 失败回滚和安全边界 +保留同计划的 `ready` journal 和历史 `helper.present`:即使历史 helper 不支持新版 +NetworkManager runtime shadow,也只能在旧 helper 与 journal pre/post-image 精确一致、 +meta 中 digest/UUID/manager/firewall/underlay 全部绑定当前方案、当前临时 helper 的权限与 +计划头核验后,交由当前实现恢复。恢复完成的 recovery-only 调用不得再次执行已恢复的 +旧 helper。跨计划仍沿历史恢复合同,不得把旧计划的权限移交给新模板;外部文件/密钥/nft +漂移必须保留现场并拒绝。新增 timer 必须在 `state=committed` 同步落盘之后才启动, +timer 启动失败只能保留 committed journal,不能回滚已提交的数据面。 + 在隔离环境中分别让 `nft -c/-f`、`firewall-cmd`、`wg`、`ip route`、 `nmcli/ifup`、systemd gate 和 timer 返回失败。每次都应看到 apply 失败并保持 `uncertain/degraded`;firewalld 模式的回滚先撤销 panel 接口的 runtime `trusted`,再以 diff --git a/engine/tests/test_fabric.py b/engine/tests/test_fabric.py index bba4aa5c..a699dc2b 100644 --- a/engine/tests/test_fabric.py +++ b/engine/tests/test_fabric.py @@ -3693,6 +3693,7 @@ def test_apply_queues_reconcile_dispatcher_only_after_durable_commit(): def _run_apply_convergence_tail( tmp_path: Path, *, start_fails: bool = False, + timer_fails: bool = False, stale_config: bool = False, emergency_present: bool = False, committed_trap_state: str | None = None, ) -> subprocess.CompletedProcess[str]: @@ -3709,6 +3710,9 @@ def _run_apply_convergence_tail( "\n}\ntrap rollback EXIT HUP INT TERM", rollback_start, ) + 2 rollback_function = rendered[rollback_start:rollback_end] + logger_start = rendered.index("vc_fabric_apply_failure_log() {") + logger_end = rendered.index("\n}\n", logger_start) + 2 + failure_logger = rendered[logger_start:logger_end] emergency_owner = tmp_path / "emergency.owner" sysctl_file = tmp_path / "fabric-sysctl.conf" sysctl_file.write_text(_sysctl_text(current), encoding="utf-8") @@ -3740,6 +3744,9 @@ def _run_apply_convergence_tail( systemctl.write_text( """#!/bin/sh case "$1:$2" in + start:virt-cloud-wg-fabric-reconcile.timer) + [ "${VC_TEST_TIMER_FAIL:-no}" = no ] || exit 1 + exit 0 ;; start:virt-cloud-wg-fabric-reconcile.service) [ "${VC_TEST_START_FAIL:-no}" = no ] || exit 1 : > "$VC_TEST_CONVERGED"; exit 0 ;; @@ -3836,6 +3843,7 @@ esac "vc_fabric_sysctl_file_owned() { return 0; }", "read_sysctl() { cat \"$1\"; }", "vc_fabric_operation_checkpoint() { :; }", + failure_logger, *trap_setup, "exec 9>>\"$LOCK\"", tail, @@ -3846,6 +3854,7 @@ esac env={ "PATH": "/usr/bin:/bin", "VC_TEST_START_FAIL": "yes" if start_fails else "no", + "VC_TEST_TIMER_FAIL": "yes" if timer_fails else "no", "VC_TEST_EMERGENCY": "yes" if emergency_present else "no", "VC_TEST_CONVERGED": str(converged), }, @@ -3884,6 +3893,16 @@ def test_post_commit_dispatcher_failure_preserves_committed_journal(tmp_path): ) == "committed\n" +def test_post_commit_timer_failure_preserves_committed_journal(tmp_path): + result = _run_apply_convergence_tail( + tmp_path, timer_fails=True, committed_trap_state="committed", + ) + assert result.returncode == 79, result.stderr + assert (tmp_path / "txn" / "state").read_text( + encoding="utf-8", + ) == "committed\n" + + def test_old_committed_trap_never_deletes_concurrent_ready_journal(tmp_path): result = _run_apply_convergence_tail( tmp_path, start_fails=True, committed_trap_state="ready", diff --git a/engine/tests/test_fabric_apply_final_readback.py b/engine/tests/test_fabric_apply_final_readback.py new file mode 100644 index 00000000..c7500f3f --- /dev/null +++ b/engine/tests/test_fabric_apply_final_readback.py @@ -0,0 +1,338 @@ +"""Execute the production-rendered final readback with real CLI serialization. + +Only external tools are replaced. The final shell body, nft owner checks, +tenant-set matcher and firewall-zone enumerator are the production code. +""" +from __future__ import annotations + +import base64 +import json +import os +from pathlib import Path +import subprocess +import sys + +import pytest + +from virsh_panel.fabric import ( + _FABRIC_NFT_VALIDATION_SH, + _FABRIC_WG_READBACK_SH, + _BOOT_HELPER, + _FIREWALL_MODE_GUARD_SH, + _TRANSACTION_SCRIPT, + _VPC_STATE_LOADER, + _apply_script, + _config_text, + _nft_rules_text, + parse_fabric_plan, +) + + +def _between(text: str, start: str, end: str) -> str: + begin = text.index(start) + return text[begin:text.index(end, begin)] + + +_TOOLS = r'''import json +import os +from pathlib import Path +import sys + +root = Path(os.environ["VC_FINAL_ROOT"]) +name = Path(sys.argv[0]).name +args = sys.argv[1:] +with (root / "calls").open("a", encoding="utf-8") as log: + log.write(json.dumps([name, *args]) + "\n") +fixture = json.loads((root / "responses").read_text()) +call = json.dumps([name, *args]) +if call not in fixture: + print("unexpected test tool invocation", file=sys.stderr) + raise SystemExit(99) +if name == "wg" and args == ["pubkey"]: + if sys.stdin.read() != "test-only-private-identity\n": + raise SystemExit(98) +response = fixture[call] +sys.stdout.write(response["stdout"]) +raise SystemExit(response["status"]) +''' + + +class FinalFixture: + def __init__(self, tmp_path: Path, manager: str, peers: int = 1): + self.root = tmp_path + self.manager = manager + self.plan = parse_fabric_plan({ + "fabric_id": "final-readback", "region": "hk", + "overlay_cidr": "100.64.1.0/24", "revision": 1, + "nodes": [ + {"host_id": str(index), "overlay_ip": f"100.64.1.{index}", + "endpoint": f"203.0.113.{index}:51820"} + for index in range(1, peers + 2) + ], + }) + self.keys = { + node.host_id: base64.b64encode(bytes([index]) * 32).decode() + for index, node in enumerate(self.plan.nodes, 1) + } + self.responses: dict[str, dict[str, object]] = {} + self.routes = "" + tool = tmp_path / "tool" + tool.write_text(f"#!{sys.executable}\n" + _TOOLS, encoding="utf-8") + tool.chmod(0o700) + for name in ("ip", "wg", "nft", "firewall-cmd"): + (tmp_path / name).symlink_to(tool) + (tmp_path / "private").write_text("test-only-private-identity\n") + (tmp_path / "config").write_text(_config_text( + self.plan, self.plan.nodes[0], self.keys, "firewalld_nft")) + for filename in ("vpc-peers", "vpc-routes", "vpc-tenants"): + (tmp_path / filename).write_text("") + + self.respond("nft", ["-s", "list", "table", "inet", + "virt_cloud_wg_fabric"], + _nft_rules_text(self.plan.digest)) + self.respond("nft", ["list", "set", "inet", "virt_cloud_wg_fabric", + "vpc_tenant_ips"], + "table inet virt_cloud_wg_fabric {\n" + "\tset vpc_tenant_ips {\n\t\ttype ipv4_addr\n" + "\t\tflags interval\n\t}\n}\n") + self.respond("wg", ["show", "wg-fabric0"], "") + self.respond("wg", ["pubkey"], self.keys["1"] + "\n") + self.respond("wg", ["show", "wg-fabric0", "public-key"], + self.keys["1"] + "\n") + self.respond("wg", ["show", "wg-fabric0", "listen-port"], "51820\n") + self.respond("ip", ["-o", "link", "show", "dev", "wg-fabric0"], + "16: wg-fabric0: mtu 1420 " + "qdisc noqueue state UNKNOWN mode DEFAULT group default qlen 1000\n") + # ip -o replaces the line break between address and lifetime by a + # backslash. The entire address record must remain one physical line. + self.respond("ip", ["-o", "-4", "address", "show", "dev", "wg-fabric0"], + "16: wg-fabric0 inet 100.64.1.1/32 scope global wg-fabric0\\" + " valid_lft forever preferred_lft forever\n") + live_peers = self.plan.nodes[1:] + self.respond("wg", ["show", "wg-fabric0", "peers"], + "".join(self.keys[node.host_id] + "\n" for node in live_peers)) + self.respond("wg", ["show", "wg-fabric0", "endpoints"], + "".join(self.keys[node.host_id] + "\t" + node.endpoint + "\n" + for node in reversed(live_peers))) + self.respond("wg", ["show", "wg-fabric0", "allowed-ips"], + "".join(self.keys[node.host_id] + "\t" + node.allowed_ip + "\n" + for node in reversed(live_peers))) + metric = " metric 50" if manager == "networkmanager_nmcli" else "" + self.routes = "".join(node.overlay_ip + " scope link" + metric + "\n" + for node in live_peers) + self.routes += ("local 100.64.1.1 table local proto kernel scope host " + "src 100.64.1.1\n") + self.respond("ip", ["-o", "route", "show", "table", "all", "dev", + "wg-fabric0"], self.routes) + for node in live_peers: + self.respond("ip", ["route", "get", node.overlay_ip, + "from", "100.64.1.1"], + node.overlay_ip + " from 100.64.1.1 dev wg-fabric0 uid 0\n" + " cache\n") + for permanent in (False, True): + prefix = ["--permanent"] if permanent else [] + self.respond("firewall-cmd", prefix + ["--get-zones"], "public trusted\n") + for zone in ("public", "trusted"): + active = not permanent and zone == "trusted" + self.respond("firewall-cmd", prefix + [f"--zone={zone}", + "--query-interface=wg-fabric0"], + "yes\n" if active else "no\n", 0 if active else 1) + + def respond(self, tool: str, args: list[str], stdout: str, status: int = 0): + self.responses[json.dumps([tool, *args])] = { + "stdout": stdout, "status": status, + } + + def add_vpc_member(self): + key = self.keys["2"] + (self.root / "vpc-peers").write_text(key + "|10.1.0.3/32\n") + (self.root / "vpc-tenants").write_text("10.1.0.3/32\n") + (self.root / "vpc-routes").write_text( + "10.1.0.3/32|wg-fabric0|100|100.64.1.2|100\n") + self.respond("nft", ["-s", "list", "table", "inet", "virt_cloud_wg_fabric"], + _nft_rules_text(self.plan.digest).replace( + "flags interval;", "flags interval;\n elements = { 10.1.0.3 }", 1)) + self.respond("nft", ["list", "set", "inet", "virt_cloud_wg_fabric", "vpc_tenant_ips"], + "table inet virt_cloud_wg_fabric {\n\tset vpc_tenant_ips {\n" + "\t\ttype ipv4_addr\n\t\tflags interval\n" + "\t\telements = { 10.1.0.3 }\n\t}\n}\n") + self.respond("ip", ["-o", "route", "show", "table", "all", "dev", "wg-fabric0"], + self.routes + "10.1.0.3 via 100.64.1.2 table 100 proto 186 metric 100\n") + + def run(self): + rendered = _apply_script( + self.plan, self.plan.nodes[0], self.keys, + deployment_mode=self.manager, firewall_mode="firewalld_nft") + final = _between(rendered, "vc_fabric_operation_checkpoint final_readback\n", + "_txn_state_tmp=$(mktemp") + final = final.split("\n", 1)[1] + functions = "\n".join(( + _FABRIC_NFT_VALIDATION_SH, + _FABRIC_WG_READBACK_SH, + _between(_TRANSACTION_SCRIPT, "nft_panel_owned() {", "snapshot_file() {"), + _between(_VPC_STATE_LOADER, "vpc_state_allowed_for_peer() {", + "vpc_state_route_owned() {"), + _VPC_STATE_LOADER[_VPC_STATE_LOADER.index("vpc_state_nft_set_matches() {"):], + _between(_FIREWALL_MODE_GUARD_SH, "vc_firewall_zone() {", + "vc_firewall_zone_exists() {"), + )) + env = {**os.environ, "VC_FINAL_ROOT": str(self.root), + "IFACE": "wg-fabric0", "EXPECTED_DIGEST": self.plan.digest, + "FIREWALL_MODE": "firewalld_nft", + "MANAGED_PEERS": " ".join(self.keys[n.host_id] for n in self.plan.nodes[1:]), + "MANAGED_ROUTES": " ".join(n.allowed_ip for n in self.plan.nodes[1:]), + "KEY": str(self.root / "private"), "CONF": str(self.root / "config"), + "VPC_PEER_FILE": str(self.root / "vpc-peers"), + "VPC_ROUTE_FILE": str(self.root / "vpc-routes"), + "VPC_TENANT_FILE": str(self.root / "vpc-tenants")} + env.update({name: str(self.root / tool) for name, tool in ( + ("WG", "wg"), ("IP", "ip"), ("NFT", "nft"), + ("FIREWALL_CMD", "firewall-cmd"))}) + (self.root / "responses").write_text(json.dumps(self.responses)) + result = subprocess.run(["/bin/sh", "-eu", "-c", functions + "\n" + final], + capture_output=True, text=True, env=env, check=False) + assert "unexpected test tool invocation" not in result.stderr + return result + + +@pytest.mark.parametrize("manager", ["ifupdown", "networkmanager_nmcli"]) +@pytest.mark.parametrize("peers", [1, 2]) +def test_actual_final_slice_accepts_debian_and_alma_serialization(tmp_path, manager, peers): + fixture = FinalFixture(tmp_path, manager, peers) + if manager == "networkmanager_nmcli": + rules = _nft_rules_text(fixture.plan.digest).replace( + "hook output priority -100;", "hook output priority dstnat;") + rules = "\n".join(line.replace(" ", "\t") for line in rules.splitlines()) + "\n" + fixture.respond("nft", ["-s", "list", "table", "inet", + "virt_cloud_wg_fabric"], rules) + result = fixture.run() + assert result.returncode == 0, result.stderr + + +@pytest.mark.parametrize("drift", ["nft", "public-key", "listen-port", "down", "address", + "extra-peer", "endpoint", "allowed", "route", "path", "zone"]) +def test_actual_final_slice_rejects_mutable_readback_drift(tmp_path, drift): + fixture = FinalFixture(tmp_path, "ifupdown") + key = fixture.keys["2"] + if drift == "nft": + fixture.respond("nft", ["-s", "list", "table", "inet", "virt_cloud_wg_fabric"], + _nft_rules_text(fixture.plan.digest).replace("counter drop", "counter accept", 1)) + elif drift in ("public-key", "listen-port"): + fixture.respond("wg", ["show", "wg-fabric0", drift], "wrong\n") + elif drift == "down": + fixture.respond("ip", ["-o", "link", "show", "dev", "wg-fabric0"], + "16: wg-fabric0: mtu 1420\n") + elif drift == "address": + fixture.respond("ip", ["-o", "-4", "address", "show", "dev", "wg-fabric0"], + "16: wg-fabric0 inet 100.64.1.1/24 scope global wg-fabric0\n") + elif drift == "extra-peer": + fixture.respond("wg", ["show", "wg-fabric0", "peers"], key + "\nforeign\n") + elif drift == "endpoint": + fixture.respond("wg", ["show", "wg-fabric0", "endpoints"], key + "\t203.0.113.9:51820\n") + elif drift == "allowed": + fixture.respond("wg", ["show", "wg-fabric0", "allowed-ips"], key + "\t100.64.1.9/32\n") + elif drift == "route": + fixture.respond("ip", ["-o", "route", "show", "table", "all", "dev", "wg-fabric0"], + fixture.routes + "100.64.1.3 scope link\n") + elif drift == "path": + fixture.respond("ip", ["route", "get", "100.64.1.2", "from", "100.64.1.1"], + "100.64.1.2 from 100.64.1.1 dev br0 uid 0\n cache\n") + else: + fixture.respond("firewall-cmd", ["--zone=trusted", "--query-interface=wg-fabric0"], "no\n", 1) + result = fixture.run() + assert result.returncode in (71, 78), result.stderr + + +@pytest.mark.parametrize("manager", ["ifupdown", "networkmanager_nmcli"]) +@pytest.mark.parametrize("reverse", [False, True]) +@pytest.mark.parametrize("peers", [1, 2]) +def test_actual_final_slice_accepts_official_multiple_allowed_ips_serialization(tmp_path, manager, reverse, peers): + fixture = FinalFixture(tmp_path, manager, peers) + key = fixture.keys["2"] + fixture.add_vpc_member() + prefixes = ["100.64.1.2/32", "10.1.0.3/32"] + if reverse: + prefixes.reverse() + fixture.respond("wg", ["show", "wg-fabric0", "allowed-ips"], + key + "\t" + " ".join(prefixes) + "\n" + "".join( + fixture.keys[node.host_id] + "\t" + node.allowed_ip + "\n" + for node in fixture.plan.nodes[2:])) + assert fixture.run().returncode == 0 + + +@pytest.mark.parametrize("drift", ["missing", "extra", "duplicate", "unknown", "multiple-key-fields"]) +def test_actual_final_slice_requires_complete_unique_peer_records(tmp_path, drift): + fixture = FinalFixture(tmp_path, "ifupdown", 2) + first, second = fixture.keys["2"], fixture.keys["3"] + if drift == "missing": + actual = first + "\n" + elif drift == "extra": + actual = first + "\n" + second + "\n" + base64.b64encode(bytes([9]) * 32).decode() + "\n" + elif drift == "duplicate": + actual = first + "\n" + second + "\n" + first + "\n" + elif drift == "unknown": + actual = first + "\n" + "bad-key\n" + else: + actual = first + "\t" + second + "\n" + fixture.respond("wg", ["show", "wg-fabric0", "peers"], actual) + assert fixture.run().returncode == 71 + + +@pytest.mark.parametrize("drift", ["wide", "public", "duplicate", "missing", "extra", "csv", + "malformed", "ipv6", "duplicate-peer", "unknown-peer"]) +def test_actual_final_slice_multiple_allowed_ips_remains_fail_closed(tmp_path, drift): + fixture = FinalFixture(tmp_path, "ifupdown") + key = fixture.keys["2"] + fixture.add_vpc_member() + values = ["100.64.1.2/32", "10.1.0.3/32"] + if drift == "wide": + values[1] = "10.1.0.3/24" + elif drift == "public": + values[1] = "203.0.113.9/32" + elif drift == "duplicate": + values.append(values[1]) + elif drift == "missing": + values.pop() + elif drift == "extra": + values.append("10.1.0.4/32") + elif drift == "malformed": + values[1] = "10.1.0.256/32" + elif drift == "ipv6": + values[1] = "fd00::3/128" + actual = key + "\t" + ("," if drift == "csv" else " ").join(values) + "\n" + if drift == "duplicate-peer": + actual += actual + elif drift == "unknown-peer": + actual += base64.b64encode(bytes([9]) * 32).decode() + "\t10.1.0.9/32\n" + fixture.respond("wg", ["show", "wg-fabric0", "allowed-ips"], actual) + assert fixture.run().returncode == 71 + + +@pytest.mark.parametrize("peers", [1, 2]) +@pytest.mark.parametrize("reverse", [False, True]) +@pytest.mark.parametrize("vpc", [False, True]) +def test_boot_actual_peer_readback_uses_identical_record_validator(tmp_path, peers, reverse, vpc): + fixture = FinalFixture(tmp_path, "ifupdown", peers) + if vpc: + fixture.add_vpc_member() + fixture.respond("wg", ["show", "wg-fabric0", "allowed-ips"], + fixture.keys["2"] + "\t10.1.0.3/32 100.64.1.2/32\n" + "".join( + fixture.keys[node.host_id] + "\t" + node.allowed_ip + "\n" + for node in fixture.plan.nodes[2:])) + live_keys = [fixture.keys[node.host_id] for node in fixture.plan.nodes[1:]] + if reverse: + live_keys.reverse() + fixture.respond("wg", ["show", "wg-fabric0", "peers"], "\n".join(live_keys) + "\n") + (tmp_path / "responses").write_text(json.dumps(fixture.responses)) + peer_body = _between(_BOOT_HELPER, 'actual_peers=$("$WG" show "$IFACE" peers', + "actual_fabric_routes=") + allowed = _between(_VPC_STATE_LOADER, "vpc_state_allowed_for_peer() {", + "vpc_state_route_owned() {") + result = subprocess.run(["/bin/sh", "-eu", "-c", _FABRIC_WG_READBACK_SH + "\n" + allowed + "\n" + peer_body], + capture_output=True, text=True, check=False, + env={**os.environ, "VC_FINAL_ROOT": str(tmp_path), + "WG": str(tmp_path / "wg"), "IFACE": "wg-fabric0", + "seen_peers": " ".join(live_keys), + "CONF": str(tmp_path / "config"), + "VPC_PEER_FILE": str(tmp_path / "vpc-peers")}) + assert result.returncode == 0, result.stderr diff --git a/engine/tests/test_fabric_apply_safe_log.py b/engine/tests/test_fabric_apply_safe_log.py new file mode 100644 index 00000000..55ba0adb --- /dev/null +++ b/engine/tests/test_fabric_apply_safe_log.py @@ -0,0 +1,243 @@ +"""Execute rendered async journal plumbing without running host mutations. + +The runner, update state machine, logger, helper EXIT trap and committed +rollback branch are production-rendered code. Only stat's platform-specific +ownership report and sync are external-tool doubles inside temporary files. +""" +from __future__ import annotations + +import base64 +import os +from pathlib import Path +import shlex +import subprocess +import sys +import time + +import pytest + +from virsh_panel.fabric import ( + FABRIC_OPERATION_ROOT, + _TRANSACTION_SCRIPT, + _apply_script, + _fabric_operation_runner_script, + _fabric_operation_update_script, + parse_fabric_plan, +) + + +_SECRET = "NEVER_LOG_raw_private_fixture_or_sensitive_path" +_OPERATION = "a" * 32 +_DIGEST = "b" * 64 + + +def _write(path: Path, body: str, mode: int = 0o700): + path.write_text(body, encoding="utf-8") + path.chmod(mode) + + +def _render(manager="ifupdown", firewall="firewalld_nft"): + plan = parse_fabric_plan({ + "fabric_id": "safe-log-test", "region": "hk", + "overlay_cidr": "100.64.1.0/24", "revision": 1, + "nodes": [ + {"host_id": "a", "overlay_ip": "100.64.1.1", + "endpoint": "203.0.113.1:51820"}, + {"host_id": "b", "overlay_ip": "100.64.1.2", + "endpoint": "203.0.113.2:51820"}, + ], + }) + keys = {node.host_id: base64.b64encode(bytes([index]) * 32).decode() + for index, node in enumerate(plan.nodes, 1)} + return _apply_script(plan, plan.nodes[0], keys, + deployment_mode=manager, firewall_mode=firewall) + + +def _apply_logger(rendered): + start = rendered.index("vc_fabric_apply_failure_log() {") + return rendered[start:rendered.index("vc_fabric_operation_checkpoint() {", start)] + + +def _helper_entry(rendered): + # This is the helper actually embedded in apply, not a retyped logger. + start = rendered.index("#!/bin/sh\n# virt-cloud-wg-fabric-v1\n# fabric-plan-digest=") + end = rendered.index("BASE=/etc/virt-cloud\n", start) + prefix = rendered[start:end] + assert "__VC_FABRIC_DIGEST__" not in prefix + assert "trap cleanup EXIT HUP INT TERM\n" in prefix + return prefix + + +def _run(script, *, env=None, args=()): + return subprocess.run(["/bin/sh", "-eu", "-c", script, "safe-log-test", *args], + capture_output=True, text=True, check=False, + env={**os.environ, **(env or {})}) + + +def _journal_wrapper(body): + return ("exec 3>&1\nexec >/dev/null 2>&1\n" + "VC_FABRIC_SAFE_LOG=1\nexport VC_FABRIC_SAFE_LOG\n" + body) + + +@pytest.mark.parametrize("detail", [ + "entry", "final_acl", "final_tenants", "final_identity", "final_link", + "final_addresses", "final_peers", "final_endpoints", "final_allowed_ips", + "final_routes", "final_overlay_paths", "final_zone", "commit", "convergence", + "complete", +]) +def test_rendered_apply_logger_has_one_fixed_fd3_record(detail): + script = _journal_wrapper(_apply_logger(_render()) + ( + f"VC_FABRIC_APPLY_DETAIL={shlex.quote(detail)}\n" + f"printf '%s\\n' {shlex.quote(_SECRET)}\n" + f"printf '%s\\n' {shlex.quote(_SECRET)} >&2\n" + "vc_fabric_apply_failure_log 71\n" + )) + result = _run(script) + assert result.returncode == 0 + assert result.stdout == ( + f"virt-cloud-fabric apply_state=failed apply_check={detail} exit_status=71\n") + assert result.stderr == "" + + +@pytest.mark.parametrize("logger", ["apply", "helper"]) +@pytest.mark.parametrize("status", ["", "-1", "256", "9" * 100, _SECRET]) +def test_unknown_phase_and_invalid_status_are_sanitized(logger, status): + rendered = _render() + phase = "unknown\n" + _SECRET + "=/sensitive/path" + if logger == "apply": + body = (_apply_logger(rendered) + + f"VC_FABRIC_APPLY_DETAIL={shlex.quote(phase)}\n" + + f"vc_fabric_apply_failure_log {shlex.quote(status)}\n") + line = "virt-cloud-fabric apply_state=failed apply_check=unknown exit_status=1\n" + else: + body = (_helper_entry(rendered) + + f"VC_FABRIC_HELPER_PHASE={shlex.quote(phase)}\n" + + f"vc_fabric_helper_failure_log {shlex.quote(status)}\nexit 0\n") + line = "virt-cloud-fabric helper_state=failed helper_phase=unknown exit_status=1\n" + result = _run(_journal_wrapper(body)) + assert result.returncode == 0 + assert result.stdout == line + assert result.stderr == "" + assert _SECRET not in result.stdout + + +@pytest.mark.parametrize("status", [1, 70, 71, 79, 127, 143, 255]) +def test_rendered_child_helper_preserves_exit_status_without_duplicate_fd3_log(status): + body = (_helper_entry(_render()) + "VC_FABRIC_HELPER_PHASE=final\n" + + f"printf '%s\\n' {shlex.quote(_SECRET)}\n" + + f"printf '%s\\n' {shlex.quote(_SECRET)} >&2\n" + + f"exit {status}\n") + result = _run(_journal_wrapper(body)) + assert result.returncode == status + assert result.stdout == ( + f"virt-cloud-fabric helper_state=failed helper_phase=final exit_status={status}\n") + assert result.stderr == "" + + +def _runner_fixture(tmp_path: Path, apply_body: str): + directory = tmp_path / _OPERATION + directory.mkdir(mode=0o700) + stat_tool = tmp_path / "stat" + _write(stat_tool, f"#!{sys.executable}\n" + ( + "import os, stat, sys\n" + "assert sys.argv[1:3] == ['-c', '%u:%a']\n" + "print('0:' + format(stat.S_IMODE(os.stat(sys.argv[3]).st_mode), 'o'))\n" + )) + sync_tool = tmp_path / "sync" + _write(sync_tool, "#!/bin/sh\nexit 0\n") + updater = _fabric_operation_update_script(_OPERATION, _DIGEST).replace( + FABRIC_OPERATION_ROOT, str(tmp_path)).replace( + "stat -c", shlex.quote(str(stat_tool)) + " -c").replace( + "\nsync\n", "\n" + shlex.quote(str(sync_tool)) + "\n") + _write(directory / "update", updater) + _write(directory / "apply", "#!/bin/sh\n" + apply_body) + _write(directory / "manifest", ( + f"schema=1\noperation_id={_OPERATION}\nplan_digest={_DIGEST}\n"), 0o600) + now = int(time.time()) + _write(directory / "status", ( + f"schema=1\noperation_id={_OPERATION}\nstatus=queued\nphase=apply\n" + "checkpoint=dispatch\nreason=none\n" + f"created_at={now}\nstarted_at=0\nupdated_at={now}\nfinished_at=0\n"), 0o600) + runner = _fabric_operation_runner_script(_OPERATION, _DIGEST).replace( + FABRIC_OPERATION_ROOT, str(tmp_path)) + return runner, directory + + +@pytest.mark.parametrize("status", [0, 71, 79]) +def test_rendered_runner_propagates_safe_descriptor_and_never_raw_output(tmp_path, status): + helper = tmp_path / "child-helper" + _write(helper, _helper_entry(_render()) + ( + "VC_FABRIC_HELPER_PHASE=gate\n" + f"printf '%s\\n' {shlex.quote(_SECRET)}\n" + f"printf '%s\\n' {shlex.quote(_SECRET)} >&2\nexit {status}\n" + )) + body = (f"printf '%s\\n' {shlex.quote(_SECRET)}\n" + + f"printf '%s\\n' {shlex.quote(_SECRET)} >&2\n" + + f"exec /bin/sh {shlex.quote(str(helper))}\n") + runner, directory = _runner_fixture(tmp_path, body) + result = _run(runner) + assert result.returncode == (0 if status == 0 else 1) + assert result.stderr == "" + assert _SECRET not in result.stdout + assert str(tmp_path) not in result.stdout + lines = result.stdout.splitlines() + assert len(lines) == (2 if status == 0 else 3) + assert lines[0] == ( + f"virt-cloud-fabric operation={_OPERATION} state=running checkpoint=dispatch reason=none") + if status: + assert lines[1] == ( + f"virt-cloud-fabric helper_state=failed helper_phase=gate exit_status={status}") + state = (directory / "status").read_text() + assert ("status=succeeded\n" if status == 0 else "status=failed\n") in state + assert lines[-1].startswith(f"virt-cloud-fabric operation={_OPERATION} state=") + + +@pytest.mark.parametrize("status", [71, 79]) +def test_first_apply_failure_precedes_secondary_rollback_cleanup_record(tmp_path, status): + rendered = _render() + helper = tmp_path / "secondary-helper" + _write(helper, _helper_entry(rendered) + "VC_FABRIC_HELPER_PHASE=gate\nexit 79\n") + start = _TRANSACTION_SCRIPT.index("rollback() {") + end = _TRANSACTION_SCRIPT.index("\ntrap rollback EXIT HUP INT TERM", start) + rollback = _TRANSACTION_SCRIPT[start:end] + body = ("COMMITTED=yes\n" + _apply_logger(rendered) + rollback + ( + "\ncleanup() {\n" + f" printf '%s\\n' {shlex.quote(_SECRET)} >&2\n" + f" /bin/sh {shlex.quote(str(helper))} >/dev/null 2>&1 || true\n" + "}\ntrap rollback EXIT HUP INT TERM\n" + f"VC_FABRIC_APPLY_DETAIL=final_routes\nexit {status}\n" + )) + result = _run(_journal_wrapper(body)) + assert result.returncode == status + assert result.stderr == "" + assert result.stdout.splitlines() == [ + f"virt-cloud-fabric apply_state=failed apply_check=final_routes exit_status={status}", + "virt-cloud-fabric helper_state=failed helper_phase=gate exit_status=79", + ] + + +@pytest.mark.parametrize("manager", ["ifupdown", "networkmanager_nmcli"]) +@pytest.mark.parametrize("firewall", ["native_nftables", "firewalld_nft"]) +def test_rendered_timer_start_strictly_follows_durable_commit(manager, firewall): + rendered = _render(manager, firewall) + timer_command = '"$SYSTEMCTL" start virt-cloud-wg-fabric-reconcile.timer >/dev/null' + assert rendered.count(timer_command) == 1 + commit_marker = rendered.rindex('committed > "$_txn_state_tmp"') + publish = rendered.index('mv -f "$_txn_state_tmp" "$TXN/state"', commit_marker) + durable_sync = rendered.index("\nsync\n", publish) + committed_flag = rendered.index("COMMITTED=yes", durable_sync) + timer_start = rendered.index(timer_command) + unlock = rendered.index("exec 9>&-", timer_start) + assert commit_marker < publish < durable_sync < committed_flag < timer_start < unlock + + +def test_success_and_disabled_safe_logging_do_not_invent_failure_records(): + logger = _apply_logger(_render()) + result = _run(_journal_wrapper(logger + "VC_FABRIC_APPLY_DETAIL=complete\n" + "vc_fabric_apply_failure_log 0\n")) + assert result.returncode == 0 + assert result.stdout == result.stderr == "" + result = _run(_journal_wrapper(logger + "VC_FABRIC_SAFE_LOG=0\n" + "vc_fabric_apply_failure_log 71\n")) + assert result.returncode == 0 + assert result.stdout == result.stderr == "" diff --git a/engine/tests/test_fabric_ready_current_recovery.py b/engine/tests/test_fabric_ready_current_recovery.py new file mode 100644 index 00000000..89ede63e --- /dev/null +++ b/engine/tests/test_fabric_ready_current_recovery.py @@ -0,0 +1,272 @@ +"""Recover a same-plan ready journal without redispatching its old helper.""" +from __future__ import annotations + +import base64 +import json +import re +import shlex +import subprocess + +import pytest + +from test_fabric_first_deploy_entry import _write +from test_fabric_nm_runtime_shadow_recovery import _failed_first_deploy +from virsh_panel.fabric import ( + FIREWALL_MODE_FIREWALLD, + FIREWALL_MODE_NATIVE, + _FABRIC_NFT_VALIDATION_SH, + _FABRIC_SYSCTL_VALIDATION_SH, + _FIREWALL_MODE_GUARD_SH, + _NMCLI_FILENAME_RESOLVER_SH, + _apply_script, +) + + +def _baseline_fixture(tmp_path): + values = _failed_first_deploy(tmp_path) + _, _, installed, _, _, _, journal, _ = values + # The real Alma host now has a journaled old helper baseline, not the + # helper.absent case covered by the first-deployment recovery path. + (journal / "helper.absent").unlink() + _write(journal / "helper.present", "") + _write(journal / "helper", installed.read_text(encoding="utf-8"), 0o700) + return values + + +def _rendered_driver(values, root, firewall_mode=FIREWALL_MODE_FIREWALLD): + plan, current, installed, base, runtime, roots, _, _ = values + keys = {host.host_id: base64.b64encode(bytes([index]) * 32).decode() + for index, host in enumerate(plan.nodes, start=1)} + rendered = _apply_script( + plan, plan.nodes[0], keys, deployment_mode="networkmanager_nmcli", + firewall_mode=firewall_mode, + ) + start = rendered.index("COMMITTED=no\nROLLBACK_FAILED=no") + end = rendered.index("vc_fabric_operation_checkpoint preflight\n", start) + # Execute the actual rendered outer recovery functions and dispatch, then + # the complete current helper. Only host resources and CLI paths redirect + # to fixture doubles; no ownership/recovery shell function is replaced. + recovery = rendered[start:end] + body = (_NMCLI_FILENAME_RESOLVER_SH + "\n" + _FABRIC_NFT_VALIDATION_SH + "\n" + + _FABRIC_SYSCTL_VALIDATION_SH + "\n" + _FIREWALL_MODE_GUARD_SH + "\n" + + recovery) + for source, destination in roots.items(): + body = body.replace(source, str(destination)) + body = body.replace("in /run ", "in " + shlex.quote(str(root / "run")) + " ") + tools = root / "tools" + identity = re.search(r'^CONNECTION_UUID="([^"]+)"$', current.read_text(), re.MULTILINE)[1] + variables = { + "BASE": base, "HELPER": installed, "_tmp_helper": current, + "OWNER": base / "wg-fabric0.owner", "CONF": base / "wg-fabric0.conf", + "NM_PROFILE": roots["/etc/NetworkManager"] / "system-connections" / + "virt-cloud-wg-fabric0.nmconnection", + "EMERGENCY_OWNER": runtime / "wg-fabric-emergency.owner", + "EXPECTED_DIGEST": plan.digest, "DEPLOYMENT_MODE": "networkmanager_nmcli", + "FIREWALL_MODE": firewall_mode, "IFACE": "wg-fabric0", + "CONNECTION": "virt-cloud-wg-fabric0", "CONNECTION_UUID": identity, + "CMP": "/usr/bin/cmp", + } + for variable, name in (("STAT", "stat"), ("NFT", "nft"), ("IP", "ip"), + ("WG", "wg"), ("SYSTEMCTL", "systemctl"), + ("NMCLI", "nmcli"), ("FIREWALL_CMD", "firewall-cmd")): + variables[variable] = tools / name + prefix = "\n".join(key + "=" + shlex.quote(str(value)) for key, value in variables.items()) + prefix += ("\nfind_tool() { printf '%s' " + shlex.quote(str(tools / "sha256sum")) + "; }\n" + "vc_fabric_operation_checkpoint() { :; }\n" + "PATH=" + shlex.quote(str(tools)) + ":/usr/bin:/bin; export PATH\n") + return "set -eu\n" + prefix + body + + +def _run(values, root, firewall_mode=FIREWALL_MODE_FIREWALLD): + return subprocess.run( + ["/bin/sh", "-c", _rendered_driver(values, root, firewall_mode)], check=False, + capture_output=True, text=True, + env={"VC_ENTRY_ROOT": str(root), "VC_FABRIC_LOCK_HELD": "1"}, timeout=45, + ) + + +def _native_baseline_fixture(tmp_path, *, empty_header=False): + values = _baseline_fixture(tmp_path) + _, current, _, _, _, roots, journal, shadow = values + # Rendered helper implementation is identical for both owners; the + # explicit literal backend selects the native branch. No shell function + # or recovery conditional is replaced in this fixture. + content = current.read_text().replace( + 'FIREWALL_MODE="firewalld_nft"', 'FIREWALL_MODE="native_nftables"', 1, + ) + native_main = tmp_path / "nftables.conf" + roots["/etc/nftables.conf"] = native_main + content = content.replace("/etc/nftables.conf", str(native_main)) + if empty_header: + content = content.replace('UNDERLAY_ENDPOINT="203.0.113.1"', 'UNDERLAY_ENDPOINT=""') + content = content.replace('UNDERLAY_PORT="51820"', 'UNDERLAY_PORT=""') + _write(current, content, 0o700) + metadata = dict(line.split("=", 1) for line in (journal / "meta").read_text().splitlines()) + metadata.update({ + "firewall_mode": FIREWALL_MODE_NATIVE, "nft_main": str(native_main), + "firewalld_runtime_zone_before": "na", "firewalld_permanent_zone_before": "na", + "underlay_endpoint": "", "underlay_port": "", "underlay_zone": "", + "underlay_rule_owner": "none", "underlay_rule_runtime_before": "na", + "underlay_rule_permanent_before": "na", + }) + _write(journal / "meta", "".join(f"{key}={value}\n" for key, value in metadata.items())) + distribution_rules = '# distribution-owned main file\ninclude "/etc/nftables.d/*.nft"\n' + _write(native_main, distribution_rules, 0o644) + (journal / "nft_main.absent").unlink() + _write(journal / "nft_main.present", "") + _write(journal / "nft_main", distribution_rules, 0o644) + _write(shadow, shadow.read_text().replace("zone=trusted", "zone=")) + _write(tmp_path / "nm-zone", "") + (tmp_path / "runtime-zone").unlink() + # Native owner fixture: nftables enabled/running, firewalld masked and + # inactive. Only external systemctl evidence differs from firewalld. + tool = tmp_path / "tools" / "external-tool" + external = tool.read_text().replace( + 'print("active" if args[1] == "firewalld" else "inactive")', + 'print("inactive" if args[1] == "firewalld" else "active")', + ).replace( + 'print("enabled" if args[1] == "firewalld" else "disabled")', + 'print("masked" if args[1] == "firewalld" else "enabled")', + ) + _write(tool, external, 0o700) + return values + + +@pytest.mark.parametrize("empty_header", [False, True]) +@pytest.mark.parametrize("failure", [None, "endpoint", "port"]) +def test_native_same_plan_handoff_requires_empty_underlay_metadata_not_unused_header( + tmp_path, empty_header, failure): + values = _native_baseline_fixture(tmp_path, empty_header=empty_header) + _, _, installed, _, runtime, _, journal, shadow = values + historical = installed.read_bytes() + if failure: + _write(journal / "meta", (journal / "meta").read_text().replace( + f"underlay_{failure}=\n", + f"underlay_{failure}=" + ("203.0.113.1" if failure == "endpoint" else "51820") + "\n", + 1, + )) + live = tmp_path / "tables" / "inet--virt_cloud_wg_fabric" + original = live.read_bytes() + result = _run(values, tmp_path, FIREWALL_MODE_NATIVE) + assert result.returncode == (0 if failure is None else 79), result.stderr + assert "LEGACY_HELPER_EXECUTED" not in result.stdout + result.stderr + assert installed.read_bytes() == historical + assert (runtime / "wg-fabric-emergency.owner").is_file() + if failure is None: + assert not journal.exists() + assert not live.exists() + assert not shadow.exists() + assert not (tmp_path / "link-present").exists() + else: + assert journal.is_dir() + assert live.read_bytes() == original + assert shadow.is_file() + + +def test_rendered_outer_replays_same_plan_old_baseline_with_current_helper(tmp_path): + values = _baseline_fixture(tmp_path) + _, current, installed, base, runtime, _, journal, shadow = values + historical = installed.read_bytes() + assert current.read_bytes() != historical + assert "vc_fabric_nm_profile_semantics_match" not in historical.decode() + result = _run(values, tmp_path) + assert result.returncode == 0, result.stderr + assert "LEGACY_HELPER_EXECUTED" not in result.stdout + result.stderr + assert not journal.exists() + # Complete recovery preserves its pre-existing, precisely journaled old + # helper but must not execute it again under --recover-current. + assert installed.read_bytes() == historical + assert not shadow.exists() + assert not (tmp_path / "link-present").exists() + assert not (base / "wg-fabric0.owner").exists() + assert not (tmp_path / "tables" / "inet--virt_cloud_wg_fabric").exists() + assert (runtime / "wg-fabric-emergency.owner").is_file() + for family in ("inet", "bridge"): + assert (tmp_path / "tables" / f"{family}--virt_cloud_fabric_emergency").is_file() + calls = [json.loads(line) for line in (tmp_path / "calls").read_text().splitlines()] + assert not any(call[0] == "systemctl" and call[1] in ("start", "enable") for call in calls) + assert not any(call[:5] == ["ip", "link", "set", "dev", "wg-fabric0"] + and call[-1] == "up" for call in calls) + + +def test_normal_boot_recovery_still_reenters_verified_historical_helper(tmp_path): + values = _baseline_fixture(tmp_path) + _, current, installed, _, runtime, _, journal, _ = values + historical = installed.read_bytes() + result = subprocess.run( + ["/bin/sh", str(current)], check=False, capture_output=True, text=True, + env={"VC_ENTRY_ROOT": str(tmp_path), "VC_FABRIC_LOCK_HELD": "1"}, timeout=45, + ) + # This complete non-recovery-only entry still uses the restored old + # implementation. Its deliberate exit99 proves the normal exec contract + # was preserved instead of silently treating a regular boot as finished. + assert result.returncode == 99, result.stderr + assert "LEGACY_HELPER_EXECUTED" in result.stderr + assert not journal.exists() + assert installed.read_bytes() == historical + assert (runtime / "wg-fabric-emergency.owner").is_file() + + +@pytest.mark.parametrize("failure", [ + "foreign_helper", "baseline_and_desired_drift", "temporary_mode", "temporary_digest", + "endpoint", "port", "runtime_key", "live_nft_verdict", +]) +def test_rendered_same_plan_handoff_rejects_unproved_ownership_or_plan(tmp_path, failure): + values = _baseline_fixture(tmp_path) + plan, current, installed, _, runtime, _, journal, shadow = values + if failure == "foreign_helper": + _write(installed, installed.read_text() + "# external replacement\n", 0o700) + elif failure == "baseline_and_desired_drift": + for name in ("helper", "helper.desired"): + _write(journal / name, (journal / name).read_text() + "# changed snapshot\n", 0o700) + elif failure == "temporary_mode": + current.chmod(0o600) + elif failure == "temporary_digest": + _write(current, current.read_text().replace(plan.digest, "a" * 64), 0o700) + elif failure in ("endpoint", "port"): + old, new = (("underlay_endpoint=203.0.113.1", "underlay_endpoint=203.0.113.9") + if failure == "endpoint" else ("underlay_port=51820", "underlay_port=51821")) + _write(journal / "meta", (journal / "meta").read_text().replace(old, new)) + elif failure == "runtime_key": + old = base64.b64encode(bytes([7]) * 32).decode() + _write(shadow, shadow.read_text().replace(old, base64.b64encode(bytes([8]) * 32).decode())) + else: + live = tmp_path / "tables" / "inet--virt_cloud_wg_fabric" + _write(live, live.read_text().replace("counter drop comment", "counter accept comment", 1)) + live = tmp_path / "tables" / "inet--virt_cloud_wg_fabric" + original = live.read_bytes() + result = _run(values, tmp_path) + assert result.returncode == 79, result.stderr + assert "LEGACY_HELPER_EXECUTED" not in result.stdout + result.stderr + assert journal.is_dir() + assert installed.is_file() + assert live.read_bytes() == original + assert (runtime / "wg-fabric-emergency.owner").is_file() + calls = [json.loads(line) for line in (tmp_path / "calls").read_text().splitlines()] + assert not any(call[0] == "nft" and call[1] == "-f" for call in calls) + + +@pytest.mark.parametrize("field,value", [ + ("plan_digest", "f" * 64), + ("connection_uuid", "ffffffff-ffff-ffff-ffff-ffffffffffff"), + ("deployment_mode", "ifupdown"), + ("firewall_mode", "native_nftables"), +]) +def test_different_plan_keeps_historical_recovery_contract_without_current_handoff( + tmp_path, field, value): + values = _baseline_fixture(tmp_path) + _, _, _, _, runtime, _, journal, _ = values + text = (journal / "meta").read_text() + _write(journal / "meta", re.sub(rf"^{field}=.*$", f"{field}={value}", text, + flags=re.MULTILINE)) + before = (tmp_path / "tables" / "inet--virt_cloud_wg_fabric").read_bytes() + result = _run(values, tmp_path) + assert result.returncode == 79, result.stderr + # This controlled historical helper rejects at exit99. Existing cross- + # revision dispatch stays intact, and the current template never assumes + # authority over the other journal identity. + assert "LEGACY_HELPER_EXECUTED" in result.stderr + assert "helper_state=failed" not in result.stderr + assert journal.is_dir() + assert (tmp_path / "tables" / "inet--virt_cloud_wg_fabric").read_bytes() == before + assert (runtime / "wg-fabric-emergency.owner").is_file() diff --git a/engine/virsh_panel/fabric.py b/engine/virsh_panel/fabric.py index 5cbcaa4d..3dd56068 100644 --- a/engine/virsh_panel/fabric.py +++ b/engine/virsh_panel/fabric.py @@ -1500,6 +1500,7 @@ operation_interrupted() {{ trap operation_interrupted HUP INT TERM /bin/sh "$UPDATE" running dispatch none || exit 1 export VC_FABRIC_OPERATION_ID="$OPERATION_ID" +export VC_FABRIC_SAFE_LOG=1 if /bin/sh "$APPLY"; then /bin/sh "$UPDATE" succeeded complete none || exit 1 exit 0 @@ -1962,6 +1963,104 @@ table inet {FABRIC_NFT_TABLE} {{ ''' +_FABRIC_WG_READBACK_SH = r'''vc_fabric_wg_peers_exact() { + vc_wg_live_peers=$1 + vc_wg_expected_peers=$2 + vc_wg_live_size=$(printf '%s' "$vc_wg_live_peers" | wc -c | tr -d '[:space:]') || return 1 + [ "$vc_wg_live_size" -le 262144 ] 2>/dev/null || return 1 + # wg(8) emits one public key per line. Compare complete records as an + # unordered, duplicate-free set; a space-delimited shell pattern cannot + # recognize keys separated by the command's internal newlines. + printf '%s\n' "$vc_wg_live_peers" | awk -v expected="$vc_wg_expected_peers" ' + function key(value) { + return length(value)==44 && value ~ /^[A-Za-z0-9+\/]+=$/ + } + BEGIN { + n=split(expected, values, /[[:space:]]+/) + for (i=1; i<=n; i++) { + if (values[i]=="") continue + if (!key(values[i]) || wanted[values[i]]++) bad=1 + expected_count++ + } + if (expected_count<1 || expected_count>63) bad=1 + } + NF { + if (NF!=1 || !key($1) || !wanted[$1] || seen[$1]++) bad=1 + actual_count++ + } + END {exit (bad || actual_count!=expected_count) ? 1 : 0} + ' +} +vc_fabric_wg_allowed_exact() { + vc_wg_live_allowed=$1 + vc_wg_wanted_key=$2 + vc_wg_expected_allowed=$3 + vc_wg_overlay_allowed=$4 + vc_wg_expected_keys=$5 + vc_wg_live_size=$(printf '%s' "$vc_wg_live_allowed" | wc -c | tr -d '[:space:]') || return 1 + [ "$vc_wg_live_size" -le 262144 ] 2>/dev/null || return 1 + # The machine-readable `wg show IFACE allowed-ips` uses spaces between + # prefixes (unlike the comma-separated set-command input). Validate the + # full peer table and every prefix of the selected row, not merely $2. + printf '%s\n' "$vc_wg_live_allowed" | awk \ + -v selected="$vc_wg_wanted_key" -v expected="$vc_wg_expected_allowed" \ + -v overlay="$vc_wg_overlay_allowed" -v peers="$vc_wg_expected_keys" ' + function key(value) { + return length(value)==44 && value ~ /^[A-Za-z0-9+\/]+=$/ + } + function host_prefix(value, parts, octets, count, pos) { + count=split(value, parts, "/") + if (count!=2 || parts[2]!="32" || split(parts[1], octets, ".")!=4) + return 0 + for (pos=1; pos<=4; pos++) + if (octets[pos] !~ /^(0|[1-9][0-9]*)$/ || octets[pos]>255) return 0 + return 1 + } + function allowed_prefix(value, parts, octets) { + if (!host_prefix(value)) return 0 + if (value==overlay) return 1 + split(value, parts, "/"); split(parts[1], octets, ".") + return (octets[1]==10 || + (octets[1]==172 && octets[2]>=16 && octets[2]<=31) || + (octets[1]==192 && octets[2]==168)) + } + BEGIN { + n=split(peers, peer_values, /[[:space:]]+/) + for (i=1; i<=n; i++) { + if (peer_values[i]=="") continue + if (!key(peer_values[i]) || peer_wanted[peer_values[i]]++) bad=1 + peer_count++ + } + if (peer_count<1 || peer_count>63 || !peer_wanted[selected] || + !host_prefix(overlay)) bad=1 + split(overlay, overlay_parts, "/"); split(overlay_parts[1], overlay_octets, ".") + if (overlay_octets[1]!=100 || overlay_octets[2]<64 || overlay_octets[2]>127) bad=1 + n=split(expected, prefix_values, ",") + for (i=1; i<=n; i++) { + value=prefix_values[i] + if (!allowed_prefix(value) || prefix_wanted[value]++) bad=1 + expected_count++ + } + if (expected_count<1 || expected_count>16384 || prefix_wanted[overlay]!=1) bad=1 + } + NF { + if (NF<2 || !key($1) || !peer_wanted[$1] || peer_seen[$1]++) bad=1 + actual_peers++ + for (i=2; i<=NF; i++) { + if (!host_prefix($i) || prefix_seen[$1, $i]++) bad=1 + if ($1==selected) { + if (!allowed_prefix($i) || !prefix_wanted[$i]) bad=1 + actual_count++ + } + } + } + END {exit (bad || actual_peers!=peer_count || peer_seen[selected]!=1 || + actual_count!=expected_count) ? 1 : 0} + ' +} +'''.strip() + + _FABRIC_ROUTE_VALIDATION_AWK = r''' function canonical_host_prefix(value, octet, count, pos) { # iproute2 omits /32 for host routes. Canonicalize only an exact IPv4 @@ -3545,6 +3644,13 @@ vc_fabric_helper_failure_log() { esac printf 'virt-cloud-fabric helper_state=failed helper_phase=%s exit_status=%s\n' \ "$vc_helper_log_phase" "$vc_helper_log_status" >&2 + # The async runner suppresses arbitrary child stdout/stderr because they + # may contain credentials. Only this closed, sanitized record may use its + # inherited journal descriptor; never redirect raw helper output there. + if [ "${VC_FABRIC_SAFE_LOG:-0}" = 1 ]; then + printf 'virt-cloud-fabric helper_state=failed helper_phase=%s exit_status=%s\n' \ + "$vc_helper_log_phase" "$vc_helper_log_status" >&3 2>/dev/null || true + fi } cleanup() { vc_helper_exit_status=$? @@ -4028,6 +4134,7 @@ __VC_NMCLI_FILENAME_RESOLVER__ __VC_FIREWALL_MODE_GUARD__ __VC_NFT_MANIFEST_VALIDATION__ __VC_FABRIC_NFT_VALIDATION__ +__VC_FABRIC_WG_READBACK__ __VC_VPC_STATE_LOADER__ NMCLI= IFQUERY= @@ -5515,6 +5622,13 @@ recover_journal() { exit 0 fi rm -rf "$JOURNAL" || exit 79 + # Recovery-only has now restored and verified the complete journaled + # baseline. A current temporary helper must not redispatch the obsolete + # baseline implementation whose compatibility defect prompted recovery. + # Ordinary boot recovery still re-enters the restored helper as before. + if [ "$RECOVER_ONLY" = yes ]; then + exit 0 + fi exec /bin/sh "$HELPER" "$@" } if [ "${VC_FABRIC_APPLYING:-0}" != 1 ]; then @@ -6697,12 +6811,16 @@ VC_FABRIC_HELPER_PHASE=vpc recover_vpc_dataplanes VC_FABRIC_HELPER_PHASE=final actual_peers=$("$WG" show "$IFACE" peers 2>/dev/null) || exit 78 -for actual_peer in $actual_peers; do - case " $seen_peers " in *" $actual_peer "*) : ;; *) exit 78 ;; esac -done -for expected_peer in $seen_peers; do - case " $actual_peers " in *" $expected_peer "*) : ;; *) exit 78 ;; esac -done +vc_fabric_wg_peers_exact "$actual_peers" "$seen_peers" || exit 78 +actual_allowed_ips=$("$WG" show "$IFACE" allowed-ips 2>/dev/null) || exit 78 +while IFS='|' read -r final_kind final_host final_peer final_endpoint final_allowed; do + [ "$final_kind" = peer ] || continue + final_vpc_extra=$(vpc_state_allowed_for_peer "$final_peer") || exit 78 + final_expected_allowed=$final_allowed + [ -z "$final_vpc_extra" ] || final_expected_allowed="$final_expected_allowed,$final_vpc_extra" + vc_fabric_wg_allowed_exact "$actual_allowed_ips" "$final_peer" \ + "$final_expected_allowed" "$final_allowed" "$seen_peers" || exit 78 +done < "$CONF" actual_fabric_routes=$("$IP" -o route show table all dev "$IFACE" 2>/dev/null) || exit 78 route_extra=$(printf '%s\n' "$actual_fabric_routes" | awk -v allowed="$allowed_routes" -v own="$overlay" -v vpc_file="$VPC_ROUTE_FILE" ' __VC_FABRIC_ROUTE_VALIDATION_AWK__ @@ -7390,6 +7508,57 @@ transaction_recovery_helper_owned() { fi return 1 } +transaction_recovery_same_current_plan() { + # A historical helper is authoritative only for the journaled pre/post + # image proved above. Replaying that exact plan with today's temporary + # helper is safe, and avoids redispatching an obsolete implementation + # which cannot understand its own NetworkManager in-memory shadow. A + # different revision still uses the existing historical-helper contract; + # never transfer its authority to this apply's new plan. + transaction_current_meta="$BASE/.wg-fabric-txn/meta" + [ -f "$transaction_current_meta" ] && + [ ! -L "$transaction_current_meta" ] && + [ "$($STAT -c '%u:%a' "$transaction_current_meta" 2>/dev/null)" = 0:600 ] || return 1 + awk -F= -v wanted_digest="$EXPECTED_DIGEST" \ + -v wanted_uuid="$CONNECTION_UUID" -v wanted_mode="$DEPLOYMENT_MODE" \ + -v wanted_firewall="$FIREWALL_MODE" ' + $1=="schema" {schema++; if (substr($0,index($0,"=")+1)!="1") bad=1; next} + $1=="plan_digest" {digest++; if (substr($0,index($0,"=")+1)!=wanted_digest) bad=1; next} + $1=="connection_uuid" {uuid++; if (substr($0,index($0,"=")+1)!=wanted_uuid) bad=1; next} + $1=="deployment_mode" {mode++; if (substr($0,index($0,"=")+1)!=wanted_mode) bad=1; next} + $1=="firewall_mode" {firewall++; if (substr($0,index($0,"=")+1)!=wanted_firewall) bad=1; next} + $1=="underlay_endpoint" {endpoint++; next} + $1=="underlay_port" {port++; next} + END { + exit (bad || schema!=1 || digest!=1 || uuid!=1 || mode!=1 || + firewall!=1 || endpoint!=1 || port!=1) ? 1 : 0 + } + ' "$transaction_current_meta" >/dev/null +} +transaction_recovery_current_underlay_matches() { + # These public constants are already rendered into the guarded current + # helper. The outer apply has not yet reached underlay preflight, so do + # not read its uninitialized mutable UNDERLAY_* variables or eval files. + if [ "$FIREWALL_MODE" = native_nftables ]; then + transaction_current_endpoint= + transaction_current_port= + else + transaction_current_endpoint=$(awk -F'"' ' + /^UNDERLAY_ENDPOINT="[0-9.]+"$/ {value=$2; count++} + END {if (count==1) print value; else exit 1} + ' "$_tmp_helper") || return 1 + transaction_current_port=$(awk -F'"' ' + /^UNDERLAY_PORT="[0-9]+"$/ {value=$2; count++} + END {if (count==1) print value; else exit 1} + ' "$_tmp_helper") || return 1 + fi + awk -F= -v wanted_endpoint="$transaction_current_endpoint" \ + -v wanted_port="$transaction_current_port" ' + $1=="underlay_endpoint" {endpoint++; if (substr($0,index($0,"=")+1)!=wanted_endpoint) bad=1} + $1=="underlay_port" {port++; if (substr($0,index($0,"=")+1)!=wanted_port) bad=1} + END {exit (bad || endpoint!=1 || port!=1) ? 1 : 0} + ' "$transaction_current_meta" >/dev/null +} if [ -L "$BASE" ] || { [ -e "$BASE" ] && [ ! -d "$BASE" ]; }; then exit 71; fi TRANSACTION_RECOVERED=no if [ -e "$BASE/.wg-fabric-txn" ]; then @@ -7502,9 +7671,28 @@ if [ -e "$BASE/.wg-fabric-txn" ]; then fi else transaction_recovery_helper_owned || exit 79 - grep -Fq -- '--recover-only' "$HELPER" || exit 79 - vc_fabric_operation_checkpoint journal_dispatch - VC_FABRIC_LOCK_HELD=1 /bin/sh "$HELPER" --recover-only || exit 79 + if transaction_recovery_same_current_plan; then + # Keep the exact historical helper/journal ownership + # proof, but run the current implementation only after + # all plan and manager/firewall identity fields match. + # The child repeats the full journal, artifact, nft and + # live ownership validation before restoring anything. + [ -n "${_tmp_helper:-}" ] && + [ -f "$_tmp_helper" ] && [ ! -L "$_tmp_helper" ] && + [ "$($STAT -c '%u:%a' "$_tmp_helper" 2>/dev/null)" = 0:700 ] || exit 79 + [ "$(sed -n '1p' "$_tmp_helper")" = '#!/bin/sh' ] && + [ "$(sed -n '2p' "$_tmp_helper")" = '# virt-cloud-wg-fabric-v1' ] && + [ "$(sed -n '3p' "$_tmp_helper")" = "# fabric-plan-digest=$EXPECTED_DIGEST" ] && + [ "$(sed -n '4p' "$_tmp_helper")" = "PLAN_DIGEST=\"$EXPECTED_DIGEST\"" ] || exit 79 + grep -Fq -- '--recover-current' "$_tmp_helper" || exit 79 + transaction_recovery_current_underlay_matches || exit 79 + vc_fabric_operation_checkpoint journal_dispatch + VC_FABRIC_LOCK_HELD=1 /bin/sh "$_tmp_helper" --recover-current || exit 79 + else + grep -Fq -- '--recover-only' "$HELPER" || exit 79 + vc_fabric_operation_checkpoint journal_dispatch + VC_FABRIC_LOCK_HELD=1 /bin/sh "$HELPER" --recover-only || exit 79 + fi fi [ ! -e "$BASE/.wg-fabric-txn" ] || exit 79 TRANSACTION_RECOVERED=yes @@ -7701,6 +7889,7 @@ sync JOURNAL_READY=yes rollback() { rollback_status=$? + vc_fabric_apply_failure_log "$rollback_status" || true trap - EXIT HUP INT TERM set +e set +u @@ -8429,6 +8618,7 @@ def _apply_script(plan: FabricPlan, host: FabricNode, "__VC_FIREWALL_MODE_GUARD__", _FIREWALL_MODE_GUARD_SH).replace( "__VC_NFT_MANIFEST_VALIDATION__", _NFT_MANIFEST_VALIDATION_SH).replace( "__VC_FABRIC_NFT_VALIDATION__", _FABRIC_NFT_VALIDATION_SH).replace( + "__VC_FABRIC_WG_READBACK__", _FABRIC_WG_READBACK_SH).replace( "__VC_FABRIC_SYSCTL_VALIDATION__", _FABRIC_SYSCTL_VALIDATION_SH) manager_exec = ( "/bin/sh /usr/local/libexec/virt-cloud-wg-fabric --ifupdown-service" @@ -8752,6 +8942,24 @@ set -eu umask 077 PATH=/usr/sbin:/usr/bin:/sbin:/bin export PATH +VC_FABRIC_APPLY_DETAIL=entry +vc_fabric_apply_failure_log() {{ + vc_apply_log_status=$1 + case "$vc_apply_log_status" in ''|*[!0-9]*) vc_apply_log_status=1 ;; esac + [ "$vc_apply_log_status" -le 255 ] 2>/dev/null || vc_apply_log_status=1 + [ "$vc_apply_log_status" != 0 ] || return 0 + case "${{VC_FABRIC_APPLY_DETAIL:-}}" in + entry|final_acl|final_tenants|final_identity|final_link|final_addresses|final_peers|final_endpoints|final_allowed_ips|final_routes|final_overlay_paths|final_zone|commit|convergence|complete) + vc_apply_log_detail=$VC_FABRIC_APPLY_DETAIL ;; + *) vc_apply_log_detail=unknown ;; + esac + # Emit before rollback so a secondary helper/service failure cannot hide + # the first failing check. FD 3 belongs to the async runner, not a raw log. + [ "${{VC_FABRIC_SAFE_LOG:-0}}" = 1 ] || return 0 + printf 'virt-cloud-fabric apply_state=failed apply_check=%s exit_status=%s\\n' \ + "$vc_apply_log_detail" "$vc_apply_log_status" >&3 2>/dev/null || true + return 0 +}} vc_fabric_operation_checkpoint() {{ vc_operation_checkpoint=$1 [ -n "${{VC_FABRIC_OPERATION_ID:-}}" ] || return 0 @@ -8834,6 +9042,7 @@ FLOCK=$(find_tool /usr/bin/flock /usr/sbin/flock /bin/flock /sbin/flock) || exit {_FIREWALL_MODE_GUARD_SH} {_NFT_MANIFEST_VALIDATION_SH} {_FABRIC_NFT_VALIDATION_SH} +{_FABRIC_WG_READBACK_SH} {_FABRIC_SYSCTL_VALIDATION_SH} PREFLIGHT_IFACE_BEFORE=no PREFLIGHT_IFACE_UP_BEFORE=no @@ -9350,52 +9559,54 @@ for gate_unit in {' '.join(FABRIC_LIBVIRT_GATE_UNITS)}; do done "$SYSTEMCTL" enable {FABRIC_SERVICE} >/dev/null "$SYSTEMCTL" enable {FABRIC_TIMER} >/dev/null -"$SYSTEMCTL" start {FABRIC_TIMER} >/dev/null vc_fabric_operation_checkpoint final_readback if [ "$FIREWALL_MODE" = firewalld_nft ] || [ -n "${{VC_FABRIC_EMERGENCY_SNAPSHOT:-}}" ]; then + VC_FABRIC_APPLY_DETAIL=final_acl final_fabric_acl=$($NFT -s list table inet virt_cloud_wg_fabric 2>/dev/null) || exit 72 nft_plan_owned "$final_fabric_acl" "$EXPECTED_DIGEST" || exit 71 + VC_FABRIC_APPLY_DETAIL=final_tenants vpc_state_nft_set_matches || exit 71 + VC_FABRIC_APPLY_DETAIL=final_identity "$WG" show "$IFACE" >/dev/null 2>&1 || exit 71 final_public_key=$($WG pubkey < "$KEY") || exit 71 [ "$($WG show "$IFACE" public-key 2>/dev/null)" = "$final_public_key" ] || exit 71 [ "$($WG show "$IFACE" listen-port 2>/dev/null)" = {_q(str(host.listen_port))} ] || exit 71 + VC_FABRIC_APPLY_DETAIL=final_link final_link=$($IP -o link show dev "$IFACE" 2>/dev/null) || exit 71 printf '%s\n' "$final_link" | awk -F'[<>]' 'NF >= 3 {{ count=split($2, flags, ",") for (i=1; i<=count; i++) if (flags[i] == "UP") found=1 }} END {{exit found ? 0 : 1}}' || exit 71 + VC_FABRIC_APPLY_DETAIL=final_addresses final_addresses=$($IP -o -4 address show dev "$IFACE" 2>/dev/null) || exit 71 printf '%s\n' "$final_addresses" | awk -v wanted={_q(host.overlay_ip)} ' NF {{split($4, address, "/"); count++; if (address[1] != wanted || address[2] != 32) bad=1}} END {{exit (!bad && count == 1) ? 0 : 1}} ' || exit 71 + VC_FABRIC_APPLY_DETAIL=final_peers final_actual_peers=$($WG show "$IFACE" peers 2>/dev/null) || exit 71 - for final_actual_peer in $final_actual_peers; do - case " $MANAGED_PEERS " in *" $final_actual_peer "*) : ;; *) exit 71 ;; esac - done - for final_expected_peer in $MANAGED_PEERS; do - case " $final_actual_peers " in *" $final_expected_peer "*) : ;; *) exit 71 ;; esac - done + vc_fabric_wg_peers_exact "$final_actual_peers" "$MANAGED_PEERS" || exit 71 + VC_FABRIC_APPLY_DETAIL=final_endpoints final_endpoints=$($WG show "$IFACE" endpoints 2>/dev/null) || exit 71 + VC_FABRIC_APPLY_DETAIL=final_allowed_ips final_allowed_ips=$($WG show "$IFACE" allowed-ips 2>/dev/null) || exit 71 while IFS='|' read -r final_kind final_host final_peer final_endpoint final_allowed; do [ "$final_kind" = peer ] || continue final_vpc_extra=$(vpc_state_allowed_for_peer "$final_peer") || exit 71 final_expected_allowed=$final_allowed [ -z "$final_vpc_extra" ] || final_expected_allowed="$final_expected_allowed,$final_vpc_extra" + VC_FABRIC_APPLY_DETAIL=final_endpoints final_live_endpoint=$(printf '%s\n' "$final_endpoints" | awk -v wanted="$final_peer" ' $1 == wanted {{value=$2; count++}} END {{if (count == 1) print value; else exit 1}} ') || exit 71 [ "$final_live_endpoint" = "$final_endpoint" ] || exit 71 - final_live_allowed=$(printf '%s\n' "$final_allowed_ips" | awk -v wanted="$final_peer" ' - $1 == wanted {{value=$2; count++}} - END {{if (count == 1) print value; else exit 1}} - ') || exit 71 - [ "$final_live_allowed" = "$final_expected_allowed" ] || exit 71 + VC_FABRIC_APPLY_DETAIL=final_allowed_ips + vc_fabric_wg_allowed_exact "$final_allowed_ips" "$final_peer" \ + "$final_expected_allowed" "$final_allowed" "$MANAGED_PEERS" || exit 71 done < "$CONF" + VC_FABRIC_APPLY_DETAIL=final_routes final_fabric_routes=$($IP -o route show table all dev "$IFACE" 2>/dev/null) || exit 71 final_route_extra=$(printf '%s\n' "$final_fabric_routes" | awk \ -v allowed="$MANAGED_ROUTES" -v own={_q(host.overlay_ip)} \ @@ -9443,8 +9654,10 @@ if [ "$FIREWALL_MODE" = firewalld_nft ] || }}') [ "$final_route_seen" = 1 ] || exit 71 done + VC_FABRIC_APPLY_DETAIL=final_overlay_paths {path_checks} if [ "$FIREWALL_MODE" = firewalld_nft ]; then + VC_FABRIC_APPLY_DETAIL=final_zone [ "$(vc_firewall_zone permanent)" = none ] || exit 71 [ "$(vc_firewall_zone runtime)" = trusted ] || exit 71 fi @@ -9453,6 +9666,7 @@ if [ "$FIREWALL_MODE" = firewalld_nft ] || # which restores and proves every VPC before it alone removes the guard. : fi +VC_FABRIC_APPLY_DETAIL=commit _txn_state_tmp=$(mktemp "$TXN/.state.XXXXXX") printf '%s\n' committed > "$_txn_state_tmp" chmod 600 "$_txn_state_tmp" @@ -9460,6 +9674,11 @@ mv -f "$_txn_state_tmp" "$TXN/state" _txn_state_tmp= sync COMMITTED=yes +# OnBootSec may already be elapsed and enqueue repair immediately. Start the +# timer only after durable commit, so a failed pre-commit readback/rollback +# cannot race a queued primary helper into the restored absent generator. +VC_FABRIC_APPLY_DETAIL=convergence +"$SYSTEMCTL" start {FABRIC_TIMER} >/dev/null 2>&1 || exit 79 # Release the transaction flock only after durable commit, then synchronously # run the independent dispatcher. It either waits for an inactive primary or, # when primary is already active, executes the complete helper to restore and @@ -9509,6 +9728,7 @@ printf '%s\n' "$final_emergency_tables" | final_emergency_tables_verify=$($NFT list tables 2>/dev/null) || exit 79 [ "$final_emergency_tables_verify" = "$final_emergency_tables" ] || exit 79 [ ! -e "$final_emergency_owner" ] && [ ! -L "$final_emergency_owner" ] || exit 79 +VC_FABRIC_APPLY_DETAIL=complete vc_fabric_operation_checkpoint complete printf '%s\\n' '__VC_FABRIC_APPLIED__' '''