Fix Fabric recovery and readback diagnostics

venix
admin 1 month ago
parent 63b4741ff2
commit a45a0da682

@ -156,7 +156,8 @@ ACL-first helper。后一分支不得 restart 主 Fabric unit 或任何 libvirt
保留时)和 `journalctl -u virt-cloud-fabric-apply-<operation_id>.service --no-pager`。
transient unit 使用 `--collect`,因此终态后 `systemctl status` 可以变成 not-found,
但 journal 仍必须可按完整 unit 名回查。journal 只得出现
operation ID、固定 state/checkpoint/reason 与 systemd 生命周期;不得出现私钥、完整脚本、
operation ID、固定 state/checkpoint/reason、固定 helper_phase/apply_check/退出码
与 systemd 生命周期;不得出现私钥、完整脚本、
peer 全量配置、SSH 路径或原始 stderr。
journal 恢复必须按实际进度显示固定子检查点 `journal_artifact_restore`、
`journal_nft_restore`、`journal_runtime_restore` 或 `journal_finalize`;任何未知值都只能投影为
@ -164,6 +165,10 @@ ACL-first helper。后一分支不得 restart 主 Fabric unit 或任何 libvirt
`journal_dispatch` 表示已验证恢复 helper 身份并即将调用;执行前检查分别显示
`preflight_sysctl/tools/artifacts/firewall/vpc/underlay/routes/persistence`(每项为独立
`preflight_` 检查点)。旧 `preflight` 的未知命令失败不得再统一误标为资源归属冲突。
async runner 必须继续丢弃原始 stdout/stderr,只通过独立描述符记录净化后的
`helper_state=failed helper_phase=... exit_status=...`。apply 在回滚开始前记录
`apply_state=failed apply_check=... exit_status=...`,用于区别首次失败与回滚/服务
的次生错误;未知子项只能为 `unknown`,不能输出命令、配置或外部错误文本。
3. `/run/virt-cloud/fabric-operations/<operation_id>` 必须为 root:root `0700`,manifest/status
必须为 `0600`,apply/update/runner 必须为 `0700`;任一符号链接、多余文件、owner/mode/hash
漂移都必须 fail-closed。达到上限时只能精确删除最旧的已验证终态目录,不得删除
@ -187,6 +192,14 @@ ACL-first helper。后一分支不得 restart 主 Fabric unit 或任何 libvirt
## 失败回滚和安全边界
保留同计划的 `ready` journal 和历史 `helper.present`:即使历史 helper 不支持新版
NetworkManager runtime shadow,也只能在旧 helper 与 journal pre/post-image 精确一致、
meta 中 digest/UUID/manager/firewall/underlay 全部绑定当前方案、当前临时 helper 的权限与
计划头核验后,交由当前实现恢复。恢复完成的 recovery-only 调用不得再次执行已恢复的
旧 helper。跨计划仍沿历史恢复合同,不得把旧计划的权限移交给新模板;外部文件/密钥/nft
漂移必须保留现场并拒绝。新增 timer 必须在 `state=committed` 同步落盘之后才启动,
timer 启动失败只能保留 committed journal,不能回滚已提交的数据面。
在隔离环境中分别让 `nft -c/-f`、`firewall-cmd`、`wg`、`ip route`、
`nmcli/ifup`、systemd gate 和 timer 返回失败。每次都应看到 apply 失败并保持
`uncertain/degraded`;firewalld 模式的回滚先撤销 panel 接口的 runtime `trusted`,再以

@ -3693,6 +3693,7 @@ def test_apply_queues_reconcile_dispatcher_only_after_durable_commit():
def _run_apply_convergence_tail(
tmp_path: Path, *, start_fails: bool = False,
timer_fails: bool = False,
stale_config: bool = False, emergency_present: bool = False,
committed_trap_state: str | None = None,
) -> subprocess.CompletedProcess[str]:
@ -3709,6 +3710,9 @@ def _run_apply_convergence_tail(
"\n}\ntrap rollback EXIT HUP INT TERM", rollback_start,
) + 2
rollback_function = rendered[rollback_start:rollback_end]
logger_start = rendered.index("vc_fabric_apply_failure_log() {")
logger_end = rendered.index("\n}\n", logger_start) + 2
failure_logger = rendered[logger_start:logger_end]
emergency_owner = tmp_path / "emergency.owner"
sysctl_file = tmp_path / "fabric-sysctl.conf"
sysctl_file.write_text(_sysctl_text(current), encoding="utf-8")
@ -3740,6 +3744,9 @@ def _run_apply_convergence_tail(
systemctl.write_text(
"""#!/bin/sh
case "$1:$2" in
start:virt-cloud-wg-fabric-reconcile.timer)
[ "${VC_TEST_TIMER_FAIL:-no}" = no ] || exit 1
exit 0 ;;
start:virt-cloud-wg-fabric-reconcile.service)
[ "${VC_TEST_START_FAIL:-no}" = no ] || exit 1
: > "$VC_TEST_CONVERGED"; exit 0 ;;
@ -3836,6 +3843,7 @@ esac
"vc_fabric_sysctl_file_owned() { return 0; }",
"read_sysctl() { cat \"$1\"; }",
"vc_fabric_operation_checkpoint() { :; }",
failure_logger,
*trap_setup,
"exec 9>>\"$LOCK\"",
tail,
@ -3846,6 +3854,7 @@ esac
env={
"PATH": "/usr/bin:/bin",
"VC_TEST_START_FAIL": "yes" if start_fails else "no",
"VC_TEST_TIMER_FAIL": "yes" if timer_fails else "no",
"VC_TEST_EMERGENCY": "yes" if emergency_present else "no",
"VC_TEST_CONVERGED": str(converged),
},
@ -3884,6 +3893,16 @@ def test_post_commit_dispatcher_failure_preserves_committed_journal(tmp_path):
) == "committed\n"
def test_post_commit_timer_failure_preserves_committed_journal(tmp_path):
result = _run_apply_convergence_tail(
tmp_path, timer_fails=True, committed_trap_state="committed",
)
assert result.returncode == 79, result.stderr
assert (tmp_path / "txn" / "state").read_text(
encoding="utf-8",
) == "committed\n"
def test_old_committed_trap_never_deletes_concurrent_ready_journal(tmp_path):
result = _run_apply_convergence_tail(
tmp_path, start_fails=True, committed_trap_state="ready",

@ -0,0 +1,338 @@
"""Execute the production-rendered final readback with real CLI serialization.
Only external tools are replaced. The final shell body, nft owner checks,
tenant-set matcher and firewall-zone enumerator are the production code.
"""
from __future__ import annotations
import base64
import json
import os
from pathlib import Path
import subprocess
import sys
import pytest
from virsh_panel.fabric import (
_FABRIC_NFT_VALIDATION_SH,
_FABRIC_WG_READBACK_SH,
_BOOT_HELPER,
_FIREWALL_MODE_GUARD_SH,
_TRANSACTION_SCRIPT,
_VPC_STATE_LOADER,
_apply_script,
_config_text,
_nft_rules_text,
parse_fabric_plan,
)
def _between(text: str, start: str, end: str) -> str:
begin = text.index(start)
return text[begin:text.index(end, begin)]
_TOOLS = r'''import json
import os
from pathlib import Path
import sys
root = Path(os.environ["VC_FINAL_ROOT"])
name = Path(sys.argv[0]).name
args = sys.argv[1:]
with (root / "calls").open("a", encoding="utf-8") as log:
log.write(json.dumps([name, *args]) + "\n")
fixture = json.loads((root / "responses").read_text())
call = json.dumps([name, *args])
if call not in fixture:
print("unexpected test tool invocation", file=sys.stderr)
raise SystemExit(99)
if name == "wg" and args == ["pubkey"]:
if sys.stdin.read() != "test-only-private-identity\n":
raise SystemExit(98)
response = fixture[call]
sys.stdout.write(response["stdout"])
raise SystemExit(response["status"])
'''
class FinalFixture:
def __init__(self, tmp_path: Path, manager: str, peers: int = 1):
self.root = tmp_path
self.manager = manager
self.plan = parse_fabric_plan({
"fabric_id": "final-readback", "region": "hk",
"overlay_cidr": "100.64.1.0/24", "revision": 1,
"nodes": [
{"host_id": str(index), "overlay_ip": f"100.64.1.{index}",
"endpoint": f"203.0.113.{index}:51820"}
for index in range(1, peers + 2)
],
})
self.keys = {
node.host_id: base64.b64encode(bytes([index]) * 32).decode()
for index, node in enumerate(self.plan.nodes, 1)
}
self.responses: dict[str, dict[str, object]] = {}
self.routes = ""
tool = tmp_path / "tool"
tool.write_text(f"#!{sys.executable}\n" + _TOOLS, encoding="utf-8")
tool.chmod(0o700)
for name in ("ip", "wg", "nft", "firewall-cmd"):
(tmp_path / name).symlink_to(tool)
(tmp_path / "private").write_text("test-only-private-identity\n")
(tmp_path / "config").write_text(_config_text(
self.plan, self.plan.nodes[0], self.keys, "firewalld_nft"))
for filename in ("vpc-peers", "vpc-routes", "vpc-tenants"):
(tmp_path / filename).write_text("")
self.respond("nft", ["-s", "list", "table", "inet",
"virt_cloud_wg_fabric"],
_nft_rules_text(self.plan.digest))
self.respond("nft", ["list", "set", "inet", "virt_cloud_wg_fabric",
"vpc_tenant_ips"],
"table inet virt_cloud_wg_fabric {\n"
"\tset vpc_tenant_ips {\n\t\ttype ipv4_addr\n"
"\t\tflags interval\n\t}\n}\n")
self.respond("wg", ["show", "wg-fabric0"], "")
self.respond("wg", ["pubkey"], self.keys["1"] + "\n")
self.respond("wg", ["show", "wg-fabric0", "public-key"],
self.keys["1"] + "\n")
self.respond("wg", ["show", "wg-fabric0", "listen-port"], "51820\n")
self.respond("ip", ["-o", "link", "show", "dev", "wg-fabric0"],
"16: wg-fabric0: <POINTOPOINT,NOARP,UP,LOWER_UP> mtu 1420 "
"qdisc noqueue state UNKNOWN mode DEFAULT group default qlen 1000\n")
# ip -o replaces the line break between address and lifetime by a
# backslash. The entire address record must remain one physical line.
self.respond("ip", ["-o", "-4", "address", "show", "dev", "wg-fabric0"],
"16: wg-fabric0 inet 100.64.1.1/32 scope global wg-fabric0\\"
" valid_lft forever preferred_lft forever\n")
live_peers = self.plan.nodes[1:]
self.respond("wg", ["show", "wg-fabric0", "peers"],
"".join(self.keys[node.host_id] + "\n" for node in live_peers))
self.respond("wg", ["show", "wg-fabric0", "endpoints"],
"".join(self.keys[node.host_id] + "\t" + node.endpoint + "\n"
for node in reversed(live_peers)))
self.respond("wg", ["show", "wg-fabric0", "allowed-ips"],
"".join(self.keys[node.host_id] + "\t" + node.allowed_ip + "\n"
for node in reversed(live_peers)))
metric = " metric 50" if manager == "networkmanager_nmcli" else ""
self.routes = "".join(node.overlay_ip + " scope link" + metric + "\n"
for node in live_peers)
self.routes += ("local 100.64.1.1 table local proto kernel scope host "
"src 100.64.1.1\n")
self.respond("ip", ["-o", "route", "show", "table", "all", "dev",
"wg-fabric0"], self.routes)
for node in live_peers:
self.respond("ip", ["route", "get", node.overlay_ip,
"from", "100.64.1.1"],
node.overlay_ip + " from 100.64.1.1 dev wg-fabric0 uid 0\n"
" cache\n")
for permanent in (False, True):
prefix = ["--permanent"] if permanent else []
self.respond("firewall-cmd", prefix + ["--get-zones"], "public trusted\n")
for zone in ("public", "trusted"):
active = not permanent and zone == "trusted"
self.respond("firewall-cmd", prefix + [f"--zone={zone}",
"--query-interface=wg-fabric0"],
"yes\n" if active else "no\n", 0 if active else 1)
def respond(self, tool: str, args: list[str], stdout: str, status: int = 0):
self.responses[json.dumps([tool, *args])] = {
"stdout": stdout, "status": status,
}
def add_vpc_member(self):
key = self.keys["2"]
(self.root / "vpc-peers").write_text(key + "|10.1.0.3/32\n")
(self.root / "vpc-tenants").write_text("10.1.0.3/32\n")
(self.root / "vpc-routes").write_text(
"10.1.0.3/32|wg-fabric0|100|100.64.1.2|100\n")
self.respond("nft", ["-s", "list", "table", "inet", "virt_cloud_wg_fabric"],
_nft_rules_text(self.plan.digest).replace(
"flags interval;", "flags interval;\n elements = { 10.1.0.3 }", 1))
self.respond("nft", ["list", "set", "inet", "virt_cloud_wg_fabric", "vpc_tenant_ips"],
"table inet virt_cloud_wg_fabric {\n\tset vpc_tenant_ips {\n"
"\t\ttype ipv4_addr\n\t\tflags interval\n"
"\t\telements = { 10.1.0.3 }\n\t}\n}\n")
self.respond("ip", ["-o", "route", "show", "table", "all", "dev", "wg-fabric0"],
self.routes + "10.1.0.3 via 100.64.1.2 table 100 proto 186 metric 100\n")
def run(self):
rendered = _apply_script(
self.plan, self.plan.nodes[0], self.keys,
deployment_mode=self.manager, firewall_mode="firewalld_nft")
final = _between(rendered, "vc_fabric_operation_checkpoint final_readback\n",
"_txn_state_tmp=$(mktemp")
final = final.split("\n", 1)[1]
functions = "\n".join((
_FABRIC_NFT_VALIDATION_SH,
_FABRIC_WG_READBACK_SH,
_between(_TRANSACTION_SCRIPT, "nft_panel_owned() {", "snapshot_file() {"),
_between(_VPC_STATE_LOADER, "vpc_state_allowed_for_peer() {",
"vpc_state_route_owned() {"),
_VPC_STATE_LOADER[_VPC_STATE_LOADER.index("vpc_state_nft_set_matches() {"):],
_between(_FIREWALL_MODE_GUARD_SH, "vc_firewall_zone() {",
"vc_firewall_zone_exists() {"),
))
env = {**os.environ, "VC_FINAL_ROOT": str(self.root),
"IFACE": "wg-fabric0", "EXPECTED_DIGEST": self.plan.digest,
"FIREWALL_MODE": "firewalld_nft",
"MANAGED_PEERS": " ".join(self.keys[n.host_id] for n in self.plan.nodes[1:]),
"MANAGED_ROUTES": " ".join(n.allowed_ip for n in self.plan.nodes[1:]),
"KEY": str(self.root / "private"), "CONF": str(self.root / "config"),
"VPC_PEER_FILE": str(self.root / "vpc-peers"),
"VPC_ROUTE_FILE": str(self.root / "vpc-routes"),
"VPC_TENANT_FILE": str(self.root / "vpc-tenants")}
env.update({name: str(self.root / tool) for name, tool in (
("WG", "wg"), ("IP", "ip"), ("NFT", "nft"),
("FIREWALL_CMD", "firewall-cmd"))})
(self.root / "responses").write_text(json.dumps(self.responses))
result = subprocess.run(["/bin/sh", "-eu", "-c", functions + "\n" + final],
capture_output=True, text=True, env=env, check=False)
assert "unexpected test tool invocation" not in result.stderr
return result
@pytest.mark.parametrize("manager", ["ifupdown", "networkmanager_nmcli"])
@pytest.mark.parametrize("peers", [1, 2])
def test_actual_final_slice_accepts_debian_and_alma_serialization(tmp_path, manager, peers):
fixture = FinalFixture(tmp_path, manager, peers)
if manager == "networkmanager_nmcli":
rules = _nft_rules_text(fixture.plan.digest).replace(
"hook output priority -100;", "hook output priority dstnat;")
rules = "\n".join(line.replace(" ", "\t") for line in rules.splitlines()) + "\n"
fixture.respond("nft", ["-s", "list", "table", "inet",
"virt_cloud_wg_fabric"], rules)
result = fixture.run()
assert result.returncode == 0, result.stderr
@pytest.mark.parametrize("drift", ["nft", "public-key", "listen-port", "down", "address",
"extra-peer", "endpoint", "allowed", "route", "path", "zone"])
def test_actual_final_slice_rejects_mutable_readback_drift(tmp_path, drift):
fixture = FinalFixture(tmp_path, "ifupdown")
key = fixture.keys["2"]
if drift == "nft":
fixture.respond("nft", ["-s", "list", "table", "inet", "virt_cloud_wg_fabric"],
_nft_rules_text(fixture.plan.digest).replace("counter drop", "counter accept", 1))
elif drift in ("public-key", "listen-port"):
fixture.respond("wg", ["show", "wg-fabric0", drift], "wrong\n")
elif drift == "down":
fixture.respond("ip", ["-o", "link", "show", "dev", "wg-fabric0"],
"16: wg-fabric0: <POINTOPOINT,NOARP> mtu 1420\n")
elif drift == "address":
fixture.respond("ip", ["-o", "-4", "address", "show", "dev", "wg-fabric0"],
"16: wg-fabric0 inet 100.64.1.1/24 scope global wg-fabric0\n")
elif drift == "extra-peer":
fixture.respond("wg", ["show", "wg-fabric0", "peers"], key + "\nforeign\n")
elif drift == "endpoint":
fixture.respond("wg", ["show", "wg-fabric0", "endpoints"], key + "\t203.0.113.9:51820\n")
elif drift == "allowed":
fixture.respond("wg", ["show", "wg-fabric0", "allowed-ips"], key + "\t100.64.1.9/32\n")
elif drift == "route":
fixture.respond("ip", ["-o", "route", "show", "table", "all", "dev", "wg-fabric0"],
fixture.routes + "100.64.1.3 scope link\n")
elif drift == "path":
fixture.respond("ip", ["route", "get", "100.64.1.2", "from", "100.64.1.1"],
"100.64.1.2 from 100.64.1.1 dev br0 uid 0\n cache\n")
else:
fixture.respond("firewall-cmd", ["--zone=trusted", "--query-interface=wg-fabric0"], "no\n", 1)
result = fixture.run()
assert result.returncode in (71, 78), result.stderr
@pytest.mark.parametrize("manager", ["ifupdown", "networkmanager_nmcli"])
@pytest.mark.parametrize("reverse", [False, True])
@pytest.mark.parametrize("peers", [1, 2])
def test_actual_final_slice_accepts_official_multiple_allowed_ips_serialization(tmp_path, manager, reverse, peers):
fixture = FinalFixture(tmp_path, manager, peers)
key = fixture.keys["2"]
fixture.add_vpc_member()
prefixes = ["100.64.1.2/32", "10.1.0.3/32"]
if reverse:
prefixes.reverse()
fixture.respond("wg", ["show", "wg-fabric0", "allowed-ips"],
key + "\t" + " ".join(prefixes) + "\n" + "".join(
fixture.keys[node.host_id] + "\t" + node.allowed_ip + "\n"
for node in fixture.plan.nodes[2:]))
assert fixture.run().returncode == 0
@pytest.mark.parametrize("drift", ["missing", "extra", "duplicate", "unknown", "multiple-key-fields"])
def test_actual_final_slice_requires_complete_unique_peer_records(tmp_path, drift):
fixture = FinalFixture(tmp_path, "ifupdown", 2)
first, second = fixture.keys["2"], fixture.keys["3"]
if drift == "missing":
actual = first + "\n"
elif drift == "extra":
actual = first + "\n" + second + "\n" + base64.b64encode(bytes([9]) * 32).decode() + "\n"
elif drift == "duplicate":
actual = first + "\n" + second + "\n" + first + "\n"
elif drift == "unknown":
actual = first + "\n" + "bad-key\n"
else:
actual = first + "\t" + second + "\n"
fixture.respond("wg", ["show", "wg-fabric0", "peers"], actual)
assert fixture.run().returncode == 71
@pytest.mark.parametrize("drift", ["wide", "public", "duplicate", "missing", "extra", "csv",
"malformed", "ipv6", "duplicate-peer", "unknown-peer"])
def test_actual_final_slice_multiple_allowed_ips_remains_fail_closed(tmp_path, drift):
fixture = FinalFixture(tmp_path, "ifupdown")
key = fixture.keys["2"]
fixture.add_vpc_member()
values = ["100.64.1.2/32", "10.1.0.3/32"]
if drift == "wide":
values[1] = "10.1.0.3/24"
elif drift == "public":
values[1] = "203.0.113.9/32"
elif drift == "duplicate":
values.append(values[1])
elif drift == "missing":
values.pop()
elif drift == "extra":
values.append("10.1.0.4/32")
elif drift == "malformed":
values[1] = "10.1.0.256/32"
elif drift == "ipv6":
values[1] = "fd00::3/128"
actual = key + "\t" + ("," if drift == "csv" else " ").join(values) + "\n"
if drift == "duplicate-peer":
actual += actual
elif drift == "unknown-peer":
actual += base64.b64encode(bytes([9]) * 32).decode() + "\t10.1.0.9/32\n"
fixture.respond("wg", ["show", "wg-fabric0", "allowed-ips"], actual)
assert fixture.run().returncode == 71
@pytest.mark.parametrize("peers", [1, 2])
@pytest.mark.parametrize("reverse", [False, True])
@pytest.mark.parametrize("vpc", [False, True])
def test_boot_actual_peer_readback_uses_identical_record_validator(tmp_path, peers, reverse, vpc):
fixture = FinalFixture(tmp_path, "ifupdown", peers)
if vpc:
fixture.add_vpc_member()
fixture.respond("wg", ["show", "wg-fabric0", "allowed-ips"],
fixture.keys["2"] + "\t10.1.0.3/32 100.64.1.2/32\n" + "".join(
fixture.keys[node.host_id] + "\t" + node.allowed_ip + "\n"
for node in fixture.plan.nodes[2:]))
live_keys = [fixture.keys[node.host_id] for node in fixture.plan.nodes[1:]]
if reverse:
live_keys.reverse()
fixture.respond("wg", ["show", "wg-fabric0", "peers"], "\n".join(live_keys) + "\n")
(tmp_path / "responses").write_text(json.dumps(fixture.responses))
peer_body = _between(_BOOT_HELPER, 'actual_peers=$("$WG" show "$IFACE" peers',
"actual_fabric_routes=")
allowed = _between(_VPC_STATE_LOADER, "vpc_state_allowed_for_peer() {",
"vpc_state_route_owned() {")
result = subprocess.run(["/bin/sh", "-eu", "-c", _FABRIC_WG_READBACK_SH + "\n" + allowed + "\n" + peer_body],
capture_output=True, text=True, check=False,
env={**os.environ, "VC_FINAL_ROOT": str(tmp_path),
"WG": str(tmp_path / "wg"), "IFACE": "wg-fabric0",
"seen_peers": " ".join(live_keys),
"CONF": str(tmp_path / "config"),
"VPC_PEER_FILE": str(tmp_path / "vpc-peers")})
assert result.returncode == 0, result.stderr

@ -0,0 +1,243 @@
"""Execute rendered async journal plumbing without running host mutations.
The runner, update state machine, logger, helper EXIT trap and committed
rollback branch are production-rendered code. Only stat's platform-specific
ownership report and sync are external-tool doubles inside temporary files.
"""
from __future__ import annotations
import base64
import os
from pathlib import Path
import shlex
import subprocess
import sys
import time
import pytest
from virsh_panel.fabric import (
FABRIC_OPERATION_ROOT,
_TRANSACTION_SCRIPT,
_apply_script,
_fabric_operation_runner_script,
_fabric_operation_update_script,
parse_fabric_plan,
)
_SECRET = "NEVER_LOG_raw_private_fixture_or_sensitive_path"
_OPERATION = "a" * 32
_DIGEST = "b" * 64
def _write(path: Path, body: str, mode: int = 0o700):
path.write_text(body, encoding="utf-8")
path.chmod(mode)
def _render(manager="ifupdown", firewall="firewalld_nft"):
plan = parse_fabric_plan({
"fabric_id": "safe-log-test", "region": "hk",
"overlay_cidr": "100.64.1.0/24", "revision": 1,
"nodes": [
{"host_id": "a", "overlay_ip": "100.64.1.1",
"endpoint": "203.0.113.1:51820"},
{"host_id": "b", "overlay_ip": "100.64.1.2",
"endpoint": "203.0.113.2:51820"},
],
})
keys = {node.host_id: base64.b64encode(bytes([index]) * 32).decode()
for index, node in enumerate(plan.nodes, 1)}
return _apply_script(plan, plan.nodes[0], keys,
deployment_mode=manager, firewall_mode=firewall)
def _apply_logger(rendered):
start = rendered.index("vc_fabric_apply_failure_log() {")
return rendered[start:rendered.index("vc_fabric_operation_checkpoint() {", start)]
def _helper_entry(rendered):
# This is the helper actually embedded in apply, not a retyped logger.
start = rendered.index("#!/bin/sh\n# virt-cloud-wg-fabric-v1\n# fabric-plan-digest=")
end = rendered.index("BASE=/etc/virt-cloud\n", start)
prefix = rendered[start:end]
assert "__VC_FABRIC_DIGEST__" not in prefix
assert "trap cleanup EXIT HUP INT TERM\n" in prefix
return prefix
def _run(script, *, env=None, args=()):
return subprocess.run(["/bin/sh", "-eu", "-c", script, "safe-log-test", *args],
capture_output=True, text=True, check=False,
env={**os.environ, **(env or {})})
def _journal_wrapper(body):
return ("exec 3>&1\nexec >/dev/null 2>&1\n"
"VC_FABRIC_SAFE_LOG=1\nexport VC_FABRIC_SAFE_LOG\n" + body)
@pytest.mark.parametrize("detail", [
"entry", "final_acl", "final_tenants", "final_identity", "final_link",
"final_addresses", "final_peers", "final_endpoints", "final_allowed_ips",
"final_routes", "final_overlay_paths", "final_zone", "commit", "convergence",
"complete",
])
def test_rendered_apply_logger_has_one_fixed_fd3_record(detail):
script = _journal_wrapper(_apply_logger(_render()) + (
f"VC_FABRIC_APPLY_DETAIL={shlex.quote(detail)}\n"
f"printf '%s\\n' {shlex.quote(_SECRET)}\n"
f"printf '%s\\n' {shlex.quote(_SECRET)} >&2\n"
"vc_fabric_apply_failure_log 71\n"
))
result = _run(script)
assert result.returncode == 0
assert result.stdout == (
f"virt-cloud-fabric apply_state=failed apply_check={detail} exit_status=71\n")
assert result.stderr == ""
@pytest.mark.parametrize("logger", ["apply", "helper"])
@pytest.mark.parametrize("status", ["", "-1", "256", "9" * 100, _SECRET])
def test_unknown_phase_and_invalid_status_are_sanitized(logger, status):
rendered = _render()
phase = "unknown\n" + _SECRET + "=/sensitive/path"
if logger == "apply":
body = (_apply_logger(rendered)
+ f"VC_FABRIC_APPLY_DETAIL={shlex.quote(phase)}\n"
+ f"vc_fabric_apply_failure_log {shlex.quote(status)}\n")
line = "virt-cloud-fabric apply_state=failed apply_check=unknown exit_status=1\n"
else:
body = (_helper_entry(rendered)
+ f"VC_FABRIC_HELPER_PHASE={shlex.quote(phase)}\n"
+ f"vc_fabric_helper_failure_log {shlex.quote(status)}\nexit 0\n")
line = "virt-cloud-fabric helper_state=failed helper_phase=unknown exit_status=1\n"
result = _run(_journal_wrapper(body))
assert result.returncode == 0
assert result.stdout == line
assert result.stderr == ""
assert _SECRET not in result.stdout
@pytest.mark.parametrize("status", [1, 70, 71, 79, 127, 143, 255])
def test_rendered_child_helper_preserves_exit_status_without_duplicate_fd3_log(status):
body = (_helper_entry(_render()) + "VC_FABRIC_HELPER_PHASE=final\n"
+ f"printf '%s\\n' {shlex.quote(_SECRET)}\n"
+ f"printf '%s\\n' {shlex.quote(_SECRET)} >&2\n"
+ f"exit {status}\n")
result = _run(_journal_wrapper(body))
assert result.returncode == status
assert result.stdout == (
f"virt-cloud-fabric helper_state=failed helper_phase=final exit_status={status}\n")
assert result.stderr == ""
def _runner_fixture(tmp_path: Path, apply_body: str):
directory = tmp_path / _OPERATION
directory.mkdir(mode=0o700)
stat_tool = tmp_path / "stat"
_write(stat_tool, f"#!{sys.executable}\n" + (
"import os, stat, sys\n"
"assert sys.argv[1:3] == ['-c', '%u:%a']\n"
"print('0:' + format(stat.S_IMODE(os.stat(sys.argv[3]).st_mode), 'o'))\n"
))
sync_tool = tmp_path / "sync"
_write(sync_tool, "#!/bin/sh\nexit 0\n")
updater = _fabric_operation_update_script(_OPERATION, _DIGEST).replace(
FABRIC_OPERATION_ROOT, str(tmp_path)).replace(
"stat -c", shlex.quote(str(stat_tool)) + " -c").replace(
"\nsync\n", "\n" + shlex.quote(str(sync_tool)) + "\n")
_write(directory / "update", updater)
_write(directory / "apply", "#!/bin/sh\n" + apply_body)
_write(directory / "manifest", (
f"schema=1\noperation_id={_OPERATION}\nplan_digest={_DIGEST}\n"), 0o600)
now = int(time.time())
_write(directory / "status", (
f"schema=1\noperation_id={_OPERATION}\nstatus=queued\nphase=apply\n"
"checkpoint=dispatch\nreason=none\n"
f"created_at={now}\nstarted_at=0\nupdated_at={now}\nfinished_at=0\n"), 0o600)
runner = _fabric_operation_runner_script(_OPERATION, _DIGEST).replace(
FABRIC_OPERATION_ROOT, str(tmp_path))
return runner, directory
@pytest.mark.parametrize("status", [0, 71, 79])
def test_rendered_runner_propagates_safe_descriptor_and_never_raw_output(tmp_path, status):
helper = tmp_path / "child-helper"
_write(helper, _helper_entry(_render()) + (
"VC_FABRIC_HELPER_PHASE=gate\n"
f"printf '%s\\n' {shlex.quote(_SECRET)}\n"
f"printf '%s\\n' {shlex.quote(_SECRET)} >&2\nexit {status}\n"
))
body = (f"printf '%s\\n' {shlex.quote(_SECRET)}\n"
+ f"printf '%s\\n' {shlex.quote(_SECRET)} >&2\n"
+ f"exec /bin/sh {shlex.quote(str(helper))}\n")
runner, directory = _runner_fixture(tmp_path, body)
result = _run(runner)
assert result.returncode == (0 if status == 0 else 1)
assert result.stderr == ""
assert _SECRET not in result.stdout
assert str(tmp_path) not in result.stdout
lines = result.stdout.splitlines()
assert len(lines) == (2 if status == 0 else 3)
assert lines[0] == (
f"virt-cloud-fabric operation={_OPERATION} state=running checkpoint=dispatch reason=none")
if status:
assert lines[1] == (
f"virt-cloud-fabric helper_state=failed helper_phase=gate exit_status={status}")
state = (directory / "status").read_text()
assert ("status=succeeded\n" if status == 0 else "status=failed\n") in state
assert lines[-1].startswith(f"virt-cloud-fabric operation={_OPERATION} state=")
@pytest.mark.parametrize("status", [71, 79])
def test_first_apply_failure_precedes_secondary_rollback_cleanup_record(tmp_path, status):
rendered = _render()
helper = tmp_path / "secondary-helper"
_write(helper, _helper_entry(rendered) + "VC_FABRIC_HELPER_PHASE=gate\nexit 79\n")
start = _TRANSACTION_SCRIPT.index("rollback() {")
end = _TRANSACTION_SCRIPT.index("\ntrap rollback EXIT HUP INT TERM", start)
rollback = _TRANSACTION_SCRIPT[start:end]
body = ("COMMITTED=yes\n" + _apply_logger(rendered) + rollback + (
"\ncleanup() {\n"
f" printf '%s\\n' {shlex.quote(_SECRET)} >&2\n"
f" /bin/sh {shlex.quote(str(helper))} >/dev/null 2>&1 || true\n"
"}\ntrap rollback EXIT HUP INT TERM\n"
f"VC_FABRIC_APPLY_DETAIL=final_routes\nexit {status}\n"
))
result = _run(_journal_wrapper(body))
assert result.returncode == status
assert result.stderr == ""
assert result.stdout.splitlines() == [
f"virt-cloud-fabric apply_state=failed apply_check=final_routes exit_status={status}",
"virt-cloud-fabric helper_state=failed helper_phase=gate exit_status=79",
]
@pytest.mark.parametrize("manager", ["ifupdown", "networkmanager_nmcli"])
@pytest.mark.parametrize("firewall", ["native_nftables", "firewalld_nft"])
def test_rendered_timer_start_strictly_follows_durable_commit(manager, firewall):
rendered = _render(manager, firewall)
timer_command = '"$SYSTEMCTL" start virt-cloud-wg-fabric-reconcile.timer >/dev/null'
assert rendered.count(timer_command) == 1
commit_marker = rendered.rindex('committed > "$_txn_state_tmp"')
publish = rendered.index('mv -f "$_txn_state_tmp" "$TXN/state"', commit_marker)
durable_sync = rendered.index("\nsync\n", publish)
committed_flag = rendered.index("COMMITTED=yes", durable_sync)
timer_start = rendered.index(timer_command)
unlock = rendered.index("exec 9>&-", timer_start)
assert commit_marker < publish < durable_sync < committed_flag < timer_start < unlock
def test_success_and_disabled_safe_logging_do_not_invent_failure_records():
logger = _apply_logger(_render())
result = _run(_journal_wrapper(logger + "VC_FABRIC_APPLY_DETAIL=complete\n"
"vc_fabric_apply_failure_log 0\n"))
assert result.returncode == 0
assert result.stdout == result.stderr == ""
result = _run(_journal_wrapper(logger + "VC_FABRIC_SAFE_LOG=0\n"
"vc_fabric_apply_failure_log 71\n"))
assert result.returncode == 0
assert result.stdout == result.stderr == ""

@ -0,0 +1,272 @@
"""Recover a same-plan ready journal without redispatching its old helper."""
from __future__ import annotations
import base64
import json
import re
import shlex
import subprocess
import pytest
from test_fabric_first_deploy_entry import _write
from test_fabric_nm_runtime_shadow_recovery import _failed_first_deploy
from virsh_panel.fabric import (
FIREWALL_MODE_FIREWALLD,
FIREWALL_MODE_NATIVE,
_FABRIC_NFT_VALIDATION_SH,
_FABRIC_SYSCTL_VALIDATION_SH,
_FIREWALL_MODE_GUARD_SH,
_NMCLI_FILENAME_RESOLVER_SH,
_apply_script,
)
def _baseline_fixture(tmp_path):
values = _failed_first_deploy(tmp_path)
_, _, installed, _, _, _, journal, _ = values
# The real Alma host now has a journaled old helper baseline, not the
# helper.absent case covered by the first-deployment recovery path.
(journal / "helper.absent").unlink()
_write(journal / "helper.present", "")
_write(journal / "helper", installed.read_text(encoding="utf-8"), 0o700)
return values
def _rendered_driver(values, root, firewall_mode=FIREWALL_MODE_FIREWALLD):
plan, current, installed, base, runtime, roots, _, _ = values
keys = {host.host_id: base64.b64encode(bytes([index]) * 32).decode()
for index, host in enumerate(plan.nodes, start=1)}
rendered = _apply_script(
plan, plan.nodes[0], keys, deployment_mode="networkmanager_nmcli",
firewall_mode=firewall_mode,
)
start = rendered.index("COMMITTED=no\nROLLBACK_FAILED=no")
end = rendered.index("vc_fabric_operation_checkpoint preflight\n", start)
# Execute the actual rendered outer recovery functions and dispatch, then
# the complete current helper. Only host resources and CLI paths redirect
# to fixture doubles; no ownership/recovery shell function is replaced.
recovery = rendered[start:end]
body = (_NMCLI_FILENAME_RESOLVER_SH + "\n" + _FABRIC_NFT_VALIDATION_SH + "\n"
+ _FABRIC_SYSCTL_VALIDATION_SH + "\n" + _FIREWALL_MODE_GUARD_SH + "\n"
+ recovery)
for source, destination in roots.items():
body = body.replace(source, str(destination))
body = body.replace("in /run ", "in " + shlex.quote(str(root / "run")) + " ")
tools = root / "tools"
identity = re.search(r'^CONNECTION_UUID="([^"]+)"$', current.read_text(), re.MULTILINE)[1]
variables = {
"BASE": base, "HELPER": installed, "_tmp_helper": current,
"OWNER": base / "wg-fabric0.owner", "CONF": base / "wg-fabric0.conf",
"NM_PROFILE": roots["/etc/NetworkManager"] / "system-connections" /
"virt-cloud-wg-fabric0.nmconnection",
"EMERGENCY_OWNER": runtime / "wg-fabric-emergency.owner",
"EXPECTED_DIGEST": plan.digest, "DEPLOYMENT_MODE": "networkmanager_nmcli",
"FIREWALL_MODE": firewall_mode, "IFACE": "wg-fabric0",
"CONNECTION": "virt-cloud-wg-fabric0", "CONNECTION_UUID": identity,
"CMP": "/usr/bin/cmp",
}
for variable, name in (("STAT", "stat"), ("NFT", "nft"), ("IP", "ip"),
("WG", "wg"), ("SYSTEMCTL", "systemctl"),
("NMCLI", "nmcli"), ("FIREWALL_CMD", "firewall-cmd")):
variables[variable] = tools / name
prefix = "\n".join(key + "=" + shlex.quote(str(value)) for key, value in variables.items())
prefix += ("\nfind_tool() { printf '%s' " + shlex.quote(str(tools / "sha256sum")) + "; }\n"
"vc_fabric_operation_checkpoint() { :; }\n"
"PATH=" + shlex.quote(str(tools)) + ":/usr/bin:/bin; export PATH\n")
return "set -eu\n" + prefix + body
def _run(values, root, firewall_mode=FIREWALL_MODE_FIREWALLD):
return subprocess.run(
["/bin/sh", "-c", _rendered_driver(values, root, firewall_mode)], check=False,
capture_output=True, text=True,
env={"VC_ENTRY_ROOT": str(root), "VC_FABRIC_LOCK_HELD": "1"}, timeout=45,
)
def _native_baseline_fixture(tmp_path, *, empty_header=False):
values = _baseline_fixture(tmp_path)
_, current, _, _, _, roots, journal, shadow = values
# Rendered helper implementation is identical for both owners; the
# explicit literal backend selects the native branch. No shell function
# or recovery conditional is replaced in this fixture.
content = current.read_text().replace(
'FIREWALL_MODE="firewalld_nft"', 'FIREWALL_MODE="native_nftables"', 1,
)
native_main = tmp_path / "nftables.conf"
roots["/etc/nftables.conf"] = native_main
content = content.replace("/etc/nftables.conf", str(native_main))
if empty_header:
content = content.replace('UNDERLAY_ENDPOINT="203.0.113.1"', 'UNDERLAY_ENDPOINT=""')
content = content.replace('UNDERLAY_PORT="51820"', 'UNDERLAY_PORT=""')
_write(current, content, 0o700)
metadata = dict(line.split("=", 1) for line in (journal / "meta").read_text().splitlines())
metadata.update({
"firewall_mode": FIREWALL_MODE_NATIVE, "nft_main": str(native_main),
"firewalld_runtime_zone_before": "na", "firewalld_permanent_zone_before": "na",
"underlay_endpoint": "", "underlay_port": "", "underlay_zone": "",
"underlay_rule_owner": "none", "underlay_rule_runtime_before": "na",
"underlay_rule_permanent_before": "na",
})
_write(journal / "meta", "".join(f"{key}={value}\n" for key, value in metadata.items()))
distribution_rules = '# distribution-owned main file\ninclude "/etc/nftables.d/*.nft"\n'
_write(native_main, distribution_rules, 0o644)
(journal / "nft_main.absent").unlink()
_write(journal / "nft_main.present", "")
_write(journal / "nft_main", distribution_rules, 0o644)
_write(shadow, shadow.read_text().replace("zone=trusted", "zone="))
_write(tmp_path / "nm-zone", "")
(tmp_path / "runtime-zone").unlink()
# Native owner fixture: nftables enabled/running, firewalld masked and
# inactive. Only external systemctl evidence differs from firewalld.
tool = tmp_path / "tools" / "external-tool"
external = tool.read_text().replace(
'print("active" if args[1] == "firewalld" else "inactive")',
'print("inactive" if args[1] == "firewalld" else "active")',
).replace(
'print("enabled" if args[1] == "firewalld" else "disabled")',
'print("masked" if args[1] == "firewalld" else "enabled")',
)
_write(tool, external, 0o700)
return values
@pytest.mark.parametrize("empty_header", [False, True])
@pytest.mark.parametrize("failure", [None, "endpoint", "port"])
def test_native_same_plan_handoff_requires_empty_underlay_metadata_not_unused_header(
tmp_path, empty_header, failure):
values = _native_baseline_fixture(tmp_path, empty_header=empty_header)
_, _, installed, _, runtime, _, journal, shadow = values
historical = installed.read_bytes()
if failure:
_write(journal / "meta", (journal / "meta").read_text().replace(
f"underlay_{failure}=\n",
f"underlay_{failure}=" + ("203.0.113.1" if failure == "endpoint" else "51820") + "\n",
1,
))
live = tmp_path / "tables" / "inet--virt_cloud_wg_fabric"
original = live.read_bytes()
result = _run(values, tmp_path, FIREWALL_MODE_NATIVE)
assert result.returncode == (0 if failure is None else 79), result.stderr
assert "LEGACY_HELPER_EXECUTED" not in result.stdout + result.stderr
assert installed.read_bytes() == historical
assert (runtime / "wg-fabric-emergency.owner").is_file()
if failure is None:
assert not journal.exists()
assert not live.exists()
assert not shadow.exists()
assert not (tmp_path / "link-present").exists()
else:
assert journal.is_dir()
assert live.read_bytes() == original
assert shadow.is_file()
def test_rendered_outer_replays_same_plan_old_baseline_with_current_helper(tmp_path):
values = _baseline_fixture(tmp_path)
_, current, installed, base, runtime, _, journal, shadow = values
historical = installed.read_bytes()
assert current.read_bytes() != historical
assert "vc_fabric_nm_profile_semantics_match" not in historical.decode()
result = _run(values, tmp_path)
assert result.returncode == 0, result.stderr
assert "LEGACY_HELPER_EXECUTED" not in result.stdout + result.stderr
assert not journal.exists()
# Complete recovery preserves its pre-existing, precisely journaled old
# helper but must not execute it again under --recover-current.
assert installed.read_bytes() == historical
assert not shadow.exists()
assert not (tmp_path / "link-present").exists()
assert not (base / "wg-fabric0.owner").exists()
assert not (tmp_path / "tables" / "inet--virt_cloud_wg_fabric").exists()
assert (runtime / "wg-fabric-emergency.owner").is_file()
for family in ("inet", "bridge"):
assert (tmp_path / "tables" / f"{family}--virt_cloud_fabric_emergency").is_file()
calls = [json.loads(line) for line in (tmp_path / "calls").read_text().splitlines()]
assert not any(call[0] == "systemctl" and call[1] in ("start", "enable") for call in calls)
assert not any(call[:5] == ["ip", "link", "set", "dev", "wg-fabric0"]
and call[-1] == "up" for call in calls)
def test_normal_boot_recovery_still_reenters_verified_historical_helper(tmp_path):
values = _baseline_fixture(tmp_path)
_, current, installed, _, runtime, _, journal, _ = values
historical = installed.read_bytes()
result = subprocess.run(
["/bin/sh", str(current)], check=False, capture_output=True, text=True,
env={"VC_ENTRY_ROOT": str(tmp_path), "VC_FABRIC_LOCK_HELD": "1"}, timeout=45,
)
# This complete non-recovery-only entry still uses the restored old
# implementation. Its deliberate exit99 proves the normal exec contract
# was preserved instead of silently treating a regular boot as finished.
assert result.returncode == 99, result.stderr
assert "LEGACY_HELPER_EXECUTED" in result.stderr
assert not journal.exists()
assert installed.read_bytes() == historical
assert (runtime / "wg-fabric-emergency.owner").is_file()
@pytest.mark.parametrize("failure", [
"foreign_helper", "baseline_and_desired_drift", "temporary_mode", "temporary_digest",
"endpoint", "port", "runtime_key", "live_nft_verdict",
])
def test_rendered_same_plan_handoff_rejects_unproved_ownership_or_plan(tmp_path, failure):
values = _baseline_fixture(tmp_path)
plan, current, installed, _, runtime, _, journal, shadow = values
if failure == "foreign_helper":
_write(installed, installed.read_text() + "# external replacement\n", 0o700)
elif failure == "baseline_and_desired_drift":
for name in ("helper", "helper.desired"):
_write(journal / name, (journal / name).read_text() + "# changed snapshot\n", 0o700)
elif failure == "temporary_mode":
current.chmod(0o600)
elif failure == "temporary_digest":
_write(current, current.read_text().replace(plan.digest, "a" * 64), 0o700)
elif failure in ("endpoint", "port"):
old, new = (("underlay_endpoint=203.0.113.1", "underlay_endpoint=203.0.113.9")
if failure == "endpoint" else ("underlay_port=51820", "underlay_port=51821"))
_write(journal / "meta", (journal / "meta").read_text().replace(old, new))
elif failure == "runtime_key":
old = base64.b64encode(bytes([7]) * 32).decode()
_write(shadow, shadow.read_text().replace(old, base64.b64encode(bytes([8]) * 32).decode()))
else:
live = tmp_path / "tables" / "inet--virt_cloud_wg_fabric"
_write(live, live.read_text().replace("counter drop comment", "counter accept comment", 1))
live = tmp_path / "tables" / "inet--virt_cloud_wg_fabric"
original = live.read_bytes()
result = _run(values, tmp_path)
assert result.returncode == 79, result.stderr
assert "LEGACY_HELPER_EXECUTED" not in result.stdout + result.stderr
assert journal.is_dir()
assert installed.is_file()
assert live.read_bytes() == original
assert (runtime / "wg-fabric-emergency.owner").is_file()
calls = [json.loads(line) for line in (tmp_path / "calls").read_text().splitlines()]
assert not any(call[0] == "nft" and call[1] == "-f" for call in calls)
@pytest.mark.parametrize("field,value", [
("plan_digest", "f" * 64),
("connection_uuid", "ffffffff-ffff-ffff-ffff-ffffffffffff"),
("deployment_mode", "ifupdown"),
("firewall_mode", "native_nftables"),
])
def test_different_plan_keeps_historical_recovery_contract_without_current_handoff(
tmp_path, field, value):
values = _baseline_fixture(tmp_path)
_, _, _, _, runtime, _, journal, _ = values
text = (journal / "meta").read_text()
_write(journal / "meta", re.sub(rf"^{field}=.*$", f"{field}={value}", text,
flags=re.MULTILINE))
before = (tmp_path / "tables" / "inet--virt_cloud_wg_fabric").read_bytes()
result = _run(values, tmp_path)
assert result.returncode == 79, result.stderr
# This controlled historical helper rejects at exit99. Existing cross-
# revision dispatch stays intact, and the current template never assumes
# authority over the other journal identity.
assert "LEGACY_HELPER_EXECUTED" in result.stderr
assert "helper_state=failed" not in result.stderr
assert journal.is_dir()
assert (tmp_path / "tables" / "inet--virt_cloud_wg_fabric").read_bytes() == before
assert (runtime / "wg-fabric-emergency.owner").is_file()

@ -1500,6 +1500,7 @@ operation_interrupted() {{
trap operation_interrupted HUP INT TERM
/bin/sh "$UPDATE" running dispatch none || exit 1
export VC_FABRIC_OPERATION_ID="$OPERATION_ID"
export VC_FABRIC_SAFE_LOG=1
if /bin/sh "$APPLY"; then
/bin/sh "$UPDATE" succeeded complete none || exit 1
exit 0
@ -1962,6 +1963,104 @@ table inet {FABRIC_NFT_TABLE} {{
'''
_FABRIC_WG_READBACK_SH = r'''vc_fabric_wg_peers_exact() {
vc_wg_live_peers=$1
vc_wg_expected_peers=$2
vc_wg_live_size=$(printf '%s' "$vc_wg_live_peers" | wc -c | tr -d '[:space:]') || return 1
[ "$vc_wg_live_size" -le 262144 ] 2>/dev/null || return 1
# wg(8) emits one public key per line. Compare complete records as an
# unordered, duplicate-free set; a space-delimited shell pattern cannot
# recognize keys separated by the command's internal newlines.
printf '%s\n' "$vc_wg_live_peers" | awk -v expected="$vc_wg_expected_peers" '
function key(value) {
return length(value)==44 && value ~ /^[A-Za-z0-9+\/]+=$/
}
BEGIN {
n=split(expected, values, /[[:space:]]+/)
for (i=1; i<=n; i++) {
if (values[i]=="") continue
if (!key(values[i]) || wanted[values[i]]++) bad=1
expected_count++
}
if (expected_count<1 || expected_count>63) bad=1
}
NF {
if (NF!=1 || !key($1) || !wanted[$1] || seen[$1]++) bad=1
actual_count++
}
END {exit (bad || actual_count!=expected_count) ? 1 : 0}
'
}
vc_fabric_wg_allowed_exact() {
vc_wg_live_allowed=$1
vc_wg_wanted_key=$2
vc_wg_expected_allowed=$3
vc_wg_overlay_allowed=$4
vc_wg_expected_keys=$5
vc_wg_live_size=$(printf '%s' "$vc_wg_live_allowed" | wc -c | tr -d '[:space:]') || return 1
[ "$vc_wg_live_size" -le 262144 ] 2>/dev/null || return 1
# The machine-readable `wg show IFACE allowed-ips` uses spaces between
# prefixes (unlike the comma-separated set-command input). Validate the
# full peer table and every prefix of the selected row, not merely $2.
printf '%s\n' "$vc_wg_live_allowed" | awk \
-v selected="$vc_wg_wanted_key" -v expected="$vc_wg_expected_allowed" \
-v overlay="$vc_wg_overlay_allowed" -v peers="$vc_wg_expected_keys" '
function key(value) {
return length(value)==44 && value ~ /^[A-Za-z0-9+\/]+=$/
}
function host_prefix(value, parts, octets, count, pos) {
count=split(value, parts, "/")
if (count!=2 || parts[2]!="32" || split(parts[1], octets, ".")!=4)
return 0
for (pos=1; pos<=4; pos++)
if (octets[pos] !~ /^(0|[1-9][0-9]*)$/ || octets[pos]>255) return 0
return 1
}
function allowed_prefix(value, parts, octets) {
if (!host_prefix(value)) return 0
if (value==overlay) return 1
split(value, parts, "/"); split(parts[1], octets, ".")
return (octets[1]==10 ||
(octets[1]==172 && octets[2]>=16 && octets[2]<=31) ||
(octets[1]==192 && octets[2]==168))
}
BEGIN {
n=split(peers, peer_values, /[[:space:]]+/)
for (i=1; i<=n; i++) {
if (peer_values[i]=="") continue
if (!key(peer_values[i]) || peer_wanted[peer_values[i]]++) bad=1
peer_count++
}
if (peer_count<1 || peer_count>63 || !peer_wanted[selected] ||
!host_prefix(overlay)) bad=1
split(overlay, overlay_parts, "/"); split(overlay_parts[1], overlay_octets, ".")
if (overlay_octets[1]!=100 || overlay_octets[2]<64 || overlay_octets[2]>127) bad=1
n=split(expected, prefix_values, ",")
for (i=1; i<=n; i++) {
value=prefix_values[i]
if (!allowed_prefix(value) || prefix_wanted[value]++) bad=1
expected_count++
}
if (expected_count<1 || expected_count>16384 || prefix_wanted[overlay]!=1) bad=1
}
NF {
if (NF<2 || !key($1) || !peer_wanted[$1] || peer_seen[$1]++) bad=1
actual_peers++
for (i=2; i<=NF; i++) {
if (!host_prefix($i) || prefix_seen[$1, $i]++) bad=1
if ($1==selected) {
if (!allowed_prefix($i) || !prefix_wanted[$i]) bad=1
actual_count++
}
}
}
END {exit (bad || actual_peers!=peer_count || peer_seen[selected]!=1 ||
actual_count!=expected_count) ? 1 : 0}
'
}
'''.strip()
_FABRIC_ROUTE_VALIDATION_AWK = r'''
function canonical_host_prefix(value, octet, count, pos) {
# iproute2 omits /32 for host routes. Canonicalize only an exact IPv4
@ -3545,6 +3644,13 @@ vc_fabric_helper_failure_log() {
esac
printf 'virt-cloud-fabric helper_state=failed helper_phase=%s exit_status=%s\n' \
"$vc_helper_log_phase" "$vc_helper_log_status" >&2
# The async runner suppresses arbitrary child stdout/stderr because they
# may contain credentials. Only this closed, sanitized record may use its
# inherited journal descriptor; never redirect raw helper output there.
if [ "${VC_FABRIC_SAFE_LOG:-0}" = 1 ]; then
printf 'virt-cloud-fabric helper_state=failed helper_phase=%s exit_status=%s\n' \
"$vc_helper_log_phase" "$vc_helper_log_status" >&3 2>/dev/null || true
fi
}
cleanup() {
vc_helper_exit_status=$?
@ -4028,6 +4134,7 @@ __VC_NMCLI_FILENAME_RESOLVER__
__VC_FIREWALL_MODE_GUARD__
__VC_NFT_MANIFEST_VALIDATION__
__VC_FABRIC_NFT_VALIDATION__
__VC_FABRIC_WG_READBACK__
__VC_VPC_STATE_LOADER__
NMCLI=
IFQUERY=
@ -5515,6 +5622,13 @@ recover_journal() {
exit 0
fi
rm -rf "$JOURNAL" || exit 79
# Recovery-only has now restored and verified the complete journaled
# baseline. A current temporary helper must not redispatch the obsolete
# baseline implementation whose compatibility defect prompted recovery.
# Ordinary boot recovery still re-enters the restored helper as before.
if [ "$RECOVER_ONLY" = yes ]; then
exit 0
fi
exec /bin/sh "$HELPER" "$@"
}
if [ "${VC_FABRIC_APPLYING:-0}" != 1 ]; then
@ -6697,12 +6811,16 @@ VC_FABRIC_HELPER_PHASE=vpc
recover_vpc_dataplanes
VC_FABRIC_HELPER_PHASE=final
actual_peers=$("$WG" show "$IFACE" peers 2>/dev/null) || exit 78
for actual_peer in $actual_peers; do
case " $seen_peers " in *" $actual_peer "*) : ;; *) exit 78 ;; esac
done
for expected_peer in $seen_peers; do
case " $actual_peers " in *" $expected_peer "*) : ;; *) exit 78 ;; esac
done
vc_fabric_wg_peers_exact "$actual_peers" "$seen_peers" || exit 78
actual_allowed_ips=$("$WG" show "$IFACE" allowed-ips 2>/dev/null) || exit 78
while IFS='|' read -r final_kind final_host final_peer final_endpoint final_allowed; do
[ "$final_kind" = peer ] || continue
final_vpc_extra=$(vpc_state_allowed_for_peer "$final_peer") || exit 78
final_expected_allowed=$final_allowed
[ -z "$final_vpc_extra" ] || final_expected_allowed="$final_expected_allowed,$final_vpc_extra"
vc_fabric_wg_allowed_exact "$actual_allowed_ips" "$final_peer" \
"$final_expected_allowed" "$final_allowed" "$seen_peers" || exit 78
done < "$CONF"
actual_fabric_routes=$("$IP" -o route show table all dev "$IFACE" 2>/dev/null) || exit 78
route_extra=$(printf '%s\n' "$actual_fabric_routes" | awk -v allowed="$allowed_routes" -v own="$overlay" -v vpc_file="$VPC_ROUTE_FILE" '
__VC_FABRIC_ROUTE_VALIDATION_AWK__
@ -7390,6 +7508,57 @@ transaction_recovery_helper_owned() {
fi
return 1
}
transaction_recovery_same_current_plan() {
# A historical helper is authoritative only for the journaled pre/post
# image proved above. Replaying that exact plan with today's temporary
# helper is safe, and avoids redispatching an obsolete implementation
# which cannot understand its own NetworkManager in-memory shadow. A
# different revision still uses the existing historical-helper contract;
# never transfer its authority to this apply's new plan.
transaction_current_meta="$BASE/.wg-fabric-txn/meta"
[ -f "$transaction_current_meta" ] &&
[ ! -L "$transaction_current_meta" ] &&
[ "$($STAT -c '%u:%a' "$transaction_current_meta" 2>/dev/null)" = 0:600 ] || return 1
awk -F= -v wanted_digest="$EXPECTED_DIGEST" \
-v wanted_uuid="$CONNECTION_UUID" -v wanted_mode="$DEPLOYMENT_MODE" \
-v wanted_firewall="$FIREWALL_MODE" '
$1=="schema" {schema++; if (substr($0,index($0,"=")+1)!="1") bad=1; next}
$1=="plan_digest" {digest++; if (substr($0,index($0,"=")+1)!=wanted_digest) bad=1; next}
$1=="connection_uuid" {uuid++; if (substr($0,index($0,"=")+1)!=wanted_uuid) bad=1; next}
$1=="deployment_mode" {mode++; if (substr($0,index($0,"=")+1)!=wanted_mode) bad=1; next}
$1=="firewall_mode" {firewall++; if (substr($0,index($0,"=")+1)!=wanted_firewall) bad=1; next}
$1=="underlay_endpoint" {endpoint++; next}
$1=="underlay_port" {port++; next}
END {
exit (bad || schema!=1 || digest!=1 || uuid!=1 || mode!=1 ||
firewall!=1 || endpoint!=1 || port!=1) ? 1 : 0
}
' "$transaction_current_meta" >/dev/null
}
transaction_recovery_current_underlay_matches() {
# These public constants are already rendered into the guarded current
# helper. The outer apply has not yet reached underlay preflight, so do
# not read its uninitialized mutable UNDERLAY_* variables or eval files.
if [ "$FIREWALL_MODE" = native_nftables ]; then
transaction_current_endpoint=
transaction_current_port=
else
transaction_current_endpoint=$(awk -F'"' '
/^UNDERLAY_ENDPOINT="[0-9.]+"$/ {value=$2; count++}
END {if (count==1) print value; else exit 1}
' "$_tmp_helper") || return 1
transaction_current_port=$(awk -F'"' '
/^UNDERLAY_PORT="[0-9]+"$/ {value=$2; count++}
END {if (count==1) print value; else exit 1}
' "$_tmp_helper") || return 1
fi
awk -F= -v wanted_endpoint="$transaction_current_endpoint" \
-v wanted_port="$transaction_current_port" '
$1=="underlay_endpoint" {endpoint++; if (substr($0,index($0,"=")+1)!=wanted_endpoint) bad=1}
$1=="underlay_port" {port++; if (substr($0,index($0,"=")+1)!=wanted_port) bad=1}
END {exit (bad || endpoint!=1 || port!=1) ? 1 : 0}
' "$transaction_current_meta" >/dev/null
}
if [ -L "$BASE" ] || { [ -e "$BASE" ] && [ ! -d "$BASE" ]; }; then exit 71; fi
TRANSACTION_RECOVERED=no
if [ -e "$BASE/.wg-fabric-txn" ]; then
@ -7502,9 +7671,28 @@ if [ -e "$BASE/.wg-fabric-txn" ]; then
fi
else
transaction_recovery_helper_owned || exit 79
grep -Fq -- '--recover-only' "$HELPER" || exit 79
vc_fabric_operation_checkpoint journal_dispatch
VC_FABRIC_LOCK_HELD=1 /bin/sh "$HELPER" --recover-only || exit 79
if transaction_recovery_same_current_plan; then
# Keep the exact historical helper/journal ownership
# proof, but run the current implementation only after
# all plan and manager/firewall identity fields match.
# The child repeats the full journal, artifact, nft and
# live ownership validation before restoring anything.
[ -n "${_tmp_helper:-}" ] &&
[ -f "$_tmp_helper" ] && [ ! -L "$_tmp_helper" ] &&
[ "$($STAT -c '%u:%a' "$_tmp_helper" 2>/dev/null)" = 0:700 ] || exit 79
[ "$(sed -n '1p' "$_tmp_helper")" = '#!/bin/sh' ] &&
[ "$(sed -n '2p' "$_tmp_helper")" = '# virt-cloud-wg-fabric-v1' ] &&
[ "$(sed -n '3p' "$_tmp_helper")" = "# fabric-plan-digest=$EXPECTED_DIGEST" ] &&
[ "$(sed -n '4p' "$_tmp_helper")" = "PLAN_DIGEST=\"$EXPECTED_DIGEST\"" ] || exit 79
grep -Fq -- '--recover-current' "$_tmp_helper" || exit 79
transaction_recovery_current_underlay_matches || exit 79
vc_fabric_operation_checkpoint journal_dispatch
VC_FABRIC_LOCK_HELD=1 /bin/sh "$_tmp_helper" --recover-current || exit 79
else
grep -Fq -- '--recover-only' "$HELPER" || exit 79
vc_fabric_operation_checkpoint journal_dispatch
VC_FABRIC_LOCK_HELD=1 /bin/sh "$HELPER" --recover-only || exit 79
fi
fi
[ ! -e "$BASE/.wg-fabric-txn" ] || exit 79
TRANSACTION_RECOVERED=yes
@ -7701,6 +7889,7 @@ sync
JOURNAL_READY=yes
rollback() {
rollback_status=$?
vc_fabric_apply_failure_log "$rollback_status" || true
trap - EXIT HUP INT TERM
set +e
set +u
@ -8429,6 +8618,7 @@ def _apply_script(plan: FabricPlan, host: FabricNode,
"__VC_FIREWALL_MODE_GUARD__", _FIREWALL_MODE_GUARD_SH).replace(
"__VC_NFT_MANIFEST_VALIDATION__", _NFT_MANIFEST_VALIDATION_SH).replace(
"__VC_FABRIC_NFT_VALIDATION__", _FABRIC_NFT_VALIDATION_SH).replace(
"__VC_FABRIC_WG_READBACK__", _FABRIC_WG_READBACK_SH).replace(
"__VC_FABRIC_SYSCTL_VALIDATION__", _FABRIC_SYSCTL_VALIDATION_SH)
manager_exec = (
"/bin/sh /usr/local/libexec/virt-cloud-wg-fabric --ifupdown-service"
@ -8752,6 +8942,24 @@ set -eu
umask 077
PATH=/usr/sbin:/usr/bin:/sbin:/bin
export PATH
VC_FABRIC_APPLY_DETAIL=entry
vc_fabric_apply_failure_log() {{
vc_apply_log_status=$1
case "$vc_apply_log_status" in ''|*[!0-9]*) vc_apply_log_status=1 ;; esac
[ "$vc_apply_log_status" -le 255 ] 2>/dev/null || vc_apply_log_status=1
[ "$vc_apply_log_status" != 0 ] || return 0
case "${{VC_FABRIC_APPLY_DETAIL:-}}" in
entry|final_acl|final_tenants|final_identity|final_link|final_addresses|final_peers|final_endpoints|final_allowed_ips|final_routes|final_overlay_paths|final_zone|commit|convergence|complete)
vc_apply_log_detail=$VC_FABRIC_APPLY_DETAIL ;;
*) vc_apply_log_detail=unknown ;;
esac
# Emit before rollback so a secondary helper/service failure cannot hide
# the first failing check. FD 3 belongs to the async runner, not a raw log.
[ "${{VC_FABRIC_SAFE_LOG:-0}}" = 1 ] || return 0
printf 'virt-cloud-fabric apply_state=failed apply_check=%s exit_status=%s\\n' \
"$vc_apply_log_detail" "$vc_apply_log_status" >&3 2>/dev/null || true
return 0
}}
vc_fabric_operation_checkpoint() {{
vc_operation_checkpoint=$1
[ -n "${{VC_FABRIC_OPERATION_ID:-}}" ] || return 0
@ -8834,6 +9042,7 @@ FLOCK=$(find_tool /usr/bin/flock /usr/sbin/flock /bin/flock /sbin/flock) || exit
{_FIREWALL_MODE_GUARD_SH}
{_NFT_MANIFEST_VALIDATION_SH}
{_FABRIC_NFT_VALIDATION_SH}
{_FABRIC_WG_READBACK_SH}
{_FABRIC_SYSCTL_VALIDATION_SH}
PREFLIGHT_IFACE_BEFORE=no
PREFLIGHT_IFACE_UP_BEFORE=no
@ -9350,52 +9559,54 @@ for gate_unit in {' '.join(FABRIC_LIBVIRT_GATE_UNITS)}; do
done
"$SYSTEMCTL" enable {FABRIC_SERVICE} >/dev/null
"$SYSTEMCTL" enable {FABRIC_TIMER} >/dev/null
"$SYSTEMCTL" start {FABRIC_TIMER} >/dev/null
vc_fabric_operation_checkpoint final_readback
if [ "$FIREWALL_MODE" = firewalld_nft ] ||
[ -n "${{VC_FABRIC_EMERGENCY_SNAPSHOT:-}}" ]; then
VC_FABRIC_APPLY_DETAIL=final_acl
final_fabric_acl=$($NFT -s list table inet virt_cloud_wg_fabric 2>/dev/null) || exit 72
nft_plan_owned "$final_fabric_acl" "$EXPECTED_DIGEST" || exit 71
VC_FABRIC_APPLY_DETAIL=final_tenants
vpc_state_nft_set_matches || exit 71
VC_FABRIC_APPLY_DETAIL=final_identity
"$WG" show "$IFACE" >/dev/null 2>&1 || exit 71
final_public_key=$($WG pubkey < "$KEY") || exit 71
[ "$($WG show "$IFACE" public-key 2>/dev/null)" = "$final_public_key" ] || exit 71
[ "$($WG show "$IFACE" listen-port 2>/dev/null)" = {_q(str(host.listen_port))} ] || exit 71
VC_FABRIC_APPLY_DETAIL=final_link
final_link=$($IP -o link show dev "$IFACE" 2>/dev/null) || exit 71
printf '%s\n' "$final_link" | awk -F'[<>]' 'NF >= 3 {{
count=split($2, flags, ",")
for (i=1; i<=count; i++) if (flags[i] == "UP") found=1
}} END {{exit found ? 0 : 1}}' || exit 71
VC_FABRIC_APPLY_DETAIL=final_addresses
final_addresses=$($IP -o -4 address show dev "$IFACE" 2>/dev/null) || exit 71
printf '%s\n' "$final_addresses" | awk -v wanted={_q(host.overlay_ip)} '
NF {{split($4, address, "/"); count++; if (address[1] != wanted || address[2] != 32) bad=1}}
END {{exit (!bad && count == 1) ? 0 : 1}}
' || exit 71
VC_FABRIC_APPLY_DETAIL=final_peers
final_actual_peers=$($WG show "$IFACE" peers 2>/dev/null) || exit 71
for final_actual_peer in $final_actual_peers; do
case " $MANAGED_PEERS " in *" $final_actual_peer "*) : ;; *) exit 71 ;; esac
done
for final_expected_peer in $MANAGED_PEERS; do
case " $final_actual_peers " in *" $final_expected_peer "*) : ;; *) exit 71 ;; esac
done
vc_fabric_wg_peers_exact "$final_actual_peers" "$MANAGED_PEERS" || exit 71
VC_FABRIC_APPLY_DETAIL=final_endpoints
final_endpoints=$($WG show "$IFACE" endpoints 2>/dev/null) || exit 71
VC_FABRIC_APPLY_DETAIL=final_allowed_ips
final_allowed_ips=$($WG show "$IFACE" allowed-ips 2>/dev/null) || exit 71
while IFS='|' read -r final_kind final_host final_peer final_endpoint final_allowed; do
[ "$final_kind" = peer ] || continue
final_vpc_extra=$(vpc_state_allowed_for_peer "$final_peer") || exit 71
final_expected_allowed=$final_allowed
[ -z "$final_vpc_extra" ] || final_expected_allowed="$final_expected_allowed,$final_vpc_extra"
VC_FABRIC_APPLY_DETAIL=final_endpoints
final_live_endpoint=$(printf '%s\n' "$final_endpoints" | awk -v wanted="$final_peer" '
$1 == wanted {{value=$2; count++}}
END {{if (count == 1) print value; else exit 1}}
') || exit 71
[ "$final_live_endpoint" = "$final_endpoint" ] || exit 71
final_live_allowed=$(printf '%s\n' "$final_allowed_ips" | awk -v wanted="$final_peer" '
$1 == wanted {{value=$2; count++}}
END {{if (count == 1) print value; else exit 1}}
') || exit 71
[ "$final_live_allowed" = "$final_expected_allowed" ] || exit 71
VC_FABRIC_APPLY_DETAIL=final_allowed_ips
vc_fabric_wg_allowed_exact "$final_allowed_ips" "$final_peer" \
"$final_expected_allowed" "$final_allowed" "$MANAGED_PEERS" || exit 71
done < "$CONF"
VC_FABRIC_APPLY_DETAIL=final_routes
final_fabric_routes=$($IP -o route show table all dev "$IFACE" 2>/dev/null) || exit 71
final_route_extra=$(printf '%s\n' "$final_fabric_routes" | awk \
-v allowed="$MANAGED_ROUTES" -v own={_q(host.overlay_ip)} \
@ -9443,8 +9654,10 @@ if [ "$FIREWALL_MODE" = firewalld_nft ] ||
}}')
[ "$final_route_seen" = 1 ] || exit 71
done
VC_FABRIC_APPLY_DETAIL=final_overlay_paths
{path_checks}
if [ "$FIREWALL_MODE" = firewalld_nft ]; then
VC_FABRIC_APPLY_DETAIL=final_zone
[ "$(vc_firewall_zone permanent)" = none ] || exit 71
[ "$(vc_firewall_zone runtime)" = trusted ] || exit 71
fi
@ -9453,6 +9666,7 @@ if [ "$FIREWALL_MODE" = firewalld_nft ] ||
# which restores and proves every VPC before it alone removes the guard.
:
fi
VC_FABRIC_APPLY_DETAIL=commit
_txn_state_tmp=$(mktemp "$TXN/.state.XXXXXX")
printf '%s\n' committed > "$_txn_state_tmp"
chmod 600 "$_txn_state_tmp"
@ -9460,6 +9674,11 @@ mv -f "$_txn_state_tmp" "$TXN/state"
_txn_state_tmp=
sync
COMMITTED=yes
# OnBootSec may already be elapsed and enqueue repair immediately. Start the
# timer only after durable commit, so a failed pre-commit readback/rollback
# cannot race a queued primary helper into the restored absent generator.
VC_FABRIC_APPLY_DETAIL=convergence
"$SYSTEMCTL" start {FABRIC_TIMER} >/dev/null 2>&1 || exit 79
# Release the transaction flock only after durable commit, then synchronously
# run the independent dispatcher. It either waits for an inactive primary or,
# when primary is already active, executes the complete helper to restore and
@ -9509,6 +9728,7 @@ printf '%s\n' "$final_emergency_tables" |
final_emergency_tables_verify=$($NFT list tables 2>/dev/null) || exit 79
[ "$final_emergency_tables_verify" = "$final_emergency_tables" ] || exit 79
[ ! -e "$final_emergency_owner" ] && [ ! -L "$final_emergency_owner" ] || exit 79
VC_FABRIC_APPLY_DETAIL=complete
vc_fabric_operation_checkpoint complete
printf '%s\\n' '__VC_FABRIC_APPLIED__'
'''

Loading…
Cancel
Save