From c12e8d7b62f7dc894cf900f1d4cae23622f2172a Mon Sep 17 00:00:00 2001 From: admin Date: Fri, 9 Oct 2026 10:17:41 +0800 Subject: [PATCH] Fix real host Fabric sysctl and first deploy recovery --- docs/VALIDATION.md | 22 + .../tests/test_fabric_first_deploy_entry.py | 401 ++++++++++++++++++ engine/tests/test_fabric_proc_sysctl.py | 205 +++++++++ engine/virsh_panel/fabric.py | 64 +-- 4 files changed, 666 insertions(+), 26 deletions(-) create mode 100644 engine/tests/test_fabric_first_deploy_entry.py create mode 100644 engine/tests/test_fabric_proc_sysctl.py diff --git a/docs/VALIDATION.md b/docs/VALIDATION.md index 8104d439..4546c274 100644 --- a/docs/VALIDATION.md +++ b/docs/VALIDATION.md @@ -1,5 +1,27 @@ # 本版验证记录 +## 2026-10-09:接管真实 SSH 后的 proc 读取与首次部署入口修复 + +- 在用户指定的 electerm Debian 13 会话真实复现:`cat /proc/sys/net/ipv4/ip_forward` + 为 1,而 dash 的逐字节 `IFS= read -r` 返回 1 且变量已是 `1`。整数 proc sysctl + 在非零读取偏移直接 EOF;原脚本把这误判为 sysctl 不兼容。apply、boot、observe 和 + rp_filter 写后回读改用缓冲读取并保留末尾换行,仅接受精确单行 0/1/2;I/O 失败、 + 空值、额外换行、多行及畸形值继续拒绝,ip_forward 必须等于 1 的门禁保留。 +- AlmaLinux 10 真实会话确认 SELinux Disabled、脚本所在文件系统无 noexec,旧 journal + 为 ready/installing,generator 与 timer 的基线为 absent。完整渲染 helper 测试进一步 + 发现入口提前展开未定义 config_fabric/network、过早要求未发布的 gate generator、 + timer.absent 分支缺少 TIMER_UNIT 声明。现用已绑定的 plan 常量初始化 loader,常规 + manifest 精确匹配;恢复-only 完整 journal/plan/nft 证明后可退出,正常启动仍校验 gate。 + 恢复全程保留 emergency 隔离,不会提前启动 libvirt。 +- 分批回归:Fabric 既有四套用例 448 项通过,新增 proc 读取与完整 helper 入口 100 项 + 通过;9 项真实 Linux /proc 用例在本地 Darwin 跳过,由上方 SSH 复现补充现场证据。 + 联合首轮中的两个尚在编辑的新 manifest 测试已由稳定文件完整重跑(10/10)通过。 + Engine boundary/VPC/dataplane 278 passed、23 warnings;py_compile、diff 检查与独立 + 安全审查通过。成功/外部 gate/错 plan/普通启动/manifest 偏移均走完整 helper 控制流, + 不注入 TIMER_UNIT/config_fabric/network 来掩盖入口缺陷。 +- 代码回归后通过管理员在线更新部署,再由同一地区的新 operation 验证旧事务收敛和 + WireGuard 回读;组网验收以实际页面、宿主 journal 和双向 overlay 流量为准。 + ## 2026-10-09:Debian 13 / AlmaLinux 10 Fabric 启动与恢复核对 - 按 Debian 13 的 systemd/ifupdown 文档与源码、RHEL 10 的 systemd 文档核对生成脚本。 diff --git a/engine/tests/test_fabric_first_deploy_entry.py b/engine/tests/test_fabric_first_deploy_entry.py new file mode 100644 index 00000000..7ed57583 --- /dev/null +++ b/engine/tests/test_fabric_first_deploy_entry.py @@ -0,0 +1,401 @@ +"""Exercise the rendered recovery helper from its real shell entry point.""" +from __future__ import annotations + +import base64 +import json +import re +import shlex +import subprocess +import sys +from pathlib import Path + +import pytest + +from virsh_panel.fabric import ( + FIREWALL_MODE_FIREWALLD, + _apply_script, + _nft_rules_text, + _sysctl_text, + parse_fabric_plan, +) + + +def _write(path: Path, content: str, mode: int = 0o600) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + path.chmod(mode) + + +def _emergency_table(family: str, token: str) -> str: + rows = [f"table {family} virt_cloud_fabric_emergency {{"] + for chain in ("input", "output", "forward"): + rows.extend((f" chain {chain} {{", ( + f" type filter hook {chain} priority 300; policy accept;" + ))) + if family == "inet": + matches = [] + if chain in ("input", "forward"): + matches.extend(('iifname "wg-fabric0"', 'iifname "vcv-*"')) + if chain in ("output", "forward"): + matches.extend(('oifname "wg-fabric0"', 'oifname "vcv-*"')) + else: + matches = [] + if chain in ("input", "forward"): + matches.append('meta ibrname "vcv-*"') + if chain in ("output", "forward"): + matches.append('meta obrname "vcv-*"') + for match in matches: + rows.append( + f' {match} counter drop comment ' + f'"virt-cloud-fabric-emergency-v2:{token}"' + ) + rows.append(" }") + rows.append("}") + return "\n".join(rows) + "\n" + + +_TOOLS = r'''import hashlib +import json +import os +import stat +import sys +from pathlib import Path + +name = Path(sys.argv[0]).name +args = sys.argv[1:] +root = Path(os.environ["VC_ENTRY_ROOT"]) +with (root / "calls").open("a", encoding="utf-8") as log: + log.write(json.dumps([name, *args]) + "\n") + +if name == "stat": + if len(args) != 3 or args[0] != "-c": + raise SystemExit(2) + value = os.stat(args[2]) + mode = format(stat.S_IMODE(value.st_mode), "o") + fields = { + "%u": "0", "%a": mode, "%u:%a": "0:" + mode, + "%u:%g:%a": "0:0:" + mode, "%s": str(value.st_size), + } + if args[1] not in fields: + raise SystemExit(2) + print(fields[args[1]]) +elif name == "sha256sum": + for filename in args: + print(hashlib.sha256(Path(filename).read_bytes()).hexdigest(), filename) +elif name == "ip": + # The interrupted first deployment never reached link creation. + raise SystemExit(1) +elif name == "nft": + table_root = root / "tables" + if args == ["-j", "list", "table", "inet", "firewalld"]: + print('{"nftables":[{"table":{"family":"inet",' + '"name":"firewalld","flags":["owner","persist"]}}]}') + elif args == ["list", "tables"]: + for path in sorted(table_root.glob("*")): + family, table = path.name.split("--", 1) + print("table", family, table) + elif len(args) in (4, 5) and args[-4:-2] == ["list", "table"]: + path = table_root / (args[-2] + "--" + args[-1]) + if not path.is_file(): + raise SystemExit(1) + print(path.read_text(encoding="utf-8"), end="") + elif len(args) in (2, 3) and args[-2] == "-f": + batch = Path(args[-1]).read_text(encoding="utf-8") + # Recovery may delete only the completely proved interrupted Fabric + # table. Releasing emergency isolation is a test failure. + if batch != "delete table inet virt_cloud_wg_fabric\n": + raise SystemExit(2) + if "-c" not in args: + (table_root / "inet--virt_cloud_wg_fabric").unlink() + else: + raise SystemExit(2) +elif name == "systemctl": + if args[:2] == ["show", "-p"]: + if args[2] == "LoadState": + print("loaded" if args[-1] in ("firewalld", "nftables") + else "not-found") + elif args[2] == "ActiveState": + print("inactive") + else: + raise SystemExit(2) + elif args[0] == "is-active" and args[1] in ("firewalld", "nftables"): + print("active" if args[1] == "firewalld" else "inactive") + elif args[0] == "is-enabled" and args[1] in ("firewalld", "nftables"): + print("enabled" if args[1] == "firewalld" else "disabled") + elif args and args[0] in ("daemon-reload", "disable"): + pass + else: + raise SystemExit(2) +elif name == "firewall-cmd": + if args == ["--state"]: + print("running") + elif "--get-zones" in args: + print("public trusted") + elif any(arg.startswith("--query-interface=") for arg in args): + raise SystemExit(1) + elif any(arg.startswith("--query-rich-rule=") for arg in args): + raise SystemExit(1) + else: + raise SystemExit(2) +elif name == "nmcli": + if args == ["-t", "--escape", "no", "-g", "connection.uuid", + "connection", "show", "virt-cloud-wg-fabric0"]: + pass + else: + raise SystemExit(2) +elif name in ("wg", "bridge", "flock", "sync", "ifup", "ifquery", "ifdown"): + pass +else: + raise SystemExit(2) +''' + + +def _fixture(tmp_path: Path, mode: str): + plan = parse_fabric_plan({ + "fabric_id": "entry-fabric", "region": "entry-region", + "overlay_cidr": "100.64.1.0/24", "revision": 1, + "nodes": [ + {"host_id": "a", "overlay_ip": "100.64.1.1", + "endpoint": "203.0.113.1:51820"}, + {"host_id": "b", "overlay_ip": "100.64.1.2", + "endpoint": "203.0.113.2:51820"}, + ], + }) + public_keys = { + "a": base64.b64encode(bytes([1]) * 32).decode(), + "b": base64.b64encode(bytes([2]) * 32).decode(), + } + apply = _apply_script( + plan, plan.nodes[0], public_keys, deployment_mode=mode, + firewall_mode=FIREWALL_MODE_FIREWALLD, + ) + start = apply.index("#!/bin/sh\n# virt-cloud-wg-fabric-v1\n") + end = re.search(r"\n__VC_FABRIC_[0-9a-f]{16}\n", apply[start:]).start() + start + helper = apply[start:end] + "\n" + assert "__VC_FABRIC_MODE__" not in helper + assert "__VC_FABRIC_NFT_RULES_SHA256__" not in helper + + tools = tmp_path / "tools" + tools.mkdir() + tool_path = tools / "external-tool" + _write(tool_path, f"#!{sys.executable}\n" + _TOOLS, 0o700) + for name in ("stat", "sha256sum", "ip", "nft", "systemctl", + "firewall-cmd", "nmcli", "wg", "bridge", "flock", "sync", + "ifup", "ifquery", "ifdown"): + (tools / name).symlink_to(tool_path) + + roots = { + "/etc/virt-cloud": tmp_path / "etc" / "virt-cloud", + "/run/virt-cloud": tmp_path / "run" / "virt-cloud", + "/etc/systemd/system": tmp_path / "units", + "/usr/local/libexec": tmp_path / "libexec", + "/usr/local/lib/systemd/system-generators": tmp_path / "generators", + "/etc/NetworkManager": tmp_path / "NetworkManager", + "/etc/network": tmp_path / "network", + "/etc/sysctl.d": tmp_path / "sysctl.d", + "/etc/nftables.d": tmp_path / "nftables.d", + "/proc/sys/net/ipv4": tmp_path / "proc" / "ipv4", + } + # Only redirect host resources/external tools. Keep all production shell + # control flow and variable declarations, notably TIMER_UNIT, intact. + for source, destination in roots.items(): + helper = helper.replace(source, str(destination)) + destination.mkdir(parents=True, exist_ok=True) + base = roots["/etc/virt-cloud"] + runtime = roots["/run/virt-cloud"] + base.chmod(0o700) + runtime.chmod(0o700) + roots["/etc/sysctl.d"].chmod(0o755) + _write(roots["/proc/sys/net/ipv4"] / "ip_forward", "1\n") + helper = helper.replace( + "PATH=/usr/sbin:/usr/bin:/sbin:/bin", + f"PATH={shlex.quote(str(tools))}:/usr/sbin:/usr/bin:/sbin:/bin", + ) + for name in ("ifup", "ifquery", "ifdown"): + helper = helper.replace(f"/usr/sbin/{name}", str(tools / name)) + for name in ("wg", "ip", "nft", "stat", "sha256sum", "systemctl", + "firewall-cmd", "nmcli", "bridge", "flock"): + helper = re.sub( + rf"find_tool (?:/(?:usr/)?s?bin/{re.escape(name)} ?)+", + f"find_tool {shlex.quote(str(tools / name))} ", + helper, + ) + helper_path = tmp_path / "helper" + _write(helper_path, helper, 0o700) + + tables = tmp_path / "tables" + tables.mkdir() + token = "A" * 24 + _write(runtime / "wg-fabric-emergency.owner", ( + f"virt-cloud-fabric-emergency-v2:{token}\n" + )) + for family in ("inet", "bridge"): + _write(tables / f"{family}--virt_cloud_fabric_emergency", ( + _emergency_table(family, token) + )) + return plan, helper_path, base, runtime, roots + + +def _journal(plan, base: Path, mode: str, helper: Path, root: Path): + journal = base / ".wg-fabric-txn" + journal.mkdir(mode=0o700) + # Read the rendered UUID; the fixture must not invent an independent plan. + uuid = re.search( + r'^CONNECTION_UUID="([^"]+)"$', + helper.read_text(encoding="utf-8"), re.MULTILINE, + ).group(1) + metadata = { + "schema": "1", "deployment_mode": mode, + "firewall_mode": FIREWALL_MODE_FIREWALLD, + "plan_digest": plan.digest, "connection_uuid": uuid, + "nft_main": str(base / "nftables.conf"), + "iface_before": "no", "iface_up_before": "no", + "rp_filter_before": "missing", "nft_before": "no", + "service_enabled_before": "not-found", + "service_active_before": "inactive", + "timer_enabled_before": "not-found", "timer_active_before": "inactive", + "firewalld_runtime_zone_before": "none", + "firewalld_permanent_zone_before": "none", + "underlay_endpoint": "203.0.113.1", "underlay_port": "51820", + "underlay_zone": "public", "underlay_rule_owner": "panel", + "underlay_rule_runtime_before": "no", + "underlay_rule_permanent_before": "no", + "old_underlay_rule_remove": "no", "old_underlay_endpoint": "", + "old_underlay_port": "", "old_underlay_zone": "", + } + _write(journal / "meta", "".join( + f"{key}={value}\n" for key, value in metadata.items() + )) + _write(journal / "state", "ready\n") + _write(journal / "nft.phase", "installing\n") + _write(journal / "nft.table", "") + for name in ("conf", "owner", "helper", "unit", "repair", "timer", + "gate_generator", "nm_profile", "ifupdown_fragment", + "sysctl", "nft_rules", "nft_main"): + _write(journal / f"{name}.absent", "") + _write(journal / "key.present", "") + _write(journal / "key", "test-private-key-snapshot\n") + _write(base / "wg-fabric0.key", "test-private-key-snapshot\n") + desired = _nft_rules_text(plan.digest) + _write(journal / "nft_rules.desired", desired) + _write(root / "tables" / "inet--virt_cloud_wg_fabric", desired) + return journal + + +def _run(helper: Path, root: Path, *args: str): + return subprocess.run( + ["/bin/sh", str(helper), *args], check=False, + capture_output=True, text=True, + env={"VC_ENTRY_ROOT": str(root), "VC_FABRIC_LOCK_HELD": "1"}, + timeout=30, + ) + + +@pytest.mark.parametrize("mode", ["networkmanager_nmcli", "ifupdown"]) +def test_first_deploy_complete_entry_recovers_absent_generator_and_timer( + tmp_path, mode): + plan, helper, base, runtime, roots = _fixture(tmp_path, mode) + journal = _journal(plan, base, mode, helper, tmp_path) + result = _run(helper, tmp_path, "--recover-installing") + assert result.returncode == 0, result.stderr + assert not journal.exists() + assert not (roots["/usr/local/lib/systemd/system-generators"] / + "virt-cloud-wg-fabric-gate").exists() + assert not (roots["/etc/systemd/system"] / + "virt-cloud-wg-fabric-reconcile.timer").exists() + assert not (tmp_path / "tables" / "inet--virt_cloud_wg_fabric").exists() + # Recovery keeps independent isolation; only a complete normal apply may + # activate interfaces or release these tables. + assert (runtime / "wg-fabric-emergency.owner").is_file() + for family in ("inet", "bridge"): + assert (tmp_path / "tables" / + f"{family}--virt_cloud_fabric_emergency").is_file() + calls = [json.loads(line) for line in + (tmp_path / "calls").read_text(encoding="utf-8").splitlines()] + assert ["systemctl", "show", "-p", "LoadState", "--value", + "virt-cloud-wg-fabric-reconcile.timer"] in calls + assert not any(call[0] == "systemctl" and call[1] in + ("start", "enable") for call in calls) + + +@pytest.mark.parametrize("field", ["fabric", "network"]) +def test_normal_complete_entry_rejects_config_outside_rendered_plan(tmp_path, field): + plan, helper, base, runtime, roots = _fixture(tmp_path, "ifupdown") + gate = (roots["/usr/local/lib/systemd/system-generators"] / + "virt-cloud-wg-fabric-gate") + _write(gate, ( + "#!/bin/sh\n# virt-cloud-wg-fabric-v1\n" + "# virt-cloud-wg-fabric-libvirt-gate-v1\n" + f"# fabric-plan-digest={plan.digest}\n" + ), 0o755) + _write(base / "wg-fabric0.owner", "virt-cloud-wg-fabric-v1\n") + _write(roots["/etc/sysctl.d"] / "90-virt-cloud-wg-fabric.conf", + _sysctl_text(plan), 0o644) + fabric = "other-fabric" if field == "fabric" else plan.fabric_id + network = "100.64.2.0/24" if field == "network" else plan.overlay_cidr + overlay = "100.64.2.1" if field == "network" else "100.64.1.1" + peer_ip = "100.64.2.2" if field == "network" else "100.64.1.2" + key = base64.b64encode(bytes([2]) * 32).decode() + _write(base / "wg-fabric0.conf", ( + f"schema|1\nfabric|{fabric}\nrevision|1\ndigest|{plan.digest}\n" + "firewall|firewalld_nft\nunderlay|local|203.0.113.1|eth0|public\n" + "firewall_rule|51820|panel\n" + f"network|{network}\noverlay|{overlay}\nlisten|51820\n" + f"peer|b|{key}|203.0.113.2:51820|{peer_ip}/32\nroute|{peer_ip}/32\n" + )) + result = _run(helper, tmp_path) + assert result.returncode == 72, result.stderr + calls = [json.loads(line) for line in + (tmp_path / "calls").read_text(encoding="utf-8").splitlines()] + assert ["nft", "-j", "list", "table", "inet", "firewalld"] in calls + # Both otherwise syntactically valid configurations fail at their exact + # identity/network assertion, before any route or link activation. + assert not any(call[0] == "ip" and "route" in call for call in calls) + assert (runtime / "wg-fabric-emergency.owner").is_file() + assert not any(call[0] == "systemctl" and call[1] in + ("start", "enable") for call in calls) + + +@pytest.mark.parametrize("failure", ["foreign_gate", "plan_digest"]) +def test_first_deploy_complete_entry_rejects_unproved_artifacts( + tmp_path, failure): + plan, helper, base, runtime, roots = _fixture(tmp_path, "ifupdown") + journal = _journal(plan, base, "ifupdown", helper, tmp_path) + if failure == "foreign_gate": + gate = (roots["/usr/local/lib/systemd/system-generators"] / + "virt-cloud-wg-fabric-gate") + _write(gate, "foreign-generator\n", 0o755) + else: + metadata = (journal / "meta").read_text(encoding="utf-8") + _write(journal / "meta", metadata.replace(plan.digest, "a" * 64)) + before = (tmp_path / "tables" / + "inet--virt_cloud_wg_fabric").read_bytes() + result = _run(helper, tmp_path, "--recover-installing") + assert result.returncode == 79, result.stderr + assert journal.is_dir() + assert (tmp_path / "tables" / + "inet--virt_cloud_wg_fabric").read_bytes() == before + assert (runtime / "wg-fabric-emergency.owner").is_file() + + +@pytest.mark.parametrize("gate_state", ["absent", "insecure", "wrong_digest", "valid"]) +def test_normal_complete_entry_still_validates_generator(tmp_path, gate_state): + plan, helper, base, runtime, roots = _fixture(tmp_path, "ifupdown") + gate = (roots["/usr/local/lib/systemd/system-generators"] / + "virt-cloud-wg-fabric-gate") + if gate_state != "absent": + digest = "a" * 64 if gate_state == "wrong_digest" else plan.digest + _write(gate, ( + "#!/bin/sh\n# virt-cloud-wg-fabric-v1\n" + "# virt-cloud-wg-fabric-libvirt-gate-v1\n" + f"# fabric-plan-digest={digest}\n" + ), 0o700 if gate_state == "insecure" else 0o755) + result = _run(helper, tmp_path) + # The valid case proceeds to the next required owner/sysctl contract. + assert result.returncode == 71, result.stderr + calls = [json.loads(line) for line in + (tmp_path / "calls").read_text(encoding="utf-8").splitlines()] + sysctl_dir_checked = ["stat", "-c", "%u:%a", + str(roots["/etc/sysctl.d"])] in calls + assert sysctl_dir_checked == (gate_state == "valid") + assert (runtime / "wg-fabric-emergency.owner").is_file() diff --git a/engine/tests/test_fabric_proc_sysctl.py b/engine/tests/test_fabric_proc_sysctl.py new file mode 100644 index 00000000..11b6b66f --- /dev/null +++ b/engine/tests/test_fabric_proc_sysctl.py @@ -0,0 +1,205 @@ +"""Exercise the production proc/sys reader with real POSIX shells.""" +from __future__ import annotations + +import base64 +import os +import re +import shlex +import subprocess +from pathlib import Path + +import pytest + +from virsh_panel.fabric import ( + _BOOT_HELPER, + _FABRIC_SYSCTL_VALIDATION_SH, + _apply_script, + _observe_script, + parse_fabric_plan, +) + + +SHELLS = tuple(path for path in ("/bin/sh", "/bin/dash", "/bin/bash") + if os.access(path, os.X_OK)) + + +@pytest.fixture(params=SHELLS) +def shell(request): + return request.param + + +def run_reader(shell, path, prefix="", reader="vc_read_proc_sysctl", + source=_FABRIC_SYSCTL_VALIDATION_SH): + return subprocess.run( + [shell, "-c", prefix + "\n" + source + + f'\n{reader} "$1"\n', "sysctl-reader", str(path)], + text=True, capture_output=True, timeout=5, check=False, + ) + + +@pytest.mark.parametrize("value", ("0", "1", "2")) +def test_buffered_reader_accepts_one_kernel_integer_line(tmp_path, shell, value): + path = tmp_path / "sysctl" + path.write_text(value + "\n", encoding="ascii") + result = run_reader(shell, path) + assert result.returncode == 0, result.stderr + assert result.stdout == value + + +@pytest.mark.parametrize("payload", ( + "", "1", "1\n\n", "1\n2\n", "1\ninvalid\n", " 1\n", "1 \n", + "1\r\n", "3\n", "-1\n", "01\n", "1.0\n", +)) +def test_buffered_reader_rejects_empty_multiline_or_malformed_values( + tmp_path, shell, payload): + path = tmp_path / "sysctl" + path.write_text(payload, encoding="ascii") + result = run_reader(shell, path) + assert result.returncode != 0 + assert result.stdout == "" + + +@pytest.mark.parametrize("target", ("missing", "directory", "partial_io")) +def test_buffered_reader_rejects_io_failure_even_after_a_valid_digit( + tmp_path, shell, target): + path = tmp_path / "sysctl" + prefix = "" + if target == "directory": + path.mkdir() + elif target == "partial_io": + path.write_text("1\n", encoding="ascii") + # A partial read must not become valid merely because stdout already + # contained a valid-looking value before cat reported an I/O error. + prefix = 'cat() { command cat "$@"; return 42; }' + result = run_reader(shell, path, prefix) + assert result.returncode != 0 + assert result.stdout == "" + + +def test_buffered_reader_survives_proc_single_read_eof(tmp_path, shell): + path = tmp_path / "sysctl" + path.write_text("1\n", encoding="ascii") + # Model the observable result of a one-byte shell read from an integer + # proc sysctl: it assigns the digit, then sees EOF before the newline. + # The buffered production reader must not depend on that builtin result. + prefix = '''read() { proc_digit=1; return 1; } +legacy_reader() { + proc_digit= + IFS= read -r proc_digit < "$1" || return 1 + printf '%s' "$proc_digit" +} +''' + legacy = run_reader(shell, path, prefix, "legacy_reader") + assert legacy.returncode != 0 + buffered = run_reader(shell, path, prefix) + assert buffered.returncode == 0, buffered.stderr + assert buffered.stdout == "1" + + +@pytest.fixture(scope="module") +def production_scripts(): + plan = parse_fabric_plan({ + "fabric_id": "proc-test", "region": "hk", "revision": 1, + "overlay_cidr": "100.64.1.0/24", "listen_port": 51820, + "nodes": [ + {"host_id": "a", "overlay_ip": "100.64.1.1", + "endpoint": "203.0.113.1:51820"}, + {"host_id": "b", "overlay_ip": "100.64.1.2", + "endpoint": "203.0.113.2:51820"}, + ], + }) + keys = {node.host_id: base64.b64encode(bytes([index + 1]) * 32).decode() + for index, node in enumerate(plan.nodes)} + return { + "apply": _apply_script(plan, plan.nodes[0], keys), + "boot": _BOOT_HELPER, + "observe": _observe_script(("100.64.1.2",), "100.64.1.1"), + } + + +@pytest.mark.parametrize("script_name", ("apply", "boot", "observe")) +def test_each_production_sysctl_reader_uses_buffered_io( + tmp_path, shell, production_scripts, script_name): + script = production_scripts[script_name] + reader = "read_observed_sysctl" if script_name == "observe" else "read_sysctl" + function = re.search( + rf"^{reader}\(\) \{{\n.*?^\}}", script, re.MULTILINE | re.DOTALL) + assert function is not None + shared_reader = re.search( + r"^vc_read_proc_sysctl\(\) \{\n.*?^\}", + script, re.MULTILINE | re.DOTALL) + assert shared_reader is not None + path = tmp_path / "sysctl" + path.write_text("1\n", encoding="ascii") + result = run_reader( + shell, path, function.group(0), reader, shared_reader.group(0)) + assert result.returncode == 0, result.stderr + assert result.stdout == "1" + assert 'IFS= read -r sysctl_value' not in script + assert 'IFS= read -r observed_sysctl_value' not in script + assert 'IFS= read -r vc_rpf_value' not in script + + +@pytest.mark.parametrize("script_name", ("apply", "boot")) +@pytest.mark.parametrize("value,expected_rc", (("0", 71), ("1", 0), ("2", 71))) +def test_preflight_and_boot_still_require_exact_ip_forward_one( + tmp_path, shell, production_scripts, script_name, value, expected_rc): + script = production_scripts[script_name] + check = re.search( + r"^read_sysctl\(\) \{\n.*?^\}\n" + r"if ! sysctl_forward=.*?^\[ \"\$sysctl_forward\" = 1 \] \|\| exit 71", + script, re.MULTILINE | re.DOTALL) + assert check is not None + path = tmp_path / "sysctl" + path.write_text(value + "\n", encoding="ascii") + check_script = check.group(0).replace("/proc/sys/net/ipv4/ip_forward", '"$1"') + result = subprocess.run( + [shell, "-c", _FABRIC_SYSCTL_VALIDATION_SH + "\n" + check_script, + "sysctl-preflight", str(path)], + text=True, capture_output=True, timeout=5, check=False, + ) + assert result.returncode == expected_rc, result.stderr + + +@pytest.mark.parametrize("io_fails", (False, True)) +def test_platform_rp_filter_write_requires_buffered_exact_two_readback( + tmp_path, shell, io_fails): + owner = tmp_path / "owner" + owner.write_text("virt-cloud-wg-fabric-v1\n", encoding="ascii") + path = tmp_path / "rp_filter" + path.write_text("1\n", encoding="ascii") + setter = re.search( + r"^vc_fabric_rp_filter_set\(\) \{\n.*?^\}", + _FABRIC_SYSCTL_VALIDATION_SH, re.MULTILINE | re.DOTALL) + assert setter is not None + source = _FABRIC_SYSCTL_VALIDATION_SH + "\n" + setter.group(0).replace( + 'vc_rpf_path="/proc/sys/net/ipv4/conf/$IFACE/rp_filter"', + 'vc_rpf_path="$RPF_FILE"', + ) + prefix = (f"OWNER={shlex.quote(str(owner))}\n" + f"RPF_FILE={shlex.quote(str(path))}\nWG=:; IFACE=wg-fabric0\n") + if io_fails: + prefix += '''cat() { + command cat "$@" || return 1 + [ "$1" != "$RPF_FILE" ] || return 42 +} +''' + result = run_reader( + shell, path, prefix, "vc_fabric_rp_filter_set", source) + assert path.read_text(encoding="ascii") == "2\n" + assert result.returncode == (1 if io_fails else 0), result.stderr + + +@pytest.mark.parametrize("path", ( + "/proc/sys/net/ipv4/ip_forward", "/proc/sys/net/ipv4/conf/all/rp_filter", + "/proc/sys/net/ipv4/conf/default/rp_filter", +)) +def test_buffered_reader_reads_real_linux_proc_sysctl(shell, path): + source = Path(path) + if not source.is_file(): + pytest.skip("real Linux proc sysctls are unavailable on this test host") + expected = source.read_text(encoding="ascii").strip() + assert expected in ("0", "1", "2") + result = run_reader(shell, source) + assert result.returncode == 0, result.stderr + assert result.stdout == expected diff --git a/engine/virsh_panel/fabric.py b/engine/virsh_panel/fabric.py index 598ca8f8..d2dad1bb 100644 --- a/engine/virsh_panel/fabric.py +++ b/engine/virsh_panel/fabric.py @@ -134,7 +134,22 @@ _RFC1918_NETWORKS = ( ) -_FABRIC_SYSCTL_VALIDATION_SH = r'''vc_fabric_sysctl_file_owned() { +_FABRIC_SYSCTL_VALIDATION_SH = r'''vc_read_proc_sysctl() { + [ -r "$1" ] || return 1 + # proc integer sysctls return EOF after the first read. Shell builtins + # that read one byte at a time can capture the digit but miss its newline + # and report failure. Buffer the read, retaining the newline with a + # sentinel so blank/multiple lines cannot be hidden by $(...) stripping. + vc_proc_sysctl_raw=$(cat "$1" 2>/dev/null && printf '.') || return 1 + case "$vc_proc_sysctl_raw" in + '0 +.'|'1 +.'|'2 +.') printf '%s' "${vc_proc_sysctl_raw%??}" ;; + *) return 1 ;; + esac +} +vc_fabric_sysctl_file_owned() { vc_sysctl_path=$1 vc_sysctl_digest=$2 [ -n "${STAT:-}" ] || return 1 @@ -204,8 +219,7 @@ vc_fabric_rp_filter_set() { "$WG" show "$IFACE" >/dev/null 2>&1 || return 1 [ -f "$vc_rpf_path" ] && [ ! -L "$vc_rpf_path" ] || return 1 printf '%s\n' 2 > "$vc_rpf_path" || return 1 - vc_rpf_value= - IFS= read -r vc_rpf_value < "$vc_rpf_path" || return 1 + vc_rpf_value=$(vc_read_proc_sysctl "$vc_rpf_path") || return 1 [ "$vc_rpf_value" = 2 ] } '''.strip() @@ -3345,6 +3359,7 @@ BASE=/etc/virt-cloud IFACE=wg-fabric0 HELPER=/usr/local/libexec/virt-cloud-wg-fabric UNIT=/etc/systemd/system/virt-cloud-wg-fabric.service +TIMER_UNIT=/etc/systemd/system/virt-cloud-wg-fabric-reconcile.timer GATE_GENERATOR=/usr/local/lib/systemd/system-generators/virt-cloud-wg-fabric-gate CONF="$BASE/wg-fabric0.conf" OWNER="$BASE/wg-fabric0.owner" @@ -4055,20 +4070,9 @@ vc_fabric_sysctl_converge() { fi vc_fabric_sysctl_file_owned "$SYSCTL_FILE" "$EXPECTED_DIGEST" } -gate_first=; gate_second=; gate_third= -[ -f "$GATE_GENERATOR" ] && [ ! -L "$GATE_GENERATOR" ] || exit 71 -[ "$($STAT -c '%u:%a' "$GATE_GENERATOR" 2>/dev/null)" = 0:755 ] || exit 71 -{ IFS= read -r gate_first || true; IFS= read -r gate_second || true; IFS= read -r gate_third || true; } < "$GATE_GENERATOR" -[ "$gate_first" = '#!/bin/sh' ] && -[ "$gate_second" = '# virt-cloud-wg-fabric-v1' ] && -[ "$gate_third" = '# virt-cloud-wg-fabric-libvirt-gate-v1' ] || exit 71 -grep -Fqx "# fabric-plan-digest=$EXPECTED_DIGEST" "$GATE_GENERATOR" || exit 71 ''' + IP_RULE_CONVERGENCE_SH + r''' read_sysctl() { - [ -r "$1" ] || return 1 - sysctl_value= - IFS= read -r sysctl_value < "$1" || return 1 - printf '%s' "$sysctl_value" + vc_read_proc_sysctl "$1" } journal_restore_rp_filter() { if [ "$journal_iface_before" = yes ]; then @@ -5305,6 +5309,19 @@ fi if [ "$RECOVER_ONLY" = yes ]; then exit 0 fi +# A first deployment can be interrupted before its generator is published. +# Recovery first proves the journal and complete plan while the emergency +# packet stop remains installed; only normal configuration needs the live +# generator. An old helper can likewise restore a journaled newer generator +# before this current-plan assertion is meaningful. +gate_first=; gate_second=; gate_third= +[ -f "$GATE_GENERATOR" ] && [ ! -L "$GATE_GENERATOR" ] || exit 71 +[ "$($STAT -c '%u:%a' "$GATE_GENERATOR" 2>/dev/null)" = 0:755 ] || exit 71 +{ IFS= read -r gate_first || true; IFS= read -r gate_second || true; IFS= read -r gate_third || true; } < "$GATE_GENERATOR" +[ "$gate_first" = '#!/bin/sh' ] && +[ "$gate_second" = '# virt-cloud-wg-fabric-v1' ] && +[ "$gate_third" = '# virt-cloud-wg-fabric-libvirt-gate-v1' ] || exit 71 +grep -Fqx "# fabric-plan-digest=$EXPECTED_DIGEST" "$GATE_GENERATOR" || exit 71 [ -d /etc/sysctl.d ] && [ ! -L /etc/sysctl.d ] || exit 71 [ "$($STAT -c '%u:%a' /etc/sysctl.d 2>/dev/null)" = 0:755 ] || exit 71 vc_fabric_sysctl_converge || exit 71 @@ -5543,6 +5560,7 @@ while IFS='|' read -r kind a b c d; do schema_seen=1 ;; fabric) is_fabric_id "$a" || exit 72 + [ "$a" = "$VPC_STATE_FABRIC" ] || exit 72 [ "$fabric_seen" = 0 ] || exit 72 fabric_seen=1 ;; revision) @@ -5581,6 +5599,7 @@ while IFS='|' read -r kind a b c d; do firewall_rule_seen=1 ;; network) is_overlay_network "$a" || exit 72 + [ "$a" = "$VPC_FABRIC_NETWORK" ] || exit 72 [ "$network_seen" = 0 ] || exit 72 network_seen=1 network=$a ;; @@ -8146,8 +8165,8 @@ def _apply_script(plan: FabricPlan, host: FabricNode, "__VC_FABRIC_HOST_ID__", host.host_id).replace( "__VC_VPC_STATE_LOADER__", _VPC_STATE_LOADER.replace( "__VC_FABRIC_HOST_ID__", host.host_id).replace( - "__VC_FABRIC_ID__", "$config_fabric").replace( - "__VC_FABRIC_NETWORK__", "$network").replace( + "__VC_FABRIC_ID__", plan.fabric_id).replace( + "__VC_FABRIC_NETWORK__", plan.overlay_cidr).replace( "__VC_VPC_STATE_OBSERVE__", "no")).replace( "__VC_NMCLI_FILENAME_RESOLVER__", _NMCLI_FILENAME_RESOLVER_SH).replace( @@ -8638,10 +8657,7 @@ vc_fabric_operation_checkpoint journal_recovery vc_fabric_operation_checkpoint preflight vc_fabric_operation_checkpoint preflight_sysctl read_sysctl() {{ - [ -r "$1" ] || return 1 - sysctl_value= - IFS= read -r sysctl_value < "$1" || return 1 - printf '%s' "$sysctl_value" + vc_read_proc_sysctl "$1" }} if ! sysctl_forward=$(read_sysctl /proc/sys/net/ipv4/ip_forward); then exit 71; fi [ "$sysctl_forward" = 1 ] || exit 71 @@ -9306,11 +9322,7 @@ FLOCK=$(find_tool /usr/bin/flock /usr/sbin/flock /bin/flock /sbin/flock) || exit firewall_mode_state=owned vc_firewall_mode_ready || firewall_mode_state=mismatch read_observed_sysctl() {{ - observed_sysctl_path=$1 - [ -r "$observed_sysctl_path" ] || return 1 - observed_sysctl_value= - IFS= read -r observed_sysctl_value < "$observed_sysctl_path" || return 1 - printf '%s' "$observed_sysctl_value" + vc_read_proc_sysctl "$1" }} observed_sysctl_or_unknown() {{ if observed_value=$(read_observed_sysctl "$1"); then