"""Fabric plan, helper, and coordinator tests.""" from __future__ import annotations import asyncio import base64 import hashlib import json import re import shlex import stat import subprocess import sys import threading import time from concurrent.futures import ThreadPoolExecutor from pathlib import Path from types import SimpleNamespace import pytest from virsh_panel.fabric import ( FABRIC_OPERATION_CHECKPOINTS, FABRIC_OPERATION_ROOT, FABRIC_OPERATION_REASONS, FABRIC_OPERATION_START_TIMEOUT_SECONDS, FABRIC_OPERATION_STATUSES, FABRIC_OPERATION_UNIT_PREFIX, FABRIC_PROBE_CONCURRENCY, FABRIC_INTERFACE, FABRIC_OWNER_MARKER, FABRIC_REMOTE_APPLY_TIMEOUT_SECONDS, FABRIC_REMOTE_READ_TIMEOUT_SECONDS, FABRIC_SYSCTL, FIREWALL_MODE_NATIVE, MAX_VPC_STATE_FILES, MAX_FABRIC_OPERATION_DIRECTORIES, FabricError, FabricHostAdapter, FabricCoordinator, _BOOT_HELPER, _FABRIC_NFT_VALIDATION_SH, _FIREWALL_MODE_GUARD_SH, _FABRIC_SYSCTL_VALIDATION_SH, _NMCLI_FILENAME_RESOLVER_SH, _REPAIR_UNIT, _UNIT, _apply_script, _fabric_endpoint_rp_filter_evidence, _fabric_operation_dispatch_script, _fabric_operation_runner_script, _fabric_operation_status_script, _identity_script, _new_operation_id, _nm_profile_template, _nft_rules_text, _observe_script, _parse_operation_status, _profile_firewall_mode, _sysctl_text, parse_fabric_plan, validate_wireguard_key, ) from virsh_panel.remote_host import RemoteHostError KEY1 = base64.b64encode(bytes([1]) * 32).decode() KEY2 = base64.b64encode(bytes([2]) * 32).decode() KEY3 = base64.b64encode(bytes([3]) * 32).decode() def plan(): return parse_fabric_plan({ "fabric_id": "hk-fabric", "region": "hk", "overlay_cidr": "100.64.1.0/24", "revision": 1, "listen_port": 51820, "nodes": [ {"host_id": "host-a", "overlay_ip": "100.64.1.1", "endpoint": "203.0.113.1:51820"}, {"host_id": "host-b", "overlay_ip": "100.64.1.2", "endpoint": "203.0.113.2:51820"}, ], }) def plan_payload(current=None): current = current or plan() return { "fabric_id": current.fabric_id, "region": current.region, "overlay_cidr": current.overlay_cidr, "revision": current.revision, "listen_port": current.listen_port, "nodes": [ { "host_id": node.host_id, "overlay_ip": node.overlay_ip, "endpoint": node.endpoint, "listen_port": node.listen_port, } for node in current.nodes ], } def test_plan_is_canonical_and_rejects_wide_or_tenant_routes(): first = plan() reordered = parse_fabric_plan({ "nodes": [ {"host_id": "host-b", "overlay_ip": "100.64.1.2", "endpoint": "203.0.113.2:51820"}, {"host_id": "host-a", "overlay_ip": "100.64.1.1", "endpoint": "203.0.113.1:51820"}, ], "fabric_id": "hk-fabric", "region": "hk", "overlay_cidr": "100.64.1.0/24", "revision": 1, }) assert first.digest == reordered.digest with pytest.raises(FabricError): parse_fabric_plan({ "fabric_id": "bad", "region": "hk", "overlay_cidr": "10.0.0.0/8", "revision": 1, "nodes": [ {"host_id": "a", "overlay_ip": "10.0.0.1", "endpoint": "203.0.113.1:51820"}, {"host_id": "b", "overlay_ip": "10.0.0.2", "endpoint": "203.0.113.2:51820"}, ], }) with pytest.raises(FabricError): parse_fabric_plan({ "fabric_id": "bad", "region": "hk", "overlay_cidr": "100.64.1.0/24", "revision": 1, "nodes": [ {"host_id": "a", "overlay_ip": "100.64.1.1", "endpoint": "100.64.1.2:51820"}, {"host_id": "b", "overlay_ip": "100.64.1.2", "endpoint": "203.0.113.2:51820"}, ], }) with pytest.raises(FabricError): parse_fabric_plan({ "fabric_id": "bad", "region": "hk", "overlay_cidr": "100.64.1.0/24", "revision": 1, "nodes": [ {"host_id": "a", "overlay_ip": "100.64.1.1", "endpoint": "203.0.113.1:51820"}, {"host_id": "b", "overlay_ip": "100.64.1.2", "endpoint": "0.0.0.0:51820"}, ], }) with pytest.raises(FabricError, match="endpoint"): parse_fabric_plan({ "fabric_id": "bad", "region": "hk", "overlay_cidr": "100.64.1.0/24", "revision": 1, "nodes": [ {"host_id": "a", "overlay_ip": "100.64.1.1", "endpoint": "203.0.113.1:51820"}, {"host_id": "b", "overlay_ip": "100.64.1.2", "endpoint": "203.0.113.1:51820"}, ], }) @pytest.mark.parametrize("address", [ "0.0.0.0", "0.1.2.3", "127.23.45.67", "169.254.12.34", "224.0.0.1", "240.0.0.1", "255.255.255.255", ]) def test_plan_rejects_unusable_endpoint_ipv4(address): with pytest.raises(FabricError, match="endpoint 地址不可路由"): parse_fabric_plan({ "fabric_id": "bad-endpoint", "region": "hk", "overlay_cidr": "100.64.1.0/24", "revision": 1, "nodes": [ {"host_id": "a", "overlay_ip": "100.64.1.1", "endpoint": f"{address}:51820"}, {"host_id": "b", "overlay_ip": "100.64.1.2", "endpoint": "203.0.113.2:51820"}, ], }) def test_wireguard_key_rejects_the_all_zero_reserved_key(): with pytest.raises(FabricError): validate_wireguard_key(base64.b64encode(bytes(32)).decode()) def test_generated_helpers_are_posix_safe_and_do_not_embed_private_keys(): current = plan() read_only_identity = _identity_script(create_key=False) scripts = ( _BOOT_HELPER, _identity_script(), read_only_identity, _observe_script(("100.64.1.2",), "100.64.1.1", ("203.0.113.2",), "networkmanager_nmcli", "12345678-1234-5678-1234-567812345678", expected_digest=current.digest), _apply_script(current, current.nodes[0], { "host-a": KEY1, "host-b": KEY2, }), ) for script in scripts: result = subprocess.run( ["/bin/sh", "-n", "-c", script], check=False, capture_output=True, text=True, ) assert result.returncode == 0, result.stderr assert "br0" not in script assert "PRIVATE KEY" not in script assert "genkey" not in read_only_identity assert 'mktemp "$BASE/.wg-key.' not in read_only_identity assert 'mv -f "$_tmp_key" "$KEY"' not in read_only_identity assert "mkdir /run/virt-cloud" in read_only_identity assert ': > "$LOCK"' in read_only_identity assert 'exec 9>>"$LOCK"' in read_only_identity assert '[ -d "$BASE" ]' in read_only_identity observe = scripts[3] assert "__VC_FABRIC_ADMIN_STATE__" in observe assert "__VC_FABRIC_SYSCTL__ip_forward" in observe assert "__VC_FABRIC_SYSCTL__rp_filter_default" in observe assert "__VC_FABRIC_RPF__wg-fabric0" in observe assert "__VC_FABRIC_SYSCTL__rp_filter_effective" in observe assert "__VC_FABRIC_SYSCTL_PERSISTENCE__" in observe assert "__VC_FABRIC_VPC_STATE__" in observe assert "__VC_FABRIC_EMERGENCY__" in observe assert "list set inet virt_cloud_wg_fabric vpc_tenant_ips" in observe assert 'table!="main" && table!="local"' in observe apply = scripts[-1] assert KEY2 in apply and KEY1 in apply assert 'KEY="$BASE/wg-fabric0.key"' in apply assert "genkey" not in apply identity_check = '[ "$($WG pubkey < "$KEY")" = "$EXPECTED_PUBLIC_KEY" ]' assert identity_check in apply assert apply.index(identity_check) < apply.index( "vc_fabric_nm_quarantine || exit 71") assert f"SYSCTL_FILE={FABRIC_SYSCTL}" in apply assert _sysctl_text(current).rstrip() in apply assert "wg-fabric0" in apply assert "address_conflict" in apply assert "existing_route" in apply assert "route_overlaps_target" in apply assert "supernet" not in apply # shell check is numeric and bounded assert "virt-cloud-wg-fabric.nft" in apply assert "include \"/etc/nftables.d/virt-cloud-wg-fabric.nft\"" in apply assert "trap rollback EXIT HUP INT TERM" in apply assert '"$WG" showconf' in apply assert "route save table all" in apply assert 'chmod 700 "$BASE" /usr/local/libexec' not in apply assert "virt-cloud-wg-fabric-v1" in apply assert "virt_cloud_wg_fabric" in apply assert 'iifname "wg-fabric0"' in apply assert 'oifname "wg-fabric0"' in apply assert 'ip protocol icmp accept' in apply assert "route get" in apply assert "virt-cloud-wg-fabric-reconcile.timer" in apply assert "Restart=on-failure" not in apply assert f"fabric-plan-digest={current.digest}" in apply assert "NetworkManager/system-connections" in apply assert "nmcli" in apply assert "flock" in apply assert "vpc_state_load" in apply assert "vpc_state_allowed_for_peer" in apply assert "vpc_state_refresh_nft" in apply assert "vpc_state_refresh_nft" in _BOOT_HELPER assert "vc_fabric_sysctl_converge" in _BOOT_HELPER assert "vc_fabric_rp_filter_set" in _BOOT_HELPER assert "rp_filter_before" in apply assert "vc_ip_rule_preflight" in _BOOT_HELPER assert "vc_ip_rule_scope_check" in _BOOT_HELPER assert "vc_ip_rule_ensure" in _BOOT_HELPER assert "rule replace" not in _BOOT_HELPER assert r"\${" not in _BOOT_HELPER assert '"$BASE"/.vpc-dataplane-*.txn' in _BOOT_HELPER assert "vpc_seen_rule_priorities" in _BOOT_HELPER assert "vpc_seen_route_tables" in _BOOT_HELPER assert 'vpc_state != "invalid"' in observe assert 'vpc[prefix "|" device "|" table "|" via "|" metric]' in observe assert "key|key.desired|wg.conf|address.save|route.save" in _BOOT_HELPER assert "nft.table|nft.installed|nft.applied|nft.phase" in _BOOT_HELPER assert 'journal_file_mode" = 600' in _BOOT_HELPER ifup = _apply_script(current, current.nodes[0], { "host-a": KEY1, "host-b": KEY2, }, "ifupdown", "/usr/bin/ifup") result = subprocess.run( ["/bin/sh", "-n", "-c", ifup], check=False, capture_output=True, text=True, ) assert result.returncode == 0, result.stderr assert "/etc/network/interfaces.d/virt-cloud-wg-fabric0" in ifup assert "ifup" in ifup and "ifdown" in ifup assert "ExecStart=/bin/sh /usr/local/libexec/virt-cloud-wg-fabric --ifupdown-service" in ifup # The Debian 13 contract must also work with classic ifupdown; its # ifquery implementation does not provide ifupdown2's --check flag. assert "ifquery\" --check" not in ifup assert "VC_FABRIC_LOCK_HELD=1" in ifup assert "VC_FABRIC_ALLOW_TRANSITION=1" in ifup assert "br0" not in ifup with pytest.raises(FabricError): _apply_script(current, current.nodes[0], { "host-a": KEY1, "host-b": KEY2, }, "ifupdown", "/tmp/ifup") @pytest.mark.parametrize("expected_digest", ( "", "a" * 63, "A" * 64, "g" * 64, )) def test_observe_rejects_invalid_expected_digest(expected_digest): with pytest.raises(FabricError, match="Fabric 计划摘要无效"): _observe_script((), expected_digest=expected_digest) @pytest.mark.parametrize( ("kind", "first_line", "mode", "expected"), ( ("helper", "#!/bin/sh", "700", "owned"), ("unit", "[Unit]", "644", "owned"), ("helper", "[Unit]", "700", "foreign"), ("unit", "#!/bin/sh", "644", "foreign"), ("unknown", "#!/bin/sh", "644", "foreign"), ), ) def test_observe_artifact_state_accepts_only_kind_specific_header( tmp_path, kind, first_line, mode, expected): current = plan() observe = _observe_script((), expected_digest=current.digest) start = observe.index("artifact_state() {") end = observe.index("\n}\nhelper_marker=", start) + 2 artifact_state = observe[start:end] artifact = tmp_path / "artifact" artifact.write_text( f"{first_line}\n" "# virt-cloud-wg-fabric-v1\n" f"# fabric-plan-digest={current.digest}\n", encoding="utf-8", ) artifact.chmod(int(mode, 8)) fake_stat = tmp_path / "stat" fake_stat.write_text( "#!/bin/sh\n" "case \"$2\" in\n" " '%u') printf '0\\n' ;;\n" " '%a') printf '%s\\n' \"$VC_TEST_MODE\" ;;\n" " *) exit 2 ;;\n" "esac\n", encoding="utf-8", ) fake_stat.chmod(0o700) script = "\n".join(( "set -eu", f"STAT={shlex.quote(str(fake_stat))}", artifact_state, "artifact_state " f"{shlex.quote(str(artifact))} {shlex.quote(kind)} " f"{shlex.quote(current.digest)}", )) result = subprocess.run( ["/bin/sh", "-c", script], env={"PATH": "/usr/bin:/bin", "VC_TEST_MODE": mode}, check=False, capture_output=True, text=True, ) assert result.returncode == 0, result.stderr assert result.stdout == expected def test_reconcile_identity_probe_recreates_only_volatile_runtime_lock(): script = _identity_script(create_key=False) # /run is empty again after reboot, including after a pre-journal apply # failure. Recreate its lock so key validation remains serialized, while # never manufacturing or replacing the persistent WireGuard identity. assert "mkdir /run/virt-cloud" in script assert ': > "$LOCK"' in script assert '"$FLOCK" -x 9' in script assert "genkey" not in script assert 'mktemp "$BASE/.wg-key.' not in script assert 'mv -f "$_tmp_key" "$KEY"' not in script assert '[ -d "$BASE" ]' in script assert '[ -f "$KEY" ]' in script def test_networkmanager_profile_has_no_firewall_zone(): current = plan() profile = _nm_profile_template( current, current.nodes[0], "12345678-1234-5678-1234-567812345678", FIREWALL_MODE_NATIVE, ) assert "autoconnect=false" in profile assert "autoconnect-retries=0" in profile assert "zone=" not in profile assert "interface-name=wg-fabric0" in profile assert "203.0.113." not in profile assert "zone=trusted" not in profile def test_endpoint_rp_filter_evidence_uses_linux_max_without_blocking_strict(): capabilities = { "network": { "local_ipv4_interfaces": {"203.0.113.1": ["br0"]}, "rp_filter": 0, "rp_filter_default": 1, "rp_filter_interfaces": {"br0": 1}, }, } evidence, error = _fabric_endpoint_rp_filter_evidence( capabilities, "203.0.113.1") assert error is None assert evidence == { "fabric_endpoint_interface": "br0", "fabric_endpoint_rp_filter_raw": 1, "fabric_endpoint_rp_filter_effective": 1, "fabric_rp_filter_raw": None, "fabric_rp_filter_effective": None, "rp_filter_default_effective": 1, } capabilities["network"]["rp_filter"] = 2 capabilities["network"]["rp_filter_interfaces"]["br0"] = 1 capabilities["network"]["rp_filter_interfaces"][FABRIC_INTERFACE] = 1 evidence, error = _fabric_endpoint_rp_filter_evidence( capabilities, "203.0.113.1") assert error is None assert evidence["fabric_endpoint_rp_filter_effective"] == 2 assert evidence["fabric_rp_filter_effective"] == 2 def test_endpoint_rp_filter_ignores_bounded_high_density_guest_interfaces(): rp_filters = {f"vnet{index}": 1 for index in range(300)} rp_filters["br0"] = 1 capabilities = { "network": { "local_ipv4_interfaces": {"203.0.113.1": ["br0"]}, "rp_filter": 0, "rp_filter_default": 1, "rp_filter_interfaces": rp_filters, }, } evidence, error = _fabric_endpoint_rp_filter_evidence( capabilities, "203.0.113.1") assert error is None assert evidence["fabric_endpoint_interface"] == "br0" assert evidence["fabric_endpoint_rp_filter_raw"] == 1 def test_endpoint_rp_filter_ignores_bounded_high_density_local_addresses(): address_map = { f"198.51.{index // 256}.{index % 256}": [f"vnet{index}"] for index in range(300) } address_map["203.0.113.1"] = ["br0"] capabilities = { "network": { "local_ipv4_interfaces": address_map, "rp_filter": 0, "rp_filter_default": 1, "rp_filter_interfaces": {"br0": 1}, }, } evidence, error = _fabric_endpoint_rp_filter_evidence( capabilities, "203.0.113.1") assert error is None assert evidence["fabric_endpoint_interface"] == "br0" def test_endpoint_rp_filter_rejects_address_inventory_above_probe_limit(): address_map = { f"address-{index}": ["br0"] for index in range(16385) } address_map["203.0.113.1"] = ["br0"] evidence, error = _fabric_endpoint_rp_filter_evidence({ "network": { "local_ipv4_interfaces": address_map, "rp_filter": 0, "rp_filter_default": 1, "rp_filter_interfaces": {"br0": 1}, }, }, "203.0.113.1") assert evidence is None assert error == "fabric_endpoint_interface_unverified" def test_endpoint_rp_filter_rejects_probe_inventory_above_4096_interfaces(): rp_filters = {f"vnet{index}": 1 for index in range(4096)} rp_filters["br0"] = 1 evidence, error = _fabric_endpoint_rp_filter_evidence({ "network": { "local_ipv4_interfaces": {"203.0.113.1": ["br0"]}, "rp_filter": 0, "rp_filter_default": 1, "rp_filter_interfaces": rp_filters, }, }, "203.0.113.1") assert evidence is None assert error == "fabric_endpoint_rp_filter_unverified" @pytest.mark.parametrize("interfaces,rp_filters,expected_error", [ (["br0", "eth0"], {"br0": 1, "eth0": 1}, "fabric_endpoint_interface_unverified"), (["br0"], {}, "fabric_endpoint_rp_filter_unverified"), (["vnet3"], {"vnet3": 1}, "fabric_endpoint_interface_unsafe_or_drifted"), (["tap3"], {"tap3": 1}, "fabric_endpoint_interface_unsafe_or_drifted"), (["macvtap3"], {"macvtap3": 1}, "fabric_endpoint_interface_unsafe_or_drifted"), (["fwpr3"], {"fwpr3": 1}, "fabric_endpoint_interface_unsafe_or_drifted"), (["fwln3"], {"fwln3": 1}, "fabric_endpoint_interface_unsafe_or_drifted"), ]) def test_endpoint_rp_filter_evidence_fails_closed_on_ambiguous_or_unsafe_data( interfaces, rp_filters, expected_error): evidence, error = _fabric_endpoint_rp_filter_evidence({ "network": { "local_ipv4_interfaces": {"203.0.113.1": interfaces}, "rp_filter": 0, "rp_filter_default": 1, "rp_filter_interfaces": rp_filters, }, }, "203.0.113.1") assert evidence is None assert error == expected_error def test_fixed_sysctl_marker_requires_owner_or_exact_ready_journal(tmp_path): current = plan() sysctl_file = tmp_path / "90-virt-cloud-wg-fabric.conf" sysctl_file.write_text(_sysctl_text(current), encoding="utf-8") sysctl_file.chmod(0o644) owner = tmp_path / "wg-fabric0.owner" journal = tmp_path / ".wg-fabric-txn" stat = tmp_path / "stat" stat.write_text( """#!/bin/sh case "$2" in '%u:%a') if [ -d "$3" ]; then printf '%s\n' 0:700 elif [ "${3##*/}" = 90-virt-cloud-wg-fabric.conf ]; then printf '%s\n' 0:644 else printf '%s\n' 0:600 fi ;; '%u') printf '%s\n' 0 ;; '%a') if [ -d "$3" ]; then printf '%s\n' 700 elif [ "${3##*/}" = 90-virt-cloud-wg-fabric.conf ]; then printf '%s\n' 644 else printf '%s\n' 600; fi ;; *) exit 2 ;; esac """, encoding="utf-8", ) stat.chmod(0o700) preflight = f"""set -eu STAT='{stat}' OWNER='{owner}' JOURNAL='{journal}' EXPECTED_DIGEST='{current.digest}' SYSCTL_FILE='{sysctl_file}' {_FABRIC_SYSCTL_VALIDATION_SH} vc_fabric_sysctl_file_owned "$SYSCTL_FILE" any || exit 71 if ! vc_fabric_owner_file_owned; then vc_fabric_sysctl_journal_owned || exit 71 fi """ foreign = subprocess.run( ["/bin/sh", "-c", preflight], check=False, capture_output=True, text=True, ) assert foreign.returncode == 71 assert sysctl_file.read_text(encoding="utf-8") == _sysctl_text(current) owner.write_text(f"{FABRIC_OWNER_MARKER}\n", encoding="utf-8") owner.chmod(0o600) owned = subprocess.run( ["/bin/sh", "-c", preflight], check=False, capture_output=True, text=True, ) assert owned.returncode == 0, owned.stderr owner.unlink() journal.mkdir(mode=0o700) (journal / "state").write_text("ready\n", encoding="utf-8") (journal / "meta").write_text( f"schema=1\nplan_digest={current.digest}\n", encoding="utf-8", ) (journal / "sysctl.absent").touch(mode=0o600) (journal / "state").chmod(0o600) (journal / "meta").chmod(0o600) journaled = subprocess.run( ["/bin/sh", "-c", preflight], check=False, capture_output=True, text=True, ) assert journaled.returncode == 0, journaled.stderr def test_apply_rollback_restores_owned_sysctl_and_runtime_rp_filter(tmp_path): current = plan() rendered = _apply_script( current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}, ) restore_start = rendered.index("restore_file() {") restore_end = rendered.index("\n}\nrollback_sysctl_file()", restore_start) + 2 sysctl_start = rendered.index("rollback_sysctl_file() {", restore_end) sysctl_end = rendered.index("\n}\nrollback_rp_filter()", sysctl_start) + 2 rpf_start = rendered.index("rollback_rp_filter() {", sysctl_end) rpf_end = rendered.index("\n}\nwrite_nft_phase()", rpf_start) + 2 functions = "\n".join(( rendered[restore_start:restore_end], rendered[sysctl_start:sysctl_end], rendered[rpf_start:rpf_end], )) txn = tmp_path / "txn" txn.mkdir() old_sysctl = ( "# virt-cloud-wg-fabric-v1\n" f"# fabric-plan-digest={'0' * 64}\n" "net.ipv4.conf.wg-fabric0.rp_filter = 2\n" ) (txn / "sysctl").write_text(old_sysctl, encoding="utf-8") (txn / "sysctl.present").touch() (txn / "sysctl.desired").write_text( _sysctl_text(current), encoding="utf-8") (txn / "sysctl.desired").chmod(0o644) sysctl_file = tmp_path / "fabric.conf" sysctl_file.write_text(_sysctl_text(current), encoding="utf-8") rpf_file = tmp_path / "rp_filter" rpf_file.write_text("2\n", encoding="utf-8") stat = tmp_path / "stat" stat.write_text( "#!/bin/sh\n" "[ \"$1:$2\" = '-c:%u:%a' ] || exit 2\n" "mode=$(/usr/bin/stat -f '%Lp' \"$3\") || exit 2\n" "printf '%s:%s\\n' \"${VC_TEST_UID:-0}\" \"$mode\"\n", encoding="utf-8", ) stat.chmod(0o700) functions = functions.replace( '"/proc/sys/net/ipv4/conf/$IFACE/rp_filter"', f"'{rpf_file}'", ).replace( "/etc/sysctl.d/.virt-cloud-wg-fabric-rollback.XXXXXX", f"{tmp_path}/.fabric-rollback.XXXXXX", ) harness = f"""set -eu TXN='{txn}' SYSCTL_FILE='{sysctl_file}' EXPECTED_DIGEST='{current.digest}' CMP=/usr/bin/cmp STAT='{stat}' IFACE=wg-fabric0 IFACE_BEFORE=yes RP_FILTER_BEFORE=1 ROLLBACK_FAILED=no ROLLBACK_IFACE_RECREATED=no read_sysctl() {{ cat "$1"; }} vc_fabric_sysctl_file_owned() {{ [ "$($STAT -c '%u:%a' "$1")" = 0:644 ] || return 1 test_digest=$(sed -n '2s/^# fabric-plan-digest=//p' "$1") [ "$2" = any ] || [ "$test_digest" = "$2" ] }} vc_fabric_sysctl_files_equal() {{ vc_fabric_sysctl_file_owned "$1" "$3" || return 1 vc_fabric_sysctl_file_owned "$2" "$3" || return 1 "$CMP" -s "$1" "$2" }} {functions} rollback_sysctl_file rollback_rp_filter [ "$ROLLBACK_FAILED" = no ] """ result = subprocess.run( ["/bin/sh", "-c", harness], check=False, capture_output=True, text=True, ) assert result.returncode == 0, result.stderr assert sysctl_file.read_text(encoding="utf-8") == old_sysctl assert rpf_file.read_text(encoding="utf-8") == "1\n" # Equal bytes are insufficient ownership evidence: metadata drift must # stop rollback before either the fixed file or runtime value is touched. sysctl_file.chmod(0o600) rpf_file.write_text("2\n", encoding="utf-8") foreign_metadata = subprocess.run( ["/bin/sh", "-c", harness], check=False, capture_output=True, text=True, ) assert foreign_metadata.returncode != 0 assert sysctl_file.read_text(encoding="utf-8") == old_sysctl assert rpf_file.read_text(encoding="utf-8") == "2\n" sysctl_file.chmod(0o644) foreign_uid = subprocess.run( ["/bin/sh", "-c", harness], env={"PATH": "/usr/bin:/bin", "VC_TEST_UID": "1"}, check=False, capture_output=True, text=True, ) assert foreign_uid.returncode != 0 assert sysctl_file.read_text(encoding="utf-8") == old_sysctl assert rpf_file.read_text(encoding="utf-8") == "2\n" sysctl_file.write_text("foreign\n", encoding="utf-8") rpf_file.write_text("2\n", encoding="utf-8") foreign = subprocess.run( ["/bin/sh", "-c", harness], check=False, capture_output=True, text=True, ) assert foreign.returncode != 0 assert sysctl_file.read_text(encoding="utf-8") == "foreign\n" assert rpf_file.read_text(encoding="utf-8") == "2\n" # Runtime drift is independent from the fixed file. Only the # transaction-installed value 2 may be replaced by the sampled old value. sysctl_file.write_text(_sysctl_text(current), encoding="utf-8") sysctl_file.chmod(0o644) rpf_file.write_text("0\n", encoding="utf-8") runtime_drift = subprocess.run( ["/bin/sh", "-c", harness], check=False, capture_output=True, text=True, ) assert runtime_drift.returncode != 0 assert rpf_file.read_text(encoding="utf-8") == "0\n" def test_fabric_sysctl_apply_and_rollback_are_transaction_bound(): current = plan() apply = _apply_script( current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}, ) snapshot = apply.index('snapshot_file "$SYSCTL_FILE" sysctl') snapshot_readback = apply.index( 'vc_fabric_sysctl_files_equal "$SYSCTL_FILE" "$TXN/sysctl" any', snapshot, ) desired = apply.index( '_tmp_sysctl_desired=$(mktemp ' '"$TXN/.sysctl.desired.XXXXXX")', snapshot_readback) desired_temp_owned = apply.index( 'vc_fabric_sysctl_file_owned ' '"$_tmp_sysctl_desired" "$EXPECTED_DIGEST"', desired) desired_publish = apply.index( 'mv -f "$_tmp_sysctl_desired" "$TXN/sysctl.desired"', desired_temp_owned, ) desired_owned = apply.index( 'vc_fabric_sysctl_file_owned ' '"$TXN/sysctl.desired" "$EXPECTED_DIGEST"', desired_publish) desired_sync = apply.index("\nsync\n", desired_owned) precommit = apply.index( 'vc_fabric_sysctl_files_equal "$SYSCTL_FILE" "$TXN/sysctl" any', desired_owned, ) install = apply.index('mv -f "$_tmp_sysctl" "$SYSCTL_FILE"', precommit) installed_readback = apply.index( 'vc_fabric_sysctl_files_equal "$SYSCTL_FILE" ' '"$TXN/sysctl.desired" "$EXPECTED_DIGEST"', install) assert snapshot < snapshot_readback < desired < desired_temp_owned assert desired_temp_owned < desired_publish < desired_owned < desired_sync assert desired_sync < precommit < install < installed_readback rollback = apply.index("rollback_sysctl_file() {") desired_proof = apply.index( 'vc_fabric_sysctl_files_equal "$SYSCTL_FILE" \\\n' ' "$TXN/sysctl.desired" "$EXPECTED_DIGEST"', rollback) pre_restore_recheck = apply.index( 'vc_fabric_sysctl_files_equal "$SYSCTL_FILE" \\\n' ' "$TXN/sysctl.desired" "$EXPECTED_DIGEST"', desired_proof, ) restore = apply.index( 'mv -f "$rollback_sysctl_tmp" "$SYSCTL_FILE"', pre_restore_recheck, ) assert rollback < desired_proof < pre_restore_recheck < restore def _root_stat_stub(tmp_path: Path) -> Path: stat = tmp_path / "stat" stat.write_text( "#!/bin/sh\n" "case \"$1:$2\" in\n" " '-c:%u') printf '%s\\n' 0 ;;\n" " '-c:%a') /usr/bin/stat -f '%Lp' \"$3\" ;;\n" " '-c:%u:%a')\n" " mode=$(/usr/bin/stat -f '%Lp' \"$3\") || exit 2\n" " printf '0:%s\\n' \"$mode\" ;;\n" " *) exit 2 ;;\n" "esac\n", encoding="utf-8", ) stat.chmod(0o700) return stat def test_generic_apply_rollback_restores_only_its_exact_desired_file(tmp_path): current = plan() apply = _apply_script( current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}, ) functions_start = apply.index("txn_file_secure() {") functions_end = apply.index( "\nrollback_sysctl_file() {", functions_start) functions = apply[functions_start:functions_end] txn = tmp_path / "txn" txn.mkdir() target = tmp_path / "managed.conf" stat = _root_stat_stub(tmp_path) harness = "\n".join(( "set -eu", f"TXN='{txn}'", f"STAT='{stat}'", "CMP=/usr/bin/cmp", "ROLLBACK_FAILED=no", "_tmp_restore=", functions, f"restore_file '{target}' conf", "printf '%s\\n' \"$ROLLBACK_FAILED\"", )) def prepare(*, baseline: str | None, desired: str | None, live: str | None, live_mode: int = 0o600) -> None: for path in tuple(txn.iterdir()): path.unlink() if target.exists() or target.is_symlink(): target.unlink() if baseline is None: (txn / "conf.absent").touch(mode=0o600) else: (txn / "conf").write_text(baseline, encoding="utf-8") (txn / "conf").chmod(0o600) (txn / "conf.present").touch(mode=0o600) if desired is None: (txn / "conf.desired.absent").touch(mode=0o600) else: (txn / "conf.desired").write_text(desired, encoding="utf-8") (txn / "conf.desired").chmod(0o600) if live is not None: target.write_text(live, encoding="utf-8") target.chmod(live_mode) def run(): return subprocess.run( ["/bin/sh", "-c", harness], check=False, capture_output=True, text=True, ) prepare(baseline="old\n", desired="new\n", live="new\n") restored = run() assert restored.returncode == 0, restored.stderr assert restored.stdout == "no\n" assert target.read_text(encoding="utf-8") == "old\n" # A crash before this path's desired snapshot was published performed no # mutation. Exact baseline bytes remain a valid no-op recovery state. prepare(baseline="old\n", desired="new\n", live="old\n") (txn / "conf.desired").unlink() untouched = run() assert untouched.returncode == 0, untouched.stderr assert untouched.stdout == "no\n" assert target.read_text(encoding="utf-8") == "old\n" prepare(baseline="old\n", desired="new\n", live="operator\n") drifted = run() assert drifted.returncode == 0, drifted.stderr assert drifted.stdout == "yes\n" assert target.read_text(encoding="utf-8") == "operator\n" prepare(baseline=None, desired="created\n", live="created\n") removed = run() assert removed.returncode == 0, removed.stderr assert removed.stdout == "no\n" assert target.exists() is False prepare(baseline="old\n", desired=None, live=None) recreated = run() assert recreated.returncode == 0, recreated.stderr assert recreated.stdout == "no\n" assert target.read_text(encoding="utf-8") == "old\n" prepare( baseline="old\n", desired="new\n", live="new\n", live_mode=0o644, ) metadata_drift = run() assert metadata_drift.returncode == 0, metadata_drift.stderr assert metadata_drift.stdout == "yes\n" assert target.read_text(encoding="utf-8") == "new\n" assert target.stat().st_mode & 0o777 == 0o644 def test_boot_recovery_restores_only_exact_transaction_post_image(tmp_path): secure_start = _BOOT_HELPER.index("journal_file_secure() {") secure_end = _BOOT_HELPER.index( "journal_cleanup_helper_owned() {", secure_start) restore_start = _BOOT_HELPER.index("restore_journal_file() {") restore_end = _BOOT_HELPER.index( "restore_journal_sysctl() {", restore_start) functions = "\n".join(( _BOOT_HELPER[secure_start:secure_end], _BOOT_HELPER[restore_start:restore_end], )) journal = tmp_path / "journal" journal.mkdir() target = tmp_path / "managed.conf" stat = _root_stat_stub(tmp_path) harness = "\n".join(( "set -eu", f"JOURNAL='{journal}'", f"STAT='{stat}'", "CMP=/usr/bin/cmp", "_tmp_journal_restore=", "stat() { \"$STAT\" \"$@\"; }", functions, f"if restore_journal_file '{target}' conf; then", " printf '%s\\n' restored", "else", " printf '%s\\n' blocked", "fi", )) def prepare(*, baseline: str | None, desired: str | None, live: str | None, live_mode: int = 0o600) -> None: for path in tuple(journal.iterdir()): path.unlink() if target.exists() or target.is_symlink(): target.unlink() if baseline is None: (journal / "conf.absent").touch(mode=0o600) else: (journal / "conf").write_text(baseline, encoding="utf-8") (journal / "conf").chmod(0o600) (journal / "conf.present").touch(mode=0o600) if desired is None: (journal / "conf.desired.absent").touch(mode=0o600) else: (journal / "conf.desired").write_text( desired, encoding="utf-8") (journal / "conf.desired").chmod(0o600) if live is not None: target.write_text(live, encoding="utf-8") target.chmod(live_mode) def run(): return subprocess.run( ["/bin/sh", "-c", harness], check=False, capture_output=True, text=True, ) prepare(baseline="old\n", desired="new\n", live="new\n") restored = run() assert restored.returncode == 0, restored.stderr assert restored.stdout == "restored\n" assert target.read_text(encoding="utf-8") == "old\n" prepare(baseline="old\n", desired="new\n", live="old\n") (journal / "conf.desired").unlink() untouched = run() assert untouched.returncode == 0, untouched.stderr assert untouched.stdout == "restored\n" assert target.read_text(encoding="utf-8") == "old\n" prepare(baseline="old\n", desired="new\n", live="operator\n") drifted = run() assert drifted.returncode == 0, drifted.stderr assert drifted.stdout == "blocked\n" assert target.read_text(encoding="utf-8") == "operator\n" prepare(baseline=None, desired="created\n", live="created\n") removed = run() assert removed.returncode == 0, removed.stderr assert removed.stdout == "restored\n" assert target.exists() is False prepare( baseline="old\n", desired="new\n", live="new\n", live_mode=0o644, ) metadata_drift = run() assert metadata_drift.returncode == 0, metadata_drift.stderr assert metadata_drift.stdout == "blocked\n" assert target.read_text(encoding="utf-8") == "new\n" assert target.stat().st_mode & 0o777 == 0o644 def test_fixed_artifacts_publish_desired_snapshot_before_atomic_install(): current = plan() apply = _apply_script( current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}, firewall_mode=FIREWALL_MODE_NATIVE, ) assert "record_desired_file() {" in apply assert "record_desired_absent() {" in apply assert ( 'txn_baseline_matches "$publish_target" "$publish_name"' in apply ) for source, target, name in ( ("$_tmp_owner", "$OWNER", "owner"), ("$_tmp_nft", "$NFT_RULES", "nft_rules"), ("$_tmp_nft_main", "$NFT_MAIN", "nft_main"), ("$_tmp_helper", "$HELPER", "helper"), ("$_tmp_nm", "$NM_PROFILE", "nm_profile"), ("$_tmp_conf", "$CONF", "conf"), ("$_tmp_unit", "$UNIT", "unit"), ("$_tmp_repair", "$REPAIR_UNIT", "repair"), ("$_tmp_timer", "$TIMER_UNIT", "timer"), ("$_tmp_gate", "$GATE_GENERATOR", "gate_generator"), ): assert ( f'publish_desired_file "{source}" "{target}" {name}' in apply ) assert 'publish_desired_file "$_tmp_key" "$KEY" key' not in apply assert 'journal_file_matches "$restore_target"' in _BOOT_HELPER assert '"$restore_name.desired"' in _BOOT_HELPER def test_rollback_keeps_recovery_helper_until_every_other_restore_succeeds(): current = plan() apply = _apply_script( current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}, firewall_mode=FIREWALL_MODE_NATIVE, ) rollback = apply.index("rollback() {") first_fixed_restore = apply.index( 'restore_file "$UNIT" unit', rollback) verify_runtime = apply.index( "\n verify_rollback_rp_filter\n", first_fixed_restore) helper_comment = apply.index( "# Keep the transaction's complete recovery helper installed", verify_runtime, ) cleanup_state = apply.index( "rollback_publish_cleanup_helper_state || ROLLBACK_FAILED=yes", helper_comment, ) helper_restore = apply.index( 'restore_file "$HELPER" helper', cleanup_state) final_decision = apply.index( 'if [ "$ROLLBACK_FAILED" = yes ]; then', helper_restore) assert first_fixed_restore < verify_runtime < helper_comment < cleanup_state assert cleanup_state < helper_restore < final_decision assert apply.count('restore_file "$HELPER" helper') == 1 def test_rollback_stops_timer_only_with_exact_transaction_unit(tmp_path): current = plan() apply = _apply_script( current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}, ) functions_start = apply.index("txn_file_secure() {") functions_end = apply.index( "\nrollback_sysctl_file() {", functions_start) functions = apply[functions_start:functions_end] txn = tmp_path / "txn" txn.mkdir() timer = tmp_path / "timer.unit" desired = "[Unit]\nDescription=transaction\n" (txn / "timer.desired").write_text(desired, encoding="utf-8") (txn / "timer.desired").chmod(0o644) timer.write_text(desired, encoding="utf-8") timer.chmod(0o644) active = tmp_path / "active" active.write_text("active\n", encoding="utf-8") log = tmp_path / "systemctl.log" systemctl = tmp_path / "systemctl" systemctl.write_text( "#!/bin/sh\n" "case \"$1:$2:$3\" in\n" " 'show:-p:LoadState') printf '%s\\n' loaded ;;\n" " 'show:-p:ActiveState') cat \"$VC_TIMER_ACTIVE\" ;;\n" " 'stop:virt-cloud-wg-fabric-reconcile.timer:')\n" " printf '%s\\n' stop >> \"$VC_TIMER_LOG\"\n" " printf '%s\\n' inactive > \"$VC_TIMER_ACTIVE\" ;;\n" " *) exit 2 ;;\n" "esac\n", encoding="utf-8", ) systemctl.chmod(0o700) stat = _root_stat_stub(tmp_path) harness = "\n".join(( "set -eu", f"TXN='{txn}'", f"TIMER_UNIT='{timer}'", f"SYSTEMCTL='{systemctl}'", f"STAT='{stat}'", "CMP=/usr/bin/cmp", functions, "rollback_stop_transaction_timer_if_owned", )) env = { "PATH": "/usr/bin:/bin", "VC_TIMER_ACTIVE": str(active), "VC_TIMER_LOG": str(log), } owned = subprocess.run( ["/bin/sh", "-c", harness], env=env, check=False, capture_output=True, text=True, ) assert owned.returncode == 0, owned.stderr assert log.read_text(encoding="utf-8") == "stop\n" timer.write_text("[Unit]\nDescription=operator\n", encoding="utf-8") active.write_text("active\n", encoding="utf-8") log.unlink() foreign = subprocess.run( ["/bin/sh", "-c", harness], env=env, check=False, capture_output=True, text=True, ) assert foreign.returncode == 1 assert log.exists() is False assert active.read_text(encoding="utf-8") == "active\n" def test_recreated_interface_may_restore_rp_filter_from_host_default(tmp_path): phase_start = _BOOT_HELPER.index("journal_iface_recovery_state() {") phase_end = _BOOT_HELPER.index("restore_journal_file() {", phase_start) phase_functions = _BOOT_HELPER[phase_start:phase_end] journal = tmp_path / "journal" journal.mkdir() phase_harness = "\n".join(( "set -eu", f"JOURNAL='{journal}'", "journal_file_secure() {", " [ -f \"$JOURNAL/$1\" ] && [ ! -L \"$JOURNAL/$1\" ] &&", " [ \"$(/usr/bin/stat -f '%Lp' \"$JOURNAL/$1\")\" = 600 ]", "}", phase_functions, "journal_publish_iface_recovery_state recreating", '[ "$(journal_iface_recovery_state)" = recreating ]', "journal_publish_iface_recovery_state recreated", "journal_iface_recovery_state", )) phase = subprocess.run( ["/bin/sh", "-c", phase_harness], check=False, capture_output=True, text=True, ) assert phase.returncode == 0, phase.stderr assert phase.stdout == "recreated" assert (journal / "iface.recovery").read_text(encoding="utf-8") == ( "recreated\n") recovery_start = _BOOT_HELPER.index("journal_restore_rp_filter() {") recovery_end = _BOOT_HELPER.index( "journal_verify_rp_filter() {", recovery_start) recovery = _BOOT_HELPER[recovery_start:recovery_end].replace( '"/proc/sys/net/ipv4/conf/$IFACE/rp_filter"', '"$VC_TEST_RPF"', ) rpf = tmp_path / "rp_filter" recovery_harness = "\n".join(( "set -eu", f"VC_TEST_RPF='{rpf}'", "journal_iface_before=yes", "journal_rp_filter_before=2", "read_sysctl() { cat \"$1\"; }", recovery, "journal_restore_rp_filter", )) rpf.write_text("1\n", encoding="utf-8") recreated = subprocess.run( ["/bin/sh", "-c", "journal_iface_recreated=yes\n" + recovery_harness], check=False, capture_output=True, text=True, ) assert recreated.returncode == 0, recreated.stderr assert rpf.read_text(encoding="utf-8") == "2\n" rpf.write_text("1\n", encoding="utf-8") persistent = subprocess.run( ["/bin/sh", "-c", "journal_iface_recreated=no\n" + recovery_harness], check=False, capture_output=True, text=True, ) assert persistent.returncode != 0 assert rpf.read_text(encoding="utf-8") == "1\n" current = plan() apply = _apply_script( current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}, ) rollback_start = apply.index("rollback_rp_filter() {") rollback_end = apply.index("verify_rollback_rp_filter() {", rollback_start) rollback = apply[rollback_start:rollback_end].replace( '"/proc/sys/net/ipv4/conf/$IFACE/rp_filter"', '"$VC_TEST_RPF"', ) rollback_harness = "\n".join(( "set -eu", f"VC_TEST_RPF='{rpf}'", "IFACE_BEFORE=yes", "RP_FILTER_BEFORE=2", "ROLLBACK_FAILED=no", "read_sysctl() { cat \"$1\"; }", rollback, "rollback_rp_filter", '[ "$ROLLBACK_FAILED" = no ]', )) rpf.write_text("1\n", encoding="utf-8") rollback_recreated = subprocess.run( ["/bin/sh", "-c", "ROLLBACK_IFACE_RECREATED=yes\n" + rollback_harness], check=False, capture_output=True, text=True, ) assert rollback_recreated.returncode == 0, rollback_recreated.stderr assert rpf.read_text(encoding="utf-8") == "2\n" rpf.write_text("1\n", encoding="utf-8") rollback_persistent = subprocess.run( ["/bin/sh", "-c", "ROLLBACK_IFACE_RECREATED=no\n" + rollback_harness], check=False, capture_output=True, text=True, ) assert rollback_persistent.returncode != 0 assert rpf.read_text(encoding="utf-8") == "1\n" @pytest.mark.parametrize("readback_rc", (0, 71)) def test_ifupdown_service_requires_readback_after_successful_ifup( tmp_path, readback_rc): start = _BOOT_HELPER.index( 'if [ "$DEPLOYMENT_MODE" = ifupdown ] &&\n' ' [ "${1:-}" = --ifupdown-service ]; then') end = _BOOT_HELPER.index( 'if [ "$DEPLOYMENT_MODE" = ifupdown ] &&\n' ' [ "${1:-}" = --ifupdown-hook ]; then', start) calls = tmp_path / "ifup.calls" ifup = tmp_path / "ifup" ifup.write_text( "#!/bin/sh\n" f"printf '%s\\n' \"$*\" >> {shlex.quote(str(calls))}\n" "exit 0\n", encoding="utf-8") ifup.chmod(0o700) harness = "\n".join(( "set -eu", "DEPLOYMENT_MODE=ifupdown", "IFACE=wg-fabric0", f"IFUP={shlex.quote(str(ifup))}", "set -- --ifupdown-service", _BOOT_HELPER[start:end], '[ "$1" = --ifupdown-configure ] || exit 99', f"exit {readback_rc}", )) result = subprocess.run( ["/bin/sh", "-c", harness], check=False, capture_output=True, text=True) assert result.returncode == readback_rc, result.stderr assert calls.read_text(encoding="utf-8").splitlines() == [ "--force wg-fabric0"] def test_ifupdown_transition_hook_executes_without_reconciling_stale_config( tmp_path): start = _BOOT_HELPER.index( 'if [ "$DEPLOYMENT_MODE" = ifupdown ] &&\n' ' [ "${1:-}" = --ifupdown-configure ]; then') end = _BOOT_HELPER.index("vc_firewall_mode_ready || exit 71", start) block = _BOOT_HELPER[start:end].replace( '"/proc/sys/net/ipv4/conf/$IFACE/rp_filter"', '"$VC_TEST_RPF"', ) rpf = tmp_path / "rp_filter" rpf.write_text("2\n", encoding="utf-8") harness = "\n".join(( "set -eu", "DEPLOYMENT_MODE=ifupdown", "IFACE=wg-fabric0", "VC_FABRIC_APPLYING=1", "VC_FABRIC_ALLOW_TRANSITION=1", "validate_ifupdown_fragment() { :; }", "vc_fabric_rp_filter_set() { printf '%s\\n' rpf >> \"$VC_TEST_LOG\"; }", "read_sysctl() { cat \"$1\"; }", "mock_ip() { printf '%s\\n' \"$*\" >> \"$VC_TEST_LOG\"; }", "IP=mock_ip", "set -- --ifupdown-configure", block, "exit 99", )) log = tmp_path / "transition.log" env = { "PATH": "/usr/bin:/bin", "VC_TEST_LOG": str(log), "VC_TEST_RPF": str(rpf), } applied = subprocess.run( ["/bin/sh", "-c", harness], env=env, check=False, capture_output=True, text=True, ) assert applied.returncode == 0, applied.stderr assert log.read_text(encoding="utf-8").splitlines() == [ "rpf", "link set dev wg-fabric0 up", "rpf", ] log.write_text("", encoding="utf-8") rpf.write_text("1\n", encoding="utf-8") recovery_env = dict(env) recovery_env["VC_FABRIC_RESTORE_RP_FILTER"] = "1" recovered = subprocess.run( ["/bin/sh", "-c", harness], env=recovery_env, check=False, capture_output=True, text=True, ) assert recovered.returncode == 0, recovered.stderr assert log.read_text(encoding="utf-8").splitlines() == [ "link set dev wg-fabric0 up", ] def test_fabric_and_vpc_rp_filter_are_set_before_manager_activation(): current = plan() apply = _apply_script( current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}, ) # The rendered temporary recovery helper contains the same ownership # checks. Anchor on the later, live apply block rather than its here-doc. apply_owner = apply.rindex("vc_fabric_owner_file_owned || exit 71") apply_persistence = apply.index( 'mv -f "$_tmp_sysctl" "$SYSCTL_FILE"', apply_owner, ) apply_create = apply.index( '"$IP" link add dev "$IFACE" type wireguard', apply_persistence) apply_runtime = apply.index( "vc_fabric_rp_filter_set || exit 71", apply_create) apply_nm_up = apply.index( '"$NMCLI" connection up uuid', apply_runtime) assert apply_owner < apply_persistence < apply_create < apply_runtime < apply_nm_up assert 'vc_fabric_sysctl_file_owned "$SYSCTL_FILE" any' in _BOOT_HELPER assert '> /proc/sys/net/ipv4/conf/all/rp_filter' not in apply assert '> /proc/sys/net/ipv4/conf/default/rp_filter' not in apply boot_create = _BOOT_HELPER.index( '"$IP" link add dev "$IFACE" type wireguard') boot_runtime = _BOOT_HELPER.index( "vc_fabric_rp_filter_set || exit 71", boot_create) boot_nm_up = _BOOT_HELPER.index( '"$NMCLI" connection up uuid "$CONNECTION_UUID"', boot_runtime) assert boot_create < boot_runtime < boot_nm_up vpc_create = _BOOT_HELPER.index( '"$IP" link add name "$vpc_bridge" type bridge') vpc_revision = _BOOT_HELPER.index( 'vpc_revision=$(awk -F\'|\' \'$1=="revision" ' "{print $2; exit}' \"$vpc_state_file\")") vpc_digest = _BOOT_HELPER.index( 'vpc_digest=$(awk -F\'|\' \'$1=="digest" ' "{print $2; exit}' \"$vpc_state_file\")", vpc_revision, ) vpc_sysctl_owned = _BOOT_HELPER.index( 'vc_vpc_sysctl_artifact_owned "$vpc_sysctl_file" "$vpc_id"', vpc_digest, ) vpc_runtime = _BOOT_HELPER.index( 'printf \'%s\\n\' 2 > ' '"/proc/sys/net/ipv4/conf/$vpc_bridge/rp_filter"', vpc_create, ) vpc_nm_up = _BOOT_HELPER.index( '"$NMCLI" connection up uuid "$vpc_nm_uuid"', vpc_runtime) vpc_ifup = _BOOT_HELPER.index( 'VC_FABRIC_APPLYING=1 "$IFUP" --force "$vpc_bridge"', vpc_runtime, ) assert vpc_revision < vpc_digest < vpc_sysctl_owned < vpc_create assert vpc_create < vpc_runtime < vpc_nm_up < vpc_ifup assert '"$vpc_revision" "$vpc_digest" "$vpc_bridge" || exit 71' \ in _BOOT_HELPER[vpc_sysctl_owned:vpc_create] assert '__VC_VPC_SYSCTL_VALIDATION__' not in _BOOT_HELPER assert ( 'grep -Fqx "net.ipv4.conf.$vpc_bridge.rp_filter = 2" ' '"$vpc_sysctl_file"' ) not in _BOOT_HELPER def test_recovery_and_rollback_restore_rp_filter_before_reactivation(): recover_start = _BOOT_HELPER.index("recover_journal() {") recover_end = _BOOT_HELPER.index( 'if [ "$RECOVER_ONLY" = yes ]; then', recover_start) recovery = _BOOT_HELPER[recover_start:recover_end] recovery_phase_load = recovery.index( "journal_iface_recovery_phase=$(journal_iface_recovery_state)") recovery_phase_adopt = recovery.index( 'if [ "$journal_iface_recovery_phase" != none ]; then\n' ' journal_iface_recreated=yes', recovery_phase_load, ) recovery_recreating = recovery.index( "journal_publish_iface_recovery_state recreating") recovery_recreate = recovery.index( '"$IP" link add dev "$IFACE" type wireguard', recovery_recreating) recovery_recreated = recovery.index( "journal_publish_iface_recovery_state recreated", recovery_recreate) recovery_restore = recovery.index("journal_restore_rp_filter || exit 79") recovery_nm_up = recovery.index( '"$journal_nmcli" connection up uuid "$journal_old_uuid"', recovery_restore, ) recovery_ifup = recovery.index( '"$IFUP_PATH" --force "$IFACE"', recovery_restore) recovery_verify = recovery.index( "if ! journal_verify_rp_filter;", recovery_restore) assert recovery_phase_load < recovery_phase_adopt < recovery_recreating assert recovery_recreating < recovery_recreate < recovery_recreated assert recovery_recreated < recovery_restore < recovery_nm_up < recovery_verify assert recovery_restore < recovery_ifup < recovery_verify assert ( 'VC_FABRIC_APPLYING=1 \\\n' ' VC_FABRIC_RESTORE_RP_FILTER="$journal_rp_filter_before"' in recovery ) current = plan() apply = _apply_script( current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}, ) rollback_start = apply.index("rollback() {") rollback_recreating = apply.index( "write_iface_recovery_phase recreating", rollback_start) rollback_recreate = apply.index( '"$IP" link add dev "$IFACE" type wireguard', rollback_recreating) rollback_recreated = apply.index( "write_iface_recovery_phase recreated", rollback_recreate) rollback_restore = apply.index( "\n rollback_rp_filter\n", rollback_start) rollback_link_up = apply.index( '"$IP" link set dev "$IFACE" up', rollback_restore) rollback_nm_up = apply.index( '"$NMCLI" connection up uuid', rollback_restore) rollback_verify = apply.index( "\n verify_rollback_rp_filter\n", rollback_restore) assert rollback_recreating < rollback_recreate < rollback_recreated assert rollback_recreated < rollback_restore < rollback_link_up < rollback_verify assert rollback_restore < rollback_nm_up < rollback_verify def test_first_deploy_recovery_and_rollback_delete_link_before_owner(): recover_start = _BOOT_HELPER.index("recover_journal() {") first_deploy = _BOOT_HELPER.index( 'if [ "$journal_iface_before" = no ] &&', recover_start) delete_link = _BOOT_HELPER.index( '"$IP" link del dev "$IFACE"', first_deploy) confirm_absent = _BOOT_HELPER.index( '"$IP" link show dev "$IFACE"', delete_link) mark_absent = _BOOT_HELPER.index( "journal_current_iface_state=absent", confirm_absent) restore_owner = _BOOT_HELPER.index( 'restore_journal_file "$OWNER" owner', mark_absent) assert first_deploy < delete_link < confirm_absent < mark_absent < restore_owner current = plan() apply = _apply_script( current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}, ) rollback_start = apply.index("rollback() {") rollback_first = apply.index( 'if [ "$IFACE_BEFORE" = no ]; then', rollback_start) rollback_delete = apply.index( '"$IP" link del dev "$IFACE"', rollback_first) rollback_confirm = apply.index( '"$IP" link show dev "$IFACE"', rollback_delete) rollback_owner = apply.index( 'restore_file "$OWNER" owner', rollback_confirm) guard = apply.rfind( 'if [ "$IFACE_BEFORE" = yes ] || [ "$ROLLBACK_FAILED" = no ]; then', rollback_confirm, rollback_owner, ) assert rollback_first < rollback_delete < rollback_confirm < guard assert guard < rollback_owner def test_first_deploy_recovery_delete_is_repeatable_and_fails_closed( tmp_path): start = _BOOT_HELPER.index( 'if [ "$journal_iface_before" = no ] &&') end = _BOOT_HELPER.index(" journal_sysctl_restore=no", start) block = _BOOT_HELPER[start:end] link_state = tmp_path / "link-present" link_state.touch() harness = "\n".join(( "set -eu", "journal_iface_before=no", "journal_current_iface_state=present", "journal_current_owner_state=present", "IFACE=wg-fabric0", "vc_fabric_recovery_checkpoint() { :; }", "vc_fabric_owner_file_owned() { :; }", "mock_wg() { [ \"$1:$2\" = show:wg-fabric0 ]; }", "mock_ip() {", " case \"$*\" in", " 'link show dev wg-fabric0') [ -e \"$VC_TEST_LINK\" ] ;;", " 'link del dev wg-fabric0')", " [ \"${VC_TEST_DELETE_FAIL:-no}\" = yes ] || " "rm -f \"$VC_TEST_LINK\" ;;", " *) return 97 ;;", " esac", "}", "WG=mock_wg", "IP=mock_ip", block, "printf '%s\\n' \"$journal_current_iface_state\"", )) env = { "PATH": "/usr/bin:/bin", "VC_TEST_LINK": str(link_state), } removed = subprocess.run( ["/bin/sh", "-c", harness], env=env, check=False, capture_output=True, text=True, ) assert removed.returncode == 0, removed.stderr assert removed.stdout == "absent\n" assert link_state.exists() is False # A crash after link deletion but before restoring the absent OWNER is # repeatable: the next pass observes no live link and leaves it untouched. repeated = subprocess.run( ["/bin/sh", "-c", harness.replace( "journal_current_iface_state=present", "journal_current_iface_state=absent", )], env=env, check=False, capture_output=True, text=True, ) assert repeated.returncode == 0, repeated.stderr assert repeated.stdout == "absent\n" link_state.touch() failed_env = dict(env) failed_env["VC_TEST_DELETE_FAIL"] = "yes" failed = subprocess.run( ["/bin/sh", "-c", harness], env=failed_env, check=False, capture_output=True, text=True, ) assert failed.returncode == 79 assert link_state.exists() is True def test_first_deploy_recovery_exits_without_absent_helper_or_timer_stop(): recover_start = _BOOT_HELPER.index("recover_journal() {") recover_end = _BOOT_HELPER.index( '\n}\nif [ "${VC_FABRIC_APPLYING:-0}" != 1 ]; then', recover_start) recovery = _BOOT_HELPER[recover_start:recover_end] helper_snapshot = recovery.index( 'if [ -f "$JOURNAL/helper.present" ]; then') timer_snapshot = recovery.index( 'if [ -f "$JOURNAL/timer.present" ]; then', helper_snapshot) stop_transaction_timer = recovery.index( "vc_stop_transaction_timer_if_loaded || exit 79", timer_snapshot, ) restore_timer_file = recovery.index( 'restore_journal_file "/etc/systemd/system/' 'virt-cloud-wg-fabric-reconcile.timer" timer', stop_transaction_timer, ) daemon_reload = recovery.index( '"$SYSTEMCTL" daemon-reload', restore_timer_file) timer_present_branch = recovery.index( 'if [ "$journal_timer_before" = present ]; then', daemon_reload) timer_restore = recovery.index( 'vc_restore_timer_activity "$journal_timer_active"', timer_present_branch, ) timer_not_found = recovery.index( '[ "$journal_timer_load_state" = not-found ]', timer_restore) timer_inactive = recovery.index( '[ "$journal_timer_active_state" = inactive ]', timer_not_found) absent_helper = recovery.index( 'if [ "$journal_helper_before" = absent ]; then', timer_inactive) cleanup_state = recovery.index( "journal_publish_cleanup_helper_state || exit 79", absent_helper) restore_absent_helper = recovery.index( 'restore_journal_file "$HELPER" helper || exit 79', cleanup_state) remove_journal = recovery.index( 'rm -rf "$JOURNAL" || exit 79', restore_absent_helper) absent_exit = recovery.index("exit 0", remove_journal) exec_helper = recovery.index('exec /bin/sh "$HELPER" "$@"', absent_exit) assert helper_snapshot < timer_snapshot < stop_transaction_timer assert stop_transaction_timer < restore_timer_file assert restore_timer_file < daemon_reload assert daemon_reload < timer_present_branch < timer_restore < timer_not_found assert timer_not_found < timer_inactive < absent_helper < cleanup_state assert cleanup_state < restore_absent_helper < remove_journal assert remove_journal < absent_exit < exec_helper present_helper_branch = recovery.index( 'if [ "$journal_helper_before" = present ]; then', timer_snapshot) present_helper_restore = recovery.index( 'restore_journal_file "$HELPER" helper || exit 79', present_helper_branch, ) assert present_helper_branch < present_helper_restore < absent_helper timer_stop_start = _BOOT_HELPER.index( "vc_stop_transaction_timer_if_loaded() {") timer_stop_end = _BOOT_HELPER.index("nft_panel_owned() {", timer_stop_start) timer_stop = _BOOT_HELPER[timer_stop_start:timer_stop_end] assert "not-found:inactive|loaded:inactive" in timer_stop assert "loaded:active|loaded:activating|loaded:deactivating" in timer_stop @pytest.mark.parametrize( ("load_state", "active_state", "sticky", "expected_rc", "stops"), ( ("not-found", "inactive", False, 0, False), ("loaded", "inactive", False, 0, False), ("loaded", "active", False, 0, True), ("loaded", "active", True, 1, True), ("not-found", "active", False, 1, False), ("loaded", "failed", False, 1, False), ), ) def test_absent_timer_recovery_stops_only_loaded_transaction_unit( tmp_path, load_state, active_state, sticky, expected_rc, stops): function_start = _BOOT_HELPER.index( "vc_stop_transaction_timer_if_loaded() {") function_end = _BOOT_HELPER.index("nft_panel_owned() {", function_start) function = _BOOT_HELPER[function_start:function_end] load = tmp_path / "load" active = tmp_path / "active" log = tmp_path / "systemctl.log" load.write_text(f"{load_state}\n", encoding="utf-8") active.write_text(f"{active_state}\n", encoding="utf-8") systemctl = tmp_path / "systemctl" systemctl.write_text( "#!/bin/sh\n" "case \"$1:$2:$3\" in\n" " 'show:-p:LoadState') cat \"$VC_TIMER_LOAD\" ;;\n" " 'show:-p:ActiveState') cat \"$VC_TIMER_ACTIVE\" ;;\n" " 'stop:virt-cloud-wg-fabric-reconcile.timer:')\n" " printf '%s\\n' stop >> \"$VC_TIMER_LOG\"\n" " [ \"$VC_TIMER_STICKY\" = yes ] || " "printf '%s\\n' inactive > \"$VC_TIMER_ACTIVE\" ;;\n" " *) exit 2 ;;\n" "esac\n", encoding="utf-8", ) systemctl.chmod(0o700) result = subprocess.run( ["/bin/sh", "-c", "\n".join(( "set -eu", f"SYSTEMCTL='{systemctl}'", function, "vc_stop_transaction_timer_if_loaded", ))], env={ "PATH": "/usr/bin:/bin", "VC_TIMER_LOAD": str(load), "VC_TIMER_ACTIVE": str(active), "VC_TIMER_LOG": str(log), "VC_TIMER_STICKY": "yes" if sticky else "no", }, check=False, capture_output=True, text=True, ) assert result.returncode == expected_rc, result.stderr assert log.exists() is stops def test_cleanup_helper_terminal_phase_is_repeatable_and_rejects_foreign_file( tmp_path): digest = "a" * 64 journal = tmp_path / "journal" helper = tmp_path / "helper" stat = tmp_path / "stat" stat.write_text( "#!/bin/sh\n" "[ \"$1:$2\" = '-c:%u:%a' ] || exit 2\n" "case \"$3\" in\n" f" '{helper}') printf '%s\\n' 0:700 ;;\n" " *) printf '%s\\n' 0:600 ;;\n" "esac\n", encoding="utf-8", ) stat.chmod(0o700) validator_start = _BOOT_HELPER.index("journal_cleanup_helper_owned() {") validator_end = _BOOT_HELPER.index( "journal_publish_cleanup_helper_state() {", validator_start) validator = _BOOT_HELPER[validator_start:validator_end] recover_start = _BOOT_HELPER.index("recover_journal() {") branch_start = _BOOT_HELPER.index(" cleanup_helper)", recover_start) branch_end = _BOOT_HELPER.index(" guarded)", branch_start) cleanup_branch = _BOOT_HELPER[branch_start:branch_end] harness = "\n".join(( "set -eu", f"JOURNAL='{journal}'", f"HELPER='{helper}'", f"STAT='{stat}'", "journal_file_secure() {", " [ -f \"$JOURNAL/$1\" ] && [ ! -L \"$JOURNAL/$1\" ]", "}", "journal_file_matches() {", " [ -f \"$1\" ] && [ ! -L \"$1\" ] &&", " /usr/bin/cmp -s \"$1\" \"$JOURNAL/$2\"", "}", "journal_value() {", " awk -F= -v wanted=\"$1\" '$1 == wanted {print substr($0, " "index($0, \"=\") + 1); found=1} END {exit found ? 0 : 1}' " "\"$JOURNAL/meta\"", "}", validator, "case cleanup_helper in", cleanup_branch, "esac", "exit 99", )) def prepare(*, helper_state: str) -> None: journal.mkdir() (journal / "state").write_text("cleanup_helper\n", encoding="utf-8") (journal / "meta").write_text( f"schema=1\nplan_digest={digest}\n", encoding="utf-8") (journal / "helper.absent").touch() for path in (journal / "state", journal / "meta", journal / "helper.absent"): path.chmod(0o600) (journal / "helper.desired").write_text( "#!/bin/sh\n" "# virt-cloud-wg-fabric-v1\n" f"# fabric-plan-digest={digest}\n" f'PLAN_DIGEST="{digest}"\n', encoding="utf-8", ) (journal / "helper.desired").chmod(0o700) if helper_state != "absent": actual_digest = digest if helper_state == "owned" else "b" * 64 helper.write_text( "#!/bin/sh\n" "# virt-cloud-wg-fabric-v1\n" f"# fabric-plan-digest={actual_digest}\n" f'PLAN_DIGEST="{actual_digest}"\n', encoding="utf-8", ) helper.chmod(0o700) prepare(helper_state="owned") owned = subprocess.run( ["/bin/sh", "-c", harness], check=False, capture_output=True, text=True, ) assert owned.returncode == 0, owned.stderr assert helper.exists() is False assert journal.exists() is False prepare(helper_state="absent") absent = subprocess.run( ["/bin/sh", "-c", harness], check=False, capture_output=True, text=True, ) assert absent.returncode == 0, absent.stderr assert journal.exists() is False prepare(helper_state="foreign") foreign = subprocess.run( ["/bin/sh", "-c", harness], check=False, capture_output=True, text=True, ) assert foreign.returncode == 79 assert helper.exists() is True assert journal.exists() is True current = plan() apply = _apply_script( current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}, ) existing = apply.index('case "$journal_existing_state" in') cleanup = apply.index("cleanup_helper)", existing) direct_absent = apply.index( '[ ! -e "$HELPER" ] && [ ! -L "$HELPER" ] || exit 79', cleanup) ready = apply.index("ready)", direct_absent) first_deploy_phase = apply.index( 'transaction_missing_installing_snapshot=no', ready) special_first = apply.index( 'if [ "$transaction_missing_installing_snapshot" = yes ]; then', first_deploy_phase, ) temporary_recovery = apply.index( 'VC_FABRIC_LOCK_HELD=1 /bin/sh "$_tmp_helper" ' '--recover-installing || exit 79', special_first, ) fixed_helper_owned = apply.index( 'transaction_recovery_helper_owned || exit 79', temporary_recovery, ) fixed_recovery = apply.index( 'VC_FABRIC_LOCK_HELD=1 /bin/sh "$HELPER" --recover-only || exit 79', fixed_helper_owned, ) assert ( existing < cleanup < direct_absent < ready < first_deploy_phase < special_first < temporary_recovery < fixed_helper_owned < fixed_recovery ) def test_transaction_journal_preparing_state_is_atomically_published(): current = plan() apply = _apply_script( current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}, ) existing = apply.index('if [ -e "$BASE/.wg-fabric-txn" ]; then') state_required = apply.index( '[ -f "$BASE/.wg-fabric-txn/state" ] &&', existing) existing_case = apply.index('case "$journal_existing_state" in', state_required) publish_start = apply.index( '_tmp_txn_dir=$(mktemp -d ' '"$BASE/.wg-fabric-txn-preparing.XXXXXX")', existing_case) preparing_state = apply.index( 'printf \'%s\\n\' preparing > "$_tmp_txn_dir/state"', publish_start) preparing_meta = apply.index( 'cat > "$_tmp_txn_dir/meta"', preparing_state) absence_proof = apply.index( '[ ! -e "$TXN" ] && [ ! -L "$TXN" ] || exit 79', preparing_meta) publish = apply.index('mv "$_tmp_txn_dir" "$TXN"', absence_proof) readback = apply.index( '[ "$(cat "$TXN/state" 2>/dev/null)" = preparing ]', publish) first_snapshot = apply.index('snapshot_file "$CONF" conf', readback) assert existing < state_required < existing_case < publish_start assert publish_start < preparing_state < preparing_meta < absence_proof assert absence_proof < publish < readback < first_snapshot assert "firewall_mode=$FIREWALL_MODE" in apply[ preparing_meta:absence_proof ] assert 'mkdir "$TXN"' not in apply def test_observe_marks_any_fabric_transaction_journal_as_mismatch(): current = plan() observe = _observe_script( ("100.64.1.2",), "100.64.1.1", ("203.0.113.2",), firewall_mode=FIREWALL_MODE_NATIVE, connection_uuid="12345678-1234-5678-1234-567812345678", host_id="host-a", fabric_id=current.fabric_id, overlay_cidr=current.overlay_cidr, expected_digest=current.digest, ) state = observe.index("firewall_mode_state=owned") journal_path = observe.index( "[ ! -e /etc/virt-cloud/.wg-fabric-txn ]", state) journal_link = observe.index( "[ ! -L /etc/virt-cloud/.wg-fabric-txn ]", journal_path) mismatch = observe.index( "|| firewall_mode_state=mismatch", journal_link) marker = observe.index("__VC_FABRIC_FIREWALL_STATE__", mismatch) assert state < journal_path < journal_link < mismatch < marker def test_networkmanager_filename_resolution_uses_supported_unique_list_rows(): assert "GENERAL.FILENAME" not in _NMCLI_FILENAME_RESOLVER_SH assert "GENERAL.FILENAME" not in _FIREWALL_MODE_GUARD_SH assert "GENERAL.FILENAME" not in _BOOT_HELPER assert ( 'LC_ALL=C "$NMCLI" -t --escape no -f UUID,FILENAME' in _NMCLI_FILENAME_RESOLVER_SH ) # Resolve before and after semantic ownership checks to reject a filename # switch during the readback transaction. assert ( _FIREWALL_MODE_GUARD_SH.count( 'vc_nm_filename_for_uuid "$CONNECTION_UUID"') == 2 ) assert ( _BOOT_HELPER.count('vc_nm_filename_for_uuid "$vpc_nm_uuid"') == 1 ) current = plan() apply = _apply_script( current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}, firewall_mode=FIREWALL_MODE_NATIVE, ) observe = _observe_script( ("100.64.1.2",), "100.64.1.1", ("203.0.113.2",), firewall_mode=FIREWALL_MODE_NATIVE, connection_uuid="12345678-1234-5678-1234-567812345678", host_id="host-a", fabric_id=current.fabric_id, overlay_cidr=current.overlay_cidr, expected_digest=current.digest, ) assert "__VC_NMCLI_FILENAME_RESOLVER__" not in apply assert "GENERAL.FILENAME" not in apply assert "GENERAL.FILENAME" not in observe # Apply embeds one resolver for its own transaction and another in the # separately installed boot helper; observe is one shell process. assert apply.count("vc_nm_filename_for_uuid() {") == 2 assert observe.count("vc_nm_filename_for_uuid() {") == 1 assert "nm_vpc_field_raw=$(" in _BOOT_HELPER assert ( 'nm_vpc_value=$(nm_vpc_field "$nm_vpc_empty" uuid ' '"$vpc_nm_uuid") || exit 71' in _BOOT_HELPER ) target = "11111111-1111-4111-8111-111111111111" expected = ( "/etc/NetworkManager/system-connections/" "bridge profile:primary.nmconnection" ) harness = "\n".join(( "set -eu", r''' vc_test_nmcli() { case "$*" in "-t --escape no -f UUID,FILENAME connection show") [ "${VC_NMCLI_FAIL:-no}" = no ] || return 3 printf '%s' "$VC_NMCLI_ROWS" ;; *GENERAL.FILENAME*) return 97 ;; *) return 98 ;; esac } NMCLI=vc_test_nmcli '''.strip(), _NMCLI_FILENAME_RESOLVER_SH, 'vc_nm_filename_for_uuid "$VC_TARGET_UUID"', )) def run(rows: str, *, uuid: str = target, fail: bool = False): return subprocess.run( ["/bin/sh", "-c", harness], env={ "PATH": "/usr/bin:/bin", "VC_TARGET_UUID": uuid, "VC_NMCLI_ROWS": rows, "VC_NMCLI_FAIL": "yes" if fail else "no", }, check=False, capture_output=True, text=True, ) accepted = run( "22222222-2222-4222-8222-222222222222:" "/run/NetworkManager/system-connections/other.nmconnection\n" f"{target}:{expected}\n" ) assert accepted.returncode == 0, accepted.stderr assert accepted.stdout == f"{expected}\n" duplicate = run(f"{target}:{expected}\n{target}:{expected}\n") assert duplicate.returncode != 0 missing = run( "22222222-2222-4222-8222-222222222222:" "/etc/NetworkManager/system-connections/other.nmconnection\n" ) assert missing.returncode != 0 malformed_inventory = run( f"{target}:{expected}\ncontinuation-without-a-uuid\n" ) assert malformed_inventory.returncode != 0 oversized_row = run(f"{target}:{'x' * 4097}\n") assert oversized_row.returncode != 0 malformed = run(f"{target}:{expected}\n", uuid="../not-a-uuid") assert malformed.returncode != 0 command_failure = run(f"{target}:{expected}\n", fail=True) assert command_failure.returncode != 0 def test_networkmanager_quarantine_keeps_owned_device_until_hard_link_down(): current = plan() helper = _apply_script( current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}, firewall_mode=FIREWALL_MODE_NATIVE, ) start = helper.index("vc_fabric_nm_quarantine() {") end = helper.index("\n}", start) block = helper[start:end] profile_read = block.index("vc_fabric_nm_profile_owned any || return 1") unmanaged = block.index( '"$NMCLI" device set "$IFACE" managed no', profile_read) hard_down = block.index( '"$IP" link set dev "$IFACE" down', unmanaged) assert profile_read < unmanaged < hard_down assert "firewall-cmd" not in block assert "connection.zone" not in block assert 'device reapply "$IFACE"' not in block assert 'connection down uuid "$CONNECTION_UUID"' not in block assert 'device disconnect "$IFACE"' not in block def _write_firewall_guard_tools(tmp_path: Path) -> Path: systemctl = tmp_path / "systemctl" systemctl.write_text("""#!/bin/sh case "$1:$2" in is-active:nftables) printf '%s\\n' "$VC_NFT_ACTIVE" ;; is-enabled:nftables) printf '%s\\n' "$VC_NFT_ENABLED" ;; is-active:firewalld) printf '%s\\n' "$VC_FIREWALLD_ACTIVE" ;; is-enabled:firewalld) printf '%s\\n' "$VC_FIREWALLD_ENABLED" ;; show:-p) case "$5" in nftables) printf '%s\\n' "$VC_NFT_LOAD" ;; firewalld) printf '%s\\n' "$VC_FIREWALLD_LOAD" ;; *) exit 2 ;; esac ;; *) exit 2 ;; esac """, encoding="utf-8") systemctl.chmod(0o700) return systemctl def _run_firewall_guard( tmp_path: Path, mode: str, *, nft_active: str, nft_enabled: str, nft_load: str, firewalld_active: str, firewalld_enabled: str, firewalld_load: str, nft_readable: bool = True, ) -> subprocess.CompletedProcess[str]: systemctl = _write_firewall_guard_tools(tmp_path) script = "\n".join(( "set -eu", f"SYSTEMCTL='{systemctl}'", f"NFT={'/usr/bin/true' if nft_readable else '/usr/bin/false'}", f"FIREWALL_MODE='{mode}'", "find_tool() { for p in \"$@\"; do [ -x \"$p\" ] && { printf '%s' \"$p\"; return 0; }; done; return 1; }", _FIREWALL_MODE_GUARD_SH, "vc_firewall_mode_ready", )) env = { "PATH": "/usr/bin:/bin", "VC_NFT_ACTIVE": nft_active, "VC_NFT_ENABLED": nft_enabled, "VC_NFT_LOAD": nft_load, "VC_FIREWALLD_ACTIVE": firewalld_active, "VC_FIREWALLD_ENABLED": firewalld_enabled, "VC_FIREWALLD_LOAD": firewalld_load, } return subprocess.run( ["/bin/sh", "-c", script], env=env, check=False, capture_output=True, text=True, ) @pytest.mark.parametrize(("active", "enabled", "load"), ( ("active", "enabled-runtime", "loaded"), ("inactive", "disabled", "loaded"), ("inactive", "masked", "masked"), ("unknown", "unknown", "not-found"), )) def test_native_runtime_guard_ignores_stock_nftables_service_state( tmp_path, active, enabled, load, ): native = _run_firewall_guard( tmp_path, FIREWALL_MODE_NATIVE, nft_active=active, nft_enabled=enabled, nft_load=load, firewalld_active="inactive", firewalld_enabled="masked", firewalld_load="masked", ) assert native.returncode == 0, native.stderr def test_native_runtime_guard_requires_readable_nft_runtime(tmp_path): native = _run_firewall_guard( tmp_path, FIREWALL_MODE_NATIVE, nft_active="inactive", nft_enabled="disabled", nft_load="loaded", firewalld_active="inactive", firewalld_enabled="masked", firewalld_load="masked", nft_readable=False, ) assert native.returncode != 0 def test_native_runtime_guard_accepts_a_not_found_firewalld_unit(tmp_path): result = _run_firewall_guard( tmp_path, FIREWALL_MODE_NATIVE, nft_active="active", nft_enabled="enabled", nft_load="loaded", firewalld_active="unknown", firewalld_enabled="unknown", firewalld_load="not-found", ) assert result.returncode == 0, result.stderr def test_native_runtime_guard_rejects_inactive_unmasked_firewalld(tmp_path): result = _run_firewall_guard( tmp_path, FIREWALL_MODE_NATIVE, nft_active="active", nft_enabled="enabled", nft_load="loaded", firewalld_active="inactive", firewalld_enabled="disabled", firewalld_load="loaded", ) assert result.returncode != 0 @pytest.mark.parametrize(("distribution", "target", "source"), [ ("debian-13-iproute2-6.15", "203.0.113.123", "203.0.113.126"), ("almalinux-10-iproute-6.17", "198.51.100.126", "198.51.100.123"), ]) def test_underlay_route_path_accepts_real_explicit_from_output( tmp_path, distribution, target, source, ): fake_ip = tmp_path / "ip" fake_ip.write_text( "#!/bin/sh\n" "if [ \"${5:-}\" = from ]; then\n" " printf '%s from %s dev %s uid 0\\n cache\\n' " "\"$4\" \"$VC_FROM_SRC\" \"$VC_FROM_DEV\"\n" "else\n" " printf '%s dev %s src %s uid 0\\n cache\\n' " "\"$4\" \"$VC_PLAIN_DEV\" \"$VC_PLAIN_SRC\"\n" "fi\n", encoding="utf-8", ) fake_ip.chmod(0o700) script = "\n".join(( "set -eu", f"IP='{fake_ip}'", "find_tool() { for p in \"$@\"; do [ -x \"$p\" ] && { printf '%s' \"$p\"; return 0; }; done; return 1; }", _FIREWALL_MODE_GUARD_SH, f"vc_underlay_route_path {target} {source}", )) base_env = { "PATH": "/usr/bin:/bin", "VC_PLAIN_DEV": "br0", "VC_FROM_DEV": "br0", "VC_FROM_SRC": source, } exact = subprocess.run( ["/bin/sh", "-c", script], env={**base_env, "VC_PLAIN_SRC": source}, check=False, capture_output=True, text=True, ) assert exact.returncode == 0, f"{distribution}: {exact.stderr}" assert exact.stdout == f"br0|{source}" @pytest.mark.parametrize(("plain_source", "explicit_source"), [ ("10.20.30.40", "203.0.113.1"), ("203.0.113.1", "10.20.30.40"), ]) def test_underlay_route_path_rejects_inexact_plain_or_explicit_source( tmp_path, plain_source, explicit_source, ): fake_ip = tmp_path / "ip" fake_ip.write_text( "#!/bin/sh\n" "if [ \"${5:-}\" = from ]; then\n" " printf '%s from %s dev eth0 uid 0\\n cache\\n' " "\"$4\" \"$VC_FROM_SRC\"\n" "else\n" " printf '%s dev eth0 src %s uid 0\\n cache\\n' " "\"$4\" \"$VC_PLAIN_SRC\"\n" "fi\n", encoding="utf-8", ) fake_ip.chmod(0o700) script = "\n".join(( "set -eu", f"IP='{fake_ip}'", "find_tool() { for p in \"$@\"; do [ -x \"$p\" ] && { printf '%s' \"$p\"; return 0; }; done; return 1; }", _FIREWALL_MODE_GUARD_SH, "vc_underlay_route_path 203.0.113.2 203.0.113.1", )) mismatch = subprocess.run( ["/bin/sh", "-c", script], env={ "PATH": "/usr/bin:/bin", "VC_PLAIN_SRC": plain_source, "VC_FROM_SRC": explicit_source, }, check=False, capture_output=True, text=True, ) assert mismatch.returncode != 0 def _run_underlay_explicit_evidence( tmp_path: Path, explicit_output: str, ) -> subprocess.CompletedProcess[str]: fake_ip = tmp_path / "ip" fake_ip.write_text( "#!/bin/sh\n" "if [ \"${5:-}\" = from ]; then\n" " printf '%s\\n' \"$VC_EXPLICIT_OUTPUT\"\n" "else\n" " printf '%s\\n' '203.0.113.2 dev br0 src 203.0.113.1 uid 0'\n" "fi\n", encoding="utf-8", ) fake_ip.chmod(0o700) script = "\n".join(( "set -eu", f"IP='{fake_ip}'", "find_tool() { for p in \"$@\"; do [ -x \"$p\" ] && { printf '%s' \"$p\"; return 0; }; done; return 1; }", _FIREWALL_MODE_GUARD_SH, "vc_underlay_route_path 203.0.113.2 203.0.113.1", )) return subprocess.run( ["/bin/sh", "-c", script], env={ "PATH": "/usr/bin:/bin", "VC_EXPLICIT_OUTPUT": explicit_output, }, check=False, capture_output=True, text=True, ) def test_underlay_route_path_accepts_matching_optional_selected_source( tmp_path, ): result = _run_underlay_explicit_evidence( tmp_path, "203.0.113.2 from 203.0.113.1 dev br0 " "src 203.0.113.1 uid 0", ) assert result.returncode == 0, result.stderr assert result.stdout == "br0|203.0.113.1" @pytest.mark.parametrize("explicit_output", [ "203.0.113.2 dev br0 uid 0", "203.0.113.2 dev br0 src 203.0.113.1 uid 0", "203.0.113.2 from 203.0.113.1 from 203.0.113.1 dev br0 uid 0", ( "203.0.113.2 from 203.0.113.1 dev br0 " "src 10.20.30.40 uid 0" ), "203.0.113.2 from 203.0.113.1 dev br0 dev br0 uid 0", "203.0.113.2 dev br0 src 203.0.113.1 from", ]) def test_underlay_route_path_rejects_missing_or_ambiguous_explicit_evidence( tmp_path, explicit_output, ): result = _run_underlay_explicit_evidence(tmp_path, explicit_output) assert result.returncode != 0 def _run_native_apply_underlay_preflight( tmp_path: Path, *, local_address: str = "203.0.113.1", local_device: str = "eth0", plain_device: str = "eth0", plain_source: str = "203.0.113.1", from_device: str = "eth0", from_source: str = "203.0.113.1", ) -> subprocess.CompletedProcess[str]: current = plan() rendered = _apply_script( current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}, firewall_mode=FIREWALL_MODE_NATIVE, ) preflight_start = rendered.index("LOCAL_ENDPOINT=203.0.113.1") preflight_end = rendered.index( 'if [ "$DEPLOYMENT_MODE" = networkmanager_nmcli ]; then', preflight_start, ) endpoint_start = rendered.index( "# The UDP endpoint must leave through the host underlay", preflight_end, ) endpoint_end = rendered.index( "# A desired overlay /32 may only be replaced", endpoint_start, ) fake_ip = tmp_path / "ip" fake_ip.write_text( "#!/bin/sh\n" "case \"$*\" in\n" " '-o -4 address show up scope global')\n" " printf '%s\\n' \"2: $VC_LOCAL_DEVICE inet $VC_LOCAL_ADDRESS/24 " "scope global $VC_LOCAL_DEVICE\" ;;\n" " '-4 route get 203.0.113.2')\n" " printf '%s\\n' \"203.0.113.2 dev $VC_PLAIN_DEV " "src $VC_PLAIN_SRC\" ;;\n" " '-4 route get 203.0.113.2 from 203.0.113.1')\n" " printf '%s\\n' \"203.0.113.2 from $VC_FROM_SRC " "dev $VC_FROM_DEV uid 0\" \" cache\" ;;\n" " *) exit 91 ;;\n" "esac\n", encoding="utf-8", ) fake_ip.chmod(0o700) shell = "\n".join(( "set -eu", f"IP='{fake_ip}'", "FIREWALL_MODE=native_nftables", "IFACE=wg-fabric0", "CONF=/nonexistent/virt-cloud-wg-fabric.conf", "OLD_UNDERLAY=", "OLD_FIREWALL_RULE_OWNER=", _FIREWALL_MODE_GUARD_SH, rendered[preflight_start:preflight_end], rendered[endpoint_start:endpoint_end], )) return subprocess.run( ["/bin/sh", "-c", shell], env={ "PATH": "/usr/bin:/bin", "VC_LOCAL_ADDRESS": local_address, "VC_LOCAL_DEVICE": local_device, "VC_PLAIN_DEV": plain_device, "VC_PLAIN_SRC": plain_source, "VC_FROM_DEV": from_device, "VC_FROM_SRC": from_source, }, check=False, capture_output=True, text=True, ) def test_native_apply_uses_local_endpoint_without_firewalld_metadata(tmp_path): result = _run_native_apply_underlay_preflight(tmp_path) assert result.returncode == 0, result.stderr current = plan() rendered = _apply_script( current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}, firewall_mode=FIREWALL_MODE_NATIVE, ) assert "LOCAL_ENDPOINT=203.0.113.1" in rendered assert 'vc_underlay_route_path 203.0.113.2 "$LOCAL_ENDPOINT"' in rendered assert "underlay|local|" not in rendered assert "firewall_rule|" not in rendered @pytest.mark.parametrize(("kwargs", "expected_status"), [ ({"local_address": "192.0.2.99"}, 71), ({"plain_source": "192.0.2.99"}, 77), ({"from_source": "192.0.2.99"}, 77), ({"from_device": "eth1"}, 77), ]) def test_native_apply_rejects_nonlocal_or_inexact_underlay_path( tmp_path, kwargs, expected_status, ): result = _run_native_apply_underlay_preflight(tmp_path, **kwargs) assert result.returncode == expected_status, result.stderr @pytest.mark.parametrize( "interface", ["vnet3", "vcv-123", "tap3", "macvtap3", "fwpr3", "fwln3"], ) def test_native_apply_rejects_guest_or_tenant_underlay_interface( tmp_path, interface): result = _run_native_apply_underlay_preflight( tmp_path, local_device=interface, plain_device=interface, from_device=interface, ) assert result.returncode == 71, result.stderr def test_fabric_nft_validator_rejects_unowned_rules_and_extra_objects(): digest = "a" * 64 rules = _nft_rules_text(digest) def validate(candidate: str) -> subprocess.CompletedProcess[str]: return subprocess.run( ["/bin/sh", "-c", "\n".join(( "set -eu", _FABRIC_NFT_VALIDATION_SH, 'vc_fabric_nft_strict_shape "$VC_NFT_TEXT"', ))], env={"PATH": "/usr/bin:/bin", "VC_NFT_TEXT": candidate}, check=False, capture_output=True, text=True, ) accepted = validate(rules) assert accepted.returncode == 0, accepted.stderr with_tenants = rules.replace( " flags interval;", " flags interval;\n" " elements = { 10.10.0.2, 172.16.4.9/32, 192.168.1.8 }", 1, ) accepted_tenants = validate(with_tenants) assert accepted_tenants.returncode == 0, accepted_tenants.stderr early_accept = rules.replace( ' iifname "wg-fabric0" ip protocol icmp accept', ' meta iifname "wg-fabric0" accept\n' ' iifname "wg-fabric0" ip protocol icmp accept', 1, ) impossible_drop = rules.replace( ' iifname "wg-fabric0" counter drop', ' iifname "foreign0" iifname "wg-fabric0" counter drop', 1, ) extra_chain = rules.replace( " chain input {", " chain bypass {\n type filter hook input priority -101; " "policy accept;\n }\n chain input {", 1, ) extra_set = rules.replace( " set vpc_tenant_ips {", " set foreign_ips { type ipv4_addr; }\n" " set vpc_tenant_ips {", 1, ) wrong_set_type = rules.replace("type ipv4_addr;", "type inet_service;", 1) wrong_set_flags = rules.replace("flags interval;", "flags timeout;", 1) extra_object = rules.replace( " chain input {", " counter foreign_counter { }\n chain input {", 1, ) non_private_element = rules.replace( " flags interval;", " flags interval;\n elements = { 0.0.0.0/0 }", 1, ) for tampered in ( early_accept, impossible_drop, extra_chain, extra_set, wrong_set_type, wrong_set_flags, extra_object, non_private_element, ): assert validate(tampered).returncode != 0 def test_observe_nft_guard_matches_exact_rendered_owner_comments(): current = plan() observe = _observe_script( (), deployment_mode="ifupdown", firewall_mode=FIREWALL_MODE_NATIVE, expected_digest=current.digest, ) owner_regex = ( f"""grep -Ec 'comment "{FABRIC_OWNER_MARKER}:[0-9a-f]{{64}}"'""" ) digest_literal = ( f'grep -Foc "comment \\"{FABRIC_OWNER_MARKER}:$config_digest\\""' ) assert owner_regex in observe assert digest_literal in observe guard_start = observe.index( 'if [ -n "$NFT" ]; then', observe.index("nft_guard=missing") ) guard_end = observe.index("emergency_state=unknown", guard_start) guard = observe[guard_start:guard_end] program = "\n".join(( "set -eu", _FABRIC_NFT_VALIDATION_SH, "vc_test_nft() {", ' [ "$#" = 5 ] || return 1', ' [ "$1" = -s ] && [ "$2" = list ] && [ "$3" = table ] &&', ' [ "$4" = inet ] && [ "$5" = virt_cloud_wg_fabric ] || return 1', ' printf \'%s\\n\' "$VC_NFT_TEXT"', "}", "NFT=vc_test_nft", f"config_digest={shlex.quote(current.digest)}", "nft_guard=missing", guard, 'printf \'%s\\n\' "$nft_guard"', )) def readback(candidate): return subprocess.run( ["/bin/sh", "-c", program], env={"PATH": "/usr/bin:/bin", "VC_NFT_TEXT": candidate}, check=False, capture_output=True, text=True, ) owned = readback(_nft_rules_text(current.digest)) assert owned.returncode == 0, owned.stderr assert owned.stdout == "owned\n" stale = readback(_nft_rules_text("0" * 64)) assert stale.returncode == 0, stale.stderr assert stale.stdout == "missing\n" def test_fabric_nft_validator_accepts_alma_output_dstnat_priority_only(): digest = "a" * 64 rules = _nft_rules_text(digest) def validate(candidate: str) -> subprocess.CompletedProcess[str]: return subprocess.run( ["/bin/sh", "-c", "\n".join(( "set -eu", _FABRIC_NFT_VALIDATION_SH, 'vc_fabric_nft_strict_shape "$VC_NFT_TEXT"', ))], env={"PATH": "/usr/bin:/bin", "VC_NFT_TEXT": candidate}, check=False, capture_output=True, text=True, ) alma_canonical = rules.replace( "type filter hook output priority -100; policy accept;", "type filter hook output priority dstnat; policy accept;", 1, ) accepted = validate(alma_canonical) assert accepted.returncode == 0, accepted.stderr for hook in ("input", "forward"): invalid = rules.replace( f"type filter hook {hook} priority -100; policy accept;", f"type filter hook {hook} priority dstnat; policy accept;", 1, ) assert validate(invalid).returncode != 0 def test_boot_recovery_accepts_alma_output_dstnat_priority(): digest = "a" * 64 alma_canonical = _nft_rules_text(digest).replace( "type filter hook output priority -100; policy accept;", "type filter hook output priority dstnat; policy accept;", 1, ) helper = _BOOT_HELPER.replace( "__VC_FABRIC_NFT_VALIDATION__", _FABRIC_NFT_VALIDATION_SH, ) function_start = helper.index("vc_fabric_nft_strict_shape() {") function_end = helper.index("\n}", function_start) + 2 validator = helper[function_start:function_end] result = subprocess.run( ["/bin/sh", "-c", "\n".join(( "set -eu", validator, 'vc_fabric_nft_strict_shape "$VC_NFT_TEXT"', ))], env={"PATH": "/usr/bin:/bin", "VC_NFT_TEXT": alma_canonical}, check=False, capture_output=True, text=True, ) assert result.returncode == 0, result.stderr def _bridge_port_snapshot_runner(tmp_path, mode): start = _BOOT_HELPER.index("vc_bridge_port_names() {") end = _BOOT_HELPER.index("\n}\nvc_vpc_emergency_down()", start) + 2 runtime = tmp_path / "run" runtime.mkdir() function = _BOOT_HELPER[start:end].replace( "/run/virt-cloud", str(runtime), ) bridge = tmp_path / "bridge" bridge.write_text( """#!/bin/sh case "$VC_BRIDGE_MODE" in failed) exit 1 ;; malformed) printf '%s\n' 'not-a-bridge-link-record' ;; duplicate) printf '%s\n' '2: tap0@if3: master vcv-0123456789a' printf '%s\n' '4: tap0@if5: master vcv-0123456789a' ;; excessive) i=0 while [ "$i" -lt 4097 ]; do printf '%s: tap%s: master vcv-0123456789a\n' "$i" "$i" i=$((i + 1)) done ;; valid) printf '%s\n' '4: vnet2@if5: master vcv-0123456789a' printf '%s\n' '2: tap1@if3: master vcv-0123456789a' ;; *) exit 2 ;; esac """, encoding="utf-8", ) bridge.chmod(0o700) stat = tmp_path / "stat" stat.write_text( "#!/bin/sh\n/usr/bin/stat -f '%z' \"$3\"\n", encoding="utf-8", ) stat.chmod(0o700) script = "\n".join(( "set -eu", f"BRIDGE_CMD='{bridge}'", f"STAT='{stat}'", "AWK=/usr/bin/awk", function, "vc_bridge_port_names vcv-0123456789a", )) result = subprocess.run( ["/bin/sh", "-c", script], env={"PATH": "/usr/bin:/bin", "VC_BRIDGE_MODE": mode}, check=False, capture_output=True, text=True, ) return result, runtime, function @pytest.mark.parametrize( "mode", ("failed", "malformed", "duplicate", "excessive"), ) def test_bridge_port_snapshot_fails_closed_on_command_or_parse_errors( tmp_path, mode, ): result, runtime, _ = _bridge_port_snapshot_runner(tmp_path, mode) assert result.returncode == 1, result.stderr assert list(runtime.iterdir()) == [] def test_bridge_port_snapshot_sorts_valid_unique_names(tmp_path): result, runtime, function = _bridge_port_snapshot_runner( tmp_path, "valid", ) assert result.returncode == 0, result.stderr assert result.stdout == "tap1\nvnet2\n" assert list(runtime.iterdir()) == [] assert '"$BRIDGE_CMD" link show master' in function assert '"$BRIDGE_CMD" link show master "$vc_bridge_ports_bridge" |' \ not in _BOOT_HELPER def _vpc_force_down_runner(tmp_path, *, link_up, nm_fails=False): start = _BOOT_HELPER.index("vc_link_down_proven() {") end = _BOOT_HELPER.index("\n}\nvc_vpc_emergency_down()", start) + 2 functions = _BOOT_HELPER[start:end] ip = tmp_path / "ip" ip.write_text( """#!/bin/sh printf '%s\n' "$*" >> "$VC_IP_LOG" case "$*" in 'link set dev vcv-0123456789a down') exit 0 ;; '-o link show dev vcv-0123456789a') if [ "$VC_LINK_UP" = yes ]; then printf '%s\n' '2: vcv-0123456789a: mtu 1500' else printf '%s\n' '2: vcv-0123456789a: mtu 1500' fi exit 0 ;; *) exit 2 ;; esac """, encoding="utf-8", ) ip.chmod(0o700) nmcli = tmp_path / "nmcli" nmcli.write_text("#!/bin/sh\nexit 1\n", encoding="utf-8") nmcli.chmod(0o700) log = tmp_path / "ip.log" body = [ "set -eu", f"IP='{ip}'", f"NMCLI='{nmcli}'", "AWK=/usr/bin/awk", functions, ] if nm_fails: body.extend(( "nm_failed=no", '"$NMCLI" device show vcv-0123456789a >/dev/null 2>&1 || ' "nm_failed=yes", "vc_vpc_bridge_force_down vcv-0123456789a", '[ "$nm_failed" = yes ]', )) else: body.append("vc_vpc_bridge_force_down vcv-0123456789a") result = subprocess.run( ["/bin/sh", "-c", "\n".join(body)], env={ "PATH": "/usr/bin:/bin", "VC_IP_LOG": str(log), "VC_LINK_UP": "yes" if link_up else "no", }, check=False, capture_output=True, text=True, ) return result, log def test_vpc_force_down_rejects_nm_race_that_restores_up_flag(tmp_path): result, log = _vpc_force_down_runner(tmp_path, link_up=True) assert result.returncode == 1, result.stderr assert log.read_text(encoding="utf-8").splitlines() == [ "link set dev vcv-0123456789a down", "-o link show dev vcv-0123456789a", ] def test_vpc_force_down_still_runs_and_proves_isolation_after_nm_failure( tmp_path, ): result, log = _vpc_force_down_runner( tmp_path, link_up=False, nm_fails=True, ) assert result.returncode == 0, result.stderr assert log.read_text(encoding="utf-8").splitlines() == [ "link set dev vcv-0123456789a down", "-o link show dev vcv-0123456789a", ] early = _BOOT_HELPER.index("vc_vpc_emergency_managed=") early_down = _BOOT_HELPER.index( 'vc_vpc_bridge_force_down "$vc_vpc_emergency_bridge"', early, ) early_after = _BOOT_HELPER.index( "vc_vpc_emergency_ports_after=", early_down, ) assert early < early_down < early_after def test_apply_queues_reconcile_dispatcher_only_after_durable_commit(): current = plan() script = _apply_script( current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}, firewall_mode=FIREWALL_MODE_NATIVE, ) committed_marker = script.rindex('committed > "$_txn_state_tmp"') committed_flag = script.rindex("COMMITTED=yes") unlock = script.rindex("exec 9>&-") queue = script.rindex( '"$SYSTEMCTL" start ' 'virt-cloud-wg-fabric-reconcile.service' ) relock = script.rindex('exec 9>>"$LOCK"') primary_readback = script.rindex( 'is-active virt-cloud-wg-fabric.service 2>/dev/null') emergency_readback = script.rindex( 'final_emergency_tables_verify=$($NFT list tables') success = script.rindex("__VC_FABRIC_APPLIED__") assert ( committed_marker < committed_flag < unlock < queue < relock < primary_readback < emergency_readback < success ) assert "Type=oneshot" in _UNIT assert "RemainAfterExit=yes" in _UNIT assert ( '"$SYSTEMCTL" start virt-cloud-wg-fabric.service' not in script[committed_flag:] ) assert "vc_primary_inactive_retry_ready()" in _BOOT_HELPER assert "retry_primary_inactive=yes" in _BOOT_HELPER primary = _BOOT_HELPER.index( '"$SYSTEMCTL" start virt-cloud-wg-fabric.service') active = _BOOT_HELPER.index( 'is-active virt-cloud-wg-fabric.service', primary) daemon = _BOOT_HELPER.index( 'vc_gate_retry_start_claimed "$retry_gate_unit"', active) assert primary < active < daemon def _run_apply_convergence_tail( tmp_path: Path, *, start_fails: bool = False, timer_fails: bool = False, stale_config: bool = False, emergency_present: bool = False, committed_trap_state: str | None = None, ) -> subprocess.CompletedProcess[str]: current = plan() rendered = _apply_script( current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}, firewall_mode=FIREWALL_MODE_NATIVE, ) tail_start = rendered.rindex("COMMITTED=yes") marker = "printf '%s\\n' '__VC_FABRIC_APPLIED__'" tail_end = rendered.rindex(marker) + len(marker) rollback_start = rendered.index("rollback() {") rollback_end = rendered.index( "\n}\ntrap rollback EXIT HUP INT TERM", rollback_start, ) + 2 rollback_function = rendered[rollback_start:rollback_end] logger_start = rendered.index("vc_fabric_apply_failure_log() {") logger_end = rendered.index("\n}\n", logger_start) + 2 failure_logger = rendered[logger_start:logger_end] emergency_owner = tmp_path / "emergency.owner" sysctl_file = tmp_path / "fabric-sysctl.conf" sysctl_file.write_text(_sysctl_text(current), encoding="utf-8") sysctl_file.chmod(0o644) rp_filter_file = tmp_path / "rp_filter" rp_filter_file.write_text("2\n", encoding="utf-8") tail = rendered[tail_start:tail_end].replace( "/run/virt-cloud/wg-fabric-emergency.owner", str(emergency_owner), ).replace( '"/proc/sys/net/ipv4/conf/$IFACE/rp_filter"', f"'{rp_filter_file}'", ) lock = tmp_path / "fabric.lock" lock.touch(mode=0o600) conf = tmp_path / "fabric.conf" conf.write_text( f"digest|{'0' * 64 if stale_config else current.digest}\n" f"firewall|{FIREWALL_MODE_NATIVE}\n", encoding="utf-8", ) conf.chmod(0o600) key = tmp_path / "fabric.key" key.write_text("private\n", encoding="utf-8") converged = tmp_path / "converged" systemctl = tmp_path / "systemctl" systemctl.write_text( """#!/bin/sh case "$1:$2" in start:virt-cloud-wg-fabric-reconcile.timer) [ "${VC_TEST_TIMER_FAIL:-no}" = no ] || exit 1 exit 0 ;; start:virt-cloud-wg-fabric-reconcile.service) [ "${VC_TEST_START_FAIL:-no}" = no ] || exit 1 : > "$VC_TEST_CONVERGED"; exit 0 ;; is-active:virt-cloud-wg-fabric.service|\ is-active:virt-cloud-wg-fabric-reconcile.timer) [ -f "$VC_TEST_CONVERGED" ] || exit 1 printf '%s\n' active; exit 0 ;; *) exit 2 ;; esac """, encoding="utf-8", ) systemctl.chmod(0o700) nft = tmp_path / "nft" nft.write_text( """#!/bin/sh case "$*" in '-s list table inet virt_cloud_wg_fabric') printf '%s\n' managed-fabric-table ;; 'list tables') if [ "${VC_TEST_EMERGENCY:-no}" = yes ]; then printf '%s\n' 'table inet virt_cloud_fabric_emergency' printf '%s\n' 'table bridge virt_cloud_fabric_emergency' else printf '%s\n' 'table inet virt_cloud_wg_fabric' fi ;; *) exit 2 ;; esac """, encoding="utf-8", ) nft.chmod(0o700) wg = tmp_path / "wg" wg.write_text( """#!/bin/sh case "$*" in 'show wg-fabric0') exit 0 ;; 'pubkey') printf '%s\n' mock-public-key ;; 'show wg-fabric0 public-key') printf '%s\n' mock-public-key ;; 'show wg-fabric0 listen-port') printf '%s\n' 51820 ;; *) exit 2 ;; esac """, encoding="utf-8", ) wg.chmod(0o700) ip = tmp_path / "ip" ip.write_text( "#!/bin/sh\n" "[ \"$*\" = '-o link show dev wg-fabric0' ] || exit 2\n" "printf '%s\\n' '1: wg-fabric0: mtu 1420'\n", encoding="utf-8", ) ip.chmod(0o700) stat = tmp_path / "stat" stat.write_text("#!/bin/sh\nprintf '%s\\n' '0:600'\n", encoding="utf-8") stat.chmod(0o700) flock = tmp_path / "flock" flock.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") flock.chmod(0o700) trap_setup: tuple[str, ...] = () if committed_trap_state is not None: journal = tmp_path / "txn" journal.mkdir() (journal / "state").write_text( f"{committed_trap_state}\n", encoding="utf-8", ) trap_setup = ( f"TXN='{journal}'", "JOURNAL_READY=yes", "cleanup() { :; }", rollback_function, "trap rollback EXIT HUP INT TERM", ) harness = "\n".join(( "set -eu", f"LOCK='{lock}'", f"CONF='{conf}'", f"KEY='{key}'", f"SYSCTL_FILE='{sysctl_file}'", "IFACE=wg-fabric0", f"EXPECTED_DIGEST='{current.digest}'", f"FIREWALL_MODE='{FIREWALL_MODE_NATIVE}'", f"SYSTEMCTL='{systemctl}'", f"NFT='{nft}'", f"WG='{wg}'", f"IP='{ip}'", f"STAT='{stat}'", f"FLOCK='{flock}'", "nft_plan_owned() { [ \"$2\" = \"$EXPECTED_DIGEST\" ]; }", "vpc_state_nft_set_matches() { return 0; }", "vc_fabric_sysctl_file_owned() { return 0; }", "read_sysctl() { cat \"$1\"; }", "vc_fabric_operation_checkpoint() { :; }", failure_logger, *trap_setup, "exec 9>>\"$LOCK\"", tail, )) return subprocess.run( ["/bin/sh", "-c", harness], check=False, capture_output=True, text=True, env={ "PATH": "/usr/bin:/bin", "VC_TEST_START_FAIL": "yes" if start_fails else "no", "VC_TEST_TIMER_FAIL": "yes" if timer_fails else "no", "VC_TEST_EMERGENCY": "yes" if emergency_present else "no", "VC_TEST_CONVERGED": str(converged), }, ) def test_apply_convergence_tail_requires_sync_success_and_final_clear(tmp_path): success = _run_apply_convergence_tail(tmp_path) assert success.returncode == 0, success.stderr assert "__VC_FABRIC_APPLIED__" in success.stdout @pytest.mark.parametrize( "failure", ["start", "stale_config", "emergency"], ) def test_apply_convergence_tail_refuses_unconfirmed_state(tmp_path, failure): result = _run_apply_convergence_tail( tmp_path, start_fails=failure == "start", stale_config=failure == "stale_config", emergency_present=failure == "emergency", ) assert result.returncode == 79, result.stderr assert "__VC_FABRIC_APPLIED__" not in result.stdout def test_post_commit_dispatcher_failure_preserves_committed_journal(tmp_path): result = _run_apply_convergence_tail( tmp_path, start_fails=True, committed_trap_state="committed", ) assert result.returncode == 79, result.stderr assert (tmp_path / "txn" / "state").read_text( encoding="utf-8", ) == "committed\n" def test_post_commit_timer_failure_preserves_committed_journal(tmp_path): result = _run_apply_convergence_tail( tmp_path, timer_fails=True, committed_trap_state="committed", ) assert result.returncode == 79, result.stderr assert (tmp_path / "txn" / "state").read_text( encoding="utf-8", ) == "committed\n" def test_old_committed_trap_never_deletes_concurrent_ready_journal(tmp_path): result = _run_apply_convergence_tail( tmp_path, start_fails=True, committed_trap_state="ready", ) assert result.returncode == 79, result.stderr assert (tmp_path / "txn" / "state").read_text( encoding="utf-8", ) == "ready\n" def test_inactive_primary_retry_requires_complete_managed_evidence(tmp_path): start = _BOOT_HELPER.index("vc_primary_inactive_retry_ready() {") end = _BOOT_HELPER.index("\n}\nvc_fabric_rpf_repair_needed()", start) + 2 function = _BOOT_HELPER[start:end] digest = "a" * 64 conf = tmp_path / "wg-fabric0.conf" conf.write_text( f"digest|{digest}\nfirewall|native_nftables\n", encoding="utf-8", ) owner = tmp_path / "wg-fabric0.owner" owner.write_text("virt-cloud-wg-fabric-v1\n", encoding="utf-8") helper = tmp_path / "helper" helper.write_text( f"#!/bin/sh\n# virt-cloud-wg-fabric-v1\n" f"# fabric-plan-digest={digest}\n", encoding="utf-8", ) unit = tmp_path / "unit" unit.write_text( "\n".join(( "# virt-cloud-wg-fabric-v1", f"# fabric-plan-digest={digest}", "Type=oneshot", "RemainAfterExit=yes", "ExecStart=/bin/sh /usr/local/libexec/virt-cloud-wg-fabric", )) + "\n", encoding="utf-8", ) stat = tmp_path / "stat" stat.write_text( """#!/bin/sh case "$3" in *wg-fabric0.conf|*wg-fabric0.owner) printf '%s\n' 0:600 ;; *helper) printf '%s\n' 0:700 ;; *unit) printf '%s\n' 0:644 ;; *) exit 1 ;; esac """, encoding="utf-8", ) stat.chmod(0o700) systemctl = tmp_path / "systemctl" systemctl.write_text( """#!/bin/sh case "$1" in is-active) printf '%s\n' inactive ;; is-enabled) printf '%s\n' enabled ;; *) exit 2 ;; esac """, encoding="utf-8", ) systemctl.chmod(0o700) wg = tmp_path / "wg" wg.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") wg.chmod(0o700) base_script = "\n".join(( "set -eu", f"SYSTEMCTL='{systemctl}'", f"STAT='{stat}'", f"WG='{wg}'", f"CONF='{conf}'", f"OWNER='{owner}'", f"HELPER='{helper}'", f"UNIT='{unit}'", f"PLAN_DIGEST='{digest}'", "FIREWALL_MODE=native_nftables", "IFACE=wg-fabric0", function, "vc_primary_inactive_retry_ready", )) accepted = subprocess.run( ["/bin/sh", "-c", base_script], check=False, capture_output=True, text=True, ) assert accepted.returncode == 0, accepted.stderr unit.write_text(unit.read_text(encoding="utf-8").replace( "RemainAfterExit=yes", "RemainAfterExit=no", ), encoding="utf-8") rejected = subprocess.run( ["/bin/sh", "-c", base_script], check=False, capture_output=True, text=True, ) assert rejected.returncode == 1 def test_fabric_nft_journal_records_only_post_refresh_as_applied(): current = plan() script = _apply_script( current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}, firewall_mode=FIREWALL_MODE_NATIVE, ) installing = script.index("write_nft_phase installing") installed_snapshot = script.index( '"$nft_installed" > "$TXN/nft.installed.tmp"', installing, ) installed = script.index("write_nft_phase installed", installed_snapshot) refreshing = script.index("write_nft_phase refreshing", installed) refresh = script.index("vpc_state_refresh_nft || exit 71", refreshing) exact_set = script.index("vpc_state_nft_set_matches || exit 71", refresh) applied_snapshot = script.index( '"$fabric_acl_applied" > "$TXN/nft.applied.tmp"', exact_set, ) refreshed = script.index("write_nft_phase refreshed", applied_snapshot) assert ( installing < installed_snapshot < installed < refreshing < refresh < exact_set < applied_snapshot < refreshed ) assert "before_mutation|installing|installed|refreshing|refreshed" in script assert 'case "$rollback_nft_phase" in' in script assert 'vpc_state_nft_set_matches || ROLLBACK_FAILED=yes' in script def test_boot_recovery_handles_both_sides_of_nft_refresh_fail_closed(): helper = _BOOT_HELPER phase_read = helper.index( 'journal_nft_phase=$(cat "$JOURNAL/nft.phase"') pre_refresh_snapshot = helper.index( 'journal_file_secure nft.installed || exit 79', phase_read) refreshing_case = helper.index("refreshing)", pre_refresh_snapshot) full_shape = helper.index( 'nft_plan_owned "$journal_refresh_text" "$journal_plan_digest"', refreshing_case, ) exact_set = helper.index("vpc_state_nft_set_matches || exit 79", full_shape) stable_recheck = helper.index( '"$CMP" -s "$JOURNAL/nft.current" ' '"$JOURNAL/nft.refresh-verify"', exact_set, ) final_snapshot = helper.index( '"$CMP" -s "$JOURNAL/nft.current" "$JOURNAL/nft.applied"', stable_recheck, ) destructive_batch = helper.index( 'delete table inet virt_cloud_wg_fabric', final_snapshot) assert ( phase_read < pre_refresh_snapshot < refreshing_case < full_shape < exact_set < stable_recheck < final_snapshot < destructive_batch ) assert ( "Crash between nft commit and its complete snapshot is\n" " # intentionally ambiguous" ) in helper assert ( 'elif [ "$journal_nft_phase" = refreshing ] &&\n' ' { [ -e "$JOURNAL/nft.applied" ] || ' '[ -L "$JOURNAL/nft.applied" ]; }; then' ) in helper assert ( '"$CMP" -s "$JOURNAL/nft.current" ' '"$JOURNAL/nft.applied" || exit 79' ) in helper def test_first_deploy_recovery_uses_only_fenced_temporary_helper(): current = plan() script = _apply_script( current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}, firewall_mode=FIREWALL_MODE_NATIVE, ) helper_created = script.index( '_tmp_helper=$(mktemp "$BASE/.wg-fabric0.helper.XXXXXX")') transaction = script.index("COMMITTED=no", helper_created) ready_case = script.index(" ready)", transaction) ready_end = script.index(" *) exit 79 ;;", ready_case) helper_published = script.index( 'publish_desired_file "$_tmp_helper" "$HELPER" helper', ready_end) ready_contract = script[ready_case:ready_end] first_deploy = ready_contract.index( 'transaction_first_deploy_helper_absent=no') missing_snapshot = ready_contract.index( 'transaction_missing_installing_snapshot=no') special_first = ready_contract.index( 'if [ "$transaction_first_deploy_helper_absent" = yes ]; then', missing_snapshot, ) temporary_installing = ready_contract.index( 'VC_FABRIC_LOCK_HELD=1 /bin/sh "$_tmp_helper" ' '--recover-installing || exit 79', special_first, ) temporary_current = ready_contract.index( 'VC_FABRIC_LOCK_HELD=1 /bin/sh "$_tmp_helper" ' '--recover-current || exit 79', temporary_installing, ) fixed_helper_owned = ready_contract.index( 'transaction_recovery_helper_owned || exit 79', temporary_current, ) fixed_recovery = ready_contract.index( 'VC_FABRIC_LOCK_HELD=1 /bin/sh "$HELPER" --recover-only || exit 79', fixed_helper_owned, ) assert helper_created < transaction < ready_case < helper_published assert ( first_deploy < missing_snapshot < special_first < temporary_installing < temporary_current < fixed_helper_owned < fixed_recovery ) assert ( 'VC_FABRIC_LOCK_HELD=1 /bin/sh "$HELPER" --recover-only || exit 79' in ready_contract ) assert ( 'VC_FABRIC_LOCK_HELD=1 /bin/sh "$_tmp_helper" ' '--recover-installing || exit 79' ) in ready_contract assert ( 'VC_FABRIC_LOCK_HELD=1 /bin/sh "$_tmp_helper" ' '--recover-current || exit 79' ) in ready_contract assert '"$_tmp_helper" --recover-only' not in ready_contract assert '"$HELPER" --recover-installing' not in ready_contract assert '"$HELPER" --recover-current' not in ready_contract def _run_transaction_recovery_helper_owned_harness( tmp_path, *, scenario, ): current = plan() apply = _apply_script( current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}, firewall_mode=FIREWALL_MODE_NATIVE, ) function_start = apply.index("transaction_recovery_helper_owned() {") function_end = apply.index( '\n}\nif [ -L "$BASE" ]', function_start, ) + 2 function = apply[function_start:function_end] base = tmp_path / "base" journal = base / ".wg-fabric-txn" journal.mkdir(parents=True) helper = tmp_path / "fixed-helper" owned = b"#!/bin/sh\n# owned helper\n" fixed = owned if scenario != "foreign_fixed_helper" else ( b"#!/bin/sh\n# foreign helper\n" ) helper.write_bytes(fixed) helper.chmod(0o700) if scenario in ("baseline_exact", "foreign_fixed_helper", "baseline_marker_drift"): marker = journal / "helper.present" marker.write_bytes( b"drift\n" if scenario == "baseline_marker_drift" else b"", ) marker.chmod(0o600) baseline = journal / "helper" baseline.write_bytes(owned) baseline.chmod(0o700) elif scenario in ("desired_exact", "desired_metadata_drift"): desired = journal / "helper.desired" desired.write_bytes(owned) desired.chmod(0o700) if scenario == "desired_metadata_drift": marker = journal / "helper.desired.absent" marker.write_bytes(b"") marker.chmod(0o600) else: raise AssertionError(f"unknown helper scenario: {scenario}") fake_stat = tmp_path / "stat" fake_stat.write_text( """#!/bin/sh case "$3" in */helper.present) printf '0:600\n' ;; */helper|*/helper.desired|*/fixed-helper) printf '0:700\n' ;; *) exit 2 ;; esac """, encoding="utf-8", ) fake_stat.chmod(0o700) script = "\n".join(( "set -eu", f"BASE={shlex.quote(str(base))}", f"HELPER={shlex.quote(str(helper))}", f"STAT={shlex.quote(str(fake_stat))}", "CMP=/usr/bin/cmp", function, "if transaction_recovery_helper_owned; then", " exit 0", "fi", "exit 17", )) return subprocess.run( ["/bin/sh", "-c", script], check=False, capture_output=True, text=True, ) @pytest.mark.parametrize("scenario", ("baseline_exact", "desired_exact")) def test_transaction_recovery_fixed_helper_accepts_exact_journal_image( tmp_path, scenario, ): result = _run_transaction_recovery_helper_owned_harness( tmp_path, scenario=scenario, ) assert result.returncode == 0, result.stderr @pytest.mark.parametrize("scenario", ( "foreign_fixed_helper", "baseline_marker_drift", "desired_metadata_drift", )) def test_transaction_recovery_fixed_helper_rejects_unproven_image( tmp_path, scenario, ): result = _run_transaction_recovery_helper_owned_harness( tmp_path, scenario=scenario, ) assert result.returncode == 17 def _run_ready_recovery_dispatch_harness( tmp_path, *, first_deploy, phase, installed, deny_direct_exec=False, ): current = plan() apply = _apply_script( current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}, firewall_mode=FIREWALL_MODE_NATIVE, ) function_start = apply.index("transaction_recovery_helper_owned() {") function_end = apply.index( '\n}\nif [ -L "$BASE" ]', function_start, ) + 2 function = apply[function_start:function_end] ready = apply.index(" ready)", function_end) ready_body_start = apply.index("\n", ready) + 1 ready_body_end = apply.index( " ;;\n *) exit 79 ;;", ready_body_start, ) ready_body = apply[ready_body_start:ready_body_end] base = tmp_path / "base" journal = base / ".wg-fabric-txn" journal.mkdir(parents=True) phase_file = journal / "nft.phase" phase_file.write_text(f"{phase}\n", encoding="utf-8") phase_file.chmod(0o600) if installed: installed_file = journal / "nft.installed" installed_file.write_bytes(b"complete nft snapshot\n") installed_file.chmod(0o600) log = tmp_path / "dispatch.log" digest = current.digest def helper_text(label, options): cases = "|".join(options) return ( "#!/bin/sh\n" "# virt-cloud-wg-fabric-v1\n" f"# fabric-plan-digest={digest}\n" f'PLAN_DIGEST="{digest}"\n' f"# accepted recovery arguments: {cases}\n" f"case \"${{1:-}}\" in {cases}) : ;; *) exit 31 ;; esac\n" f"printf '%s:%s\\n' {shlex.quote(label)} \"$1\" > " f"{shlex.quote(str(log))}\n" f"rm -rf {shlex.quote(str(journal))}\n" ) temporary = tmp_path / "temporary-helper" temporary.write_text( helper_text("temporary", ("--recover-current", "--recover-installing")), encoding="utf-8", ) temporary.chmod(0o600 if deny_direct_exec else 0o700) fixed = tmp_path / "fixed-helper" if first_deploy: marker = journal / "helper.absent" marker.write_bytes(b"") marker.chmod(0o600) else: fixed_bytes = helper_text("fixed", ("--recover-only",)).encode() fixed.write_bytes(fixed_bytes) fixed.chmod(0o600 if deny_direct_exec else 0o700) marker = journal / "helper.present" marker.write_bytes(b"") marker.chmod(0o600) baseline = journal / "helper" baseline.write_bytes(fixed_bytes) baseline.chmod(0o700) fake_stat = tmp_path / "stat" fake_stat.write_text( f"""#!{sys.executable} import os import stat import sys if len(sys.argv) != 4 or sys.argv[1] != "-c": raise SystemExit(2) value = os.lstat(sys.argv[3]) mode = format(stat.S_IMODE(value.st_mode), "o") if {deny_direct_exec!r} and os.path.basename(sys.argv[3]) in ( "temporary-helper", "fixed-helper"): # Model an executable root-owned file on a noexec mount: its metadata # passes the contract while the OS rejects a direct exec. mode = "700" if sys.argv[2] == "%a": print(mode) elif sys.argv[2] == "%u": print("0") elif sys.argv[2] == "%u:%a": print(f"0:{{mode}}") else: raise SystemExit(2) """, encoding="utf-8", ) fake_stat.chmod(0o700) harness = "\n".join(( "set -eu", f"BASE={shlex.quote(str(base))}", f"HELPER={shlex.quote(str(fixed))}", f"_tmp_helper={shlex.quote(str(temporary))}", f"STAT={shlex.quote(str(fake_stat))}", "CMP=/usr/bin/cmp", f"EXPECTED_DIGEST={shlex.quote(digest)}", "TRANSACTION_RECOVERED=no", "vc_fabric_operation_checkpoint() { :; }", function, ready_body, '[ "$TRANSACTION_RECOVERED" = yes ]', )) result = subprocess.run( ["/bin/sh", "-c", harness], check=False, capture_output=True, text=True, ) return result, log @pytest.mark.parametrize(("phase", "installed", "expected"), ( ("before_mutation", False, "--recover-current"), ("installing", False, "--recover-installing"), ("installing", True, "--recover-current"), ("installed", True, "--recover-current"), ("refreshing", True, "--recover-current"), ("refreshed", True, "--recover-current"), )) def test_first_deploy_ready_dispatch_uses_current_plan_helper( tmp_path, phase, installed, expected, ): result, log = _run_ready_recovery_dispatch_harness( tmp_path, first_deploy=True, phase=phase, installed=installed, ) assert result.returncode == 0, result.stderr assert log.read_text(encoding="utf-8") == f"temporary:{expected}\n" @pytest.mark.parametrize(("first_deploy", "phase", "installed", "expected"), ( (True, "installing", False, "temporary:--recover-installing"), (True, "installed", True, "temporary:--recover-current"), (False, "installed", True, "fixed:--recover-only"), )) def test_ready_recovery_uses_interpreter_when_direct_execution_is_denied( tmp_path, first_deploy, phase, installed, expected): result, log = _run_ready_recovery_dispatch_harness( tmp_path, first_deploy=first_deploy, phase=phase, installed=installed, deny_direct_exec=True) helper = tmp_path / ( "temporary-helper" if first_deploy else "fixed-helper") with pytest.raises(PermissionError): subprocess.run([str(helper)], check=False, capture_output=True) assert result.returncode == 0, result.stderr assert log.read_text(encoding="utf-8") == expected + "\n" @pytest.mark.parametrize(("phase", "installed"), ( ("before_mutation", False), ("installing", False), ("installing", True), ("installed", True), ("refreshing", True), ("refreshed", True), )) def test_existing_ready_dispatch_uses_exact_owned_fixed_helper( tmp_path, phase, installed, ): result, log = _run_ready_recovery_dispatch_harness( tmp_path, first_deploy=False, phase=phase, installed=installed, ) assert result.returncode == 0, result.stderr assert log.read_text(encoding="utf-8") == "fixed:--recover-only\n" def test_installing_recovery_requires_exact_stable_first_deploy_evidence(): current = plan() script = _apply_script( current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}, firewall_mode=FIREWALL_MODE_NATIVE, ) recover_start = script.index("recover_journal() {") recover_end = script.index( '\n}\nif [ "${VC_FABRIC_APPLYING:-0}" != 1 ]; then', recover_start, ) + 2 recover = script[recover_start:recover_end] digest_fence = recover.index( 'if [ "${RECOVER_ONLY:-no}" != yes ] ||\n' ' [ "${RECOVER_CURRENT:-no}" = yes ]; then') digest_match = recover.index( '[ "$journal_plan_digest" = "$PLAN_DIGEST" ]', digest_fence) connection_match = recover.index( '[ "$journal_connection_uuid" = "$CONNECTION_UUID" ]', digest_match, ) deployment_match = recover.index( '[ "$journal_mode" = "$DEPLOYMENT_MODE" ]', connection_match) firewall_match = recover.index( '[ "$journal_firewall_mode" = "$FIREWALL_MODE" ]', deployment_match, ) special = recover.index( 'if [ "${RECOVER_INSTALLING:-no}" = yes ]; then', firewall_match) phase_gate = recover.index( '[ "$journal_nft_phase" = installing ]', special) absent_baseline = recover.index( '[ "$journal_nft_before" = no ]', phase_gate) clean_first = recover.index( 'for journal_clean_first in conf owner helper unit repair timer ' 'gate_generator nm_profile ifupdown_fragment sysctl nft_rules; do', absent_baseline, ) clean_markers = recover.index( 'journal_marker_secure "$journal_clean_first.absent" || exit 79', clean_first, ) desired_rules = recover.index( 'journal_file_secure nft_rules.desired || exit 79', clean_markers) desired_mode = recover.index( '$STAT -c \'%u:%a\' "$JOURNAL/nft_rules.desired"', desired_rules, ) desired_sha = recover.index( '"$SHA256" "$JOURNAL/nft_rules.desired"', desired_mode) expected_sha = recover.index( '[ "$journal_desired_nft_sha" = ' '"$EXPECTED_NFT_RULES_SHA256" ] || exit 79', desired_sha, ) desired_shape = recover.index( 'nft_plan_owned "$journal_desired_nft" ' '"$journal_plan_digest" || exit 79', expected_sha, ) desired_empty = recover.index( 'vc_fabric_nft_empty_tenant_set "$journal_desired_nft" ' '|| exit 79', desired_shape, ) owner_artifact = recover.index( 'journal_first_deploy_artifact_safe "$OWNER" owner || exit 79', desired_empty, ) sysctl_artifact = recover.index( 'journal_first_deploy_artifact_safe "$SYSCTL_FILE" sysctl ' '|| exit 79', owner_artifact, ) nft_rules_artifact = recover.index( 'journal_first_deploy_artifact_safe "$NFT_RULES" nft_rules ' '|| exit 79', sysctl_artifact, ) native_baseline = recover.index( 'journal_native_nft_main_safe || exit 79', nft_rules_artifact, ) current_owner = recover.index( 'journal_current_owner_state=absent', native_baseline) installing_owner_gate = recover.index( 'case "$journal_current_owner_state:' '$journal_current_iface_state" in', current_owner, ) first_read = recover.index( '$NFT -s list table inet virt_cloud_wg_fabric > ' '"$JOURNAL/nft.current"', desired_empty, ) live_shape = recover.index( 'nft_plan_owned "$journal_installing_text" ' '"$journal_plan_digest" || exit 79', first_read, ) live_empty = recover.index( 'vc_fabric_nft_empty_tenant_set "$journal_installing_text" ' '|| exit 79', live_shape, ) second_read = recover.index( '"$NFT" -s list table inet virt_cloud_wg_fabric > ' '"$_tmp_nft_installed"', live_empty, ) stable_compare = recover.index( '"$CMP" -s "$JOURNAL/nft.current" ' '"$_tmp_nft_installed" || exit 79', second_read, ) atomic_snapshot = recover.index( 'mv -f "$_tmp_nft_installed" "$JOURNAL/nft.installed"', stable_compare, ) installed_compare = recover.index( '"$CMP" -s "$JOURNAL/nft.current" ' '"$JOURNAL/nft.installed"', atomic_snapshot, ) expected_file = recover.index( 'journal_nft_expected_file="$JOURNAL/nft.installed"', installed_compare, ) allow_replace = recover.index( "journal_nft_action=replace", expected_file) verify_read = recover.index( '$NFT -s list table inet virt_cloud_wg_fabric > ' '"$JOURNAL/nft.verify"', allow_replace, ) verify_compare = recover.index( '"$CMP" -s "$JOURNAL/nft.verify" ' '"$journal_nft_expected_file" || exit 79', verify_read, ) precommit_read = recover.index( '$NFT -s list table inet virt_cloud_wg_fabric > ' '"$JOURNAL/nft.precommit"', verify_compare, ) precommit_compare = recover.index( '"$CMP" -s "$JOURNAL/nft.precommit" ' '"$journal_nft_expected_file" || exit 79', precommit_read, ) destructive_apply = recover.index( '"$NFT" -f "$_tmp_nft_rules"', precommit_compare) assert ( digest_fence < digest_match < connection_match < deployment_match < firewall_match < special < phase_gate < absent_baseline < clean_first < clean_markers < desired_rules < desired_mode < desired_sha < expected_sha < desired_shape < desired_empty < owner_artifact < sysctl_artifact < nft_rules_artifact < native_baseline < current_owner < installing_owner_gate < first_read < live_shape < live_empty < second_read < stable_compare < atomic_snapshot < installed_compare < expected_file < allow_replace < verify_read < verify_compare < precommit_read < precommit_compare < destructive_apply ) def _installing_recovery_fragments(): helper = _BOOT_HELPER.replace( "__VC_FABRIC_NFT_VALIDATION__", _FABRIC_NFT_VALIDATION_SH, ) ownership_start = helper.index("nft_panel_owned() {") ownership_end = helper.index("\nrecover_journal() {", ownership_start) ownership = helper[ownership_start:ownership_end] artifact_start = helper.index("journal_file_secure() {") artifact_end = helper.index( "\njournal_cleanup_helper_owned() {", artifact_start) artifacts = helper[artifact_start:artifact_end] special_start = helper.index( ' if [ "${RECOVER_INSTALLING:-no}" = yes ]; then') special_end = helper.index( " journal_applied_present=no", special_start) special = helper[special_start:special_end] current_owner_start = helper.index( " journal_current_owner_state=absent", special_end) current_owner_end = helper.index( " journal_nmcli=", current_owner_start) current_owner = helper[current_owner_start:current_owner_end] snapshot_start = helper.index( ' if [ "$journal_recover_installing" = yes ]; then') snapshot_end = helper.index( ' if [ -f "$JOURNAL/nft.installed" ] &&', snapshot_start, ) snapshot = helper[snapshot_start:snapshot_end] proof_end = helper.index("\n installed)", snapshot_end) installed_proof = helper[snapshot_end:proof_end] assert re.search(r"fi ;;\s*$", installed_proof) installed_proof = re.sub( r"fi ;;\s*$", "fi", installed_proof, ) return artifacts, ownership, special, current_owner, snapshot, installed_proof def _run_installing_recovery_harness( tmp_path, *, failure="none", artifact_state="post_trap_absent", firewall_mode=FIREWALL_MODE_NATIVE, native_state="baseline_current_no_desired", ): digest = "a" * 64 desired = _nft_rules_text(digest) if failure == "nonempty_tenant_set": desired = desired.replace( " flags interval;", " flags interval;\n" " elements = { 10.10.0.2 }", 1, ) elif failure == "extra_object": desired = desired.replace( " set vpc_tenant_ips {", " set foreign_ips { type ipv4_addr; }\n" " set vpc_tenant_ips {", 1, ) desired_bytes = desired.encode() expected_sha = hashlib.sha256(desired_bytes).hexdigest() if failure == "desired_hash": expected_sha = "0" * 64 alma_live = _nft_rules_text(digest).replace( "type filter hook output priority -100; policy accept;", "type filter hook output priority dstnat; policy accept;", 1, ) second_live = alma_live if failure == "second_read_drift": second_live = alma_live.replace( "type filter hook output priority dstnat; policy accept;", "type filter hook output priority -100; policy accept;", 1, ) journal = tmp_path / "journal" journal.mkdir(mode=0o700) clean_first = ( "conf", "owner", "helper", "unit", "repair", "timer", "gate_generator", "nm_profile", "ifupdown_fragment", "sysctl", "nft_rules", ) for name in clean_first: marker = journal / f"{name}.absent" marker.write_bytes(b"") marker.chmod(0o600) nft_main_marker = journal / ( "nft_main.present" if firewall_mode == FIREWALL_MODE_NATIVE else "nft_main.absent" ) nft_main_marker.write_bytes(b"") nft_main_marker.chmod(0o600) nft_before = journal / "nft.table" nft_before.write_bytes(b"") nft_before.chmod(0o600) desired_path = journal / "nft_rules.desired" desired_path.write_bytes(desired_bytes) desired_path.chmod(0o600) evidence = journal / "operator.evidence" evidence.write_bytes(b"do-not-overwrite\n") evidence.chmod(0o600) targets_dir = tmp_path / "targets" targets_dir.mkdir() targets = { "conf": targets_dir / "wg-fabric0.conf", "owner": targets_dir / "wg-fabric0.owner", "helper": targets_dir / "virt-cloud-wg-fabric", "unit": targets_dir / "virt-cloud-wg-fabric.service", "repair": targets_dir / "virt-cloud-wg-fabric-reconcile.service", "timer": targets_dir / "virt-cloud-wg-fabric-reconcile.timer", "gate_generator": targets_dir / "virt-cloud-wg-fabric-gate", "nm_profile": targets_dir / "virt-cloud-wg-fabric0.nmconnection", "ifupdown_fragment": targets_dir / "virt-cloud-wg-fabric0", "sysctl": targets_dir / "90-virt-cloud-wg-fabric.conf", "nft_rules": targets_dir / "virt-cloud-wg-fabric.nft", "nft_main": targets_dir / "nftables.conf", } artifact_values = { "owner": b"virt-cloud-wg-fabric-v1\n", "sysctl": b"net.ipv4.ip_forward = 1\n", "nft_rules": desired_bytes, "nft_main": b'include "/etc/nftables.d/*.nft"\n', } for name, value in artifact_values.items(): if name in ("nft_rules", "nft_main"): continue desired_artifact = journal / f"{name}.desired" desired_artifact.write_bytes(value) desired_artifact.chmod(0o600) if artifact_state == "post_trap_absent": present_artifacts = set() elif artifact_state == "hard_crash_exact_desired": present_artifacts = set(artifact_values) elif artifact_state == "mixed": present_artifacts = {"owner", "nft_rules"} else: raise AssertionError(f"unknown artifact state: {artifact_state}") if failure == "artifact_foreign_bytes": present_artifacts.add("sysctl") elif failure == "artifact_wrong_mode": present_artifacts.add("nft_rules") elif failure == "artifact_symlink": present_artifacts.add("nft_main") elif failure in ("artifact_desired_absent", "owner_iface_present"): present_artifacts.add("owner") artifact_symlink_target = tmp_path / "artifact-symlink-target" for name in present_artifacts: target = targets[name] if failure == "artifact_symlink" and name == "nft_main": artifact_symlink_target.write_bytes(artifact_values[name]) target.symlink_to(artifact_symlink_target) continue value = artifact_values[name] if failure == "artifact_foreign_bytes" and name == "sysctl": value = b"foreign sysctl bytes\n" target.write_bytes(value) target.chmod( 0o644 if failure == "artifact_wrong_mode" and name == "nft_rules" else 0o600 ) if failure == "artifact_desired_absent": desired_absent = journal / "owner.desired.absent" desired_absent.write_bytes(b"") desired_absent.chmod(0o600) if firewall_mode == FIREWALL_MODE_NATIVE: native_baseline = ( b"# baseline\ninclude \"/etc/nftables.d/*.nft\"\n" ) native_desired = ( b"# desired\n" b"include \"/etc/nftables.d/virt-cloud-wg-fabric.nft\"\n" ) baseline = journal / "nft_main" native_snapshot_target = tmp_path / "native-snapshot-target" if failure == "native_baseline_symlink": native_snapshot_target.write_bytes(native_baseline) baseline.symlink_to(native_snapshot_target) elif failure != "native_baseline_missing": baseline.write_bytes( b"# invalid baseline\n" if failure == "native_baseline_content" else native_baseline ) baseline.chmod( 0o666 if failure == "native_baseline_mode" else 0o600 ) desired_needed = native_state != "baseline_current_no_desired" or ( failure in { "native_desired_symlink", "native_desired_mode", "native_desired_content", "native_desired_absent_conflict", } ) desired_main = journal / "nft_main.desired" native_desired_target = tmp_path / "native-desired-target" if desired_needed: if failure == "native_desired_symlink": native_desired_target.write_bytes(native_desired) desired_main.symlink_to(native_desired_target) else: desired_main.write_bytes( b"# invalid desired\n" if failure == "native_desired_content" else native_desired ) desired_main.chmod( 0o666 if failure == "native_desired_mode" else 0o600 ) if failure in ("native_marker_conflict", "native_desired_absent_conflict"): conflict = journal / ( "nft_main.absent" if failure == "native_marker_conflict" else "nft_main.desired.absent" ) conflict.write_bytes(b"") conflict.chmod(0o600) live = targets["nft_main"] native_live_target = tmp_path / "native-live-target" if failure == "native_live_symlink": native_live_target.write_bytes(native_baseline) live.symlink_to(native_live_target) elif failure != "native_live_missing": live_value = ( native_desired if native_state == "desired_current" else native_baseline ) if failure == "native_live_foreign": live_value = b"# foreign live\ninclude \"/etc/nftables.d/*.nft\"\n" live.write_bytes(live_value) live.chmod( 0o644 if failure == "native_live_mode" else 0o600 ) else: native_snapshot_target = tmp_path / "unused-native-snapshot" native_desired_target = tmp_path / "unused-native-desired" native_live_target = tmp_path / "unused-native-live" desired_main = journal / "nft_main.desired" desired_main.write_bytes(artifact_values["nft_main"]) desired_main.chmod(0o600) installed_target = tmp_path / "installed-target" if failure == "installed_symlink": installed_target.write_bytes(b"foreign-snapshot\n") (journal / "nft.installed").symlink_to(installed_target) first_output = tmp_path / "nft-first" first_output.write_text(alma_live, encoding="utf-8") second_output = tmp_path / "nft-second" second_output.write_text(second_live, encoding="utf-8") call_count = tmp_path / "nft-count" nft = tmp_path / "nft" nft.write_text( """#!/bin/sh [ "$*" = "-s list table inet virt_cloud_wg_fabric" ] || exit 2 vc_count=0 if [ -f "$VC_NFT_CALL_COUNT" ]; then vc_count=$(cat "$VC_NFT_CALL_COUNT") fi vc_count=$((vc_count + 1)) printf '%s\n' "$vc_count" > "$VC_NFT_CALL_COUNT" if [ "$vc_count" -eq 1 ]; then cat "$VC_NFT_FIRST" else cat "$VC_NFT_SECOND" fi """, encoding="utf-8", ) nft.chmod(0o700) sha256 = tmp_path / "sha256sum" sha256.write_text( f"""#!{sys.executable} import hashlib import sys for filename in sys.argv[1:]: with open(filename, "rb") as stream: digest = hashlib.sha256(stream.read()).hexdigest() print(f"{{digest}} {{filename}}") """, encoding="utf-8", ) sha256.chmod(0o700) fake_stat = tmp_path / "stat" fake_stat.write_text( f"""#!{sys.executable} import os import stat import sys if len(sys.argv) != 4 or sys.argv[1] != "-c": raise SystemExit(2) value = os.lstat(sys.argv[3]) mode = format(stat.S_IMODE(value.st_mode), "o") if sys.argv[2] == "%a": print(mode) elif sys.argv[2] == "%u": print("0") elif sys.argv[2] == "%u:%a": print(f"0:{{mode}}") else: raise SystemExit(2) """, encoding="utf-8", ) fake_stat.chmod(0o700) ip = tmp_path / "ip" ip.write_text( """#!/bin/sh if [ "${VC_IFACE_PRESENT:-no}" = yes ] && [ "$*" = "link show dev wg-fabric0" ]; then exit 0 fi exit 1 """, encoding="utf-8", ) ip.chmod(0o700) wg = tmp_path / "wg" wg.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") wg.chmod(0o700) artifacts, ownership, special, current_owner, snapshot, installed_proof = ( _installing_recovery_fragments() ) special = special.replace( "/etc/systemd/system/virt-cloud-wg-fabric-reconcile.service", str(targets["repair"]), ).replace( "/etc/systemd/system/virt-cloud-wg-fabric-reconcile.timer", str(targets["timer"]), ) script = "\n".join(( "set -eu", "umask 077", f"JOURNAL={shlex.quote(str(journal))}", f"CONF={shlex.quote(str(targets['conf']))}", f"OWNER={shlex.quote(str(targets['owner']))}", f"HELPER={shlex.quote(str(targets['helper']))}", f"UNIT={shlex.quote(str(targets['unit']))}", f"GATE_GENERATOR={shlex.quote(str(targets['gate_generator']))}", f"NM_PROFILE={shlex.quote(str(targets['nm_profile']))}", f"IFUPDOWN_FRAGMENT={shlex.quote(str(targets['ifupdown_fragment']))}", f"SYSCTL_FILE={shlex.quote(str(targets['sysctl']))}", f"NFT_RULES={shlex.quote(str(targets['nft_rules']))}", f"journal_nft_main={shlex.quote(str(targets['nft_main']))}", f"IP={shlex.quote(str(ip))}", f"WG={shlex.quote(str(wg))}", "IFACE=wg-fabric0", f"NFT={shlex.quote(str(nft))}", f"STAT={shlex.quote(str(fake_stat))}", f"SHA256={shlex.quote(str(sha256))}", f"CMP={shlex.quote('/usr/bin/cmp')}", f"EXPECTED_NFT_RULES_SHA256={shlex.quote(expected_sha)}", f"journal_plan_digest={shlex.quote(digest)}", f"journal_firewall_mode={firewall_mode}", "journal_nft_phase=installing", "journal_nft_before=no", "journal_iface_before=no", "journal_iface_up_before=no", "journal_rp_filter_before=missing", "journal_helper_before=absent", "journal_iface_recovery_phase=none", "journal_recover_installing=no", "RECOVER_CURRENT=yes", "RECOVER_INSTALLING=yes", "_tmp_nft_installed=", "cleanup() {", " cleanup_status=$?", " trap - EXIT", ' [ -z "$_tmp_nft_installed" ] || ' 'rm -f "$_tmp_nft_installed" || true', ' exit "$cleanup_status"', "}", "trap cleanup EXIT", "sync() { :; }", 'stat() { "$STAT" "$@"; }', artifacts, _FABRIC_NFT_VALIDATION_SH, ownership, special, '[ "$journal_recover_installing" = yes ] || exit 79', current_owner, '"$NFT" -s list table inet virt_cloud_wg_fabric > ' '"$JOURNAL/nft.current" 2>/dev/null || exit 79', 'chmod 600 "$JOURNAL/nft.current" || exit 79', snapshot, installed_proof, "journal_nft_action=replace", "printf '%s\\n' \"journal_nft_action=$journal_nft_action\"", )) result = subprocess.run( ["/bin/sh", "-c", script], env={ "PATH": "/usr/bin:/bin", "VC_NFT_CALL_COUNT": str(call_count), "VC_NFT_FIRST": str(first_output), "VC_NFT_SECOND": str(second_output), "VC_IFACE_PRESENT": ( "yes" if failure == "owner_iface_present" else "no" ), }, check=False, capture_output=True, text=True, ) return result, SimpleNamespace( journal=journal, desired=desired_bytes, evidence=evidence, alma_live=alma_live.encode(), call_count=call_count, installed_target=installed_target, artifact_symlink_target=artifact_symlink_target, artifact_state=artifact_state, artifact_values=artifact_values, targets=targets, native_snapshot_target=native_snapshot_target, native_desired_target=native_desired_target, native_live_target=native_live_target, native_baseline=native_baseline, ) @pytest.mark.parametrize("artifact_state", ( "post_trap_absent", "hard_crash_exact_desired", "mixed", )) def test_installing_recovery_execution_snapshots_stable_alma_live_table( tmp_path, artifact_state, ): result, fixture = _run_installing_recovery_harness( tmp_path, artifact_state=artifact_state, ) assert result.returncode == 0, result.stderr assert result.stdout.splitlines() == ["journal_nft_action=replace"] installed = fixture.journal / "nft.installed" assert installed.read_bytes() == fixture.alma_live assert stat.S_IMODE(installed.stat().st_mode) == 0o600 assert (fixture.journal / "nft_rules.desired").read_bytes() == ( fixture.desired ) assert fixture.evidence.read_bytes() == b"do-not-overwrite\n" assert fixture.call_count.read_text(encoding="utf-8") == "2\n" assert not list(fixture.journal.glob(".nft-installed.*")) expected_present = { "post_trap_absent": {"nft_main"}, "hard_crash_exact_desired": set(fixture.artifact_values), "mixed": {"owner", "nft_rules", "nft_main"}, }[artifact_state] for name, value in fixture.artifact_values.items(): target = fixture.targets[name] if name in expected_present: expected = ( fixture.native_baseline if name == "nft_main" else value ) assert target.read_bytes() == expected assert stat.S_IMODE(target.stat().st_mode) == 0o600 else: assert not target.exists() assert not target.is_symlink() @pytest.mark.parametrize("failure", ( "desired_hash", "nonempty_tenant_set", "extra_object", "installed_symlink", "second_read_drift", "artifact_foreign_bytes", "artifact_wrong_mode", "artifact_symlink", "artifact_desired_absent", "owner_iface_present", )) def test_installing_recovery_execution_rejects_ambiguous_evidence( tmp_path, failure, ): result, fixture = _run_installing_recovery_harness( tmp_path, failure=failure, ) assert result.returncode == 79 assert (fixture.journal / "nft_rules.desired").read_bytes() == ( fixture.desired ) assert fixture.evidence.read_bytes() == b"do-not-overwrite\n" installed = fixture.journal / "nft.installed" if failure == "installed_symlink": assert installed.is_symlink() assert fixture.installed_target.read_bytes() == b"foreign-snapshot\n" else: assert not installed.exists() assert not installed.is_symlink() if failure == "second_read_drift": assert (fixture.journal / "nft.current").read_bytes() == ( fixture.alma_live ) assert fixture.call_count.read_text(encoding="utf-8") == "2\n" else: assert not (fixture.journal / "nft.current").exists() assert not fixture.call_count.exists() if failure == "artifact_foreign_bytes": assert fixture.targets["sysctl"].read_bytes() == ( b"foreign sysctl bytes\n" ) elif failure == "artifact_wrong_mode": target = fixture.targets["nft_rules"] assert target.read_bytes() == fixture.artifact_values["nft_rules"] assert stat.S_IMODE(target.stat().st_mode) == 0o644 elif failure == "artifact_symlink": assert fixture.targets["nft_main"].is_symlink() assert fixture.artifact_symlink_target.read_bytes() == ( fixture.native_baseline ) elif failure == "artifact_desired_absent": assert (fixture.journal / "owner.desired.absent").is_file() assert fixture.targets["owner"].read_bytes() == ( fixture.artifact_values["owner"] ) elif failure == "owner_iface_present": assert fixture.targets["owner"].read_bytes() == ( fixture.artifact_values["owner"] ) assert not list(fixture.journal.glob(".nft-installed.*")) @pytest.mark.parametrize("native_state", ( "baseline_current_no_desired", "desired_current", "baseline_current_with_desired", )) def test_native_installing_recovery_accepts_proven_nft_main_images( tmp_path, native_state, ): result, fixture = _run_installing_recovery_harness( tmp_path, firewall_mode=FIREWALL_MODE_NATIVE, native_state=native_state, ) assert result.returncode == 0, result.stderr assert result.stdout.splitlines() == ["journal_nft_action=replace"] assert (fixture.journal / "nft.installed").read_bytes() == ( fixture.alma_live ) assert (fixture.journal / "nft_main.present").is_file() assert not (fixture.journal / "nft_main.absent").exists() assert (fixture.journal / "nft_main").read_bytes().startswith( b"# baseline\n", ) desired = fixture.journal / "nft_main.desired" if native_state == "baseline_current_no_desired": assert not desired.exists() assert fixture.targets["nft_main"].read_bytes().startswith( b"# baseline\n", ) elif native_state == "desired_current": assert desired.read_bytes().startswith(b"# desired\n") assert fixture.targets["nft_main"].read_bytes() == desired.read_bytes() else: assert desired.read_bytes().startswith(b"# desired\n") assert fixture.targets["nft_main"].read_bytes() == ( fixture.journal / "nft_main" ).read_bytes() assert fixture.call_count.read_text(encoding="utf-8") == "2\n" @pytest.mark.parametrize("failure", ( "native_baseline_missing", "native_baseline_symlink", "native_baseline_mode", "native_baseline_content", "native_marker_conflict", "native_live_missing", "native_live_symlink", "native_live_mode", "native_live_foreign", "native_desired_symlink", "native_desired_mode", "native_desired_content", "native_desired_absent_conflict", )) def test_native_installing_recovery_rejects_unproven_nft_main_images( tmp_path, failure, ): result, fixture = _run_installing_recovery_harness( tmp_path, failure=failure, firewall_mode=FIREWALL_MODE_NATIVE, ) assert result.returncode == 79 assert fixture.evidence.read_bytes() == b"do-not-overwrite\n" assert not (fixture.journal / "nft.current").exists() assert not fixture.call_count.exists() assert not (fixture.journal / "nft.installed").exists() assert not list(fixture.journal.glob(".nft-installed.*")) if failure == "native_baseline_symlink": assert (fixture.journal / "nft_main").is_symlink() assert fixture.native_snapshot_target.read_bytes().startswith( b"# baseline\n", ) elif failure == "native_desired_symlink": assert (fixture.journal / "nft_main.desired").is_symlink() assert fixture.native_desired_target.read_bytes().startswith( b"# desired\n", ) elif failure == "native_live_symlink": assert fixture.targets["nft_main"].is_symlink() assert fixture.native_live_target.read_bytes().startswith( b"# baseline\n", ) def _timer_activity_recovery_runner( tmp_path, *, wanted, initial, failure="none", ): start = _BOOT_HELPER.index("vc_restore_timer_activity() {") end = _BOOT_HELPER.index("\n}\nnft_panel_owned()", start) + 2 function = _BOOT_HELPER[start:end] state = tmp_path / "timer.state" state.write_text(f"{initial}\n", encoding="utf-8") log = tmp_path / "systemctl.log" systemctl = tmp_path / "systemctl" systemctl.write_text( """#!/bin/sh printf '%s\n' "$*" >> "$VC_SYSTEMCTL_LOG" case "$1" in start) [ "$VC_SYSTEMCTL_FAILURE" != start ] || exit 1 printf '%s\n' active > "$VC_TIMER_STATE" ;; stop) [ "$VC_SYSTEMCTL_FAILURE" != stop ] || exit 1 [ "$VC_SYSTEMCTL_FAILURE" = sticky ] || \ printf '%s\n' inactive > "$VC_TIMER_STATE" ;; is-active) state=$(cat "$VC_TIMER_STATE") printf '%s\n' "$state" [ "$state" = active ] || [ "$state" = activating ] ;; *) exit 2 ;; esac """, encoding="utf-8", ) systemctl.chmod(0o700) result = subprocess.run( ["/bin/sh", "-c", "\n".join(( "set -eu", f"SYSTEMCTL='{systemctl}'", function, f"vc_restore_timer_activity {wanted}", ))], env={ "PATH": "/usr/bin:/bin", "VC_SYSTEMCTL_LOG": str(log), "VC_TIMER_STATE": str(state), "VC_SYSTEMCTL_FAILURE": failure, }, check=False, capture_output=True, text=True, ) return result, state, log def test_crash_recovery_stops_timer_started_after_inactive_snapshot(tmp_path): result, state, log = _timer_activity_recovery_runner( tmp_path, wanted="inactive", initial="active", ) assert result.returncode == 0, result.stderr assert state.read_text(encoding="utf-8") == "inactive\n" assert log.read_text(encoding="utf-8").splitlines() == [ "stop virt-cloud-wg-fabric-reconcile.timer", "is-active virt-cloud-wg-fabric-reconcile.timer", ] helper = _BOOT_HELPER restore_unit = helper.index( 'restore_journal_file "/etc/systemd/system/' 'virt-cloud-wg-fabric-reconcile.timer" timer' ) daemon_reload = helper.index( '"$SYSTEMCTL" daemon-reload', restore_unit, ) restore_activity = helper.index( 'vc_restore_timer_activity "$journal_timer_active"', daemon_reload, ) remove_journal = helper.index('rm -rf "$JOURNAL"', restore_activity) assert restore_unit < daemon_reload < restore_activity < remove_journal def test_crash_recovery_restarts_previously_active_timer(tmp_path): result, state, log = _timer_activity_recovery_runner( tmp_path, wanted="active", initial="inactive", ) assert result.returncode == 0, result.stderr assert state.read_text(encoding="utf-8") == "active\n" assert log.read_text(encoding="utf-8").splitlines() == [ "start virt-cloud-wg-fabric-reconcile.timer", "is-active virt-cloud-wg-fabric-reconcile.timer", ] @pytest.mark.parametrize( ("wanted", "initial", "failure"), (("active", "inactive", "start"), ("inactive", "active", "stop"), ("inactive", "active", "sticky")), ) def test_crash_recovery_keeps_journal_when_timer_state_cannot_be_restored( tmp_path, wanted, initial, failure, ): result, _, _ = _timer_activity_recovery_runner( tmp_path, wanted=wanted, initial=initial, failure=failure, ) assert result.returncode == 1 def test_apply_rollback_recovers_refresh_snapshot_before_phase_marker(): current = plan() script = _apply_script( current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}, firewall_mode=FIREWALL_MODE_NATIVE, ) snapshot_publish = script.index( 'mv -f "$TXN/nft.applied.tmp" "$TXN/nft.applied"') phase_publish = script.index( "write_nft_phase refreshed", snapshot_publish) rollback_window = script.index( 'elif [ "$rollback_nft_phase" = refreshing ] &&') exact_snapshot = script.index( '"$CMP" -s "$TXN/nft.current" "$TXN/nft.applied" ' '|| ROLLBACK_FAILED=yes', rollback_window, ) exact_set = script.index( "vpc_state_nft_set_matches || ROLLBACK_FAILED=yes", exact_snapshot, ) stable_snapshot = script.index( '"$CMP" -s "$TXN/nft.applied" "$TXN/nft.refresh-verify" ' '|| ROLLBACK_FAILED=yes', exact_set, ) assert snapshot_publish < phase_publish assert rollback_window < exact_snapshot < exact_set < stable_snapshot def test_repair_unit_contains_only_the_retry_dispatcher(): current = plan() script = _apply_script( current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}, firewall_mode=FIREWALL_MODE_NATIVE, ) rendered = _REPAIR_UNIT.replace( "__VC_FABRIC_DIGEST__", current.digest) assert rendered in script assert ( 'ExecCondition=/bin/sh /usr/local/libexec/virt-cloud-wg-fabric ' '--needs-repair' ) in rendered assert [ line for line in rendered.splitlines() if line.startswith("ExecStart=") ] == [ "ExecStart=/bin/sh /usr/local/libexec/virt-cloud-wg-fabric " "--retry-gated-units", ] assert "ExecStartPost=" not in rendered assert "__VC_FABRIC_EXEC__" not in rendered def test_repair_dispatcher_queues_primary_without_proven_dependency_jobs(tmp_path): tool = tmp_path / "tool" tool.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") tool.chmod(0o700) systemctl = tmp_path / "systemctl" systemctl.write_text( """#!/bin/sh printf '%s\\n' "$*" >> "$VC_TEST_SYSTEMCTL_LOG" case "$1:$2:$3:$5" in is-failed:--quiet:virt-cloud-wg-fabric.service:) exit 0 ;; show:-p:LoadState:libvirtd.service) printf '%s\\n' loaded ;; show:-p:Result:libvirtd.service) printf '%s\\n' dependency ;; show:-p:LoadState:virtqemud.service) printf '%s\\n' loaded ;; show:-p:Result:virtqemud.service) printf '%s\\n' success ;; reset-failed:libvirtd.service::) exit 0 ;; reset-failed:virt-cloud-wg-fabric.service::) exit 0 ;; start:virt-cloud-wg-fabric.service::) exit 0 ;; *) exit 2 ;; esac """, encoding="utf-8", ) systemctl.chmod(0o700) helper = _BOOT_HELPER.replace( "__VC_LIBVIRT_GATE_UNITS__", "libvirtd.service virtqemud.service", ).replace( "__VC_FABRIC_SYSCTL_VALIDATION__", _FABRIC_SYSCTL_VALIDATION_SH, ) tool_variables = { "WG", "IP", "NFT", "STAT", "SHA256", "CMP", "FIREWALL_CMD", "AWK", "BRIDGE_CMD", } helper_lines = [] for line in helper.splitlines(): if line == "BASE=/etc/virt-cloud": line = 'BASE="$VC_TEST_BASE"' elif line.startswith("SYSTEMCTL=$(find_tool "): line = 'SYSTEMCTL="$VC_TEST_SYSTEMCTL"' elif any( line.startswith(f"{name}=$(find_tool ") for name in tool_variables ): name = line.split("=", 1)[0] line = f'{name}="$VC_TEST_TOOL"' helper_lines.append(line) helper_path = tmp_path / "virt-cloud-wg-fabric" helper_path.write_text( "\n".join(helper_lines) + "\n", encoding="utf-8", ) helper_path.chmod(0o700) base = tmp_path / "state" base.mkdir() log = tmp_path / "systemctl.log" env = { "VC_TEST_BASE": str(base), "VC_TEST_TOOL": str(tool), "VC_TEST_SYSTEMCTL": str(systemctl), "VC_TEST_SYSTEMCTL_LOG": str(log), } condition = subprocess.run( ["/bin/sh", str(helper_path), "--needs-repair"], check=False, capture_output=True, text=True, env=env, ) assert condition.returncode == 0, condition.stderr dispatcher = subprocess.run( ["/bin/sh", str(helper_path), "--retry-gated-units"], check=False, capture_output=True, text=True, env=env, ) assert dispatcher.returncode == 0, dispatcher.stderr calls = log.read_text(encoding="utf-8").splitlines() start_calls = [call for call in calls if call.startswith("start ")] assert start_calls == [ "start virt-cloud-wg-fabric.service", ] assert not any("show -p Result" in call for call in calls) def test_repair_retry_dispatches_before_lock_and_only_requeues_failures(): retry = _BOOT_HELPER.index('if [ "${1:-}" = --retry-gated-units ]') lock = _BOOT_HELPER.index('exec 9>>"$LOCK"') assert retry < lock assert 'vc_gate_retry_evidence' in _BOOT_HELPER assert 'value!="dependency"' in _BOOT_HELPER assert 'show -p Result' not in _BOOT_HELPER assert 'start virt-cloud-wg-fabric.service' in _BOOT_HELPER assert 'exec /bin/sh "$HELPER" --ifupdown-service' in _BOOT_HELPER assert 'exec /bin/sh "$HELPER"' in _BOOT_HELPER assert 'restart --no-block virt-cloud-wg-fabric.service' not in _BOOT_HELPER needs_repair = _BOOT_HELPER.index( 'if [ "${1:-}" = --needs-repair ]; then') lock = _BOOT_HELPER.index('exec 9>>"$LOCK"') assert needs_repair < lock assert 'is-failed --quiet virt-cloud-wg-fabric.service' in _BOOT_HELPER assert ( 'ExecCondition=/bin/sh /usr/local/libexec/virt-cloud-wg-fabric ' '--needs-repair' ) in _REPAIR_UNIT def _operation_status_output( operation_id, *, status="queued", checkpoint="dispatch", reason="none", created_at=100, started_at=0, updated_at=100, finished_at=0): return "\n".join(( "__VC_FABRIC_OPERATION_SCHEMA__=1", f"__VC_FABRIC_OPERATION_ID__={operation_id}", f"__VC_FABRIC_OPERATION_STATUS__={status}", "__VC_FABRIC_OPERATION_PHASE__=apply", f"__VC_FABRIC_OPERATION_CHECKPOINT__={checkpoint}", f"__VC_FABRIC_OPERATION_REASON__={reason}", f"__VC_FABRIC_OPERATION_CREATED_AT__={created_at}", f"__VC_FABRIC_OPERATION_STARTED_AT__={started_at}", f"__VC_FABRIC_OPERATION_UPDATED_AT__={updated_at}", f"__VC_FABRIC_OPERATION_FINISHED_AT__={finished_at}", "", )) def test_fabric_operation_ids_are_opaque_fixed_lowercase_hex(): operation_ids = {_new_operation_id() for _index in range(128)} assert len(operation_ids) == 128 assert all(re.fullmatch(r"[0-9a-f]{32}", value) for value in operation_ids) @pytest.mark.parametrize("operation_id", [ "", "a" * 31, "a" * 33, "A" * 32, "g" * 32, "../" + "a" * 29, "a" * 31 + "/", "a" * 31 + "\n", "$(id)".ljust(32, "a"), ]) def test_fabric_operation_id_is_validated_before_rendering_remote_shell( operation_id): with pytest.raises(FabricError) as error: _fabric_operation_status_script(operation_id) assert error.value.error_code == "fabric_operation_invalid" @pytest.mark.parametrize(("kwargs", "expected"), [ ({}, { "status": "queued", "checkpoint": "dispatch", "reason": "none", "started_at": None, "finished_at": None, }), ({ "status": "running", "checkpoint": "journal_recovery", "created_at": 100, "started_at": 101, "updated_at": 102, }, { "status": "running", "checkpoint": "journal_recovery", "reason": "none", "started_at": 101, "finished_at": None, }), ({ "status": "succeeded", "checkpoint": "complete", "created_at": 100, "started_at": 101, "updated_at": 103, "finished_at": 103, }, { "status": "succeeded", "checkpoint": "complete", "reason": "none", "started_at": 101, "finished_at": 103, }), ({ "status": "failed", "checkpoint": "nft_install", "reason": "nft_failed", "created_at": 100, "started_at": 101, "updated_at": 104, "finished_at": 104, }, { "status": "failed", "checkpoint": "nft_install", "reason": "nft_failed", "started_at": 101, "finished_at": 104, }), ]) def test_fabric_operation_status_parser_accepts_only_complete_safe_projection( kwargs, expected): operation_id = "1" * 32 result = _parse_operation_status( _operation_status_output(operation_id, **kwargs), operation_id) assert result["operation_id"] == operation_id assert result["phase"] == "apply" for field, value in expected.items(): assert result[field] == value def _replace_status_line(output, prefix, replacement): lines = output.splitlines() return "\n".join( replacement if line.startswith(prefix) else line for line in lines ) + "\n" @pytest.mark.parametrize("mutation", [ lambda value: value + "__VC_FABRIC_OPERATION_EXTRA__=value\n", lambda value: value + "__VC_FABRIC_OPERATION_STATUS__=queued\n", lambda value: value.replace( "__VC_FABRIC_OPERATION_REASON__=none\n", ""), lambda value: "attacker-controlled diagnostic\n" + value, lambda value: _replace_status_line( value, "__VC_FABRIC_OPERATION_STATUS__=", "__VC_FABRIC_OPERATION_STATUS__=unknown"), lambda value: _replace_status_line( value, "__VC_FABRIC_OPERATION_PHASE__=", "__VC_FABRIC_OPERATION_PHASE__=identity"), lambda value: _replace_status_line( value, "__VC_FABRIC_OPERATION_CHECKPOINT__=", "__VC_FABRIC_OPERATION_CHECKPOINT__=arbitrary"), lambda value: _replace_status_line( value, "__VC_FABRIC_OPERATION_REASON__=", "__VC_FABRIC_OPERATION_REASON__=/root/private-key"), lambda value: _replace_status_line( value, "__VC_FABRIC_OPERATION_CREATED_AT__=", "__VC_FABRIC_OPERATION_CREATED_AT__=-1"), lambda value: _replace_status_line( value, "__VC_FABRIC_OPERATION_UPDATED_AT__=", "__VC_FABRIC_OPERATION_UPDATED_AT__=99"), ]) def test_fabric_operation_status_parser_rejects_extra_duplicate_or_invalid_data( mutation): operation_id = "2" * 32 output = mutation(_operation_status_output(operation_id)) with pytest.raises(FabricError) as error: _parse_operation_status(output, operation_id) assert error.value.error_code == "fabric_operation_status_invalid" safe_error = json.dumps({ "message": str(error.value), "code": error.value.error_code, "phase": error.value.failure_phase, }) assert "attacker-controlled" not in safe_error assert "/root/private-key" not in safe_error def test_fabric_operation_status_parser_fences_different_attempts(): first = "3" * 32 second = "4" * 32 with pytest.raises(FabricError) as error: _parse_operation_status(_operation_status_output(first), second) assert error.value.error_code == "fabric_operation_status_invalid" @pytest.mark.parametrize(("status", "checkpoint", "reason", "started", "finished"), [ ("queued", "dispatch", "none", 1, 0), ("running", "preflight", "none", 0, 0), ("running", "preflight", "nft_failed", 101, 0), ("succeeded", "final_readback", "none", 101, 102), ("succeeded", "complete", "command_failed", 101, 102), ("failed", "nft_install", "none", 101, 102), ]) def test_fabric_operation_status_parser_rejects_impossible_state_combinations( status, checkpoint, reason, started, finished): operation_id = "5" * 32 output = _operation_status_output( operation_id, status=status, checkpoint=checkpoint, reason=reason, started_at=started, updated_at=102, finished_at=finished) with pytest.raises(FabricError) as error: _parse_operation_status(output, operation_id) assert error.value.error_code == "fabric_operation_status_invalid" def test_fabric_operation_protocol_has_fixed_bounded_vocabulary(): assert FABRIC_OPERATION_STATUSES == { "queued", "running", "succeeded", "failed", "uncertain", "interrupted", } assert FABRIC_OPERATION_CHECKPOINTS == { "dispatch", "host_lock", "identity", "journal_recovery", "journal_dispatch", "preflight_sysctl", "preflight_tools", "preflight_artifacts", "preflight_firewall", "preflight_vpc", "preflight_underlay", "preflight_routes", "preflight_persistence", "journal_artifact_restore", "journal_nft_restore", "journal_runtime_restore", "journal_finalize", "preflight", "transaction", "nft_install", "network_activate", "persistence", "final_readback", "complete", } assert FABRIC_OPERATION_REASONS == { "none", "tool_missing", "ownership_conflict", "journal_conflict", "sysctl_conflict", "persistence_conflict", "firewall_conflict", "underlay_conflict", "network_manager_failed", "nft_failed", "systemd_failed", "readback_failed", "command_failed", "transport_failed", "status_invalid", } def test_fabric_operation_status_reader_is_strictly_read_only(): operation_id = "6" * 32 script = _fabric_operation_status_script(operation_id) assert f"OPERATION_ID={operation_id}" in script assert "__VC_FABRIC_OPERATION_STATUS__=$state" in script assert '"$UPDATE" interrupted' not in script assert "mv " not in script assert "rm " not in script assert "systemctl start" not in script assert "systemctl restart" not in script def test_fabric_operation_dispatch_uses_fixed_nonblocking_systemd_command(): operation_id = "7" * 32 secret = "private-key-material-must-not-be-logged" rendered_apply = f"#!/bin/sh\nprintf '%s' {shlex.quote(secret)} >/dev/null\n" script = _fabric_operation_dispatch_script( operation_id, rendered_apply, "8" * 64) systemd_start = script.index('"$SYSTEMD_RUN" --unit="$UNIT"') assert f"UNIT={FABRIC_OPERATION_UNIT_PREFIX}{operation_id}.service" in script assert "--collect --no-block --quiet" in script[systemd_start:] assert "--property=Type=oneshot" in script[systemd_start:] assert "--property=StandardOutput=journal" in script[systemd_start:] assert "--property=StandardError=null" in script[systemd_start:] assert script.count( f"--property=TimeoutStartSec={FABRIC_OPERATION_START_TIMEOUT_SECONDS}s" ) == 2 assert "RuntimeMaxSec=" not in script assert script.count('/bin/sh "$OPERATION_DIR/runner"') == 2 assert not re.search(r'^\s*"\$OPERATION_DIR/runner"', script, re.MULTILINE) assert "/bin/sh -c" not in script[systemd_start:] assert "exec >/dev/null 2>&1" in script assert "operation=%s state=%s checkpoint=%s reason=%s" in script assert secret not in script[systemd_start:] def test_fabric_operation_runner_preserves_apply_exit_status_for_safe_reason(): runner = _fabric_operation_runner_script("7" * 32, "8" * 64) run_apply = runner.index('if /bin/sh "$APPLY"; then') success = runner.index( '/bin/sh "$UPDATE" succeeded complete none', run_apply) capture = runner.index("else\n apply_rc=$?\nfi", success) tool_reason = runner.index("70) reason=tool_missing", capture) journal_reason = runner.index("79) reason=journal_conflict", tool_reason) failed = runner.index( '/bin/sh "$UPDATE" failed "$checkpoint" "$reason"', journal_reason) assert run_apply < success < capture < tool_reason < journal_reason < failed assert runner.count('/bin/sh "$UPDATE"') == 4 assert not re.search(r'^\s*"\$UPDATE"', runner, re.MULTILINE) @pytest.mark.parametrize(("apply_exit", "checkpoint", "expected_calls"), [ (0, "dispatch", ["running dispatch none", "succeeded complete none"]), (79, "journal_recovery", [ "running dispatch none", "failed journal_recovery journal_conflict", ]), (79, "journal_artifact_restore", [ "running dispatch none", "failed journal_artifact_restore journal_conflict", ]), (79, "journal_nft_restore", [ "running dispatch none", "failed journal_nft_restore journal_conflict", ]), (79, "journal_runtime_restore", [ "running dispatch none", "failed journal_runtime_restore journal_conflict", ]), (79, "journal_finalize", [ "running dispatch none", "failed journal_finalize journal_conflict", ]), *[(71, checkpoint, [ "running dispatch none", f"failed {checkpoint} {reason}", ]) for checkpoint, reason in ( ("preflight", "command_failed"), ("preflight_sysctl", "sysctl_conflict"), ("preflight_tools", "tool_missing"), ("preflight_artifacts", "ownership_conflict"), ("preflight_firewall", "firewall_conflict"), ("preflight_vpc", "readback_failed"), ("preflight_underlay", "underlay_conflict"), ("preflight_routes", "underlay_conflict"), ("preflight_persistence", "persistence_conflict"), ("journal_dispatch", "journal_conflict"), )], ]) def test_fabric_operation_runner_uses_shell_for_non_executable_spool_scripts( tmp_path, monkeypatch, apply_exit, checkpoint, expected_calls): operation_id = "7" * 32 operation_root = tmp_path / "fabric-operations" operation_dir = operation_root / operation_id operation_dir.mkdir(parents=True) calls = operation_dir / "calls" monkeypatch.setattr( "virsh_panel.fabric.FABRIC_OPERATION_ROOT", str(operation_root)) update = operation_dir / "update" update.write_text( "#!/bin/sh\n" f"printf '%s\\n' \"$*\" >> {shlex.quote(str(calls))}\n", encoding="utf-8", ) apply = operation_dir / "apply" apply.write_text( f"#!/bin/sh\nexit {apply_exit}\n", encoding="utf-8", ) (operation_dir / "status").write_text( "\n".join(( "schema=1", f"operation_id={operation_id}", "status=running", "phase=apply", f"checkpoint={checkpoint}", "reason=none", "created_at=100", "started_at=101", "updated_at=101", "finished_at=0", "", )), encoding="utf-8", ) runner = operation_dir / "runner" runner.write_text( _fabric_operation_runner_script(operation_id, "8" * 64), encoding="utf-8", ) for spool_script in (update, apply, runner): spool_script.chmod(0o600) result = subprocess.run( ["/bin/sh", str(runner)], check=False, capture_output=True, text=True, ) assert result.returncode == (0 if apply_exit == 0 else 1) assert calls.read_text(encoding="utf-8").splitlines() == expected_calls def test_fabric_operation_spool_is_ephemeral_bounded_and_fail_closed(): operation_id = "8" * 32 script = _fabric_operation_dispatch_script( operation_id, "#!/bin/sh\nexit 0\n", "9" * 64) assert FABRIC_OPERATION_ROOT == "/run/virt-cloud/fabric-operations" assert f"ROOT={FABRIC_OPERATION_ROOT}" in script assert ( f'if [ "$operation_count" -ge {MAX_FABRIC_OPERATION_DIRECTORIES} ]; ' "then" ) in script assert '"__VC_FABRIC_OPERATION_DISPATCHED__=limit"' in script assert 'case "$operation_name" in' in script assert "*[!0-9a-f]*) exit 71" in script assert '[ "${#operation_name}" = 32 ] || exit 71' in script assert 'validate_retained_operation "$operation_path" || exit 71' in script assert '[ "$operation_hidden" = "$LOCK" ] || exit 71' in script limit = script.index( f'if [ "$operation_count" -ge {MAX_FABRIC_OPERATION_DIRECTORIES} ];') validate = script.index( 'validate_retained_operation "$oldest_terminal_path"', limit) terminal = script.index( 'succeeded|failed|uncertain|interrupted)', validate) remove = script.index( 'for retained_file in manifest status apply update runner; do', terminal) rmdir = script.index('rmdir -- "$oldest_terminal_path"', remove) limit_result = script.index( '"__VC_FABRIC_OPERATION_DISPATCHED__=limit"', rmdir) assert limit < validate < terminal < remove < rmdir < limit_result assert 'rm -rf -- "$oldest_terminal_path"' not in script def test_fabric_operation_existing_attempt_recovers_only_orphaned_queue(): operation_id = "9" * 32 script = _fabric_operation_dispatch_script( operation_id, "#!/bin/sh\nexit 0\n", "a" * 64) queued = script.index('if [ "$retained_state" = queued ]; then') unit_read = script.index( '"$SYSTEMCTL" show "$UNIT" --property=LoadState --value', queued) missing = script.index("not-found)", unit_read) recover = script.index("recover_operation_unit", missing) loaded = script.index("loaded)", recover) existing = script.index( '"__VC_FABRIC_OPERATION_DISPATCHED__=existing"', loaded) existing_exit = script.index(" exit 0", existing) assert queued < unit_read < missing < recover < loaded < existing assert existing < existing_exit assert script.count("recover_operation_unit") == 2 recover_function = script[script.index("recover_operation_unit()"): script.index("validate_retained_operation()")] assert recover_function.count('"$SYSTEMD_RUN"') == 1 assert '/bin/sh "$OPERATION_DIR/runner"' in recover_function assert script.count('/bin/sh "$OPERATION_DIR/update"') == 3 assert not re.search( r'^\s*"\$OPERATION_DIR/update"', script, re.MULTILINE) assert ( '[ "$(sed -n \'2p\' "$OPERATION_DIR/manifest")" = ' '"operation_id=$OPERATION_ID" ]' ) in script assert ( '[ "$(sed -n \'3p\' "$OPERATION_DIR/manifest")" = ' '"plan_digest=$PLAN_DIGEST" ]' ) in script assert ( '[ "$(sed -n \'4p\' "$OPERATION_DIR/manifest")" = ' '"script_digest=$SCRIPT_DIGEST" ]' ) in script assert ( 'case "${operation_file##*/}" in ' "manifest|status|apply|update|runner) : ;; *) exit 71 ;; esac" ) in script assert 'for operation_file in manifest status; do' in script assert ' = 0:600 ] || exit 71' in script assert 'for operation_file in apply update runner; do' in script assert ' = 0:700 ] || exit 71' in script assert ( '$SHA256SUM "$OPERATION_DIR/apply"' in script and '$SHA256SUM "$OPERATION_DIR/update"' in script and '$SHA256SUM "$OPERATION_DIR/runner"' in script ) def test_fabric_dispatch_retires_only_proven_unstarted_legacy_spool(): operation_id = "9" * 32 script = _fabric_operation_dispatch_script( operation_id, "#!/bin/sh\nexit 0\n", "a" * 64) retained = script.index( 'if [ "$retained_apply:$retained_update:$retained_runner" !=') queued = script.index( 'queued:dispatch:none ] || exit 71', retained) never_started = script.index( '[ "$retained_started:$retained_finished" = 0:0 ] || exit 71', queued) never_updated = script.index( '[ "$retained_updated" = "$retained_created" ] || exit 71', never_started) absent = script.index('not-found:inactive) : ;;', never_updated) exec_failed = script.index( 'exit-code:exited:203 ] || exit 71', absent) retire = script.index( 'status=failed phase=apply checkpoint=dispatch', exec_failed) current_manifest = script.index( '"plan_digest=$PLAN_DIGEST" ] || exit 71', retire) assert retained < queued < never_started < never_updated < absent assert absent < exec_failed assert exec_failed < retire < current_manifest assert '/bin/sh "$OPERATION_DIR/update"' not in script[ retained:retire] assert "possibly started operation" in script def test_fabric_async_apply_records_recovery_before_new_transaction(): current = plan() script = _apply_script( current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}) recovery_checkpoint = script.index( "vc_fabric_operation_checkpoint journal_recovery") recovery = script.index( 'case "$journal_existing_state" in', recovery_checkpoint) preflight = script.index( "vc_fabric_operation_checkpoint preflight", recovery) transaction = script.index( "vc_fabric_operation_checkpoint transaction", preflight) assert recovery_checkpoint < recovery < preflight < transaction for checkpoint in FABRIC_OPERATION_CHECKPOINTS - {"dispatch"}: assert ( f"vc_fabric_operation_checkpoint {checkpoint}" in script or f"vc_fabric_recovery_checkpoint {checkpoint}" in script ) def test_fabric_journal_recovery_reports_fixed_safe_subcheckpoints(): helper = _BOOT_HELPER artifact = helper.index( "vc_fabric_recovery_checkpoint journal_artifact_restore") artifact_restore = helper.index("journal_sysctl_restore=no", artifact) nft = helper.index( "vc_fabric_recovery_checkpoint journal_nft_restore", artifact_restore) nft_read = helper.index( "journal_tables=$($NFT list tables", nft) runtime = helper.index( "vc_fabric_recovery_checkpoint journal_runtime_restore", nft_read) runtime_restore = helper.index( 'if [ "$journal_iface_before" = yes ]; then', runtime) finalize = helper.index( "vc_fabric_recovery_checkpoint journal_finalize", runtime_restore) helper_cleanup = helper.index( 'if [ "$journal_helper_before" = absent ]; then', finalize) assert artifact < artifact_restore < nft < nft_read assert nft_read < runtime < runtime_restore < finalize < helper_cleanup checkpoint_function = helper[ helper.index("vc_fabric_recovery_checkpoint()"): helper.index("vc_legacy_firewalld_state_detected()") ] assert "/run/virt-cloud/fabric-operations/$VC_FABRIC_OPERATION_ID" in ( checkpoint_function) assert "0:0:700" in checkpoint_function assert '/bin/sh "$vc_recovery_update"' in checkpoint_function assert "|| true" in checkpoint_function def test_fabric_apply_operation_checkpoint_uses_shell_on_noexec_run(): current = plan() script = _apply_script( current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}) assert '/bin/sh "$vc_operation_update"' in script assert not re.search( r'^\s*"\$vc_operation_update"', script, re.MULTILINE) class FakeManager: ssh_binary = "ssh" def __init__(self, public_key=KEY1, extra_address=False, self_route=False, config_digest=None, ping_ok=True, endpoint_device="eth0", host=None, fabric_endpoint_ip="", firewall_mode=FIREWALL_MODE_NATIVE, live_public_key=None): self.public_key = public_key self.live_public_key = live_public_key or public_key self.extra_address = extra_address self.self_route = self_route self.config_digest = config_digest self.ping_ok = ping_ok self.endpoint_device = endpoint_device self.scripts = [] self.timeouts = [] self.reprobe_calls = 0 self.admin_state = "up" self.ip_forward = "1" self.rp_filter = "2" self.rp_filter_default = "2" self.rp_filter_iface = "2" self.rp_filter_effective = "2" self.sysctl_persistence = "owned" self.emergency_state = "clear" self.firewall_mode = firewall_mode effective_host = host or ( "203.0.113.1" if public_key == KEY1 else "203.0.113.2") self.effective_host = effective_host self.underlay_endpoint = fabric_endpoint_ip or effective_host self.profile = { "host": effective_host, "fabric_endpoint_ip": fabric_endpoint_ip, "ssh_config_file": "/tmp/fake-ssh-config", "ssh_target": "vp-managed", "key_locked": False, "capabilities": { "vpc_ready": True, "wireguard": { "interface": FABRIC_INTERFACE, "deployment_mode": "networkmanager_nmcli", "ready": True, }, "network": { "network_manager": "networkmanager", "network_manager_supported": True, "firewall_manager": "nftables", "firewall_mode": firewall_mode, "ip_probe_valid": True, "local_ipv4_addresses": [ fabric_endpoint_ip or effective_host, ], "local_ipv4_interfaces": { fabric_endpoint_ip or effective_host: [endpoint_device], }, "rp_filter": 2, "rp_filter_default": 2, "rp_filter_default_effective": 2, "rp_filter_interfaces": { endpoint_device: 1, }, "networkmanager_persistent_interface_zones": { endpoint_device: "", }, "ifcfg_zone_interfaces": [], }, "nftables": { "ready": True, "firewall_mode": firewall_mode, }, }, } def current(self): return self.profile def reprobe_current(self): self.reprobe_calls += 1 return self.profile["capabilities"] def _exec(self, argv, *, timeout, input_text=None): assert argv[-1] == "/bin/sh -s" self.timeouts.append(timeout) self.scripts.append(input_text) if input_text.startswith("#!/bin/sh\nset -eu") and "__VC_FABRIC_PUBLIC_KEY__=" in input_text: output = f"__VC_FABRIC_PUBLIC_KEY__={self.public_key}\n" elif "__VC_FABRIC_APPLIED__" in input_text: self.live_public_key = self.public_key output = "__VC_FABRIC_APPLIED__\n" else: now = int(time.time()) peer = KEY2 if self.public_key == KEY1 else KEY1 address = "100.64.1.1" if self.public_key == KEY1 else "100.64.1.2" target = "100.64.1.2" if self.public_key == KEY1 else "100.64.1.1" endpoint = "203.0.113.2:51820" if self.public_key == KEY1 else "203.0.113.1:51820" extra_address = ( "__VC_FABRIC_ADDRESS__=100.64.1.99\n" if self.extra_address else "" ) self_route = ( f"__VC_FABRIC_ROUTE__={address}|local|{FABRIC_INTERFACE}\n" if self.self_route else "" ) output = ( f"__VC_FABRIC_OWNER__={FABRIC_OWNER_MARKER}\n" "__VC_FABRIC_CONFIG_FABRIC__=hk-fabric\n" "__VC_FABRIC_CONFIG_REVISION__=1\n" f"__VC_FABRIC_CONFIG_DIGEST__={self.config_digest or plan().digest}\n" f"__VC_FABRIC_CONFIG_FIREWALL__={self.firewall_mode}\n" "__VC_FABRIC_CONFIG_NETWORK__=100.64.1.0/24\n" f"__VC_FABRIC_CONFIG_OVERLAY__={address}\n" "__VC_FABRIC_CONFIG_OK__=yes\n" f"__VC_FABRIC_CONFIG_PEER__={peer}|{endpoint}|{target}/32\n" f"__VC_FABRIC_CONFIG_ROUTE__={target}/32\n" f"__VC_FABRIC_PUBLIC_KEY__={self.public_key}\n" "__VC_FABRIC_INTERFACE__=wireguard\n" f"__VC_FABRIC_LIVE_PUBLIC_KEY__={self.live_public_key}\n" "__VC_FABRIC_LISTEN__=51820\n" f"__VC_FABRIC_ADDRESS__={address}\n" f"{extra_address}" f"__VC_FABRIC_PEER__={peer}|{endpoint}|{target}/32\n" f"__VC_FABRIC_HANDSHAKE__={peer}|{now}\n" f"{self_route}" f"__VC_FABRIC_ROUTE__={target}/32|main|{FABRIC_INTERFACE}\n" f"__VC_FABRIC_PATH__={target}|{FABRIC_INTERFACE}\n" f"__VC_FABRIC_ENDPOINT_PATH__={endpoint.split(':', 1)[0]}|{self.endpoint_device}|{self.underlay_endpoint}\n" f"__VC_FABRIC_PING__={target}|{'ok' if self.ping_ok else 'failed'}\n" f"__VC_FABRIC_ROUTE_OVERLAP__={target}|clean\n" "__VC_FABRIC_MANAGER_MODE__=networkmanager_nmcli\n" "__VC_FABRIC_MANAGER_OWNER__=owned\n" "__VC_FABRIC_MANAGER_ACTIVE__=active\n" f"__VC_FABRIC_FIREWALL_MODE__={self.firewall_mode}\n" "__VC_FABRIC_FIREWALL_STATE__=owned\n" f"__VC_FABRIC_ADMIN_STATE__={self.admin_state}\n" f"__VC_FABRIC_SYSCTL__ip_forward={self.ip_forward}\n" f"__VC_FABRIC_SYSCTL__rp_filter={self.rp_filter}\n" f"__VC_FABRIC_SYSCTL__rp_filter_default={self.rp_filter_default}\n" f"__VC_FABRIC_SYSCTL__rp_filter_effective={self.rp_filter_effective}\n" f"__VC_FABRIC_RPF__wg-fabric0={self.rp_filter_iface}\n" f"__VC_FABRIC_SYSCTL_PERSISTENCE__={self.sysctl_persistence}\n" "__VC_FABRIC_ENABLED__=enabled\n" "__VC_FABRIC_ACTIVE__=active\n" "__VC_FABRIC_TIMER_ENABLED__=enabled\n" "__VC_FABRIC_TIMER_ACTIVE__=active\n" "__VC_FABRIC_NFT_GUARD__=owned\n" "__VC_FABRIC_NFT_PERSISTENCE__=owned\n" f"__VC_FABRIC_EMERGENCY__={self.emergency_state}\n" "__VC_FABRIC_VPC_NFT_STATE__=owned\n" "__VC_FABRIC_KEY_SECURE__=yes\n" "__VC_FABRIC_CONFIG_SECURE__=yes\n" "__VC_FABRIC_HELPER__=owned\n" "__VC_FABRIC_UNIT__=owned\n" "__VC_FABRIC_REPAIR__=owned\n" "__VC_FABRIC_TIMER__=owned\n" "__VC_FABRIC_LIBVIRT_GATE__=owned\n" "__VC_FABRIC_LIBVIRT_DEPENDENCIES__=owned\n" ) return subprocess.CompletedProcess(argv, 0, output, "") class AsyncOperationManager(FakeManager): def __init__(self, *args, operation_id, status="queued", checkpoint="dispatch", reason="none", **kwargs): super().__init__(*args, **kwargs) self.operation_id = operation_id self.operation_status = status self.operation_checkpoint = checkpoint self.operation_reason = reason self.dispatch_result = "yes" self.disconnect_dispatch = False def _status_output(self): started_at = 0 finished_at = 0 if self.operation_status != "queued": started_at = 101 if self.operation_status in { "succeeded", "failed", "uncertain", "interrupted"}: finished_at = 103 return _operation_status_output( self.operation_id, status=self.operation_status, checkpoint=self.operation_checkpoint, reason=self.operation_reason, started_at=started_at, updated_at=103 if finished_at else 102, finished_at=finished_at, ) def _exec(self, argv, *, timeout, input_text=None): assert argv[-1] == "/bin/sh -s" self.timeouts.append(timeout) self.scripts.append(input_text) if 'SYSTEMD_RUN=$(find_tool ' in input_text: if self.disconnect_dispatch: raise RemoteHostError( "ambiguous disconnect with /secret/path", 502, "host_remote_command_failed") output = ( f"__VC_FABRIC_OPERATION_DISPATCHED__={self.dispatch_result}\n" f"__VC_FABRIC_OPERATION_ID__={self.operation_id}\n" ) return subprocess.CompletedProcess(argv, 0, output, "") if "__VC_FABRIC_OPERATION_SCHEMA__=$schema" in input_text: return subprocess.CompletedProcess( argv, 0, self._status_output(), "") raise AssertionError("unexpected asynchronous Fabric command") def test_adapter_dispatches_apply_and_reads_status_without_waiting_for_job(): current = plan() operation_id = "b" * 32 manager = AsyncOperationManager(KEY1, operation_id=operation_id) adapter = FabricHostAdapter(manager) result = adapter.dispatch_apply( current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}, operation_id) assert result == { "operation_id": operation_id, "status": "queued", "phase": "apply", "checkpoint": "dispatch", "reason": "none", "created_at": 100, "started_at": None, "updated_at": 102, "finished_at": None, } serialized = json.dumps(result) assert KEY1 not in serialized assert KEY2 not in serialized assert "/run/virt-cloud" not in serialized assert manager.timeouts == [ FABRIC_REMOTE_READ_TIMEOUT_SECONDS, FABRIC_REMOTE_READ_TIMEOUT_SECONDS, ] assert len(manager.scripts) == 2 assert "--no-block" in manager.scripts[0] assert "__VC_FABRIC_APPLIED__" in manager.scripts[0] assert "__VC_FABRIC_OPERATION_STATUS__=$state" in manager.scripts[1] @pytest.mark.parametrize(("status", "checkpoint", "reason"), [ ("running", "journal_recovery", "none"), ("succeeded", "complete", "none"), ("failed", "nft_install", "nft_failed"), ]) def test_adapter_can_read_exact_attempt_after_dispatch_connection_loss( status, checkpoint, reason): current = plan() operation_id = "c" * 32 manager = AsyncOperationManager( KEY1, operation_id=operation_id, status=status, checkpoint=checkpoint, reason=reason) manager.disconnect_dispatch = True adapter = FabricHostAdapter(manager) with pytest.raises(FabricError) as error: adapter.dispatch_apply( current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}, operation_id) assert error.value.error_code == "fabric_host_connection_failed" assert error.value.failure_phase == "apply" assert len(manager.scripts) == 1 # A lost dispatch response is reconciled only by its exact operation id; # the adapter does not submit the mutation a second time. manager.disconnect_dispatch = False observed = adapter.apply_status(operation_id) assert observed["operation_id"] == operation_id assert observed["status"] == status assert observed["checkpoint"] == checkpoint assert observed["reason"] == reason assert len(manager.scripts) == 2 assert 'SYSTEMD_RUN=$(find_tool ' not in manager.scripts[-1] def test_adapter_rejects_cross_attempt_status_without_redispatch(): operation_id = "d" * 32 manager = AsyncOperationManager(KEY1, operation_id="e" * 32) adapter = FabricHostAdapter(manager) with pytest.raises(FabricError) as error: adapter.apply_status(operation_id) assert error.value.error_code == "fabric_operation_status_invalid" assert len(manager.scripts) == 1 assert 'SYSTEMD_RUN=$(find_tool ' not in manager.scripts[0] def test_adapter_reports_exact_missing_operation_without_leaking_paths(): operation_id = "a" * 32 manager = AsyncOperationManager(KEY1, operation_id=operation_id) manager._status_output = lambda: ( "__VC_FABRIC_OPERATION_MISSING__=yes\n" f"__VC_FABRIC_OPERATION_ID__={operation_id}\n" ) adapter = FabricHostAdapter(manager) with pytest.raises(FabricError) as error: adapter.apply_status(operation_id) assert error.value.status == 409 assert error.value.error_code == "fabric_operation_not_found" assert error.value.failure_phase == "observe" assert "/run/virt-cloud" not in str(error.value) assert KEY1 not in str(error.value) assert len(manager.scripts) == 1 assert 'SYSTEMD_RUN=$(find_tool ' not in manager.scripts[0] def test_adapter_validates_operation_id_before_remote_dispatch(): current = plan() manager = AsyncOperationManager(KEY1, operation_id="f" * 32) adapter = FabricHostAdapter(manager) with pytest.raises(FabricError) as error: adapter.dispatch_apply( current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}, "../not-an-operation") assert error.value.error_code == "fabric_operation_invalid" assert manager.scripts == [] def test_adapter_readback_requires_exact_peer_route_and_persistence(): current = plan() manager = FakeManager(KEY1) adapter = FabricHostAdapter(manager) assert adapter.identity() == KEY1 assert adapter.observe( current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2} )["transport_verified"] is True rendered_observe = manager.scripts[-1] digest_assignment = f"EXPECTED_DIGEST={shlex.quote(current.digest)}" assert rendered_observe.index(digest_assignment) < rendered_observe.index( "vc_fabric_nm_profile_owned expected" ) def test_adapter_uses_stage_specific_remote_timeouts(): current = plan() manager = FakeManager(KEY1) adapter = FabricHostAdapter(manager) public_keys = {"host-a": KEY1, "host-b": KEY2} assert adapter.identity() == KEY1 adapter.apply(current, current.nodes[0], public_keys) adapter.observe(current, current.nodes[0], public_keys) assert [ FABRIC_REMOTE_READ_TIMEOUT_SECONDS, FABRIC_REMOTE_APPLY_TIMEOUT_SECONDS, FABRIC_REMOTE_READ_TIMEOUT_SECONDS, ] == [35, 300, 35] assert manager.timeouts == [35, 300, 35] @pytest.mark.parametrize("failure_phase", ["identity", "apply", "observe"]) def test_adapter_remote_failure_exposes_only_whitelisted_phase(failure_phase): class FailedRemoteManager(FakeManager): def _exec(self, argv, *, timeout, input_text=None): self.scripts.append(input_text) return subprocess.CompletedProcess( argv, 71, "", "secret /root/path private-key-material") current = plan() adapter = FabricHostAdapter(FailedRemoteManager(KEY1)) with pytest.raises(FabricError) as error: if failure_phase == "identity": adapter.identity() elif failure_phase == "apply": adapter.apply( current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}) else: adapter.observe( current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}) assert error.value.error_code == "fabric_remote_command_failed" assert error.value.failure_phase == failure_phase safe_error = json.dumps({ "message": str(error.value), "error_code": error.value.error_code, "failure_phase": error.value.failure_phase, }) assert "secret" not in safe_error assert "/root/path" not in safe_error assert "private-key-material" not in safe_error def test_profile_firewall_mode_rejects_missing_or_crossed_capability_evidence(): manager = FakeManager(KEY1) assert _profile_firewall_mode(manager.profile) == FIREWALL_MODE_NATIVE manager.profile["capabilities"]["network"].update({ "firewall_manager": "firewalld", "firewall_mode": "firewalld_nft", }) manager.profile["capabilities"]["nftables"]["firewall_mode"] = ( "firewalld_nft") with pytest.raises(FabricError) as error: _profile_firewall_mode(manager.profile) assert error.value.error_code == "fabric_host_firewall_mode_conflict" manager.profile["capabilities"]["nftables"]["firewall_mode"] = ( FIREWALL_MODE_NATIVE) with pytest.raises(FabricError) as error: _profile_firewall_mode(manager.profile) assert error.value.error_code == "fabric_host_firewall_mode_conflict" def test_adapter_rejects_an_unexpected_overlay_address(): current = plan() manager = FakeManager(KEY1, extra_address=True) observed = FabricHostAdapter(manager).observe( current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2} ) assert observed["config_ok"] is False assert observed["state"] == "degraded" def test_adapter_ignores_the_kernel_connected_route_for_its_own_address(): current = plan() manager = FakeManager(KEY1, self_route=True) observed = FabricHostAdapter(manager).observe( current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2} ) assert observed["config_ok"] is True def test_adapter_requires_admin_up_and_fabric_sysctl_readback(): current = plan() manager = FakeManager(KEY1) manager.admin_state = "down" assert FabricHostAdapter(manager).observe( current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2} )["config_ok"] is False manager.admin_state = "up" manager.rp_filter_iface = "1" observed = FabricHostAdapter(manager).observe( current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2} ) assert observed["config_ok"] is False assert observed["observed"]["admin_state"] == "up" assert observed["observed"]["rp_filters"][FABRIC_INTERFACE] == "1" def test_adapter_accepts_strict_global_defaults_when_owned_fabric_is_loose(): current = plan() manager = FakeManager(KEY1) manager.rp_filter = "0" manager.rp_filter_default = "1" manager.rp_filter_iface = "2" manager.rp_filter_effective = "2" observed = FabricHostAdapter(manager).observe( current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}, ) assert observed["config_ok"] is True assert observed["observed"]["sysctls"] == { "ip_forward": "1", "rp_filter": "0", "rp_filter_default": "1", "rp_filter_effective": "2", } def test_adapter_requires_exact_owned_sysctl_persistence(): current = plan() manager = FakeManager(KEY1) manager.sysctl_persistence = "foreign" observed = FabricHostAdapter(manager).observe( current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}, ) assert observed["config_ok"] is False assert observed["observed"]["sysctl_persistence"] == "foreign" @pytest.mark.parametrize("emergency_state", ["present", "unknown"]) def test_adapter_never_publishes_active_while_emergency_guard_remains( emergency_state): current = plan() manager = FakeManager(KEY1) manager.emergency_state = emergency_state observed = FabricHostAdapter(manager).observe( current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}, ) assert observed["config_ok"] is False assert observed["state"] == "degraded" assert observed["observed"]["emergency_state"] == emergency_state def test_adapter_rejects_stale_config_digest_and_failed_probe(): current = plan() stale = FakeManager(KEY1, config_digest="0" * 64) observed = FabricHostAdapter(stale).observe( current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2} ) assert observed["config_ok"] is False def test_adapter_rejects_endpoint_egress_drift_to_fabric_or_missing_device(): current = plan() for device in (FABRIC_INTERFACE, ""): observed = FabricHostAdapter(FakeManager(KEY1, endpoint_device=device)).observe( current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2} ) assert observed["config_ok"] is False failed_ping = FakeManager(KEY1, ping_ok=False) observed = FabricHostAdapter(failed_ping).observe( current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2} ) assert observed["config_ok"] is True assert observed["state"] == "active" assert observed["transport_verified"] is False def test_adapter_rejects_live_public_key_drift_from_persistent_key(): current = plan() observed = FabricHostAdapter(FakeManager( KEY1, live_public_key=KEY3, )).observe( current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}, ) assert observed["public_key"] == KEY1 assert observed["observed"]["live_public_key"] == KEY3 assert observed["config_ok"] is False assert observed["state"] == "degraded" class MemoryStore: def __init__(self): self.values = {} def get(self, kind, key): return self.values.get((kind, key)) def set(self, kind, key, value): self.values[(kind, key)] = value def list(self, kind): return {key: value for (row_kind, key), value in self.values.items() if row_kind == kind} class FakeAdapter: failures = set() observe_failures = set() def __init__(self, manager, profile=None): self.manager = manager def identity(self): return self.manager.public_key def apply(self, *_args): if self.manager.host_id in self.failures: raise FabricError("offline", 502, "fabric_remote_command_failed") def observe(self, current, host, keys): if (self.manager.host_id in self.failures or self.manager.host_id in self.observe_failures): raise FabricError("offline", 502, "fabric_remote_command_failed") return { "state": "active", "transport_verified": True, "observed": {"interface": FABRIC_INTERFACE}, } class Registry: def __init__(self, components): self.components = components async def get(self, host_id): return self.components[host_id] class UnconfiguredRegistry: def __init__(self): self.components = {} self.get_called = False async def is_configured(self, _host_id): return False async def get(self, _host_id): self.get_called = True raise AssertionError("readiness probe constructed an unknown host") class ProbeManager: def __init__(self, host_id, *, host="203.0.113.10", fabric_endpoint_ip="", ready=True, failure=None, counter=None): self.host_id = host_id self.failure = failure self.counter = counter self.profile = { "host_id": host_id, "host": host, "fabric_endpoint_ip": fabric_endpoint_ip, "region": "hk", "ssh_config_file": "/secret/ssh/config", "private_key": "must-not-leak", } effective_endpoint = fabric_endpoint_ip or host endpoint_is_ipv4 = bool(re.fullmatch( r"(?:[0-9]{1,3}\.){3}[0-9]{1,3}", effective_endpoint)) self.capabilities = { "vpc_ready": ready, "reasons": [] if ready else [ "firewall_owner_not_nftables", "rp_filter_not_compatible", ], "os": {"id": "almalinux", "version_id": "10", "name": "must-not-be-projected"}, "network": { "network_manager": "networkmanager", "network_manager_expected": "networkmanager", "network_manager_supported": True, "firewall_manager": "nftables" if ready else "firewalld", "firewall_mode": "native_nftables" if ready else "unsupported", "ip_forward": 1, "rp_filter": 2 if ready else 1, "rp_filter_default": 1, "rp_filter_default_effective": 2 if ready else 1, "rp_filter_interfaces": ( {"br0": 1} if endpoint_is_ipv4 else {}), "local_ipv4_addresses": ( [effective_endpoint] if endpoint_is_ipv4 else []), "local_ipv4_interfaces": ( {effective_endpoint: ["br0"]} if endpoint_is_ipv4 else {}), "fabric_rp_filter_raw": None, "fabric_rp_filter_effective": None, "raw_output": "must-not-leak", }, "wireguard": { "deployment_mode": "networkmanager_nmcli", "ready": True, "owned_by_panel": False, "ownership_marker": "must-not-be-projected", }, "nftables": { "ready": ready, "firewall_mode": "native_nftables" if ready else "unsupported", "service_active": ready, "service_persistent": ready, }, "network_roles": { "public": {"selector": "bridge:br0", "verified": True}, }, "tools": {"wg": {"path": "/usr/bin/wg"}}, "raw_output": "must-not-leak", } def current(self): return self.profile def reprobe_current_snapshot(self): if self.counter is not None: with self.counter["lock"]: self.counter["active"] += 1 self.counter["maximum"] = max( self.counter["maximum"], self.counter["active"]) if self.counter["active"] == self.counter.get("target"): self.counter["gate"].set() try: gate = self.counter.get("gate") if gate is not None: gate.wait(timeout=2) else: time.sleep(0.02) finally: with self.counter["lock"]: self.counter["active"] -= 1 if self.failure is not None: raise self.failure return self.profile, self.capabilities def test_fabric_components_treat_host_region_as_a_display_label(): components = {} labels = { "host-a": "中国香港", "host-b": "Hong Kong", } for host_id, key in (("host-a", KEY1), ("host-b", KEY2)): manager = FakeManager(key) manager.profile["host_id"] = host_id manager.profile["region"] = labels[host_id] components[host_id] = SimpleNamespace(manager=manager) resolved = asyncio.run(FabricCoordinator( MemoryStore(), Registry(components), )._components(plan())) assert set(resolved) == {"host-a", "host-b"} assert resolved["host-a"][1]["region"] == "中国香港" assert resolved["host-b"][1]["region"] == "Hong Kong" assert all(component.manager.reprobe_calls == 1 for component, _profile in resolved.values()) def test_fabric_components_accept_configured_local_underlay_endpoints(): components = {} for host_id, key, endpoint in ( ("host-a", KEY1, "203.0.113.1"), ("host-b", KEY2, "203.0.113.2"), ): manager = FakeManager( key, host="host.example.test", fabric_endpoint_ip=endpoint) manager.profile["host_id"] = host_id components[host_id] = SimpleNamespace(manager=manager) resolved = asyncio.run(FabricCoordinator( MemoryStore(), Registry(components), )._components(plan())) assert set(resolved) == {"host-a", "host-b"} def test_native_ssh_ipv4_fallback_rejects_nat_without_local_mapping(): first = FakeManager(KEY1, host="203.0.113.1") first.profile["host_id"] = "host-a" first.profile["capabilities"]["network"]["local_ipv4_addresses"] = [] first.profile["capabilities"]["network"]["local_ipv4_interfaces"] = {} first.profile["capabilities"]["network"]["rp_filter_interfaces"] = {} second = FakeManager(KEY2) second.profile["host_id"] = "host-b" with pytest.raises(FabricError) as error: asyncio.run(FabricCoordinator( MemoryStore(), Registry({ "host-a": SimpleNamespace(manager=first), "host-b": SimpleNamespace(manager=second), }), )._components(plan())) assert error.value.error_code == "fabric_endpoint_not_local" @pytest.mark.parametrize(("profile_changes", "error_code"), [ ({"host": "203.0.113.9"}, "fabric_endpoint_not_local"), ({"fabric_endpoint_ip": "127.0.0.1"}, "fabric_endpoint_invalid"), ]) def test_fabric_components_reject_mismatched_or_invalid_effective_endpoint( profile_changes, error_code, ): first = FakeManager(KEY1) first.profile.update(profile_changes) first.profile["host_id"] = "host-a" second = FakeManager(KEY2) second.profile["host_id"] = "host-b" coordinator = FabricCoordinator(MemoryStore(), Registry({ "host-a": SimpleNamespace(manager=first), "host-b": SimpleNamespace(manager=second), })) with pytest.raises(FabricError) as error: asyncio.run(coordinator._components(plan())) assert error.value.status == 409 assert error.value.error_code == error_code def test_fabric_components_reject_configured_endpoint_without_local_evidence(): first = FakeManager(KEY1, fabric_endpoint_ip="203.0.113.1") first.profile["host_id"] = "host-a" first.profile["capabilities"]["network"]["local_ipv4_addresses"] = [] second = FakeManager(KEY2) second.profile["host_id"] = "host-b" coordinator = FabricCoordinator(MemoryStore(), Registry({ "host-a": SimpleNamespace(manager=first), "host-b": SimpleNamespace(manager=second), })) with pytest.raises(FabricError) as error: asyncio.run(coordinator._components(plan())) assert error.value.status == 409 assert error.value.error_code == "fabric_endpoint_not_local" first.profile["capabilities"]["network"]["local_ipv4_addresses"] = [ "203.0.113.1", ] first.profile["capabilities"]["network"]["local_ipv4_interfaces"] = { "203.0.113.1": ["br0"], } first.profile["capabilities"]["network"]["rp_filter_interfaces"] = {} with pytest.raises(FabricError) as error: asyncio.run(coordinator._components(plan())) assert error.value.error_code == "fabric_endpoint_rp_filter_unverified" @pytest.mark.parametrize("value", [ {}, {"host_ids": []}, {"host_ids": ["host-a"], "extra": True}, {"host_ids": ["host-a", "host-a"]}, {"host_ids": ["bad.id"]}, {"host_ids": ["host-a"] * 65}, ]) def test_readiness_probe_rejects_non_exact_or_unbounded_host_ids(value): coordinator = FabricCoordinator(MemoryStore(), Registry({})) with pytest.raises(FabricError) as error: asyncio.run(coordinator.probe_hosts(value)) assert error.value.status == 422 assert error.value.error_code == "fabric_probe_invalid" def test_readiness_probe_is_bounded_isolated_and_secret_free(): counter = { "active": 0, "maximum": 0, "lock": threading.Lock(), "target": FABRIC_PROBE_CONCURRENCY, "gate": threading.Event(), } components = {} host_ids = [] for index in range(FABRIC_PROBE_CONCURRENCY + 4): host_id = f"host-{index}" host_ids.append(host_id) failure = None if index == 1: failure = RemoteHostError( "remote output /secret/path", 502, "host_capability_probe_failed") manager = ProbeManager( host_id, host="host.example.test" if index == 2 else f"203.0.113.{index + 1}", ready=index != 3, failure=failure, counter=counter, ) components[host_id] = SimpleNamespace(manager=manager) coordinator = FabricCoordinator(MemoryStore(), Registry(components)) async def run_with_constrained_default_executor(): # Prove readiness uses its own 16-worker executor rather than merely # inheriting whatever size asyncio chose for the process-wide pool. asyncio.get_running_loop().set_default_executor(ThreadPoolExecutor( max_workers=1, thread_name_prefix="test-default", )) return await coordinator.probe_hosts({"host_ids": host_ids}) result = asyncio.run(run_with_constrained_default_executor()) coordinator.close() assert [item["host_id"] for item in result["hosts"]] == host_ids assert counter["maximum"] == FABRIC_PROBE_CONCURRENCY assert result["hosts"][0] == { "host_id": "host-0", "configured": True, "reachable": True, "region": "hk", "endpoint_ip": "203.0.113.1", "endpoint_source": "ssh_host_ipv4", "probed_at": result["hosts"][0]["probed_at"], "vpc_ready": True, "reasons": [], "capabilities": { "os_id": "almalinux", "os_version": "10", "network_manager": "networkmanager", "network_manager_expected": "networkmanager", "network_manager_supported": True, "deployment_mode": "networkmanager_nmcli", "firewall_manager": "nftables", "firewall_mode": "native_nftables", "underlay_interface": "br0", "ip_forward": 1, "rp_filter": 2, "rp_filter_all": 2, "rp_filter_default": 1, "rp_filter_default_effective": 2, "rp_filter_underlay_raw": 1, "rp_filter_underlay_effective": 2, "rp_filter_fabric_raw": None, "rp_filter_fabric_effective": None, "wireguard_ready": True, "wireguard_owned": False, "nftables_ready": True, "nftables_active": True, "nftables_persistent": True, "public_selector": "bridge:br0", "public_verified": True, }, } failed = result["hosts"][1] assert failed["configured"] is True assert failed["reachable"] is False assert failed["error_code"] == "fabric_host_capability_probe_failed" assert failed["reasons"] == ["fabric_host_capability_probe_failed"] assert result["hosts"][2]["endpoint_ip"] is None assert result["hosts"][2]["endpoint_source"] is None assert result["hosts"][3]["vpc_ready"] is False assert result["hosts"][3]["reasons"] == [ "firewall_owner_not_nftables", "rp_filter_not_compatible", ] @pytest.mark.parametrize(("raw", "projected"), [ (FIREWALL_MODE_NATIVE, FIREWALL_MODE_NATIVE), ("firewalld_nft", "unsupported"), ("../../operator-data", "unsupported"), (None, "unsupported"), ]) def test_readiness_projects_only_supported_firewall_modes(raw, projected): result = FabricCoordinator._readiness_capabilities({ "network": {"firewall_mode": raw}, }) assert result["firewall_mode"] == projected serialized = json.dumps(result) assert "must-not-leak" not in serialized assert "/secret/" not in serialized assert "/usr/bin/wg" not in serialized def test_readiness_probe_does_not_construct_an_unconfigured_host(): registry = UnconfiguredRegistry() result = asyncio.run(FabricCoordinator( MemoryStore(), registry, ).probe_hosts({"host_ids": ["host-unknown"]})) assert registry.get_called is False item = result["hosts"][0] assert item == { "host_id": "host-unknown", "configured": False, "reachable": False, "region": "", "endpoint_ip": None, "endpoint_source": None, "probed_at": item["probed_at"], "vpc_ready": False, "reasons": ["fabric_host_not_configured"], "capabilities": {}, "error_code": "fabric_host_not_configured", } @pytest.mark.parametrize("address", [ "0.0.0.0", "0.1.2.3", "127.23.45.67", "169.254.12.34", "224.0.0.1", "240.0.0.1", "255.255.255.255", ]) def test_readiness_probe_hides_unusable_endpoint_ipv4(address): manager = ProbeManager("host-a", host=address) coordinator = FabricCoordinator( MemoryStore(), Registry({"host-a": SimpleNamespace(manager=manager)}), ) try: result = asyncio.run(coordinator.probe_hosts({ "host_ids": ["host-a"], })) finally: coordinator.close() readiness = result["hosts"][0] assert readiness["configured"] is True assert readiness["reachable"] is True assert readiness["vpc_ready"] is False assert readiness["endpoint_ip"] is None assert readiness["endpoint_source"] is None assert readiness["error_code"] == "fabric_endpoint_not_local" def test_readiness_probe_prefers_configured_fabric_endpoint(): manager = ProbeManager( "host-a", host="host.example.test", fabric_endpoint_ip="10.20.30.40", ) coordinator = FabricCoordinator( MemoryStore(), Registry({"host-a": SimpleNamespace(manager=manager)}), ) try: result = asyncio.run(coordinator.probe_hosts({ "host_ids": ["host-a"], })) finally: coordinator.close() readiness = result["hosts"][0] assert readiness["endpoint_ip"] == "10.20.30.40" assert readiness["endpoint_source"] == "configured" def test_readiness_probe_rejects_native_fallback_without_local_evidence(): manager = ProbeManager("host-a", host="203.0.113.10") manager.capabilities["network"]["local_ipv4_addresses"] = [] manager.capabilities["network"]["local_ipv4_interfaces"] = {} manager.capabilities["network"]["rp_filter_interfaces"] = {} coordinator = FabricCoordinator( MemoryStore(), Registry({"host-a": SimpleNamespace(manager=manager)}), ) try: result = asyncio.run(coordinator.probe_hosts({ "host_ids": ["host-a"], })) finally: coordinator.close() readiness = result["hosts"][0] assert readiness["endpoint_ip"] == "203.0.113.10" assert readiness["endpoint_source"] == "ssh_host_ipv4" assert readiness["error_code"] == "fabric_endpoint_not_local" assert readiness["reasons"] == ["fabric_endpoint_not_local"] manager.capabilities["network"]["local_ipv4_addresses"] = [ "203.0.113.10", ] manager.capabilities["network"]["local_ipv4_interfaces"] = { "203.0.113.10": ["br0"], } coordinator = FabricCoordinator( MemoryStore(), Registry({"host-a": SimpleNamespace(manager=manager)}), ) try: result = asyncio.run(coordinator.probe_hosts({ "host_ids": ["host-a"], })) finally: coordinator.close() assert result["hosts"][0]["error_code"] == ( "fabric_endpoint_rp_filter_unverified" ) def test_readiness_probe_rejects_overlapping_batch_without_queueing(): started = threading.Event() release = threading.Event() manager = ProbeManager("host-a") def blocking_reprobe(): started.set() release.wait(timeout=2) return manager.profile, manager.capabilities manager.reprobe_current_snapshot = blocking_reprobe coordinator = FabricCoordinator( MemoryStore(), Registry({"host-a": SimpleNamespace(manager=manager)}), ) async def exercise(): first = asyncio.create_task(coordinator.probe_hosts({ "host_ids": ["host-a"], })) try: for _ in range(100): if started.is_set(): break await asyncio.sleep(0.001) assert started.is_set() with pytest.raises(FabricError) as error: await coordinator.probe_hosts({"host_ids": ["host-a"]}) assert error.value.status == 409 assert error.value.error_code == "fabric_probe_busy" finally: release.set() return await first try: result = asyncio.run(exercise()) finally: release.set() coordinator.close() assert result["hosts"][0]["reachable"] is True def test_coordinator_keeps_successful_hosts_and_blocks_next_stage(monkeypatch): store = MemoryStore() managers = {} components = {} for host_id, key in (("host-a", KEY1), ("host-b", KEY2)): manager = FakeManager(key) manager.host_id = host_id manager.profile["host_id"] = host_id manager.profile["region"] = "hk" manager.profile["capabilities"] = { "vpc_ready": True, "wireguard": { "interface": FABRIC_INTERFACE, "deployment_mode": "networkmanager_nmcli", "ready": True, }, "network": { "network_manager": "networkmanager", "network_manager_supported": True, "ip_probe_valid": True, "local_ipv4_addresses": [manager.effective_host], "local_ipv4_interfaces": { manager.effective_host: ["eth0"], }, "rp_filter": 2, "rp_filter_default": 2, "rp_filter_interfaces": {"eth0": 1}, }, } managers[host_id] = manager components[host_id] = SimpleNamespace(manager=manager) monkeypatch.setattr("virsh_panel.fabric.FabricHostAdapter", FakeAdapter) FakeAdapter.failures = {"host-b"} result = asyncio.run(FabricCoordinator(store, Registry(components)).apply({ "fabric_id": "hk-fabric", "region": "hk", "overlay_cidr": "100.64.1.0/24", "revision": 1, "nodes": [ {"host_id": "host-a", "overlay_ip": "100.64.1.1", "endpoint": "203.0.113.1:51820"}, {"host_id": "host-b", "overlay_ip": "100.64.1.2", "endpoint": "203.0.113.2:51820"}, ], })) assert result["state"] == "uncertain" assert result["next_stage_allowed"] is False states = {node["host_id"]: node["state"] for node in result["nodes"]} assert states == {"host-a": "active", "host-b": "uncertain"} failed = next( node for node in result["nodes"] if node["host_id"] == "host-b") assert failed["last_error_phase"] == "apply" persisted = json.loads(store.get( "fabric_node", f"{result['fabric_id']}.host-b")) assert persisted["last_error_phase"] == "apply" assert managers["host-a"].reprobe_calls == 1 assert managers["host-b"].reprobe_calls == 1 serialized = "\n".join(store.values.values()) assert "PRIVATE KEY" not in serialized assert KEY1 in serialized and KEY2 in serialized FakeAdapter.failures = set() def test_initial_apply_identity_failure_carries_safe_phase_without_record( monkeypatch): current = plan() store = MemoryStore() components = _fabric_components(current) class IdentityFailureAdapter(FakeAdapter): def identity(self): raise FabricError( "safe", 502, "fabric_remote_command_failed") monkeypatch.setattr( "virsh_panel.fabric.FabricHostAdapter", IdentityFailureAdapter) with pytest.raises(FabricError) as error: asyncio.run(FabricCoordinator( store, Registry(components)).apply(plan_payload(current))) assert error.value.error_code == "fabric_remote_command_failed" assert error.value.failure_phase == "identity" assert store.get("fabric_desired", current.fabric_id) is None def test_apply_records_post_apply_observe_failure_phase(monkeypatch): current = plan() store = MemoryStore() components = _fabric_components(current) monkeypatch.setattr("virsh_panel.fabric.FabricHostAdapter", FakeAdapter) FakeAdapter.failures = set() FakeAdapter.observe_failures = {"host-a"} try: result = asyncio.run(FabricCoordinator( store, Registry(components)).apply(plan_payload(current))) finally: FakeAdapter.observe_failures = set() nodes = {node["host_id"]: node for node in result["nodes"]} assert nodes["host-a"]["last_error_code"] == ( "fabric_remote_command_failed") assert nodes["host-a"]["last_error_phase"] == "observe" assert "last_error_phase" not in nodes["host-b"] persisted = json.loads(store.get( "fabric_node", f"{current.fabric_id}.host-a")) assert persisted["last_error_phase"] == "observe" def test_recover_does_not_replay_an_interrupted_write(): store = MemoryStore() desired = {"fabric_id": "hk-fabric", "state": "applying", "next_stage_allowed": False, "revision": 1, "plan_digest": "x", "nodes": [{"host_id": "host-a", "state": "active", "transport_verified": True}]} store.set("fabric_desired", "hk-fabric", json.dumps(desired)) coordinator = FabricCoordinator(store, Registry({})) coordinator.recover() recovered = json.loads(store.get("fabric_desired", "hk-fabric")) assert recovered["state"] == "uncertain" assert recovered["next_stage_allowed"] is False assert recovered["nodes"][0]["transport_verified"] is False desired["state"] = "active" desired["next_stage_allowed"] = True desired["nodes"][0]["state"] = "active" desired["nodes"][0]["transport_verified"] = True store.set("fabric_desired", "hk-fabric", json.dumps(desired)) coordinator.recover() restarted = json.loads(store.get("fabric_desired", "hk-fabric")) assert restarted["state"] == "uncertain" assert restarted["next_stage_allowed"] is False def test_public_status_does_not_echo_nested_secrets_or_commands(): public = FabricCoordinator._public({ "fabric_id": "f", "state": "uncertain", "listen_port": 51820, "private_key": "secret", "command": "rm -rf /", "nodes": [{ "host_id": "host-a", "public_key": KEY1, "last_error_code": "fabric_remote_command_failed", "last_error_phase": "apply", "private_key": "secret", "observed": { "interface": FABRIC_INTERFACE, "command": "wg show", "peers": [(KEY2, "203.0.113.2:51820", "100.64.1.2/32")], "handshakes": {KEY2: 123}, }, }, { "host_id": "host-b", "public_key": KEY2, "last_error_phase": "observe", }, { "host_id": "host-c", "public_key": KEY3, "last_error_code": "fabric_remote_command_failed", "last_error_phase": "stderr:/secret/path", }], }) serialized = json.dumps(public) assert public["listen_port"] == 51820 assert "private_key" not in serialized assert "command" not in public assert "command" not in public["nodes"][0]["observed"] assert KEY1 in serialized and KEY2 in serialized assert public["nodes"][0]["last_error_phase"] == "apply" assert "last_error_phase" not in public["nodes"][1] assert "last_error_phase" not in public["nodes"][2] assert FabricError( "safe", failure_phase="stderr:/secret/path").failure_phase is None def test_reconcile_rejects_a_tampered_plan_digest(): store = MemoryStore() current = plan() store.set("fabric_desired", current.fabric_id, json.dumps({ "fabric_id": current.fabric_id, "region": current.region, "overlay_cidr": current.overlay_cidr, "revision": current.revision, "listen_port": current.listen_port, "plan_digest": "0" * 64, "public_keys": {node.host_id: key for node, key in zip( current.nodes, (KEY1, KEY2))}, "nodes": [ {"host_id": node.host_id, "overlay_ip": node.overlay_ip, "endpoint": node.endpoint, "listen_port": node.listen_port} for node in current.nodes ], })) coordinator = FabricCoordinator(store, Registry({})) with pytest.raises(FabricError) as error: asyncio.run(coordinator.reconcile(current.fabric_id)) assert error.value.error_code == "fabric_record_corrupt" def _active_fabric_record(current): keys = {node.host_id: key for node, key in zip( current.nodes, (KEY1, KEY2))} return { "schema_version": 1, "fabric_id": current.fabric_id, "region": current.region, "overlay_cidr": current.overlay_cidr, "revision": current.revision, "listen_port": current.listen_port, "plan_digest": current.digest, "state": "active", "next_stage_allowed": True, "updated_at": int(time.time()), "public_keys": keys, "nodes": [ { "host_id": node.host_id, "overlay_ip": node.overlay_ip, "endpoint": node.endpoint, "listen_port": node.listen_port, "state": "active", "transport_verified": True, "observed_revision": current.revision, "observed_plan_digest": current.digest, } for node in current.nodes ], } def _fabric_components(current): components = {} for node, key in zip(current.nodes, (KEY1, KEY2)): manager = FakeManager(key) manager.host_id = node.host_id manager.profile["host_id"] = node.host_id manager.profile["region"] = current.region components[node.host_id] = SimpleNamespace(manager=manager) return components def _remote_operation_status( operation_id, status, *, checkpoint=None, reason=None): checkpoint = checkpoint or ( "complete" if status == "succeeded" else "preflight") reason = reason or ( "none" if status in {"queued", "running", "succeeded"} else "command_failed") started_at = None if status == "queued" else 101 finished_at = ( 103 if status in {"succeeded", "failed", "uncertain", "interrupted"} else None) return { "operation_id": operation_id, "status": status, "phase": "apply", "checkpoint": checkpoint, "reason": reason, "created_at": 100, "started_at": started_at, "updated_at": 103 if finished_at else 102, "finished_at": finished_at, } class AsyncCoordinatorAdapter: """Host stub that separates one dispatch from read-only status polling.""" def __init__(self, manager, _profile=None): self.manager = manager def identity(self, *, create_key=True): self.manager.async_identity_calls += 1 return self.manager.public_key def dispatch_apply(self, _plan, _host, _keys, operation_id=None): self.manager.async_dispatch_calls += 1 self.manager.async_operation_id = operation_id if self.manager.async_dispatch_entered is not None: self.manager.async_dispatch_entered.set() if self.manager.async_dispatch_release is not None: assert self.manager.async_dispatch_release.wait(timeout=2) if self.manager.async_dispatch_error is not None: error = self.manager.async_dispatch_error self.manager.async_dispatch_error = None raise error return _remote_operation_status(operation_id, "queued") def apply_status(self, operation_id): self.manager.async_status_calls += 1 assert operation_id == self.manager.async_operation_id statuses = self.manager.async_statuses if len(statuses) > 1: status = statuses.pop(0) else: status = statuses[0] if isinstance(status, Exception): raise status if isinstance(status, dict): return dict(status) return _remote_operation_status( operation_id, status, checkpoint=("nft_install" if status == "failed" else None), reason=("nft_failed" if status == "failed" else None), ) def observe(self, current, host, _keys): self.manager.async_observe_calls += 1 return _reconcile_observation(current, host) class PublishedQueuedGapAdapter(AsyncCoordinatorAdapter): """First dispatch publishes the directory but dies before systemd-run.""" def dispatch_apply(self, _plan, _host, _keys, operation_id=None): self.manager.async_dispatch_calls += 1 self.manager.async_operation_id = operation_id if self.manager.async_dispatch_calls == 1: raise FabricError( "ambiguous transport /root/private", 502, "fabric_host_connection_failed", "apply") self.manager.async_runner_starts += 1 self.manager.async_statuses = ["succeeded"] return _remote_operation_status(operation_id, "queued") class RecoveredQueuedGapAdapter(AsyncCoordinatorAdapter): """Restart joins a published queue and starts its original runner once.""" def dispatch_apply(self, _plan, _host, _keys, operation_id=None): self.manager.async_dispatch_calls += 1 assert operation_id == self.manager.async_operation_id self.manager.async_runner_starts += 1 self.manager.async_statuses = ["succeeded"] return _remote_operation_status(operation_id, "queued") class PartialHostCrashAdapter(AsyncCoordinatorAdapter): """Host A was admitted; host B still needs same-id continuation.""" def identity(self, *, create_key=True): self.manager.async_identity_calls += 1 self.manager.partial_identity_create_flags.append(create_key) return self.manager.public_key def dispatch_apply(self, _plan, _host, _keys, operation_id=None): assert self.manager.partial_configured is False self.manager.async_dispatch_calls += 1 self.manager.async_operation_id = operation_id self.manager.partial_configured = True return _remote_operation_status(operation_id, "succeeded") def observe(self, current, host, _keys): self.manager.async_observe_calls += 1 if self.manager.partial_configured: return _reconcile_observation(current, host) return { "state": "degraded", "config_ok": False, "transport_verified": False, "public_key": self.manager.public_key, "observed": { "interface": None, "config_revision": None, "config_digest": None, }, } def _async_components(current): components = _fabric_components(current) for component in components.values(): manager = component.manager manager.async_identity_calls = 0 manager.async_dispatch_calls = 0 manager.async_status_calls = 0 manager.async_observe_calls = 0 manager.async_operation_id = None manager.async_dispatch_entered = None manager.async_dispatch_release = None manager.async_dispatch_error = None manager.async_runner_starts = 0 manager.partial_configured = False manager.partial_identity_create_flags = [] manager.async_statuses = ["succeeded"] return components async def _await_fabric_operation(coordinator, fabric_id): for _attempt in range(500): task = coordinator._operation_tasks_by_fabric.get(fabric_id) if task is None: return await asyncio.sleep(0.002) raise AssertionError("Fabric background operation did not finish") def test_enqueue_apply_returns_before_host_systemd_job_finishes(monkeypatch): current = plan() store = MemoryStore() components = _async_components(current) entered = threading.Event() release = threading.Event() components["host-a"].manager.async_dispatch_entered = entered components["host-a"].manager.async_dispatch_release = release coordinator = FabricCoordinator(store, Registry(components)) monkeypatch.setattr( "virsh_panel.fabric.FabricHostAdapter", AsyncCoordinatorAdapter) monkeypatch.setattr( "virsh_panel.fabric.FABRIC_OPERATION_POLL_INTERVAL_SECONDS", 0) async def exercise(): admitted = await coordinator.enqueue_apply(plan_payload(current)) assert admitted["operation"]["status"] == "queued" assert re.fullmatch(r"[0-9a-f]{32}", admitted["operation"]["id"]) assert not entered.is_set() for _attempt in range(100): if entered.is_set(): break await asyncio.sleep(0.002) assert entered.is_set() live = coordinator.get_status(current.fabric_id) assert live["operation"]["status"] == "running" release.set() await _await_fabric_operation(coordinator, current.fabric_id) return admitted, coordinator.get_status(current.fabric_id) admitted, final = asyncio.run(exercise()) assert final["operation"]["id"] == admitted["operation"]["id"] assert final["operation"]["status"] == "succeeded" assert all(node["operation"]["status"] == "succeeded" for node in final["nodes"]) assert all(component.manager.async_dispatch_calls == 1 for component in components.values()) def test_cancelled_async_dispatch_drains_then_recovers_without_redispatch( monkeypatch): current = plan() store = MemoryStore() components = _async_components(current) entered = threading.Event() release = threading.Event() first = components["host-a"].manager first.async_dispatch_entered = entered first.async_dispatch_release = release coordinator = FabricCoordinator(store, Registry(components)) monkeypatch.setattr( "virsh_panel.fabric.FabricHostAdapter", AsyncCoordinatorAdapter) async def exercise(): admitted = await coordinator.enqueue_apply(plan_payload(current)) for _attempt in range(100): if entered.is_set(): break await asyncio.sleep(0.002) assert entered.is_set() operation_task = coordinator._operation_tasks_by_fabric[ current.fabric_id] lock_entered = asyncio.Event() async def wait_for_fabric_lock(): async with coordinator._locks[current.fabric_id]: lock_entered.set() waiter = asyncio.create_task(wait_for_fabric_lock()) operation_task.cancel() await asyncio.sleep(0) assert operation_task.done() is False assert lock_entered.is_set() is False release.set() with pytest.raises(asyncio.CancelledError): await operation_task await asyncio.wait_for(waiter, timeout=1) assert lock_entered.is_set() is True coordinator.recover() first.async_operation_id = admitted["operation"]["id"] first.async_statuses = ["succeeded"] resumed = await coordinator.enqueue_reconcile(current.fabric_id) assert resumed["operation"]["id"] == admitted["operation"]["id"] await _await_fabric_operation(coordinator, current.fabric_id) return admitted, coordinator.get_status(current.fabric_id) try: admitted, final = asyncio.run(exercise()) finally: release.set() assert final["operation"]["id"] == admitted["operation"]["id"] assert first.async_dispatch_calls == 1 assert components["host-b"].manager.async_dispatch_calls == 0 @pytest.mark.parametrize(("terminal", "expected_root"), [ ("succeeded", "succeeded"), ("failed", "failed"), ]) def test_dispatch_disconnect_is_reconciled_by_status_without_replay( monkeypatch, terminal, expected_root): current = plan() store = MemoryStore() components = _async_components(current) disconnected = components["host-a"].manager disconnected.async_dispatch_error = FabricError( "safe disconnect", 502, "fabric_host_connection_failed", "apply") disconnected.async_statuses = ["running", terminal] coordinator = FabricCoordinator(store, Registry(components)) monkeypatch.setattr( "virsh_panel.fabric.FabricHostAdapter", AsyncCoordinatorAdapter) monkeypatch.setattr( "virsh_panel.fabric.FABRIC_OPERATION_POLL_INTERVAL_SECONDS", 0) async def exercise(): admitted = await coordinator.enqueue_apply(plan_payload(current)) await _await_fabric_operation(coordinator, current.fabric_id) return admitted, coordinator.get_status(current.fabric_id) admitted, final = asyncio.run(exercise()) assert final["operation"]["id"] == admitted["operation"]["id"] assert final["operation"]["status"] == expected_root failed_or_recovered = next( node for node in final["nodes"] if node["host_id"] == "host-a") assert failed_or_recovered["operation"]["status"] == terminal assert disconnected.async_dispatch_calls == 1 assert disconnected.async_status_calls >= 2 if terminal == "succeeded": assert disconnected.async_observe_calls == 1 else: assert disconnected.async_observe_calls == 0 def test_async_apply_timeout_is_uncertain_and_never_redispatched(monkeypatch): current = plan() store = MemoryStore() components = _async_components(current) stalled = components["host-a"].manager stalled.async_statuses = ["running"] coordinator = FabricCoordinator(store, Registry(components)) monkeypatch.setattr( "virsh_panel.fabric.FabricHostAdapter", AsyncCoordinatorAdapter) monkeypatch.setattr( "virsh_panel.fabric.FABRIC_OPERATION_POLL_INTERVAL_SECONDS", 0.002) monkeypatch.setattr( "virsh_panel.fabric.FABRIC_OPERATION_POLL_TIMEOUT_SECONDS", 0.01) async def exercise(): await coordinator.enqueue_apply(plan_payload(current)) await _await_fabric_operation(coordinator, current.fabric_id) return coordinator.get_status(current.fabric_id) final = asyncio.run(exercise()) stalled_node = next( node for node in final["nodes"] if node["host_id"] == "host-a") assert final["operation"]["status"] == "uncertain" assert stalled_node["operation"]["status"] == "uncertain" assert stalled_node["operation"]["reason"] == "transport_failed" assert stalled.async_dispatch_calls == 1 assert stalled.async_status_calls >= 1 @pytest.mark.parametrize("error_code", [ "fabric_operation_status_invalid", "fabric_operation_not_found", "fabric_remote_command_failed", ]) def test_confirmed_dispatch_status_error_stays_uncertain_and_resumes_same_id( monkeypatch, error_code): current = plan() store = MemoryStore() components = _async_components(current) for component in components.values(): component.manager.async_statuses = [ FabricError("unsafe /root/private", 502, error_code, "observe"), "succeeded", ] coordinator = FabricCoordinator(store, Registry(components)) monkeypatch.setattr( "virsh_panel.fabric.FabricHostAdapter", AsyncCoordinatorAdapter) monkeypatch.setattr( "virsh_panel.fabric.FABRIC_OPERATION_POLL_INTERVAL_SECONDS", 0) async def exercise(): admitted = await coordinator.enqueue_apply(plan_payload(current)) await _await_fabric_operation(coordinator, current.fabric_id) uncertain = coordinator.get_status(current.fabric_id) assert uncertain["operation"]["status"] == "uncertain" assert uncertain["next_stage_allowed"] is False resumed = await coordinator.enqueue_reconcile(current.fabric_id) assert resumed["operation"]["id"] == admitted["operation"]["id"] await _await_fabric_operation(coordinator, current.fabric_id) return admitted, coordinator.get_status(current.fabric_id) admitted, final = asyncio.run(exercise()) assert final["operation"]["id"] == admitted["operation"]["id"] assert final["operation"]["status"] == "succeeded" assert all(component.manager.async_dispatch_calls == 1 for component in components.values()) assert all(component.manager.async_status_calls >= 2 for component in components.values()) assert "/root/private" not in json.dumps(final) def test_published_queue_gap_recovers_original_runner_once_with_same_id( monkeypatch): current = plan() store = MemoryStore() components = _async_components(current) for component in components.values(): component.manager.async_statuses = ["queued"] coordinator = FabricCoordinator(store, Registry(components)) monkeypatch.setattr( "virsh_panel.fabric.FabricHostAdapter", PublishedQueuedGapAdapter) monkeypatch.setattr( "virsh_panel.fabric.FABRIC_OPERATION_POLL_INTERVAL_SECONDS", 0.001) monkeypatch.setattr( "virsh_panel.fabric.FABRIC_OPERATION_POLL_TIMEOUT_SECONDS", 0.004) async def exercise(): admitted = await coordinator.enqueue_apply(plan_payload(current)) await _await_fabric_operation(coordinator, current.fabric_id) stalled = coordinator.get_status(current.fabric_id) assert stalled["operation"]["status"] == "uncertain" assert stalled["next_stage_allowed"] is False resumed = await coordinator.enqueue_reconcile(current.fabric_id) assert resumed["operation"]["id"] == admitted["operation"]["id"] await _await_fabric_operation(coordinator, current.fabric_id) restarted = coordinator.get_status(current.fabric_id) assert restarted["operation"]["status"] == "running" assert restarted["next_stage_allowed"] is False await coordinator.reconcile_all() return admitted, coordinator.get_status(current.fabric_id) admitted, final = asyncio.run(exercise()) assert final["operation"]["id"] == admitted["operation"]["id"] assert final["operation"]["status"] == "succeeded" assert final["next_stage_allowed"] is True assert all(component.manager.async_dispatch_calls == 2 for component in components.values()) assert all(component.manager.async_runner_starts == 1 for component in components.values()) assert all(component.manager.async_operation_id == admitted["operation"]["id"] for component in components.values()) def test_restart_periodic_queued_status_still_allows_same_id_runner_recovery( monkeypatch): current = plan() operation_id = "a" * 32 store = MemoryStore() components = _async_components(current) coordinator = FabricCoordinator(store, Registry(components)) admitted = coordinator._admit_apply_record(current, operation_id) admitted["operation"].update( status="running", started_at=101, updated_at=101) admitted["public_keys"] = {"host-a": KEY1, "host-b": KEY2} for node, key in zip(admitted["nodes"], (KEY1, KEY2)): node["public_key"] = key node["operation"].update( status="running", phase="apply", checkpoint="dispatch", started_at=101, updated_at=101) store.set( "fabric_desired", current.fabric_id, json.dumps(admitted)) coordinator.recover() for component in components.values(): manager = component.manager manager.async_operation_id = operation_id manager.async_statuses = ["queued"] monkeypatch.setattr( "virsh_panel.fabric.FabricHostAdapter", RecoveredQueuedGapAdapter) async def exercise(): await coordinator.reconcile_all() observed = coordinator.get_status(current.fabric_id) assert observed["operation"]["id"] == operation_id assert observed["operation"]["status"] == "running" assert observed["next_stage_allowed"] is False assert all(component.manager.async_dispatch_calls == 0 for component in components.values()) resumed = await coordinator.enqueue_reconcile(current.fabric_id) assert resumed["operation"]["id"] == operation_id await _await_fabric_operation(coordinator, current.fabric_id) restarted = coordinator.get_status(current.fabric_id) assert restarted["operation"]["status"] == "running" assert restarted["next_stage_allowed"] is False await coordinator.reconcile_all() return coordinator.get_status(current.fabric_id) final = asyncio.run(exercise()) assert final["operation"]["id"] == operation_id assert final["operation"]["status"] == "succeeded" assert final["next_stage_allowed"] is True assert all(component.manager.async_dispatch_calls == 1 for component in components.values()) assert all(component.manager.async_runner_starts == 1 for component in components.values()) assert all(component.manager.async_operation_id == operation_id for component in components.values()) def _interrupted_async_record(current, operation_id): record = _active_fabric_record(current) record.update(state="uncertain", next_stage_allowed=False) record["operation"] = { "id": operation_id, "kind": "apply", "status": "interrupted", "created_at": 100, "started_at": 101, "updated_at": 102, "finished_at": 102, } for node in record["nodes"]: node.update( state="uncertain", transport_verified=False, observed_revision=None, observed_plan_digest=None, operation={ "id": operation_id, "status": "uncertain", "phase": "apply", "checkpoint": "preflight", "reason": "transport_failed", "created_at": 100, "started_at": 101, "updated_at": 102, "finished_at": 102, }, ) return record @pytest.mark.parametrize("crash_stage", [ "before_identity", "mid_identity", "identity_fenced", ]) def test_pre_dispatch_crash_resumes_same_operation_id_without_replay( monkeypatch, crash_stage): current = plan() operation_id = "5" * 32 store = MemoryStore() components = _async_components(current) coordinator = FabricCoordinator(store, Registry(components)) admitted = coordinator._admit_apply_record(current, operation_id) if crash_stage in {"mid_identity", "identity_fenced"}: admitted["operation"].update( status="running", started_at=101, updated_at=101) admitted["nodes"][0]["operation"].update( status="running", phase="identity", checkpoint="identity", started_at=101, updated_at=101) if crash_stage == "identity_fenced": admitted["public_keys"] = {"host-a": KEY1, "host-b": KEY2} for node, key in zip(admitted["nodes"], (KEY1, KEY2)): node["public_key"] = key node["operation"].update( status="running", phase="identity", checkpoint="identity", started_at=101, updated_at=101) store.set( "fabric_desired", current.fabric_id, json.dumps(admitted)) coordinator.recover() for component in components.values(): component.manager.async_operation_id = operation_id component.manager.async_statuses = ["succeeded"] monkeypatch.setattr( "virsh_panel.fabric.FabricHostAdapter", AsyncCoordinatorAdapter) monkeypatch.setattr( "virsh_panel.fabric.FABRIC_OPERATION_POLL_INTERVAL_SECONDS", 0) async def exercise(): resumed = await coordinator.enqueue_reconcile(current.fabric_id) assert resumed["operation"]["id"] == operation_id await _await_fabric_operation(coordinator, current.fabric_id) return coordinator.get_status(current.fabric_id) final = asyncio.run(exercise()) assert final["operation"]["id"] == operation_id assert final["operation"]["status"] == "succeeded" assert all(component.manager.async_dispatch_calls == 1 for component in components.values()) def _partial_host_crash_record(coordinator, current, operation_id): admitted = coordinator._admit_apply_record(current, operation_id) admitted["operation"].update( status="running", started_at=101, updated_at=101) admitted["public_keys"] = {"host-a": KEY1, "host-b": KEY2} for node, key in zip(admitted["nodes"], (KEY1, KEY2)): node["public_key"] = key node["operation"].update( status="running", phase="identity", checkpoint="identity", started_at=101, updated_at=101) admitted["nodes"][0]["operation"].update( phase="apply", checkpoint="dispatch") return admitted def test_crash_between_hosts_continues_only_undispatched_host_with_same_id( monkeypatch): current = plan() operation_id = "6" * 32 store = MemoryStore() components = _async_components(current) coordinator = FabricCoordinator(store, Registry(components)) admitted = _partial_host_crash_record( coordinator, current, operation_id) store.set( "fabric_desired", current.fabric_id, json.dumps(admitted)) coordinator.recover() first = components["host-a"].manager second = components["host-b"].manager first.async_operation_id = operation_id first.async_statuses = ["succeeded"] first.partial_configured = True second.async_statuses = ["succeeded"] monkeypatch.setattr( "virsh_panel.fabric.FabricHostAdapter", PartialHostCrashAdapter) async def exercise(): resumed = await coordinator.enqueue_reconcile(current.fabric_id) assert resumed["operation"]["id"] == operation_id await _await_fabric_operation(coordinator, current.fabric_id) continuing = coordinator.get_status(current.fabric_id) assert continuing["operation"]["status"] == "running" assert continuing["next_stage_allowed"] is False await coordinator.reconcile_all() return coordinator.get_status(current.fabric_id) final = asyncio.run(exercise()) assert final["operation"]["id"] == operation_id assert final["operation"]["status"] == "succeeded" assert final["next_stage_allowed"] is True assert first.async_dispatch_calls == 0 assert second.async_dispatch_calls == 1 assert first.async_identity_calls == 0 assert second.async_identity_calls == 1 assert second.partial_identity_create_flags == [False] assert second.async_operation_id == operation_id def test_partial_remote_failed_but_exact_continues_only_undispatched_host( monkeypatch): current = plan() operation_id = "7" * 32 store = MemoryStore() components = _async_components(current) coordinator = FabricCoordinator(store, Registry(components)) store.set( "fabric_desired", current.fabric_id, json.dumps(_partial_host_crash_record( coordinator, current, operation_id))) coordinator.recover() first = components["host-a"].manager second = components["host-b"].manager first.async_operation_id = operation_id first.async_statuses = ["failed"] first.partial_configured = True second.async_statuses = ["succeeded"] monkeypatch.setattr( "virsh_panel.fabric.FabricHostAdapter", PartialHostCrashAdapter) async def exercise(): resumed = await coordinator.enqueue_reconcile(current.fabric_id) assert resumed["operation"]["id"] == operation_id await _await_fabric_operation(coordinator, current.fabric_id) continuing = coordinator.get_status(current.fabric_id) assert continuing["operation"]["status"] == "running" assert continuing["next_stage_allowed"] is False await coordinator.reconcile_all() return coordinator.get_status(current.fabric_id) final = asyncio.run(exercise()) assert final["operation"]["id"] == operation_id assert final["operation"]["status"] == "succeeded" assert final["next_stage_allowed"] is True assert first.async_dispatch_calls == 0 assert second.async_dispatch_calls == 1 assert first.async_identity_calls == 0 assert second.async_identity_calls == 1 assert second.partial_identity_create_flags == [False] assert second.async_operation_id == operation_id def test_partial_remote_failed_nonexact_stays_terminal_then_uses_new_id( monkeypatch): current = plan() operation_id = "8" * 32 store = MemoryStore() components = _async_components(current) coordinator = FabricCoordinator(store, Registry(components)) store.set( "fabric_desired", current.fabric_id, json.dumps(_partial_host_crash_record( coordinator, current, operation_id))) coordinator.recover() first = components["host-a"].manager second = components["host-b"].manager first.async_operation_id = operation_id first.async_statuses = ["failed"] first.partial_configured = False second.partial_configured = False monkeypatch.setattr( "virsh_panel.fabric.FabricHostAdapter", PartialHostCrashAdapter) async def exercise(): resumed = await coordinator.enqueue_reconcile(current.fabric_id) assert resumed["operation"]["id"] == operation_id await _await_fabric_operation(coordinator, current.fabric_id) failed = coordinator.get_status(current.fabric_id) assert failed["operation"]["status"] == "failed" first.async_statuses = ["succeeded"] retried = await coordinator.enqueue_reconcile(current.fabric_id) assert retried["operation"]["id"] != operation_id await _await_fabric_operation(coordinator, current.fabric_id) return failed, coordinator.get_status(current.fabric_id) failed, final = asyncio.run(exercise()) assert failed["operation"]["id"] == operation_id assert failed["state"] == "uncertain" assert failed["next_stage_allowed"] is False assert final["operation"]["id"] != operation_id assert final["operation"]["status"] == "succeeded" assert final["next_stage_allowed"] is True assert first.async_dispatch_calls == 1 assert second.async_dispatch_calls == 1 assert first.async_identity_calls == 1 assert second.async_identity_calls == 1 class NonExactAsyncCoordinatorAdapter(AsyncCoordinatorAdapter): def observe(self, current, host, keys): result = super().observe(current, host, keys) result.update( state="degraded", config_ok=False, transport_verified=False) result["observed"]["config_digest"] = "0" * 64 return result def test_remote_success_with_non_exact_readback_is_terminal_uncertain( monkeypatch): current = plan() store = MemoryStore() components = _async_components(current) coordinator = FabricCoordinator(store, Registry(components)) monkeypatch.setattr( "virsh_panel.fabric.FabricHostAdapter", NonExactAsyncCoordinatorAdapter) monkeypatch.setattr( "virsh_panel.fabric.FABRIC_OPERATION_POLL_INTERVAL_SECONDS", 0) async def exercise(): admitted = await coordinator.enqueue_apply(plan_payload(current)) await _await_fabric_operation(coordinator, current.fabric_id) degraded = coordinator.get_status(current.fabric_id) assert degraded["operation"]["status"] == "uncertain" assert degraded["next_stage_allowed"] is False durable = json.loads(store.get( "fabric_desired", current.fabric_id)) assert all( node["operation"]["phase"] == "observe" and node["operation"]["checkpoint"] == "final_readback" for node in durable["nodes"]) assert coordinator._operation_may_have_dispatched( durable, admitted["operation"]["id"]) is True resumed = await coordinator.enqueue_reconcile(current.fabric_id) assert resumed["operation"]["id"] == admitted["operation"]["id"] await _await_fabric_operation(coordinator, current.fabric_id) return admitted, coordinator.get_status(current.fabric_id) admitted, final = asyncio.run(exercise()) assert final["operation"]["id"] == admitted["operation"]["id"] assert final["operation"]["status"] == "uncertain" assert final["state"] == "uncertain" assert final["next_stage_allowed"] is False assert all(node["operation"]["status"] == "uncertain" for node in final["nodes"]) assert all(component.manager.async_dispatch_calls == 1 for component in components.values()) def test_restart_resume_reads_exact_operation_without_redispatch(monkeypatch): current = plan() operation_id = "1" * 32 store = MemoryStore() store.set( "fabric_desired", current.fabric_id, json.dumps(_interrupted_async_record(current, operation_id))) components = _async_components(current) for component in components.values(): component.manager.async_operation_id = operation_id component.manager.async_statuses = ["succeeded"] coordinator = FabricCoordinator(store, Registry(components)) monkeypatch.setattr( "virsh_panel.fabric.FabricHostAdapter", AsyncCoordinatorAdapter) async def exercise(): admitted = await coordinator.enqueue_reconcile(current.fabric_id) assert admitted["operation"]["id"] == operation_id await _await_fabric_operation(coordinator, current.fabric_id) return coordinator.get_status(current.fabric_id) final = asyncio.run(exercise()) assert final["operation"]["id"] == operation_id assert final["operation"]["status"] == "succeeded" assert final["next_stage_allowed"] is True assert all(component.manager.async_dispatch_calls == 0 for component in components.values()) assert all(component.manager.async_status_calls == 1 for component in components.values()) assert all(component.manager.async_observe_calls == 1 for component in components.values()) def test_restart_resume_does_not_publish_running_job_as_succeeded(monkeypatch): current = plan() operation_id = "2" * 32 store = MemoryStore() store.set( "fabric_desired", current.fabric_id, json.dumps(_interrupted_async_record(current, operation_id))) components = _async_components(current) for component in components.values(): component.manager.async_operation_id = operation_id component.manager.async_statuses = ["running"] coordinator = FabricCoordinator(store, Registry(components)) monkeypatch.setattr( "virsh_panel.fabric.FabricHostAdapter", AsyncCoordinatorAdapter) async def exercise(): await coordinator.enqueue_reconcile(current.fabric_id) await _await_fabric_operation(coordinator, current.fabric_id) return coordinator.get_status(current.fabric_id) final = asyncio.run(exercise()) assert final["operation"]["id"] == operation_id assert final["operation"]["status"] == "running" assert final["next_stage_allowed"] is False assert all(node["operation"]["status"] == "running" for node in final["nodes"]) assert all(component.manager.async_dispatch_calls == 0 for component in components.values()) assert all(component.manager.async_observe_calls == 0 for component in components.values()) def test_restart_missing_runtime_operation_can_converge_by_exact_observation( monkeypatch): current = plan() operation_id = "3" * 32 store = MemoryStore() store.set( "fabric_desired", current.fabric_id, json.dumps(_interrupted_async_record(current, operation_id))) components = _async_components(current) for component in components.values(): component.manager.async_operation_id = operation_id component.manager.async_statuses = [FabricError( "safe missing", 409, "fabric_operation_not_found", "observe")] coordinator = FabricCoordinator(store, Registry(components)) monkeypatch.setattr( "virsh_panel.fabric.FabricHostAdapter", AsyncCoordinatorAdapter) asyncio.run(coordinator.reconcile_all()) final = coordinator.get_status(current.fabric_id) assert final["operation"]["id"] == operation_id assert final["operation"]["status"] == "succeeded" assert final["state"] == "active" assert final["next_stage_allowed"] is True assert all(component.manager.async_dispatch_calls == 0 for component in components.values()) assert all(component.manager.async_observe_calls == 1 for component in components.values()) @pytest.mark.parametrize("error_code", [ "fabric_host_connection_failed", "fabric_operation_status_invalid", "fabric_remote_command_failed", ]) def test_restart_status_failure_cannot_use_old_exact_observation( monkeypatch, error_code): current = plan() operation_id = "4" * 32 store = MemoryStore() store.set( "fabric_desired", current.fabric_id, json.dumps(_interrupted_async_record(current, operation_id))) components = _async_components(current) for component in components.values(): component.manager.async_operation_id = operation_id component.manager.async_statuses = [FabricError( "unsafe transport output /root/private", 502, error_code, "observe")] coordinator = FabricCoordinator(store, Registry(components)) monkeypatch.setattr( "virsh_panel.fabric.FabricHostAdapter", AsyncCoordinatorAdapter) asyncio.run(coordinator.reconcile_all()) final = coordinator.get_status(current.fabric_id) durable = json.loads(store.get("fabric_desired", current.fabric_id)) assert final["operation"]["id"] == operation_id assert final["operation"]["status"] == "uncertain" assert final["state"] == "uncertain" assert final["next_stage_allowed"] is False assert all(node["operation"]["status"] == "uncertain" for node in final["nodes"]) assert all(component.manager.async_dispatch_calls == 0 for component in components.values()) assert "/root/private" not in json.dumps(durable) def test_async_identity_is_durably_fenced_before_first_host_dispatch( monkeypatch): current = plan() store = MemoryStore() components = _async_components(current) entered = threading.Event() release = threading.Event() components["host-a"].manager.async_dispatch_entered = entered components["host-a"].manager.async_dispatch_release = release coordinator = FabricCoordinator(store, Registry(components)) monkeypatch.setattr( "virsh_panel.fabric.FabricHostAdapter", AsyncCoordinatorAdapter) monkeypatch.setattr( "virsh_panel.fabric.FABRIC_OPERATION_POLL_INTERVAL_SECONDS", 0) async def exercise(): admitted = await coordinator.enqueue_apply(plan_payload(current)) for _attempt in range(100): if entered.is_set(): break await asyncio.sleep(0.002) assert entered.is_set() durable = json.loads(store.get( "fabric_desired", current.fabric_id)) release.set() await _await_fabric_operation(coordinator, current.fabric_id) return admitted, durable try: admitted, durable = asyncio.run(exercise()) finally: release.set() assert durable["operation"]["id"] == admitted["operation"]["id"] assert durable["public_keys"] == {"host-a": KEY1, "host-b": KEY2} assert { node["host_id"]: node.get("public_key") for node in durable["nodes"] } == {"host-a": KEY1, "host-b": KEY2} def test_observe_only_probe_failure_closes_durable_active_gate(monkeypatch): current = plan() store = MemoryStore() store.set( "fabric_desired", current.fabric_id, json.dumps(_active_fabric_record(current))) components = _fabric_components(current) def probe_failed(): raise RemoteHostError( "unsafe remote detail /root/private", 502, "host_capability_probe_failed") components["host-a"].manager.reprobe_current = probe_failed coordinator = FabricCoordinator(store, Registry(components)) asyncio.run(coordinator.reconcile_all()) durable = json.loads(store.get("fabric_desired", current.fabric_id)) assert durable["state"] == "uncertain" assert durable["next_stage_allowed"] is False assert all(node["transport_verified"] is False for node in durable["nodes"]) assert "/root/private" not in json.dumps(durable) def test_enqueue_reconcile_admission_uses_shared_catalog_lock(monkeypatch): current = plan() store = MemoryStore() store.set( "fabric_desired", current.fabric_id, json.dumps(_active_fabric_record(current))) components = _async_components(current) coordinator = FabricCoordinator(store, Registry(components)) monkeypatch.setattr( "virsh_panel.fabric.FabricHostAdapter", AsyncCoordinatorAdapter) async def exercise(): shared = coordinator.catalog_mutation_lock() async with shared: admission = asyncio.create_task( coordinator.enqueue_reconcile(current.fabric_id)) await asyncio.sleep(0.02) assert admission.done() is False durable = json.loads(store.get( "fabric_desired", current.fabric_id)) assert "operation" not in durable receipt = await asyncio.wait_for(admission, timeout=1) await coordinator.shutdown_operations() return receipt receipt = asyncio.run(exercise()) assert receipt["operation"]["kind"] == "reconcile" assert receipt["operation"]["status"] == "queued" def test_background_internal_error_cannot_leave_operation_running(monkeypatch): current = plan() store = MemoryStore() components = _async_components(current) class UnexpectedFailureAdapter(AsyncCoordinatorAdapter): def dispatch_apply(self, *_args, **_kwargs): self.manager.async_dispatch_calls += 1 raise RuntimeError("unsafe implementation detail /root/private") coordinator = FabricCoordinator(store, Registry(components)) monkeypatch.setattr( "virsh_panel.fabric.FabricHostAdapter", UnexpectedFailureAdapter) async def exercise(): await coordinator.enqueue_apply(plan_payload(current)) await _await_fabric_operation(coordinator, current.fabric_id) return coordinator.get_status(current.fabric_id) final = asyncio.run(exercise()) assert final["operation"]["status"] in { "failed", "uncertain", "interrupted", } assert final["operation"]["finished_at"] is not None assert all(node["operation"]["status"] not in {"pending", "running"} for node in final["nodes"]) assert "/root/private" not in json.dumps(final) def test_periodic_reconcile_is_read_only_and_never_dispatches(monkeypatch): current = plan() store = MemoryStore() store.set( "fabric_desired", current.fabric_id, json.dumps(_active_fabric_record(current))) components = _async_components(current) coordinator = FabricCoordinator(store, Registry(components)) monkeypatch.setattr( "virsh_panel.fabric.FabricHostAdapter", AsyncCoordinatorAdapter) asyncio.run(coordinator.reconcile_all()) final = coordinator.get_status(current.fabric_id) assert final["state"] == "active" assert final["next_stage_allowed"] is True assert all(component.manager.async_dispatch_calls == 0 for component in components.values()) assert all(component.manager.async_status_calls == 0 for component in components.values()) assert all(component.manager.async_observe_calls == 1 for component in components.values()) def _reconcile_observation(current, host): public_key = KEY1 if host.host_id == "host-a" else KEY2 return { "state": "active", "config_ok": True, "transport_verified": True, "public_key": public_key, "observed": { "interface": FABRIC_INTERFACE, "config_revision": current.revision, "config_digest": current.digest, }, } class ReconcileRecoveryAdapter: """Stateful host stub for fixed-plan reconcile recovery tests.""" def __init__(self, manager, _profile=None): self.manager = manager def observe(self, current, host, _keys): self.manager.reconcile_observe_calls += 1 if self.manager.reconcile_observe_error is not None: error = self.manager.reconcile_observe_error if self.manager.reconcile_observe_error_once: self.manager.reconcile_observe_error = None raise error if self.manager.reconcile_configured: result = _reconcile_observation(current, host) result["public_key"] = self.manager.public_key result["transport_verified"] = ( self.manager.reconcile_transport_verified) return result return { "state": "degraded", "config_ok": False, "transport_verified": False, "public_key": self.manager.public_key, "observed": { "interface": None, "config_revision": None, "config_digest": None, "emergency_state": "owned", }, } def identity(self, *, create_key=True): self.manager.reconcile_identity_calls += 1 self.manager.reconcile_identity_create_flags.append(create_key) entered = self.manager.reconcile_identity_entered release = self.manager.reconcile_identity_release if entered is not None: entered.set() if release is not None: assert release.wait(timeout=2) if self.manager.reconcile_identity_error is not None: raise self.manager.reconcile_identity_error return self.manager.public_key def apply(self, *_args): self.manager.reconcile_apply_calls += 1 entered = self.manager.reconcile_apply_entered release = self.manager.reconcile_apply_release if entered is not None: entered.set() if release is not None: assert release.wait(timeout=2) if self.manager.reconcile_apply_error is not None: raise self.manager.reconcile_apply_error self.manager.reconcile_configured = True def _recovery_components(current, *, configured=False): components = _fabric_components(current) for component in components.values(): manager = component.manager manager.reconcile_configured = configured manager.reconcile_transport_verified = True manager.reconcile_observe_calls = 0 manager.reconcile_observe_error = None manager.reconcile_observe_error_once = False manager.reconcile_identity_calls = 0 manager.reconcile_identity_create_flags = [] manager.reconcile_identity_error = None manager.reconcile_identity_entered = None manager.reconcile_identity_release = None manager.reconcile_apply_calls = 0 manager.reconcile_apply_error = None manager.reconcile_apply_entered = None manager.reconcile_apply_release = None return components def test_successful_apply_clears_stale_failure_phase(): current = plan() store = MemoryStore() record = _active_fabric_record(current) for node in record["nodes"]: node["last_error_code"] = "fabric_remote_command_failed" node["last_error_phase"] = "observe" store.set("fabric_desired", current.fabric_id, json.dumps(record)) coordinator = FabricCoordinator( store, Registry(_fabric_components(current))) result = asyncio.run(coordinator.apply(plan_payload(current))) assert result["state"] == "active" assert all("last_error_phase" not in node for node in result["nodes"]) durable = json.loads(store.get("fabric_desired", current.fabric_id)) assert all("last_error_phase" not in node for node in durable["nodes"]) for host in current.nodes: persisted = json.loads(store.get( "fabric_node", f"{current.fabric_id}.{host.host_id}")) assert "last_error_phase" not in persisted assert persisted["last_error_code"] is None def test_reconcile_recovers_when_incomplete_host_observe_initially_fails( monkeypatch): current = plan() store = MemoryStore() store.set("fabric_desired", current.fabric_id, json.dumps( _active_fabric_record(current))) components = _recovery_components(current) first = components["host-a"].manager first.reconcile_observe_error = FabricError( "incomplete", 502, "fabric_remote_command_failed") first.reconcile_observe_error_once = True coordinator = FabricCoordinator(store, Registry(components)) monkeypatch.setattr( "virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter) result = asyncio.run(coordinator.reconcile(current.fabric_id)) assert result["state"] == "active" assert result["next_stage_allowed"] is True assert all("last_error_phase" not in node for node in result["nodes"]) assert all(component.manager.reconcile_identity_calls == 1 for component in components.values()) assert all(component.manager.reconcile_apply_calls == 1 for component in components.values()) assert first.reconcile_observe_calls == 2 def test_reconcile_persists_post_apply_observe_failure_phase(monkeypatch): current = plan() store = MemoryStore() store.set("fabric_desired", current.fabric_id, json.dumps( _active_fabric_record(current))) components = _recovery_components(current) failed = components["host-a"].manager class PostApplyObserveFailureAdapter(ReconcileRecoveryAdapter): def observe(self, plan_value, host, keys): if self.manager is failed and self.manager.reconcile_configured: self.manager.reconcile_observe_calls += 1 raise FabricError( "incomplete", 502, "fabric_remote_command_failed") return super().observe(plan_value, host, keys) coordinator = FabricCoordinator(store, Registry(components)) monkeypatch.setattr( "virsh_panel.fabric.FabricHostAdapter", PostApplyObserveFailureAdapter) result = asyncio.run(coordinator.reconcile(current.fabric_id)) nodes = {node["host_id"]: node for node in result["nodes"]} assert result["state"] == "uncertain" assert nodes["host-a"]["last_error_code"] == ( "fabric_remote_command_failed") assert nodes["host-a"]["last_error_phase"] == "observe" persisted = json.loads(store.get( "fabric_node", f"{current.fabric_id}.host-a")) assert persisted["last_error_phase"] == "observe" assert failed.reconcile_observe_calls == 2 assert failed.reconcile_apply_calls == 1 def test_reconcile_identity_gate_keeps_other_host_observe_phase(monkeypatch): current = plan() store = MemoryStore() store.set("fabric_desired", current.fabric_id, json.dumps( _active_fabric_record(current))) components = _recovery_components(current) observed_failed = components["host-a"].manager identity_failed = components["host-b"].manager observed_failed.reconcile_observe_error = FabricError( "incomplete", 502, "fabric_remote_command_failed") identity_failed.reconcile_identity_error = FabricError( "offline", 502, "fabric_remote_command_failed") coordinator = FabricCoordinator(store, Registry(components)) monkeypatch.setattr( "virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter) result = asyncio.run(coordinator.reconcile(current.fabric_id)) nodes = {node["host_id"]: node for node in result["nodes"]} assert nodes["host-a"]["last_error_phase"] == "observe" assert nodes["host-b"]["last_error_phase"] == "identity" assert all(component.manager.reconcile_apply_calls == 0 for component in components.values()) def test_successful_reconcile_observation_clears_stale_failure_phase( monkeypatch): current = plan() store = MemoryStore() record = _active_fabric_record(current) for node in record["nodes"]: node["last_error_code"] = "fabric_remote_command_failed" node["last_error_phase"] = "apply" store.set("fabric_desired", current.fabric_id, json.dumps(record)) components = _recovery_components(current, configured=True) coordinator = FabricCoordinator(store, Registry(components)) monkeypatch.setattr( "virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter) result = asyncio.run(coordinator.reconcile(current.fabric_id)) assert result["state"] == "active" assert all(node["last_error_code"] is None for node in result["nodes"]) assert all("last_error_phase" not in node for node in result["nodes"]) durable = json.loads(store.get("fabric_desired", current.fabric_id)) assert all("last_error_phase" not in node for node in durable["nodes"]) for host in current.nodes: persisted = json.loads(store.get( "fabric_node", f"{current.fabric_id}.{host.host_id}")) assert "last_error_phase" not in persisted assert persisted["last_error_code"] is None def test_reconcile_recovers_key_and_emergency_only_then_is_idempotent( monkeypatch): current = plan() store = MemoryStore() store.set("fabric_desired", current.fabric_id, json.dumps( _active_fabric_record(current))) components = _recovery_components(current) coordinator = FabricCoordinator(store, Registry(components)) monkeypatch.setattr( "virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter) first = asyncio.run(coordinator.reconcile(current.fabric_id)) second = asyncio.run(coordinator.reconcile(current.fabric_id)) assert first["state"] == "active" assert first["next_stage_allowed"] is True assert second["state"] == "active" assert second["next_stage_allowed"] is True for component in components.values(): manager = component.manager assert manager.reconcile_apply_calls == 1 assert manager.reconcile_identity_calls == 1 assert manager.reconcile_identity_create_flags == [False] # initial degraded + post-apply exact + second reconcile exact assert manager.reconcile_observe_calls == 3 def test_reconcile_does_not_reapply_exact_config_for_stale_transport( monkeypatch): current = plan() store = MemoryStore() store.set("fabric_desired", current.fabric_id, json.dumps( _active_fabric_record(current))) components = _recovery_components(current, configured=True) for component in components.values(): component.manager.reconcile_transport_verified = False coordinator = FabricCoordinator(store, Registry(components)) monkeypatch.setattr( "virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter) result = asyncio.run(coordinator.reconcile(current.fabric_id)) assert result["state"] == "degraded" assert result["next_stage_allowed"] is False for component in components.values(): manager = component.manager assert manager.reconcile_apply_calls == 0 assert manager.reconcile_identity_calls == 0 assert manager.reconcile_observe_calls == 1 def test_reconcile_does_not_reapply_exact_config_for_failed_ping(): current = plan() store = MemoryStore() store.set("fabric_desired", current.fabric_id, json.dumps( _active_fabric_record(current))) components = _fabric_components(current) for component in components.values(): component.manager.ping_ok = False coordinator = FabricCoordinator(store, Registry(components)) result = asyncio.run(coordinator.reconcile(current.fabric_id)) assert result["state"] == "degraded" assert result["next_stage_allowed"] is False assert all(node["state"] == "active" for node in result["nodes"]) assert all(node["transport_verified"] is False for node in result["nodes"]) # One script is the initial observe. An identity probe or apply would add # another script and prove reconcile mistook liveness for config drift. assert all(len(component.manager.scripts) == 1 for component in components.values()) def test_reconcile_repairs_live_public_key_drift(): current = plan() store = MemoryStore() store.set("fabric_desired", current.fabric_id, json.dumps( _active_fabric_record(current))) components = _fabric_components(current) drifted = components["host-a"].manager drifted.live_public_key = KEY3 coordinator = FabricCoordinator(store, Registry(components)) result = asyncio.run(coordinator.reconcile(current.fabric_id)) assert result["state"] == "active" assert result["next_stage_allowed"] is True assert drifted.live_public_key == KEY1 # observe -> fixed-key identity -> apply -> exact post-observe assert len(drifted.scripts) == 4 # The already exact peer receives only its initial observation. assert len(components["host-b"].manager.scripts) == 1 @pytest.mark.parametrize("identity_failure", ["drift", "unavailable"]) def test_reconcile_identity_failure_blocks_all_host_writes( monkeypatch, identity_failure): current = plan() store = MemoryStore() store.set("fabric_desired", current.fabric_id, json.dumps( _active_fabric_record(current))) components = _recovery_components(current) failed = components["host-b"].manager expected_error = "fabric_identity_conflict" if identity_failure == "drift": failed.public_key = KEY3 else: expected_error = "fabric_remote_command_failed" failed.reconcile_identity_error = FabricError( "offline", 502, expected_error) coordinator = FabricCoordinator(store, Registry(components)) monkeypatch.setattr( "virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter) result = asyncio.run(coordinator.reconcile(current.fabric_id)) nodes = {node["host_id"]: node for node in result["nodes"]} assert result["state"] == "uncertain" assert result["next_stage_allowed"] is False assert nodes["host-b"]["last_error_code"] == expected_error assert nodes["host-b"]["last_error_phase"] == "identity" persisted = json.loads(store.get( "fabric_node", f"{current.fabric_id}.host-b")) assert persisted["last_error_phase"] == "identity" for component in components.values(): manager = component.manager assert manager.reconcile_identity_calls == 1 assert manager.reconcile_identity_create_flags == [False] assert manager.reconcile_apply_calls == 0 def test_reconcile_apply_failure_on_one_node_does_not_stop_other_node( monkeypatch): current = plan() store = MemoryStore() store.set("fabric_desired", current.fabric_id, json.dumps( _active_fabric_record(current))) components = _recovery_components(current) components["host-a"].manager.reconcile_apply_error = FabricError( "offline", 502, "fabric_remote_command_failed") coordinator = FabricCoordinator(store, Registry(components)) monkeypatch.setattr( "virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter) result = asyncio.run(coordinator.reconcile(current.fabric_id)) nodes = {node["host_id"]: node for node in result["nodes"]} assert result["state"] == "uncertain" assert nodes["host-a"]["last_error_code"] == ( "fabric_remote_command_failed") assert nodes["host-a"]["last_error_phase"] == "apply" assert nodes["host-b"]["state"] == "active" assert nodes["host-b"]["observed_plan_digest"] == current.digest assert all(component.manager.reconcile_identity_calls == 1 for component in components.values()) assert all(component.manager.reconcile_apply_calls == 1 for component in components.values()) def test_cancelled_reconcile_drains_admitted_host_apply_before_unlock( monkeypatch): current = plan() store = MemoryStore() store.set("fabric_desired", current.fabric_id, json.dumps( _active_fabric_record(current))) components = _recovery_components(current) entered = threading.Event() release = threading.Event() first = components["host-a"].manager first.reconcile_apply_entered = entered first.reconcile_apply_release = release coordinator = FabricCoordinator(store, Registry(components)) monkeypatch.setattr( "virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter) async def exercise(): reconcile_task = asyncio.create_task( coordinator.reconcile(current.fabric_id)) for _attempt in range(100): if entered.is_set(): break await asyncio.sleep(0.01) assert entered.is_set() lock_entered = asyncio.Event() async def wait_for_fabric_lock(): async with coordinator._locks[current.fabric_id]: lock_entered.set() waiter = asyncio.create_task(wait_for_fabric_lock()) reconcile_task.cancel() await asyncio.sleep(0) assert reconcile_task.done() is False assert lock_entered.is_set() is False release.set() with pytest.raises(asyncio.CancelledError): await reconcile_task await asyncio.wait_for(waiter, timeout=1) assert lock_entered.is_set() is True try: asyncio.run(exercise()) finally: release.set() assert first.reconcile_apply_calls == 1 assert first.reconcile_configured is True assert components["host-b"].manager.reconcile_apply_calls == 0 def test_cancelled_reconcile_drains_runtime_identity_lock_before_unlock( monkeypatch): current = plan() store = MemoryStore() store.set("fabric_desired", current.fabric_id, json.dumps( _active_fabric_record(current))) components = _recovery_components(current) entered = threading.Event() release = threading.Event() first = components["host-a"].manager first.reconcile_identity_entered = entered first.reconcile_identity_release = release coordinator = FabricCoordinator(store, Registry(components)) monkeypatch.setattr( "virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter) async def exercise(): reconcile_task = asyncio.create_task( coordinator.reconcile(current.fabric_id)) for _attempt in range(100): if entered.is_set(): break await asyncio.sleep(0.01) assert entered.is_set() lock_entered = asyncio.Event() async def wait_for_fabric_lock(): async with coordinator._locks[current.fabric_id]: lock_entered.set() waiter = asyncio.create_task(wait_for_fabric_lock()) reconcile_task.cancel() await asyncio.sleep(0) assert reconcile_task.done() is False assert lock_entered.is_set() is False release.set() with pytest.raises(asyncio.CancelledError): await reconcile_task await asyncio.wait_for(waiter, timeout=1) assert lock_entered.is_set() is True try: asyncio.run(exercise()) finally: release.set() assert first.reconcile_identity_calls == 1 assert first.reconcile_apply_calls == 0 assert components["host-b"].manager.reconcile_identity_calls == 0 assert components["host-b"].manager.reconcile_apply_calls == 0 def _vpc_payload_for_fabric(current, *, vpc_id="vpc-race"): return { "schema_version": 1, "vpc_id": vpc_id, "region": current.region, "fabric_id": current.fabric_id, "fabric_revision": current.revision, "cidr": "10.20.0.0/24", "desired_revision": 1, "operation_id": f"{vpc_id}-operation-0001", "subnets": [ {"subnet_id": "subnet-a", "cidr": "10.20.0.0/25"}, ], "hosts": [{ "host_id": "host-a", "members": [{ "member_id": "member-a", "instance_uuid": "11111111-1111-4111-8111-111111111111", "private_ip_cidr": "10.20.0.10/32", "subnet_id": "subnet-a", }], }], "routes": [], "acls": [], } def test_reconcile_waits_for_admitted_vpc_apply_before_invalidating( monkeypatch): from virsh_panel.vpc import VpcCoordinator current = plan() store = MemoryStore() store.set("fabric_desired", current.fabric_id, json.dumps( _active_fabric_record(current))) coordinator = FabricCoordinator( store, Registry(_fabric_components(current))) vpc = VpcCoordinator( store, coordinator, catalog_lock=coordinator.catalog_mutation_lock, ) vpc_entered = asyncio.Event() vpc_release = asyncio.Event() observe_entered = threading.Event() observe_release = threading.Event() async def blocked_vpc_apply(_plan): vpc_entered.set() await vpc_release.wait() return {"vpc_id": "vpc-race", "state": "active"} class BlockingObserveAdapter: def __init__(self, _manager, _profile=None): pass def observe(self, _plan, host, _keys): observe_entered.set() assert observe_release.wait(timeout=2) return _reconcile_observation(current, host) vpc._apply_locked = blocked_vpc_apply monkeypatch.setattr( "virsh_panel.fabric.FabricHostAdapter", BlockingObserveAdapter) vpc_payload = _vpc_payload_for_fabric(current) async def exercise(): vpc_task = asyncio.create_task(vpc.apply(vpc_payload)) await vpc_entered.wait() reconcile_task = asyncio.create_task( coordinator.reconcile(current.fabric_id)) await asyncio.sleep(0) await asyncio.sleep(0) before = json.loads(store.get( "fabric_desired", current.fabric_id)) assert before["state"] == "active" assert before["next_stage_allowed"] is True vpc_release.set() await vpc_task for _attempt in range(100): if observe_entered.is_set(): break await asyncio.sleep(0.01) assert observe_entered.is_set() invalidated = json.loads(store.get( "fabric_desired", current.fabric_id)) assert invalidated["state"] == "uncertain" assert invalidated["next_stage_allowed"] is False observe_release.set() return await asyncio.wait_for(reconcile_task, timeout=1) try: result = asyncio.run(exercise()) finally: observe_release.set() assert result["state"] == "active" assert result["next_stage_allowed"] is True def test_reconcile_first_durable_invalidation_blocks_vpc_host_mutation( monkeypatch): from virsh_panel.vpc import VpcCoordinator current = plan() class BlockingInvalidationStore(MemoryStore): def __init__(self): super().__init__() self.block = False self.entered = threading.Event() self.release = threading.Event() def set(self, kind, key, value): parsed = json.loads(value) if (self.block and kind == "fabric_desired" and parsed.get("state") == "uncertain"): self.entered.set() assert self.release.wait(timeout=2) super().set(kind, key, value) observe_entered = threading.Event() observe_release = threading.Event() class BlockingObserveAdapter: def __init__(self, _manager, _profile=None): pass def observe(self, _plan, host, _keys): observe_entered.set() assert observe_release.wait(timeout=2) return _reconcile_observation(current, host) store = BlockingInvalidationStore() store.set("fabric_desired", current.fabric_id, json.dumps( _active_fabric_record(current))) store.block = True coordinator = FabricCoordinator( store, Registry(_fabric_components(current))) vpc = VpcCoordinator( store, coordinator, registry=SimpleNamespace(components={}), catalog_lock=coordinator.catalog_mutation_lock, ) host_mutation_called = False def forbidden_host_validation(_plan): nonlocal host_mutation_called host_mutation_called = True raise AssertionError("inactive Fabric reached host mutation") vpc._validate_dataplane_resources = forbidden_host_validation monkeypatch.setattr( "virsh_panel.fabric.FabricHostAdapter", BlockingObserveAdapter) async def exercise(): reconcile_task = asyncio.create_task( coordinator.reconcile(current.fabric_id)) for _attempt in range(100): if store.entered.is_set(): break await asyncio.sleep(0.01) assert store.entered.is_set() vpc_task = asyncio.create_task(vpc.apply( _vpc_payload_for_fabric(current, vpc_id="vpc-after-reconcile"))) await asyncio.sleep(0) assert vpc_task.done() is False assert host_mutation_called is False still_active = json.loads(store.get( "fabric_desired", current.fabric_id)) assert still_active["state"] == "active" store.release.set() for _attempt in range(100): if observe_entered.is_set(): break await asyncio.sleep(0.01) assert observe_entered.is_set() result = await asyncio.wait_for(vpc_task, timeout=1) assert result["state"] == "degraded" assert result["next_stage_allowed"] is False assert result["fabric_gate"]["error_code"] == ( "vpc_fabric_not_active") assert host_mutation_called is False durable_vpc = json.loads(store.get( "vpc_desired", "vpc-after-reconcile")) assert durable_vpc["state"] == "degraded" observe_release.set() await asyncio.wait_for(reconcile_task, timeout=1) try: asyncio.run(exercise()) finally: store.release.set() observe_release.set() def test_reconcile_releases_catalog_during_remote_observe_and_relocks_publish( monkeypatch): current = plan() store = MemoryStore() store.set("fabric_desired", current.fabric_id, json.dumps( _active_fabric_record(current))) coordinator = FabricCoordinator( store, Registry(_fabric_components(current))) observe_entered = threading.Event() observe_release = threading.Event() all_observed = threading.Event() count_lock = threading.Lock() observation_count = 0 class BlockingObserveAdapter: def __init__(self, _manager, _profile=None): pass def observe(self, _plan, host, _keys): nonlocal observation_count observe_entered.set() assert observe_release.wait(timeout=2) with count_lock: observation_count += 1 if observation_count == len(current.nodes): all_observed.set() return _reconcile_observation(current, host) monkeypatch.setattr( "virsh_panel.fabric.FabricHostAdapter", BlockingObserveAdapter) async def exercise(): reconcile_task = asyncio.create_task( coordinator.reconcile(current.fabric_id)) for _attempt in range(100): if observe_entered.is_set(): break await asyncio.sleep(0.01) assert observe_entered.is_set() shared_lock = coordinator.catalog_mutation_lock() await asyncio.wait_for(shared_lock.acquire(), timeout=1) try: observe_release.set() for _attempt in range(100): if all_observed.is_set(): break await asyncio.sleep(0.01) assert all_observed.is_set() assert reconcile_task.done() is False durable = json.loads(store.get( "fabric_desired", current.fabric_id)) assert durable["state"] == "uncertain" assert durable["next_stage_allowed"] is False finally: shared_lock.release() return await asyncio.wait_for(reconcile_task, timeout=1) try: result = asyncio.run(exercise()) finally: observe_release.set() assert result["state"] == "active" assert result["next_stage_allowed"] is True @pytest.mark.parametrize("blocked_state", ["uncertain", "active"]) def test_cancelled_reconcile_drains_root_save_before_catalog_unlock( monkeypatch, blocked_state): current = plan() class BlockingRootStore(MemoryStore): def __init__(self): super().__init__() self.block = False self.entered = threading.Event() self.release = threading.Event() def set(self, kind, key, value): parsed = json.loads(value) if (self.block and kind == "fabric_desired" and parsed.get("state") == blocked_state): self.entered.set() assert self.release.wait(timeout=2) super().set(kind, key, value) class ImmediateObserveAdapter: def __init__(self, _manager, _profile=None): pass def observe(self, _plan, host, _keys): return _reconcile_observation(current, host) store = BlockingRootStore() store.set("fabric_desired", current.fabric_id, json.dumps( _active_fabric_record(current))) store.block = True coordinator = FabricCoordinator( store, Registry(_fabric_components(current))) monkeypatch.setattr( "virsh_panel.fabric.FabricHostAdapter", ImmediateObserveAdapter) async def exercise(): reconcile_task = asyncio.create_task( coordinator.reconcile(current.fabric_id)) for _attempt in range(100): if store.entered.is_set(): break await asyncio.sleep(0.01) assert store.entered.is_set() catalog_entered = asyncio.Event() async def competing_catalog_apply(): async with coordinator.catalog_mutation_lock(): catalog_entered.set() competitor = asyncio.create_task(competing_catalog_apply()) await asyncio.sleep(0) reconcile_task.cancel() await asyncio.sleep(0) assert reconcile_task.done() is False assert catalog_entered.is_set() is False store.release.set() with pytest.raises(asyncio.CancelledError): await reconcile_task await asyncio.wait_for(competitor, timeout=1) assert catalog_entered.is_set() is True try: asyncio.run(exercise()) finally: store.release.set() durable = json.loads(store.get("fabric_desired", current.fabric_id)) assert durable["state"] == blocked_state assert durable["next_stage_allowed"] is (blocked_state == "active") def test_new_fabric_cannot_reuse_a_host_or_overlay_catalog_entry(): store = MemoryStore() store.set("fabric_desired", "existing", json.dumps({ "fabric_id": "existing", "overlay_cidr": "100.64.2.0/24", "nodes": [{"host_id": "host-a"}, {"host_id": "host-c"}], })) coordinator = FabricCoordinator(store, Registry({})) value = { "fabric_id": "new-fabric", "region": "hk", "overlay_cidr": "100.64.2.0/24", "revision": 1, "nodes": [ {"host_id": "host-a", "overlay_ip": "100.64.2.10", "endpoint": "203.0.113.1:51820"}, {"host_id": "host-b", "overlay_ip": "100.64.2.11", "endpoint": "203.0.113.2:51820"}, ], } with pytest.raises(FabricError) as error: asyncio.run(coordinator.apply(value)) assert error.value.error_code == "fabric_host_interface_conflict" def test_existing_fabric_region_is_immutable_without_migration(): store = MemoryStore() current = plan() store.set("fabric_desired", current.fabric_id, json.dumps({ "fabric_id": current.fabric_id, "region": current.region, "revision": current.revision, "plan_digest": current.digest, "nodes": [{"host_id": node.host_id} for node in current.nodes], })) coordinator = FabricCoordinator(store, Registry({})) value = { "fabric_id": current.fabric_id, "region": "sg", "overlay_cidr": current.overlay_cidr, "revision": current.revision + 1, "nodes": [ {"host_id": node.host_id, "overlay_ip": node.overlay_ip, "endpoint": node.endpoint} for node in current.nodes ], } with pytest.raises(FabricError) as error: asyncio.run(coordinator.apply(value)) assert error.value.error_code == "fabric_region_conflict" @pytest.mark.parametrize(("tenant_fabric", "tenant_region", "expected"), [ ("hk-fabric", "sg", "fabric_region_conflict"), ("other-fabric", "sg", "fabric_endpoint_vpc_conflict"), ]) def test_fabric_checks_region_identity_and_all_tenant_cidrs( tenant_fabric, tenant_region, expected): from virsh_panel.vpc import parse_vpc_plan store = MemoryStore() tenant = parse_vpc_plan({ "schema_version": 1, "vpc_id": "vpc-existing", "region": tenant_region, "fabric_id": tenant_fabric, "fabric_revision": 1, "cidr": "192.168.50.0/24", "desired_revision": 1, "operation_id": "vpc-op-region-0001", "subnets": [{"subnet_id": "subnet-a", "cidr": "192.168.50.0/25"}], "hosts": [{"host_id": "host-a", "members": [{ "member_id": "member-a", "instance_uuid": "11111111-1111-4111-8111-111111111111", "private_ip_cidr": "192.168.50.10/32", "subnet_id": "subnet-a", }]}], "routes": [], "acls": [], }) record = tenant.canonical() record.update({ "operation_id": tenant.operation_id, "desired_digest": tenant.desired_digest, }) store.set("vpc_desired", tenant.vpc_id, json.dumps(record)) coordinator = FabricCoordinator(store, Registry({})) candidate = parse_fabric_plan({ "fabric_id": "hk-fabric", "region": "hk", "overlay_cidr": "100.64.1.0/24", "revision": 1, "nodes": [ {"host_id": "host-a", "overlay_ip": "100.64.1.1", "endpoint": "192.168.50.20:51820"}, {"host_id": "host-b", "overlay_ip": "100.64.1.2", "endpoint": "203.0.113.2:51820"}, ], }) with pytest.raises(FabricError) as error: coordinator._validate_resource_exclusivity(candidate) assert error.value.error_code == expected def test_new_fabric_rejects_endpoint_inside_existing_tenant_cidr(): from virsh_panel.vpc import parse_vpc_plan store = MemoryStore() tenant = parse_vpc_plan({ "schema_version": 1, "vpc_id": "vpc-hk-1", "region": "hk", "fabric_id": "hk-fabric", "fabric_revision": 1, "cidr": "10.20.0.0/24", "desired_revision": 1, "operation_id": "vpc-op-00000001", "subnets": [{"subnet_id": "subnet-a", "cidr": "10.20.0.0/25"}], "hosts": [{"host_id": "host-a", "members": [{ "member_id": "member-a", "instance_uuid": "11111111-1111-4111-8111-111111111111", "private_ip_cidr": "10.20.0.10/32", "subnet_id": "subnet-a", }]}], "routes": [], "acls": [], }) tenant_record = tenant.canonical() tenant_record.update({ "operation_id": tenant.operation_id, "desired_digest": tenant.desired_digest, }) store.set("vpc_desired", tenant.vpc_id, json.dumps(tenant_record)) coordinator = FabricCoordinator(store, Registry({})) value = { "fabric_id": "hk-fabric", "region": "hk", "overlay_cidr": "100.64.1.0/24", "revision": 1, "nodes": [ {"host_id": "host-a", "overlay_ip": "100.64.1.1", "endpoint": "10.20.0.50:51820"}, {"host_id": "host-b", "overlay_ip": "100.64.1.2", "endpoint": "192.168.50.2:51820"}, ], } with pytest.raises(FabricError) as error: asyncio.run(coordinator.apply(value)) assert error.value.error_code == "fabric_endpoint_vpc_conflict" current = parse_fabric_plan(value) store.set("fabric_desired", current.fabric_id, json.dumps({ "schema_version": 1, "fabric_id": current.fabric_id, "region": current.region, "overlay_cidr": current.overlay_cidr, "revision": current.revision, "listen_port": current.listen_port, "plan_digest": current.digest, "state": "active", "next_stage_allowed": True, "updated_at": int(time.time()), "public_keys": {"host-a": KEY1, "host-b": KEY2}, "nodes": [ {"host_id": node.host_id, "overlay_ip": node.overlay_ip, "endpoint": node.endpoint, "listen_port": node.listen_port, "state": "active", "transport_verified": True} for node in current.nodes ], })) with pytest.raises(FabricError) as reconcile_error: asyncio.run(coordinator.reconcile(current.fabric_id)) assert reconcile_error.value.error_code == "fabric_endpoint_vpc_conflict" invalidated = json.loads(store.get("fabric_desired", current.fabric_id)) assert invalidated["state"] == "uncertain" assert invalidated["next_stage_allowed"] is False class TenantReadbackManager(FakeManager): """Synthetic readback with panel-owned VPC tenant state.""" def __init__(self, *args, tenant_ip="10.10.0.2/32", **kwargs): super().__init__(*args, **kwargs) self.tenant_ip = tenant_ip def _exec(self, argv, *, timeout, input_text=None): result = super()._exec(argv, timeout=timeout, input_text=input_text) if input_text.startswith("#!/bin/sh\nset -eu"): return result peer = KEY2 if self.public_key == KEY1 else KEY1 target = "100.64.1.2" if self.public_key == KEY1 else "100.64.1.1" endpoint = "203.0.113.2:51820" if self.public_key == KEY1 else "203.0.113.1:51820" marker = f"__VC_FABRIC_PEER__={peer}|{endpoint}|{target}/32" output = result.stdout.replace(marker, f"{marker},{self.tenant_ip}") output += ( "__VC_FABRIC_VPC_STATE__=owned\n" f"__VC_FABRIC_VPC_TENANT__={peer}|{self.tenant_ip}\n" "__VC_FABRIC_VPC_NFT_STATE__=owned\n" f"__VC_FABRIC_VPC_NFT_IP__={self.tenant_ip}\n" ) return result.__class__(result.args, result.returncode, output, result.stderr) def test_fabric_merges_and_reads_panel_owned_vpc_allowed_ips(): current = plan() observed = FabricHostAdapter(TenantReadbackManager(KEY1)).observe( current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2} ) assert observed["config_ok"] is True assert observed["observed"]["peer_allowed_exact"] is True assert f"{KEY2}|10.10.0.2/32" in observed["observed"]["vpc_tenants"] assert all("10001" not in route for route in observed["observed"]["routes"]) def test_fabric_rejects_unbacked_vpc_allowed_ip(): current = plan() manager = FakeManager(KEY1) original_exec = manager._exec def unbacked(argv, *, timeout, input_text=None): result = original_exec(argv, timeout=timeout, input_text=input_text) if input_text.startswith("#!/bin/sh\nset -eu"): return result peer = KEY2 marker = f"__VC_FABRIC_PEER__={peer}|203.0.113.2:51820|100.64.1.2/32" output = result.stdout.replace(marker, f"{marker},10.10.0.2/32") return result.__class__(result.args, result.returncode, output, result.stderr) manager._exec = unbacked observed = FabricHostAdapter(manager).observe( current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2} ) assert observed["config_ok"] is False assert observed["observed"]["peer_allowed_exact"] is False def test_fabric_rejects_non_rfc1918_vpc_tenant_prefix(): current = plan() observed = FabricHostAdapter( TenantReadbackManager(KEY1, tenant_ip="169.254.1.2/32") ).observe(current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}) assert observed["config_ok"] is False assert observed["observed"]["peer_allowed_exact"] is False def test_fabric_rejects_stale_vpc_nft_tenant_set(): current = plan() manager = TenantReadbackManager(KEY1) original_exec = manager._exec def stale_nft(argv, *, timeout, input_text=None): result = original_exec(argv, timeout=timeout, input_text=input_text) if input_text.startswith("#!/bin/sh\nset -eu"): return result output = result.stdout.replace( "__VC_FABRIC_VPC_NFT_IP__=10.10.0.2/32\n", "") return result.__class__(result.args, result.returncode, output, result.stderr) manager._exec = stale_nft observed = FabricHostAdapter(manager).observe( current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2} ) assert observed["config_ok"] is False assert observed["observed"]["vpc_nft_exact"] is False def test_fabric_does_not_hide_foreign_custom_route(): current = plan() manager = FakeManager(KEY1) original_exec = manager._exec def foreign_route(argv, *, timeout, input_text=None): result = original_exec(argv, timeout=timeout, input_text=input_text) if input_text.startswith("#!/bin/sh\nset -eu"): return result output = result.stdout + ( f"__VC_FABRIC_ROUTE__=10.10.0.2/32|10001|{FABRIC_INTERFACE}\n" ) return result.__class__(result.args, result.returncode, output, result.stderr) manager._exec = foreign_route observed = FabricHostAdapter(manager).observe( current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2} ) assert observed["config_ok"] is False assert "10.10.0.2/32|10001|wg-fabric0" in observed["observed"]["routes"]