You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/engine/tests/test_fabric.py

8794 lines
320 KiB
Python

"""Fabric plan, helper, and coordinator tests."""
from __future__ import annotations
import asyncio
import base64
import hashlib
import json
import re
import shlex
import stat
import subprocess
import sys
import threading
import time
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from types import SimpleNamespace
import pytest
from virsh_panel.fabric import (
FABRIC_OPERATION_CHECKPOINTS,
FABRIC_OPERATION_ROOT,
FABRIC_OPERATION_REASONS,
FABRIC_OPERATION_START_TIMEOUT_SECONDS,
FABRIC_OPERATION_STATUSES,
FABRIC_OPERATION_UNIT_PREFIX,
FABRIC_PROBE_CONCURRENCY,
FABRIC_INTERFACE,
FABRIC_OWNER_MARKER,
FABRIC_REMOTE_APPLY_TIMEOUT_SECONDS,
FABRIC_REMOTE_READ_TIMEOUT_SECONDS,
FABRIC_SYSCTL,
FIREWALL_MODE_NATIVE,
MAX_VPC_STATE_FILES,
MAX_FABRIC_OPERATION_DIRECTORIES,
FabricError,
FabricHostAdapter,
FabricCoordinator,
_BOOT_HELPER,
_FABRIC_NFT_VALIDATION_SH,
_FIREWALL_MODE_GUARD_SH,
_FABRIC_SYSCTL_VALIDATION_SH,
_NMCLI_FILENAME_RESOLVER_SH,
_REPAIR_UNIT,
_UNIT,
_apply_script,
_fabric_endpoint_rp_filter_evidence,
_fabric_operation_dispatch_script,
_fabric_operation_runner_script,
_fabric_operation_status_script,
_identity_script,
_new_operation_id,
_nm_profile_template,
_nft_rules_text,
_observe_script,
_parse_operation_status,
_profile_firewall_mode,
_sysctl_text,
parse_fabric_plan,
validate_wireguard_key,
)
from virsh_panel.remote_host import RemoteHostError
KEY1 = base64.b64encode(bytes([1]) * 32).decode()
KEY2 = base64.b64encode(bytes([2]) * 32).decode()
KEY3 = base64.b64encode(bytes([3]) * 32).decode()
def plan():
return parse_fabric_plan({
"fabric_id": "hk-fabric",
"region": "hk",
"overlay_cidr": "100.64.1.0/24",
"revision": 1,
"listen_port": 51820,
"nodes": [
{"host_id": "host-a", "overlay_ip": "100.64.1.1",
"endpoint": "203.0.113.1:51820"},
{"host_id": "host-b", "overlay_ip": "100.64.1.2",
"endpoint": "203.0.113.2:51820"},
],
})
def plan_payload(current=None):
current = current or plan()
return {
"fabric_id": current.fabric_id,
"region": current.region,
"overlay_cidr": current.overlay_cidr,
"revision": current.revision,
"listen_port": current.listen_port,
"nodes": [
{
"host_id": node.host_id,
"overlay_ip": node.overlay_ip,
"endpoint": node.endpoint,
"listen_port": node.listen_port,
}
for node in current.nodes
],
}
def test_plan_is_canonical_and_rejects_wide_or_tenant_routes():
first = plan()
reordered = parse_fabric_plan({
"nodes": [
{"host_id": "host-b", "overlay_ip": "100.64.1.2",
"endpoint": "203.0.113.2:51820"},
{"host_id": "host-a", "overlay_ip": "100.64.1.1",
"endpoint": "203.0.113.1:51820"},
],
"fabric_id": "hk-fabric", "region": "hk",
"overlay_cidr": "100.64.1.0/24", "revision": 1,
})
assert first.digest == reordered.digest
with pytest.raises(FabricError):
parse_fabric_plan({
"fabric_id": "bad", "region": "hk",
"overlay_cidr": "10.0.0.0/8", "revision": 1,
"nodes": [
{"host_id": "a", "overlay_ip": "10.0.0.1",
"endpoint": "203.0.113.1:51820"},
{"host_id": "b", "overlay_ip": "10.0.0.2",
"endpoint": "203.0.113.2:51820"},
],
})
with pytest.raises(FabricError):
parse_fabric_plan({
"fabric_id": "bad", "region": "hk",
"overlay_cidr": "100.64.1.0/24", "revision": 1,
"nodes": [
{"host_id": "a", "overlay_ip": "100.64.1.1",
"endpoint": "100.64.1.2:51820"},
{"host_id": "b", "overlay_ip": "100.64.1.2",
"endpoint": "203.0.113.2:51820"},
],
})
with pytest.raises(FabricError):
parse_fabric_plan({
"fabric_id": "bad", "region": "hk",
"overlay_cidr": "100.64.1.0/24", "revision": 1,
"nodes": [
{"host_id": "a", "overlay_ip": "100.64.1.1",
"endpoint": "203.0.113.1:51820"},
{"host_id": "b", "overlay_ip": "100.64.1.2",
"endpoint": "0.0.0.0:51820"},
],
})
with pytest.raises(FabricError, match="endpoint"):
parse_fabric_plan({
"fabric_id": "bad", "region": "hk",
"overlay_cidr": "100.64.1.0/24", "revision": 1,
"nodes": [
{"host_id": "a", "overlay_ip": "100.64.1.1",
"endpoint": "203.0.113.1:51820"},
{"host_id": "b", "overlay_ip": "100.64.1.2",
"endpoint": "203.0.113.1:51820"},
],
})
@pytest.mark.parametrize("address", [
"0.0.0.0",
"0.1.2.3",
"127.23.45.67",
"169.254.12.34",
"224.0.0.1",
"240.0.0.1",
"255.255.255.255",
])
def test_plan_rejects_unusable_endpoint_ipv4(address):
with pytest.raises(FabricError, match="endpoint 地址不可路由"):
parse_fabric_plan({
"fabric_id": "bad-endpoint", "region": "hk",
"overlay_cidr": "100.64.1.0/24", "revision": 1,
"nodes": [
{"host_id": "a", "overlay_ip": "100.64.1.1",
"endpoint": f"{address}:51820"},
{"host_id": "b", "overlay_ip": "100.64.1.2",
"endpoint": "203.0.113.2:51820"},
],
})
def test_wireguard_key_rejects_the_all_zero_reserved_key():
with pytest.raises(FabricError):
validate_wireguard_key(base64.b64encode(bytes(32)).decode())
def test_generated_helpers_are_posix_safe_and_do_not_embed_private_keys():
current = plan()
read_only_identity = _identity_script(create_key=False)
scripts = (
_BOOT_HELPER,
_identity_script(),
read_only_identity,
_observe_script(("100.64.1.2",), "100.64.1.1",
("203.0.113.2",), "networkmanager_nmcli",
"12345678-1234-5678-1234-567812345678",
expected_digest=current.digest),
_apply_script(current, current.nodes[0], {
"host-a": KEY1, "host-b": KEY2,
}),
)
for script in scripts:
result = subprocess.run(
["/bin/sh", "-n", "-c", script],
check=False, capture_output=True, text=True,
)
assert result.returncode == 0, result.stderr
assert "br0" not in script
assert "PRIVATE KEY" not in script
assert "genkey" not in read_only_identity
assert 'mktemp "$BASE/.wg-key.' not in read_only_identity
assert 'mv -f "$_tmp_key" "$KEY"' not in read_only_identity
assert "mkdir /run/virt-cloud" in read_only_identity
assert ': > "$LOCK"' in read_only_identity
assert 'exec 9>>"$LOCK"' in read_only_identity
assert '[ -d "$BASE" ]' in read_only_identity
observe = scripts[3]
assert "__VC_FABRIC_ADMIN_STATE__" in observe
assert "__VC_FABRIC_SYSCTL__ip_forward" in observe
assert "__VC_FABRIC_SYSCTL__rp_filter_default" in observe
assert "__VC_FABRIC_RPF__wg-fabric0" in observe
assert "__VC_FABRIC_SYSCTL__rp_filter_effective" in observe
assert "__VC_FABRIC_SYSCTL_PERSISTENCE__" in observe
assert "__VC_FABRIC_VPC_STATE__" in observe
assert "__VC_FABRIC_EMERGENCY__" in observe
assert "list set inet virt_cloud_wg_fabric vpc_tenant_ips" in observe
assert 'table!="main" && table!="local"' in observe
apply = scripts[-1]
assert KEY2 in apply and KEY1 in apply
assert 'KEY="$BASE/wg-fabric0.key"' in apply
assert "genkey" not in apply
identity_check = '[ "$($WG pubkey < "$KEY")" = "$EXPECTED_PUBLIC_KEY" ]'
assert identity_check in apply
assert apply.index(identity_check) < apply.index(
"vc_fabric_nm_quarantine || exit 71")
assert f"SYSCTL_FILE={FABRIC_SYSCTL}" in apply
assert _sysctl_text(current).rstrip() in apply
assert "wg-fabric0" in apply
assert "address_conflict" in apply
assert "existing_route" in apply
assert "route_overlaps_target" in apply
assert "supernet" not in apply # shell check is numeric and bounded
assert "virt-cloud-wg-fabric.nft" in apply
assert "include \"/etc/nftables.d/virt-cloud-wg-fabric.nft\"" in apply
assert "trap rollback EXIT HUP INT TERM" in apply
assert '"$WG" showconf' in apply
assert "route save table all" in apply
assert 'chmod 700 "$BASE" /usr/local/libexec' not in apply
assert "virt-cloud-wg-fabric-v1" in apply
assert "virt_cloud_wg_fabric" in apply
assert 'iifname "wg-fabric0"' in apply
assert 'oifname "wg-fabric0"' in apply
assert 'ip protocol icmp accept' in apply
assert "route get" in apply
assert "virt-cloud-wg-fabric-reconcile.timer" in apply
assert "Restart=on-failure" not in apply
assert f"fabric-plan-digest={current.digest}" in apply
assert "NetworkManager/system-connections" in apply
assert "nmcli" in apply
assert "flock" in apply
assert "vpc_state_load" in apply
assert "vpc_state_allowed_for_peer" in apply
assert "vpc_state_refresh_nft" in apply
assert "vpc_state_refresh_nft" in _BOOT_HELPER
assert "vc_fabric_sysctl_converge" in _BOOT_HELPER
assert "vc_fabric_rp_filter_set" in _BOOT_HELPER
assert "rp_filter_before" in apply
assert "vc_ip_rule_preflight" in _BOOT_HELPER
assert "vc_ip_rule_scope_check" in _BOOT_HELPER
assert "vc_ip_rule_ensure" in _BOOT_HELPER
assert "rule replace" not in _BOOT_HELPER
assert r"\${" not in _BOOT_HELPER
assert '"$BASE"/.vpc-dataplane-*.txn' in _BOOT_HELPER
assert "vpc_seen_rule_priorities" in _BOOT_HELPER
assert "vpc_seen_route_tables" in _BOOT_HELPER
assert 'vpc_state != "invalid"' in observe
assert 'vpc[prefix "|" device "|" table "|" via "|" metric]' in observe
assert "key|key.desired|wg.conf|address.save|route.save" in _BOOT_HELPER
assert "nft.table|nft.installed|nft.applied|nft.phase" in _BOOT_HELPER
assert 'journal_file_mode" = 600' in _BOOT_HELPER
ifup = _apply_script(current, current.nodes[0], {
"host-a": KEY1, "host-b": KEY2,
}, "ifupdown", "/usr/bin/ifup")
result = subprocess.run(
["/bin/sh", "-n", "-c", ifup],
check=False, capture_output=True, text=True,
)
assert result.returncode == 0, result.stderr
assert "/etc/network/interfaces.d/virt-cloud-wg-fabric0" in ifup
assert "ifup" in ifup and "ifdown" in ifup
assert "ExecStart=/bin/sh /usr/local/libexec/virt-cloud-wg-fabric --ifupdown-service" in ifup
# The Debian 13 contract must also work with classic ifupdown; its
# ifquery implementation does not provide ifupdown2's --check flag.
assert "ifquery\" --check" not in ifup
assert "VC_FABRIC_LOCK_HELD=1" in ifup
assert "VC_FABRIC_ALLOW_TRANSITION=1" in ifup
assert "br0" not in ifup
with pytest.raises(FabricError):
_apply_script(current, current.nodes[0], {
"host-a": KEY1, "host-b": KEY2,
}, "ifupdown", "/tmp/ifup")
@pytest.mark.parametrize("expected_digest", (
"", "a" * 63, "A" * 64, "g" * 64,
))
def test_observe_rejects_invalid_expected_digest(expected_digest):
with pytest.raises(FabricError, match="Fabric 计划摘要无效"):
_observe_script((), expected_digest=expected_digest)
@pytest.mark.parametrize(
("kind", "first_line", "mode", "expected"),
(
("helper", "#!/bin/sh", "700", "owned"),
("unit", "[Unit]", "644", "owned"),
("helper", "[Unit]", "700", "foreign"),
("unit", "#!/bin/sh", "644", "foreign"),
("unknown", "#!/bin/sh", "644", "foreign"),
),
)
def test_observe_artifact_state_accepts_only_kind_specific_header(
tmp_path, kind, first_line, mode, expected):
current = plan()
observe = _observe_script((), expected_digest=current.digest)
start = observe.index("artifact_state() {")
end = observe.index("\n}\nhelper_marker=", start) + 2
artifact_state = observe[start:end]
artifact = tmp_path / "artifact"
artifact.write_text(
f"{first_line}\n"
"# virt-cloud-wg-fabric-v1\n"
f"# fabric-plan-digest={current.digest}\n"
"# libvirt-startup-independent-v1\n"
"# vpc-local-neighbor-contract-v1\n"
"# vpc-state-route-scope-contract-v4\n",
encoding="utf-8",
)
artifact.chmod(int(mode, 8))
fake_stat = tmp_path / "stat"
fake_stat.write_text(
"#!/bin/sh\n"
"case \"$2\" in\n"
" '%u') printf '0\\n' ;;\n"
" '%a') printf '%s\\n' \"$VC_TEST_MODE\" ;;\n"
" *) exit 2 ;;\n"
"esac\n",
encoding="utf-8",
)
fake_stat.chmod(0o700)
script = "\n".join((
"set -eu",
f"STAT={shlex.quote(str(fake_stat))}",
f"UNIT={shlex.quote(str(artifact))}",
f"GATE_GENERATOR={shlex.quote(str(tmp_path / 'missing-generator'))}",
artifact_state,
"artifact_state "
f"{shlex.quote(str(artifact))} {shlex.quote(kind)} "
f"{shlex.quote(current.digest)}",
))
result = subprocess.run(
["/bin/sh", "-c", script],
env={"PATH": "/usr/bin:/bin", "VC_TEST_MODE": mode},
check=False, capture_output=True, text=True,
)
assert result.returncode == 0, result.stderr
assert result.stdout == expected
def test_reconcile_identity_probe_recreates_only_volatile_runtime_lock():
script = _identity_script(create_key=False)
# /run is empty again after reboot, including after a pre-journal apply
# failure. Recreate its lock so key validation remains serialized, while
# never manufacturing or replacing the persistent WireGuard identity.
assert "mkdir /run/virt-cloud" in script
assert ': > "$LOCK"' in script
assert '"$FLOCK" -x 9' in script
assert "genkey" not in script
assert 'mktemp "$BASE/.wg-key.' not in script
assert 'mv -f "$_tmp_key" "$KEY"' not in script
assert '[ -d "$BASE" ]' in script
assert '[ -f "$KEY" ]' in script
def test_networkmanager_profile_has_no_firewall_zone():
current = plan()
profile = _nm_profile_template(
current, current.nodes[0],
"12345678-1234-5678-1234-567812345678",
FIREWALL_MODE_NATIVE,
)
assert "autoconnect=false" in profile
assert "autoconnect-retries=0" in profile
assert "zone=" not in profile
assert "interface-name=wg-fabric0" in profile
assert "203.0.113." not in profile
assert "zone=trusted" not in profile
def test_endpoint_rp_filter_evidence_uses_linux_max_without_blocking_strict():
capabilities = {
"network": {
"local_ipv4_interfaces": {"203.0.113.1": ["br0"]},
"rp_filter": 0,
"rp_filter_default": 1,
"rp_filter_interfaces": {"br0": 1},
},
}
evidence, error = _fabric_endpoint_rp_filter_evidence(
capabilities, "203.0.113.1")
assert error is None
assert evidence == {
"fabric_endpoint_interface": "br0",
"fabric_endpoint_rp_filter_raw": 1,
"fabric_endpoint_rp_filter_effective": 1,
"fabric_rp_filter_raw": None,
"fabric_rp_filter_effective": None,
"rp_filter_default_effective": 1,
}
capabilities["network"]["rp_filter"] = 2
capabilities["network"]["rp_filter_interfaces"]["br0"] = 1
capabilities["network"]["rp_filter_interfaces"][FABRIC_INTERFACE] = 1
evidence, error = _fabric_endpoint_rp_filter_evidence(
capabilities, "203.0.113.1")
assert error is None
assert evidence["fabric_endpoint_rp_filter_effective"] == 2
assert evidence["fabric_rp_filter_effective"] == 2
def test_endpoint_rp_filter_ignores_bounded_high_density_guest_interfaces():
rp_filters = {f"vnet{index}": 1 for index in range(300)}
rp_filters["br0"] = 1
capabilities = {
"network": {
"local_ipv4_interfaces": {"203.0.113.1": ["br0"]},
"rp_filter": 0,
"rp_filter_default": 1,
"rp_filter_interfaces": rp_filters,
},
}
evidence, error = _fabric_endpoint_rp_filter_evidence(
capabilities, "203.0.113.1")
assert error is None
assert evidence["fabric_endpoint_interface"] == "br0"
assert evidence["fabric_endpoint_rp_filter_raw"] == 1
def test_endpoint_rp_filter_ignores_bounded_high_density_local_addresses():
address_map = {
f"198.51.{index // 256}.{index % 256}": [f"vnet{index}"]
for index in range(300)
}
address_map["203.0.113.1"] = ["br0"]
capabilities = {
"network": {
"local_ipv4_interfaces": address_map,
"rp_filter": 0,
"rp_filter_default": 1,
"rp_filter_interfaces": {"br0": 1},
},
}
evidence, error = _fabric_endpoint_rp_filter_evidence(
capabilities, "203.0.113.1")
assert error is None
assert evidence["fabric_endpoint_interface"] == "br0"
def test_endpoint_rp_filter_rejects_address_inventory_above_probe_limit():
address_map = {
f"address-{index}": ["br0"]
for index in range(16385)
}
address_map["203.0.113.1"] = ["br0"]
evidence, error = _fabric_endpoint_rp_filter_evidence({
"network": {
"local_ipv4_interfaces": address_map,
"rp_filter": 0,
"rp_filter_default": 1,
"rp_filter_interfaces": {"br0": 1},
},
}, "203.0.113.1")
assert evidence is None
assert error == "fabric_endpoint_interface_unverified"
def test_endpoint_rp_filter_rejects_probe_inventory_above_4096_interfaces():
rp_filters = {f"vnet{index}": 1 for index in range(4096)}
rp_filters["br0"] = 1
evidence, error = _fabric_endpoint_rp_filter_evidence({
"network": {
"local_ipv4_interfaces": {"203.0.113.1": ["br0"]},
"rp_filter": 0,
"rp_filter_default": 1,
"rp_filter_interfaces": rp_filters,
},
}, "203.0.113.1")
assert evidence is None
assert error == "fabric_endpoint_rp_filter_unverified"
@pytest.mark.parametrize("interfaces,rp_filters,expected_error", [
(["br0", "eth0"], {"br0": 1, "eth0": 1},
"fabric_endpoint_interface_unverified"),
(["br0"], {}, "fabric_endpoint_rp_filter_unverified"),
(["vnet3"], {"vnet3": 1},
"fabric_endpoint_interface_unsafe_or_drifted"),
(["tap3"], {"tap3": 1},
"fabric_endpoint_interface_unsafe_or_drifted"),
(["macvtap3"], {"macvtap3": 1},
"fabric_endpoint_interface_unsafe_or_drifted"),
(["fwpr3"], {"fwpr3": 1},
"fabric_endpoint_interface_unsafe_or_drifted"),
(["fwln3"], {"fwln3": 1},
"fabric_endpoint_interface_unsafe_or_drifted"),
])
def test_endpoint_rp_filter_evidence_fails_closed_on_ambiguous_or_unsafe_data(
interfaces, rp_filters, expected_error):
evidence, error = _fabric_endpoint_rp_filter_evidence({
"network": {
"local_ipv4_interfaces": {"203.0.113.1": interfaces},
"rp_filter": 0,
"rp_filter_default": 1,
"rp_filter_interfaces": rp_filters,
},
}, "203.0.113.1")
assert evidence is None
assert error == expected_error
def test_fixed_sysctl_marker_requires_owner_or_exact_ready_journal(tmp_path):
current = plan()
sysctl_file = tmp_path / "90-virt-cloud-wg-fabric.conf"
sysctl_file.write_text(_sysctl_text(current), encoding="utf-8")
sysctl_file.chmod(0o644)
owner = tmp_path / "wg-fabric0.owner"
journal = tmp_path / ".wg-fabric-txn"
stat = tmp_path / "stat"
stat.write_text(
"""#!/bin/sh
case "$2" in
'%u:%a')
if [ -d "$3" ]; then
printf '%s\n' 0:700
elif [ "${3##*/}" = 90-virt-cloud-wg-fabric.conf ]; then
printf '%s\n' 0:644
else
printf '%s\n' 0:600
fi ;;
'%u') printf '%s\n' 0 ;;
'%a')
if [ -d "$3" ]; then printf '%s\n' 700
elif [ "${3##*/}" = 90-virt-cloud-wg-fabric.conf ]; then
printf '%s\n' 644
else printf '%s\n' 600; fi ;;
*) exit 2 ;;
esac
""",
encoding="utf-8",
)
stat.chmod(0o700)
preflight = f"""set -eu
STAT='{stat}'
OWNER='{owner}'
JOURNAL='{journal}'
EXPECTED_DIGEST='{current.digest}'
SYSCTL_FILE='{sysctl_file}'
{_FABRIC_SYSCTL_VALIDATION_SH}
vc_fabric_sysctl_file_owned "$SYSCTL_FILE" any || exit 71
if ! vc_fabric_owner_file_owned; then
vc_fabric_sysctl_journal_owned || exit 71
fi
"""
foreign = subprocess.run(
["/bin/sh", "-c", preflight], check=False,
capture_output=True, text=True,
)
assert foreign.returncode == 71
assert sysctl_file.read_text(encoding="utf-8") == _sysctl_text(current)
owner.write_text(f"{FABRIC_OWNER_MARKER}\n", encoding="utf-8")
owner.chmod(0o600)
owned = subprocess.run(
["/bin/sh", "-c", preflight], check=False,
capture_output=True, text=True,
)
assert owned.returncode == 0, owned.stderr
owner.unlink()
journal.mkdir(mode=0o700)
(journal / "state").write_text("ready\n", encoding="utf-8")
(journal / "meta").write_text(
f"schema=1\nplan_digest={current.digest}\n", encoding="utf-8",
)
(journal / "sysctl.absent").touch(mode=0o600)
(journal / "state").chmod(0o600)
(journal / "meta").chmod(0o600)
journaled = subprocess.run(
["/bin/sh", "-c", preflight], check=False,
capture_output=True, text=True,
)
assert journaled.returncode == 0, journaled.stderr
def test_apply_rollback_restores_owned_sysctl_and_runtime_rp_filter(tmp_path):
current = plan()
rendered = _apply_script(
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
)
restore_start = rendered.index("restore_file() {")
restore_end = rendered.index("\n}\nrollback_sysctl_file()", restore_start) + 2
sysctl_start = rendered.index("rollback_sysctl_file() {", restore_end)
sysctl_end = rendered.index("\n}\nrollback_rp_filter()", sysctl_start) + 2
rpf_start = rendered.index("rollback_rp_filter() {", sysctl_end)
rpf_end = rendered.index("\n}\nwrite_nft_phase()", rpf_start) + 2
functions = "\n".join((
rendered[restore_start:restore_end],
rendered[sysctl_start:sysctl_end],
rendered[rpf_start:rpf_end],
))
txn = tmp_path / "txn"
txn.mkdir()
old_sysctl = (
"# virt-cloud-wg-fabric-v1\n"
f"# fabric-plan-digest={'0' * 64}\n"
"net.ipv4.conf.wg-fabric0.rp_filter = 2\n"
)
(txn / "sysctl").write_text(old_sysctl, encoding="utf-8")
(txn / "sysctl.present").touch()
(txn / "sysctl.desired").write_text(
_sysctl_text(current), encoding="utf-8")
(txn / "sysctl.desired").chmod(0o644)
sysctl_file = tmp_path / "fabric.conf"
sysctl_file.write_text(_sysctl_text(current), encoding="utf-8")
rpf_file = tmp_path / "rp_filter"
rpf_file.write_text("2\n", encoding="utf-8")
stat = tmp_path / "stat"
stat.write_text(
"#!/bin/sh\n"
"[ \"$1:$2\" = '-c:%u:%a' ] || exit 2\n"
"mode=$(/usr/bin/stat -f '%Lp' \"$3\") || exit 2\n"
"printf '%s:%s\\n' \"${VC_TEST_UID:-0}\" \"$mode\"\n",
encoding="utf-8",
)
stat.chmod(0o700)
functions = functions.replace(
'"/proc/sys/net/ipv4/conf/$IFACE/rp_filter"',
f"'{rpf_file}'",
).replace(
"/etc/sysctl.d/.virt-cloud-wg-fabric-rollback.XXXXXX",
f"{tmp_path}/.fabric-rollback.XXXXXX",
)
harness = f"""set -eu
TXN='{txn}'
SYSCTL_FILE='{sysctl_file}'
EXPECTED_DIGEST='{current.digest}'
CMP=/usr/bin/cmp
STAT='{stat}'
IFACE=wg-fabric0
IFACE_BEFORE=yes
RP_FILTER_BEFORE=1
ROLLBACK_FAILED=no
ROLLBACK_IFACE_RECREATED=no
read_sysctl() {{ cat "$1"; }}
vc_fabric_sysctl_file_owned() {{
[ "$($STAT -c '%u:%a' "$1")" = 0:644 ] || return 1
test_digest=$(sed -n '2s/^# fabric-plan-digest=//p' "$1")
[ "$2" = any ] || [ "$test_digest" = "$2" ]
}}
vc_fabric_sysctl_files_equal() {{
vc_fabric_sysctl_file_owned "$1" "$3" || return 1
vc_fabric_sysctl_file_owned "$2" "$3" || return 1
"$CMP" -s "$1" "$2"
}}
{functions}
rollback_sysctl_file
rollback_rp_filter
[ "$ROLLBACK_FAILED" = no ]
"""
result = subprocess.run(
["/bin/sh", "-c", harness], check=False,
capture_output=True, text=True,
)
assert result.returncode == 0, result.stderr
assert sysctl_file.read_text(encoding="utf-8") == old_sysctl
assert rpf_file.read_text(encoding="utf-8") == "1\n"
# Equal bytes are insufficient ownership evidence: metadata drift must
# stop rollback before either the fixed file or runtime value is touched.
sysctl_file.chmod(0o600)
rpf_file.write_text("2\n", encoding="utf-8")
foreign_metadata = subprocess.run(
["/bin/sh", "-c", harness], check=False,
capture_output=True, text=True,
)
assert foreign_metadata.returncode != 0
assert sysctl_file.read_text(encoding="utf-8") == old_sysctl
assert rpf_file.read_text(encoding="utf-8") == "2\n"
sysctl_file.chmod(0o644)
foreign_uid = subprocess.run(
["/bin/sh", "-c", harness],
env={"PATH": "/usr/bin:/bin", "VC_TEST_UID": "1"},
check=False, capture_output=True, text=True,
)
assert foreign_uid.returncode != 0
assert sysctl_file.read_text(encoding="utf-8") == old_sysctl
assert rpf_file.read_text(encoding="utf-8") == "2\n"
sysctl_file.write_text("foreign\n", encoding="utf-8")
rpf_file.write_text("2\n", encoding="utf-8")
foreign = subprocess.run(
["/bin/sh", "-c", harness], check=False,
capture_output=True, text=True,
)
assert foreign.returncode != 0
assert sysctl_file.read_text(encoding="utf-8") == "foreign\n"
assert rpf_file.read_text(encoding="utf-8") == "2\n"
# Runtime drift is independent from the fixed file. Only the
# transaction-installed value 2 may be replaced by the sampled old value.
sysctl_file.write_text(_sysctl_text(current), encoding="utf-8")
sysctl_file.chmod(0o644)
rpf_file.write_text("0\n", encoding="utf-8")
runtime_drift = subprocess.run(
["/bin/sh", "-c", harness], check=False,
capture_output=True, text=True,
)
assert runtime_drift.returncode != 0
assert rpf_file.read_text(encoding="utf-8") == "0\n"
def test_fabric_sysctl_apply_and_rollback_are_transaction_bound():
current = plan()
apply = _apply_script(
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
)
snapshot = apply.index('snapshot_file "$SYSCTL_FILE" sysctl')
snapshot_readback = apply.index(
'vc_fabric_sysctl_files_equal "$SYSCTL_FILE" "$TXN/sysctl" any',
snapshot,
)
desired = apply.index(
'_tmp_sysctl_desired=$(mktemp '
'"$TXN/.sysctl.desired.XXXXXX")', snapshot_readback)
desired_temp_owned = apply.index(
'vc_fabric_sysctl_file_owned '
'"$_tmp_sysctl_desired" "$EXPECTED_DIGEST"', desired)
desired_publish = apply.index(
'mv -f "$_tmp_sysctl_desired" "$TXN/sysctl.desired"',
desired_temp_owned,
)
desired_owned = apply.index(
'vc_fabric_sysctl_file_owned '
'"$TXN/sysctl.desired" "$EXPECTED_DIGEST"', desired_publish)
desired_sync = apply.index("\nsync\n", desired_owned)
precommit = apply.index(
'vc_fabric_sysctl_files_equal "$SYSCTL_FILE" "$TXN/sysctl" any',
desired_owned,
)
install = apply.index('mv -f "$_tmp_sysctl" "$SYSCTL_FILE"', precommit)
installed_readback = apply.index(
'vc_fabric_sysctl_files_equal "$SYSCTL_FILE" '
'"$TXN/sysctl.desired" "$EXPECTED_DIGEST"', install)
assert snapshot < snapshot_readback < desired < desired_temp_owned
assert desired_temp_owned < desired_publish < desired_owned < desired_sync
assert desired_sync < precommit < install < installed_readback
rollback = apply.index("rollback_sysctl_file() {")
desired_proof = apply.index(
'vc_fabric_sysctl_files_equal "$SYSCTL_FILE" \\\n'
' "$TXN/sysctl.desired" "$EXPECTED_DIGEST"', rollback)
pre_restore_recheck = apply.index(
'vc_fabric_sysctl_files_equal "$SYSCTL_FILE" \\\n'
' "$TXN/sysctl.desired" "$EXPECTED_DIGEST"',
desired_proof,
)
restore = apply.index(
'mv -f "$rollback_sysctl_tmp" "$SYSCTL_FILE"',
pre_restore_recheck,
)
assert rollback < desired_proof < pre_restore_recheck < restore
def _root_stat_stub(tmp_path: Path) -> Path:
stat = tmp_path / "stat"
stat.write_text(
"#!/bin/sh\n"
"case \"$1:$2\" in\n"
" '-c:%u') printf '%s\\n' 0 ;;\n"
" '-c:%a') /usr/bin/stat -f '%Lp' \"$3\" ;;\n"
" '-c:%u:%a')\n"
" mode=$(/usr/bin/stat -f '%Lp' \"$3\") || exit 2\n"
" printf '0:%s\\n' \"$mode\" ;;\n"
" *) exit 2 ;;\n"
"esac\n",
encoding="utf-8",
)
stat.chmod(0o700)
return stat
def test_generic_apply_rollback_restores_only_its_exact_desired_file(tmp_path):
current = plan()
apply = _apply_script(
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
)
functions_start = apply.index("txn_file_secure() {")
functions_end = apply.index(
"\nrollback_sysctl_file() {", functions_start)
functions = apply[functions_start:functions_end]
txn = tmp_path / "txn"
txn.mkdir()
target = tmp_path / "managed.conf"
stat = _root_stat_stub(tmp_path)
harness = "\n".join((
"set -eu",
f"TXN='{txn}'",
f"STAT='{stat}'",
"CMP=/usr/bin/cmp",
"ROLLBACK_FAILED=no",
"_tmp_restore=",
functions,
f"restore_file '{target}' conf",
"printf '%s\\n' \"$ROLLBACK_FAILED\"",
))
def prepare(*, baseline: str | None, desired: str | None,
live: str | None, live_mode: int = 0o600) -> None:
for path in tuple(txn.iterdir()):
path.unlink()
if target.exists() or target.is_symlink():
target.unlink()
if baseline is None:
(txn / "conf.absent").touch(mode=0o600)
else:
(txn / "conf").write_text(baseline, encoding="utf-8")
(txn / "conf").chmod(0o600)
(txn / "conf.present").touch(mode=0o600)
if desired is None:
(txn / "conf.desired.absent").touch(mode=0o600)
else:
(txn / "conf.desired").write_text(desired, encoding="utf-8")
(txn / "conf.desired").chmod(0o600)
if live is not None:
target.write_text(live, encoding="utf-8")
target.chmod(live_mode)
def run():
return subprocess.run(
["/bin/sh", "-c", harness], check=False,
capture_output=True, text=True,
)
prepare(baseline="old\n", desired="new\n", live="new\n")
restored = run()
assert restored.returncode == 0, restored.stderr
assert restored.stdout == "no\n"
assert target.read_text(encoding="utf-8") == "old\n"
# A crash before this path's desired snapshot was published performed no
# mutation. Exact baseline bytes remain a valid no-op recovery state.
prepare(baseline="old\n", desired="new\n", live="old\n")
(txn / "conf.desired").unlink()
untouched = run()
assert untouched.returncode == 0, untouched.stderr
assert untouched.stdout == "no\n"
assert target.read_text(encoding="utf-8") == "old\n"
prepare(baseline="old\n", desired="new\n", live="operator\n")
drifted = run()
assert drifted.returncode == 0, drifted.stderr
assert drifted.stdout == "yes\n"
assert target.read_text(encoding="utf-8") == "operator\n"
prepare(baseline=None, desired="created\n", live="created\n")
removed = run()
assert removed.returncode == 0, removed.stderr
assert removed.stdout == "no\n"
assert target.exists() is False
prepare(baseline="old\n", desired=None, live=None)
recreated = run()
assert recreated.returncode == 0, recreated.stderr
assert recreated.stdout == "no\n"
assert target.read_text(encoding="utf-8") == "old\n"
prepare(
baseline="old\n", desired="new\n", live="new\n",
live_mode=0o644,
)
metadata_drift = run()
assert metadata_drift.returncode == 0, metadata_drift.stderr
assert metadata_drift.stdout == "yes\n"
assert target.read_text(encoding="utf-8") == "new\n"
assert target.stat().st_mode & 0o777 == 0o644
def test_boot_recovery_restores_only_exact_transaction_post_image(tmp_path):
secure_start = _BOOT_HELPER.index("journal_file_secure() {")
secure_end = _BOOT_HELPER.index(
"journal_cleanup_helper_owned() {", secure_start)
restore_start = _BOOT_HELPER.index("restore_journal_file() {")
restore_end = _BOOT_HELPER.index(
"restore_journal_sysctl() {", restore_start)
functions = "\n".join((
_BOOT_HELPER[secure_start:secure_end],
_BOOT_HELPER[restore_start:restore_end],
))
journal = tmp_path / "journal"
journal.mkdir()
target = tmp_path / "managed.conf"
stat = _root_stat_stub(tmp_path)
harness = "\n".join((
"set -eu",
f"JOURNAL='{journal}'",
f"STAT='{stat}'",
"CMP=/usr/bin/cmp",
"_tmp_journal_restore=",
"stat() { \"$STAT\" \"$@\"; }",
functions,
f"if restore_journal_file '{target}' conf; then",
" printf '%s\\n' restored",
"else",
" printf '%s\\n' blocked",
"fi",
))
def prepare(*, baseline: str | None, desired: str | None,
live: str | None, live_mode: int = 0o600) -> None:
for path in tuple(journal.iterdir()):
path.unlink()
if target.exists() or target.is_symlink():
target.unlink()
if baseline is None:
(journal / "conf.absent").touch(mode=0o600)
else:
(journal / "conf").write_text(baseline, encoding="utf-8")
(journal / "conf").chmod(0o600)
(journal / "conf.present").touch(mode=0o600)
if desired is None:
(journal / "conf.desired.absent").touch(mode=0o600)
else:
(journal / "conf.desired").write_text(
desired, encoding="utf-8")
(journal / "conf.desired").chmod(0o600)
if live is not None:
target.write_text(live, encoding="utf-8")
target.chmod(live_mode)
def run():
return subprocess.run(
["/bin/sh", "-c", harness], check=False,
capture_output=True, text=True,
)
prepare(baseline="old\n", desired="new\n", live="new\n")
restored = run()
assert restored.returncode == 0, restored.stderr
assert restored.stdout == "restored\n"
assert target.read_text(encoding="utf-8") == "old\n"
prepare(baseline="old\n", desired="new\n", live="old\n")
(journal / "conf.desired").unlink()
untouched = run()
assert untouched.returncode == 0, untouched.stderr
assert untouched.stdout == "restored\n"
assert target.read_text(encoding="utf-8") == "old\n"
prepare(baseline="old\n", desired="new\n", live="operator\n")
drifted = run()
assert drifted.returncode == 0, drifted.stderr
assert drifted.stdout == "blocked\n"
assert target.read_text(encoding="utf-8") == "operator\n"
prepare(baseline=None, desired="created\n", live="created\n")
removed = run()
assert removed.returncode == 0, removed.stderr
assert removed.stdout == "restored\n"
assert target.exists() is False
prepare(
baseline="old\n", desired="new\n", live="new\n",
live_mode=0o644,
)
metadata_drift = run()
assert metadata_drift.returncode == 0, metadata_drift.stderr
assert metadata_drift.stdout == "blocked\n"
assert target.read_text(encoding="utf-8") == "new\n"
assert target.stat().st_mode & 0o777 == 0o644
def test_fixed_artifacts_publish_desired_snapshot_before_atomic_install():
current = plan()
apply = _apply_script(
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
firewall_mode=FIREWALL_MODE_NATIVE,
)
assert "record_desired_file() {" in apply
assert "record_desired_absent() {" in apply
assert (
'txn_baseline_matches "$publish_target" "$publish_name"'
in apply
)
for source, target, name in (
("$_tmp_owner", "$OWNER", "owner"),
("$_tmp_nft", "$NFT_RULES", "nft_rules"),
("$_tmp_nft_main", "$NFT_MAIN", "nft_main"),
("$_tmp_helper", "$HELPER", "helper"),
("$_tmp_nm", "$NM_PROFILE", "nm_profile"),
("$_tmp_conf", "$CONF", "conf"),
("$_tmp_unit", "$UNIT", "unit"),
("$_tmp_repair", "$REPAIR_UNIT", "repair"),
("$_tmp_timer", "$TIMER_UNIT", "timer"),
):
assert (
f'publish_desired_file "{source}" "{target}" {name}' in apply
)
assert 'publish_desired_file "$_tmp_key" "$KEY" key' not in apply
assert 'journal_file_matches "$restore_target"' in _BOOT_HELPER
assert '"$restore_name.desired"' in _BOOT_HELPER
def test_rollback_keeps_recovery_helper_until_every_other_restore_succeeds():
current = plan()
apply = _apply_script(
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
firewall_mode=FIREWALL_MODE_NATIVE,
)
rollback = apply.index("rollback() {")
first_fixed_restore = apply.index(
'restore_file "$UNIT" unit', rollback)
verify_runtime = apply.index(
"\n verify_rollback_rp_filter\n", first_fixed_restore)
helper_comment = apply.index(
"# Keep the transaction's complete recovery helper installed",
verify_runtime,
)
cleanup_state = apply.index(
"rollback_publish_cleanup_helper_state || ROLLBACK_FAILED=yes",
helper_comment,
)
helper_restore = apply.index(
'restore_file "$HELPER" helper', cleanup_state)
final_decision = apply.index(
'if [ "$ROLLBACK_FAILED" = yes ]; then', helper_restore)
assert first_fixed_restore < verify_runtime < helper_comment < cleanup_state
assert cleanup_state < helper_restore < final_decision
assert apply.count('restore_file "$HELPER" helper') == 1
def test_rollback_stops_timer_only_with_exact_transaction_unit(tmp_path):
current = plan()
apply = _apply_script(
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
)
functions_start = apply.index("txn_file_secure() {")
functions_end = apply.index(
"\nrollback_sysctl_file() {", functions_start)
functions = apply[functions_start:functions_end]
txn = tmp_path / "txn"
txn.mkdir()
timer = tmp_path / "timer.unit"
desired = "[Unit]\nDescription=transaction\n"
(txn / "timer.desired").write_text(desired, encoding="utf-8")
(txn / "timer.desired").chmod(0o644)
timer.write_text(desired, encoding="utf-8")
timer.chmod(0o644)
active = tmp_path / "active"
active.write_text("active\n", encoding="utf-8")
log = tmp_path / "systemctl.log"
systemctl = tmp_path / "systemctl"
systemctl.write_text(
"#!/bin/sh\n"
"case \"$1:$2:$3\" in\n"
" 'show:-p:LoadState') printf '%s\\n' loaded ;;\n"
" 'show:-p:ActiveState') cat \"$VC_TIMER_ACTIVE\" ;;\n"
" 'stop:virt-cloud-wg-fabric-reconcile.timer:')\n"
" printf '%s\\n' stop >> \"$VC_TIMER_LOG\"\n"
" printf '%s\\n' inactive > \"$VC_TIMER_ACTIVE\" ;;\n"
" *) exit 2 ;;\n"
"esac\n",
encoding="utf-8",
)
systemctl.chmod(0o700)
stat = _root_stat_stub(tmp_path)
harness = "\n".join((
"set -eu",
f"TXN='{txn}'",
f"TIMER_UNIT='{timer}'",
f"SYSTEMCTL='{systemctl}'",
f"STAT='{stat}'",
"CMP=/usr/bin/cmp",
functions,
"rollback_stop_transaction_timer_if_owned",
))
env = {
"PATH": "/usr/bin:/bin",
"VC_TIMER_ACTIVE": str(active),
"VC_TIMER_LOG": str(log),
}
owned = subprocess.run(
["/bin/sh", "-c", harness], env=env, check=False,
capture_output=True, text=True,
)
assert owned.returncode == 0, owned.stderr
assert log.read_text(encoding="utf-8") == "stop\n"
timer.write_text("[Unit]\nDescription=operator\n", encoding="utf-8")
active.write_text("active\n", encoding="utf-8")
log.unlink()
foreign = subprocess.run(
["/bin/sh", "-c", harness], env=env, check=False,
capture_output=True, text=True,
)
assert foreign.returncode == 1
assert log.exists() is False
assert active.read_text(encoding="utf-8") == "active\n"
def test_recreated_interface_may_restore_rp_filter_from_host_default(tmp_path):
phase_start = _BOOT_HELPER.index("journal_iface_recovery_state() {")
phase_end = _BOOT_HELPER.index("restore_journal_file() {", phase_start)
phase_functions = _BOOT_HELPER[phase_start:phase_end]
journal = tmp_path / "journal"
journal.mkdir()
phase_harness = "\n".join((
"set -eu",
f"JOURNAL='{journal}'",
"journal_file_secure() {",
" [ -f \"$JOURNAL/$1\" ] && [ ! -L \"$JOURNAL/$1\" ] &&",
" [ \"$(/usr/bin/stat -f '%Lp' \"$JOURNAL/$1\")\" = 600 ]",
"}",
phase_functions,
"journal_publish_iface_recovery_state recreating",
'[ "$(journal_iface_recovery_state)" = recreating ]',
"journal_publish_iface_recovery_state recreated",
"journal_iface_recovery_state",
))
phase = subprocess.run(
["/bin/sh", "-c", phase_harness], check=False,
capture_output=True, text=True,
)
assert phase.returncode == 0, phase.stderr
assert phase.stdout == "recreated"
assert (journal / "iface.recovery").read_text(encoding="utf-8") == (
"recreated\n")
recovery_start = _BOOT_HELPER.index("journal_restore_rp_filter() {")
recovery_end = _BOOT_HELPER.index(
"journal_verify_rp_filter() {", recovery_start)
recovery = _BOOT_HELPER[recovery_start:recovery_end].replace(
'"/proc/sys/net/ipv4/conf/$IFACE/rp_filter"',
'"$VC_TEST_RPF"',
)
rpf = tmp_path / "rp_filter"
recovery_harness = "\n".join((
"set -eu",
f"VC_TEST_RPF='{rpf}'",
"journal_iface_before=yes",
"journal_rp_filter_before=2",
"read_sysctl() { cat \"$1\"; }",
recovery,
"journal_restore_rp_filter",
))
rpf.write_text("1\n", encoding="utf-8")
recreated = subprocess.run(
["/bin/sh", "-c", "journal_iface_recreated=yes\n" + recovery_harness],
check=False, capture_output=True, text=True,
)
assert recreated.returncode == 0, recreated.stderr
assert rpf.read_text(encoding="utf-8") == "2\n"
rpf.write_text("1\n", encoding="utf-8")
persistent = subprocess.run(
["/bin/sh", "-c", "journal_iface_recreated=no\n" + recovery_harness],
check=False, capture_output=True, text=True,
)
assert persistent.returncode != 0
assert rpf.read_text(encoding="utf-8") == "1\n"
current = plan()
apply = _apply_script(
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
)
rollback_start = apply.index("rollback_rp_filter() {")
rollback_end = apply.index("verify_rollback_rp_filter() {", rollback_start)
rollback = apply[rollback_start:rollback_end].replace(
'"/proc/sys/net/ipv4/conf/$IFACE/rp_filter"',
'"$VC_TEST_RPF"',
)
rollback_harness = "\n".join((
"set -eu",
f"VC_TEST_RPF='{rpf}'",
"IFACE_BEFORE=yes",
"RP_FILTER_BEFORE=2",
"ROLLBACK_FAILED=no",
"read_sysctl() { cat \"$1\"; }",
rollback,
"rollback_rp_filter",
'[ "$ROLLBACK_FAILED" = no ]',
))
rpf.write_text("1\n", encoding="utf-8")
rollback_recreated = subprocess.run(
["/bin/sh", "-c", "ROLLBACK_IFACE_RECREATED=yes\n" + rollback_harness],
check=False, capture_output=True, text=True,
)
assert rollback_recreated.returncode == 0, rollback_recreated.stderr
assert rpf.read_text(encoding="utf-8") == "2\n"
rpf.write_text("1\n", encoding="utf-8")
rollback_persistent = subprocess.run(
["/bin/sh", "-c", "ROLLBACK_IFACE_RECREATED=no\n" + rollback_harness],
check=False, capture_output=True, text=True,
)
assert rollback_persistent.returncode != 0
assert rpf.read_text(encoding="utf-8") == "1\n"
@pytest.mark.parametrize("readback_rc", (0, 71))
def test_ifupdown_service_requires_readback_after_successful_ifup(
tmp_path, readback_rc):
start = _BOOT_HELPER.index(
'if [ "$DEPLOYMENT_MODE" = ifupdown ] &&\n'
' [ "${1:-}" = --ifupdown-service ]; then')
end = _BOOT_HELPER.index(
'if [ "$DEPLOYMENT_MODE" = ifupdown ] &&\n'
' [ "${1:-}" = --ifupdown-hook ]; then', start)
calls = tmp_path / "ifup.calls"
ifup = tmp_path / "ifup"
ifup.write_text(
"#!/bin/sh\n"
f"printf '%s\\n' \"$*\" >> {shlex.quote(str(calls))}\n"
"exit 0\n", encoding="utf-8")
ifup.chmod(0o700)
harness = "\n".join((
"set -eu", "DEPLOYMENT_MODE=ifupdown", "IFACE=wg-fabric0",
f"IFUP={shlex.quote(str(ifup))}", "set -- --ifupdown-service",
_BOOT_HELPER[start:end],
'[ "$1" = --ifupdown-configure ] || exit 99',
f"exit {readback_rc}",
))
result = subprocess.run(
["/bin/sh", "-c", harness], check=False,
capture_output=True, text=True)
assert result.returncode == readback_rc, result.stderr
assert calls.read_text(encoding="utf-8").splitlines() == [
"--force wg-fabric0"]
def test_ifupdown_transition_hook_executes_without_reconciling_stale_config(
tmp_path):
start = _BOOT_HELPER.index(
'if [ "$DEPLOYMENT_MODE" = ifupdown ] &&\n'
' [ "${1:-}" = --ifupdown-configure ]; then')
end = _BOOT_HELPER.index("vc_firewall_mode_ready || exit 71", start)
block = _BOOT_HELPER[start:end].replace(
'"/proc/sys/net/ipv4/conf/$IFACE/rp_filter"',
'"$VC_TEST_RPF"',
)
rpf = tmp_path / "rp_filter"
rpf.write_text("2\n", encoding="utf-8")
harness = "\n".join((
"set -eu",
"DEPLOYMENT_MODE=ifupdown",
"IFACE=wg-fabric0",
"VC_FABRIC_APPLYING=1",
"VC_FABRIC_ALLOW_TRANSITION=1",
"validate_ifupdown_fragment() { :; }",
"vc_fabric_rp_filter_set() { printf '%s\\n' rpf >> \"$VC_TEST_LOG\"; }",
"read_sysctl() { cat \"$1\"; }",
"mock_ip() { printf '%s\\n' \"$*\" >> \"$VC_TEST_LOG\"; }",
"IP=mock_ip",
"set -- --ifupdown-configure",
block,
"exit 99",
))
log = tmp_path / "transition.log"
env = {
"PATH": "/usr/bin:/bin",
"VC_TEST_LOG": str(log),
"VC_TEST_RPF": str(rpf),
}
applied = subprocess.run(
["/bin/sh", "-c", harness], env=env, check=False,
capture_output=True, text=True,
)
assert applied.returncode == 0, applied.stderr
assert log.read_text(encoding="utf-8").splitlines() == [
"rpf", "link set dev wg-fabric0 up", "rpf",
]
log.write_text("", encoding="utf-8")
rpf.write_text("1\n", encoding="utf-8")
recovery_env = dict(env)
recovery_env["VC_FABRIC_RESTORE_RP_FILTER"] = "1"
recovered = subprocess.run(
["/bin/sh", "-c", harness], env=recovery_env, check=False,
capture_output=True, text=True,
)
assert recovered.returncode == 0, recovered.stderr
assert log.read_text(encoding="utf-8").splitlines() == [
"link set dev wg-fabric0 up",
]
def test_fabric_and_vpc_rp_filter_are_set_before_manager_activation():
current = plan()
apply = _apply_script(
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
)
# The rendered temporary recovery helper contains the same ownership
# checks. Anchor on the later, live apply block rather than its here-doc.
apply_owner = apply.rindex("vc_fabric_owner_file_owned || exit 71")
apply_persistence = apply.index(
'mv -f "$_tmp_sysctl" "$SYSCTL_FILE"',
apply_owner,
)
apply_create = apply.index(
'"$IP" link add dev "$IFACE" type wireguard', apply_persistence)
apply_runtime = apply.index(
"vc_fabric_rp_filter_set || exit 71", apply_create)
apply_nm_up = apply.index(
'"$NMCLI" connection up uuid', apply_runtime)
assert apply_owner < apply_persistence < apply_create < apply_runtime < apply_nm_up
assert 'vc_fabric_sysctl_file_owned "$SYSCTL_FILE" any' in _BOOT_HELPER
assert '> /proc/sys/net/ipv4/conf/all/rp_filter' not in apply
assert '> /proc/sys/net/ipv4/conf/default/rp_filter' not in apply
boot_create = _BOOT_HELPER.index(
'"$IP" link add dev "$IFACE" type wireguard')
boot_runtime = _BOOT_HELPER.index(
"vc_fabric_rp_filter_set || exit 71", boot_create)
boot_nm_up = _BOOT_HELPER.index(
'"$NMCLI" connection up uuid "$CONNECTION_UUID"', boot_runtime)
assert boot_create < boot_runtime < boot_nm_up
vpc_create = _BOOT_HELPER.index(
'"$IP" link add name "$vpc_bridge" type bridge')
vpc_revision = _BOOT_HELPER.index(
'vpc_revision=$(awk -F\'|\' \'$1=="revision" '
"{print $2; exit}' \"$vpc_state_file\")")
vpc_digest = _BOOT_HELPER.index(
'vpc_digest=$(awk -F\'|\' \'$1=="digest" '
"{print $2; exit}' \"$vpc_state_file\")",
vpc_revision,
)
vpc_sysctl_owned = _BOOT_HELPER.index(
'vc_vpc_sysctl_artifact_owned "$vpc_sysctl_file" "$vpc_id"',
vpc_digest,
)
vpc_runtime = _BOOT_HELPER.index(
'printf \'%s\\n\' 2 > '
'"/proc/sys/net/ipv4/conf/$vpc_bridge/rp_filter"',
vpc_create,
)
vpc_nm_up = _BOOT_HELPER.index(
'"$NMCLI" connection up uuid "$vpc_nm_uuid"', vpc_runtime)
vpc_ifup = _BOOT_HELPER.index(
'VC_FABRIC_APPLYING=1 "$IFUP" --force "$vpc_bridge"',
vpc_runtime,
)
assert vpc_revision < vpc_digest < vpc_sysctl_owned < vpc_create
assert vpc_create < vpc_runtime < vpc_nm_up < vpc_ifup
assert '"$vpc_revision" "$vpc_digest" "$vpc_bridge" || exit 71' \
in _BOOT_HELPER[vpc_sysctl_owned:vpc_create]
assert '__VC_VPC_SYSCTL_VALIDATION__' not in _BOOT_HELPER
assert (
'grep -Fqx "net.ipv4.conf.$vpc_bridge.rp_filter = 2" '
'"$vpc_sysctl_file"'
) not in _BOOT_HELPER
def test_recovery_and_rollback_restore_rp_filter_before_reactivation():
recover_start = _BOOT_HELPER.index("recover_journal() {")
recover_end = _BOOT_HELPER.index(
'if [ "$RECOVER_ONLY" = yes ]; then', recover_start)
recovery = _BOOT_HELPER[recover_start:recover_end]
recovery_phase_load = recovery.index(
"journal_iface_recovery_phase=$(journal_iface_recovery_state)")
recovery_phase_adopt = recovery.index(
'if [ "$journal_iface_recovery_phase" != none ]; then\n'
' journal_iface_recreated=yes',
recovery_phase_load,
)
recovery_recreating = recovery.index(
"journal_publish_iface_recovery_state recreating")
recovery_recreate = recovery.index(
'"$IP" link add dev "$IFACE" type wireguard', recovery_recreating)
recovery_recreated = recovery.index(
"journal_publish_iface_recovery_state recreated", recovery_recreate)
recovery_restore = recovery.index("journal_restore_rp_filter || exit 79")
recovery_nm_up = recovery.index(
'"$journal_nmcli" connection up uuid "$journal_old_uuid"',
recovery_restore,
)
recovery_ifup = recovery.index(
'"$IFUP_PATH" --force "$IFACE"', recovery_restore)
recovery_verify = recovery.index(
"if ! journal_verify_rp_filter;", recovery_restore)
assert recovery_phase_load < recovery_phase_adopt < recovery_recreating
assert recovery_recreating < recovery_recreate < recovery_recreated
assert recovery_recreated < recovery_restore < recovery_nm_up < recovery_verify
assert recovery_restore < recovery_ifup < recovery_verify
assert (
'VC_FABRIC_APPLYING=1 \\\n'
' VC_FABRIC_RESTORE_RP_FILTER="$journal_rp_filter_before"'
in recovery
)
current = plan()
apply = _apply_script(
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
)
rollback_start = apply.index("rollback() {")
rollback_recreating = apply.index(
"write_iface_recovery_phase recreating", rollback_start)
rollback_recreate = apply.index(
'"$IP" link add dev "$IFACE" type wireguard', rollback_recreating)
rollback_recreated = apply.index(
"write_iface_recovery_phase recreated", rollback_recreate)
rollback_restore = apply.index(
"\n rollback_rp_filter\n", rollback_start)
rollback_link_up = apply.index(
'"$IP" link set dev "$IFACE" up', rollback_restore)
rollback_nm_up = apply.index(
'"$NMCLI" connection up uuid', rollback_restore)
rollback_verify = apply.index(
"\n verify_rollback_rp_filter\n", rollback_restore)
assert rollback_recreating < rollback_recreate < rollback_recreated
assert rollback_recreated < rollback_restore < rollback_link_up < rollback_verify
assert rollback_restore < rollback_nm_up < rollback_verify
def test_first_deploy_recovery_and_rollback_delete_link_before_owner():
recover_start = _BOOT_HELPER.index("recover_journal() {")
first_deploy = _BOOT_HELPER.index(
'if [ "$journal_iface_before" = no ] &&', recover_start)
delete_link = _BOOT_HELPER.index(
'"$IP" link del dev "$IFACE"', first_deploy)
confirm_absent = _BOOT_HELPER.index(
'"$IP" link show dev "$IFACE"', delete_link)
mark_absent = _BOOT_HELPER.index(
"journal_current_iface_state=absent", confirm_absent)
restore_owner = _BOOT_HELPER.index(
'restore_journal_file "$OWNER" owner', mark_absent)
assert first_deploy < delete_link < confirm_absent < mark_absent < restore_owner
current = plan()
apply = _apply_script(
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
)
rollback_start = apply.index("rollback() {")
rollback_first = apply.index(
'if [ "$IFACE_BEFORE" = no ]; then', rollback_start)
rollback_delete = apply.index(
'"$IP" link del dev "$IFACE"', rollback_first)
rollback_confirm = apply.index(
'"$IP" link show dev "$IFACE"', rollback_delete)
rollback_owner = apply.index(
'restore_file "$OWNER" owner', rollback_confirm)
guard = apply.rfind(
'if [ "$IFACE_BEFORE" = yes ] || [ "$ROLLBACK_FAILED" = no ]; then',
rollback_confirm,
rollback_owner,
)
assert rollback_first < rollback_delete < rollback_confirm < guard
assert guard < rollback_owner
def test_first_deploy_recovery_delete_is_repeatable_and_fails_closed(
tmp_path):
start = _BOOT_HELPER.index(
'if [ "$journal_iface_before" = no ] &&')
end = _BOOT_HELPER.index(" journal_sysctl_restore=no", start)
block = _BOOT_HELPER[start:end]
link_state = tmp_path / "link-present"
link_state.touch()
harness = "\n".join((
"set -eu",
"journal_iface_before=no",
"journal_current_iface_state=present",
"journal_current_owner_state=present",
"IFACE=wg-fabric0",
"vc_fabric_recovery_checkpoint() { :; }",
"vc_fabric_owner_file_owned() { :; }",
"mock_wg() { [ \"$1:$2\" = show:wg-fabric0 ]; }",
"mock_ip() {",
" case \"$*\" in",
" 'link show dev wg-fabric0') [ -e \"$VC_TEST_LINK\" ] ;;",
" 'link del dev wg-fabric0')",
" [ \"${VC_TEST_DELETE_FAIL:-no}\" = yes ] || "
"rm -f \"$VC_TEST_LINK\" ;;",
" *) return 97 ;;",
" esac",
"}",
"WG=mock_wg",
"IP=mock_ip",
block,
"printf '%s\\n' \"$journal_current_iface_state\"",
))
env = {
"PATH": "/usr/bin:/bin",
"VC_TEST_LINK": str(link_state),
}
removed = subprocess.run(
["/bin/sh", "-c", harness], env=env, check=False,
capture_output=True, text=True,
)
assert removed.returncode == 0, removed.stderr
assert removed.stdout == "absent\n"
assert link_state.exists() is False
# A crash after link deletion but before restoring the absent OWNER is
# repeatable: the next pass observes no live link and leaves it untouched.
repeated = subprocess.run(
["/bin/sh", "-c", harness.replace(
"journal_current_iface_state=present",
"journal_current_iface_state=absent",
)], env=env, check=False, capture_output=True, text=True,
)
assert repeated.returncode == 0, repeated.stderr
assert repeated.stdout == "absent\n"
link_state.touch()
failed_env = dict(env)
failed_env["VC_TEST_DELETE_FAIL"] = "yes"
failed = subprocess.run(
["/bin/sh", "-c", harness], env=failed_env, check=False,
capture_output=True, text=True,
)
assert failed.returncode == 79
assert link_state.exists() is True
def test_first_deploy_recovery_exits_without_absent_helper_or_timer_stop():
recover_start = _BOOT_HELPER.index("recover_journal() {")
recover_end = _BOOT_HELPER.index(
'\n}\nif [ "${VC_FABRIC_APPLYING:-0}" != 1 ]; then', recover_start)
recovery = _BOOT_HELPER[recover_start:recover_end]
helper_snapshot = recovery.index(
'if [ -f "$JOURNAL/helper.present" ]; then')
timer_snapshot = recovery.index(
'if [ -f "$JOURNAL/timer.present" ]; then', helper_snapshot)
stop_transaction_timer = recovery.index(
"vc_stop_transaction_timer_if_loaded || exit 79",
timer_snapshot,
)
restore_timer_file = recovery.index(
'restore_journal_file "/etc/systemd/system/'
'virt-cloud-wg-fabric-reconcile.timer" timer',
stop_transaction_timer,
)
daemon_reload = recovery.index(
'"$SYSTEMCTL" daemon-reload', restore_timer_file)
timer_present_branch = recovery.index(
'if [ "$journal_timer_before" = present ]; then', daemon_reload)
timer_restore = recovery.index(
'vc_restore_timer_activity "$journal_timer_active"',
timer_present_branch,
)
timer_not_found = recovery.index(
'[ "$journal_timer_load_state" = not-found ]', timer_restore)
timer_inactive = recovery.index(
'[ "$journal_timer_active_state" = inactive ]', timer_not_found)
absent_helper = recovery.index(
'if [ "$journal_helper_before" = absent ]; then', timer_inactive)
cleanup_state = recovery.index(
"journal_publish_cleanup_helper_state || exit 79", absent_helper)
restore_absent_helper = recovery.index(
'restore_journal_file "$HELPER" helper || exit 79', cleanup_state)
remove_journal = recovery.index(
'rm -rf "$JOURNAL" || exit 79', restore_absent_helper)
absent_exit = recovery.index("exit 0", remove_journal)
exec_helper = recovery.index('exec /bin/sh "$HELPER" "$@"', absent_exit)
assert helper_snapshot < timer_snapshot < stop_transaction_timer
assert stop_transaction_timer < restore_timer_file
assert restore_timer_file < daemon_reload
assert daemon_reload < timer_present_branch < timer_restore < timer_not_found
assert timer_not_found < timer_inactive < absent_helper < cleanup_state
assert cleanup_state < restore_absent_helper < remove_journal
assert remove_journal < absent_exit < exec_helper
present_helper_branch = recovery.index(
'if [ "$journal_helper_before" = present ]; then', timer_snapshot)
present_helper_restore = recovery.index(
'restore_journal_file "$HELPER" helper || exit 79',
present_helper_branch,
)
assert present_helper_branch < present_helper_restore < absent_helper
timer_stop_start = _BOOT_HELPER.index(
"vc_stop_transaction_timer_if_loaded() {")
timer_stop_end = _BOOT_HELPER.index("nft_panel_owned() {", timer_stop_start)
timer_stop = _BOOT_HELPER[timer_stop_start:timer_stop_end]
assert "not-found:inactive|loaded:inactive" in timer_stop
assert "loaded:active|loaded:activating|loaded:deactivating" in timer_stop
@pytest.mark.parametrize(
("load_state", "active_state", "sticky", "expected_rc", "stops"),
(
("not-found", "inactive", False, 0, False),
("loaded", "inactive", False, 0, False),
("loaded", "active", False, 0, True),
("loaded", "active", True, 1, True),
("not-found", "active", False, 1, False),
("loaded", "failed", False, 1, False),
),
)
def test_absent_timer_recovery_stops_only_loaded_transaction_unit(
tmp_path, load_state, active_state, sticky, expected_rc, stops):
function_start = _BOOT_HELPER.index(
"vc_stop_transaction_timer_if_loaded() {")
function_end = _BOOT_HELPER.index("nft_panel_owned() {", function_start)
function = _BOOT_HELPER[function_start:function_end]
load = tmp_path / "load"
active = tmp_path / "active"
log = tmp_path / "systemctl.log"
load.write_text(f"{load_state}\n", encoding="utf-8")
active.write_text(f"{active_state}\n", encoding="utf-8")
systemctl = tmp_path / "systemctl"
systemctl.write_text(
"#!/bin/sh\n"
"case \"$1:$2:$3\" in\n"
" 'show:-p:LoadState') cat \"$VC_TIMER_LOAD\" ;;\n"
" 'show:-p:ActiveState') cat \"$VC_TIMER_ACTIVE\" ;;\n"
" 'stop:virt-cloud-wg-fabric-reconcile.timer:')\n"
" printf '%s\\n' stop >> \"$VC_TIMER_LOG\"\n"
" [ \"$VC_TIMER_STICKY\" = yes ] || "
"printf '%s\\n' inactive > \"$VC_TIMER_ACTIVE\" ;;\n"
" *) exit 2 ;;\n"
"esac\n",
encoding="utf-8",
)
systemctl.chmod(0o700)
result = subprocess.run(
["/bin/sh", "-c", "\n".join((
"set -eu",
f"SYSTEMCTL='{systemctl}'",
function,
"vc_stop_transaction_timer_if_loaded",
))],
env={
"PATH": "/usr/bin:/bin",
"VC_TIMER_LOAD": str(load),
"VC_TIMER_ACTIVE": str(active),
"VC_TIMER_LOG": str(log),
"VC_TIMER_STICKY": "yes" if sticky else "no",
},
check=False, capture_output=True, text=True,
)
assert result.returncode == expected_rc, result.stderr
assert log.exists() is stops
def test_cleanup_helper_terminal_phase_is_repeatable_and_rejects_foreign_file(
tmp_path):
digest = "a" * 64
journal = tmp_path / "journal"
helper = tmp_path / "helper"
stat = tmp_path / "stat"
stat.write_text(
"#!/bin/sh\n"
"[ \"$1:$2\" = '-c:%u:%a' ] || exit 2\n"
"case \"$3\" in\n"
f" '{helper}') printf '%s\\n' 0:700 ;;\n"
" *) printf '%s\\n' 0:600 ;;\n"
"esac\n",
encoding="utf-8",
)
stat.chmod(0o700)
validator_start = _BOOT_HELPER.index("journal_cleanup_helper_owned() {")
validator_end = _BOOT_HELPER.index(
"journal_publish_cleanup_helper_state() {", validator_start)
validator = _BOOT_HELPER[validator_start:validator_end]
recover_start = _BOOT_HELPER.index("recover_journal() {")
branch_start = _BOOT_HELPER.index(" cleanup_helper)", recover_start)
branch_end = _BOOT_HELPER.index(" guarded)", branch_start)
cleanup_branch = _BOOT_HELPER[branch_start:branch_end]
harness = "\n".join((
"set -eu",
f"JOURNAL='{journal}'",
f"HELPER='{helper}'",
f"STAT='{stat}'",
"journal_file_secure() {",
" [ -f \"$JOURNAL/$1\" ] && [ ! -L \"$JOURNAL/$1\" ]",
"}",
"journal_file_matches() {",
" [ -f \"$1\" ] && [ ! -L \"$1\" ] &&",
" /usr/bin/cmp -s \"$1\" \"$JOURNAL/$2\"",
"}",
"journal_value() {",
" awk -F= -v wanted=\"$1\" '$1 == wanted {print substr($0, "
"index($0, \"=\") + 1); found=1} END {exit found ? 0 : 1}' "
"\"$JOURNAL/meta\"",
"}",
validator,
"case cleanup_helper in",
cleanup_branch,
"esac",
"exit 99",
))
def prepare(*, helper_state: str) -> None:
journal.mkdir()
(journal / "state").write_text("cleanup_helper\n", encoding="utf-8")
(journal / "meta").write_text(
f"schema=1\nplan_digest={digest}\n", encoding="utf-8")
(journal / "helper.absent").touch()
for path in (journal / "state", journal / "meta",
journal / "helper.absent"):
path.chmod(0o600)
(journal / "helper.desired").write_text(
"#!/bin/sh\n"
"# virt-cloud-wg-fabric-v1\n"
f"# fabric-plan-digest={digest}\n"
f'PLAN_DIGEST="{digest}"\n',
encoding="utf-8",
)
(journal / "helper.desired").chmod(0o700)
if helper_state != "absent":
actual_digest = digest if helper_state == "owned" else "b" * 64
helper.write_text(
"#!/bin/sh\n"
"# virt-cloud-wg-fabric-v1\n"
f"# fabric-plan-digest={actual_digest}\n"
f'PLAN_DIGEST="{actual_digest}"\n',
encoding="utf-8",
)
helper.chmod(0o700)
prepare(helper_state="owned")
owned = subprocess.run(
["/bin/sh", "-c", harness], check=False,
capture_output=True, text=True,
)
assert owned.returncode == 0, owned.stderr
assert helper.exists() is False
assert journal.exists() is False
prepare(helper_state="absent")
absent = subprocess.run(
["/bin/sh", "-c", harness], check=False,
capture_output=True, text=True,
)
assert absent.returncode == 0, absent.stderr
assert journal.exists() is False
prepare(helper_state="foreign")
foreign = subprocess.run(
["/bin/sh", "-c", harness], check=False,
capture_output=True, text=True,
)
assert foreign.returncode == 79
assert helper.exists() is True
assert journal.exists() is True
current = plan()
apply = _apply_script(
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
)
existing = apply.index('case "$journal_existing_state" in')
cleanup = apply.index("cleanup_helper)", existing)
direct_absent = apply.index(
'[ ! -e "$HELPER" ] && [ ! -L "$HELPER" ] || exit 79', cleanup)
ready = apply.index("ready)", direct_absent)
first_deploy_phase = apply.index(
'transaction_missing_installing_snapshot=no', ready)
special_first = apply.index(
'if [ "$transaction_missing_installing_snapshot" = yes ]; then',
first_deploy_phase,
)
temporary_recovery = apply.index(
'VC_FABRIC_LOCK_HELD=1 /bin/sh "$_tmp_helper" '
'--recover-installing || exit 79',
special_first,
)
fixed_helper_owned = apply.index(
'transaction_recovery_helper_owned || exit 79',
temporary_recovery,
)
fixed_recovery = apply.index(
'VC_FABRIC_LOCK_HELD=1 /bin/sh "$HELPER" --recover-only || exit 79',
fixed_helper_owned,
)
assert (
existing < cleanup < direct_absent < ready < first_deploy_phase
< special_first < temporary_recovery < fixed_helper_owned
< fixed_recovery
)
def test_transaction_journal_preparing_state_is_atomically_published():
current = plan()
apply = _apply_script(
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
)
existing = apply.index('if [ -e "$BASE/.wg-fabric-txn" ]; then')
state_required = apply.index(
'[ -f "$BASE/.wg-fabric-txn/state" ] &&', existing)
existing_case = apply.index('case "$journal_existing_state" in', state_required)
publish_start = apply.index(
'_tmp_txn_dir=$(mktemp -d '
'"$BASE/.wg-fabric-txn-preparing.XXXXXX")', existing_case)
preparing_state = apply.index(
'printf \'%s\\n\' preparing > "$_tmp_txn_dir/state"', publish_start)
preparing_meta = apply.index(
'cat > "$_tmp_txn_dir/meta"', preparing_state)
absence_proof = apply.index(
'[ ! -e "$TXN" ] && [ ! -L "$TXN" ] || exit 79', preparing_meta)
publish = apply.index('mv "$_tmp_txn_dir" "$TXN"', absence_proof)
readback = apply.index(
'[ "$(cat "$TXN/state" 2>/dev/null)" = preparing ]', publish)
first_snapshot = apply.index('snapshot_file "$CONF" conf', readback)
assert existing < state_required < existing_case < publish_start
assert publish_start < preparing_state < preparing_meta < absence_proof
assert absence_proof < publish < readback < first_snapshot
assert "firewall_mode=$FIREWALL_MODE" in apply[
preparing_meta:absence_proof
]
assert 'mkdir "$TXN"' not in apply
def test_observe_marks_any_fabric_transaction_journal_as_mismatch():
current = plan()
observe = _observe_script(
("100.64.1.2",), "100.64.1.1", ("203.0.113.2",),
firewall_mode=FIREWALL_MODE_NATIVE,
connection_uuid="12345678-1234-5678-1234-567812345678",
host_id="host-a", fabric_id=current.fabric_id,
overlay_cidr=current.overlay_cidr,
expected_digest=current.digest,
)
state = observe.index("firewall_mode_state=owned")
journal_path = observe.index(
"[ ! -e /etc/virt-cloud/.wg-fabric-txn ]", state)
journal_link = observe.index(
"[ ! -L /etc/virt-cloud/.wg-fabric-txn ]", journal_path)
mismatch = observe.index(
"|| firewall_mode_state=mismatch", journal_link)
marker = observe.index("__VC_FABRIC_FIREWALL_STATE__", mismatch)
assert state < journal_path < journal_link < mismatch < marker
def test_networkmanager_filename_resolution_uses_supported_unique_list_rows():
assert "GENERAL.FILENAME" not in _NMCLI_FILENAME_RESOLVER_SH
assert "GENERAL.FILENAME" not in _FIREWALL_MODE_GUARD_SH
assert "GENERAL.FILENAME" not in _BOOT_HELPER
assert (
'LC_ALL=C "$NMCLI" -t --escape no -f UUID,FILENAME'
in _NMCLI_FILENAME_RESOLVER_SH
)
# Resolve before and after semantic ownership checks to reject a filename
# switch during the readback transaction.
assert (
_FIREWALL_MODE_GUARD_SH.count(
'vc_nm_filename_for_uuid "$CONNECTION_UUID"') == 2
)
assert (
_BOOT_HELPER.count('vc_nm_filename_for_uuid "$vpc_nm_uuid"') == 1
)
current = plan()
apply = _apply_script(
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
firewall_mode=FIREWALL_MODE_NATIVE,
)
observe = _observe_script(
("100.64.1.2",), "100.64.1.1", ("203.0.113.2",),
firewall_mode=FIREWALL_MODE_NATIVE,
connection_uuid="12345678-1234-5678-1234-567812345678",
host_id="host-a", fabric_id=current.fabric_id,
overlay_cidr=current.overlay_cidr,
expected_digest=current.digest,
)
assert "__VC_NMCLI_FILENAME_RESOLVER__" not in apply
assert "GENERAL.FILENAME" not in apply
assert "GENERAL.FILENAME" not in observe
# Apply embeds one resolver for its own transaction and another in the
# separately installed boot helper; observe is one shell process.
assert apply.count("vc_nm_filename_for_uuid() {") == 2
assert observe.count("vc_nm_filename_for_uuid() {") == 1
assert "nm_vpc_field_raw=$(" in _BOOT_HELPER
assert (
'nm_vpc_value=$(nm_vpc_field "$nm_vpc_empty" uuid '
'"$vpc_nm_uuid") || exit 71' in _BOOT_HELPER
)
target = "11111111-1111-4111-8111-111111111111"
expected = (
"/etc/NetworkManager/system-connections/"
"bridge profile:primary.nmconnection"
)
harness = "\n".join((
"set -eu",
r'''
vc_test_nmcli() {
case "$*" in
"-t --escape no -f UUID,FILENAME connection show")
[ "${VC_NMCLI_FAIL:-no}" = no ] || return 3
printf '%s' "$VC_NMCLI_ROWS"
;;
*GENERAL.FILENAME*) return 97 ;;
*) return 98 ;;
esac
}
NMCLI=vc_test_nmcli
'''.strip(),
_NMCLI_FILENAME_RESOLVER_SH,
'vc_nm_filename_for_uuid "$VC_TARGET_UUID"',
))
def run(rows: str, *, uuid: str = target, fail: bool = False):
return subprocess.run(
["/bin/sh", "-c", harness],
env={
"PATH": "/usr/bin:/bin",
"VC_TARGET_UUID": uuid,
"VC_NMCLI_ROWS": rows,
"VC_NMCLI_FAIL": "yes" if fail else "no",
},
check=False, capture_output=True, text=True,
)
accepted = run(
"22222222-2222-4222-8222-222222222222:"
"/run/NetworkManager/system-connections/other.nmconnection\n"
f"{target}:{expected}\n"
)
assert accepted.returncode == 0, accepted.stderr
assert accepted.stdout == f"{expected}\n"
duplicate = run(f"{target}:{expected}\n{target}:{expected}\n")
assert duplicate.returncode != 0
missing = run(
"22222222-2222-4222-8222-222222222222:"
"/etc/NetworkManager/system-connections/other.nmconnection\n"
)
assert missing.returncode != 0
malformed_inventory = run(
f"{target}:{expected}\ncontinuation-without-a-uuid\n"
)
assert malformed_inventory.returncode != 0
oversized_row = run(f"{target}:{'x' * 4097}\n")
assert oversized_row.returncode != 0
malformed = run(f"{target}:{expected}\n", uuid="../not-a-uuid")
assert malformed.returncode != 0
command_failure = run(f"{target}:{expected}\n", fail=True)
assert command_failure.returncode != 0
def test_networkmanager_quarantine_keeps_owned_device_until_hard_link_down():
current = plan()
helper = _apply_script(
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
firewall_mode=FIREWALL_MODE_NATIVE,
)
start = helper.index("vc_fabric_nm_quarantine() {")
end = helper.index("\n}", start)
block = helper[start:end]
profile_read = block.index("vc_fabric_nm_profile_owned any || return 1")
unmanaged = block.index(
'"$NMCLI" device set "$IFACE" managed no', profile_read)
hard_down = block.index(
'"$IP" link set dev "$IFACE" down', unmanaged)
assert profile_read < unmanaged < hard_down
assert "firewall-cmd" not in block
assert "connection.zone" not in block
assert 'device reapply "$IFACE"' not in block
assert 'connection down uuid "$CONNECTION_UUID"' not in block
assert 'device disconnect "$IFACE"' not in block
def _write_firewall_guard_tools(tmp_path: Path) -> Path:
systemctl = tmp_path / "systemctl"
systemctl.write_text("""#!/bin/sh
case "$1:$2" in
is-active:nftables) printf '%s\\n' "$VC_NFT_ACTIVE" ;;
is-enabled:nftables) printf '%s\\n' "$VC_NFT_ENABLED" ;;
is-active:firewalld) printf '%s\\n' "$VC_FIREWALLD_ACTIVE" ;;
is-enabled:firewalld) printf '%s\\n' "$VC_FIREWALLD_ENABLED" ;;
show:-p)
case "$5" in
nftables) printf '%s\\n' "$VC_NFT_LOAD" ;;
firewalld) printf '%s\\n' "$VC_FIREWALLD_LOAD" ;;
*) exit 2 ;;
esac ;;
*) exit 2 ;;
esac
""", encoding="utf-8")
systemctl.chmod(0o700)
return systemctl
def _run_firewall_guard(
tmp_path: Path, mode: str, *, nft_active: str, nft_enabled: str,
nft_load: str, firewalld_active: str, firewalld_enabled: str,
firewalld_load: str, nft_readable: bool = True,
) -> subprocess.CompletedProcess[str]:
systemctl = _write_firewall_guard_tools(tmp_path)
script = "\n".join((
"set -eu",
f"SYSTEMCTL='{systemctl}'",
f"NFT={'/usr/bin/true' if nft_readable else '/usr/bin/false'}",
f"FIREWALL_MODE='{mode}'",
"find_tool() { for p in \"$@\"; do [ -x \"$p\" ] && { printf '%s' \"$p\"; return 0; }; done; return 1; }",
_FIREWALL_MODE_GUARD_SH,
"vc_firewall_mode_ready",
))
env = {
"PATH": "/usr/bin:/bin",
"VC_NFT_ACTIVE": nft_active,
"VC_NFT_ENABLED": nft_enabled,
"VC_NFT_LOAD": nft_load,
"VC_FIREWALLD_ACTIVE": firewalld_active,
"VC_FIREWALLD_ENABLED": firewalld_enabled,
"VC_FIREWALLD_LOAD": firewalld_load,
}
return subprocess.run(
["/bin/sh", "-c", script], env=env,
check=False, capture_output=True, text=True,
)
@pytest.mark.parametrize(("active", "enabled", "load"), (
("active", "enabled-runtime", "loaded"),
("inactive", "disabled", "loaded"),
("inactive", "masked", "masked"),
("unknown", "unknown", "not-found"),
))
def test_native_runtime_guard_ignores_stock_nftables_service_state(
tmp_path, active, enabled, load,
):
native = _run_firewall_guard(
tmp_path, FIREWALL_MODE_NATIVE,
nft_active=active, nft_enabled=enabled, nft_load=load,
firewalld_active="inactive", firewalld_enabled="masked",
firewalld_load="masked",
)
assert native.returncode == 0, native.stderr
def test_native_runtime_guard_requires_readable_nft_runtime(tmp_path):
native = _run_firewall_guard(
tmp_path, FIREWALL_MODE_NATIVE,
nft_active="inactive", nft_enabled="disabled", nft_load="loaded",
firewalld_active="inactive", firewalld_enabled="masked",
firewalld_load="masked", nft_readable=False,
)
assert native.returncode != 0
def test_native_runtime_guard_accepts_a_not_found_firewalld_unit(tmp_path):
result = _run_firewall_guard(
tmp_path, FIREWALL_MODE_NATIVE,
nft_active="active", nft_enabled="enabled", nft_load="loaded",
firewalld_active="unknown", firewalld_enabled="unknown",
firewalld_load="not-found",
)
assert result.returncode == 0, result.stderr
def test_native_runtime_guard_rejects_inactive_unmasked_firewalld(tmp_path):
result = _run_firewall_guard(
tmp_path, FIREWALL_MODE_NATIVE,
nft_active="active", nft_enabled="enabled", nft_load="loaded",
firewalld_active="inactive", firewalld_enabled="disabled",
firewalld_load="loaded",
)
assert result.returncode != 0
@pytest.mark.parametrize(("distribution", "target", "source"), [
("debian-13-iproute2-6.15", "203.0.113.123", "203.0.113.126"),
("almalinux-10-iproute-6.17", "198.51.100.126", "198.51.100.123"),
])
def test_underlay_route_path_accepts_real_explicit_from_output(
tmp_path, distribution, target, source,
):
fake_ip = tmp_path / "ip"
fake_ip.write_text(
"#!/bin/sh\n"
"if [ \"${5:-}\" = from ]; then\n"
" printf '%s from %s dev %s uid 0\\n cache\\n' "
"\"$4\" \"$VC_FROM_SRC\" \"$VC_FROM_DEV\"\n"
"else\n"
" printf '%s dev %s src %s uid 0\\n cache\\n' "
"\"$4\" \"$VC_PLAIN_DEV\" \"$VC_PLAIN_SRC\"\n"
"fi\n",
encoding="utf-8",
)
fake_ip.chmod(0o700)
script = "\n".join((
"set -eu",
f"IP='{fake_ip}'",
"find_tool() { for p in \"$@\"; do [ -x \"$p\" ] && { printf '%s' \"$p\"; return 0; }; done; return 1; }",
_FIREWALL_MODE_GUARD_SH,
f"vc_underlay_route_path {target} {source}",
))
base_env = {
"PATH": "/usr/bin:/bin",
"VC_PLAIN_DEV": "br0",
"VC_FROM_DEV": "br0",
"VC_FROM_SRC": source,
}
exact = subprocess.run(
["/bin/sh", "-c", script],
env={**base_env, "VC_PLAIN_SRC": source},
check=False, capture_output=True, text=True,
)
assert exact.returncode == 0, f"{distribution}: {exact.stderr}"
assert exact.stdout == f"br0|{source}"
@pytest.mark.parametrize(("plain_source", "explicit_source"), [
("10.20.30.40", "203.0.113.1"),
("203.0.113.1", "10.20.30.40"),
])
def test_underlay_route_path_rejects_inexact_plain_or_explicit_source(
tmp_path, plain_source, explicit_source,
):
fake_ip = tmp_path / "ip"
fake_ip.write_text(
"#!/bin/sh\n"
"if [ \"${5:-}\" = from ]; then\n"
" printf '%s from %s dev eth0 uid 0\\n cache\\n' "
"\"$4\" \"$VC_FROM_SRC\"\n"
"else\n"
" printf '%s dev eth0 src %s uid 0\\n cache\\n' "
"\"$4\" \"$VC_PLAIN_SRC\"\n"
"fi\n",
encoding="utf-8",
)
fake_ip.chmod(0o700)
script = "\n".join((
"set -eu",
f"IP='{fake_ip}'",
"find_tool() { for p in \"$@\"; do [ -x \"$p\" ] && { printf '%s' \"$p\"; return 0; }; done; return 1; }",
_FIREWALL_MODE_GUARD_SH,
"vc_underlay_route_path 203.0.113.2 203.0.113.1",
))
mismatch = subprocess.run(
["/bin/sh", "-c", script],
env={
"PATH": "/usr/bin:/bin",
"VC_PLAIN_SRC": plain_source,
"VC_FROM_SRC": explicit_source,
},
check=False, capture_output=True, text=True,
)
assert mismatch.returncode != 0
def _run_underlay_explicit_evidence(
tmp_path: Path, explicit_output: str,
) -> subprocess.CompletedProcess[str]:
fake_ip = tmp_path / "ip"
fake_ip.write_text(
"#!/bin/sh\n"
"if [ \"${5:-}\" = from ]; then\n"
" printf '%s\\n' \"$VC_EXPLICIT_OUTPUT\"\n"
"else\n"
" printf '%s\\n' '203.0.113.2 dev br0 src 203.0.113.1 uid 0'\n"
"fi\n",
encoding="utf-8",
)
fake_ip.chmod(0o700)
script = "\n".join((
"set -eu",
f"IP='{fake_ip}'",
"find_tool() { for p in \"$@\"; do [ -x \"$p\" ] && { printf '%s' \"$p\"; return 0; }; done; return 1; }",
_FIREWALL_MODE_GUARD_SH,
"vc_underlay_route_path 203.0.113.2 203.0.113.1",
))
return subprocess.run(
["/bin/sh", "-c", script],
env={
"PATH": "/usr/bin:/bin",
"VC_EXPLICIT_OUTPUT": explicit_output,
},
check=False, capture_output=True, text=True,
)
def test_underlay_route_path_accepts_matching_optional_selected_source(
tmp_path,
):
result = _run_underlay_explicit_evidence(
tmp_path,
"203.0.113.2 from 203.0.113.1 dev br0 "
"src 203.0.113.1 uid 0",
)
assert result.returncode == 0, result.stderr
assert result.stdout == "br0|203.0.113.1"
@pytest.mark.parametrize("explicit_output", [
"203.0.113.2 dev br0 uid 0",
"203.0.113.2 dev br0 src 203.0.113.1 uid 0",
"203.0.113.2 from 203.0.113.1 from 203.0.113.1 dev br0 uid 0",
(
"203.0.113.2 from 203.0.113.1 dev br0 "
"src 10.20.30.40 uid 0"
),
"203.0.113.2 from 203.0.113.1 dev br0 dev br0 uid 0",
"203.0.113.2 dev br0 src 203.0.113.1 from",
])
def test_underlay_route_path_rejects_missing_or_ambiguous_explicit_evidence(
tmp_path, explicit_output,
):
result = _run_underlay_explicit_evidence(tmp_path, explicit_output)
assert result.returncode != 0
def _run_native_apply_underlay_preflight(
tmp_path: Path, *, local_address: str = "203.0.113.1",
local_device: str = "eth0",
plain_device: str = "eth0", plain_source: str = "203.0.113.1",
from_device: str = "eth0", from_source: str = "203.0.113.1",
) -> subprocess.CompletedProcess[str]:
current = plan()
rendered = _apply_script(
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
firewall_mode=FIREWALL_MODE_NATIVE,
)
preflight_start = rendered.index("LOCAL_ENDPOINT=203.0.113.1")
preflight_end = rendered.index(
'if [ "$DEPLOYMENT_MODE" = networkmanager_nmcli ]; then',
preflight_start,
)
endpoint_start = rendered.index(
"# The UDP endpoint must leave through the host underlay",
preflight_end,
)
endpoint_end = rendered.index(
"# A desired overlay /32 may only be replaced", endpoint_start,
)
fake_ip = tmp_path / "ip"
fake_ip.write_text(
"#!/bin/sh\n"
"case \"$*\" in\n"
" '-o -4 address show up scope global')\n"
" printf '%s\\n' \"2: $VC_LOCAL_DEVICE inet $VC_LOCAL_ADDRESS/24 "
"scope global $VC_LOCAL_DEVICE\" ;;\n"
" '-4 route get 203.0.113.2')\n"
" printf '%s\\n' \"203.0.113.2 dev $VC_PLAIN_DEV "
"src $VC_PLAIN_SRC\" ;;\n"
" '-4 route get 203.0.113.2 from 203.0.113.1')\n"
" printf '%s\\n' \"203.0.113.2 from $VC_FROM_SRC "
"dev $VC_FROM_DEV uid 0\" \" cache\" ;;\n"
" *) exit 91 ;;\n"
"esac\n",
encoding="utf-8",
)
fake_ip.chmod(0o700)
shell = "\n".join((
"set -eu",
f"IP='{fake_ip}'",
"FIREWALL_MODE=native_nftables",
"IFACE=wg-fabric0",
"CONF=/nonexistent/virt-cloud-wg-fabric.conf",
"OLD_UNDERLAY=",
"OLD_FIREWALL_RULE_OWNER=",
_FIREWALL_MODE_GUARD_SH,
rendered[preflight_start:preflight_end],
rendered[endpoint_start:endpoint_end],
))
return subprocess.run(
["/bin/sh", "-c", shell],
env={
"PATH": "/usr/bin:/bin",
"VC_LOCAL_ADDRESS": local_address,
"VC_LOCAL_DEVICE": local_device,
"VC_PLAIN_DEV": plain_device,
"VC_PLAIN_SRC": plain_source,
"VC_FROM_DEV": from_device,
"VC_FROM_SRC": from_source,
},
check=False, capture_output=True, text=True,
)
def test_native_apply_uses_local_endpoint_without_firewalld_metadata(tmp_path):
result = _run_native_apply_underlay_preflight(tmp_path)
assert result.returncode == 0, result.stderr
current = plan()
rendered = _apply_script(
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
firewall_mode=FIREWALL_MODE_NATIVE,
)
assert "LOCAL_ENDPOINT=203.0.113.1" in rendered
assert 'vc_underlay_route_path 203.0.113.2 "$LOCAL_ENDPOINT"' in rendered
assert "underlay|local|" not in rendered
assert "firewall_rule|" not in rendered
@pytest.mark.parametrize(("kwargs", "expected_status"), [
({"local_address": "192.0.2.99"}, 71),
({"plain_source": "192.0.2.99"}, 77),
({"from_source": "192.0.2.99"}, 77),
({"from_device": "eth1"}, 77),
])
def test_native_apply_rejects_nonlocal_or_inexact_underlay_path(
tmp_path, kwargs, expected_status,
):
result = _run_native_apply_underlay_preflight(tmp_path, **kwargs)
assert result.returncode == expected_status, result.stderr
@pytest.mark.parametrize(
"interface", ["vnet3", "vcv-123", "tap3", "macvtap3", "fwpr3", "fwln3"],
)
def test_native_apply_rejects_guest_or_tenant_underlay_interface(
tmp_path, interface):
result = _run_native_apply_underlay_preflight(
tmp_path,
local_device=interface,
plain_device=interface,
from_device=interface,
)
assert result.returncode == 71, result.stderr
def test_fabric_nft_validator_rejects_unowned_rules_and_extra_objects():
digest = "a" * 64
rules = _nft_rules_text(digest)
def validate(candidate: str) -> subprocess.CompletedProcess[str]:
return subprocess.run(
["/bin/sh", "-c", "\n".join((
"set -eu",
_FABRIC_NFT_VALIDATION_SH,
'vc_fabric_nft_strict_shape "$VC_NFT_TEXT"',
))],
env={"PATH": "/usr/bin:/bin", "VC_NFT_TEXT": candidate},
check=False, capture_output=True, text=True,
)
accepted = validate(rules)
assert accepted.returncode == 0, accepted.stderr
with_tenants = rules.replace(
" flags interval;",
" flags interval;\n"
" elements = { 10.10.0.2, 172.16.4.9/32, 192.168.1.8 }",
1,
)
accepted_tenants = validate(with_tenants)
assert accepted_tenants.returncode == 0, accepted_tenants.stderr
early_accept = rules.replace(
' iifname "wg-fabric0" ip protocol icmp accept',
' meta iifname "wg-fabric0" accept\n'
' iifname "wg-fabric0" ip protocol icmp accept',
1,
)
impossible_drop = rules.replace(
' iifname "wg-fabric0" counter drop',
' iifname "foreign0" iifname "wg-fabric0" counter drop',
1,
)
extra_chain = rules.replace(
" chain input {",
" chain bypass {\n type filter hook input priority -101; "
"policy accept;\n }\n chain input {",
1,
)
extra_set = rules.replace(
" set vpc_tenant_ips {",
" set foreign_ips { type ipv4_addr; }\n"
" set vpc_tenant_ips {",
1,
)
wrong_set_type = rules.replace("type ipv4_addr;", "type inet_service;", 1)
wrong_set_flags = rules.replace("flags interval;", "flags timeout;", 1)
extra_object = rules.replace(
" chain input {",
" counter foreign_counter { }\n chain input {",
1,
)
non_private_element = rules.replace(
" flags interval;",
" flags interval;\n elements = { 0.0.0.0/0 }",
1,
)
for tampered in (
early_accept, impossible_drop, extra_chain, extra_set,
wrong_set_type, wrong_set_flags, extra_object, non_private_element,
):
assert validate(tampered).returncode != 0
def test_observe_nft_guard_matches_exact_rendered_owner_comments():
current = plan()
observe = _observe_script(
(),
deployment_mode="ifupdown",
firewall_mode=FIREWALL_MODE_NATIVE,
expected_digest=current.digest,
)
owner_regex = (
f"""grep -Ec 'comment "{FABRIC_OWNER_MARKER}:[0-9a-f]{{64}}"'"""
)
digest_literal = (
f'grep -Foc "comment \\"{FABRIC_OWNER_MARKER}:$config_digest\\""'
)
assert owner_regex in observe
assert digest_literal in observe
guard_start = observe.index(
'if [ -n "$NFT" ]; then', observe.index("nft_guard=missing")
)
guard_end = observe.index("emergency_state=unknown", guard_start)
guard = observe[guard_start:guard_end]
program = "\n".join((
"set -eu",
_FABRIC_NFT_VALIDATION_SH,
"vc_test_nft() {",
' [ "$#" = 5 ] || return 1',
' [ "$1" = -s ] && [ "$2" = list ] && [ "$3" = table ] &&',
' [ "$4" = inet ] && [ "$5" = virt_cloud_wg_fabric ] || return 1',
' printf \'%s\\n\' "$VC_NFT_TEXT"',
"}",
"NFT=vc_test_nft",
f"config_digest={shlex.quote(current.digest)}",
"nft_guard=missing",
guard,
'printf \'%s\\n\' "$nft_guard"',
))
def readback(candidate):
return subprocess.run(
["/bin/sh", "-c", program],
env={"PATH": "/usr/bin:/bin", "VC_NFT_TEXT": candidate},
check=False,
capture_output=True,
text=True,
)
owned = readback(_nft_rules_text(current.digest))
assert owned.returncode == 0, owned.stderr
assert owned.stdout == "owned\n"
stale = readback(_nft_rules_text("0" * 64))
assert stale.returncode == 0, stale.stderr
assert stale.stdout == "missing\n"
def test_fabric_nft_validator_accepts_alma_output_dstnat_priority_only():
digest = "a" * 64
rules = _nft_rules_text(digest)
def validate(candidate: str) -> subprocess.CompletedProcess[str]:
return subprocess.run(
["/bin/sh", "-c", "\n".join((
"set -eu",
_FABRIC_NFT_VALIDATION_SH,
'vc_fabric_nft_strict_shape "$VC_NFT_TEXT"',
))],
env={"PATH": "/usr/bin:/bin", "VC_NFT_TEXT": candidate},
check=False, capture_output=True, text=True,
)
alma_canonical = rules.replace(
"type filter hook output priority -100; policy accept;",
"type filter hook output priority dstnat; policy accept;",
1,
)
accepted = validate(alma_canonical)
assert accepted.returncode == 0, accepted.stderr
for hook in ("input", "forward"):
invalid = rules.replace(
f"type filter hook {hook} priority -100; policy accept;",
f"type filter hook {hook} priority dstnat; policy accept;",
1,
)
assert validate(invalid).returncode != 0
def test_boot_recovery_accepts_alma_output_dstnat_priority():
digest = "a" * 64
alma_canonical = _nft_rules_text(digest).replace(
"type filter hook output priority -100; policy accept;",
"type filter hook output priority dstnat; policy accept;",
1,
)
helper = _BOOT_HELPER.replace(
"__VC_FABRIC_NFT_VALIDATION__", _FABRIC_NFT_VALIDATION_SH,
)
function_start = helper.index("vc_fabric_nft_strict_shape() {")
function_end = helper.index("\n}", function_start) + 2
validator = helper[function_start:function_end]
result = subprocess.run(
["/bin/sh", "-c", "\n".join((
"set -eu",
validator,
'vc_fabric_nft_strict_shape "$VC_NFT_TEXT"',
))],
env={"PATH": "/usr/bin:/bin", "VC_NFT_TEXT": alma_canonical},
check=False, capture_output=True, text=True,
)
assert result.returncode == 0, result.stderr
def _bridge_port_snapshot_runner(tmp_path, mode):
start = _BOOT_HELPER.index("vc_bridge_port_records() {")
end = _BOOT_HELPER.index("\n}\nvc_vpc_emergency_down()", start) + 2
runtime = tmp_path / "run"
runtime.mkdir()
function = _BOOT_HELPER[start:end].replace(
"/run/virt-cloud", str(runtime),
)
bridge = tmp_path / "bridge"
bridge.write_text(
"""#!/bin/sh
[ "$*" = "-o link show master vcv-0123456789a" ] || exit 83
case "$VC_BRIDGE_MODE" in
failed) exit 1 ;;
malformed) printf '%s\n' 'not-a-bridge-link-record' ;;
duplicate)
printf '%s\n' '2: tap0@if3: <BROADCAST> master vcv-0123456789a'
printf '%s\n' '4: tap0@if5: <BROADCAST> master vcv-0123456789a' ;;
excessive)
i=0
while [ "$i" -lt 4097 ]; do
printf '%s: tap%s: <BROADCAST> master vcv-0123456789a\n' "$i" "$i"
i=$((i + 1))
done ;;
valid)
printf '%s\n' '4: vnet2@if5: <BROADCAST> master vcv-0123456789a'
printf '%s\n' '2: tap1@if3: <BROADCAST> master vcv-0123456789a' ;;
*) exit 2 ;;
esac
""",
encoding="utf-8",
)
bridge.chmod(0o700)
stat = tmp_path / "stat"
stat.write_text(
"#!/bin/sh\n/usr/bin/stat -f '%z' \"$3\"\n",
encoding="utf-8",
)
stat.chmod(0o700)
script = "\n".join((
"set -eu",
f"IP='{bridge}'",
f"STAT='{stat}'",
"AWK=/usr/bin/awk",
function,
"vc_bridge_port_names vcv-0123456789a",
))
result = subprocess.run(
["/bin/sh", "-c", script],
env={"PATH": "/usr/bin:/bin", "VC_BRIDGE_MODE": mode},
check=False, capture_output=True, text=True,
)
return result, runtime, function
@pytest.mark.parametrize(
"mode", ("failed", "malformed", "duplicate", "excessive"),
)
def test_bridge_port_snapshot_fails_closed_on_command_or_parse_errors(
tmp_path, mode,
):
result, runtime, _ = _bridge_port_snapshot_runner(tmp_path, mode)
assert result.returncode == 1, result.stderr
assert list(runtime.iterdir()) == []
def test_bridge_port_snapshot_sorts_valid_unique_names(tmp_path):
result, runtime, function = _bridge_port_snapshot_runner(
tmp_path, "valid",
)
assert result.returncode == 0, result.stderr
assert result.stdout == "tap1\nvnet2\n"
assert list(runtime.iterdir()) == []
assert 'vc_bridge_port_records "$vc_bridge_ports_bridge"' in function
assert '"$BRIDGE_CMD" link show master "$vc_bridge_ports_bridge" |' \
not in _BOOT_HELPER
def _vpc_force_down_runner(tmp_path, *, link_up, nm_fails=False):
start = _BOOT_HELPER.index("vc_link_down_proven() {")
end = _BOOT_HELPER.index("\n}\nvc_vpc_emergency_down()", start) + 2
functions = _BOOT_HELPER[start:end]
ip = tmp_path / "ip"
ip.write_text(
"""#!/bin/sh
printf '%s\n' "$*" >> "$VC_IP_LOG"
case "$*" in
'link set dev vcv-0123456789a down') exit 0 ;;
'-o link show dev vcv-0123456789a')
if [ "$VC_LINK_UP" = yes ]; then
printf '%s\n' '2: vcv-0123456789a: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500'
else
printf '%s\n' '2: vcv-0123456789a: <BROADCAST,MULTICAST> mtu 1500'
fi
exit 0 ;;
*) exit 2 ;;
esac
""",
encoding="utf-8",
)
ip.chmod(0o700)
nmcli = tmp_path / "nmcli"
nmcli.write_text("#!/bin/sh\nexit 1\n", encoding="utf-8")
nmcli.chmod(0o700)
log = tmp_path / "ip.log"
body = [
"set -eu",
f"IP='{ip}'",
f"NMCLI='{nmcli}'",
"AWK=/usr/bin/awk",
functions,
]
if nm_fails:
body.extend((
"nm_failed=no",
'"$NMCLI" device show vcv-0123456789a >/dev/null 2>&1 || '
"nm_failed=yes",
"vc_vpc_bridge_force_down vcv-0123456789a",
'[ "$nm_failed" = yes ]',
))
else:
body.append("vc_vpc_bridge_force_down vcv-0123456789a")
result = subprocess.run(
["/bin/sh", "-c", "\n".join(body)],
env={
"PATH": "/usr/bin:/bin",
"VC_IP_LOG": str(log),
"VC_LINK_UP": "yes" if link_up else "no",
},
check=False, capture_output=True, text=True,
)
return result, log
def test_vpc_force_down_rejects_nm_race_that_restores_up_flag(tmp_path):
result, log = _vpc_force_down_runner(tmp_path, link_up=True)
assert result.returncode == 1, result.stderr
assert log.read_text(encoding="utf-8").splitlines() == [
"link set dev vcv-0123456789a down",
"-o link show dev vcv-0123456789a",
]
def test_vpc_force_down_still_runs_and_proves_isolation_after_nm_failure(
tmp_path,
):
result, log = _vpc_force_down_runner(
tmp_path, link_up=False, nm_fails=True,
)
assert result.returncode == 0, result.stderr
assert log.read_text(encoding="utf-8").splitlines() == [
"link set dev vcv-0123456789a down",
"-o link show dev vcv-0123456789a",
]
early = _BOOT_HELPER.index("vc_vpc_emergency_managed=")
early_down = _BOOT_HELPER.index(
'vc_vpc_bridge_force_down "$vc_vpc_emergency_bridge"', early,
)
early_after = _BOOT_HELPER.index(
"vc_vpc_emergency_ports_after=", early_down,
)
assert early < early_down < early_after
def test_apply_queues_reconcile_dispatcher_only_after_durable_commit():
current = plan()
script = _apply_script(
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
firewall_mode=FIREWALL_MODE_NATIVE,
)
committed_marker = script.rindex('committed > "$_txn_state_tmp"')
committed_flag = script.rindex("COMMITTED=yes")
unlock = script.rindex("exec 9>&-")
queue = script.rindex(
'"$SYSTEMCTL" start '
'virt-cloud-wg-fabric-reconcile.service'
)
relock = script.rindex('exec 9>>"$LOCK"')
primary_readback = script.rindex(
'is-active virt-cloud-wg-fabric.service 2>/dev/null')
emergency_readback = script.rindex(
'final_emergency_tables_verify=$($NFT list tables')
success = script.rindex("__VC_FABRIC_APPLIED__")
assert (
committed_marker < committed_flag < unlock < queue < relock
< primary_readback < emergency_readback < success
)
assert "Type=oneshot" in _UNIT
assert "RemainAfterExit=yes" in _UNIT
assert (
'"$SYSTEMCTL" start virt-cloud-wg-fabric.service'
not in script[committed_flag:]
)
assert "vc_primary_inactive_retry_ready()" in _BOOT_HELPER
assert "retry_primary_inactive=yes" in _BOOT_HELPER
primary = _BOOT_HELPER.index(
'"$SYSTEMCTL" start virt-cloud-wg-fabric.service')
assert primary > 0
assert 'vc_gate_retry_start_claimed' not in _BOOT_HELPER
assert 'start libvirtd.service' not in _BOOT_HELPER
def _run_apply_convergence_tail(
tmp_path: Path, *, start_fails: bool = False,
timer_fails: bool = False,
stale_config: bool = False, emergency_present: bool = False,
committed_trap_state: str | None = None,
) -> subprocess.CompletedProcess[str]:
current = plan()
rendered = _apply_script(
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
firewall_mode=FIREWALL_MODE_NATIVE,
)
tail_start = rendered.rindex("COMMITTED=yes")
marker = "printf '%s\\n' '__VC_FABRIC_APPLIED__'"
tail_end = rendered.rindex(marker) + len(marker)
rollback_start = rendered.index("rollback() {")
rollback_end = rendered.index(
"\n}\ntrap rollback EXIT HUP INT TERM", rollback_start,
) + 2
rollback_function = rendered[rollback_start:rollback_end]
logger_start = rendered.index("vc_fabric_apply_failure_log() {")
logger_end = rendered.index("\n}\n", logger_start) + 2
failure_logger = rendered[logger_start:logger_end]
emergency_owner = tmp_path / "emergency.owner"
sysctl_file = tmp_path / "fabric-sysctl.conf"
sysctl_file.write_text(_sysctl_text(current), encoding="utf-8")
sysctl_file.chmod(0o644)
rp_filter_file = tmp_path / "rp_filter"
rp_filter_file.write_text("2\n", encoding="utf-8")
tail = rendered[tail_start:tail_end].replace(
"/run/virt-cloud/wg-fabric-emergency.owner",
str(emergency_owner),
).replace(
'"/proc/sys/net/ipv4/conf/$IFACE/rp_filter"',
f"'{rp_filter_file}'",
)
lock = tmp_path / "fabric.lock"
lock.touch(mode=0o600)
conf = tmp_path / "fabric.conf"
conf.write_text(
f"digest|{'0' * 64 if stale_config else current.digest}\n"
f"firewall|{FIREWALL_MODE_NATIVE}\n",
encoding="utf-8",
)
conf.chmod(0o600)
key = tmp_path / "fabric.key"
key.write_text("private\n", encoding="utf-8")
converged = tmp_path / "converged"
systemctl = tmp_path / "systemctl"
systemctl.write_text(
"""#!/bin/sh
case "$1:$2" in
start:virt-cloud-wg-fabric-reconcile.timer)
[ "${VC_TEST_TIMER_FAIL:-no}" = no ] || exit 1
exit 0 ;;
start:virt-cloud-wg-fabric-reconcile.service)
[ "${VC_TEST_START_FAIL:-no}" = no ] || exit 1
: > "$VC_TEST_CONVERGED"; exit 0 ;;
is-active:virt-cloud-wg-fabric.service|\
is-active:virt-cloud-wg-fabric-reconcile.timer)
[ -f "$VC_TEST_CONVERGED" ] || exit 1
printf '%s\n' active; exit 0 ;;
*) exit 2 ;;
esac
""",
encoding="utf-8",
)
systemctl.chmod(0o700)
nft = tmp_path / "nft"
nft.write_text(
"""#!/bin/sh
case "$*" in
'-s list table inet virt_cloud_wg_fabric')
printf '%s\n' managed-fabric-table ;;
'list tables')
if [ "${VC_TEST_EMERGENCY:-no}" = yes ]; then
printf '%s\n' 'table inet virt_cloud_fabric_emergency'
printf '%s\n' 'table bridge virt_cloud_fabric_emergency'
else
printf '%s\n' 'table inet virt_cloud_wg_fabric'
fi ;;
*) exit 2 ;;
esac
""",
encoding="utf-8",
)
nft.chmod(0o700)
wg = tmp_path / "wg"
wg.write_text(
"""#!/bin/sh
case "$*" in
'show wg-fabric0') exit 0 ;;
'pubkey') printf '%s\n' mock-public-key ;;
'show wg-fabric0 public-key') printf '%s\n' mock-public-key ;;
'show wg-fabric0 listen-port') printf '%s\n' 51820 ;;
*) exit 2 ;;
esac
""",
encoding="utf-8",
)
wg.chmod(0o700)
ip = tmp_path / "ip"
ip.write_text(
"#!/bin/sh\n"
"[ \"$*\" = '-o link show dev wg-fabric0' ] || exit 2\n"
"printf '%s\\n' '1: wg-fabric0: <UP,LOWER_UP> mtu 1420'\n",
encoding="utf-8",
)
ip.chmod(0o700)
stat = tmp_path / "stat"
stat.write_text("#!/bin/sh\nprintf '%s\\n' '0:600'\n", encoding="utf-8")
stat.chmod(0o700)
flock = tmp_path / "flock"
flock.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8")
flock.chmod(0o700)
trap_setup: tuple[str, ...] = ()
if committed_trap_state is not None:
journal = tmp_path / "txn"
journal.mkdir()
(journal / "state").write_text(
f"{committed_trap_state}\n", encoding="utf-8",
)
trap_setup = (
f"TXN='{journal}'",
"JOURNAL_READY=yes",
"cleanup() { :; }",
rollback_function,
"trap rollback EXIT HUP INT TERM",
)
harness = "\n".join((
"set -eu",
f"LOCK='{lock}'",
f"CONF='{conf}'",
f"KEY='{key}'",
f"SYSCTL_FILE='{sysctl_file}'",
"IFACE=wg-fabric0",
f"EXPECTED_DIGEST='{current.digest}'",
f"FIREWALL_MODE='{FIREWALL_MODE_NATIVE}'",
f"SYSTEMCTL='{systemctl}'",
f"NFT='{nft}'",
f"WG='{wg}'",
f"IP='{ip}'",
f"STAT='{stat}'",
f"FLOCK='{flock}'",
"nft_plan_owned() { [ \"$2\" = \"$EXPECTED_DIGEST\" ]; }",
"vpc_state_nft_set_matches() { return 0; }",
"vc_fabric_sysctl_file_owned() { return 0; }",
"read_sysctl() { cat \"$1\"; }",
"vc_fabric_operation_checkpoint() { :; }",
failure_logger,
*trap_setup,
"exec 9>>\"$LOCK\"",
tail,
))
return subprocess.run(
["/bin/sh", "-c", harness], check=False,
capture_output=True, text=True,
env={
"PATH": "/usr/bin:/bin",
"VC_TEST_START_FAIL": "yes" if start_fails else "no",
"VC_TEST_TIMER_FAIL": "yes" if timer_fails else "no",
"VC_TEST_EMERGENCY": "yes" if emergency_present else "no",
"VC_TEST_CONVERGED": str(converged),
},
)
def test_apply_convergence_tail_requires_sync_success_and_final_clear(tmp_path):
success = _run_apply_convergence_tail(tmp_path)
assert success.returncode == 0, success.stderr
assert "__VC_FABRIC_APPLIED__" in success.stdout
@pytest.mark.parametrize(
"failure",
["start", "stale_config", "emergency"],
)
def test_apply_convergence_tail_refuses_unconfirmed_state(tmp_path, failure):
result = _run_apply_convergence_tail(
tmp_path,
start_fails=failure == "start",
stale_config=failure == "stale_config",
emergency_present=failure == "emergency",
)
assert result.returncode == 79, result.stderr
assert "__VC_FABRIC_APPLIED__" not in result.stdout
def test_post_commit_dispatcher_failure_preserves_committed_journal(tmp_path):
result = _run_apply_convergence_tail(
tmp_path, start_fails=True, committed_trap_state="committed",
)
assert result.returncode == 79, result.stderr
assert (tmp_path / "txn" / "state").read_text(
encoding="utf-8",
) == "committed\n"
def test_post_commit_timer_failure_preserves_committed_journal(tmp_path):
result = _run_apply_convergence_tail(
tmp_path, timer_fails=True, committed_trap_state="committed",
)
assert result.returncode == 79, result.stderr
assert (tmp_path / "txn" / "state").read_text(
encoding="utf-8",
) == "committed\n"
def test_old_committed_trap_never_deletes_concurrent_ready_journal(tmp_path):
result = _run_apply_convergence_tail(
tmp_path, start_fails=True, committed_trap_state="ready",
)
assert result.returncode == 79, result.stderr
assert (tmp_path / "txn" / "state").read_text(
encoding="utf-8",
) == "ready\n"
def test_inactive_primary_retry_requires_complete_managed_evidence(tmp_path):
start = _BOOT_HELPER.index("vc_primary_inactive_retry_ready() {")
end = _BOOT_HELPER.index("\n}\nvc_fabric_rpf_repair_needed()", start) + 2
function = _BOOT_HELPER[start:end]
digest = "a" * 64
conf = tmp_path / "wg-fabric0.conf"
conf.write_text(
f"digest|{digest}\nfirewall|native_nftables\n", encoding="utf-8",
)
owner = tmp_path / "wg-fabric0.owner"
owner.write_text("virt-cloud-wg-fabric-v1\n", encoding="utf-8")
helper = tmp_path / "helper"
helper.write_text(
f"#!/bin/sh\n# virt-cloud-wg-fabric-v1\n"
f"# fabric-plan-digest={digest}\n",
encoding="utf-8",
)
unit = tmp_path / "unit"
unit.write_text(
"\n".join((
"# virt-cloud-wg-fabric-v1",
f"# fabric-plan-digest={digest}",
"Type=oneshot",
"RemainAfterExit=yes",
"ExecStart=/bin/sh /usr/local/libexec/virt-cloud-wg-fabric",
)) + "\n",
encoding="utf-8",
)
stat = tmp_path / "stat"
stat.write_text(
"""#!/bin/sh
case "$3" in
*wg-fabric0.conf|*wg-fabric0.owner) printf '%s\n' 0:600 ;;
*helper) printf '%s\n' 0:700 ;;
*unit) printf '%s\n' 0:644 ;;
*) exit 1 ;;
esac
""",
encoding="utf-8",
)
stat.chmod(0o700)
systemctl = tmp_path / "systemctl"
systemctl.write_text(
"""#!/bin/sh
case "$1" in
is-active) printf '%s\n' inactive ;;
is-enabled) printf '%s\n' enabled ;;
*) exit 2 ;;
esac
""",
encoding="utf-8",
)
systemctl.chmod(0o700)
wg = tmp_path / "wg"
wg.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8")
wg.chmod(0o700)
base_script = "\n".join((
"set -eu",
f"SYSTEMCTL='{systemctl}'",
f"STAT='{stat}'",
f"WG='{wg}'",
f"CONF='{conf}'",
f"OWNER='{owner}'",
f"HELPER='{helper}'",
f"UNIT='{unit}'",
f"PLAN_DIGEST='{digest}'",
"FIREWALL_MODE=native_nftables",
"IFACE=wg-fabric0",
function,
"vc_primary_inactive_retry_ready",
))
accepted = subprocess.run(
["/bin/sh", "-c", base_script], check=False,
capture_output=True, text=True,
)
assert accepted.returncode == 0, accepted.stderr
unit.write_text(unit.read_text(encoding="utf-8").replace(
"RemainAfterExit=yes", "RemainAfterExit=no",
), encoding="utf-8")
rejected = subprocess.run(
["/bin/sh", "-c", base_script], check=False,
capture_output=True, text=True,
)
assert rejected.returncode == 1
def test_fabric_nft_journal_records_only_post_refresh_as_applied():
current = plan()
script = _apply_script(
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
firewall_mode=FIREWALL_MODE_NATIVE,
)
installing = script.index("write_nft_phase installing")
installed_snapshot = script.index(
'"$nft_installed" > "$TXN/nft.installed.tmp"',
installing,
)
installed = script.index("write_nft_phase installed", installed_snapshot)
refreshing = script.index("write_nft_phase refreshing", installed)
refresh = script.index("vpc_state_refresh_nft || exit 71", refreshing)
exact_set = script.index("vpc_state_nft_set_matches || exit 71", refresh)
applied_snapshot = script.index(
'"$fabric_acl_applied" > "$TXN/nft.applied.tmp"',
exact_set,
)
refreshed = script.index("write_nft_phase refreshed", applied_snapshot)
assert (
installing < installed_snapshot < installed < refreshing < refresh
< exact_set < applied_snapshot < refreshed
)
assert "before_mutation|installing|installed|refreshing|refreshed" in script
assert 'case "$rollback_nft_phase" in' in script
assert 'vpc_state_nft_set_matches || ROLLBACK_FAILED=yes' in script
def test_boot_recovery_handles_both_sides_of_nft_refresh_fail_closed():
helper = _BOOT_HELPER
phase_read = helper.index(
'journal_nft_phase=$(cat "$JOURNAL/nft.phase"')
pre_refresh_snapshot = helper.index(
'journal_file_secure nft.installed || exit 79', phase_read)
refreshing_case = helper.index("refreshing)", pre_refresh_snapshot)
full_shape = helper.index(
'nft_plan_owned "$journal_refresh_text" "$journal_plan_digest"',
refreshing_case,
)
exact_set = helper.index("vpc_state_nft_set_matches || exit 79", full_shape)
stable_recheck = helper.index(
'"$CMP" -s "$JOURNAL/nft.current" '
'"$JOURNAL/nft.refresh-verify"',
exact_set,
)
final_snapshot = helper.index(
'"$CMP" -s "$JOURNAL/nft.current" "$JOURNAL/nft.applied"',
stable_recheck,
)
destructive_batch = helper.index(
'delete table inet virt_cloud_wg_fabric', final_snapshot)
assert (
phase_read < pre_refresh_snapshot < refreshing_case < full_shape
< exact_set < stable_recheck < final_snapshot < destructive_batch
)
assert (
"Crash between nft commit and its complete snapshot is\n"
" # intentionally ambiguous"
) in helper
assert (
'elif [ "$journal_nft_phase" = refreshing ] &&\n'
' { [ -e "$JOURNAL/nft.applied" ] || '
'[ -L "$JOURNAL/nft.applied" ]; }; then'
) in helper
assert (
'"$CMP" -s "$JOURNAL/nft.current" '
'"$JOURNAL/nft.applied" || exit 79'
) in helper
def test_first_deploy_recovery_uses_only_fenced_temporary_helper():
current = plan()
script = _apply_script(
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
firewall_mode=FIREWALL_MODE_NATIVE,
)
helper_created = script.index(
'_tmp_helper=$(mktemp "$BASE/.wg-fabric0.helper.XXXXXX")')
transaction = script.index("COMMITTED=no", helper_created)
ready_case = script.index(" ready)", transaction)
ready_end = script.index(" *) exit 79 ;;", ready_case)
helper_published = script.index(
'publish_desired_file "$_tmp_helper" "$HELPER" helper', ready_end)
ready_contract = script[ready_case:ready_end]
first_deploy = ready_contract.index(
'transaction_first_deploy_helper_absent=no')
missing_snapshot = ready_contract.index(
'transaction_missing_installing_snapshot=no')
special_first = ready_contract.index(
'if [ "$transaction_first_deploy_helper_absent" = yes ]; then',
missing_snapshot,
)
temporary_installing = ready_contract.index(
'VC_FABRIC_LOCK_HELD=1 /bin/sh "$_tmp_helper" '
'--recover-installing || exit 79',
special_first,
)
temporary_current = ready_contract.index(
'VC_FABRIC_LOCK_HELD=1 /bin/sh "$_tmp_helper" '
'--recover-current || exit 79',
temporary_installing,
)
fixed_helper_owned = ready_contract.index(
'transaction_recovery_helper_owned || exit 79',
temporary_current,
)
fixed_recovery = ready_contract.index(
'VC_FABRIC_LOCK_HELD=1 /bin/sh "$HELPER" --recover-only || exit 79',
fixed_helper_owned,
)
assert helper_created < transaction < ready_case < helper_published
assert (
first_deploy < missing_snapshot < special_first < temporary_installing
< temporary_current
< fixed_helper_owned < fixed_recovery
)
assert (
'VC_FABRIC_LOCK_HELD=1 /bin/sh "$HELPER" --recover-only || exit 79'
in ready_contract
)
assert (
'VC_FABRIC_LOCK_HELD=1 /bin/sh "$_tmp_helper" '
'--recover-installing || exit 79'
) in ready_contract
assert (
'VC_FABRIC_LOCK_HELD=1 /bin/sh "$_tmp_helper" '
'--recover-current || exit 79'
) in ready_contract
assert '"$_tmp_helper" --recover-only' not in ready_contract
assert '"$HELPER" --recover-installing' not in ready_contract
assert '"$HELPER" --recover-current' not in ready_contract
def _run_transaction_recovery_helper_owned_harness(
tmp_path, *, scenario,
):
current = plan()
apply = _apply_script(
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
firewall_mode=FIREWALL_MODE_NATIVE,
)
function_start = apply.index("transaction_recovery_helper_owned() {")
function_end = apply.index(
'\n}\nif [ -L "$BASE" ]', function_start,
) + 2
function = apply[function_start:function_end]
base = tmp_path / "base"
journal = base / ".wg-fabric-txn"
journal.mkdir(parents=True)
helper = tmp_path / "fixed-helper"
owned = b"#!/bin/sh\n# owned helper\n"
fixed = owned if scenario != "foreign_fixed_helper" else (
b"#!/bin/sh\n# foreign helper\n"
)
helper.write_bytes(fixed)
helper.chmod(0o700)
if scenario in ("baseline_exact", "foreign_fixed_helper",
"baseline_marker_drift"):
marker = journal / "helper.present"
marker.write_bytes(
b"drift\n" if scenario == "baseline_marker_drift" else b"",
)
marker.chmod(0o600)
baseline = journal / "helper"
baseline.write_bytes(owned)
baseline.chmod(0o700)
elif scenario in ("desired_exact", "desired_metadata_drift"):
desired = journal / "helper.desired"
desired.write_bytes(owned)
desired.chmod(0o700)
if scenario == "desired_metadata_drift":
marker = journal / "helper.desired.absent"
marker.write_bytes(b"")
marker.chmod(0o600)
else:
raise AssertionError(f"unknown helper scenario: {scenario}")
fake_stat = tmp_path / "stat"
fake_stat.write_text(
"""#!/bin/sh
case "$3" in
*/helper.present) printf '0:600\n' ;;
*/helper|*/helper.desired|*/fixed-helper) printf '0:700\n' ;;
*) exit 2 ;;
esac
""",
encoding="utf-8",
)
fake_stat.chmod(0o700)
script = "\n".join((
"set -eu",
f"BASE={shlex.quote(str(base))}",
f"HELPER={shlex.quote(str(helper))}",
f"STAT={shlex.quote(str(fake_stat))}",
"CMP=/usr/bin/cmp",
function,
"if transaction_recovery_helper_owned; then",
" exit 0",
"fi",
"exit 17",
))
return subprocess.run(
["/bin/sh", "-c", script], check=False,
capture_output=True, text=True,
)
@pytest.mark.parametrize("scenario", ("baseline_exact", "desired_exact"))
def test_transaction_recovery_fixed_helper_accepts_exact_journal_image(
tmp_path, scenario,
):
result = _run_transaction_recovery_helper_owned_harness(
tmp_path, scenario=scenario,
)
assert result.returncode == 0, result.stderr
@pytest.mark.parametrize("scenario", (
"foreign_fixed_helper",
"baseline_marker_drift",
"desired_metadata_drift",
))
def test_transaction_recovery_fixed_helper_rejects_unproven_image(
tmp_path, scenario,
):
result = _run_transaction_recovery_helper_owned_harness(
tmp_path, scenario=scenario,
)
assert result.returncode == 17
def _run_ready_recovery_dispatch_harness(
tmp_path, *, first_deploy, phase, installed, deny_direct_exec=False,
):
current = plan()
apply = _apply_script(
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
firewall_mode=FIREWALL_MODE_NATIVE,
)
function_start = apply.index("transaction_recovery_helper_owned() {")
function_end = apply.index(
'\n}\nif [ -L "$BASE" ]', function_start,
) + 2
function = apply[function_start:function_end]
ready = apply.index(" ready)", function_end)
ready_body_start = apply.index("\n", ready) + 1
ready_body_end = apply.index(
" ;;\n *) exit 79 ;;", ready_body_start,
)
ready_body = apply[ready_body_start:ready_body_end]
base = tmp_path / "base"
journal = base / ".wg-fabric-txn"
journal.mkdir(parents=True)
phase_file = journal / "nft.phase"
phase_file.write_text(f"{phase}\n", encoding="utf-8")
phase_file.chmod(0o600)
if installed:
installed_file = journal / "nft.installed"
installed_file.write_bytes(b"complete nft snapshot\n")
installed_file.chmod(0o600)
log = tmp_path / "dispatch.log"
digest = current.digest
def helper_text(label, options):
cases = "|".join(options)
return (
"#!/bin/sh\n"
"# virt-cloud-wg-fabric-v1\n"
f"# fabric-plan-digest={digest}\n"
f'PLAN_DIGEST="{digest}"\n'
f"# accepted recovery arguments: {cases}\n"
f"case \"${{1:-}}\" in {cases}) : ;; *) exit 31 ;; esac\n"
f"printf '%s:%s\\n' {shlex.quote(label)} \"$1\" > "
f"{shlex.quote(str(log))}\n"
f"rm -rf {shlex.quote(str(journal))}\n"
)
temporary = tmp_path / "temporary-helper"
temporary.write_text(
helper_text("temporary", ("--recover-current",
"--recover-installing")),
encoding="utf-8",
)
temporary.chmod(0o600 if deny_direct_exec else 0o700)
fixed = tmp_path / "fixed-helper"
if first_deploy:
marker = journal / "helper.absent"
marker.write_bytes(b"")
marker.chmod(0o600)
else:
fixed_bytes = helper_text("fixed", ("--recover-only",)).encode()
fixed.write_bytes(fixed_bytes)
fixed.chmod(0o600 if deny_direct_exec else 0o700)
marker = journal / "helper.present"
marker.write_bytes(b"")
marker.chmod(0o600)
baseline = journal / "helper"
baseline.write_bytes(fixed_bytes)
baseline.chmod(0o700)
fake_stat = tmp_path / "stat"
fake_stat.write_text(
f"""#!{sys.executable}
import os
import stat
import sys
if len(sys.argv) != 4 or sys.argv[1] != "-c":
raise SystemExit(2)
value = os.lstat(sys.argv[3])
mode = format(stat.S_IMODE(value.st_mode), "o")
if {deny_direct_exec!r} and os.path.basename(sys.argv[3]) in (
"temporary-helper", "fixed-helper"):
# Model an executable root-owned file on a noexec mount: its metadata
# passes the contract while the OS rejects a direct exec.
mode = "700"
if sys.argv[2] == "%a":
print(mode)
elif sys.argv[2] == "%u":
print("0")
elif sys.argv[2] == "%u:%a":
print(f"0:{{mode}}")
else:
raise SystemExit(2)
""",
encoding="utf-8",
)
fake_stat.chmod(0o700)
harness = "\n".join((
"set -eu",
f"BASE={shlex.quote(str(base))}",
f"HELPER={shlex.quote(str(fixed))}",
f"_tmp_helper={shlex.quote(str(temporary))}",
f"STAT={shlex.quote(str(fake_stat))}",
"CMP=/usr/bin/cmp",
f"EXPECTED_DIGEST={shlex.quote(digest)}",
"TRANSACTION_RECOVERED=no",
"vc_fabric_operation_checkpoint() { :; }",
function,
ready_body,
'[ "$TRANSACTION_RECOVERED" = yes ]',
))
result = subprocess.run(
["/bin/sh", "-c", harness], check=False,
capture_output=True, text=True,
)
return result, log
@pytest.mark.parametrize(("phase", "installed", "expected"), (
("before_mutation", False, "--recover-current"),
("installing", False, "--recover-installing"),
("installing", True, "--recover-current"),
("installed", True, "--recover-current"),
("refreshing", True, "--recover-current"),
("refreshed", True, "--recover-current"),
))
def test_first_deploy_ready_dispatch_uses_current_plan_helper(
tmp_path, phase, installed, expected,
):
result, log = _run_ready_recovery_dispatch_harness(
tmp_path, first_deploy=True, phase=phase, installed=installed,
)
assert result.returncode == 0, result.stderr
assert log.read_text(encoding="utf-8") == f"temporary:{expected}\n"
@pytest.mark.parametrize(("first_deploy", "phase", "installed", "expected"), (
(True, "installing", False, "temporary:--recover-installing"),
(True, "installed", True, "temporary:--recover-current"),
(False, "installed", True, "fixed:--recover-only"),
))
def test_ready_recovery_uses_interpreter_when_direct_execution_is_denied(
tmp_path, first_deploy, phase, installed, expected):
result, log = _run_ready_recovery_dispatch_harness(
tmp_path, first_deploy=first_deploy, phase=phase, installed=installed,
deny_direct_exec=True)
helper = tmp_path / (
"temporary-helper" if first_deploy else "fixed-helper")
with pytest.raises(PermissionError):
subprocess.run([str(helper)], check=False, capture_output=True)
assert result.returncode == 0, result.stderr
assert log.read_text(encoding="utf-8") == expected + "\n"
@pytest.mark.parametrize(("phase", "installed"), (
("before_mutation", False),
("installing", False),
("installing", True),
("installed", True),
("refreshing", True),
("refreshed", True),
))
def test_existing_ready_dispatch_uses_exact_owned_fixed_helper(
tmp_path, phase, installed,
):
result, log = _run_ready_recovery_dispatch_harness(
tmp_path, first_deploy=False, phase=phase, installed=installed,
)
assert result.returncode == 0, result.stderr
assert log.read_text(encoding="utf-8") == "fixed:--recover-only\n"
def test_installing_recovery_requires_exact_stable_first_deploy_evidence():
current = plan()
script = _apply_script(
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
firewall_mode=FIREWALL_MODE_NATIVE,
)
recover_start = script.index("recover_journal() {")
recover_end = script.index(
'\n}\nif [ "${VC_FABRIC_APPLYING:-0}" != 1 ]; then',
recover_start,
) + 2
recover = script[recover_start:recover_end]
digest_fence = recover.index(
'if [ "${RECOVER_ONLY:-no}" != yes ] ||\n'
' [ "${RECOVER_CURRENT:-no}" = yes ]; then')
digest_match = recover.index(
'[ "$journal_plan_digest" = "$PLAN_DIGEST" ]', digest_fence)
connection_match = recover.index(
'[ "$journal_connection_uuid" = "$CONNECTION_UUID" ]',
digest_match,
)
deployment_match = recover.index(
'[ "$journal_mode" = "$DEPLOYMENT_MODE" ]', connection_match)
firewall_match = recover.index(
'[ "$journal_firewall_mode" = "$FIREWALL_MODE" ]',
deployment_match,
)
special = recover.index(
'if [ "${RECOVER_INSTALLING:-no}" = yes ]; then', firewall_match)
phase_gate = recover.index(
'[ "$journal_nft_phase" = installing ]', special)
absent_baseline = recover.index(
'[ "$journal_nft_before" = no ]', phase_gate)
clean_first = recover.index(
'for journal_clean_first in conf owner helper unit repair timer '
'gate_generator nm_profile ifupdown_fragment sysctl nft_rules; do',
absent_baseline,
)
clean_markers = recover.index(
'journal_marker_secure "$journal_clean_first.absent" || exit 79',
clean_first,
)
desired_rules = recover.index(
'journal_file_secure nft_rules.desired || exit 79', clean_markers)
desired_mode = recover.index(
'$STAT -c \'%u:%a\' "$JOURNAL/nft_rules.desired"',
desired_rules,
)
desired_sha = recover.index(
'"$SHA256" "$JOURNAL/nft_rules.desired"', desired_mode)
expected_sha = recover.index(
'[ "$journal_desired_nft_sha" = '
'"$EXPECTED_NFT_RULES_SHA256" ] || exit 79',
desired_sha,
)
desired_shape = recover.index(
'nft_plan_owned "$journal_desired_nft" '
'"$journal_plan_digest" || exit 79',
expected_sha,
)
desired_empty = recover.index(
'vc_fabric_nft_empty_tenant_set "$journal_desired_nft" '
'|| exit 79',
desired_shape,
)
owner_artifact = recover.index(
'journal_first_deploy_artifact_safe "$OWNER" owner || exit 79',
desired_empty,
)
sysctl_artifact = recover.index(
'journal_first_deploy_artifact_safe "$SYSCTL_FILE" sysctl '
'|| exit 79',
owner_artifact,
)
nft_rules_artifact = recover.index(
'journal_first_deploy_artifact_safe "$NFT_RULES" nft_rules '
'|| exit 79',
sysctl_artifact,
)
native_baseline = recover.index(
'journal_native_nft_main_safe || exit 79', nft_rules_artifact,
)
current_owner = recover.index(
'journal_current_owner_state=absent', native_baseline)
installing_owner_gate = recover.index(
'case "$journal_current_owner_state:'
'$journal_current_iface_state" in',
current_owner,
)
first_read = recover.index(
'$NFT -s list table inet virt_cloud_wg_fabric > '
'"$JOURNAL/nft.current"',
desired_empty,
)
live_shape = recover.index(
'nft_plan_owned "$journal_installing_text" '
'"$journal_plan_digest" || exit 79',
first_read,
)
live_empty = recover.index(
'vc_fabric_nft_empty_tenant_set "$journal_installing_text" '
'|| exit 79',
live_shape,
)
second_read = recover.index(
'"$NFT" -s list table inet virt_cloud_wg_fabric > '
'"$_tmp_nft_installed"',
live_empty,
)
stable_compare = recover.index(
'"$CMP" -s "$JOURNAL/nft.current" '
'"$_tmp_nft_installed" || exit 79',
second_read,
)
atomic_snapshot = recover.index(
'mv -f "$_tmp_nft_installed" "$JOURNAL/nft.installed"',
stable_compare,
)
installed_compare = recover.index(
'"$CMP" -s "$JOURNAL/nft.current" '
'"$JOURNAL/nft.installed"',
atomic_snapshot,
)
expected_file = recover.index(
'journal_nft_expected_file="$JOURNAL/nft.installed"',
installed_compare,
)
allow_replace = recover.index(
"journal_nft_action=replace", expected_file)
verify_read = recover.index(
'$NFT -s list table inet virt_cloud_wg_fabric > '
'"$JOURNAL/nft.verify"',
allow_replace,
)
verify_compare = recover.index(
'"$CMP" -s "$JOURNAL/nft.verify" '
'"$journal_nft_expected_file" || exit 79',
verify_read,
)
precommit_read = recover.index(
'$NFT -s list table inet virt_cloud_wg_fabric > '
'"$JOURNAL/nft.precommit"',
verify_compare,
)
precommit_compare = recover.index(
'"$CMP" -s "$JOURNAL/nft.precommit" '
'"$journal_nft_expected_file" || exit 79',
precommit_read,
)
destructive_apply = recover.index(
'"$NFT" -f "$_tmp_nft_rules"', precommit_compare)
assert (
digest_fence < digest_match < connection_match < deployment_match
< firewall_match < special < phase_gate < absent_baseline < clean_first
< clean_markers < desired_rules < desired_mode < desired_sha < expected_sha
< desired_shape < desired_empty < owner_artifact < sysctl_artifact
< nft_rules_artifact < native_baseline < current_owner
< installing_owner_gate < first_read < live_shape
< live_empty < second_read < stable_compare < atomic_snapshot
< installed_compare < expected_file < allow_replace < verify_read
< verify_compare < precommit_read < precommit_compare
< destructive_apply
)
def _installing_recovery_fragments():
helper = _BOOT_HELPER.replace(
"__VC_FABRIC_NFT_VALIDATION__", _FABRIC_NFT_VALIDATION_SH,
)
ownership_start = helper.index("nft_panel_owned() {")
ownership_end = helper.index("\nrecover_journal() {", ownership_start)
ownership = helper[ownership_start:ownership_end]
artifact_start = helper.index("journal_file_secure() {")
artifact_end = helper.index(
"\njournal_cleanup_helper_owned() {", artifact_start)
artifacts = helper[artifact_start:artifact_end]
special_start = helper.index(
' if [ "${RECOVER_INSTALLING:-no}" = yes ]; then')
special_end = helper.index(
" journal_applied_present=no", special_start)
special = helper[special_start:special_end]
current_owner_start = helper.index(
" journal_current_owner_state=absent", special_end)
current_owner_end = helper.index(
" journal_nmcli=", current_owner_start)
current_owner = helper[current_owner_start:current_owner_end]
snapshot_start = helper.index(
' if [ "$journal_recover_installing" = yes ]; then')
snapshot_end = helper.index(
' if [ -f "$JOURNAL/nft.installed" ] &&',
snapshot_start,
)
snapshot = helper[snapshot_start:snapshot_end]
proof_end = helper.index("\n installed)", snapshot_end)
installed_proof = helper[snapshot_end:proof_end]
assert re.search(r"fi ;;\s*$", installed_proof)
installed_proof = re.sub(
r"fi ;;\s*$", "fi", installed_proof,
)
return artifacts, ownership, special, current_owner, snapshot, installed_proof
def _run_installing_recovery_harness(
tmp_path, *, failure="none", artifact_state="post_trap_absent",
firewall_mode=FIREWALL_MODE_NATIVE,
native_state="baseline_current_no_desired",
):
digest = "a" * 64
desired = _nft_rules_text(digest)
if failure == "nonempty_tenant_set":
desired = desired.replace(
" flags interval;",
" flags interval;\n"
" elements = { 10.10.0.2 }",
1,
)
elif failure == "extra_object":
desired = desired.replace(
" set vpc_tenant_ips {",
" set foreign_ips { type ipv4_addr; }\n"
" set vpc_tenant_ips {",
1,
)
desired_bytes = desired.encode()
expected_sha = hashlib.sha256(desired_bytes).hexdigest()
if failure == "desired_hash":
expected_sha = "0" * 64
alma_live = _nft_rules_text(digest).replace(
"type filter hook output priority -100; policy accept;",
"type filter hook output priority dstnat; policy accept;",
1,
)
second_live = alma_live
if failure == "second_read_drift":
second_live = alma_live.replace(
"type filter hook output priority dstnat; policy accept;",
"type filter hook output priority -100; policy accept;",
1,
)
journal = tmp_path / "journal"
journal.mkdir(mode=0o700)
clean_first = (
"conf", "owner", "helper", "unit", "repair", "timer",
"gate_generator", "nm_profile", "ifupdown_fragment", "sysctl",
"nft_rules",
)
for name in clean_first:
marker = journal / f"{name}.absent"
marker.write_bytes(b"")
marker.chmod(0o600)
nft_main_marker = journal / (
"nft_main.present"
if firewall_mode == FIREWALL_MODE_NATIVE
else "nft_main.absent"
)
nft_main_marker.write_bytes(b"")
nft_main_marker.chmod(0o600)
nft_before = journal / "nft.table"
nft_before.write_bytes(b"")
nft_before.chmod(0o600)
desired_path = journal / "nft_rules.desired"
desired_path.write_bytes(desired_bytes)
desired_path.chmod(0o600)
evidence = journal / "operator.evidence"
evidence.write_bytes(b"do-not-overwrite\n")
evidence.chmod(0o600)
targets_dir = tmp_path / "targets"
targets_dir.mkdir()
targets = {
"conf": targets_dir / "wg-fabric0.conf",
"owner": targets_dir / "wg-fabric0.owner",
"helper": targets_dir / "virt-cloud-wg-fabric",
"unit": targets_dir / "virt-cloud-wg-fabric.service",
"repair": targets_dir / "virt-cloud-wg-fabric-reconcile.service",
"timer": targets_dir / "virt-cloud-wg-fabric-reconcile.timer",
"gate_generator": targets_dir / "virt-cloud-wg-fabric-gate",
"nm_profile": targets_dir / "virt-cloud-wg-fabric0.nmconnection",
"ifupdown_fragment": targets_dir / "virt-cloud-wg-fabric0",
"sysctl": targets_dir / "90-virt-cloud-wg-fabric.conf",
"nft_rules": targets_dir / "virt-cloud-wg-fabric.nft",
"nft_main": targets_dir / "nftables.conf",
}
artifact_values = {
"owner": b"virt-cloud-wg-fabric-v1\n",
"sysctl": b"net.ipv4.ip_forward = 1\n",
"nft_rules": desired_bytes,
"nft_main": b'include "/etc/nftables.d/*.nft"\n',
}
for name, value in artifact_values.items():
if name in ("nft_rules", "nft_main"):
continue
desired_artifact = journal / f"{name}.desired"
desired_artifact.write_bytes(value)
desired_artifact.chmod(0o600)
if artifact_state == "post_trap_absent":
present_artifacts = set()
elif artifact_state == "hard_crash_exact_desired":
present_artifacts = set(artifact_values)
elif artifact_state == "mixed":
present_artifacts = {"owner", "nft_rules"}
else:
raise AssertionError(f"unknown artifact state: {artifact_state}")
if failure == "artifact_foreign_bytes":
present_artifacts.add("sysctl")
elif failure == "artifact_wrong_mode":
present_artifacts.add("nft_rules")
elif failure == "artifact_symlink":
present_artifacts.add("nft_main")
elif failure in ("artifact_desired_absent", "owner_iface_present"):
present_artifacts.add("owner")
artifact_symlink_target = tmp_path / "artifact-symlink-target"
for name in present_artifacts:
target = targets[name]
if failure == "artifact_symlink" and name == "nft_main":
artifact_symlink_target.write_bytes(artifact_values[name])
target.symlink_to(artifact_symlink_target)
continue
value = artifact_values[name]
if failure == "artifact_foreign_bytes" and name == "sysctl":
value = b"foreign sysctl bytes\n"
target.write_bytes(value)
target.chmod(
0o644
if failure == "artifact_wrong_mode" and name == "nft_rules"
else 0o600
)
if failure == "artifact_desired_absent":
desired_absent = journal / "owner.desired.absent"
desired_absent.write_bytes(b"")
desired_absent.chmod(0o600)
if firewall_mode == FIREWALL_MODE_NATIVE:
native_baseline = (
b"# baseline\ninclude \"/etc/nftables.d/*.nft\"\n"
)
native_desired = (
b"# desired\n"
b"include \"/etc/nftables.d/virt-cloud-wg-fabric.nft\"\n"
)
baseline = journal / "nft_main"
native_snapshot_target = tmp_path / "native-snapshot-target"
if failure == "native_baseline_symlink":
native_snapshot_target.write_bytes(native_baseline)
baseline.symlink_to(native_snapshot_target)
elif failure != "native_baseline_missing":
baseline.write_bytes(
b"# invalid baseline\n"
if failure == "native_baseline_content"
else native_baseline
)
baseline.chmod(
0o666 if failure == "native_baseline_mode" else 0o600
)
desired_needed = native_state != "baseline_current_no_desired" or (
failure in {
"native_desired_symlink",
"native_desired_mode",
"native_desired_content",
"native_desired_absent_conflict",
}
)
desired_main = journal / "nft_main.desired"
native_desired_target = tmp_path / "native-desired-target"
if desired_needed:
if failure == "native_desired_symlink":
native_desired_target.write_bytes(native_desired)
desired_main.symlink_to(native_desired_target)
else:
desired_main.write_bytes(
b"# invalid desired\n"
if failure == "native_desired_content"
else native_desired
)
desired_main.chmod(
0o666 if failure == "native_desired_mode" else 0o600
)
if failure in ("native_marker_conflict",
"native_desired_absent_conflict"):
conflict = journal / (
"nft_main.absent"
if failure == "native_marker_conflict"
else "nft_main.desired.absent"
)
conflict.write_bytes(b"")
conflict.chmod(0o600)
live = targets["nft_main"]
native_live_target = tmp_path / "native-live-target"
if failure == "native_live_symlink":
native_live_target.write_bytes(native_baseline)
live.symlink_to(native_live_target)
elif failure != "native_live_missing":
live_value = (
native_desired
if native_state == "desired_current"
else native_baseline
)
if failure == "native_live_foreign":
live_value = b"# foreign live\ninclude \"/etc/nftables.d/*.nft\"\n"
live.write_bytes(live_value)
live.chmod(
0o644 if failure == "native_live_mode" else 0o600
)
else:
native_snapshot_target = tmp_path / "unused-native-snapshot"
native_desired_target = tmp_path / "unused-native-desired"
native_live_target = tmp_path / "unused-native-live"
desired_main = journal / "nft_main.desired"
desired_main.write_bytes(artifact_values["nft_main"])
desired_main.chmod(0o600)
installed_target = tmp_path / "installed-target"
if failure == "installed_symlink":
installed_target.write_bytes(b"foreign-snapshot\n")
(journal / "nft.installed").symlink_to(installed_target)
first_output = tmp_path / "nft-first"
first_output.write_text(alma_live, encoding="utf-8")
second_output = tmp_path / "nft-second"
second_output.write_text(second_live, encoding="utf-8")
call_count = tmp_path / "nft-count"
nft = tmp_path / "nft"
nft.write_text(
"""#!/bin/sh
[ "$*" = "-s list table inet virt_cloud_wg_fabric" ] || exit 2
vc_count=0
if [ -f "$VC_NFT_CALL_COUNT" ]; then
vc_count=$(cat "$VC_NFT_CALL_COUNT")
fi
vc_count=$((vc_count + 1))
printf '%s\n' "$vc_count" > "$VC_NFT_CALL_COUNT"
if [ "$vc_count" -eq 1 ]; then
cat "$VC_NFT_FIRST"
else
cat "$VC_NFT_SECOND"
fi
""",
encoding="utf-8",
)
nft.chmod(0o700)
sha256 = tmp_path / "sha256sum"
sha256.write_text(
f"""#!{sys.executable}
import hashlib
import sys
for filename in sys.argv[1:]:
with open(filename, "rb") as stream:
digest = hashlib.sha256(stream.read()).hexdigest()
print(f"{{digest}} {{filename}}")
""",
encoding="utf-8",
)
sha256.chmod(0o700)
fake_stat = tmp_path / "stat"
fake_stat.write_text(
f"""#!{sys.executable}
import os
import stat
import sys
if len(sys.argv) != 4 or sys.argv[1] != "-c":
raise SystemExit(2)
value = os.lstat(sys.argv[3])
mode = format(stat.S_IMODE(value.st_mode), "o")
if sys.argv[2] == "%a":
print(mode)
elif sys.argv[2] == "%u":
print("0")
elif sys.argv[2] == "%u:%a":
print(f"0:{{mode}}")
else:
raise SystemExit(2)
""",
encoding="utf-8",
)
fake_stat.chmod(0o700)
ip = tmp_path / "ip"
ip.write_text(
"""#!/bin/sh
if [ "${VC_IFACE_PRESENT:-no}" = yes ] &&
[ "$*" = "link show dev wg-fabric0" ]; then
exit 0
fi
exit 1
""",
encoding="utf-8",
)
ip.chmod(0o700)
wg = tmp_path / "wg"
wg.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8")
wg.chmod(0o700)
artifacts, ownership, special, current_owner, snapshot, installed_proof = (
_installing_recovery_fragments()
)
special = special.replace(
"/etc/systemd/system/virt-cloud-wg-fabric-reconcile.service",
str(targets["repair"]),
).replace(
"/etc/systemd/system/virt-cloud-wg-fabric-reconcile.timer",
str(targets["timer"]),
)
script = "\n".join((
"set -eu",
"umask 077",
f"JOURNAL={shlex.quote(str(journal))}",
f"CONF={shlex.quote(str(targets['conf']))}",
f"OWNER={shlex.quote(str(targets['owner']))}",
f"HELPER={shlex.quote(str(targets['helper']))}",
f"UNIT={shlex.quote(str(targets['unit']))}",
f"GATE_GENERATOR={shlex.quote(str(targets['gate_generator']))}",
f"NM_PROFILE={shlex.quote(str(targets['nm_profile']))}",
f"IFUPDOWN_FRAGMENT={shlex.quote(str(targets['ifupdown_fragment']))}",
f"SYSCTL_FILE={shlex.quote(str(targets['sysctl']))}",
f"NFT_RULES={shlex.quote(str(targets['nft_rules']))}",
f"journal_nft_main={shlex.quote(str(targets['nft_main']))}",
f"IP={shlex.quote(str(ip))}",
f"WG={shlex.quote(str(wg))}",
"IFACE=wg-fabric0",
f"NFT={shlex.quote(str(nft))}",
f"STAT={shlex.quote(str(fake_stat))}",
f"SHA256={shlex.quote(str(sha256))}",
f"CMP={shlex.quote('/usr/bin/cmp')}",
f"EXPECTED_NFT_RULES_SHA256={shlex.quote(expected_sha)}",
f"journal_plan_digest={shlex.quote(digest)}",
f"journal_firewall_mode={firewall_mode}",
"journal_nft_phase=installing",
"journal_nft_before=no",
"journal_iface_before=no",
"journal_iface_up_before=no",
"journal_rp_filter_before=missing",
"journal_helper_before=absent",
"journal_iface_recovery_phase=none",
"journal_recover_installing=no",
"RECOVER_CURRENT=yes",
"RECOVER_INSTALLING=yes",
"_tmp_nft_installed=",
"cleanup() {",
" cleanup_status=$?",
" trap - EXIT",
' [ -z "$_tmp_nft_installed" ] || '
'rm -f "$_tmp_nft_installed" || true',
' exit "$cleanup_status"',
"}",
"trap cleanup EXIT",
"sync() { :; }",
'stat() { "$STAT" "$@"; }',
artifacts,
_FABRIC_NFT_VALIDATION_SH,
ownership,
special,
'[ "$journal_recover_installing" = yes ] || exit 79',
current_owner,
'"$NFT" -s list table inet virt_cloud_wg_fabric > '
'"$JOURNAL/nft.current" 2>/dev/null || exit 79',
'chmod 600 "$JOURNAL/nft.current" || exit 79',
snapshot,
installed_proof,
"journal_nft_action=replace",
"printf '%s\\n' \"journal_nft_action=$journal_nft_action\"",
))
result = subprocess.run(
["/bin/sh", "-c", script],
env={
"PATH": "/usr/bin:/bin",
"VC_NFT_CALL_COUNT": str(call_count),
"VC_NFT_FIRST": str(first_output),
"VC_NFT_SECOND": str(second_output),
"VC_IFACE_PRESENT": (
"yes" if failure == "owner_iface_present" else "no"
),
},
check=False, capture_output=True, text=True,
)
return result, SimpleNamespace(
journal=journal,
desired=desired_bytes,
evidence=evidence,
alma_live=alma_live.encode(),
call_count=call_count,
installed_target=installed_target,
artifact_symlink_target=artifact_symlink_target,
artifact_state=artifact_state,
artifact_values=artifact_values,
targets=targets,
native_snapshot_target=native_snapshot_target,
native_desired_target=native_desired_target,
native_live_target=native_live_target,
native_baseline=native_baseline,
)
@pytest.mark.parametrize("artifact_state", (
"post_trap_absent",
"hard_crash_exact_desired",
"mixed",
))
def test_installing_recovery_execution_snapshots_stable_alma_live_table(
tmp_path, artifact_state,
):
result, fixture = _run_installing_recovery_harness(
tmp_path, artifact_state=artifact_state,
)
assert result.returncode == 0, result.stderr
assert result.stdout.splitlines() == ["journal_nft_action=replace"]
installed = fixture.journal / "nft.installed"
assert installed.read_bytes() == fixture.alma_live
assert stat.S_IMODE(installed.stat().st_mode) == 0o600
assert (fixture.journal / "nft_rules.desired").read_bytes() == (
fixture.desired
)
assert fixture.evidence.read_bytes() == b"do-not-overwrite\n"
assert fixture.call_count.read_text(encoding="utf-8") == "2\n"
assert not list(fixture.journal.glob(".nft-installed.*"))
expected_present = {
"post_trap_absent": {"nft_main"},
"hard_crash_exact_desired": set(fixture.artifact_values),
"mixed": {"owner", "nft_rules", "nft_main"},
}[artifact_state]
for name, value in fixture.artifact_values.items():
target = fixture.targets[name]
if name in expected_present:
expected = (
fixture.native_baseline if name == "nft_main" else value
)
assert target.read_bytes() == expected
assert stat.S_IMODE(target.stat().st_mode) == 0o600
else:
assert not target.exists()
assert not target.is_symlink()
@pytest.mark.parametrize("failure", (
"desired_hash",
"nonempty_tenant_set",
"extra_object",
"installed_symlink",
"second_read_drift",
"artifact_foreign_bytes",
"artifact_wrong_mode",
"artifact_symlink",
"artifact_desired_absent",
"owner_iface_present",
))
def test_installing_recovery_execution_rejects_ambiguous_evidence(
tmp_path, failure,
):
result, fixture = _run_installing_recovery_harness(
tmp_path, failure=failure,
)
assert result.returncode == 79
assert (fixture.journal / "nft_rules.desired").read_bytes() == (
fixture.desired
)
assert fixture.evidence.read_bytes() == b"do-not-overwrite\n"
installed = fixture.journal / "nft.installed"
if failure == "installed_symlink":
assert installed.is_symlink()
assert fixture.installed_target.read_bytes() == b"foreign-snapshot\n"
else:
assert not installed.exists()
assert not installed.is_symlink()
if failure == "second_read_drift":
assert (fixture.journal / "nft.current").read_bytes() == (
fixture.alma_live
)
assert fixture.call_count.read_text(encoding="utf-8") == "2\n"
else:
assert not (fixture.journal / "nft.current").exists()
assert not fixture.call_count.exists()
if failure == "artifact_foreign_bytes":
assert fixture.targets["sysctl"].read_bytes() == (
b"foreign sysctl bytes\n"
)
elif failure == "artifact_wrong_mode":
target = fixture.targets["nft_rules"]
assert target.read_bytes() == fixture.artifact_values["nft_rules"]
assert stat.S_IMODE(target.stat().st_mode) == 0o644
elif failure == "artifact_symlink":
assert fixture.targets["nft_main"].is_symlink()
assert fixture.artifact_symlink_target.read_bytes() == (
fixture.native_baseline
)
elif failure == "artifact_desired_absent":
assert (fixture.journal / "owner.desired.absent").is_file()
assert fixture.targets["owner"].read_bytes() == (
fixture.artifact_values["owner"]
)
elif failure == "owner_iface_present":
assert fixture.targets["owner"].read_bytes() == (
fixture.artifact_values["owner"]
)
assert not list(fixture.journal.glob(".nft-installed.*"))
@pytest.mark.parametrize("native_state", (
"baseline_current_no_desired",
"desired_current",
"baseline_current_with_desired",
))
def test_native_installing_recovery_accepts_proven_nft_main_images(
tmp_path, native_state,
):
result, fixture = _run_installing_recovery_harness(
tmp_path,
firewall_mode=FIREWALL_MODE_NATIVE,
native_state=native_state,
)
assert result.returncode == 0, result.stderr
assert result.stdout.splitlines() == ["journal_nft_action=replace"]
assert (fixture.journal / "nft.installed").read_bytes() == (
fixture.alma_live
)
assert (fixture.journal / "nft_main.present").is_file()
assert not (fixture.journal / "nft_main.absent").exists()
assert (fixture.journal / "nft_main").read_bytes().startswith(
b"# baseline\n",
)
desired = fixture.journal / "nft_main.desired"
if native_state == "baseline_current_no_desired":
assert not desired.exists()
assert fixture.targets["nft_main"].read_bytes().startswith(
b"# baseline\n",
)
elif native_state == "desired_current":
assert desired.read_bytes().startswith(b"# desired\n")
assert fixture.targets["nft_main"].read_bytes() == desired.read_bytes()
else:
assert desired.read_bytes().startswith(b"# desired\n")
assert fixture.targets["nft_main"].read_bytes() == (
fixture.journal / "nft_main"
).read_bytes()
assert fixture.call_count.read_text(encoding="utf-8") == "2\n"
@pytest.mark.parametrize("failure", (
"native_baseline_missing",
"native_baseline_symlink",
"native_baseline_mode",
"native_baseline_content",
"native_marker_conflict",
"native_live_missing",
"native_live_symlink",
"native_live_mode",
"native_live_foreign",
"native_desired_symlink",
"native_desired_mode",
"native_desired_content",
"native_desired_absent_conflict",
))
def test_native_installing_recovery_rejects_unproven_nft_main_images(
tmp_path, failure,
):
result, fixture = _run_installing_recovery_harness(
tmp_path,
failure=failure,
firewall_mode=FIREWALL_MODE_NATIVE,
)
assert result.returncode == 79
assert fixture.evidence.read_bytes() == b"do-not-overwrite\n"
assert not (fixture.journal / "nft.current").exists()
assert not fixture.call_count.exists()
assert not (fixture.journal / "nft.installed").exists()
assert not list(fixture.journal.glob(".nft-installed.*"))
if failure == "native_baseline_symlink":
assert (fixture.journal / "nft_main").is_symlink()
assert fixture.native_snapshot_target.read_bytes().startswith(
b"# baseline\n",
)
elif failure == "native_desired_symlink":
assert (fixture.journal / "nft_main.desired").is_symlink()
assert fixture.native_desired_target.read_bytes().startswith(
b"# desired\n",
)
elif failure == "native_live_symlink":
assert fixture.targets["nft_main"].is_symlink()
assert fixture.native_live_target.read_bytes().startswith(
b"# baseline\n",
)
def _timer_activity_recovery_runner(
tmp_path, *, wanted, initial, failure="none",
):
start = _BOOT_HELPER.index("vc_restore_timer_activity() {")
end = _BOOT_HELPER.index("\n}\nnft_panel_owned()", start) + 2
function = _BOOT_HELPER[start:end]
state = tmp_path / "timer.state"
state.write_text(f"{initial}\n", encoding="utf-8")
log = tmp_path / "systemctl.log"
systemctl = tmp_path / "systemctl"
systemctl.write_text(
"""#!/bin/sh
printf '%s\n' "$*" >> "$VC_SYSTEMCTL_LOG"
case "$1" in
start)
[ "$VC_SYSTEMCTL_FAILURE" != start ] || exit 1
printf '%s\n' active > "$VC_TIMER_STATE" ;;
stop)
[ "$VC_SYSTEMCTL_FAILURE" != stop ] || exit 1
[ "$VC_SYSTEMCTL_FAILURE" = sticky ] || \
printf '%s\n' inactive > "$VC_TIMER_STATE" ;;
is-active)
state=$(cat "$VC_TIMER_STATE")
printf '%s\n' "$state"
[ "$state" = active ] || [ "$state" = activating ] ;;
*) exit 2 ;;
esac
""",
encoding="utf-8",
)
systemctl.chmod(0o700)
result = subprocess.run(
["/bin/sh", "-c", "\n".join((
"set -eu",
f"SYSTEMCTL='{systemctl}'",
function,
f"vc_restore_timer_activity {wanted}",
))],
env={
"PATH": "/usr/bin:/bin",
"VC_SYSTEMCTL_LOG": str(log),
"VC_TIMER_STATE": str(state),
"VC_SYSTEMCTL_FAILURE": failure,
},
check=False, capture_output=True, text=True,
)
return result, state, log
def test_crash_recovery_stops_timer_started_after_inactive_snapshot(tmp_path):
result, state, log = _timer_activity_recovery_runner(
tmp_path, wanted="inactive", initial="active",
)
assert result.returncode == 0, result.stderr
assert state.read_text(encoding="utf-8") == "inactive\n"
assert log.read_text(encoding="utf-8").splitlines() == [
"stop virt-cloud-wg-fabric-reconcile.timer",
"is-active virt-cloud-wg-fabric-reconcile.timer",
]
helper = _BOOT_HELPER
restore_unit = helper.index(
'restore_journal_file "/etc/systemd/system/'
'virt-cloud-wg-fabric-reconcile.timer" timer'
)
daemon_reload = helper.index(
'"$SYSTEMCTL" daemon-reload', restore_unit,
)
restore_activity = helper.index(
'vc_restore_timer_activity "$journal_timer_active"', daemon_reload,
)
remove_journal = helper.index('rm -rf "$JOURNAL"', restore_activity)
assert restore_unit < daemon_reload < restore_activity < remove_journal
def test_crash_recovery_restarts_previously_active_timer(tmp_path):
result, state, log = _timer_activity_recovery_runner(
tmp_path, wanted="active", initial="inactive",
)
assert result.returncode == 0, result.stderr
assert state.read_text(encoding="utf-8") == "active\n"
assert log.read_text(encoding="utf-8").splitlines() == [
"start virt-cloud-wg-fabric-reconcile.timer",
"is-active virt-cloud-wg-fabric-reconcile.timer",
]
@pytest.mark.parametrize(
("wanted", "initial", "failure"),
(("active", "inactive", "start"),
("inactive", "active", "stop"),
("inactive", "active", "sticky")),
)
def test_crash_recovery_keeps_journal_when_timer_state_cannot_be_restored(
tmp_path, wanted, initial, failure,
):
result, _, _ = _timer_activity_recovery_runner(
tmp_path, wanted=wanted, initial=initial, failure=failure,
)
assert result.returncode == 1
def test_apply_rollback_recovers_refresh_snapshot_before_phase_marker():
current = plan()
script = _apply_script(
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
firewall_mode=FIREWALL_MODE_NATIVE,
)
snapshot_publish = script.index(
'mv -f "$TXN/nft.applied.tmp" "$TXN/nft.applied"')
phase_publish = script.index(
"write_nft_phase refreshed", snapshot_publish)
rollback_window = script.index(
'elif [ "$rollback_nft_phase" = refreshing ] &&')
exact_snapshot = script.index(
'"$CMP" -s "$TXN/nft.current" "$TXN/nft.applied" '
'|| ROLLBACK_FAILED=yes',
rollback_window,
)
exact_set = script.index(
"vpc_state_nft_set_matches || ROLLBACK_FAILED=yes",
exact_snapshot,
)
stable_snapshot = script.index(
'"$CMP" -s "$TXN/nft.applied" "$TXN/nft.refresh-verify" '
'|| ROLLBACK_FAILED=yes',
exact_set,
)
assert snapshot_publish < phase_publish
assert rollback_window < exact_snapshot < exact_set < stable_snapshot
def test_repair_unit_contains_only_the_retry_dispatcher():
current = plan()
script = _apply_script(
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
firewall_mode=FIREWALL_MODE_NATIVE,
)
rendered = _REPAIR_UNIT.replace(
"__VC_FABRIC_DIGEST__", current.digest)
assert rendered in script
assert (
'ExecCondition=/bin/sh /usr/local/libexec/virt-cloud-wg-fabric '
'--needs-repair'
) in rendered
assert [
line for line in rendered.splitlines()
if line.startswith("ExecStart=")
] == [
"ExecStart=/bin/sh /usr/local/libexec/virt-cloud-wg-fabric "
"--retry-gated-units",
]
assert "ExecStartPost=" not in rendered
assert "__VC_FABRIC_EXEC__" not in rendered
def test_repair_dispatcher_queues_primary_without_proven_dependency_jobs(tmp_path):
tool = tmp_path / "tool"
tool.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8")
tool.chmod(0o700)
systemctl = tmp_path / "systemctl"
systemctl.write_text(
"""#!/bin/sh
printf '%s\\n' "$*" >> "$VC_TEST_SYSTEMCTL_LOG"
case "$1:$2:$3:$5" in
is-failed:--quiet:virt-cloud-wg-fabric.service:) exit 0 ;;
show:-p:LoadState:libvirtd.service) printf '%s\\n' loaded ;;
show:-p:Result:libvirtd.service) printf '%s\\n' dependency ;;
show:-p:LoadState:virtqemud.service) printf '%s\\n' loaded ;;
show:-p:Result:virtqemud.service) printf '%s\\n' success ;;
reset-failed:libvirtd.service::) exit 0 ;;
reset-failed:virt-cloud-wg-fabric.service::) exit 0 ;;
start:virt-cloud-wg-fabric.service::) exit 0 ;;
*) exit 2 ;;
esac
""",
encoding="utf-8",
)
systemctl.chmod(0o700)
helper = _BOOT_HELPER.replace(
"__VC_FIREWALL_MODE_GUARD__", _FIREWALL_MODE_GUARD_SH,
).replace(
"__VC_LIBVIRT_GATE_UNITS__",
"libvirtd.service virtqemud.service",
).replace(
"__VC_FABRIC_SYSCTL_VALIDATION__",
_FABRIC_SYSCTL_VALIDATION_SH,
)
tool_variables = {
"WG", "IP", "NFT", "STAT", "SHA256", "CMP", "FIREWALL_CMD",
"AWK", "BRIDGE_CMD",
}
helper_lines = []
for line in helper.splitlines():
if line == "BASE=/etc/virt-cloud":
line = 'BASE="$VC_TEST_BASE"'
elif line.startswith("SYSTEMCTL=$(find_tool "):
line = 'SYSTEMCTL="$VC_TEST_SYSTEMCTL"'
elif any(
line.startswith(f"{name}=$(find_tool ")
for name in tool_variables
):
name = line.split("=", 1)[0]
line = f'{name}="$VC_TEST_TOOL"'
helper_lines.append(line)
helper_path = tmp_path / "virt-cloud-wg-fabric"
helper_path.write_text(
"\n".join(helper_lines) + "\n",
encoding="utf-8",
)
helper_path.chmod(0o700)
base = tmp_path / "state"
base.mkdir()
log = tmp_path / "systemctl.log"
env = {
"VC_TEST_BASE": str(base),
"VC_TEST_TOOL": str(tool),
"VC_TEST_SYSTEMCTL": str(systemctl),
"VC_TEST_SYSTEMCTL_LOG": str(log),
}
condition = subprocess.run(
["/bin/sh", str(helper_path), "--needs-repair"],
check=False, capture_output=True, text=True, env=env,
)
assert condition.returncode == 0, condition.stderr
dispatcher = subprocess.run(
["/bin/sh", str(helper_path), "--retry-gated-units"],
check=False, capture_output=True, text=True, env=env,
)
assert dispatcher.returncode == 0, dispatcher.stderr
calls = log.read_text(encoding="utf-8").splitlines()
start_calls = [call for call in calls if call.startswith("start ")]
assert start_calls == [
"start virt-cloud-wg-fabric.service",
]
assert not any("show -p Result" in call for call in calls)
def test_repair_retry_dispatches_before_lock_and_only_requeues_failures():
retry = _BOOT_HELPER.index('if [ "${1:-}" = --retry-gated-units ]')
lock = _BOOT_HELPER.index('exec 9>>"$LOCK"')
assert retry < lock
assert 'vc_gate_retry_evidence' not in _BOOT_HELPER
assert 'value!="dependency"' not in _BOOT_HELPER
assert 'show -p Result' not in _BOOT_HELPER
assert 'start virt-cloud-wg-fabric.service' in _BOOT_HELPER
assert 'exec /bin/sh "$HELPER" --ifupdown-service' in _BOOT_HELPER
assert 'exec /bin/sh "$HELPER"' in _BOOT_HELPER
assert 'restart --no-block virt-cloud-wg-fabric.service' not in _BOOT_HELPER
needs_repair = _BOOT_HELPER.index(
'if [ "${1:-}" = --needs-repair ]; then')
lock = _BOOT_HELPER.index('exec 9>>"$LOCK"')
assert needs_repair < lock
assert 'is-failed --quiet virt-cloud-wg-fabric.service' in _BOOT_HELPER
assert (
'ExecCondition=/bin/sh /usr/local/libexec/virt-cloud-wg-fabric '
'--needs-repair'
) in _REPAIR_UNIT
def _operation_status_output(
operation_id, *, status="queued", checkpoint="dispatch",
reason="none", created_at=100, started_at=0, updated_at=100,
finished_at=0):
return "\n".join((
"__VC_FABRIC_OPERATION_SCHEMA__=1",
f"__VC_FABRIC_OPERATION_ID__={operation_id}",
f"__VC_FABRIC_OPERATION_STATUS__={status}",
"__VC_FABRIC_OPERATION_PHASE__=apply",
f"__VC_FABRIC_OPERATION_CHECKPOINT__={checkpoint}",
f"__VC_FABRIC_OPERATION_REASON__={reason}",
f"__VC_FABRIC_OPERATION_CREATED_AT__={created_at}",
f"__VC_FABRIC_OPERATION_STARTED_AT__={started_at}",
f"__VC_FABRIC_OPERATION_UPDATED_AT__={updated_at}",
f"__VC_FABRIC_OPERATION_FINISHED_AT__={finished_at}",
"",
))
def test_fabric_operation_ids_are_opaque_fixed_lowercase_hex():
operation_ids = {_new_operation_id() for _index in range(128)}
assert len(operation_ids) == 128
assert all(re.fullmatch(r"[0-9a-f]{32}", value)
for value in operation_ids)
@pytest.mark.parametrize("operation_id", [
"", "a" * 31, "a" * 33, "A" * 32, "g" * 32,
"../" + "a" * 29, "a" * 31 + "/", "a" * 31 + "\n",
"$(id)".ljust(32, "a"),
])
def test_fabric_operation_id_is_validated_before_rendering_remote_shell(
operation_id):
with pytest.raises(FabricError) as error:
_fabric_operation_status_script(operation_id)
assert error.value.error_code == "fabric_operation_invalid"
@pytest.mark.parametrize(("kwargs", "expected"), [
({}, {
"status": "queued", "checkpoint": "dispatch", "reason": "none",
"started_at": None, "finished_at": None,
}),
({
"status": "running", "checkpoint": "journal_recovery",
"created_at": 100, "started_at": 101, "updated_at": 102,
}, {
"status": "running", "checkpoint": "journal_recovery",
"reason": "none", "started_at": 101, "finished_at": None,
}),
({
"status": "succeeded", "checkpoint": "complete",
"created_at": 100, "started_at": 101, "updated_at": 103,
"finished_at": 103,
}, {
"status": "succeeded", "checkpoint": "complete",
"reason": "none", "started_at": 101, "finished_at": 103,
}),
({
"status": "failed", "checkpoint": "nft_install",
"reason": "nft_failed", "created_at": 100, "started_at": 101,
"updated_at": 104, "finished_at": 104,
}, {
"status": "failed", "checkpoint": "nft_install",
"reason": "nft_failed", "started_at": 101, "finished_at": 104,
}),
])
def test_fabric_operation_status_parser_accepts_only_complete_safe_projection(
kwargs, expected):
operation_id = "1" * 32
result = _parse_operation_status(
_operation_status_output(operation_id, **kwargs), operation_id)
assert result["operation_id"] == operation_id
assert result["phase"] == "apply"
for field, value in expected.items():
assert result[field] == value
def _replace_status_line(output, prefix, replacement):
lines = output.splitlines()
return "\n".join(
replacement if line.startswith(prefix) else line
for line in lines
) + "\n"
@pytest.mark.parametrize("mutation", [
lambda value: value + "__VC_FABRIC_OPERATION_EXTRA__=value\n",
lambda value: value + "__VC_FABRIC_OPERATION_STATUS__=queued\n",
lambda value: value.replace(
"__VC_FABRIC_OPERATION_REASON__=none\n", ""),
lambda value: "attacker-controlled diagnostic\n" + value,
lambda value: _replace_status_line(
value, "__VC_FABRIC_OPERATION_STATUS__=",
"__VC_FABRIC_OPERATION_STATUS__=unknown"),
lambda value: _replace_status_line(
value, "__VC_FABRIC_OPERATION_PHASE__=",
"__VC_FABRIC_OPERATION_PHASE__=identity"),
lambda value: _replace_status_line(
value, "__VC_FABRIC_OPERATION_CHECKPOINT__=",
"__VC_FABRIC_OPERATION_CHECKPOINT__=arbitrary"),
lambda value: _replace_status_line(
value, "__VC_FABRIC_OPERATION_REASON__=",
"__VC_FABRIC_OPERATION_REASON__=/root/private-key"),
lambda value: _replace_status_line(
value, "__VC_FABRIC_OPERATION_CREATED_AT__=",
"__VC_FABRIC_OPERATION_CREATED_AT__=-1"),
lambda value: _replace_status_line(
value, "__VC_FABRIC_OPERATION_UPDATED_AT__=",
"__VC_FABRIC_OPERATION_UPDATED_AT__=99"),
])
def test_fabric_operation_status_parser_rejects_extra_duplicate_or_invalid_data(
mutation):
operation_id = "2" * 32
output = mutation(_operation_status_output(operation_id))
with pytest.raises(FabricError) as error:
_parse_operation_status(output, operation_id)
assert error.value.error_code == "fabric_operation_status_invalid"
safe_error = json.dumps({
"message": str(error.value),
"code": error.value.error_code,
"phase": error.value.failure_phase,
})
assert "attacker-controlled" not in safe_error
assert "/root/private-key" not in safe_error
def test_fabric_operation_status_parser_fences_different_attempts():
first = "3" * 32
second = "4" * 32
with pytest.raises(FabricError) as error:
_parse_operation_status(_operation_status_output(first), second)
assert error.value.error_code == "fabric_operation_status_invalid"
@pytest.mark.parametrize(("status", "checkpoint", "reason", "started", "finished"), [
("queued", "dispatch", "none", 1, 0),
("running", "preflight", "none", 0, 0),
("running", "preflight", "nft_failed", 101, 0),
("succeeded", "final_readback", "none", 101, 102),
("succeeded", "complete", "command_failed", 101, 102),
("failed", "nft_install", "none", 101, 102),
])
def test_fabric_operation_status_parser_rejects_impossible_state_combinations(
status, checkpoint, reason, started, finished):
operation_id = "5" * 32
output = _operation_status_output(
operation_id, status=status, checkpoint=checkpoint, reason=reason,
started_at=started, updated_at=102, finished_at=finished)
with pytest.raises(FabricError) as error:
_parse_operation_status(output, operation_id)
assert error.value.error_code == "fabric_operation_status_invalid"
def test_fabric_operation_protocol_has_fixed_bounded_vocabulary():
assert FABRIC_OPERATION_STATUSES == {
"queued", "running", "succeeded", "failed", "uncertain",
"interrupted",
}
assert FABRIC_OPERATION_CHECKPOINTS == {
"dispatch", "host_lock", "identity", "journal_recovery",
"journal_dispatch", "preflight_sysctl", "preflight_tools",
"preflight_artifacts", "preflight_firewall", "preflight_vpc",
"preflight_underlay", "preflight_routes", "preflight_persistence",
"journal_artifact_restore", "journal_nft_restore",
"journal_runtime_restore", "journal_finalize", "preflight",
"transaction", "nft_install", "network_activate", "persistence",
"final_readback", "complete",
}
assert FABRIC_OPERATION_REASONS == {
"none", "tool_missing", "ownership_conflict", "journal_conflict",
"sysctl_conflict", "persistence_conflict",
"firewall_conflict", "underlay_conflict", "network_manager_failed",
"nft_failed", "systemd_failed", "readback_failed",
"command_failed", "transport_failed", "status_invalid",
}
def test_fabric_operation_status_reader_is_strictly_read_only():
operation_id = "6" * 32
script = _fabric_operation_status_script(operation_id)
assert f"OPERATION_ID={operation_id}" in script
assert "__VC_FABRIC_OPERATION_STATUS__=$state" in script
assert '"$UPDATE" interrupted' not in script
assert "mv " not in script
assert "rm " not in script
assert "systemctl start" not in script
assert "systemctl restart" not in script
def test_fabric_operation_dispatch_uses_fixed_nonblocking_systemd_command():
operation_id = "7" * 32
secret = "private-key-material-must-not-be-logged"
rendered_apply = f"#!/bin/sh\nprintf '%s' {shlex.quote(secret)} >/dev/null\n"
script = _fabric_operation_dispatch_script(
operation_id, rendered_apply, "8" * 64)
systemd_start = script.index('"$SYSTEMD_RUN" --unit="$UNIT"')
assert f"UNIT={FABRIC_OPERATION_UNIT_PREFIX}{operation_id}.service" in script
assert "--collect --no-block --quiet" in script[systemd_start:]
assert "--property=Type=oneshot" in script[systemd_start:]
assert "--property=StandardOutput=journal" in script[systemd_start:]
assert "--property=StandardError=null" in script[systemd_start:]
assert script.count(
f"--property=TimeoutStartSec={FABRIC_OPERATION_START_TIMEOUT_SECONDS}s"
) == 2
assert "RuntimeMaxSec=" not in script
assert script.count('/bin/sh "$OPERATION_DIR/runner"') == 2
assert not re.search(r'^\s*"\$OPERATION_DIR/runner"', script, re.MULTILINE)
assert "/bin/sh -c" not in script[systemd_start:]
assert "exec >/dev/null 2>&1" in script
assert "operation=%s state=%s checkpoint=%s reason=%s" in script
assert secret not in script[systemd_start:]
def test_fabric_operation_runner_preserves_apply_exit_status_for_safe_reason():
runner = _fabric_operation_runner_script("7" * 32, "8" * 64)
run_apply = runner.index('if /bin/sh "$APPLY"; then')
success = runner.index(
'/bin/sh "$UPDATE" succeeded complete none', run_apply)
capture = runner.index("else\n apply_rc=$?\nfi", success)
tool_reason = runner.index("70) reason=tool_missing", capture)
journal_reason = runner.index("79) reason=journal_conflict", tool_reason)
failed = runner.index(
'/bin/sh "$UPDATE" failed "$checkpoint" "$reason"',
journal_reason)
assert run_apply < success < capture < tool_reason < journal_reason < failed
assert runner.count('/bin/sh "$UPDATE"') == 4
assert not re.search(r'^\s*"\$UPDATE"', runner, re.MULTILINE)
@pytest.mark.parametrize(("apply_exit", "checkpoint", "expected_calls"), [
(0, "dispatch", ["running dispatch none", "succeeded complete none"]),
(79, "journal_recovery", [
"running dispatch none",
"failed journal_recovery journal_conflict",
]),
(79, "journal_artifact_restore", [
"running dispatch none",
"failed journal_artifact_restore journal_conflict",
]),
(79, "journal_nft_restore", [
"running dispatch none",
"failed journal_nft_restore journal_conflict",
]),
(79, "journal_runtime_restore", [
"running dispatch none",
"failed journal_runtime_restore journal_conflict",
]),
(79, "journal_finalize", [
"running dispatch none",
"failed journal_finalize journal_conflict",
]),
*[(71, checkpoint, [
"running dispatch none", f"failed {checkpoint} {reason}",
]) for checkpoint, reason in (
("preflight", "command_failed"),
("preflight_sysctl", "sysctl_conflict"),
("preflight_tools", "tool_missing"),
("preflight_artifacts", "ownership_conflict"),
("preflight_firewall", "firewall_conflict"),
("preflight_vpc", "readback_failed"),
("preflight_underlay", "underlay_conflict"),
("preflight_routes", "underlay_conflict"),
("preflight_persistence", "persistence_conflict"),
("journal_dispatch", "journal_conflict"),
)],
])
def test_fabric_operation_runner_uses_shell_for_non_executable_spool_scripts(
tmp_path, monkeypatch, apply_exit, checkpoint, expected_calls):
operation_id = "7" * 32
operation_root = tmp_path / "fabric-operations"
operation_dir = operation_root / operation_id
operation_dir.mkdir(parents=True)
calls = operation_dir / "calls"
monkeypatch.setattr(
"virsh_panel.fabric.FABRIC_OPERATION_ROOT", str(operation_root))
update = operation_dir / "update"
update.write_text(
"#!/bin/sh\n"
f"printf '%s\\n' \"$*\" >> {shlex.quote(str(calls))}\n",
encoding="utf-8",
)
apply = operation_dir / "apply"
apply.write_text(
f"#!/bin/sh\nexit {apply_exit}\n",
encoding="utf-8",
)
(operation_dir / "status").write_text(
"\n".join((
"schema=1",
f"operation_id={operation_id}",
"status=running",
"phase=apply",
f"checkpoint={checkpoint}",
"reason=none",
"created_at=100",
"started_at=101",
"updated_at=101",
"finished_at=0",
"",
)),
encoding="utf-8",
)
runner = operation_dir / "runner"
runner.write_text(
_fabric_operation_runner_script(operation_id, "8" * 64),
encoding="utf-8",
)
for spool_script in (update, apply, runner):
spool_script.chmod(0o600)
result = subprocess.run(
["/bin/sh", str(runner)],
check=False,
capture_output=True,
text=True,
)
assert result.returncode == (0 if apply_exit == 0 else 1)
assert calls.read_text(encoding="utf-8").splitlines() == expected_calls
def test_fabric_operation_spool_is_ephemeral_bounded_and_fail_closed():
operation_id = "8" * 32
script = _fabric_operation_dispatch_script(
operation_id, "#!/bin/sh\nexit 0\n", "9" * 64)
assert FABRIC_OPERATION_ROOT == "/run/virt-cloud/fabric-operations"
assert f"ROOT={FABRIC_OPERATION_ROOT}" in script
assert (
f'if [ "$operation_count" -ge {MAX_FABRIC_OPERATION_DIRECTORIES} ]; '
"then"
) in script
assert '"__VC_FABRIC_OPERATION_DISPATCHED__=limit"' in script
assert 'case "$operation_name" in' in script
assert "*[!0-9a-f]*) exit 71" in script
assert '[ "${#operation_name}" = 32 ] || exit 71' in script
assert 'validate_retained_operation "$operation_path" || exit 71' in script
assert '[ "$operation_hidden" = "$LOCK" ] || exit 71' in script
limit = script.index(
f'if [ "$operation_count" -ge {MAX_FABRIC_OPERATION_DIRECTORIES} ];')
validate = script.index(
'validate_retained_operation "$oldest_terminal_path"', limit)
terminal = script.index(
'succeeded|failed|uncertain|interrupted)', validate)
remove = script.index(
'for retained_file in manifest status apply update runner; do',
terminal)
rmdir = script.index('rmdir -- "$oldest_terminal_path"', remove)
limit_result = script.index(
'"__VC_FABRIC_OPERATION_DISPATCHED__=limit"', rmdir)
assert limit < validate < terminal < remove < rmdir < limit_result
assert 'rm -rf -- "$oldest_terminal_path"' not in script
def test_fabric_operation_existing_attempt_recovers_only_orphaned_queue():
operation_id = "9" * 32
script = _fabric_operation_dispatch_script(
operation_id, "#!/bin/sh\nexit 0\n", "a" * 64)
queued = script.index('if [ "$retained_state" = queued ]; then')
unit_read = script.index(
'"$SYSTEMCTL" show "$UNIT" --property=LoadState --value',
queued)
missing = script.index("not-found)", unit_read)
recover = script.index("recover_operation_unit", missing)
loaded = script.index("loaded)", recover)
existing = script.index(
'"__VC_FABRIC_OPERATION_DISPATCHED__=existing"', loaded)
existing_exit = script.index(" exit 0", existing)
assert queued < unit_read < missing < recover < loaded < existing
assert existing < existing_exit
assert script.count("recover_operation_unit") == 2
recover_function = script[script.index("recover_operation_unit()"):
script.index("validate_retained_operation()")]
assert recover_function.count('"$SYSTEMD_RUN"') == 1
assert '/bin/sh "$OPERATION_DIR/runner"' in recover_function
assert script.count('/bin/sh "$OPERATION_DIR/update"') == 3
assert not re.search(
r'^\s*"\$OPERATION_DIR/update"', script, re.MULTILINE)
assert (
'[ "$(sed -n \'2p\' "$OPERATION_DIR/manifest")" = '
'"operation_id=$OPERATION_ID" ]'
) in script
assert (
'[ "$(sed -n \'3p\' "$OPERATION_DIR/manifest")" = '
'"plan_digest=$PLAN_DIGEST" ]'
) in script
assert (
'[ "$(sed -n \'4p\' "$OPERATION_DIR/manifest")" = '
'"script_digest=$SCRIPT_DIGEST" ]'
) in script
assert (
'case "${operation_file##*/}" in '
"manifest|status|apply|update|runner) : ;; *) exit 71 ;; esac"
) in script
assert 'for operation_file in manifest status; do' in script
assert ' = 0:600 ] || exit 71' in script
assert 'for operation_file in apply update runner; do' in script
assert ' = 0:700 ] || exit 71' in script
assert (
'$SHA256SUM "$OPERATION_DIR/apply"' in script
and '$SHA256SUM "$OPERATION_DIR/update"' in script
and '$SHA256SUM "$OPERATION_DIR/runner"' in script
)
def test_fabric_dispatch_retires_only_proven_unstarted_legacy_spool():
operation_id = "9" * 32
script = _fabric_operation_dispatch_script(
operation_id, "#!/bin/sh\nexit 0\n", "a" * 64)
retained = script.index(
'if [ "$retained_apply:$retained_update:$retained_runner" !=')
queued = script.index(
'queued:dispatch:none ] || exit 71', retained)
never_started = script.index(
'[ "$retained_started:$retained_finished" = 0:0 ] || exit 71',
queued)
never_updated = script.index(
'[ "$retained_updated" = "$retained_created" ] || exit 71',
never_started)
absent = script.index('not-found:inactive) : ;;', never_updated)
exec_failed = script.index(
'exit-code:exited:203 ] || exit 71', absent)
retire = script.index(
'status=failed phase=apply checkpoint=dispatch', exec_failed)
current_manifest = script.index(
'"plan_digest=$PLAN_DIGEST" ] || exit 71', retire)
assert retained < queued < never_started < never_updated < absent
assert absent < exec_failed
assert exec_failed < retire < current_manifest
assert '/bin/sh "$OPERATION_DIR/update"' not in script[
retained:retire]
assert "possibly started operation" in script
def test_fabric_async_apply_records_recovery_before_new_transaction():
current = plan()
script = _apply_script(
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2})
recovery_checkpoint = script.index(
"vc_fabric_operation_checkpoint journal_recovery")
recovery = script.index(
'case "$journal_existing_state" in', recovery_checkpoint)
preflight = script.index(
"vc_fabric_operation_checkpoint preflight", recovery)
transaction = script.index(
"vc_fabric_operation_checkpoint transaction", preflight)
assert recovery_checkpoint < recovery < preflight < transaction
for checkpoint in FABRIC_OPERATION_CHECKPOINTS - {"dispatch"}:
assert (
f"vc_fabric_operation_checkpoint {checkpoint}" in script
or f"vc_fabric_recovery_checkpoint {checkpoint}" in script
)
def test_fabric_journal_recovery_reports_fixed_safe_subcheckpoints():
helper = _BOOT_HELPER
artifact = helper.index(
"vc_fabric_recovery_checkpoint journal_artifact_restore")
artifact_restore = helper.index("journal_sysctl_restore=no", artifact)
nft = helper.index(
"vc_fabric_recovery_checkpoint journal_nft_restore",
artifact_restore)
nft_read = helper.index(
"journal_tables=$($NFT list tables", nft)
runtime = helper.index(
"vc_fabric_recovery_checkpoint journal_runtime_restore", nft_read)
runtime_restore = helper.index(
'if [ "$journal_iface_before" = yes ]; then', runtime)
finalize = helper.index(
"vc_fabric_recovery_checkpoint journal_finalize", runtime_restore)
helper_cleanup = helper.index(
'if [ "$journal_helper_before" = absent ]; then', finalize)
assert artifact < artifact_restore < nft < nft_read
assert nft_read < runtime < runtime_restore < finalize < helper_cleanup
checkpoint_function = helper[
helper.index("vc_fabric_recovery_checkpoint()"):
helper.index("vc_legacy_firewalld_state_detected()")
]
assert "/run/virt-cloud/fabric-operations/$VC_FABRIC_OPERATION_ID" in (
checkpoint_function)
assert "0:0:700" in checkpoint_function
assert '/bin/sh "$vc_recovery_update"' in checkpoint_function
assert "|| true" in checkpoint_function
def test_fabric_apply_operation_checkpoint_uses_shell_on_noexec_run():
current = plan()
script = _apply_script(
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2})
assert '/bin/sh "$vc_operation_update"' in script
assert not re.search(
r'^\s*"\$vc_operation_update"', script, re.MULTILINE)
class FakeManager:
ssh_binary = "ssh"
def __init__(self, public_key=KEY1, extra_address=False, self_route=False,
config_digest=None, ping_ok=True, endpoint_device="eth0",
host=None, fabric_endpoint_ip="",
firewall_mode=FIREWALL_MODE_NATIVE,
live_public_key=None):
self.public_key = public_key
self.live_public_key = live_public_key or public_key
self.extra_address = extra_address
self.self_route = self_route
self.config_digest = config_digest
self.ping_ok = ping_ok
self.endpoint_device = endpoint_device
self.scripts = []
self.timeouts = []
self.reprobe_calls = 0
self.admin_state = "up"
self.ip_forward = "1"
self.rp_filter = "2"
self.rp_filter_default = "2"
self.rp_filter_iface = "2"
self.rp_filter_effective = "2"
self.sysctl_persistence = "owned"
self.emergency_state = "clear"
self.firewall_mode = firewall_mode
effective_host = host or (
"203.0.113.1" if public_key == KEY1 else "203.0.113.2")
self.effective_host = effective_host
self.underlay_endpoint = fabric_endpoint_ip or effective_host
self.profile = {
"host": effective_host,
"fabric_endpoint_ip": fabric_endpoint_ip,
"ssh_config_file": "/tmp/fake-ssh-config",
"ssh_target": "vp-managed",
"key_locked": False,
"capabilities": {
"vpc_ready": True,
"wireguard": {
"interface": FABRIC_INTERFACE,
"deployment_mode": "networkmanager_nmcli",
"ready": True,
},
"network": {
"network_manager": "networkmanager",
"network_manager_supported": True,
"firewall_manager": "nftables",
"firewall_mode": firewall_mode,
"ip_probe_valid": True,
"local_ipv4_addresses": [
fabric_endpoint_ip or effective_host,
],
"local_ipv4_interfaces": {
fabric_endpoint_ip or effective_host:
[endpoint_device],
},
"rp_filter": 2,
"rp_filter_default": 2,
"rp_filter_default_effective": 2,
"rp_filter_interfaces": {
endpoint_device: 1,
},
"networkmanager_persistent_interface_zones": {
endpoint_device: "",
},
"ifcfg_zone_interfaces": [],
},
"nftables": {
"ready": True,
"firewall_mode": firewall_mode,
},
},
}
def current(self):
return self.profile
def reprobe_current(self):
self.reprobe_calls += 1
return self.profile["capabilities"]
def _exec(self, argv, *, timeout, input_text=None):
assert argv[-1] == "/bin/sh -s"
self.timeouts.append(timeout)
self.scripts.append(input_text)
if input_text.startswith("#!/bin/sh\nset -eu") and "__VC_FABRIC_PUBLIC_KEY__=" in input_text:
output = f"__VC_FABRIC_PUBLIC_KEY__={self.public_key}\n"
elif "__VC_FABRIC_APPLIED__" in input_text:
self.live_public_key = self.public_key
output = "__VC_FABRIC_APPLIED__\n"
else:
now = int(time.time())
peer = KEY2 if self.public_key == KEY1 else KEY1
address = "100.64.1.1" if self.public_key == KEY1 else "100.64.1.2"
target = "100.64.1.2" if self.public_key == KEY1 else "100.64.1.1"
endpoint = "203.0.113.2:51820" if self.public_key == KEY1 else "203.0.113.1:51820"
extra_address = (
"__VC_FABRIC_ADDRESS__=100.64.1.99\n"
if self.extra_address else ""
)
self_route = (
f"__VC_FABRIC_ROUTE__={address}|local|{FABRIC_INTERFACE}\n"
if self.self_route else ""
)
output = (
f"__VC_FABRIC_OWNER__={FABRIC_OWNER_MARKER}\n"
"__VC_FABRIC_CONFIG_FABRIC__=hk-fabric\n"
"__VC_FABRIC_CONFIG_REVISION__=1\n"
f"__VC_FABRIC_CONFIG_DIGEST__={self.config_digest or plan().digest}\n"
f"__VC_FABRIC_CONFIG_FIREWALL__={self.firewall_mode}\n"
"__VC_FABRIC_CONFIG_NETWORK__=100.64.1.0/24\n"
f"__VC_FABRIC_CONFIG_OVERLAY__={address}\n"
"__VC_FABRIC_CONFIG_OK__=yes\n"
f"__VC_FABRIC_CONFIG_PEER__={peer}|{endpoint}|{target}/32\n"
f"__VC_FABRIC_CONFIG_ROUTE__={target}/32\n"
f"__VC_FABRIC_PUBLIC_KEY__={self.public_key}\n"
"__VC_FABRIC_INTERFACE__=wireguard\n"
f"__VC_FABRIC_LIVE_PUBLIC_KEY__={self.live_public_key}\n"
"__VC_FABRIC_LISTEN__=51820\n"
f"__VC_FABRIC_ADDRESS__={address}\n"
f"{extra_address}"
f"__VC_FABRIC_PEER__={peer}|{endpoint}|{target}/32\n"
f"__VC_FABRIC_HANDSHAKE__={peer}|{now}\n"
f"{self_route}"
f"__VC_FABRIC_ROUTE__={target}/32|main|{FABRIC_INTERFACE}\n"
f"__VC_FABRIC_PATH__={target}|{FABRIC_INTERFACE}\n"
f"__VC_FABRIC_ENDPOINT_PATH__={endpoint.split(':', 1)[0]}|{self.endpoint_device}|{self.underlay_endpoint}\n"
f"__VC_FABRIC_PING__={target}|{'ok' if self.ping_ok else 'failed'}\n"
f"__VC_FABRIC_ROUTE_OVERLAP__={target}|clean\n"
"__VC_FABRIC_MANAGER_MODE__=networkmanager_nmcli\n"
"__VC_FABRIC_MANAGER_OWNER__=owned\n"
"__VC_FABRIC_MANAGER_ACTIVE__=active\n"
f"__VC_FABRIC_FIREWALL_MODE__={self.firewall_mode}\n"
"__VC_FABRIC_FIREWALL_STATE__=owned\n"
f"__VC_FABRIC_ADMIN_STATE__={self.admin_state}\n"
f"__VC_FABRIC_SYSCTL__ip_forward={self.ip_forward}\n"
f"__VC_FABRIC_SYSCTL__rp_filter={self.rp_filter}\n"
f"__VC_FABRIC_SYSCTL__rp_filter_default={self.rp_filter_default}\n"
f"__VC_FABRIC_SYSCTL__rp_filter_effective={self.rp_filter_effective}\n"
f"__VC_FABRIC_RPF__wg-fabric0={self.rp_filter_iface}\n"
f"__VC_FABRIC_SYSCTL_PERSISTENCE__={self.sysctl_persistence}\n"
"__VC_FABRIC_ENABLED__=enabled\n"
"__VC_FABRIC_ACTIVE__=active\n"
"__VC_FABRIC_TIMER_ENABLED__=enabled\n"
"__VC_FABRIC_TIMER_ACTIVE__=active\n"
"__VC_FABRIC_NFT_GUARD__=owned\n"
"__VC_FABRIC_NFT_PERSISTENCE__=owned\n"
f"__VC_FABRIC_EMERGENCY__={self.emergency_state}\n"
"__VC_FABRIC_VPC_NFT_STATE__=owned\n"
"__VC_FABRIC_KEY_SECURE__=yes\n"
"__VC_FABRIC_CONFIG_SECURE__=yes\n"
"__VC_FABRIC_HELPER__=owned\n"
"__VC_FABRIC_UNIT__=owned\n"
"__VC_FABRIC_REPAIR__=owned\n"
"__VC_FABRIC_TIMER__=owned\n"
"__VC_FABRIC_LIBVIRT_GATE__=owned\n"
"__VC_FABRIC_LIBVIRT_DEPENDENCIES__=owned\n"
)
return subprocess.CompletedProcess(argv, 0, output, "")
class AsyncOperationManager(FakeManager):
def __init__(self, *args, operation_id, status="queued",
checkpoint="dispatch", reason="none", **kwargs):
super().__init__(*args, **kwargs)
self.operation_id = operation_id
self.operation_status = status
self.operation_checkpoint = checkpoint
self.operation_reason = reason
self.dispatch_result = "yes"
self.disconnect_dispatch = False
def _status_output(self):
started_at = 0
finished_at = 0
if self.operation_status != "queued":
started_at = 101
if self.operation_status in {
"succeeded", "failed", "uncertain", "interrupted"}:
finished_at = 103
return _operation_status_output(
self.operation_id,
status=self.operation_status,
checkpoint=self.operation_checkpoint,
reason=self.operation_reason,
started_at=started_at,
updated_at=103 if finished_at else 102,
finished_at=finished_at,
)
def _exec(self, argv, *, timeout, input_text=None):
assert argv[-1] == "/bin/sh -s"
self.timeouts.append(timeout)
self.scripts.append(input_text)
if 'SYSTEMD_RUN=$(find_tool ' in input_text:
if self.disconnect_dispatch:
raise RemoteHostError(
"ambiguous disconnect with /secret/path", 502,
"host_remote_command_failed")
output = (
f"__VC_FABRIC_OPERATION_DISPATCHED__={self.dispatch_result}\n"
f"__VC_FABRIC_OPERATION_ID__={self.operation_id}\n"
)
return subprocess.CompletedProcess(argv, 0, output, "")
if "__VC_FABRIC_OPERATION_SCHEMA__=$schema" in input_text:
return subprocess.CompletedProcess(
argv, 0, self._status_output(), "")
raise AssertionError("unexpected asynchronous Fabric command")
def test_adapter_dispatches_apply_and_reads_status_without_waiting_for_job():
current = plan()
operation_id = "b" * 32
manager = AsyncOperationManager(KEY1, operation_id=operation_id)
adapter = FabricHostAdapter(manager)
result = adapter.dispatch_apply(
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
operation_id)
assert result == {
"operation_id": operation_id,
"status": "queued",
"phase": "apply",
"checkpoint": "dispatch",
"reason": "none",
"created_at": 100,
"started_at": None,
"updated_at": 102,
"finished_at": None,
}
serialized = json.dumps(result)
assert KEY1 not in serialized
assert KEY2 not in serialized
assert "/run/virt-cloud" not in serialized
assert manager.timeouts == [
FABRIC_REMOTE_READ_TIMEOUT_SECONDS,
FABRIC_REMOTE_READ_TIMEOUT_SECONDS,
]
assert len(manager.scripts) == 2
assert "--no-block" in manager.scripts[0]
assert "__VC_FABRIC_APPLIED__" in manager.scripts[0]
assert "__VC_FABRIC_OPERATION_STATUS__=$state" in manager.scripts[1]
@pytest.mark.parametrize(("status", "checkpoint", "reason"), [
("running", "journal_recovery", "none"),
("succeeded", "complete", "none"),
("failed", "nft_install", "nft_failed"),
])
def test_adapter_can_read_exact_attempt_after_dispatch_connection_loss(
status, checkpoint, reason):
current = plan()
operation_id = "c" * 32
manager = AsyncOperationManager(
KEY1, operation_id=operation_id, status=status,
checkpoint=checkpoint, reason=reason)
manager.disconnect_dispatch = True
adapter = FabricHostAdapter(manager)
with pytest.raises(FabricError) as error:
adapter.dispatch_apply(
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
operation_id)
assert error.value.error_code == "fabric_host_connection_failed"
assert error.value.failure_phase == "apply"
assert len(manager.scripts) == 1
# A lost dispatch response is reconciled only by its exact operation id;
# the adapter does not submit the mutation a second time.
manager.disconnect_dispatch = False
observed = adapter.apply_status(operation_id)
assert observed["operation_id"] == operation_id
assert observed["status"] == status
assert observed["checkpoint"] == checkpoint
assert observed["reason"] == reason
assert len(manager.scripts) == 2
assert 'SYSTEMD_RUN=$(find_tool ' not in manager.scripts[-1]
def test_adapter_rejects_cross_attempt_status_without_redispatch():
operation_id = "d" * 32
manager = AsyncOperationManager(KEY1, operation_id="e" * 32)
adapter = FabricHostAdapter(manager)
with pytest.raises(FabricError) as error:
adapter.apply_status(operation_id)
assert error.value.error_code == "fabric_operation_status_invalid"
assert len(manager.scripts) == 1
assert 'SYSTEMD_RUN=$(find_tool ' not in manager.scripts[0]
def test_adapter_reports_exact_missing_operation_without_leaking_paths():
operation_id = "a" * 32
manager = AsyncOperationManager(KEY1, operation_id=operation_id)
manager._status_output = lambda: (
"__VC_FABRIC_OPERATION_MISSING__=yes\n"
f"__VC_FABRIC_OPERATION_ID__={operation_id}\n"
)
adapter = FabricHostAdapter(manager)
with pytest.raises(FabricError) as error:
adapter.apply_status(operation_id)
assert error.value.status == 409
assert error.value.error_code == "fabric_operation_not_found"
assert error.value.failure_phase == "observe"
assert "/run/virt-cloud" not in str(error.value)
assert KEY1 not in str(error.value)
assert len(manager.scripts) == 1
assert 'SYSTEMD_RUN=$(find_tool ' not in manager.scripts[0]
def test_adapter_validates_operation_id_before_remote_dispatch():
current = plan()
manager = AsyncOperationManager(KEY1, operation_id="f" * 32)
adapter = FabricHostAdapter(manager)
with pytest.raises(FabricError) as error:
adapter.dispatch_apply(
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
"../not-an-operation")
assert error.value.error_code == "fabric_operation_invalid"
assert manager.scripts == []
def test_adapter_readback_requires_exact_peer_route_and_persistence():
current = plan()
manager = FakeManager(KEY1)
adapter = FabricHostAdapter(manager)
assert adapter.identity() == KEY1
assert adapter.observe(
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}
)["transport_verified"] is True
rendered_observe = manager.scripts[-1]
digest_assignment = f"EXPECTED_DIGEST={shlex.quote(current.digest)}"
assert rendered_observe.index(digest_assignment) < rendered_observe.index(
"vc_fabric_nm_profile_owned expected"
)
def test_adapter_uses_stage_specific_remote_timeouts():
current = plan()
manager = FakeManager(KEY1)
adapter = FabricHostAdapter(manager)
public_keys = {"host-a": KEY1, "host-b": KEY2}
assert adapter.identity() == KEY1
adapter.apply(current, current.nodes[0], public_keys)
adapter.observe(current, current.nodes[0], public_keys)
assert [
FABRIC_REMOTE_READ_TIMEOUT_SECONDS,
FABRIC_REMOTE_APPLY_TIMEOUT_SECONDS,
FABRIC_REMOTE_READ_TIMEOUT_SECONDS,
] == [35, 300, 35]
assert manager.timeouts == [35, 300, 35]
@pytest.mark.parametrize("failure_phase", ["identity", "apply", "observe"])
def test_adapter_remote_failure_exposes_only_whitelisted_phase(failure_phase):
class FailedRemoteManager(FakeManager):
def _exec(self, argv, *, timeout, input_text=None):
self.scripts.append(input_text)
return subprocess.CompletedProcess(
argv, 71, "", "secret /root/path private-key-material")
current = plan()
adapter = FabricHostAdapter(FailedRemoteManager(KEY1))
with pytest.raises(FabricError) as error:
if failure_phase == "identity":
adapter.identity()
elif failure_phase == "apply":
adapter.apply(
current, current.nodes[0],
{"host-a": KEY1, "host-b": KEY2})
else:
adapter.observe(
current, current.nodes[0],
{"host-a": KEY1, "host-b": KEY2})
assert error.value.error_code == "fabric_remote_command_failed"
assert error.value.failure_phase == failure_phase
safe_error = json.dumps({
"message": str(error.value),
"error_code": error.value.error_code,
"failure_phase": error.value.failure_phase,
})
assert "secret" not in safe_error
assert "/root/path" not in safe_error
assert "private-key-material" not in safe_error
def test_profile_firewall_mode_rejects_missing_or_crossed_capability_evidence():
manager = FakeManager(KEY1)
assert _profile_firewall_mode(manager.profile) == FIREWALL_MODE_NATIVE
manager.profile["capabilities"]["network"].update({
"firewall_manager": "firewalld",
"firewall_mode": "firewalld_nft",
})
manager.profile["capabilities"]["nftables"]["firewall_mode"] = (
"firewalld_nft")
with pytest.raises(FabricError) as error:
_profile_firewall_mode(manager.profile)
assert error.value.error_code == "fabric_host_firewall_mode_conflict"
manager.profile["capabilities"]["nftables"]["firewall_mode"] = (
FIREWALL_MODE_NATIVE)
with pytest.raises(FabricError) as error:
_profile_firewall_mode(manager.profile)
assert error.value.error_code == "fabric_host_firewall_mode_conflict"
def test_adapter_rejects_an_unexpected_overlay_address():
current = plan()
manager = FakeManager(KEY1, extra_address=True)
observed = FabricHostAdapter(manager).observe(
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}
)
assert observed["config_ok"] is False
assert observed["state"] == "degraded"
def test_adapter_ignores_the_kernel_connected_route_for_its_own_address():
current = plan()
manager = FakeManager(KEY1, self_route=True)
observed = FabricHostAdapter(manager).observe(
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}
)
assert observed["config_ok"] is True
def test_adapter_requires_admin_up_and_fabric_sysctl_readback():
current = plan()
manager = FakeManager(KEY1)
manager.admin_state = "down"
assert FabricHostAdapter(manager).observe(
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}
)["config_ok"] is False
manager.admin_state = "up"
manager.rp_filter_iface = "1"
observed = FabricHostAdapter(manager).observe(
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}
)
assert observed["config_ok"] is False
assert observed["observed"]["admin_state"] == "up"
assert observed["observed"]["rp_filters"][FABRIC_INTERFACE] == "1"
def test_adapter_accepts_strict_global_defaults_when_owned_fabric_is_loose():
current = plan()
manager = FakeManager(KEY1)
manager.rp_filter = "0"
manager.rp_filter_default = "1"
manager.rp_filter_iface = "2"
manager.rp_filter_effective = "2"
observed = FabricHostAdapter(manager).observe(
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
)
assert observed["config_ok"] is True
assert observed["observed"]["sysctls"] == {
"ip_forward": "1",
"rp_filter": "0",
"rp_filter_default": "1",
"rp_filter_effective": "2",
}
def test_adapter_requires_exact_owned_sysctl_persistence():
current = plan()
manager = FakeManager(KEY1)
manager.sysctl_persistence = "foreign"
observed = FabricHostAdapter(manager).observe(
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
)
assert observed["config_ok"] is False
assert observed["observed"]["sysctl_persistence"] == "foreign"
@pytest.mark.parametrize("emergency_state", ["present", "unknown"])
def test_adapter_never_publishes_active_while_emergency_guard_remains(
emergency_state):
current = plan()
manager = FakeManager(KEY1)
manager.emergency_state = emergency_state
observed = FabricHostAdapter(manager).observe(
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
)
assert observed["config_ok"] is False
assert observed["state"] == "degraded"
assert observed["observed"]["emergency_state"] == emergency_state
def test_adapter_rejects_stale_config_digest_and_failed_probe():
current = plan()
stale = FakeManager(KEY1, config_digest="0" * 64)
observed = FabricHostAdapter(stale).observe(
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}
)
assert observed["config_ok"] is False
def test_adapter_rejects_endpoint_egress_drift_to_fabric_or_missing_device():
current = plan()
for device in (FABRIC_INTERFACE, ""):
observed = FabricHostAdapter(FakeManager(KEY1, endpoint_device=device)).observe(
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}
)
assert observed["config_ok"] is False
failed_ping = FakeManager(KEY1, ping_ok=False)
observed = FabricHostAdapter(failed_ping).observe(
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}
)
assert observed["config_ok"] is True
assert observed["state"] == "active"
assert observed["transport_verified"] is False
def test_adapter_rejects_live_public_key_drift_from_persistent_key():
current = plan()
observed = FabricHostAdapter(FakeManager(
KEY1, live_public_key=KEY3,
)).observe(
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
)
assert observed["public_key"] == KEY1
assert observed["observed"]["live_public_key"] == KEY3
assert observed["config_ok"] is False
assert observed["state"] == "degraded"
class MemoryStore:
def __init__(self):
self.values = {}
def get(self, kind, key):
return self.values.get((kind, key))
def set(self, kind, key, value):
self.values[(kind, key)] = value
def list(self, kind):
return {key: value for (row_kind, key), value in self.values.items()
if row_kind == kind}
class FakeAdapter:
failures = set()
observe_failures = set()
def __init__(self, manager, profile=None):
self.manager = manager
def identity(self):
return self.manager.public_key
def apply(self, *_args):
if self.manager.host_id in self.failures:
raise FabricError("offline", 502, "fabric_remote_command_failed")
def observe(self, current, host, keys):
if (self.manager.host_id in self.failures
or self.manager.host_id in self.observe_failures):
raise FabricError("offline", 502, "fabric_remote_command_failed")
return {
"state": "active", "transport_verified": True,
"observed": {"interface": FABRIC_INTERFACE},
}
class Registry:
def __init__(self, components):
self.components = components
async def get(self, host_id):
return self.components[host_id]
class UnconfiguredRegistry:
def __init__(self):
self.components = {}
self.get_called = False
async def is_configured(self, _host_id):
return False
async def get(self, _host_id):
self.get_called = True
raise AssertionError("readiness probe constructed an unknown host")
class ProbeManager:
def __init__(self, host_id, *, host="203.0.113.10",
fabric_endpoint_ip="", ready=True, failure=None, counter=None):
self.host_id = host_id
self.failure = failure
self.counter = counter
self.profile = {
"host_id": host_id,
"host": host,
"fabric_endpoint_ip": fabric_endpoint_ip,
"region": "hk",
"ssh_config_file": "/secret/ssh/config",
"private_key": "must-not-leak",
}
effective_endpoint = fabric_endpoint_ip or host
endpoint_is_ipv4 = bool(re.fullmatch(
r"(?:[0-9]{1,3}\.){3}[0-9]{1,3}", effective_endpoint))
self.capabilities = {
"vpc_ready": ready,
"reasons": [] if ready else [
"firewall_owner_not_nftables", "rp_filter_not_compatible",
],
"os": {"id": "almalinux", "version_id": "10",
"name": "must-not-be-projected"},
"network": {
"ip_probe_valid": True,
"network_manager": "networkmanager",
"network_manager_expected": "networkmanager",
"network_manager_supported": True,
"firewall_manager": "nftables" if ready else "firewalld",
"firewall_mode": "native_nftables" if ready else "unsupported",
"ip_forward": 1,
"rp_filter": 2 if ready else 1,
"rp_filter_default": 1,
"rp_filter_default_effective": 2 if ready else 1,
"rp_filter_interfaces": (
{"br0": 1} if endpoint_is_ipv4 else {}),
"local_ipv4_addresses": (
[effective_endpoint] if endpoint_is_ipv4 else []),
"local_ipv4_interfaces": (
{effective_endpoint: ["br0"]}
if endpoint_is_ipv4 else {}),
"fabric_rp_filter_raw": None,
"fabric_rp_filter_effective": None,
"raw_output": "must-not-leak",
},
"wireguard": {
"deployment_mode": "networkmanager_nmcli",
"ready": True,
"owned_by_panel": False,
"ownership_marker": "must-not-be-projected",
},
"nftables": {
"ready": ready,
"firewall_mode": "native_nftables" if ready else "unsupported",
"service_active": ready,
"service_persistent": ready,
},
"network_roles": {
"public": {"selector": "bridge:br0", "verified": True},
},
"tools": {"wg": {"path": "/usr/bin/wg"}},
"raw_output": "must-not-leak",
}
def current(self):
return self.profile
def reprobe_current_snapshot(self):
if self.counter is not None:
with self.counter["lock"]:
self.counter["active"] += 1
self.counter["maximum"] = max(
self.counter["maximum"], self.counter["active"])
if self.counter["active"] == self.counter.get("target"):
self.counter["gate"].set()
try:
gate = self.counter.get("gate")
if gate is not None:
gate.wait(timeout=2)
else:
time.sleep(0.02)
finally:
with self.counter["lock"]:
self.counter["active"] -= 1
if self.failure is not None:
raise self.failure
return self.profile, self.capabilities
def test_fabric_components_treat_host_region_as_a_display_label():
components = {}
labels = {
"host-a": "中国香港",
"host-b": "Hong Kong",
}
for host_id, key in (("host-a", KEY1), ("host-b", KEY2)):
manager = FakeManager(key)
manager.profile["host_id"] = host_id
manager.profile["region"] = labels[host_id]
components[host_id] = SimpleNamespace(manager=manager)
resolved = asyncio.run(FabricCoordinator(
MemoryStore(), Registry(components),
)._components(plan()))
assert set(resolved) == {"host-a", "host-b"}
assert resolved["host-a"][1]["region"] == "中国香港"
assert resolved["host-b"][1]["region"] == "Hong Kong"
assert all(component.manager.reprobe_calls == 1
for component, _profile in resolved.values())
def test_fabric_components_accept_configured_local_underlay_endpoints():
components = {}
for host_id, key, endpoint in (
("host-a", KEY1, "203.0.113.1"),
("host-b", KEY2, "203.0.113.2"),
):
manager = FakeManager(
key, host="host.example.test", fabric_endpoint_ip=endpoint)
manager.profile["host_id"] = host_id
components[host_id] = SimpleNamespace(manager=manager)
resolved = asyncio.run(FabricCoordinator(
MemoryStore(), Registry(components),
)._components(plan()))
assert set(resolved) == {"host-a", "host-b"}
def test_native_ssh_ipv4_fallback_rejects_nat_without_local_mapping():
first = FakeManager(KEY1, host="203.0.113.1")
first.profile["host_id"] = "host-a"
first.profile["capabilities"]["network"]["local_ipv4_addresses"] = []
first.profile["capabilities"]["network"]["local_ipv4_interfaces"] = {}
first.profile["capabilities"]["network"]["rp_filter_interfaces"] = {}
second = FakeManager(KEY2)
second.profile["host_id"] = "host-b"
with pytest.raises(FabricError) as error:
asyncio.run(FabricCoordinator(
MemoryStore(), Registry({
"host-a": SimpleNamespace(manager=first),
"host-b": SimpleNamespace(manager=second),
}),
)._components(plan()))
assert error.value.error_code == "fabric_endpoint_not_local"
@pytest.mark.parametrize(("profile_changes", "error_code"), [
({"host": "203.0.113.9"}, "fabric_endpoint_not_local"),
({"fabric_endpoint_ip": "127.0.0.1"}, "fabric_endpoint_invalid"),
])
def test_fabric_components_reject_mismatched_or_invalid_effective_endpoint(
profile_changes, error_code,
):
first = FakeManager(KEY1)
first.profile.update(profile_changes)
first.profile["host_id"] = "host-a"
second = FakeManager(KEY2)
second.profile["host_id"] = "host-b"
coordinator = FabricCoordinator(MemoryStore(), Registry({
"host-a": SimpleNamespace(manager=first),
"host-b": SimpleNamespace(manager=second),
}))
with pytest.raises(FabricError) as error:
asyncio.run(coordinator._components(plan()))
assert error.value.status == 409
assert error.value.error_code == error_code
def test_fabric_components_reject_configured_endpoint_without_local_evidence():
first = FakeManager(KEY1, fabric_endpoint_ip="203.0.113.1")
first.profile["host_id"] = "host-a"
first.profile["capabilities"]["network"]["local_ipv4_addresses"] = []
second = FakeManager(KEY2)
second.profile["host_id"] = "host-b"
coordinator = FabricCoordinator(MemoryStore(), Registry({
"host-a": SimpleNamespace(manager=first),
"host-b": SimpleNamespace(manager=second),
}))
with pytest.raises(FabricError) as error:
asyncio.run(coordinator._components(plan()))
assert error.value.status == 409
assert error.value.error_code == "fabric_endpoint_not_local"
first.profile["capabilities"]["network"]["local_ipv4_addresses"] = [
"203.0.113.1",
]
first.profile["capabilities"]["network"]["local_ipv4_interfaces"] = {
"203.0.113.1": ["br0"],
}
first.profile["capabilities"]["network"]["rp_filter_interfaces"] = {}
with pytest.raises(FabricError) as error:
asyncio.run(coordinator._components(plan()))
assert error.value.error_code == "fabric_endpoint_rp_filter_unverified"
@pytest.mark.parametrize("value", [
{},
{"host_ids": []},
{"host_ids": ["host-a"], "extra": True},
{"host_ids": ["host-a", "host-a"]},
{"host_ids": ["bad.id"]},
{"host_ids": ["host-a"] * 65},
])
def test_readiness_probe_rejects_non_exact_or_unbounded_host_ids(value):
coordinator = FabricCoordinator(MemoryStore(), Registry({}))
with pytest.raises(FabricError) as error:
asyncio.run(coordinator.probe_hosts(value))
assert error.value.status == 422
assert error.value.error_code == "fabric_probe_invalid"
def test_readiness_probe_is_bounded_isolated_and_secret_free():
counter = {
"active": 0,
"maximum": 0,
"lock": threading.Lock(),
"target": FABRIC_PROBE_CONCURRENCY,
"gate": threading.Event(),
}
components = {}
host_ids = []
for index in range(FABRIC_PROBE_CONCURRENCY + 4):
host_id = f"host-{index}"
host_ids.append(host_id)
failure = None
if index == 1:
failure = RemoteHostError(
"remote output /secret/path", 502,
"host_capability_probe_failed")
manager = ProbeManager(
host_id,
host="host.example.test" if index == 2 else f"203.0.113.{index + 1}",
ready=index != 3,
failure=failure,
counter=counter,
)
components[host_id] = SimpleNamespace(manager=manager)
coordinator = FabricCoordinator(MemoryStore(), Registry(components))
async def run_with_constrained_default_executor():
# Prove readiness uses its own 16-worker executor rather than merely
# inheriting whatever size asyncio chose for the process-wide pool.
asyncio.get_running_loop().set_default_executor(ThreadPoolExecutor(
max_workers=1, thread_name_prefix="test-default",
))
return await coordinator.probe_hosts({"host_ids": host_ids})
result = asyncio.run(run_with_constrained_default_executor())
coordinator.close()
assert [item["host_id"] for item in result["hosts"]] == host_ids
assert counter["maximum"] == FABRIC_PROBE_CONCURRENCY
assert result["hosts"][0] == {
"host_id": "host-0",
"configured": True,
"reachable": True,
"region": "hk",
"endpoint_ip": "203.0.113.1",
"endpoint_source": "ssh_host_ipv4",
"probed_at": result["hosts"][0]["probed_at"],
"vpc_ready": True,
"reasons": [],
"capabilities": {
"os_id": "almalinux",
"os_version": "10",
"network_manager": "networkmanager",
"network_manager_expected": "networkmanager",
"network_manager_supported": True,
"deployment_mode": "networkmanager_nmcli",
"firewall_manager": "nftables",
"firewall_mode": "native_nftables",
"underlay_interface": "br0",
"ip_forward": 1,
"rp_filter": 2,
"rp_filter_all": 2,
"rp_filter_default": 1,
"rp_filter_default_effective": 2,
"rp_filter_underlay_raw": 1,
"rp_filter_underlay_effective": 2,
"rp_filter_fabric_raw": None,
"rp_filter_fabric_effective": None,
"wireguard_ready": True,
"wireguard_owned": False,
"nftables_ready": True,
"nftables_active": True,
"nftables_persistent": True,
"public_selector": "bridge:br0",
"public_verified": True,
},
}
failed = result["hosts"][1]
assert failed["configured"] is True
assert failed["reachable"] is False
assert failed["error_code"] == "fabric_host_capability_probe_failed"
assert failed["reasons"] == ["fabric_host_capability_probe_failed"]
assert result["hosts"][2]["endpoint_ip"] is None
assert result["hosts"][2]["endpoint_source"] is None
assert result["hosts"][3]["vpc_ready"] is False
assert result["hosts"][3]["reasons"] == [
"firewall_owner_not_nftables", "rp_filter_not_compatible",
]
@pytest.mark.parametrize(("raw", "projected"), [
(FIREWALL_MODE_NATIVE, FIREWALL_MODE_NATIVE),
("firewalld_nft", "unsupported"),
("../../operator-data", "unsupported"),
(None, "unsupported"),
])
def test_readiness_projects_only_supported_firewall_modes(raw, projected):
result = FabricCoordinator._readiness_capabilities({
"network": {"firewall_mode": raw},
})
assert result["firewall_mode"] == projected
serialized = json.dumps(result)
assert "must-not-leak" not in serialized
assert "/secret/" not in serialized
assert "/usr/bin/wg" not in serialized
def test_readiness_probe_does_not_construct_an_unconfigured_host():
registry = UnconfiguredRegistry()
result = asyncio.run(FabricCoordinator(
MemoryStore(), registry,
).probe_hosts({"host_ids": ["host-unknown"]}))
assert registry.get_called is False
item = result["hosts"][0]
assert item == {
"host_id": "host-unknown",
"configured": False,
"reachable": False,
"region": "",
"endpoint_ip": None,
"endpoint_source": None,
"probed_at": item["probed_at"],
"vpc_ready": False,
"reasons": ["fabric_host_not_configured"],
"capabilities": {},
"error_code": "fabric_host_not_configured",
}
@pytest.mark.parametrize("address", [
"0.0.0.0",
"0.1.2.3",
"127.23.45.67",
"169.254.12.34",
"224.0.0.1",
"240.0.0.1",
"255.255.255.255",
])
def test_readiness_probe_hides_unusable_endpoint_ipv4(address):
manager = ProbeManager("host-a", host=address)
coordinator = FabricCoordinator(
MemoryStore(),
Registry({"host-a": SimpleNamespace(manager=manager)}),
)
try:
result = asyncio.run(coordinator.probe_hosts({
"host_ids": ["host-a"],
}))
finally:
coordinator.close()
readiness = result["hosts"][0]
assert readiness["configured"] is True
assert readiness["reachable"] is True
assert readiness["vpc_ready"] is False
assert readiness["endpoint_ip"] is None
assert readiness["endpoint_source"] is None
assert readiness["error_code"] == "fabric_endpoint_not_local"
def test_readiness_probe_prefers_configured_fabric_endpoint():
manager = ProbeManager(
"host-a", host="host.example.test",
fabric_endpoint_ip="10.20.30.40",
)
coordinator = FabricCoordinator(
MemoryStore(),
Registry({"host-a": SimpleNamespace(manager=manager)}),
)
try:
result = asyncio.run(coordinator.probe_hosts({
"host_ids": ["host-a"],
}))
finally:
coordinator.close()
readiness = result["hosts"][0]
assert readiness["endpoint_ip"] == "10.20.30.40"
assert readiness["endpoint_source"] == "configured"
def test_readiness_probe_rejects_native_fallback_without_local_evidence():
manager = ProbeManager("host-a", host="203.0.113.10")
manager.capabilities["network"]["local_ipv4_addresses"] = []
manager.capabilities["network"]["local_ipv4_interfaces"] = {}
manager.capabilities["network"]["rp_filter_interfaces"] = {}
coordinator = FabricCoordinator(
MemoryStore(),
Registry({"host-a": SimpleNamespace(manager=manager)}),
)
try:
result = asyncio.run(coordinator.probe_hosts({
"host_ids": ["host-a"],
}))
finally:
coordinator.close()
readiness = result["hosts"][0]
assert readiness["endpoint_ip"] == "203.0.113.10"
assert readiness["endpoint_source"] == "ssh_host_ipv4"
assert readiness["error_code"] == "fabric_endpoint_not_local"
assert readiness["reasons"] == ["fabric_endpoint_not_local"]
manager.capabilities["network"]["local_ipv4_addresses"] = [
"203.0.113.10",
]
manager.capabilities["network"]["local_ipv4_interfaces"] = {
"203.0.113.10": ["br0"],
}
coordinator = FabricCoordinator(
MemoryStore(),
Registry({"host-a": SimpleNamespace(manager=manager)}),
)
try:
result = asyncio.run(coordinator.probe_hosts({
"host_ids": ["host-a"],
}))
finally:
coordinator.close()
assert result["hosts"][0]["error_code"] == (
"fabric_endpoint_rp_filter_unverified"
)
def test_readiness_probe_rejects_overlapping_batch_without_queueing():
started = threading.Event()
release = threading.Event()
manager = ProbeManager("host-a")
def blocking_reprobe():
started.set()
release.wait(timeout=2)
return manager.profile, manager.capabilities
manager.reprobe_current_snapshot = blocking_reprobe
coordinator = FabricCoordinator(
MemoryStore(),
Registry({"host-a": SimpleNamespace(manager=manager)}),
)
async def exercise():
first = asyncio.create_task(coordinator.probe_hosts({
"host_ids": ["host-a"],
}))
try:
for _ in range(100):
if started.is_set():
break
await asyncio.sleep(0.001)
assert started.is_set()
with pytest.raises(FabricError) as error:
await coordinator.probe_hosts({"host_ids": ["host-a"]})
assert error.value.status == 409
assert error.value.error_code == "fabric_probe_busy"
finally:
release.set()
return await first
try:
result = asyncio.run(exercise())
finally:
release.set()
coordinator.close()
assert result["hosts"][0]["reachable"] is True
def test_coordinator_keeps_successful_hosts_and_blocks_next_stage(monkeypatch):
store = MemoryStore()
managers = {}
components = {}
for host_id, key in (("host-a", KEY1), ("host-b", KEY2)):
manager = FakeManager(key)
manager.host_id = host_id
manager.profile["host_id"] = host_id
manager.profile["region"] = "hk"
manager.profile["capabilities"] = {
"vpc_ready": True,
"wireguard": {
"interface": FABRIC_INTERFACE,
"deployment_mode": "networkmanager_nmcli",
"ready": True,
},
"network": {
"network_manager": "networkmanager",
"network_manager_supported": True,
"ip_probe_valid": True,
"local_ipv4_addresses": [manager.effective_host],
"local_ipv4_interfaces": {
manager.effective_host: ["eth0"],
},
"rp_filter": 2,
"rp_filter_default": 2,
"rp_filter_interfaces": {"eth0": 1},
},
}
managers[host_id] = manager
components[host_id] = SimpleNamespace(manager=manager)
monkeypatch.setattr("virsh_panel.fabric.FabricHostAdapter", FakeAdapter)
FakeAdapter.failures = {"host-b"}
result = asyncio.run(FabricCoordinator(store, Registry(components)).apply({
"fabric_id": "hk-fabric", "region": "hk",
"overlay_cidr": "100.64.1.0/24", "revision": 1,
"nodes": [
{"host_id": "host-a", "overlay_ip": "100.64.1.1",
"endpoint": "203.0.113.1:51820"},
{"host_id": "host-b", "overlay_ip": "100.64.1.2",
"endpoint": "203.0.113.2:51820"},
],
}))
assert result["state"] == "uncertain"
assert result["next_stage_allowed"] is False
states = {node["host_id"]: node["state"] for node in result["nodes"]}
assert states == {"host-a": "active", "host-b": "uncertain"}
failed = next(
node for node in result["nodes"] if node["host_id"] == "host-b")
assert failed["last_error_phase"] == "apply"
persisted = json.loads(store.get(
"fabric_node", f"{result['fabric_id']}.host-b"))
assert persisted["last_error_phase"] == "apply"
assert managers["host-a"].reprobe_calls == 1
assert managers["host-b"].reprobe_calls == 1
serialized = "\n".join(store.values.values())
assert "PRIVATE KEY" not in serialized
assert KEY1 in serialized and KEY2 in serialized
FakeAdapter.failures = set()
def test_initial_apply_identity_failure_carries_safe_phase_without_record(
monkeypatch):
current = plan()
store = MemoryStore()
components = _fabric_components(current)
class IdentityFailureAdapter(FakeAdapter):
def identity(self):
raise FabricError(
"safe", 502, "fabric_remote_command_failed")
monkeypatch.setattr(
"virsh_panel.fabric.FabricHostAdapter", IdentityFailureAdapter)
with pytest.raises(FabricError) as error:
asyncio.run(FabricCoordinator(
store, Registry(components)).apply(plan_payload(current)))
assert error.value.error_code == "fabric_remote_command_failed"
assert error.value.failure_phase == "identity"
assert store.get("fabric_desired", current.fabric_id) is None
def test_apply_records_post_apply_observe_failure_phase(monkeypatch):
current = plan()
store = MemoryStore()
components = _fabric_components(current)
monkeypatch.setattr("virsh_panel.fabric.FabricHostAdapter", FakeAdapter)
FakeAdapter.failures = set()
FakeAdapter.observe_failures = {"host-a"}
try:
result = asyncio.run(FabricCoordinator(
store, Registry(components)).apply(plan_payload(current)))
finally:
FakeAdapter.observe_failures = set()
nodes = {node["host_id"]: node for node in result["nodes"]}
assert nodes["host-a"]["last_error_code"] == (
"fabric_remote_command_failed")
assert nodes["host-a"]["last_error_phase"] == "observe"
assert "last_error_phase" not in nodes["host-b"]
persisted = json.loads(store.get(
"fabric_node", f"{current.fabric_id}.host-a"))
assert persisted["last_error_phase"] == "observe"
def test_recover_does_not_replay_an_interrupted_write():
store = MemoryStore()
desired = {"fabric_id": "hk-fabric", "state": "applying",
"next_stage_allowed": False,
"revision": 1, "plan_digest": "x",
"nodes": [{"host_id": "host-a", "state": "active",
"transport_verified": True}]}
store.set("fabric_desired", "hk-fabric", json.dumps(desired))
coordinator = FabricCoordinator(store, Registry({}))
coordinator.recover()
recovered = json.loads(store.get("fabric_desired", "hk-fabric"))
assert recovered["state"] == "uncertain"
assert recovered["next_stage_allowed"] is False
assert recovered["nodes"][0]["transport_verified"] is False
desired["state"] = "active"
desired["next_stage_allowed"] = True
desired["nodes"][0]["state"] = "active"
desired["nodes"][0]["transport_verified"] = True
store.set("fabric_desired", "hk-fabric", json.dumps(desired))
coordinator.recover()
restarted = json.loads(store.get("fabric_desired", "hk-fabric"))
assert restarted["state"] == "uncertain"
assert restarted["next_stage_allowed"] is False
def test_public_status_does_not_echo_nested_secrets_or_commands():
public = FabricCoordinator._public({
"fabric_id": "f", "state": "uncertain",
"listen_port": 51820,
"private_key": "secret",
"command": "rm -rf /",
"nodes": [{
"host_id": "host-a", "public_key": KEY1,
"last_error_code": "fabric_remote_command_failed",
"last_error_phase": "apply",
"private_key": "secret",
"observed": {
"interface": FABRIC_INTERFACE,
"command": "wg show",
"peers": [(KEY2, "203.0.113.2:51820", "100.64.1.2/32")],
"handshakes": {KEY2: 123},
},
}, {
"host_id": "host-b", "public_key": KEY2,
"last_error_phase": "observe",
}, {
"host_id": "host-c", "public_key": KEY3,
"last_error_code": "fabric_remote_command_failed",
"last_error_phase": "stderr:/secret/path",
}],
})
serialized = json.dumps(public)
assert public["listen_port"] == 51820
assert "private_key" not in serialized
assert "command" not in public
assert "command" not in public["nodes"][0]["observed"]
assert KEY1 in serialized and KEY2 in serialized
assert public["nodes"][0]["last_error_phase"] == "apply"
assert "last_error_phase" not in public["nodes"][1]
assert "last_error_phase" not in public["nodes"][2]
assert FabricError(
"safe", failure_phase="stderr:/secret/path").failure_phase is None
def test_reconcile_rejects_a_tampered_plan_digest():
store = MemoryStore()
current = plan()
store.set("fabric_desired", current.fabric_id, json.dumps({
"fabric_id": current.fabric_id,
"region": current.region,
"overlay_cidr": current.overlay_cidr,
"revision": current.revision,
"listen_port": current.listen_port,
"plan_digest": "0" * 64,
"public_keys": {node.host_id: key for node, key in zip(
current.nodes, (KEY1, KEY2))},
"nodes": [
{"host_id": node.host_id, "overlay_ip": node.overlay_ip,
"endpoint": node.endpoint, "listen_port": node.listen_port}
for node in current.nodes
],
}))
coordinator = FabricCoordinator(store, Registry({}))
with pytest.raises(FabricError) as error:
asyncio.run(coordinator.reconcile(current.fabric_id))
assert error.value.error_code == "fabric_record_corrupt"
def _active_fabric_record(current):
keys = {node.host_id: key for node, key in zip(
current.nodes, (KEY1, KEY2))}
return {
"schema_version": 1,
"fabric_id": current.fabric_id,
"region": current.region,
"overlay_cidr": current.overlay_cidr,
"revision": current.revision,
"listen_port": current.listen_port,
"plan_digest": current.digest,
"state": "active",
"next_stage_allowed": True,
"updated_at": int(time.time()),
"public_keys": keys,
"nodes": [
{
"host_id": node.host_id,
"overlay_ip": node.overlay_ip,
"endpoint": node.endpoint,
"listen_port": node.listen_port,
"state": "active",
"transport_verified": True,
"observed_revision": current.revision,
"observed_plan_digest": current.digest,
}
for node in current.nodes
],
}
def _fabric_components(current):
components = {}
for node, key in zip(current.nodes, (KEY1, KEY2)):
manager = FakeManager(key)
manager.host_id = node.host_id
manager.profile["host_id"] = node.host_id
manager.profile["region"] = current.region
components[node.host_id] = SimpleNamespace(manager=manager)
return components
def _remote_operation_status(
operation_id, status, *, checkpoint=None, reason=None):
checkpoint = checkpoint or (
"complete" if status == "succeeded" else "preflight")
reason = reason or (
"none" if status in {"queued", "running", "succeeded"}
else "command_failed")
started_at = None if status == "queued" else 101
finished_at = (
103 if status in {"succeeded", "failed", "uncertain", "interrupted"}
else None)
return {
"operation_id": operation_id,
"status": status,
"phase": "apply",
"checkpoint": checkpoint,
"reason": reason,
"created_at": 100,
"started_at": started_at,
"updated_at": 103 if finished_at else 102,
"finished_at": finished_at,
}
class AsyncCoordinatorAdapter:
"""Host stub that separates one dispatch from read-only status polling."""
def __init__(self, manager, _profile=None):
self.manager = manager
def identity(self, *, create_key=True):
self.manager.async_identity_calls += 1
return self.manager.public_key
def dispatch_apply(self, _plan, _host, _keys, operation_id=None):
self.manager.async_dispatch_calls += 1
self.manager.async_operation_id = operation_id
if self.manager.async_dispatch_entered is not None:
self.manager.async_dispatch_entered.set()
if self.manager.async_dispatch_release is not None:
assert self.manager.async_dispatch_release.wait(timeout=2)
if self.manager.async_dispatch_error is not None:
error = self.manager.async_dispatch_error
self.manager.async_dispatch_error = None
raise error
return _remote_operation_status(operation_id, "queued")
def apply_status(self, operation_id):
self.manager.async_status_calls += 1
assert operation_id == self.manager.async_operation_id
statuses = self.manager.async_statuses
if len(statuses) > 1:
status = statuses.pop(0)
else:
status = statuses[0]
if isinstance(status, Exception):
raise status
if isinstance(status, dict):
return dict(status)
return _remote_operation_status(
operation_id, status,
checkpoint=("nft_install" if status == "failed" else None),
reason=("nft_failed" if status == "failed" else None),
)
def observe(self, current, host, _keys):
self.manager.async_observe_calls += 1
return _reconcile_observation(current, host)
class PublishedQueuedGapAdapter(AsyncCoordinatorAdapter):
"""First dispatch publishes the directory but dies before systemd-run."""
def dispatch_apply(self, _plan, _host, _keys, operation_id=None):
self.manager.async_dispatch_calls += 1
self.manager.async_operation_id = operation_id
if self.manager.async_dispatch_calls == 1:
raise FabricError(
"ambiguous transport /root/private", 502,
"fabric_host_connection_failed", "apply")
self.manager.async_runner_starts += 1
self.manager.async_statuses = ["succeeded"]
return _remote_operation_status(operation_id, "queued")
class RecoveredQueuedGapAdapter(AsyncCoordinatorAdapter):
"""Restart joins a published queue and starts its original runner once."""
def dispatch_apply(self, _plan, _host, _keys, operation_id=None):
self.manager.async_dispatch_calls += 1
assert operation_id == self.manager.async_operation_id
self.manager.async_runner_starts += 1
self.manager.async_statuses = ["succeeded"]
return _remote_operation_status(operation_id, "queued")
class PartialHostCrashAdapter(AsyncCoordinatorAdapter):
"""Host A was admitted; host B still needs same-id continuation."""
def identity(self, *, create_key=True):
self.manager.async_identity_calls += 1
self.manager.partial_identity_create_flags.append(create_key)
return self.manager.public_key
def dispatch_apply(self, _plan, _host, _keys, operation_id=None):
assert self.manager.partial_configured is False
self.manager.async_dispatch_calls += 1
self.manager.async_operation_id = operation_id
self.manager.partial_configured = True
return _remote_operation_status(operation_id, "succeeded")
def observe(self, current, host, _keys):
self.manager.async_observe_calls += 1
if self.manager.partial_configured:
return _reconcile_observation(current, host)
return {
"state": "degraded",
"config_ok": False,
"transport_verified": False,
"public_key": self.manager.public_key,
"observed": {
"interface": None,
"config_revision": None,
"config_digest": None,
},
}
def _async_components(current):
components = _fabric_components(current)
for component in components.values():
manager = component.manager
manager.async_identity_calls = 0
manager.async_dispatch_calls = 0
manager.async_status_calls = 0
manager.async_observe_calls = 0
manager.async_operation_id = None
manager.async_dispatch_entered = None
manager.async_dispatch_release = None
manager.async_dispatch_error = None
manager.async_runner_starts = 0
manager.partial_configured = False
manager.partial_identity_create_flags = []
manager.async_statuses = ["succeeded"]
return components
async def _await_fabric_operation(coordinator, fabric_id):
for _attempt in range(500):
task = coordinator._operation_tasks_by_fabric.get(fabric_id)
if task is None:
return
await asyncio.sleep(0.002)
raise AssertionError("Fabric background operation did not finish")
def test_enqueue_apply_returns_before_host_systemd_job_finishes(monkeypatch):
current = plan()
store = MemoryStore()
components = _async_components(current)
entered = threading.Event()
release = threading.Event()
components["host-a"].manager.async_dispatch_entered = entered
components["host-a"].manager.async_dispatch_release = release
coordinator = FabricCoordinator(store, Registry(components))
monkeypatch.setattr(
"virsh_panel.fabric.FabricHostAdapter", AsyncCoordinatorAdapter)
monkeypatch.setattr(
"virsh_panel.fabric.FABRIC_OPERATION_POLL_INTERVAL_SECONDS", 0)
async def exercise():
admitted = await coordinator.enqueue_apply(plan_payload(current))
assert admitted["operation"]["status"] == "queued"
assert re.fullmatch(r"[0-9a-f]{32}", admitted["operation"]["id"])
assert not entered.is_set()
for _attempt in range(100):
if entered.is_set():
break
await asyncio.sleep(0.002)
assert entered.is_set()
live = coordinator.get_status(current.fabric_id)
assert live["operation"]["status"] == "running"
release.set()
await _await_fabric_operation(coordinator, current.fabric_id)
return admitted, coordinator.get_status(current.fabric_id)
admitted, final = asyncio.run(exercise())
assert final["operation"]["id"] == admitted["operation"]["id"]
assert final["operation"]["status"] == "succeeded"
assert all(node["operation"]["status"] == "succeeded"
for node in final["nodes"])
assert all(component.manager.async_dispatch_calls == 1
for component in components.values())
def test_cancelled_async_dispatch_drains_then_recovers_without_redispatch(
monkeypatch):
current = plan()
store = MemoryStore()
components = _async_components(current)
entered = threading.Event()
release = threading.Event()
first = components["host-a"].manager
first.async_dispatch_entered = entered
first.async_dispatch_release = release
coordinator = FabricCoordinator(store, Registry(components))
monkeypatch.setattr(
"virsh_panel.fabric.FabricHostAdapter", AsyncCoordinatorAdapter)
async def exercise():
admitted = await coordinator.enqueue_apply(plan_payload(current))
for _attempt in range(100):
if entered.is_set():
break
await asyncio.sleep(0.002)
assert entered.is_set()
operation_task = coordinator._operation_tasks_by_fabric[
current.fabric_id]
lock_entered = asyncio.Event()
async def wait_for_fabric_lock():
async with coordinator._locks[current.fabric_id]:
lock_entered.set()
waiter = asyncio.create_task(wait_for_fabric_lock())
operation_task.cancel()
await asyncio.sleep(0)
assert operation_task.done() is False
assert lock_entered.is_set() is False
release.set()
with pytest.raises(asyncio.CancelledError):
await operation_task
await asyncio.wait_for(waiter, timeout=1)
assert lock_entered.is_set() is True
coordinator.recover()
first.async_operation_id = admitted["operation"]["id"]
first.async_statuses = ["succeeded"]
resumed = await coordinator.enqueue_reconcile(current.fabric_id)
assert resumed["operation"]["id"] == admitted["operation"]["id"]
await _await_fabric_operation(coordinator, current.fabric_id)
return admitted, coordinator.get_status(current.fabric_id)
try:
admitted, final = asyncio.run(exercise())
finally:
release.set()
assert final["operation"]["id"] == admitted["operation"]["id"]
assert first.async_dispatch_calls == 1
assert components["host-b"].manager.async_dispatch_calls == 0
@pytest.mark.parametrize(("terminal", "expected_root"), [
("succeeded", "succeeded"),
("failed", "failed"),
])
def test_dispatch_disconnect_is_reconciled_by_status_without_replay(
monkeypatch, terminal, expected_root):
current = plan()
store = MemoryStore()
components = _async_components(current)
disconnected = components["host-a"].manager
disconnected.async_dispatch_error = FabricError(
"safe disconnect", 502, "fabric_host_connection_failed", "apply")
disconnected.async_statuses = ["running", terminal]
coordinator = FabricCoordinator(store, Registry(components))
monkeypatch.setattr(
"virsh_panel.fabric.FabricHostAdapter", AsyncCoordinatorAdapter)
monkeypatch.setattr(
"virsh_panel.fabric.FABRIC_OPERATION_POLL_INTERVAL_SECONDS", 0)
async def exercise():
admitted = await coordinator.enqueue_apply(plan_payload(current))
await _await_fabric_operation(coordinator, current.fabric_id)
return admitted, coordinator.get_status(current.fabric_id)
admitted, final = asyncio.run(exercise())
assert final["operation"]["id"] == admitted["operation"]["id"]
assert final["operation"]["status"] == expected_root
failed_or_recovered = next(
node for node in final["nodes"] if node["host_id"] == "host-a")
assert failed_or_recovered["operation"]["status"] == terminal
assert disconnected.async_dispatch_calls == 1
assert disconnected.async_status_calls >= 2
if terminal == "succeeded":
assert disconnected.async_observe_calls == 1
else:
assert disconnected.async_observe_calls == 0
def test_async_apply_timeout_is_uncertain_and_never_redispatched(monkeypatch):
current = plan()
store = MemoryStore()
components = _async_components(current)
stalled = components["host-a"].manager
stalled.async_statuses = ["running"]
coordinator = FabricCoordinator(store, Registry(components))
monkeypatch.setattr(
"virsh_panel.fabric.FabricHostAdapter", AsyncCoordinatorAdapter)
monkeypatch.setattr(
"virsh_panel.fabric.FABRIC_OPERATION_POLL_INTERVAL_SECONDS", 0.002)
monkeypatch.setattr(
"virsh_panel.fabric.FABRIC_OPERATION_POLL_TIMEOUT_SECONDS", 0.01)
async def exercise():
await coordinator.enqueue_apply(plan_payload(current))
await _await_fabric_operation(coordinator, current.fabric_id)
return coordinator.get_status(current.fabric_id)
final = asyncio.run(exercise())
stalled_node = next(
node for node in final["nodes"] if node["host_id"] == "host-a")
assert final["operation"]["status"] == "uncertain"
assert stalled_node["operation"]["status"] == "uncertain"
assert stalled_node["operation"]["reason"] == "transport_failed"
assert stalled.async_dispatch_calls == 1
assert stalled.async_status_calls >= 1
@pytest.mark.parametrize("error_code", [
"fabric_operation_status_invalid",
"fabric_operation_not_found",
"fabric_remote_command_failed",
])
def test_confirmed_dispatch_status_error_stays_uncertain_and_resumes_same_id(
monkeypatch, error_code):
current = plan()
store = MemoryStore()
components = _async_components(current)
for component in components.values():
component.manager.async_statuses = [
FabricError("unsafe /root/private", 502, error_code, "observe"),
"succeeded",
]
coordinator = FabricCoordinator(store, Registry(components))
monkeypatch.setattr(
"virsh_panel.fabric.FabricHostAdapter", AsyncCoordinatorAdapter)
monkeypatch.setattr(
"virsh_panel.fabric.FABRIC_OPERATION_POLL_INTERVAL_SECONDS", 0)
async def exercise():
admitted = await coordinator.enqueue_apply(plan_payload(current))
await _await_fabric_operation(coordinator, current.fabric_id)
uncertain = coordinator.get_status(current.fabric_id)
assert uncertain["operation"]["status"] == "uncertain"
assert uncertain["next_stage_allowed"] is False
resumed = await coordinator.enqueue_reconcile(current.fabric_id)
assert resumed["operation"]["id"] == admitted["operation"]["id"]
await _await_fabric_operation(coordinator, current.fabric_id)
return admitted, coordinator.get_status(current.fabric_id)
admitted, final = asyncio.run(exercise())
assert final["operation"]["id"] == admitted["operation"]["id"]
assert final["operation"]["status"] == "succeeded"
assert all(component.manager.async_dispatch_calls == 1
for component in components.values())
assert all(component.manager.async_status_calls >= 2
for component in components.values())
assert "/root/private" not in json.dumps(final)
def test_published_queue_gap_recovers_original_runner_once_with_same_id(
monkeypatch):
current = plan()
store = MemoryStore()
components = _async_components(current)
for component in components.values():
component.manager.async_statuses = ["queued"]
coordinator = FabricCoordinator(store, Registry(components))
monkeypatch.setattr(
"virsh_panel.fabric.FabricHostAdapter", PublishedQueuedGapAdapter)
monkeypatch.setattr(
"virsh_panel.fabric.FABRIC_OPERATION_POLL_INTERVAL_SECONDS", 0.001)
monkeypatch.setattr(
"virsh_panel.fabric.FABRIC_OPERATION_POLL_TIMEOUT_SECONDS", 0.004)
async def exercise():
admitted = await coordinator.enqueue_apply(plan_payload(current))
await _await_fabric_operation(coordinator, current.fabric_id)
stalled = coordinator.get_status(current.fabric_id)
assert stalled["operation"]["status"] == "uncertain"
assert stalled["next_stage_allowed"] is False
resumed = await coordinator.enqueue_reconcile(current.fabric_id)
assert resumed["operation"]["id"] == admitted["operation"]["id"]
await _await_fabric_operation(coordinator, current.fabric_id)
restarted = coordinator.get_status(current.fabric_id)
assert restarted["operation"]["status"] == "running"
assert restarted["next_stage_allowed"] is False
await coordinator.reconcile_all()
return admitted, coordinator.get_status(current.fabric_id)
admitted, final = asyncio.run(exercise())
assert final["operation"]["id"] == admitted["operation"]["id"]
assert final["operation"]["status"] == "succeeded"
assert final["next_stage_allowed"] is True
assert all(component.manager.async_dispatch_calls == 2
for component in components.values())
assert all(component.manager.async_runner_starts == 1
for component in components.values())
assert all(component.manager.async_operation_id
== admitted["operation"]["id"]
for component in components.values())
def test_restart_periodic_queued_status_still_allows_same_id_runner_recovery(
monkeypatch):
current = plan()
operation_id = "a" * 32
store = MemoryStore()
components = _async_components(current)
coordinator = FabricCoordinator(store, Registry(components))
admitted = coordinator._admit_apply_record(current, operation_id)
admitted["operation"].update(
status="running", started_at=101, updated_at=101)
admitted["public_keys"] = {"host-a": KEY1, "host-b": KEY2}
for node, key in zip(admitted["nodes"], (KEY1, KEY2)):
node["public_key"] = key
node["operation"].update(
status="running", phase="apply", checkpoint="dispatch",
started_at=101, updated_at=101)
store.set(
"fabric_desired", current.fabric_id,
json.dumps(admitted))
coordinator.recover()
for component in components.values():
manager = component.manager
manager.async_operation_id = operation_id
manager.async_statuses = ["queued"]
monkeypatch.setattr(
"virsh_panel.fabric.FabricHostAdapter", RecoveredQueuedGapAdapter)
async def exercise():
await coordinator.reconcile_all()
observed = coordinator.get_status(current.fabric_id)
assert observed["operation"]["id"] == operation_id
assert observed["operation"]["status"] == "running"
assert observed["next_stage_allowed"] is False
assert all(component.manager.async_dispatch_calls == 0
for component in components.values())
resumed = await coordinator.enqueue_reconcile(current.fabric_id)
assert resumed["operation"]["id"] == operation_id
await _await_fabric_operation(coordinator, current.fabric_id)
restarted = coordinator.get_status(current.fabric_id)
assert restarted["operation"]["status"] == "running"
assert restarted["next_stage_allowed"] is False
await coordinator.reconcile_all()
return coordinator.get_status(current.fabric_id)
final = asyncio.run(exercise())
assert final["operation"]["id"] == operation_id
assert final["operation"]["status"] == "succeeded"
assert final["next_stage_allowed"] is True
assert all(component.manager.async_dispatch_calls == 1
for component in components.values())
assert all(component.manager.async_runner_starts == 1
for component in components.values())
assert all(component.manager.async_operation_id == operation_id
for component in components.values())
def _interrupted_async_record(current, operation_id):
record = _active_fabric_record(current)
record.update(state="uncertain", next_stage_allowed=False)
record["operation"] = {
"id": operation_id,
"kind": "apply",
"status": "interrupted",
"created_at": 100,
"started_at": 101,
"updated_at": 102,
"finished_at": 102,
}
for node in record["nodes"]:
node.update(
state="uncertain", transport_verified=False,
observed_revision=None, observed_plan_digest=None,
operation={
"id": operation_id,
"status": "uncertain",
"phase": "apply",
"checkpoint": "preflight",
"reason": "transport_failed",
"created_at": 100,
"started_at": 101,
"updated_at": 102,
"finished_at": 102,
},
)
return record
@pytest.mark.parametrize("crash_stage", [
"before_identity",
"mid_identity",
"identity_fenced",
])
def test_pre_dispatch_crash_resumes_same_operation_id_without_replay(
monkeypatch, crash_stage):
current = plan()
operation_id = "5" * 32
store = MemoryStore()
components = _async_components(current)
coordinator = FabricCoordinator(store, Registry(components))
admitted = coordinator._admit_apply_record(current, operation_id)
if crash_stage in {"mid_identity", "identity_fenced"}:
admitted["operation"].update(
status="running", started_at=101, updated_at=101)
admitted["nodes"][0]["operation"].update(
status="running", phase="identity", checkpoint="identity",
started_at=101, updated_at=101)
if crash_stage == "identity_fenced":
admitted["public_keys"] = {"host-a": KEY1, "host-b": KEY2}
for node, key in zip(admitted["nodes"], (KEY1, KEY2)):
node["public_key"] = key
node["operation"].update(
status="running", phase="identity",
checkpoint="identity", started_at=101, updated_at=101)
store.set(
"fabric_desired", current.fabric_id,
json.dumps(admitted))
coordinator.recover()
for component in components.values():
component.manager.async_operation_id = operation_id
component.manager.async_statuses = ["succeeded"]
monkeypatch.setattr(
"virsh_panel.fabric.FabricHostAdapter", AsyncCoordinatorAdapter)
monkeypatch.setattr(
"virsh_panel.fabric.FABRIC_OPERATION_POLL_INTERVAL_SECONDS", 0)
async def exercise():
resumed = await coordinator.enqueue_reconcile(current.fabric_id)
assert resumed["operation"]["id"] == operation_id
await _await_fabric_operation(coordinator, current.fabric_id)
return coordinator.get_status(current.fabric_id)
final = asyncio.run(exercise())
assert final["operation"]["id"] == operation_id
assert final["operation"]["status"] == "succeeded"
assert all(component.manager.async_dispatch_calls == 1
for component in components.values())
def _partial_host_crash_record(coordinator, current, operation_id):
admitted = coordinator._admit_apply_record(current, operation_id)
admitted["operation"].update(
status="running", started_at=101, updated_at=101)
admitted["public_keys"] = {"host-a": KEY1, "host-b": KEY2}
for node, key in zip(admitted["nodes"], (KEY1, KEY2)):
node["public_key"] = key
node["operation"].update(
status="running", phase="identity",
checkpoint="identity", started_at=101, updated_at=101)
admitted["nodes"][0]["operation"].update(
phase="apply", checkpoint="dispatch")
return admitted
def test_crash_between_hosts_continues_only_undispatched_host_with_same_id(
monkeypatch):
current = plan()
operation_id = "6" * 32
store = MemoryStore()
components = _async_components(current)
coordinator = FabricCoordinator(store, Registry(components))
admitted = _partial_host_crash_record(
coordinator, current, operation_id)
store.set(
"fabric_desired", current.fabric_id,
json.dumps(admitted))
coordinator.recover()
first = components["host-a"].manager
second = components["host-b"].manager
first.async_operation_id = operation_id
first.async_statuses = ["succeeded"]
first.partial_configured = True
second.async_statuses = ["succeeded"]
monkeypatch.setattr(
"virsh_panel.fabric.FabricHostAdapter", PartialHostCrashAdapter)
async def exercise():
resumed = await coordinator.enqueue_reconcile(current.fabric_id)
assert resumed["operation"]["id"] == operation_id
await _await_fabric_operation(coordinator, current.fabric_id)
continuing = coordinator.get_status(current.fabric_id)
assert continuing["operation"]["status"] == "running"
assert continuing["next_stage_allowed"] is False
await coordinator.reconcile_all()
return coordinator.get_status(current.fabric_id)
final = asyncio.run(exercise())
assert final["operation"]["id"] == operation_id
assert final["operation"]["status"] == "succeeded"
assert final["next_stage_allowed"] is True
assert first.async_dispatch_calls == 0
assert second.async_dispatch_calls == 1
assert first.async_identity_calls == 0
assert second.async_identity_calls == 1
assert second.partial_identity_create_flags == [False]
assert second.async_operation_id == operation_id
def test_partial_remote_failed_but_exact_continues_only_undispatched_host(
monkeypatch):
current = plan()
operation_id = "7" * 32
store = MemoryStore()
components = _async_components(current)
coordinator = FabricCoordinator(store, Registry(components))
store.set(
"fabric_desired", current.fabric_id,
json.dumps(_partial_host_crash_record(
coordinator, current, operation_id)))
coordinator.recover()
first = components["host-a"].manager
second = components["host-b"].manager
first.async_operation_id = operation_id
first.async_statuses = ["failed"]
first.partial_configured = True
second.async_statuses = ["succeeded"]
monkeypatch.setattr(
"virsh_panel.fabric.FabricHostAdapter", PartialHostCrashAdapter)
async def exercise():
resumed = await coordinator.enqueue_reconcile(current.fabric_id)
assert resumed["operation"]["id"] == operation_id
await _await_fabric_operation(coordinator, current.fabric_id)
continuing = coordinator.get_status(current.fabric_id)
assert continuing["operation"]["status"] == "running"
assert continuing["next_stage_allowed"] is False
await coordinator.reconcile_all()
return coordinator.get_status(current.fabric_id)
final = asyncio.run(exercise())
assert final["operation"]["id"] == operation_id
assert final["operation"]["status"] == "succeeded"
assert final["next_stage_allowed"] is True
assert first.async_dispatch_calls == 0
assert second.async_dispatch_calls == 1
assert first.async_identity_calls == 0
assert second.async_identity_calls == 1
assert second.partial_identity_create_flags == [False]
assert second.async_operation_id == operation_id
def test_partial_remote_failed_nonexact_stays_terminal_then_uses_new_id(
monkeypatch):
current = plan()
operation_id = "8" * 32
store = MemoryStore()
components = _async_components(current)
coordinator = FabricCoordinator(store, Registry(components))
store.set(
"fabric_desired", current.fabric_id,
json.dumps(_partial_host_crash_record(
coordinator, current, operation_id)))
coordinator.recover()
first = components["host-a"].manager
second = components["host-b"].manager
first.async_operation_id = operation_id
first.async_statuses = ["failed"]
first.partial_configured = False
second.partial_configured = False
monkeypatch.setattr(
"virsh_panel.fabric.FabricHostAdapter", PartialHostCrashAdapter)
async def exercise():
resumed = await coordinator.enqueue_reconcile(current.fabric_id)
assert resumed["operation"]["id"] == operation_id
await _await_fabric_operation(coordinator, current.fabric_id)
failed = coordinator.get_status(current.fabric_id)
assert failed["operation"]["status"] == "failed"
first.async_statuses = ["succeeded"]
retried = await coordinator.enqueue_reconcile(current.fabric_id)
assert retried["operation"]["id"] != operation_id
await _await_fabric_operation(coordinator, current.fabric_id)
return failed, coordinator.get_status(current.fabric_id)
failed, final = asyncio.run(exercise())
assert failed["operation"]["id"] == operation_id
assert failed["state"] == "uncertain"
assert failed["next_stage_allowed"] is False
assert final["operation"]["id"] != operation_id
assert final["operation"]["status"] == "succeeded"
assert final["next_stage_allowed"] is True
assert first.async_dispatch_calls == 1
assert second.async_dispatch_calls == 1
assert first.async_identity_calls == 1
assert second.async_identity_calls == 1
class NonExactAsyncCoordinatorAdapter(AsyncCoordinatorAdapter):
def observe(self, current, host, keys):
result = super().observe(current, host, keys)
result.update(
state="degraded", config_ok=False,
transport_verified=False)
result["observed"]["config_digest"] = "0" * 64
return result
def test_remote_success_with_non_exact_readback_is_terminal_uncertain(
monkeypatch):
current = plan()
store = MemoryStore()
components = _async_components(current)
coordinator = FabricCoordinator(store, Registry(components))
monkeypatch.setattr(
"virsh_panel.fabric.FabricHostAdapter",
NonExactAsyncCoordinatorAdapter)
monkeypatch.setattr(
"virsh_panel.fabric.FABRIC_OPERATION_POLL_INTERVAL_SECONDS", 0)
async def exercise():
admitted = await coordinator.enqueue_apply(plan_payload(current))
await _await_fabric_operation(coordinator, current.fabric_id)
degraded = coordinator.get_status(current.fabric_id)
assert degraded["operation"]["status"] == "uncertain"
assert degraded["next_stage_allowed"] is False
durable = json.loads(store.get(
"fabric_desired", current.fabric_id))
assert all(
node["operation"]["phase"] == "observe"
and node["operation"]["checkpoint"] == "final_readback"
for node in durable["nodes"])
assert coordinator._operation_may_have_dispatched(
durable, admitted["operation"]["id"]) is True
resumed = await coordinator.enqueue_reconcile(current.fabric_id)
assert resumed["operation"]["id"] == admitted["operation"]["id"]
await _await_fabric_operation(coordinator, current.fabric_id)
return admitted, coordinator.get_status(current.fabric_id)
admitted, final = asyncio.run(exercise())
assert final["operation"]["id"] == admitted["operation"]["id"]
assert final["operation"]["status"] == "uncertain"
assert final["state"] == "uncertain"
assert final["next_stage_allowed"] is False
assert all(node["operation"]["status"] == "uncertain"
for node in final["nodes"])
assert all(component.manager.async_dispatch_calls == 1
for component in components.values())
def test_restart_resume_reads_exact_operation_without_redispatch(monkeypatch):
current = plan()
operation_id = "1" * 32
store = MemoryStore()
store.set(
"fabric_desired", current.fabric_id,
json.dumps(_interrupted_async_record(current, operation_id)))
components = _async_components(current)
for component in components.values():
component.manager.async_operation_id = operation_id
component.manager.async_statuses = ["succeeded"]
coordinator = FabricCoordinator(store, Registry(components))
monkeypatch.setattr(
"virsh_panel.fabric.FabricHostAdapter", AsyncCoordinatorAdapter)
async def exercise():
admitted = await coordinator.enqueue_reconcile(current.fabric_id)
assert admitted["operation"]["id"] == operation_id
await _await_fabric_operation(coordinator, current.fabric_id)
return coordinator.get_status(current.fabric_id)
final = asyncio.run(exercise())
assert final["operation"]["id"] == operation_id
assert final["operation"]["status"] == "succeeded"
assert final["next_stage_allowed"] is True
assert all(component.manager.async_dispatch_calls == 0
for component in components.values())
assert all(component.manager.async_status_calls == 1
for component in components.values())
assert all(component.manager.async_observe_calls == 1
for component in components.values())
def test_restart_resume_does_not_publish_running_job_as_succeeded(monkeypatch):
current = plan()
operation_id = "2" * 32
store = MemoryStore()
store.set(
"fabric_desired", current.fabric_id,
json.dumps(_interrupted_async_record(current, operation_id)))
components = _async_components(current)
for component in components.values():
component.manager.async_operation_id = operation_id
component.manager.async_statuses = ["running"]
coordinator = FabricCoordinator(store, Registry(components))
monkeypatch.setattr(
"virsh_panel.fabric.FabricHostAdapter", AsyncCoordinatorAdapter)
async def exercise():
await coordinator.enqueue_reconcile(current.fabric_id)
await _await_fabric_operation(coordinator, current.fabric_id)
return coordinator.get_status(current.fabric_id)
final = asyncio.run(exercise())
assert final["operation"]["id"] == operation_id
assert final["operation"]["status"] == "running"
assert final["next_stage_allowed"] is False
assert all(node["operation"]["status"] == "running"
for node in final["nodes"])
assert all(component.manager.async_dispatch_calls == 0
for component in components.values())
assert all(component.manager.async_observe_calls == 0
for component in components.values())
def test_restart_missing_runtime_operation_can_converge_by_exact_observation(
monkeypatch):
current = plan()
operation_id = "3" * 32
store = MemoryStore()
store.set(
"fabric_desired", current.fabric_id,
json.dumps(_interrupted_async_record(current, operation_id)))
components = _async_components(current)
for component in components.values():
component.manager.async_operation_id = operation_id
component.manager.async_statuses = [FabricError(
"safe missing", 409, "fabric_operation_not_found", "observe")]
coordinator = FabricCoordinator(store, Registry(components))
monkeypatch.setattr(
"virsh_panel.fabric.FabricHostAdapter", AsyncCoordinatorAdapter)
asyncio.run(coordinator.reconcile_all())
final = coordinator.get_status(current.fabric_id)
assert final["operation"]["id"] == operation_id
assert final["operation"]["status"] == "succeeded"
assert final["state"] == "active"
assert final["next_stage_allowed"] is True
assert all(component.manager.async_dispatch_calls == 0
for component in components.values())
assert all(component.manager.async_observe_calls == 1
for component in components.values())
@pytest.mark.parametrize("error_code", [
"fabric_host_connection_failed",
"fabric_operation_status_invalid",
"fabric_remote_command_failed",
])
def test_restart_status_failure_cannot_use_old_exact_observation(
monkeypatch, error_code):
current = plan()
operation_id = "4" * 32
store = MemoryStore()
store.set(
"fabric_desired", current.fabric_id,
json.dumps(_interrupted_async_record(current, operation_id)))
components = _async_components(current)
for component in components.values():
component.manager.async_operation_id = operation_id
component.manager.async_statuses = [FabricError(
"unsafe transport output /root/private", 502,
error_code, "observe")]
coordinator = FabricCoordinator(store, Registry(components))
monkeypatch.setattr(
"virsh_panel.fabric.FabricHostAdapter", AsyncCoordinatorAdapter)
asyncio.run(coordinator.reconcile_all())
final = coordinator.get_status(current.fabric_id)
durable = json.loads(store.get("fabric_desired", current.fabric_id))
assert final["operation"]["id"] == operation_id
assert final["operation"]["status"] == "uncertain"
assert final["state"] == "uncertain"
assert final["next_stage_allowed"] is False
assert all(node["operation"]["status"] == "uncertain"
for node in final["nodes"])
assert all(component.manager.async_dispatch_calls == 0
for component in components.values())
assert "/root/private" not in json.dumps(durable)
def test_async_identity_is_durably_fenced_before_first_host_dispatch(
monkeypatch):
current = plan()
store = MemoryStore()
components = _async_components(current)
entered = threading.Event()
release = threading.Event()
components["host-a"].manager.async_dispatch_entered = entered
components["host-a"].manager.async_dispatch_release = release
coordinator = FabricCoordinator(store, Registry(components))
monkeypatch.setattr(
"virsh_panel.fabric.FabricHostAdapter", AsyncCoordinatorAdapter)
monkeypatch.setattr(
"virsh_panel.fabric.FABRIC_OPERATION_POLL_INTERVAL_SECONDS", 0)
async def exercise():
admitted = await coordinator.enqueue_apply(plan_payload(current))
for _attempt in range(100):
if entered.is_set():
break
await asyncio.sleep(0.002)
assert entered.is_set()
durable = json.loads(store.get(
"fabric_desired", current.fabric_id))
release.set()
await _await_fabric_operation(coordinator, current.fabric_id)
return admitted, durable
try:
admitted, durable = asyncio.run(exercise())
finally:
release.set()
assert durable["operation"]["id"] == admitted["operation"]["id"]
assert durable["public_keys"] == {"host-a": KEY1, "host-b": KEY2}
assert {
node["host_id"]: node.get("public_key")
for node in durable["nodes"]
} == {"host-a": KEY1, "host-b": KEY2}
def test_observe_only_probe_failure_closes_durable_active_gate(monkeypatch):
current = plan()
store = MemoryStore()
store.set(
"fabric_desired", current.fabric_id,
json.dumps(_active_fabric_record(current)))
components = _fabric_components(current)
def probe_failed():
raise RemoteHostError(
"unsafe remote detail /root/private", 502,
"host_capability_probe_failed")
components["host-a"].manager.reprobe_current = probe_failed
coordinator = FabricCoordinator(store, Registry(components))
asyncio.run(coordinator.reconcile_all())
durable = json.loads(store.get("fabric_desired", current.fabric_id))
assert durable["state"] == "uncertain"
assert durable["next_stage_allowed"] is False
assert all(node["transport_verified"] is False
for node in durable["nodes"])
assert "/root/private" not in json.dumps(durable)
def test_enqueue_reconcile_admission_uses_shared_catalog_lock(monkeypatch):
current = plan()
store = MemoryStore()
store.set(
"fabric_desired", current.fabric_id,
json.dumps(_active_fabric_record(current)))
components = _async_components(current)
coordinator = FabricCoordinator(store, Registry(components))
monkeypatch.setattr(
"virsh_panel.fabric.FabricHostAdapter", AsyncCoordinatorAdapter)
async def exercise():
shared = coordinator.catalog_mutation_lock()
async with shared:
admission = asyncio.create_task(
coordinator.enqueue_reconcile(current.fabric_id))
await asyncio.sleep(0.02)
assert admission.done() is False
durable = json.loads(store.get(
"fabric_desired", current.fabric_id))
assert "operation" not in durable
receipt = await asyncio.wait_for(admission, timeout=1)
await coordinator.shutdown_operations()
return receipt
receipt = asyncio.run(exercise())
assert receipt["operation"]["kind"] == "reconcile"
assert receipt["operation"]["status"] == "queued"
def test_background_internal_error_cannot_leave_operation_running(monkeypatch):
current = plan()
store = MemoryStore()
components = _async_components(current)
class UnexpectedFailureAdapter(AsyncCoordinatorAdapter):
def dispatch_apply(self, *_args, **_kwargs):
self.manager.async_dispatch_calls += 1
raise RuntimeError("unsafe implementation detail /root/private")
coordinator = FabricCoordinator(store, Registry(components))
monkeypatch.setattr(
"virsh_panel.fabric.FabricHostAdapter", UnexpectedFailureAdapter)
async def exercise():
await coordinator.enqueue_apply(plan_payload(current))
await _await_fabric_operation(coordinator, current.fabric_id)
return coordinator.get_status(current.fabric_id)
final = asyncio.run(exercise())
assert final["operation"]["status"] in {
"failed", "uncertain", "interrupted",
}
assert final["operation"]["finished_at"] is not None
assert all(node["operation"]["status"] not in {"pending", "running"}
for node in final["nodes"])
assert "/root/private" not in json.dumps(final)
def test_periodic_reconcile_is_read_only_and_never_dispatches(monkeypatch):
current = plan()
store = MemoryStore()
store.set(
"fabric_desired", current.fabric_id,
json.dumps(_active_fabric_record(current)))
components = _async_components(current)
coordinator = FabricCoordinator(store, Registry(components))
monkeypatch.setattr(
"virsh_panel.fabric.FabricHostAdapter", AsyncCoordinatorAdapter)
asyncio.run(coordinator.reconcile_all())
final = coordinator.get_status(current.fabric_id)
assert final["state"] == "active"
assert final["next_stage_allowed"] is True
assert all(component.manager.async_dispatch_calls == 0
for component in components.values())
assert all(component.manager.async_status_calls == 0
for component in components.values())
assert all(component.manager.async_observe_calls == 1
for component in components.values())
@pytest.mark.parametrize("observed_state", ["active", "degraded", "uncertain"])
def test_readback_wakes_vpc_only_after_durable_active_publication(
monkeypatch, observed_state):
current = plan()
store = MemoryStore()
store.set("fabric_desired", current.fabric_id, json.dumps(
_active_fabric_record(current)))
published = []
def on_active_readback():
published.append(json.loads(store.get(
"fabric_desired", current.fabric_id)))
class ReadbackAdapter:
def __init__(self, _manager, _profile=None):
pass
def observe(self, _plan, host, _keys):
observed = _reconcile_observation(current, host)
observed.update({
"state": "degraded" if observed_state == "uncertain" else "active",
"config_ok": observed_state != "uncertain",
"transport_verified": observed_state == "active",
})
return observed
coordinator = FabricCoordinator(
store, Registry(_fabric_components(current)),
on_active_readback=on_active_readback,
)
monkeypatch.setattr(
"virsh_panel.fabric.FabricHostAdapter", ReadbackAdapter)
asyncio.run(coordinator.reconcile_all())
durable = json.loads(store.get("fabric_desired", current.fabric_id))
assert durable["state"] == observed_state
assert len(published) == (1 if observed_state == "active" else 0)
if published:
assert published[0] == durable
assert published[0]["next_stage_allowed"] is True
assert all(node["transport_verified"] is True
for node in published[0]["nodes"])
@pytest.mark.parametrize("blocked_by", ["operation_unsettled", "save_failed"])
def test_active_readback_does_not_wake_vpc_before_publication_succeeds(blocked_by):
current = plan()
class FailingPublishStore(MemoryStore):
reject_active = False
def set(self, kind, key, value):
if (self.reject_active and kind == "fabric_desired"
and json.loads(value).get("state") == "active"):
raise RuntimeError("publication unavailable")
super().set(kind, key, value)
store = FailingPublishStore()
desired = _active_fabric_record(current)
if blocked_by == "operation_unsettled":
desired["operation"] = {"id": "1" * 32, "status": "queued"}
pending = json.loads(json.dumps(desired))
FabricCoordinator._invalidate(pending)
store.set("fabric_desired", current.fabric_id, json.dumps(pending))
store.reject_active = blocked_by == "save_failed"
notifications = []
coordinator = FabricCoordinator(
store, Registry({}), on_active_readback=lambda: notifications.append(True),
)
async def exercise():
lock = coordinator._locks.setdefault(current.fabric_id, asyncio.Lock())
async with lock, coordinator.catalog_mutation_lock():
if blocked_by == "save_failed":
with pytest.raises(RuntimeError, match="publication unavailable"):
await coordinator._publish_reconcile(
desired, current, respect_operation=True)
else:
await coordinator._publish_reconcile(
desired, current, respect_operation=True)
asyncio.run(exercise())
assert notifications == []
durable = json.loads(store.get("fabric_desired", current.fabric_id))
assert durable["next_stage_allowed"] is False
assert durable["state"] != "active"
def _reconcile_observation(current, host):
public_key = KEY1 if host.host_id == "host-a" else KEY2
return {
"state": "active",
"config_ok": True,
"transport_verified": True,
"public_key": public_key,
"observed": {
"interface": FABRIC_INTERFACE,
"config_revision": current.revision,
"config_digest": current.digest,
},
}
class ReconcileRecoveryAdapter:
"""Stateful host stub for fixed-plan reconcile recovery tests."""
def __init__(self, manager, _profile=None):
self.manager = manager
def observe(self, current, host, _keys):
self.manager.reconcile_observe_calls += 1
if self.manager.reconcile_observe_error is not None:
error = self.manager.reconcile_observe_error
if self.manager.reconcile_observe_error_once:
self.manager.reconcile_observe_error = None
raise error
if self.manager.reconcile_configured:
result = _reconcile_observation(current, host)
result["public_key"] = self.manager.public_key
result["transport_verified"] = (
self.manager.reconcile_transport_verified)
return result
return {
"state": "degraded",
"config_ok": False,
"transport_verified": False,
"public_key": self.manager.public_key,
"observed": {
"interface": None,
"config_revision": None,
"config_digest": None,
"emergency_state": "owned",
},
}
def identity(self, *, create_key=True):
self.manager.reconcile_identity_calls += 1
self.manager.reconcile_identity_create_flags.append(create_key)
entered = self.manager.reconcile_identity_entered
release = self.manager.reconcile_identity_release
if entered is not None:
entered.set()
if release is not None:
assert release.wait(timeout=2)
if self.manager.reconcile_identity_error is not None:
raise self.manager.reconcile_identity_error
return self.manager.public_key
def apply(self, *_args):
self.manager.reconcile_apply_calls += 1
entered = self.manager.reconcile_apply_entered
release = self.manager.reconcile_apply_release
if entered is not None:
entered.set()
if release is not None:
assert release.wait(timeout=2)
if self.manager.reconcile_apply_error is not None:
raise self.manager.reconcile_apply_error
self.manager.reconcile_configured = True
def _recovery_components(current, *, configured=False):
components = _fabric_components(current)
for component in components.values():
manager = component.manager
manager.reconcile_configured = configured
manager.reconcile_transport_verified = True
manager.reconcile_observe_calls = 0
manager.reconcile_observe_error = None
manager.reconcile_observe_error_once = False
manager.reconcile_identity_calls = 0
manager.reconcile_identity_create_flags = []
manager.reconcile_identity_error = None
manager.reconcile_identity_entered = None
manager.reconcile_identity_release = None
manager.reconcile_apply_calls = 0
manager.reconcile_apply_error = None
manager.reconcile_apply_entered = None
manager.reconcile_apply_release = None
return components
def test_successful_apply_clears_stale_failure_phase():
current = plan()
store = MemoryStore()
record = _active_fabric_record(current)
for node in record["nodes"]:
node["last_error_code"] = "fabric_remote_command_failed"
node["last_error_phase"] = "observe"
store.set("fabric_desired", current.fabric_id, json.dumps(record))
coordinator = FabricCoordinator(
store, Registry(_fabric_components(current)))
result = asyncio.run(coordinator.apply(plan_payload(current)))
assert result["state"] == "active"
assert all("last_error_phase" not in node for node in result["nodes"])
durable = json.loads(store.get("fabric_desired", current.fabric_id))
assert all("last_error_phase" not in node for node in durable["nodes"])
for host in current.nodes:
persisted = json.loads(store.get(
"fabric_node", f"{current.fabric_id}.{host.host_id}"))
assert "last_error_phase" not in persisted
assert persisted["last_error_code"] is None
def test_reconcile_recovers_when_incomplete_host_observe_initially_fails(
monkeypatch):
current = plan()
store = MemoryStore()
store.set("fabric_desired", current.fabric_id, json.dumps(
_active_fabric_record(current)))
components = _recovery_components(current)
first = components["host-a"].manager
first.reconcile_observe_error = FabricError(
"incomplete", 502, "fabric_remote_command_failed")
first.reconcile_observe_error_once = True
coordinator = FabricCoordinator(store, Registry(components))
monkeypatch.setattr(
"virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter)
result = asyncio.run(coordinator.reconcile(current.fabric_id))
assert result["state"] == "active"
assert result["next_stage_allowed"] is True
assert all("last_error_phase" not in node for node in result["nodes"])
assert all(component.manager.reconcile_identity_calls == 1
for component in components.values())
assert all(component.manager.reconcile_apply_calls == 1
for component in components.values())
assert first.reconcile_observe_calls == 2
def test_reconcile_persists_post_apply_observe_failure_phase(monkeypatch):
current = plan()
store = MemoryStore()
store.set("fabric_desired", current.fabric_id, json.dumps(
_active_fabric_record(current)))
components = _recovery_components(current)
failed = components["host-a"].manager
class PostApplyObserveFailureAdapter(ReconcileRecoveryAdapter):
def observe(self, plan_value, host, keys):
if self.manager is failed and self.manager.reconcile_configured:
self.manager.reconcile_observe_calls += 1
raise FabricError(
"incomplete", 502, "fabric_remote_command_failed")
return super().observe(plan_value, host, keys)
coordinator = FabricCoordinator(store, Registry(components))
monkeypatch.setattr(
"virsh_panel.fabric.FabricHostAdapter",
PostApplyObserveFailureAdapter)
result = asyncio.run(coordinator.reconcile(current.fabric_id))
nodes = {node["host_id"]: node for node in result["nodes"]}
assert result["state"] == "uncertain"
assert nodes["host-a"]["last_error_code"] == (
"fabric_remote_command_failed")
assert nodes["host-a"]["last_error_phase"] == "observe"
persisted = json.loads(store.get(
"fabric_node", f"{current.fabric_id}.host-a"))
assert persisted["last_error_phase"] == "observe"
assert failed.reconcile_observe_calls == 2
assert failed.reconcile_apply_calls == 1
def test_reconcile_identity_gate_keeps_other_host_observe_phase(monkeypatch):
current = plan()
store = MemoryStore()
store.set("fabric_desired", current.fabric_id, json.dumps(
_active_fabric_record(current)))
components = _recovery_components(current)
observed_failed = components["host-a"].manager
identity_failed = components["host-b"].manager
observed_failed.reconcile_observe_error = FabricError(
"incomplete", 502, "fabric_remote_command_failed")
identity_failed.reconcile_identity_error = FabricError(
"offline", 502, "fabric_remote_command_failed")
coordinator = FabricCoordinator(store, Registry(components))
monkeypatch.setattr(
"virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter)
result = asyncio.run(coordinator.reconcile(current.fabric_id))
nodes = {node["host_id"]: node for node in result["nodes"]}
assert nodes["host-a"]["last_error_phase"] == "observe"
assert nodes["host-b"]["last_error_phase"] == "identity"
assert all(component.manager.reconcile_apply_calls == 0
for component in components.values())
def test_successful_reconcile_observation_clears_stale_failure_phase(
monkeypatch):
current = plan()
store = MemoryStore()
record = _active_fabric_record(current)
for node in record["nodes"]:
node["last_error_code"] = "fabric_remote_command_failed"
node["last_error_phase"] = "apply"
store.set("fabric_desired", current.fabric_id, json.dumps(record))
components = _recovery_components(current, configured=True)
coordinator = FabricCoordinator(store, Registry(components))
monkeypatch.setattr(
"virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter)
result = asyncio.run(coordinator.reconcile(current.fabric_id))
assert result["state"] == "active"
assert all(node["last_error_code"] is None for node in result["nodes"])
assert all("last_error_phase" not in node for node in result["nodes"])
durable = json.loads(store.get("fabric_desired", current.fabric_id))
assert all("last_error_phase" not in node for node in durable["nodes"])
for host in current.nodes:
persisted = json.loads(store.get(
"fabric_node", f"{current.fabric_id}.{host.host_id}"))
assert "last_error_phase" not in persisted
assert persisted["last_error_code"] is None
def test_reconcile_recovers_key_and_emergency_only_then_is_idempotent(
monkeypatch):
current = plan()
store = MemoryStore()
store.set("fabric_desired", current.fabric_id, json.dumps(
_active_fabric_record(current)))
components = _recovery_components(current)
coordinator = FabricCoordinator(store, Registry(components))
monkeypatch.setattr(
"virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter)
first = asyncio.run(coordinator.reconcile(current.fabric_id))
second = asyncio.run(coordinator.reconcile(current.fabric_id))
assert first["state"] == "active"
assert first["next_stage_allowed"] is True
assert second["state"] == "active"
assert second["next_stage_allowed"] is True
for component in components.values():
manager = component.manager
assert manager.reconcile_apply_calls == 1
assert manager.reconcile_identity_calls == 1
assert manager.reconcile_identity_create_flags == [False]
# initial degraded + post-apply exact + second reconcile exact
assert manager.reconcile_observe_calls == 3
def test_reconcile_does_not_reapply_exact_config_for_stale_transport(
monkeypatch):
current = plan()
store = MemoryStore()
store.set("fabric_desired", current.fabric_id, json.dumps(
_active_fabric_record(current)))
components = _recovery_components(current, configured=True)
for component in components.values():
component.manager.reconcile_transport_verified = False
coordinator = FabricCoordinator(store, Registry(components))
monkeypatch.setattr(
"virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter)
result = asyncio.run(coordinator.reconcile(current.fabric_id))
assert result["state"] == "degraded"
assert result["next_stage_allowed"] is False
for component in components.values():
manager = component.manager
assert manager.reconcile_apply_calls == 0
assert manager.reconcile_identity_calls == 0
assert manager.reconcile_observe_calls == 1
def test_reconcile_does_not_reapply_exact_config_for_failed_ping():
current = plan()
store = MemoryStore()
store.set("fabric_desired", current.fabric_id, json.dumps(
_active_fabric_record(current)))
components = _fabric_components(current)
for component in components.values():
component.manager.ping_ok = False
coordinator = FabricCoordinator(store, Registry(components))
result = asyncio.run(coordinator.reconcile(current.fabric_id))
assert result["state"] == "degraded"
assert result["next_stage_allowed"] is False
assert all(node["state"] == "active" for node in result["nodes"])
assert all(node["transport_verified"] is False
for node in result["nodes"])
# One script is the initial observe. An identity probe or apply would add
# another script and prove reconcile mistook liveness for config drift.
assert all(len(component.manager.scripts) == 1
for component in components.values())
def test_reconcile_repairs_live_public_key_drift():
current = plan()
store = MemoryStore()
store.set("fabric_desired", current.fabric_id, json.dumps(
_active_fabric_record(current)))
components = _fabric_components(current)
drifted = components["host-a"].manager
drifted.live_public_key = KEY3
coordinator = FabricCoordinator(store, Registry(components))
result = asyncio.run(coordinator.reconcile(current.fabric_id))
assert result["state"] == "active"
assert result["next_stage_allowed"] is True
assert drifted.live_public_key == KEY1
# observe -> fixed-key identity -> apply -> exact post-observe
assert len(drifted.scripts) == 4
# The already exact peer receives only its initial observation.
assert len(components["host-b"].manager.scripts) == 1
@pytest.mark.parametrize("identity_failure", ["drift", "unavailable"])
def test_reconcile_identity_failure_blocks_all_host_writes(
monkeypatch, identity_failure):
current = plan()
store = MemoryStore()
store.set("fabric_desired", current.fabric_id, json.dumps(
_active_fabric_record(current)))
components = _recovery_components(current)
failed = components["host-b"].manager
expected_error = "fabric_identity_conflict"
if identity_failure == "drift":
failed.public_key = KEY3
else:
expected_error = "fabric_remote_command_failed"
failed.reconcile_identity_error = FabricError(
"offline", 502, expected_error)
coordinator = FabricCoordinator(store, Registry(components))
monkeypatch.setattr(
"virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter)
result = asyncio.run(coordinator.reconcile(current.fabric_id))
nodes = {node["host_id"]: node for node in result["nodes"]}
assert result["state"] == "uncertain"
assert result["next_stage_allowed"] is False
assert nodes["host-b"]["last_error_code"] == expected_error
assert nodes["host-b"]["last_error_phase"] == "identity"
persisted = json.loads(store.get(
"fabric_node", f"{current.fabric_id}.host-b"))
assert persisted["last_error_phase"] == "identity"
for component in components.values():
manager = component.manager
assert manager.reconcile_identity_calls == 1
assert manager.reconcile_identity_create_flags == [False]
assert manager.reconcile_apply_calls == 0
def test_reconcile_apply_failure_on_one_node_does_not_stop_other_node(
monkeypatch):
current = plan()
store = MemoryStore()
store.set("fabric_desired", current.fabric_id, json.dumps(
_active_fabric_record(current)))
components = _recovery_components(current)
components["host-a"].manager.reconcile_apply_error = FabricError(
"offline", 502, "fabric_remote_command_failed")
coordinator = FabricCoordinator(store, Registry(components))
monkeypatch.setattr(
"virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter)
result = asyncio.run(coordinator.reconcile(current.fabric_id))
nodes = {node["host_id"]: node for node in result["nodes"]}
assert result["state"] == "uncertain"
assert nodes["host-a"]["last_error_code"] == (
"fabric_remote_command_failed")
assert nodes["host-a"]["last_error_phase"] == "apply"
assert nodes["host-b"]["state"] == "active"
assert nodes["host-b"]["observed_plan_digest"] == current.digest
assert all(component.manager.reconcile_identity_calls == 1
for component in components.values())
assert all(component.manager.reconcile_apply_calls == 1
for component in components.values())
def test_cancelled_reconcile_drains_admitted_host_apply_before_unlock(
monkeypatch):
current = plan()
store = MemoryStore()
store.set("fabric_desired", current.fabric_id, json.dumps(
_active_fabric_record(current)))
components = _recovery_components(current)
entered = threading.Event()
release = threading.Event()
first = components["host-a"].manager
first.reconcile_apply_entered = entered
first.reconcile_apply_release = release
coordinator = FabricCoordinator(store, Registry(components))
monkeypatch.setattr(
"virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter)
async def exercise():
reconcile_task = asyncio.create_task(
coordinator.reconcile(current.fabric_id))
for _attempt in range(100):
if entered.is_set():
break
await asyncio.sleep(0.01)
assert entered.is_set()
lock_entered = asyncio.Event()
async def wait_for_fabric_lock():
async with coordinator._locks[current.fabric_id]:
lock_entered.set()
waiter = asyncio.create_task(wait_for_fabric_lock())
reconcile_task.cancel()
await asyncio.sleep(0)
assert reconcile_task.done() is False
assert lock_entered.is_set() is False
release.set()
with pytest.raises(asyncio.CancelledError):
await reconcile_task
await asyncio.wait_for(waiter, timeout=1)
assert lock_entered.is_set() is True
try:
asyncio.run(exercise())
finally:
release.set()
assert first.reconcile_apply_calls == 1
assert first.reconcile_configured is True
assert components["host-b"].manager.reconcile_apply_calls == 0
def test_cancelled_reconcile_drains_runtime_identity_lock_before_unlock(
monkeypatch):
current = plan()
store = MemoryStore()
store.set("fabric_desired", current.fabric_id, json.dumps(
_active_fabric_record(current)))
components = _recovery_components(current)
entered = threading.Event()
release = threading.Event()
first = components["host-a"].manager
first.reconcile_identity_entered = entered
first.reconcile_identity_release = release
coordinator = FabricCoordinator(store, Registry(components))
monkeypatch.setattr(
"virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter)
async def exercise():
reconcile_task = asyncio.create_task(
coordinator.reconcile(current.fabric_id))
for _attempt in range(100):
if entered.is_set():
break
await asyncio.sleep(0.01)
assert entered.is_set()
lock_entered = asyncio.Event()
async def wait_for_fabric_lock():
async with coordinator._locks[current.fabric_id]:
lock_entered.set()
waiter = asyncio.create_task(wait_for_fabric_lock())
reconcile_task.cancel()
await asyncio.sleep(0)
assert reconcile_task.done() is False
assert lock_entered.is_set() is False
release.set()
with pytest.raises(asyncio.CancelledError):
await reconcile_task
await asyncio.wait_for(waiter, timeout=1)
assert lock_entered.is_set() is True
try:
asyncio.run(exercise())
finally:
release.set()
assert first.reconcile_identity_calls == 1
assert first.reconcile_apply_calls == 0
assert components["host-b"].manager.reconcile_identity_calls == 0
assert components["host-b"].manager.reconcile_apply_calls == 0
def _vpc_payload_for_fabric(current, *, vpc_id="vpc-race"):
return {
"schema_version": 1,
"vpc_id": vpc_id,
"region": current.region,
"fabric_id": current.fabric_id,
"fabric_revision": current.revision,
"cidr": "10.20.0.0/24",
"desired_revision": 1,
"operation_id": f"{vpc_id}-operation-0001",
"subnets": [
{"subnet_id": "subnet-a", "cidr": "10.20.0.0/25"},
],
"hosts": [{
"host_id": "host-a",
"members": [{
"member_id": "member-a",
"instance_uuid": "11111111-1111-4111-8111-111111111111",
"private_ip_cidr": "10.20.0.10/32",
"subnet_id": "subnet-a",
}],
}],
"routes": [],
"acls": [],
}
def test_fabric_readback_wakes_vpc_after_periodic_closed_gate_race(monkeypatch):
from virsh_panel.vpc import VpcCoordinator, parse_vpc_plan
current = plan()
store = MemoryStore()
store.set("fabric_desired", current.fabric_id, json.dumps(
_active_fabric_record(current)))
observe_entered = threading.Event()
observe_release = threading.Event()
published = []
def on_active_readback():
published.append(json.loads(store.get(
"fabric_desired", current.fabric_id))["state"])
vpc.wake_reconcile()
coordinator = FabricCoordinator(
store, Registry(_fabric_components(current)),
on_active_readback=on_active_readback,
)
vpc = VpcCoordinator(
store, coordinator, catalog_lock=coordinator.catalog_mutation_lock,
)
vpc_plan = parse_vpc_plan(_vpc_payload_for_fabric(current))
initial = vpc._desired_record(
vpc_plan, vpc._fabric_gate(vpc_plan), state="applying")
store.set("vpc_desired", vpc_plan.vpc_id, json.dumps(initial))
host_readbacks = []
original_dataplane = vpc._reconcile_dataplane
async def readback_hosts(plan, record, gate, previous_plan=None, **kwargs):
if gate.get("active") is not True:
return await original_dataplane(
plan, record, gate, previous_plan, **kwargs)
host_readbacks.append(plan.desired_revision)
for host in record["hosts"]:
host.update({
"state": "active",
"transport_verified": True,
"private_dataplane_verified": True,
"observed_revision": plan.desired_revision,
"observed_digest": plan.desired_digest,
})
vpc._finish(record, gate)
class BlockingObserveAdapter:
def __init__(self, _manager, _profile=None):
pass
def observe(self, _plan, host, _keys):
observe_entered.set()
assert observe_release.wait(timeout=5)
return _reconcile_observation(current, host)
monkeypatch.setattr(vpc, "_reconcile_dataplane", readback_hosts)
monkeypatch.setattr(
"virsh_panel.fabric.FabricHostAdapter", BlockingObserveAdapter)
async def exercise():
results = asyncio.Queue()
original_reconcile = vpc.reconcile
async def record_reconcile(vpc_id):
result = await original_reconcile(vpc_id)
results.put_nowait(result)
return result
monkeypatch.setattr(vpc, "reconcile", record_reconcile)
fabric_task = asyncio.create_task(coordinator.reconcile_all())
vpc_task = None
try:
assert await asyncio.to_thread(observe_entered.wait, 2)
assert coordinator.get_status(current.fabric_id)["state"] == "uncertain"
vpc_task = asyncio.create_task(vpc.reconcile_loop(interval=3600))
closed = await asyncio.wait_for(results.get(), timeout=2)
assert closed["state"] == "degraded"
assert closed["fabric_gate"]["active"] is False
assert closed["private_dataplane_verified"] is False
assert host_readbacks == []
assert published == []
observe_release.set()
await asyncio.wait_for(fabric_task, timeout=2)
refreshed = await asyncio.wait_for(results.get(), timeout=2)
assert refreshed["state"] == "active"
assert refreshed["fabric_gate"]["active"] is True
assert refreshed["private_dataplane_verified"] is True
assert host_readbacks == [vpc_plan.desired_revision]
assert published == ["active"]
finally:
observe_release.set()
fabric_task.cancel()
if vpc_task is not None:
vpc_task.cancel()
await asyncio.gather(
fabric_task, *([vpc_task] if vpc_task is not None else []),
return_exceptions=True)
assert coordinator.catalog_mutation_lock().locked() is False
assert coordinator._locks[current.fabric_id].locked() is False
assert vpc._lock(vpc_plan.vpc_id).locked() is False
try:
asyncio.run(exercise())
finally:
observe_release.set()
def test_reconcile_waits_for_admitted_vpc_apply_before_invalidating(
monkeypatch):
from virsh_panel.vpc import VpcCoordinator
current = plan()
store = MemoryStore()
store.set("fabric_desired", current.fabric_id, json.dumps(
_active_fabric_record(current)))
coordinator = FabricCoordinator(
store, Registry(_fabric_components(current)))
vpc = VpcCoordinator(
store, coordinator,
catalog_lock=coordinator.catalog_mutation_lock,
)
vpc_entered = asyncio.Event()
vpc_release = asyncio.Event()
observe_entered = threading.Event()
observe_release = threading.Event()
async def blocked_vpc_apply(_plan):
vpc_entered.set()
await vpc_release.wait()
return {"vpc_id": "vpc-race", "state": "active"}
class BlockingObserveAdapter:
def __init__(self, _manager, _profile=None):
pass
def observe(self, _plan, host, _keys):
observe_entered.set()
assert observe_release.wait(timeout=2)
return _reconcile_observation(current, host)
vpc._apply_locked = blocked_vpc_apply
monkeypatch.setattr(
"virsh_panel.fabric.FabricHostAdapter", BlockingObserveAdapter)
vpc_payload = _vpc_payload_for_fabric(current)
async def exercise():
vpc_task = asyncio.create_task(vpc.apply(vpc_payload))
await vpc_entered.wait()
reconcile_task = asyncio.create_task(
coordinator.reconcile(current.fabric_id))
await asyncio.sleep(0)
await asyncio.sleep(0)
before = json.loads(store.get(
"fabric_desired", current.fabric_id))
assert before["state"] == "active"
assert before["next_stage_allowed"] is True
vpc_release.set()
await vpc_task
for _attempt in range(100):
if observe_entered.is_set():
break
await asyncio.sleep(0.01)
assert observe_entered.is_set()
invalidated = json.loads(store.get(
"fabric_desired", current.fabric_id))
assert invalidated["state"] == "uncertain"
assert invalidated["next_stage_allowed"] is False
observe_release.set()
return await asyncio.wait_for(reconcile_task, timeout=1)
try:
result = asyncio.run(exercise())
finally:
observe_release.set()
assert result["state"] == "active"
assert result["next_stage_allowed"] is True
def test_reconcile_first_durable_invalidation_blocks_vpc_host_mutation(
monkeypatch):
from virsh_panel.vpc import VpcCoordinator
current = plan()
class BlockingInvalidationStore(MemoryStore):
def __init__(self):
super().__init__()
self.block = False
self.entered = threading.Event()
self.release = threading.Event()
def set(self, kind, key, value):
parsed = json.loads(value)
if (self.block and kind == "fabric_desired"
and parsed.get("state") == "uncertain"):
self.entered.set()
assert self.release.wait(timeout=2)
super().set(kind, key, value)
observe_entered = threading.Event()
observe_release = threading.Event()
class BlockingObserveAdapter:
def __init__(self, _manager, _profile=None):
pass
def observe(self, _plan, host, _keys):
observe_entered.set()
assert observe_release.wait(timeout=2)
return _reconcile_observation(current, host)
store = BlockingInvalidationStore()
store.set("fabric_desired", current.fabric_id, json.dumps(
_active_fabric_record(current)))
store.block = True
coordinator = FabricCoordinator(
store, Registry(_fabric_components(current)))
vpc = VpcCoordinator(
store, coordinator,
registry=SimpleNamespace(components={}),
catalog_lock=coordinator.catalog_mutation_lock,
)
host_mutation_called = False
def forbidden_host_validation(_plan):
nonlocal host_mutation_called
host_mutation_called = True
raise AssertionError("inactive Fabric reached host mutation")
vpc._validate_dataplane_resources = forbidden_host_validation
monkeypatch.setattr(
"virsh_panel.fabric.FabricHostAdapter", BlockingObserveAdapter)
async def exercise():
reconcile_task = asyncio.create_task(
coordinator.reconcile(current.fabric_id))
for _attempt in range(100):
if store.entered.is_set():
break
await asyncio.sleep(0.01)
assert store.entered.is_set()
vpc_task = asyncio.create_task(vpc.apply(
_vpc_payload_for_fabric(current, vpc_id="vpc-after-reconcile")))
await asyncio.sleep(0)
assert vpc_task.done() is False
assert host_mutation_called is False
still_active = json.loads(store.get(
"fabric_desired", current.fabric_id))
assert still_active["state"] == "active"
store.release.set()
for _attempt in range(100):
if observe_entered.is_set():
break
await asyncio.sleep(0.01)
assert observe_entered.is_set()
result = await asyncio.wait_for(vpc_task, timeout=1)
assert result["state"] == "degraded"
assert result["next_stage_allowed"] is False
assert result["fabric_gate"]["error_code"] == (
"vpc_fabric_not_active")
assert host_mutation_called is False
durable_vpc = json.loads(store.get(
"vpc_desired", "vpc-after-reconcile"))
assert durable_vpc["state"] == "degraded"
observe_release.set()
await asyncio.wait_for(reconcile_task, timeout=1)
try:
asyncio.run(exercise())
finally:
store.release.set()
observe_release.set()
def test_reconcile_releases_catalog_during_remote_observe_and_relocks_publish(
monkeypatch):
current = plan()
store = MemoryStore()
store.set("fabric_desired", current.fabric_id, json.dumps(
_active_fabric_record(current)))
coordinator = FabricCoordinator(
store, Registry(_fabric_components(current)))
observe_entered = threading.Event()
observe_release = threading.Event()
all_observed = threading.Event()
count_lock = threading.Lock()
observation_count = 0
class BlockingObserveAdapter:
def __init__(self, _manager, _profile=None):
pass
def observe(self, _plan, host, _keys):
nonlocal observation_count
observe_entered.set()
assert observe_release.wait(timeout=2)
with count_lock:
observation_count += 1
if observation_count == len(current.nodes):
all_observed.set()
return _reconcile_observation(current, host)
monkeypatch.setattr(
"virsh_panel.fabric.FabricHostAdapter", BlockingObserveAdapter)
async def exercise():
reconcile_task = asyncio.create_task(
coordinator.reconcile(current.fabric_id))
for _attempt in range(100):
if observe_entered.is_set():
break
await asyncio.sleep(0.01)
assert observe_entered.is_set()
shared_lock = coordinator.catalog_mutation_lock()
await asyncio.wait_for(shared_lock.acquire(), timeout=1)
try:
observe_release.set()
for _attempt in range(100):
if all_observed.is_set():
break
await asyncio.sleep(0.01)
assert all_observed.is_set()
assert reconcile_task.done() is False
durable = json.loads(store.get(
"fabric_desired", current.fabric_id))
assert durable["state"] == "uncertain"
assert durable["next_stage_allowed"] is False
finally:
shared_lock.release()
return await asyncio.wait_for(reconcile_task, timeout=1)
try:
result = asyncio.run(exercise())
finally:
observe_release.set()
assert result["state"] == "active"
assert result["next_stage_allowed"] is True
@pytest.mark.parametrize("blocked_state", ["uncertain", "active"])
def test_cancelled_reconcile_drains_root_save_before_catalog_unlock(
monkeypatch, blocked_state):
current = plan()
class BlockingRootStore(MemoryStore):
def __init__(self):
super().__init__()
self.block = False
self.entered = threading.Event()
self.release = threading.Event()
def set(self, kind, key, value):
parsed = json.loads(value)
if (self.block and kind == "fabric_desired"
and parsed.get("state") == blocked_state):
self.entered.set()
assert self.release.wait(timeout=2)
super().set(kind, key, value)
class ImmediateObserveAdapter:
def __init__(self, _manager, _profile=None):
pass
def observe(self, _plan, host, _keys):
return _reconcile_observation(current, host)
store = BlockingRootStore()
store.set("fabric_desired", current.fabric_id, json.dumps(
_active_fabric_record(current)))
store.block = True
coordinator = FabricCoordinator(
store, Registry(_fabric_components(current)))
monkeypatch.setattr(
"virsh_panel.fabric.FabricHostAdapter", ImmediateObserveAdapter)
async def exercise():
reconcile_task = asyncio.create_task(
coordinator.reconcile(current.fabric_id))
for _attempt in range(100):
if store.entered.is_set():
break
await asyncio.sleep(0.01)
assert store.entered.is_set()
catalog_entered = asyncio.Event()
async def competing_catalog_apply():
async with coordinator.catalog_mutation_lock():
catalog_entered.set()
competitor = asyncio.create_task(competing_catalog_apply())
await asyncio.sleep(0)
reconcile_task.cancel()
await asyncio.sleep(0)
assert reconcile_task.done() is False
assert catalog_entered.is_set() is False
store.release.set()
with pytest.raises(asyncio.CancelledError):
await reconcile_task
await asyncio.wait_for(competitor, timeout=1)
assert catalog_entered.is_set() is True
try:
asyncio.run(exercise())
finally:
store.release.set()
durable = json.loads(store.get("fabric_desired", current.fabric_id))
assert durable["state"] == blocked_state
assert durable["next_stage_allowed"] is (blocked_state == "active")
def test_new_fabric_cannot_reuse_a_host_or_overlay_catalog_entry():
store = MemoryStore()
store.set("fabric_desired", "existing", json.dumps({
"fabric_id": "existing",
"overlay_cidr": "100.64.2.0/24",
"nodes": [{"host_id": "host-a"}, {"host_id": "host-c"}],
}))
coordinator = FabricCoordinator(store, Registry({}))
value = {
"fabric_id": "new-fabric", "region": "hk",
"overlay_cidr": "100.64.2.0/24", "revision": 1,
"nodes": [
{"host_id": "host-a", "overlay_ip": "100.64.2.10",
"endpoint": "203.0.113.1:51820"},
{"host_id": "host-b", "overlay_ip": "100.64.2.11",
"endpoint": "203.0.113.2:51820"},
],
}
with pytest.raises(FabricError) as error:
asyncio.run(coordinator.apply(value))
assert error.value.error_code == "fabric_host_interface_conflict"
def test_existing_fabric_region_is_immutable_without_migration():
store = MemoryStore()
current = plan()
store.set("fabric_desired", current.fabric_id, json.dumps({
"fabric_id": current.fabric_id,
"region": current.region,
"revision": current.revision,
"plan_digest": current.digest,
"nodes": [{"host_id": node.host_id} for node in current.nodes],
}))
coordinator = FabricCoordinator(store, Registry({}))
value = {
"fabric_id": current.fabric_id,
"region": "sg",
"overlay_cidr": current.overlay_cidr,
"revision": current.revision + 1,
"nodes": [
{"host_id": node.host_id, "overlay_ip": node.overlay_ip,
"endpoint": node.endpoint}
for node in current.nodes
],
}
with pytest.raises(FabricError) as error:
asyncio.run(coordinator.apply(value))
assert error.value.error_code == "fabric_region_conflict"
@pytest.mark.parametrize(("tenant_fabric", "tenant_region", "expected"), [
("hk-fabric", "sg", "fabric_region_conflict"),
("other-fabric", "sg", "fabric_endpoint_vpc_conflict"),
])
def test_fabric_checks_region_identity_and_all_tenant_cidrs(
tenant_fabric, tenant_region, expected):
from virsh_panel.vpc import parse_vpc_plan
store = MemoryStore()
tenant = parse_vpc_plan({
"schema_version": 1,
"vpc_id": "vpc-existing",
"region": tenant_region,
"fabric_id": tenant_fabric,
"fabric_revision": 1,
"cidr": "192.168.50.0/24",
"desired_revision": 1,
"operation_id": "vpc-op-region-0001",
"subnets": [{"subnet_id": "subnet-a",
"cidr": "192.168.50.0/25"}],
"hosts": [{"host_id": "host-a", "members": [{
"member_id": "member-a",
"instance_uuid": "11111111-1111-4111-8111-111111111111",
"private_ip_cidr": "192.168.50.10/32",
"subnet_id": "subnet-a",
}]}],
"routes": [],
"acls": [],
})
record = tenant.canonical()
record.update({
"operation_id": tenant.operation_id,
"desired_digest": tenant.desired_digest,
})
store.set("vpc_desired", tenant.vpc_id, json.dumps(record))
coordinator = FabricCoordinator(store, Registry({}))
candidate = parse_fabric_plan({
"fabric_id": "hk-fabric", "region": "hk",
"overlay_cidr": "100.64.1.0/24", "revision": 1,
"nodes": [
{"host_id": "host-a", "overlay_ip": "100.64.1.1",
"endpoint": "192.168.50.20:51820"},
{"host_id": "host-b", "overlay_ip": "100.64.1.2",
"endpoint": "203.0.113.2:51820"},
],
})
with pytest.raises(FabricError) as error:
coordinator._validate_resource_exclusivity(candidate)
assert error.value.error_code == expected
def test_new_fabric_rejects_endpoint_inside_existing_tenant_cidr():
from virsh_panel.vpc import parse_vpc_plan
store = MemoryStore()
tenant = parse_vpc_plan({
"schema_version": 1,
"vpc_id": "vpc-hk-1",
"region": "hk",
"fabric_id": "hk-fabric",
"fabric_revision": 1,
"cidr": "10.20.0.0/24",
"desired_revision": 1,
"operation_id": "vpc-op-00000001",
"subnets": [{"subnet_id": "subnet-a", "cidr": "10.20.0.0/25"}],
"hosts": [{"host_id": "host-a", "members": [{
"member_id": "member-a",
"instance_uuid": "11111111-1111-4111-8111-111111111111",
"private_ip_cidr": "10.20.0.10/32",
"subnet_id": "subnet-a",
}]}],
"routes": [],
"acls": [],
})
tenant_record = tenant.canonical()
tenant_record.update({
"operation_id": tenant.operation_id,
"desired_digest": tenant.desired_digest,
})
store.set("vpc_desired", tenant.vpc_id, json.dumps(tenant_record))
coordinator = FabricCoordinator(store, Registry({}))
value = {
"fabric_id": "hk-fabric", "region": "hk",
"overlay_cidr": "100.64.1.0/24", "revision": 1,
"nodes": [
{"host_id": "host-a", "overlay_ip": "100.64.1.1",
"endpoint": "10.20.0.50:51820"},
{"host_id": "host-b", "overlay_ip": "100.64.1.2",
"endpoint": "192.168.50.2:51820"},
],
}
with pytest.raises(FabricError) as error:
asyncio.run(coordinator.apply(value))
assert error.value.error_code == "fabric_endpoint_vpc_conflict"
current = parse_fabric_plan(value)
store.set("fabric_desired", current.fabric_id, json.dumps({
"schema_version": 1,
"fabric_id": current.fabric_id,
"region": current.region,
"overlay_cidr": current.overlay_cidr,
"revision": current.revision,
"listen_port": current.listen_port,
"plan_digest": current.digest,
"state": "active",
"next_stage_allowed": True,
"updated_at": int(time.time()),
"public_keys": {"host-a": KEY1, "host-b": KEY2},
"nodes": [
{"host_id": node.host_id, "overlay_ip": node.overlay_ip,
"endpoint": node.endpoint, "listen_port": node.listen_port,
"state": "active", "transport_verified": True}
for node in current.nodes
],
}))
with pytest.raises(FabricError) as reconcile_error:
asyncio.run(coordinator.reconcile(current.fabric_id))
assert reconcile_error.value.error_code == "fabric_endpoint_vpc_conflict"
invalidated = json.loads(store.get("fabric_desired", current.fabric_id))
assert invalidated["state"] == "uncertain"
assert invalidated["next_stage_allowed"] is False
class TenantReadbackManager(FakeManager):
"""Synthetic readback with panel-owned VPC tenant state."""
def __init__(self, *args, tenant_ip="10.10.0.2/32", **kwargs):
super().__init__(*args, **kwargs)
self.tenant_ip = tenant_ip
def _exec(self, argv, *, timeout, input_text=None):
result = super()._exec(argv, timeout=timeout, input_text=input_text)
if input_text.startswith("#!/bin/sh\nset -eu"):
return result
peer = KEY2 if self.public_key == KEY1 else KEY1
target = "100.64.1.2" if self.public_key == KEY1 else "100.64.1.1"
endpoint = "203.0.113.2:51820" if self.public_key == KEY1 else "203.0.113.1:51820"
marker = f"__VC_FABRIC_PEER__={peer}|{endpoint}|{target}/32"
output = result.stdout.replace(marker, f"{marker},{self.tenant_ip}")
output += (
"__VC_FABRIC_VPC_STATE__=owned\n"
f"__VC_FABRIC_VPC_TENANT__={peer}|{self.tenant_ip}\n"
"__VC_FABRIC_VPC_NFT_STATE__=owned\n"
f"__VC_FABRIC_VPC_NFT_IP__={self.tenant_ip}\n"
)
return result.__class__(result.args, result.returncode, output, result.stderr)
def test_fabric_merges_and_reads_panel_owned_vpc_allowed_ips():
current = plan()
observed = FabricHostAdapter(TenantReadbackManager(KEY1)).observe(
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}
)
assert observed["config_ok"] is True
assert observed["observed"]["peer_allowed_exact"] is True
assert f"{KEY2}|10.10.0.2/32" in observed["observed"]["vpc_tenants"]
assert all("10001" not in route for route in observed["observed"]["routes"])
def test_fabric_rejects_unbacked_vpc_allowed_ip():
current = plan()
manager = FakeManager(KEY1)
original_exec = manager._exec
def unbacked(argv, *, timeout, input_text=None):
result = original_exec(argv, timeout=timeout, input_text=input_text)
if input_text.startswith("#!/bin/sh\nset -eu"):
return result
peer = KEY2
marker = f"__VC_FABRIC_PEER__={peer}|203.0.113.2:51820|100.64.1.2/32"
output = result.stdout.replace(marker, f"{marker},10.10.0.2/32")
return result.__class__(result.args, result.returncode, output, result.stderr)
manager._exec = unbacked
observed = FabricHostAdapter(manager).observe(
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}
)
assert observed["config_ok"] is False
assert observed["observed"]["peer_allowed_exact"] is False
def test_fabric_rejects_non_rfc1918_vpc_tenant_prefix():
current = plan()
observed = FabricHostAdapter(
TenantReadbackManager(KEY1, tenant_ip="169.254.1.2/32")
).observe(current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2})
assert observed["config_ok"] is False
assert observed["observed"]["peer_allowed_exact"] is False
def test_fabric_rejects_stale_vpc_nft_tenant_set():
current = plan()
manager = TenantReadbackManager(KEY1)
original_exec = manager._exec
def stale_nft(argv, *, timeout, input_text=None):
result = original_exec(argv, timeout=timeout, input_text=input_text)
if input_text.startswith("#!/bin/sh\nset -eu"):
return result
output = result.stdout.replace(
"__VC_FABRIC_VPC_NFT_IP__=10.10.0.2/32\n", "")
return result.__class__(result.args, result.returncode, output, result.stderr)
manager._exec = stale_nft
observed = FabricHostAdapter(manager).observe(
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}
)
assert observed["config_ok"] is False
assert observed["observed"]["vpc_nft_exact"] is False
def test_fabric_does_not_hide_foreign_custom_route():
current = plan()
manager = FakeManager(KEY1)
original_exec = manager._exec
def foreign_route(argv, *, timeout, input_text=None):
result = original_exec(argv, timeout=timeout, input_text=input_text)
if input_text.startswith("#!/bin/sh\nset -eu"):
return result
output = result.stdout + (
f"__VC_FABRIC_ROUTE__=10.10.0.2/32|10001|{FABRIC_INTERFACE}\n"
)
return result.__class__(result.args, result.returncode, output, result.stderr)
manager._exec = foreign_route
observed = FabricHostAdapter(manager).observe(
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}
)
assert observed["config_ok"] is False
assert "10.10.0.2/32|10001|wg-fabric0" in observed["observed"]["routes"]