You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
8593 lines
312 KiB
Python
8593 lines
312 KiB
Python
"""Fabric plan, helper, and coordinator tests."""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import base64
|
|
import hashlib
|
|
import json
|
|
import re
|
|
import shlex
|
|
import stat
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
from virsh_panel.fabric import (
|
|
FABRIC_OPERATION_CHECKPOINTS,
|
|
FABRIC_OPERATION_ROOT,
|
|
FABRIC_OPERATION_REASONS,
|
|
FABRIC_OPERATION_START_TIMEOUT_SECONDS,
|
|
FABRIC_OPERATION_STATUSES,
|
|
FABRIC_OPERATION_UNIT_PREFIX,
|
|
FABRIC_PROBE_CONCURRENCY,
|
|
FABRIC_INTERFACE,
|
|
FABRIC_OWNER_MARKER,
|
|
FABRIC_REMOTE_APPLY_TIMEOUT_SECONDS,
|
|
FABRIC_REMOTE_READ_TIMEOUT_SECONDS,
|
|
FABRIC_SYSCTL,
|
|
FIREWALL_MODE_NATIVE,
|
|
MAX_VPC_STATE_FILES,
|
|
MAX_FABRIC_OPERATION_DIRECTORIES,
|
|
FabricError,
|
|
FabricHostAdapter,
|
|
FabricCoordinator,
|
|
_BOOT_HELPER,
|
|
_FABRIC_NFT_VALIDATION_SH,
|
|
_FIREWALL_MODE_GUARD_SH,
|
|
_FABRIC_SYSCTL_VALIDATION_SH,
|
|
_NMCLI_FILENAME_RESOLVER_SH,
|
|
_REPAIR_UNIT,
|
|
_UNIT,
|
|
_apply_script,
|
|
_fabric_endpoint_rp_filter_evidence,
|
|
_fabric_operation_dispatch_script,
|
|
_fabric_operation_runner_script,
|
|
_fabric_operation_status_script,
|
|
_identity_script,
|
|
_new_operation_id,
|
|
_nm_profile_template,
|
|
_nft_rules_text,
|
|
_observe_script,
|
|
_parse_operation_status,
|
|
_profile_firewall_mode,
|
|
_sysctl_text,
|
|
parse_fabric_plan,
|
|
validate_wireguard_key,
|
|
)
|
|
from virsh_panel.remote_host import RemoteHostError
|
|
|
|
|
|
KEY1 = base64.b64encode(bytes([1]) * 32).decode()
|
|
KEY2 = base64.b64encode(bytes([2]) * 32).decode()
|
|
KEY3 = base64.b64encode(bytes([3]) * 32).decode()
|
|
|
|
|
|
def plan():
|
|
return parse_fabric_plan({
|
|
"fabric_id": "hk-fabric",
|
|
"region": "hk",
|
|
"overlay_cidr": "100.64.1.0/24",
|
|
"revision": 1,
|
|
"listen_port": 51820,
|
|
"nodes": [
|
|
{"host_id": "host-a", "overlay_ip": "100.64.1.1",
|
|
"endpoint": "203.0.113.1:51820"},
|
|
{"host_id": "host-b", "overlay_ip": "100.64.1.2",
|
|
"endpoint": "203.0.113.2:51820"},
|
|
],
|
|
})
|
|
|
|
|
|
def plan_payload(current=None):
|
|
current = current or plan()
|
|
return {
|
|
"fabric_id": current.fabric_id,
|
|
"region": current.region,
|
|
"overlay_cidr": current.overlay_cidr,
|
|
"revision": current.revision,
|
|
"listen_port": current.listen_port,
|
|
"nodes": [
|
|
{
|
|
"host_id": node.host_id,
|
|
"overlay_ip": node.overlay_ip,
|
|
"endpoint": node.endpoint,
|
|
"listen_port": node.listen_port,
|
|
}
|
|
for node in current.nodes
|
|
],
|
|
}
|
|
|
|
|
|
def test_plan_is_canonical_and_rejects_wide_or_tenant_routes():
|
|
first = plan()
|
|
reordered = parse_fabric_plan({
|
|
"nodes": [
|
|
{"host_id": "host-b", "overlay_ip": "100.64.1.2",
|
|
"endpoint": "203.0.113.2:51820"},
|
|
{"host_id": "host-a", "overlay_ip": "100.64.1.1",
|
|
"endpoint": "203.0.113.1:51820"},
|
|
],
|
|
"fabric_id": "hk-fabric", "region": "hk",
|
|
"overlay_cidr": "100.64.1.0/24", "revision": 1,
|
|
})
|
|
assert first.digest == reordered.digest
|
|
with pytest.raises(FabricError):
|
|
parse_fabric_plan({
|
|
"fabric_id": "bad", "region": "hk",
|
|
"overlay_cidr": "10.0.0.0/8", "revision": 1,
|
|
"nodes": [
|
|
{"host_id": "a", "overlay_ip": "10.0.0.1",
|
|
"endpoint": "203.0.113.1:51820"},
|
|
{"host_id": "b", "overlay_ip": "10.0.0.2",
|
|
"endpoint": "203.0.113.2:51820"},
|
|
],
|
|
})
|
|
with pytest.raises(FabricError):
|
|
parse_fabric_plan({
|
|
"fabric_id": "bad", "region": "hk",
|
|
"overlay_cidr": "100.64.1.0/24", "revision": 1,
|
|
"nodes": [
|
|
{"host_id": "a", "overlay_ip": "100.64.1.1",
|
|
"endpoint": "100.64.1.2:51820"},
|
|
{"host_id": "b", "overlay_ip": "100.64.1.2",
|
|
"endpoint": "203.0.113.2:51820"},
|
|
],
|
|
})
|
|
with pytest.raises(FabricError):
|
|
parse_fabric_plan({
|
|
"fabric_id": "bad", "region": "hk",
|
|
"overlay_cidr": "100.64.1.0/24", "revision": 1,
|
|
"nodes": [
|
|
{"host_id": "a", "overlay_ip": "100.64.1.1",
|
|
"endpoint": "203.0.113.1:51820"},
|
|
{"host_id": "b", "overlay_ip": "100.64.1.2",
|
|
"endpoint": "0.0.0.0:51820"},
|
|
],
|
|
})
|
|
with pytest.raises(FabricError, match="endpoint"):
|
|
parse_fabric_plan({
|
|
"fabric_id": "bad", "region": "hk",
|
|
"overlay_cidr": "100.64.1.0/24", "revision": 1,
|
|
"nodes": [
|
|
{"host_id": "a", "overlay_ip": "100.64.1.1",
|
|
"endpoint": "203.0.113.1:51820"},
|
|
{"host_id": "b", "overlay_ip": "100.64.1.2",
|
|
"endpoint": "203.0.113.1:51820"},
|
|
],
|
|
})
|
|
|
|
|
|
@pytest.mark.parametrize("address", [
|
|
"0.0.0.0",
|
|
"0.1.2.3",
|
|
"127.23.45.67",
|
|
"169.254.12.34",
|
|
"224.0.0.1",
|
|
"240.0.0.1",
|
|
"255.255.255.255",
|
|
])
|
|
def test_plan_rejects_unusable_endpoint_ipv4(address):
|
|
with pytest.raises(FabricError, match="endpoint 地址不可路由"):
|
|
parse_fabric_plan({
|
|
"fabric_id": "bad-endpoint", "region": "hk",
|
|
"overlay_cidr": "100.64.1.0/24", "revision": 1,
|
|
"nodes": [
|
|
{"host_id": "a", "overlay_ip": "100.64.1.1",
|
|
"endpoint": f"{address}:51820"},
|
|
{"host_id": "b", "overlay_ip": "100.64.1.2",
|
|
"endpoint": "203.0.113.2:51820"},
|
|
],
|
|
})
|
|
|
|
|
|
def test_wireguard_key_rejects_the_all_zero_reserved_key():
|
|
with pytest.raises(FabricError):
|
|
validate_wireguard_key(base64.b64encode(bytes(32)).decode())
|
|
|
|
|
|
def test_generated_helpers_are_posix_safe_and_do_not_embed_private_keys():
|
|
current = plan()
|
|
read_only_identity = _identity_script(create_key=False)
|
|
scripts = (
|
|
_BOOT_HELPER,
|
|
_identity_script(),
|
|
read_only_identity,
|
|
_observe_script(("100.64.1.2",), "100.64.1.1",
|
|
("203.0.113.2",), "networkmanager_nmcli",
|
|
"12345678-1234-5678-1234-567812345678",
|
|
expected_digest=current.digest),
|
|
_apply_script(current, current.nodes[0], {
|
|
"host-a": KEY1, "host-b": KEY2,
|
|
}),
|
|
)
|
|
for script in scripts:
|
|
result = subprocess.run(
|
|
["/bin/sh", "-n", "-c", script],
|
|
check=False, capture_output=True, text=True,
|
|
)
|
|
assert result.returncode == 0, result.stderr
|
|
assert "br0" not in script
|
|
assert "PRIVATE KEY" not in script
|
|
assert "genkey" not in read_only_identity
|
|
assert 'mktemp "$BASE/.wg-key.' not in read_only_identity
|
|
assert 'mv -f "$_tmp_key" "$KEY"' not in read_only_identity
|
|
assert "mkdir /run/virt-cloud" in read_only_identity
|
|
assert ': > "$LOCK"' in read_only_identity
|
|
assert 'exec 9>>"$LOCK"' in read_only_identity
|
|
assert '[ -d "$BASE" ]' in read_only_identity
|
|
observe = scripts[3]
|
|
assert "__VC_FABRIC_ADMIN_STATE__" in observe
|
|
assert "__VC_FABRIC_SYSCTL__ip_forward" in observe
|
|
assert "__VC_FABRIC_SYSCTL__rp_filter_default" in observe
|
|
assert "__VC_FABRIC_RPF__wg-fabric0" in observe
|
|
assert "__VC_FABRIC_SYSCTL__rp_filter_effective" in observe
|
|
assert "__VC_FABRIC_SYSCTL_PERSISTENCE__" in observe
|
|
assert "__VC_FABRIC_VPC_STATE__" in observe
|
|
assert "__VC_FABRIC_EMERGENCY__" in observe
|
|
assert "list set inet virt_cloud_wg_fabric vpc_tenant_ips" in observe
|
|
assert 'table!="main" && table!="local"' in observe
|
|
apply = scripts[-1]
|
|
assert KEY2 in apply and KEY1 in apply
|
|
assert 'KEY="$BASE/wg-fabric0.key"' in apply
|
|
assert "genkey" not in apply
|
|
identity_check = '[ "$($WG pubkey < "$KEY")" = "$EXPECTED_PUBLIC_KEY" ]'
|
|
assert identity_check in apply
|
|
assert apply.index(identity_check) < apply.index(
|
|
"vc_fabric_nm_quarantine || exit 71")
|
|
assert f"SYSCTL_FILE={FABRIC_SYSCTL}" in apply
|
|
assert _sysctl_text(current).rstrip() in apply
|
|
assert "wg-fabric0" in apply
|
|
assert "address_conflict" in apply
|
|
assert "existing_route" in apply
|
|
assert "route_overlaps_target" in apply
|
|
assert "supernet" not in apply # shell check is numeric and bounded
|
|
assert "virt-cloud-wg-fabric.nft" in apply
|
|
assert "include \"/etc/nftables.d/virt-cloud-wg-fabric.nft\"" in apply
|
|
assert "trap rollback EXIT HUP INT TERM" in apply
|
|
assert '"$WG" showconf' in apply
|
|
assert "route save table all" in apply
|
|
assert 'chmod 700 "$BASE" /usr/local/libexec' not in apply
|
|
assert "virt-cloud-wg-fabric-v1" in apply
|
|
assert "virt_cloud_wg_fabric" in apply
|
|
assert 'iifname "wg-fabric0"' in apply
|
|
assert 'oifname "wg-fabric0"' in apply
|
|
assert 'ip protocol icmp accept' in apply
|
|
assert "route get" in apply
|
|
assert "virt-cloud-wg-fabric-reconcile.timer" in apply
|
|
assert "Restart=on-failure" not in apply
|
|
assert f"fabric-plan-digest={current.digest}" in apply
|
|
assert "NetworkManager/system-connections" in apply
|
|
assert "nmcli" in apply
|
|
assert "flock" in apply
|
|
assert "vpc_state_load" in apply
|
|
assert "vpc_state_allowed_for_peer" in apply
|
|
assert "vpc_state_refresh_nft" in apply
|
|
assert "vpc_state_refresh_nft" in _BOOT_HELPER
|
|
assert "vc_fabric_sysctl_converge" in _BOOT_HELPER
|
|
assert "vc_fabric_rp_filter_set" in _BOOT_HELPER
|
|
assert "rp_filter_before" in apply
|
|
assert "vc_ip_rule_preflight" in _BOOT_HELPER
|
|
assert "vc_ip_rule_scope_check" in _BOOT_HELPER
|
|
assert "vc_ip_rule_ensure" in _BOOT_HELPER
|
|
assert "rule replace" not in _BOOT_HELPER
|
|
assert r"\${" not in _BOOT_HELPER
|
|
assert '"$BASE"/.vpc-dataplane-*.txn' in _BOOT_HELPER
|
|
assert "vpc_seen_rule_priorities" in _BOOT_HELPER
|
|
assert "vpc_seen_route_tables" in _BOOT_HELPER
|
|
assert 'vpc_state != "invalid"' in observe
|
|
assert 'vpc[prefix "|" device "|" table "|" via "|" metric]' in observe
|
|
assert "key|key.desired|wg.conf|address.save|route.save" in _BOOT_HELPER
|
|
assert "nft.table|nft.installed|nft.applied|nft.phase" in _BOOT_HELPER
|
|
assert 'journal_file_mode" = 600' in _BOOT_HELPER
|
|
ifup = _apply_script(current, current.nodes[0], {
|
|
"host-a": KEY1, "host-b": KEY2,
|
|
}, "ifupdown", "/usr/bin/ifup")
|
|
result = subprocess.run(
|
|
["/bin/sh", "-n", "-c", ifup],
|
|
check=False, capture_output=True, text=True,
|
|
)
|
|
assert result.returncode == 0, result.stderr
|
|
assert "/etc/network/interfaces.d/virt-cloud-wg-fabric0" in ifup
|
|
assert "ifup" in ifup and "ifdown" in ifup
|
|
assert "ExecStart=/bin/sh /usr/local/libexec/virt-cloud-wg-fabric --ifupdown-service" in ifup
|
|
# The Debian 13 contract must also work with classic ifupdown; its
|
|
# ifquery implementation does not provide ifupdown2's --check flag.
|
|
assert "ifquery\" --check" not in ifup
|
|
assert "VC_FABRIC_LOCK_HELD=1" in ifup
|
|
assert "VC_FABRIC_ALLOW_TRANSITION=1" in ifup
|
|
assert "br0" not in ifup
|
|
with pytest.raises(FabricError):
|
|
_apply_script(current, current.nodes[0], {
|
|
"host-a": KEY1, "host-b": KEY2,
|
|
}, "ifupdown", "/tmp/ifup")
|
|
|
|
|
|
@pytest.mark.parametrize("expected_digest", (
|
|
"", "a" * 63, "A" * 64, "g" * 64,
|
|
))
|
|
def test_observe_rejects_invalid_expected_digest(expected_digest):
|
|
with pytest.raises(FabricError, match="Fabric 计划摘要无效"):
|
|
_observe_script((), expected_digest=expected_digest)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("kind", "first_line", "mode", "expected"),
|
|
(
|
|
("helper", "#!/bin/sh", "700", "owned"),
|
|
("unit", "[Unit]", "644", "owned"),
|
|
("helper", "[Unit]", "700", "foreign"),
|
|
("unit", "#!/bin/sh", "644", "foreign"),
|
|
("unknown", "#!/bin/sh", "644", "foreign"),
|
|
),
|
|
)
|
|
def test_observe_artifact_state_accepts_only_kind_specific_header(
|
|
tmp_path, kind, first_line, mode, expected):
|
|
current = plan()
|
|
observe = _observe_script((), expected_digest=current.digest)
|
|
start = observe.index("artifact_state() {")
|
|
end = observe.index("\n}\nhelper_marker=", start) + 2
|
|
artifact_state = observe[start:end]
|
|
|
|
artifact = tmp_path / "artifact"
|
|
artifact.write_text(
|
|
f"{first_line}\n"
|
|
"# virt-cloud-wg-fabric-v1\n"
|
|
f"# fabric-plan-digest={current.digest}\n",
|
|
encoding="utf-8",
|
|
)
|
|
artifact.chmod(int(mode, 8))
|
|
|
|
fake_stat = tmp_path / "stat"
|
|
fake_stat.write_text(
|
|
"#!/bin/sh\n"
|
|
"case \"$2\" in\n"
|
|
" '%u') printf '0\\n' ;;\n"
|
|
" '%a') printf '%s\\n' \"$VC_TEST_MODE\" ;;\n"
|
|
" *) exit 2 ;;\n"
|
|
"esac\n",
|
|
encoding="utf-8",
|
|
)
|
|
fake_stat.chmod(0o700)
|
|
|
|
script = "\n".join((
|
|
"set -eu",
|
|
f"STAT={shlex.quote(str(fake_stat))}",
|
|
artifact_state,
|
|
"artifact_state "
|
|
f"{shlex.quote(str(artifact))} {shlex.quote(kind)} "
|
|
f"{shlex.quote(current.digest)}",
|
|
))
|
|
result = subprocess.run(
|
|
["/bin/sh", "-c", script],
|
|
env={"PATH": "/usr/bin:/bin", "VC_TEST_MODE": mode},
|
|
check=False, capture_output=True, text=True,
|
|
)
|
|
|
|
assert result.returncode == 0, result.stderr
|
|
assert result.stdout == expected
|
|
|
|
|
|
def test_reconcile_identity_probe_recreates_only_volatile_runtime_lock():
|
|
script = _identity_script(create_key=False)
|
|
|
|
# /run is empty again after reboot, including after a pre-journal apply
|
|
# failure. Recreate its lock so key validation remains serialized, while
|
|
# never manufacturing or replacing the persistent WireGuard identity.
|
|
assert "mkdir /run/virt-cloud" in script
|
|
assert ': > "$LOCK"' in script
|
|
assert '"$FLOCK" -x 9' in script
|
|
assert "genkey" not in script
|
|
assert 'mktemp "$BASE/.wg-key.' not in script
|
|
assert 'mv -f "$_tmp_key" "$KEY"' not in script
|
|
assert '[ -d "$BASE" ]' in script
|
|
assert '[ -f "$KEY" ]' in script
|
|
|
|
|
|
|
|
|
|
def test_networkmanager_profile_has_no_firewall_zone():
|
|
current = plan()
|
|
profile = _nm_profile_template(
|
|
current, current.nodes[0],
|
|
"12345678-1234-5678-1234-567812345678",
|
|
FIREWALL_MODE_NATIVE,
|
|
)
|
|
assert "autoconnect=false" in profile
|
|
assert "autoconnect-retries=0" in profile
|
|
assert "zone=" not in profile
|
|
assert "interface-name=wg-fabric0" in profile
|
|
assert "203.0.113." not in profile
|
|
assert "zone=trusted" not in profile
|
|
|
|
|
|
def test_endpoint_rp_filter_evidence_uses_linux_max_without_blocking_strict():
|
|
capabilities = {
|
|
"network": {
|
|
"local_ipv4_interfaces": {"203.0.113.1": ["br0"]},
|
|
"rp_filter": 0,
|
|
"rp_filter_default": 1,
|
|
"rp_filter_interfaces": {"br0": 1},
|
|
},
|
|
}
|
|
|
|
evidence, error = _fabric_endpoint_rp_filter_evidence(
|
|
capabilities, "203.0.113.1")
|
|
|
|
assert error is None
|
|
assert evidence == {
|
|
"fabric_endpoint_interface": "br0",
|
|
"fabric_endpoint_rp_filter_raw": 1,
|
|
"fabric_endpoint_rp_filter_effective": 1,
|
|
"fabric_rp_filter_raw": None,
|
|
"fabric_rp_filter_effective": None,
|
|
"rp_filter_default_effective": 1,
|
|
}
|
|
|
|
capabilities["network"]["rp_filter"] = 2
|
|
capabilities["network"]["rp_filter_interfaces"]["br0"] = 1
|
|
capabilities["network"]["rp_filter_interfaces"][FABRIC_INTERFACE] = 1
|
|
evidence, error = _fabric_endpoint_rp_filter_evidence(
|
|
capabilities, "203.0.113.1")
|
|
assert error is None
|
|
assert evidence["fabric_endpoint_rp_filter_effective"] == 2
|
|
assert evidence["fabric_rp_filter_effective"] == 2
|
|
|
|
|
|
def test_endpoint_rp_filter_ignores_bounded_high_density_guest_interfaces():
|
|
rp_filters = {f"vnet{index}": 1 for index in range(300)}
|
|
rp_filters["br0"] = 1
|
|
capabilities = {
|
|
"network": {
|
|
"local_ipv4_interfaces": {"203.0.113.1": ["br0"]},
|
|
"rp_filter": 0,
|
|
"rp_filter_default": 1,
|
|
"rp_filter_interfaces": rp_filters,
|
|
},
|
|
}
|
|
|
|
evidence, error = _fabric_endpoint_rp_filter_evidence(
|
|
capabilities, "203.0.113.1")
|
|
|
|
assert error is None
|
|
assert evidence["fabric_endpoint_interface"] == "br0"
|
|
assert evidence["fabric_endpoint_rp_filter_raw"] == 1
|
|
|
|
|
|
def test_endpoint_rp_filter_ignores_bounded_high_density_local_addresses():
|
|
address_map = {
|
|
f"198.51.{index // 256}.{index % 256}": [f"vnet{index}"]
|
|
for index in range(300)
|
|
}
|
|
address_map["203.0.113.1"] = ["br0"]
|
|
capabilities = {
|
|
"network": {
|
|
"local_ipv4_interfaces": address_map,
|
|
"rp_filter": 0,
|
|
"rp_filter_default": 1,
|
|
"rp_filter_interfaces": {"br0": 1},
|
|
},
|
|
}
|
|
|
|
evidence, error = _fabric_endpoint_rp_filter_evidence(
|
|
capabilities, "203.0.113.1")
|
|
|
|
assert error is None
|
|
assert evidence["fabric_endpoint_interface"] == "br0"
|
|
|
|
|
|
def test_endpoint_rp_filter_rejects_address_inventory_above_probe_limit():
|
|
address_map = {
|
|
f"address-{index}": ["br0"]
|
|
for index in range(16385)
|
|
}
|
|
address_map["203.0.113.1"] = ["br0"]
|
|
|
|
evidence, error = _fabric_endpoint_rp_filter_evidence({
|
|
"network": {
|
|
"local_ipv4_interfaces": address_map,
|
|
"rp_filter": 0,
|
|
"rp_filter_default": 1,
|
|
"rp_filter_interfaces": {"br0": 1},
|
|
},
|
|
}, "203.0.113.1")
|
|
|
|
assert evidence is None
|
|
assert error == "fabric_endpoint_interface_unverified"
|
|
|
|
|
|
def test_endpoint_rp_filter_rejects_probe_inventory_above_4096_interfaces():
|
|
rp_filters = {f"vnet{index}": 1 for index in range(4096)}
|
|
rp_filters["br0"] = 1
|
|
evidence, error = _fabric_endpoint_rp_filter_evidence({
|
|
"network": {
|
|
"local_ipv4_interfaces": {"203.0.113.1": ["br0"]},
|
|
"rp_filter": 0,
|
|
"rp_filter_default": 1,
|
|
"rp_filter_interfaces": rp_filters,
|
|
},
|
|
}, "203.0.113.1")
|
|
|
|
assert evidence is None
|
|
assert error == "fabric_endpoint_rp_filter_unverified"
|
|
|
|
|
|
@pytest.mark.parametrize("interfaces,rp_filters,expected_error", [
|
|
(["br0", "eth0"], {"br0": 1, "eth0": 1},
|
|
"fabric_endpoint_interface_unverified"),
|
|
(["br0"], {}, "fabric_endpoint_rp_filter_unverified"),
|
|
(["vnet3"], {"vnet3": 1},
|
|
"fabric_endpoint_interface_unsafe_or_drifted"),
|
|
(["tap3"], {"tap3": 1},
|
|
"fabric_endpoint_interface_unsafe_or_drifted"),
|
|
(["macvtap3"], {"macvtap3": 1},
|
|
"fabric_endpoint_interface_unsafe_or_drifted"),
|
|
(["fwpr3"], {"fwpr3": 1},
|
|
"fabric_endpoint_interface_unsafe_or_drifted"),
|
|
(["fwln3"], {"fwln3": 1},
|
|
"fabric_endpoint_interface_unsafe_or_drifted"),
|
|
])
|
|
def test_endpoint_rp_filter_evidence_fails_closed_on_ambiguous_or_unsafe_data(
|
|
interfaces, rp_filters, expected_error):
|
|
evidence, error = _fabric_endpoint_rp_filter_evidence({
|
|
"network": {
|
|
"local_ipv4_interfaces": {"203.0.113.1": interfaces},
|
|
"rp_filter": 0,
|
|
"rp_filter_default": 1,
|
|
"rp_filter_interfaces": rp_filters,
|
|
},
|
|
}, "203.0.113.1")
|
|
|
|
assert evidence is None
|
|
assert error == expected_error
|
|
|
|
|
|
def test_fixed_sysctl_marker_requires_owner_or_exact_ready_journal(tmp_path):
|
|
current = plan()
|
|
sysctl_file = tmp_path / "90-virt-cloud-wg-fabric.conf"
|
|
sysctl_file.write_text(_sysctl_text(current), encoding="utf-8")
|
|
sysctl_file.chmod(0o644)
|
|
owner = tmp_path / "wg-fabric0.owner"
|
|
journal = tmp_path / ".wg-fabric-txn"
|
|
stat = tmp_path / "stat"
|
|
stat.write_text(
|
|
"""#!/bin/sh
|
|
case "$2" in
|
|
'%u:%a')
|
|
if [ -d "$3" ]; then
|
|
printf '%s\n' 0:700
|
|
elif [ "${3##*/}" = 90-virt-cloud-wg-fabric.conf ]; then
|
|
printf '%s\n' 0:644
|
|
else
|
|
printf '%s\n' 0:600
|
|
fi ;;
|
|
'%u') printf '%s\n' 0 ;;
|
|
'%a')
|
|
if [ -d "$3" ]; then printf '%s\n' 700
|
|
elif [ "${3##*/}" = 90-virt-cloud-wg-fabric.conf ]; then
|
|
printf '%s\n' 644
|
|
else printf '%s\n' 600; fi ;;
|
|
*) exit 2 ;;
|
|
esac
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
stat.chmod(0o700)
|
|
preflight = f"""set -eu
|
|
STAT='{stat}'
|
|
OWNER='{owner}'
|
|
JOURNAL='{journal}'
|
|
EXPECTED_DIGEST='{current.digest}'
|
|
SYSCTL_FILE='{sysctl_file}'
|
|
{_FABRIC_SYSCTL_VALIDATION_SH}
|
|
vc_fabric_sysctl_file_owned "$SYSCTL_FILE" any || exit 71
|
|
if ! vc_fabric_owner_file_owned; then
|
|
vc_fabric_sysctl_journal_owned || exit 71
|
|
fi
|
|
"""
|
|
|
|
foreign = subprocess.run(
|
|
["/bin/sh", "-c", preflight], check=False,
|
|
capture_output=True, text=True,
|
|
)
|
|
assert foreign.returncode == 71
|
|
assert sysctl_file.read_text(encoding="utf-8") == _sysctl_text(current)
|
|
|
|
owner.write_text(f"{FABRIC_OWNER_MARKER}\n", encoding="utf-8")
|
|
owner.chmod(0o600)
|
|
owned = subprocess.run(
|
|
["/bin/sh", "-c", preflight], check=False,
|
|
capture_output=True, text=True,
|
|
)
|
|
assert owned.returncode == 0, owned.stderr
|
|
|
|
owner.unlink()
|
|
journal.mkdir(mode=0o700)
|
|
(journal / "state").write_text("ready\n", encoding="utf-8")
|
|
(journal / "meta").write_text(
|
|
f"schema=1\nplan_digest={current.digest}\n", encoding="utf-8",
|
|
)
|
|
(journal / "sysctl.absent").touch(mode=0o600)
|
|
(journal / "state").chmod(0o600)
|
|
(journal / "meta").chmod(0o600)
|
|
journaled = subprocess.run(
|
|
["/bin/sh", "-c", preflight], check=False,
|
|
capture_output=True, text=True,
|
|
)
|
|
assert journaled.returncode == 0, journaled.stderr
|
|
|
|
|
|
def test_apply_rollback_restores_owned_sysctl_and_runtime_rp_filter(tmp_path):
|
|
current = plan()
|
|
rendered = _apply_script(
|
|
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
|
|
)
|
|
restore_start = rendered.index("restore_file() {")
|
|
restore_end = rendered.index("\n}\nrollback_sysctl_file()", restore_start) + 2
|
|
sysctl_start = rendered.index("rollback_sysctl_file() {", restore_end)
|
|
sysctl_end = rendered.index("\n}\nrollback_rp_filter()", sysctl_start) + 2
|
|
rpf_start = rendered.index("rollback_rp_filter() {", sysctl_end)
|
|
rpf_end = rendered.index("\n}\nwrite_nft_phase()", rpf_start) + 2
|
|
functions = "\n".join((
|
|
rendered[restore_start:restore_end],
|
|
rendered[sysctl_start:sysctl_end],
|
|
rendered[rpf_start:rpf_end],
|
|
))
|
|
|
|
txn = tmp_path / "txn"
|
|
txn.mkdir()
|
|
old_sysctl = (
|
|
"# virt-cloud-wg-fabric-v1\n"
|
|
f"# fabric-plan-digest={'0' * 64}\n"
|
|
"net.ipv4.conf.wg-fabric0.rp_filter = 2\n"
|
|
)
|
|
(txn / "sysctl").write_text(old_sysctl, encoding="utf-8")
|
|
(txn / "sysctl.present").touch()
|
|
(txn / "sysctl.desired").write_text(
|
|
_sysctl_text(current), encoding="utf-8")
|
|
(txn / "sysctl.desired").chmod(0o644)
|
|
sysctl_file = tmp_path / "fabric.conf"
|
|
sysctl_file.write_text(_sysctl_text(current), encoding="utf-8")
|
|
rpf_file = tmp_path / "rp_filter"
|
|
rpf_file.write_text("2\n", encoding="utf-8")
|
|
stat = tmp_path / "stat"
|
|
stat.write_text(
|
|
"#!/bin/sh\n"
|
|
"[ \"$1:$2\" = '-c:%u:%a' ] || exit 2\n"
|
|
"mode=$(/usr/bin/stat -f '%Lp' \"$3\") || exit 2\n"
|
|
"printf '%s:%s\\n' \"${VC_TEST_UID:-0}\" \"$mode\"\n",
|
|
encoding="utf-8",
|
|
)
|
|
stat.chmod(0o700)
|
|
functions = functions.replace(
|
|
'"/proc/sys/net/ipv4/conf/$IFACE/rp_filter"',
|
|
f"'{rpf_file}'",
|
|
).replace(
|
|
"/etc/sysctl.d/.virt-cloud-wg-fabric-rollback.XXXXXX",
|
|
f"{tmp_path}/.fabric-rollback.XXXXXX",
|
|
)
|
|
harness = f"""set -eu
|
|
TXN='{txn}'
|
|
SYSCTL_FILE='{sysctl_file}'
|
|
EXPECTED_DIGEST='{current.digest}'
|
|
CMP=/usr/bin/cmp
|
|
STAT='{stat}'
|
|
IFACE=wg-fabric0
|
|
IFACE_BEFORE=yes
|
|
RP_FILTER_BEFORE=1
|
|
ROLLBACK_FAILED=no
|
|
ROLLBACK_IFACE_RECREATED=no
|
|
read_sysctl() {{ cat "$1"; }}
|
|
vc_fabric_sysctl_file_owned() {{
|
|
[ "$($STAT -c '%u:%a' "$1")" = 0:644 ] || return 1
|
|
test_digest=$(sed -n '2s/^# fabric-plan-digest=//p' "$1")
|
|
[ "$2" = any ] || [ "$test_digest" = "$2" ]
|
|
}}
|
|
vc_fabric_sysctl_files_equal() {{
|
|
vc_fabric_sysctl_file_owned "$1" "$3" || return 1
|
|
vc_fabric_sysctl_file_owned "$2" "$3" || return 1
|
|
"$CMP" -s "$1" "$2"
|
|
}}
|
|
{functions}
|
|
rollback_sysctl_file
|
|
rollback_rp_filter
|
|
[ "$ROLLBACK_FAILED" = no ]
|
|
"""
|
|
result = subprocess.run(
|
|
["/bin/sh", "-c", harness], check=False,
|
|
capture_output=True, text=True,
|
|
)
|
|
assert result.returncode == 0, result.stderr
|
|
assert sysctl_file.read_text(encoding="utf-8") == old_sysctl
|
|
assert rpf_file.read_text(encoding="utf-8") == "1\n"
|
|
|
|
# Equal bytes are insufficient ownership evidence: metadata drift must
|
|
# stop rollback before either the fixed file or runtime value is touched.
|
|
sysctl_file.chmod(0o600)
|
|
rpf_file.write_text("2\n", encoding="utf-8")
|
|
foreign_metadata = subprocess.run(
|
|
["/bin/sh", "-c", harness], check=False,
|
|
capture_output=True, text=True,
|
|
)
|
|
assert foreign_metadata.returncode != 0
|
|
assert sysctl_file.read_text(encoding="utf-8") == old_sysctl
|
|
assert rpf_file.read_text(encoding="utf-8") == "2\n"
|
|
|
|
sysctl_file.chmod(0o644)
|
|
foreign_uid = subprocess.run(
|
|
["/bin/sh", "-c", harness],
|
|
env={"PATH": "/usr/bin:/bin", "VC_TEST_UID": "1"},
|
|
check=False, capture_output=True, text=True,
|
|
)
|
|
assert foreign_uid.returncode != 0
|
|
assert sysctl_file.read_text(encoding="utf-8") == old_sysctl
|
|
assert rpf_file.read_text(encoding="utf-8") == "2\n"
|
|
|
|
sysctl_file.write_text("foreign\n", encoding="utf-8")
|
|
rpf_file.write_text("2\n", encoding="utf-8")
|
|
foreign = subprocess.run(
|
|
["/bin/sh", "-c", harness], check=False,
|
|
capture_output=True, text=True,
|
|
)
|
|
assert foreign.returncode != 0
|
|
assert sysctl_file.read_text(encoding="utf-8") == "foreign\n"
|
|
assert rpf_file.read_text(encoding="utf-8") == "2\n"
|
|
|
|
# Runtime drift is independent from the fixed file. Only the
|
|
# transaction-installed value 2 may be replaced by the sampled old value.
|
|
sysctl_file.write_text(_sysctl_text(current), encoding="utf-8")
|
|
sysctl_file.chmod(0o644)
|
|
rpf_file.write_text("0\n", encoding="utf-8")
|
|
runtime_drift = subprocess.run(
|
|
["/bin/sh", "-c", harness], check=False,
|
|
capture_output=True, text=True,
|
|
)
|
|
assert runtime_drift.returncode != 0
|
|
assert rpf_file.read_text(encoding="utf-8") == "0\n"
|
|
|
|
|
|
def test_fabric_sysctl_apply_and_rollback_are_transaction_bound():
|
|
current = plan()
|
|
apply = _apply_script(
|
|
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
|
|
)
|
|
|
|
snapshot = apply.index('snapshot_file "$SYSCTL_FILE" sysctl')
|
|
snapshot_readback = apply.index(
|
|
'vc_fabric_sysctl_files_equal "$SYSCTL_FILE" "$TXN/sysctl" any',
|
|
snapshot,
|
|
)
|
|
desired = apply.index(
|
|
'_tmp_sysctl_desired=$(mktemp '
|
|
'"$TXN/.sysctl.desired.XXXXXX")', snapshot_readback)
|
|
desired_temp_owned = apply.index(
|
|
'vc_fabric_sysctl_file_owned '
|
|
'"$_tmp_sysctl_desired" "$EXPECTED_DIGEST"', desired)
|
|
desired_publish = apply.index(
|
|
'mv -f "$_tmp_sysctl_desired" "$TXN/sysctl.desired"',
|
|
desired_temp_owned,
|
|
)
|
|
desired_owned = apply.index(
|
|
'vc_fabric_sysctl_file_owned '
|
|
'"$TXN/sysctl.desired" "$EXPECTED_DIGEST"', desired_publish)
|
|
desired_sync = apply.index("\nsync\n", desired_owned)
|
|
precommit = apply.index(
|
|
'vc_fabric_sysctl_files_equal "$SYSCTL_FILE" "$TXN/sysctl" any',
|
|
desired_owned,
|
|
)
|
|
install = apply.index('mv -f "$_tmp_sysctl" "$SYSCTL_FILE"', precommit)
|
|
installed_readback = apply.index(
|
|
'vc_fabric_sysctl_files_equal "$SYSCTL_FILE" '
|
|
'"$TXN/sysctl.desired" "$EXPECTED_DIGEST"', install)
|
|
assert snapshot < snapshot_readback < desired < desired_temp_owned
|
|
assert desired_temp_owned < desired_publish < desired_owned < desired_sync
|
|
assert desired_sync < precommit < install < installed_readback
|
|
|
|
rollback = apply.index("rollback_sysctl_file() {")
|
|
desired_proof = apply.index(
|
|
'vc_fabric_sysctl_files_equal "$SYSCTL_FILE" \\\n'
|
|
' "$TXN/sysctl.desired" "$EXPECTED_DIGEST"', rollback)
|
|
pre_restore_recheck = apply.index(
|
|
'vc_fabric_sysctl_files_equal "$SYSCTL_FILE" \\\n'
|
|
' "$TXN/sysctl.desired" "$EXPECTED_DIGEST"',
|
|
desired_proof,
|
|
)
|
|
restore = apply.index(
|
|
'mv -f "$rollback_sysctl_tmp" "$SYSCTL_FILE"',
|
|
pre_restore_recheck,
|
|
)
|
|
assert rollback < desired_proof < pre_restore_recheck < restore
|
|
|
|
|
|
def _root_stat_stub(tmp_path: Path) -> Path:
|
|
stat = tmp_path / "stat"
|
|
stat.write_text(
|
|
"#!/bin/sh\n"
|
|
"case \"$1:$2\" in\n"
|
|
" '-c:%u') printf '%s\\n' 0 ;;\n"
|
|
" '-c:%a') /usr/bin/stat -f '%Lp' \"$3\" ;;\n"
|
|
" '-c:%u:%a')\n"
|
|
" mode=$(/usr/bin/stat -f '%Lp' \"$3\") || exit 2\n"
|
|
" printf '0:%s\\n' \"$mode\" ;;\n"
|
|
" *) exit 2 ;;\n"
|
|
"esac\n",
|
|
encoding="utf-8",
|
|
)
|
|
stat.chmod(0o700)
|
|
return stat
|
|
|
|
|
|
def test_generic_apply_rollback_restores_only_its_exact_desired_file(tmp_path):
|
|
current = plan()
|
|
apply = _apply_script(
|
|
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
|
|
)
|
|
functions_start = apply.index("txn_file_secure() {")
|
|
functions_end = apply.index(
|
|
"\nrollback_sysctl_file() {", functions_start)
|
|
functions = apply[functions_start:functions_end]
|
|
txn = tmp_path / "txn"
|
|
txn.mkdir()
|
|
target = tmp_path / "managed.conf"
|
|
stat = _root_stat_stub(tmp_path)
|
|
harness = "\n".join((
|
|
"set -eu",
|
|
f"TXN='{txn}'",
|
|
f"STAT='{stat}'",
|
|
"CMP=/usr/bin/cmp",
|
|
"ROLLBACK_FAILED=no",
|
|
"_tmp_restore=",
|
|
functions,
|
|
f"restore_file '{target}' conf",
|
|
"printf '%s\\n' \"$ROLLBACK_FAILED\"",
|
|
))
|
|
|
|
def prepare(*, baseline: str | None, desired: str | None,
|
|
live: str | None, live_mode: int = 0o600) -> None:
|
|
for path in tuple(txn.iterdir()):
|
|
path.unlink()
|
|
if target.exists() or target.is_symlink():
|
|
target.unlink()
|
|
if baseline is None:
|
|
(txn / "conf.absent").touch(mode=0o600)
|
|
else:
|
|
(txn / "conf").write_text(baseline, encoding="utf-8")
|
|
(txn / "conf").chmod(0o600)
|
|
(txn / "conf.present").touch(mode=0o600)
|
|
if desired is None:
|
|
(txn / "conf.desired.absent").touch(mode=0o600)
|
|
else:
|
|
(txn / "conf.desired").write_text(desired, encoding="utf-8")
|
|
(txn / "conf.desired").chmod(0o600)
|
|
if live is not None:
|
|
target.write_text(live, encoding="utf-8")
|
|
target.chmod(live_mode)
|
|
|
|
def run():
|
|
return subprocess.run(
|
|
["/bin/sh", "-c", harness], check=False,
|
|
capture_output=True, text=True,
|
|
)
|
|
|
|
prepare(baseline="old\n", desired="new\n", live="new\n")
|
|
restored = run()
|
|
assert restored.returncode == 0, restored.stderr
|
|
assert restored.stdout == "no\n"
|
|
assert target.read_text(encoding="utf-8") == "old\n"
|
|
|
|
# A crash before this path's desired snapshot was published performed no
|
|
# mutation. Exact baseline bytes remain a valid no-op recovery state.
|
|
prepare(baseline="old\n", desired="new\n", live="old\n")
|
|
(txn / "conf.desired").unlink()
|
|
untouched = run()
|
|
assert untouched.returncode == 0, untouched.stderr
|
|
assert untouched.stdout == "no\n"
|
|
assert target.read_text(encoding="utf-8") == "old\n"
|
|
|
|
prepare(baseline="old\n", desired="new\n", live="operator\n")
|
|
drifted = run()
|
|
assert drifted.returncode == 0, drifted.stderr
|
|
assert drifted.stdout == "yes\n"
|
|
assert target.read_text(encoding="utf-8") == "operator\n"
|
|
|
|
prepare(baseline=None, desired="created\n", live="created\n")
|
|
removed = run()
|
|
assert removed.returncode == 0, removed.stderr
|
|
assert removed.stdout == "no\n"
|
|
assert target.exists() is False
|
|
|
|
prepare(baseline="old\n", desired=None, live=None)
|
|
recreated = run()
|
|
assert recreated.returncode == 0, recreated.stderr
|
|
assert recreated.stdout == "no\n"
|
|
assert target.read_text(encoding="utf-8") == "old\n"
|
|
|
|
prepare(
|
|
baseline="old\n", desired="new\n", live="new\n",
|
|
live_mode=0o644,
|
|
)
|
|
metadata_drift = run()
|
|
assert metadata_drift.returncode == 0, metadata_drift.stderr
|
|
assert metadata_drift.stdout == "yes\n"
|
|
assert target.read_text(encoding="utf-8") == "new\n"
|
|
assert target.stat().st_mode & 0o777 == 0o644
|
|
|
|
|
|
def test_boot_recovery_restores_only_exact_transaction_post_image(tmp_path):
|
|
secure_start = _BOOT_HELPER.index("journal_file_secure() {")
|
|
secure_end = _BOOT_HELPER.index(
|
|
"journal_cleanup_helper_owned() {", secure_start)
|
|
restore_start = _BOOT_HELPER.index("restore_journal_file() {")
|
|
restore_end = _BOOT_HELPER.index(
|
|
"restore_journal_sysctl() {", restore_start)
|
|
functions = "\n".join((
|
|
_BOOT_HELPER[secure_start:secure_end],
|
|
_BOOT_HELPER[restore_start:restore_end],
|
|
))
|
|
journal = tmp_path / "journal"
|
|
journal.mkdir()
|
|
target = tmp_path / "managed.conf"
|
|
stat = _root_stat_stub(tmp_path)
|
|
harness = "\n".join((
|
|
"set -eu",
|
|
f"JOURNAL='{journal}'",
|
|
f"STAT='{stat}'",
|
|
"CMP=/usr/bin/cmp",
|
|
"_tmp_journal_restore=",
|
|
"stat() { \"$STAT\" \"$@\"; }",
|
|
functions,
|
|
f"if restore_journal_file '{target}' conf; then",
|
|
" printf '%s\\n' restored",
|
|
"else",
|
|
" printf '%s\\n' blocked",
|
|
"fi",
|
|
))
|
|
|
|
def prepare(*, baseline: str | None, desired: str | None,
|
|
live: str | None, live_mode: int = 0o600) -> None:
|
|
for path in tuple(journal.iterdir()):
|
|
path.unlink()
|
|
if target.exists() or target.is_symlink():
|
|
target.unlink()
|
|
if baseline is None:
|
|
(journal / "conf.absent").touch(mode=0o600)
|
|
else:
|
|
(journal / "conf").write_text(baseline, encoding="utf-8")
|
|
(journal / "conf").chmod(0o600)
|
|
(journal / "conf.present").touch(mode=0o600)
|
|
if desired is None:
|
|
(journal / "conf.desired.absent").touch(mode=0o600)
|
|
else:
|
|
(journal / "conf.desired").write_text(
|
|
desired, encoding="utf-8")
|
|
(journal / "conf.desired").chmod(0o600)
|
|
if live is not None:
|
|
target.write_text(live, encoding="utf-8")
|
|
target.chmod(live_mode)
|
|
|
|
def run():
|
|
return subprocess.run(
|
|
["/bin/sh", "-c", harness], check=False,
|
|
capture_output=True, text=True,
|
|
)
|
|
|
|
prepare(baseline="old\n", desired="new\n", live="new\n")
|
|
restored = run()
|
|
assert restored.returncode == 0, restored.stderr
|
|
assert restored.stdout == "restored\n"
|
|
assert target.read_text(encoding="utf-8") == "old\n"
|
|
|
|
prepare(baseline="old\n", desired="new\n", live="old\n")
|
|
(journal / "conf.desired").unlink()
|
|
untouched = run()
|
|
assert untouched.returncode == 0, untouched.stderr
|
|
assert untouched.stdout == "restored\n"
|
|
assert target.read_text(encoding="utf-8") == "old\n"
|
|
|
|
prepare(baseline="old\n", desired="new\n", live="operator\n")
|
|
drifted = run()
|
|
assert drifted.returncode == 0, drifted.stderr
|
|
assert drifted.stdout == "blocked\n"
|
|
assert target.read_text(encoding="utf-8") == "operator\n"
|
|
|
|
prepare(baseline=None, desired="created\n", live="created\n")
|
|
removed = run()
|
|
assert removed.returncode == 0, removed.stderr
|
|
assert removed.stdout == "restored\n"
|
|
assert target.exists() is False
|
|
|
|
prepare(
|
|
baseline="old\n", desired="new\n", live="new\n",
|
|
live_mode=0o644,
|
|
)
|
|
metadata_drift = run()
|
|
assert metadata_drift.returncode == 0, metadata_drift.stderr
|
|
assert metadata_drift.stdout == "blocked\n"
|
|
assert target.read_text(encoding="utf-8") == "new\n"
|
|
assert target.stat().st_mode & 0o777 == 0o644
|
|
|
|
|
|
def test_fixed_artifacts_publish_desired_snapshot_before_atomic_install():
|
|
current = plan()
|
|
apply = _apply_script(
|
|
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
|
|
firewall_mode=FIREWALL_MODE_NATIVE,
|
|
)
|
|
assert "record_desired_file() {" in apply
|
|
assert "record_desired_absent() {" in apply
|
|
assert (
|
|
'txn_baseline_matches "$publish_target" "$publish_name"'
|
|
in apply
|
|
)
|
|
for source, target, name in (
|
|
("$_tmp_owner", "$OWNER", "owner"),
|
|
("$_tmp_nft", "$NFT_RULES", "nft_rules"),
|
|
("$_tmp_nft_main", "$NFT_MAIN", "nft_main"),
|
|
("$_tmp_helper", "$HELPER", "helper"),
|
|
("$_tmp_nm", "$NM_PROFILE", "nm_profile"),
|
|
("$_tmp_conf", "$CONF", "conf"),
|
|
("$_tmp_unit", "$UNIT", "unit"),
|
|
("$_tmp_repair", "$REPAIR_UNIT", "repair"),
|
|
("$_tmp_timer", "$TIMER_UNIT", "timer"),
|
|
("$_tmp_gate", "$GATE_GENERATOR", "gate_generator"),
|
|
):
|
|
assert (
|
|
f'publish_desired_file "{source}" "{target}" {name}' in apply
|
|
)
|
|
assert 'publish_desired_file "$_tmp_key" "$KEY" key' not in apply
|
|
assert 'journal_file_matches "$restore_target"' in _BOOT_HELPER
|
|
assert '"$restore_name.desired"' in _BOOT_HELPER
|
|
|
|
|
|
def test_rollback_keeps_recovery_helper_until_every_other_restore_succeeds():
|
|
current = plan()
|
|
apply = _apply_script(
|
|
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
|
|
firewall_mode=FIREWALL_MODE_NATIVE,
|
|
)
|
|
rollback = apply.index("rollback() {")
|
|
first_fixed_restore = apply.index(
|
|
'restore_file "$UNIT" unit', rollback)
|
|
verify_runtime = apply.index(
|
|
"\n verify_rollback_rp_filter\n", first_fixed_restore)
|
|
helper_comment = apply.index(
|
|
"# Keep the transaction's complete recovery helper installed",
|
|
verify_runtime,
|
|
)
|
|
cleanup_state = apply.index(
|
|
"rollback_publish_cleanup_helper_state || ROLLBACK_FAILED=yes",
|
|
helper_comment,
|
|
)
|
|
helper_restore = apply.index(
|
|
'restore_file "$HELPER" helper', cleanup_state)
|
|
final_decision = apply.index(
|
|
'if [ "$ROLLBACK_FAILED" = yes ]; then', helper_restore)
|
|
|
|
assert first_fixed_restore < verify_runtime < helper_comment < cleanup_state
|
|
assert cleanup_state < helper_restore < final_decision
|
|
assert apply.count('restore_file "$HELPER" helper') == 1
|
|
|
|
|
|
def test_rollback_stops_timer_only_with_exact_transaction_unit(tmp_path):
|
|
current = plan()
|
|
apply = _apply_script(
|
|
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
|
|
)
|
|
functions_start = apply.index("txn_file_secure() {")
|
|
functions_end = apply.index(
|
|
"\nrollback_sysctl_file() {", functions_start)
|
|
functions = apply[functions_start:functions_end]
|
|
txn = tmp_path / "txn"
|
|
txn.mkdir()
|
|
timer = tmp_path / "timer.unit"
|
|
desired = "[Unit]\nDescription=transaction\n"
|
|
(txn / "timer.desired").write_text(desired, encoding="utf-8")
|
|
(txn / "timer.desired").chmod(0o644)
|
|
timer.write_text(desired, encoding="utf-8")
|
|
timer.chmod(0o644)
|
|
active = tmp_path / "active"
|
|
active.write_text("active\n", encoding="utf-8")
|
|
log = tmp_path / "systemctl.log"
|
|
systemctl = tmp_path / "systemctl"
|
|
systemctl.write_text(
|
|
"#!/bin/sh\n"
|
|
"case \"$1:$2:$3\" in\n"
|
|
" 'show:-p:LoadState') printf '%s\\n' loaded ;;\n"
|
|
" 'show:-p:ActiveState') cat \"$VC_TIMER_ACTIVE\" ;;\n"
|
|
" 'stop:virt-cloud-wg-fabric-reconcile.timer:')\n"
|
|
" printf '%s\\n' stop >> \"$VC_TIMER_LOG\"\n"
|
|
" printf '%s\\n' inactive > \"$VC_TIMER_ACTIVE\" ;;\n"
|
|
" *) exit 2 ;;\n"
|
|
"esac\n",
|
|
encoding="utf-8",
|
|
)
|
|
systemctl.chmod(0o700)
|
|
stat = _root_stat_stub(tmp_path)
|
|
harness = "\n".join((
|
|
"set -eu",
|
|
f"TXN='{txn}'",
|
|
f"TIMER_UNIT='{timer}'",
|
|
f"SYSTEMCTL='{systemctl}'",
|
|
f"STAT='{stat}'",
|
|
"CMP=/usr/bin/cmp",
|
|
functions,
|
|
"rollback_stop_transaction_timer_if_owned",
|
|
))
|
|
env = {
|
|
"PATH": "/usr/bin:/bin",
|
|
"VC_TIMER_ACTIVE": str(active),
|
|
"VC_TIMER_LOG": str(log),
|
|
}
|
|
owned = subprocess.run(
|
|
["/bin/sh", "-c", harness], env=env, check=False,
|
|
capture_output=True, text=True,
|
|
)
|
|
assert owned.returncode == 0, owned.stderr
|
|
assert log.read_text(encoding="utf-8") == "stop\n"
|
|
|
|
timer.write_text("[Unit]\nDescription=operator\n", encoding="utf-8")
|
|
active.write_text("active\n", encoding="utf-8")
|
|
log.unlink()
|
|
foreign = subprocess.run(
|
|
["/bin/sh", "-c", harness], env=env, check=False,
|
|
capture_output=True, text=True,
|
|
)
|
|
assert foreign.returncode == 1
|
|
assert log.exists() is False
|
|
assert active.read_text(encoding="utf-8") == "active\n"
|
|
|
|
|
|
def test_recreated_interface_may_restore_rp_filter_from_host_default(tmp_path):
|
|
phase_start = _BOOT_HELPER.index("journal_iface_recovery_state() {")
|
|
phase_end = _BOOT_HELPER.index("restore_journal_file() {", phase_start)
|
|
phase_functions = _BOOT_HELPER[phase_start:phase_end]
|
|
journal = tmp_path / "journal"
|
|
journal.mkdir()
|
|
phase_harness = "\n".join((
|
|
"set -eu",
|
|
f"JOURNAL='{journal}'",
|
|
"journal_file_secure() {",
|
|
" [ -f \"$JOURNAL/$1\" ] && [ ! -L \"$JOURNAL/$1\" ] &&",
|
|
" [ \"$(/usr/bin/stat -f '%Lp' \"$JOURNAL/$1\")\" = 600 ]",
|
|
"}",
|
|
phase_functions,
|
|
"journal_publish_iface_recovery_state recreating",
|
|
'[ "$(journal_iface_recovery_state)" = recreating ]',
|
|
"journal_publish_iface_recovery_state recreated",
|
|
"journal_iface_recovery_state",
|
|
))
|
|
phase = subprocess.run(
|
|
["/bin/sh", "-c", phase_harness], check=False,
|
|
capture_output=True, text=True,
|
|
)
|
|
assert phase.returncode == 0, phase.stderr
|
|
assert phase.stdout == "recreated"
|
|
assert (journal / "iface.recovery").read_text(encoding="utf-8") == (
|
|
"recreated\n")
|
|
|
|
recovery_start = _BOOT_HELPER.index("journal_restore_rp_filter() {")
|
|
recovery_end = _BOOT_HELPER.index(
|
|
"journal_verify_rp_filter() {", recovery_start)
|
|
recovery = _BOOT_HELPER[recovery_start:recovery_end].replace(
|
|
'"/proc/sys/net/ipv4/conf/$IFACE/rp_filter"',
|
|
'"$VC_TEST_RPF"',
|
|
)
|
|
rpf = tmp_path / "rp_filter"
|
|
recovery_harness = "\n".join((
|
|
"set -eu",
|
|
f"VC_TEST_RPF='{rpf}'",
|
|
"journal_iface_before=yes",
|
|
"journal_rp_filter_before=2",
|
|
"read_sysctl() { cat \"$1\"; }",
|
|
recovery,
|
|
"journal_restore_rp_filter",
|
|
))
|
|
|
|
rpf.write_text("1\n", encoding="utf-8")
|
|
recreated = subprocess.run(
|
|
["/bin/sh", "-c", "journal_iface_recreated=yes\n" + recovery_harness],
|
|
check=False, capture_output=True, text=True,
|
|
)
|
|
assert recreated.returncode == 0, recreated.stderr
|
|
assert rpf.read_text(encoding="utf-8") == "2\n"
|
|
|
|
rpf.write_text("1\n", encoding="utf-8")
|
|
persistent = subprocess.run(
|
|
["/bin/sh", "-c", "journal_iface_recreated=no\n" + recovery_harness],
|
|
check=False, capture_output=True, text=True,
|
|
)
|
|
assert persistent.returncode != 0
|
|
assert rpf.read_text(encoding="utf-8") == "1\n"
|
|
|
|
current = plan()
|
|
apply = _apply_script(
|
|
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
|
|
)
|
|
rollback_start = apply.index("rollback_rp_filter() {")
|
|
rollback_end = apply.index("verify_rollback_rp_filter() {", rollback_start)
|
|
rollback = apply[rollback_start:rollback_end].replace(
|
|
'"/proc/sys/net/ipv4/conf/$IFACE/rp_filter"',
|
|
'"$VC_TEST_RPF"',
|
|
)
|
|
rollback_harness = "\n".join((
|
|
"set -eu",
|
|
f"VC_TEST_RPF='{rpf}'",
|
|
"IFACE_BEFORE=yes",
|
|
"RP_FILTER_BEFORE=2",
|
|
"ROLLBACK_FAILED=no",
|
|
"read_sysctl() { cat \"$1\"; }",
|
|
rollback,
|
|
"rollback_rp_filter",
|
|
'[ "$ROLLBACK_FAILED" = no ]',
|
|
))
|
|
|
|
rpf.write_text("1\n", encoding="utf-8")
|
|
rollback_recreated = subprocess.run(
|
|
["/bin/sh", "-c", "ROLLBACK_IFACE_RECREATED=yes\n" + rollback_harness],
|
|
check=False, capture_output=True, text=True,
|
|
)
|
|
assert rollback_recreated.returncode == 0, rollback_recreated.stderr
|
|
assert rpf.read_text(encoding="utf-8") == "2\n"
|
|
|
|
rpf.write_text("1\n", encoding="utf-8")
|
|
rollback_persistent = subprocess.run(
|
|
["/bin/sh", "-c", "ROLLBACK_IFACE_RECREATED=no\n" + rollback_harness],
|
|
check=False, capture_output=True, text=True,
|
|
)
|
|
assert rollback_persistent.returncode != 0
|
|
assert rpf.read_text(encoding="utf-8") == "1\n"
|
|
|
|
|
|
@pytest.mark.parametrize("readback_rc", (0, 71))
|
|
def test_ifupdown_service_requires_readback_after_successful_ifup(
|
|
tmp_path, readback_rc):
|
|
start = _BOOT_HELPER.index(
|
|
'if [ "$DEPLOYMENT_MODE" = ifupdown ] &&\n'
|
|
' [ "${1:-}" = --ifupdown-service ]; then')
|
|
end = _BOOT_HELPER.index(
|
|
'if [ "$DEPLOYMENT_MODE" = ifupdown ] &&\n'
|
|
' [ "${1:-}" = --ifupdown-hook ]; then', start)
|
|
calls = tmp_path / "ifup.calls"
|
|
ifup = tmp_path / "ifup"
|
|
ifup.write_text(
|
|
"#!/bin/sh\n"
|
|
f"printf '%s\\n' \"$*\" >> {shlex.quote(str(calls))}\n"
|
|
"exit 0\n", encoding="utf-8")
|
|
ifup.chmod(0o700)
|
|
harness = "\n".join((
|
|
"set -eu", "DEPLOYMENT_MODE=ifupdown", "IFACE=wg-fabric0",
|
|
f"IFUP={shlex.quote(str(ifup))}", "set -- --ifupdown-service",
|
|
_BOOT_HELPER[start:end],
|
|
'[ "$1" = --ifupdown-configure ] || exit 99',
|
|
f"exit {readback_rc}",
|
|
))
|
|
result = subprocess.run(
|
|
["/bin/sh", "-c", harness], check=False,
|
|
capture_output=True, text=True)
|
|
|
|
assert result.returncode == readback_rc, result.stderr
|
|
assert calls.read_text(encoding="utf-8").splitlines() == [
|
|
"--force wg-fabric0"]
|
|
|
|
|
|
def test_ifupdown_transition_hook_executes_without_reconciling_stale_config(
|
|
tmp_path):
|
|
start = _BOOT_HELPER.index(
|
|
'if [ "$DEPLOYMENT_MODE" = ifupdown ] &&\n'
|
|
' [ "${1:-}" = --ifupdown-configure ]; then')
|
|
end = _BOOT_HELPER.index("vc_firewall_mode_ready || exit 71", start)
|
|
block = _BOOT_HELPER[start:end].replace(
|
|
'"/proc/sys/net/ipv4/conf/$IFACE/rp_filter"',
|
|
'"$VC_TEST_RPF"',
|
|
)
|
|
rpf = tmp_path / "rp_filter"
|
|
rpf.write_text("2\n", encoding="utf-8")
|
|
harness = "\n".join((
|
|
"set -eu",
|
|
"DEPLOYMENT_MODE=ifupdown",
|
|
"IFACE=wg-fabric0",
|
|
"VC_FABRIC_APPLYING=1",
|
|
"VC_FABRIC_ALLOW_TRANSITION=1",
|
|
"validate_ifupdown_fragment() { :; }",
|
|
"vc_fabric_rp_filter_set() { printf '%s\\n' rpf >> \"$VC_TEST_LOG\"; }",
|
|
"read_sysctl() { cat \"$1\"; }",
|
|
"mock_ip() { printf '%s\\n' \"$*\" >> \"$VC_TEST_LOG\"; }",
|
|
"IP=mock_ip",
|
|
"set -- --ifupdown-configure",
|
|
block,
|
|
"exit 99",
|
|
))
|
|
log = tmp_path / "transition.log"
|
|
env = {
|
|
"PATH": "/usr/bin:/bin",
|
|
"VC_TEST_LOG": str(log),
|
|
"VC_TEST_RPF": str(rpf),
|
|
}
|
|
|
|
applied = subprocess.run(
|
|
["/bin/sh", "-c", harness], env=env, check=False,
|
|
capture_output=True, text=True,
|
|
)
|
|
assert applied.returncode == 0, applied.stderr
|
|
assert log.read_text(encoding="utf-8").splitlines() == [
|
|
"rpf", "link set dev wg-fabric0 up", "rpf",
|
|
]
|
|
|
|
log.write_text("", encoding="utf-8")
|
|
rpf.write_text("1\n", encoding="utf-8")
|
|
recovery_env = dict(env)
|
|
recovery_env["VC_FABRIC_RESTORE_RP_FILTER"] = "1"
|
|
recovered = subprocess.run(
|
|
["/bin/sh", "-c", harness], env=recovery_env, check=False,
|
|
capture_output=True, text=True,
|
|
)
|
|
assert recovered.returncode == 0, recovered.stderr
|
|
assert log.read_text(encoding="utf-8").splitlines() == [
|
|
"link set dev wg-fabric0 up",
|
|
]
|
|
|
|
|
|
def test_fabric_and_vpc_rp_filter_are_set_before_manager_activation():
|
|
current = plan()
|
|
apply = _apply_script(
|
|
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
|
|
)
|
|
# The rendered temporary recovery helper contains the same ownership
|
|
# checks. Anchor on the later, live apply block rather than its here-doc.
|
|
apply_owner = apply.rindex("vc_fabric_owner_file_owned || exit 71")
|
|
apply_persistence = apply.index(
|
|
'mv -f "$_tmp_sysctl" "$SYSCTL_FILE"',
|
|
apply_owner,
|
|
)
|
|
apply_create = apply.index(
|
|
'"$IP" link add dev "$IFACE" type wireguard', apply_persistence)
|
|
apply_runtime = apply.index(
|
|
"vc_fabric_rp_filter_set || exit 71", apply_create)
|
|
apply_nm_up = apply.index(
|
|
'"$NMCLI" connection up uuid', apply_runtime)
|
|
assert apply_owner < apply_persistence < apply_create < apply_runtime < apply_nm_up
|
|
assert 'vc_fabric_sysctl_file_owned "$SYSCTL_FILE" any' in _BOOT_HELPER
|
|
assert '> /proc/sys/net/ipv4/conf/all/rp_filter' not in apply
|
|
assert '> /proc/sys/net/ipv4/conf/default/rp_filter' not in apply
|
|
|
|
boot_create = _BOOT_HELPER.index(
|
|
'"$IP" link add dev "$IFACE" type wireguard')
|
|
boot_runtime = _BOOT_HELPER.index(
|
|
"vc_fabric_rp_filter_set || exit 71", boot_create)
|
|
boot_nm_up = _BOOT_HELPER.index(
|
|
'"$NMCLI" connection up uuid "$CONNECTION_UUID"', boot_runtime)
|
|
assert boot_create < boot_runtime < boot_nm_up
|
|
|
|
vpc_create = _BOOT_HELPER.index(
|
|
'"$IP" link add name "$vpc_bridge" type bridge')
|
|
vpc_revision = _BOOT_HELPER.index(
|
|
'vpc_revision=$(awk -F\'|\' \'$1=="revision" '
|
|
"{print $2; exit}' \"$vpc_state_file\")")
|
|
vpc_digest = _BOOT_HELPER.index(
|
|
'vpc_digest=$(awk -F\'|\' \'$1=="digest" '
|
|
"{print $2; exit}' \"$vpc_state_file\")",
|
|
vpc_revision,
|
|
)
|
|
vpc_sysctl_owned = _BOOT_HELPER.index(
|
|
'vc_vpc_sysctl_artifact_owned "$vpc_sysctl_file" "$vpc_id"',
|
|
vpc_digest,
|
|
)
|
|
vpc_runtime = _BOOT_HELPER.index(
|
|
'printf \'%s\\n\' 2 > '
|
|
'"/proc/sys/net/ipv4/conf/$vpc_bridge/rp_filter"',
|
|
vpc_create,
|
|
)
|
|
vpc_nm_up = _BOOT_HELPER.index(
|
|
'"$NMCLI" connection up uuid "$vpc_nm_uuid"', vpc_runtime)
|
|
vpc_ifup = _BOOT_HELPER.index(
|
|
'VC_FABRIC_APPLYING=1 "$IFUP" --force "$vpc_bridge"',
|
|
vpc_runtime,
|
|
)
|
|
assert vpc_revision < vpc_digest < vpc_sysctl_owned < vpc_create
|
|
assert vpc_create < vpc_runtime < vpc_nm_up < vpc_ifup
|
|
assert '"$vpc_revision" "$vpc_digest" "$vpc_bridge" || exit 71' \
|
|
in _BOOT_HELPER[vpc_sysctl_owned:vpc_create]
|
|
assert '__VC_VPC_SYSCTL_VALIDATION__' not in _BOOT_HELPER
|
|
assert (
|
|
'grep -Fqx "net.ipv4.conf.$vpc_bridge.rp_filter = 2" '
|
|
'"$vpc_sysctl_file"'
|
|
) not in _BOOT_HELPER
|
|
|
|
|
|
def test_recovery_and_rollback_restore_rp_filter_before_reactivation():
|
|
recover_start = _BOOT_HELPER.index("recover_journal() {")
|
|
recover_end = _BOOT_HELPER.index(
|
|
'if [ "$RECOVER_ONLY" = yes ]; then', recover_start)
|
|
recovery = _BOOT_HELPER[recover_start:recover_end]
|
|
recovery_phase_load = recovery.index(
|
|
"journal_iface_recovery_phase=$(journal_iface_recovery_state)")
|
|
recovery_phase_adopt = recovery.index(
|
|
'if [ "$journal_iface_recovery_phase" != none ]; then\n'
|
|
' journal_iface_recreated=yes',
|
|
recovery_phase_load,
|
|
)
|
|
recovery_recreating = recovery.index(
|
|
"journal_publish_iface_recovery_state recreating")
|
|
recovery_recreate = recovery.index(
|
|
'"$IP" link add dev "$IFACE" type wireguard', recovery_recreating)
|
|
recovery_recreated = recovery.index(
|
|
"journal_publish_iface_recovery_state recreated", recovery_recreate)
|
|
recovery_restore = recovery.index("journal_restore_rp_filter || exit 79")
|
|
recovery_nm_up = recovery.index(
|
|
'"$journal_nmcli" connection up uuid "$journal_old_uuid"',
|
|
recovery_restore,
|
|
)
|
|
recovery_ifup = recovery.index(
|
|
'"$IFUP_PATH" --force "$IFACE"', recovery_restore)
|
|
recovery_verify = recovery.index(
|
|
"if ! journal_verify_rp_filter;", recovery_restore)
|
|
assert recovery_phase_load < recovery_phase_adopt < recovery_recreating
|
|
assert recovery_recreating < recovery_recreate < recovery_recreated
|
|
assert recovery_recreated < recovery_restore < recovery_nm_up < recovery_verify
|
|
assert recovery_restore < recovery_ifup < recovery_verify
|
|
assert (
|
|
'VC_FABRIC_APPLYING=1 \\\n'
|
|
' VC_FABRIC_RESTORE_RP_FILTER="$journal_rp_filter_before"'
|
|
in recovery
|
|
)
|
|
|
|
current = plan()
|
|
apply = _apply_script(
|
|
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
|
|
)
|
|
rollback_start = apply.index("rollback() {")
|
|
rollback_recreating = apply.index(
|
|
"write_iface_recovery_phase recreating", rollback_start)
|
|
rollback_recreate = apply.index(
|
|
'"$IP" link add dev "$IFACE" type wireguard', rollback_recreating)
|
|
rollback_recreated = apply.index(
|
|
"write_iface_recovery_phase recreated", rollback_recreate)
|
|
rollback_restore = apply.index(
|
|
"\n rollback_rp_filter\n", rollback_start)
|
|
rollback_link_up = apply.index(
|
|
'"$IP" link set dev "$IFACE" up', rollback_restore)
|
|
rollback_nm_up = apply.index(
|
|
'"$NMCLI" connection up uuid', rollback_restore)
|
|
rollback_verify = apply.index(
|
|
"\n verify_rollback_rp_filter\n", rollback_restore)
|
|
assert rollback_recreating < rollback_recreate < rollback_recreated
|
|
assert rollback_recreated < rollback_restore < rollback_link_up < rollback_verify
|
|
assert rollback_restore < rollback_nm_up < rollback_verify
|
|
|
|
|
|
def test_first_deploy_recovery_and_rollback_delete_link_before_owner():
|
|
recover_start = _BOOT_HELPER.index("recover_journal() {")
|
|
first_deploy = _BOOT_HELPER.index(
|
|
'if [ "$journal_iface_before" = no ] &&', recover_start)
|
|
delete_link = _BOOT_HELPER.index(
|
|
'"$IP" link del dev "$IFACE"', first_deploy)
|
|
confirm_absent = _BOOT_HELPER.index(
|
|
'"$IP" link show dev "$IFACE"', delete_link)
|
|
mark_absent = _BOOT_HELPER.index(
|
|
"journal_current_iface_state=absent", confirm_absent)
|
|
restore_owner = _BOOT_HELPER.index(
|
|
'restore_journal_file "$OWNER" owner', mark_absent)
|
|
assert first_deploy < delete_link < confirm_absent < mark_absent < restore_owner
|
|
|
|
current = plan()
|
|
apply = _apply_script(
|
|
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
|
|
)
|
|
rollback_start = apply.index("rollback() {")
|
|
rollback_first = apply.index(
|
|
'if [ "$IFACE_BEFORE" = no ]; then', rollback_start)
|
|
rollback_delete = apply.index(
|
|
'"$IP" link del dev "$IFACE"', rollback_first)
|
|
rollback_confirm = apply.index(
|
|
'"$IP" link show dev "$IFACE"', rollback_delete)
|
|
rollback_owner = apply.index(
|
|
'restore_file "$OWNER" owner', rollback_confirm)
|
|
guard = apply.rfind(
|
|
'if [ "$IFACE_BEFORE" = yes ] || [ "$ROLLBACK_FAILED" = no ]; then',
|
|
rollback_confirm,
|
|
rollback_owner,
|
|
)
|
|
assert rollback_first < rollback_delete < rollback_confirm < guard
|
|
assert guard < rollback_owner
|
|
|
|
|
|
def test_first_deploy_recovery_delete_is_repeatable_and_fails_closed(
|
|
tmp_path):
|
|
start = _BOOT_HELPER.index(
|
|
'if [ "$journal_iface_before" = no ] &&')
|
|
end = _BOOT_HELPER.index(" journal_sysctl_restore=no", start)
|
|
block = _BOOT_HELPER[start:end]
|
|
link_state = tmp_path / "link-present"
|
|
link_state.touch()
|
|
harness = "\n".join((
|
|
"set -eu",
|
|
"journal_iface_before=no",
|
|
"journal_current_iface_state=present",
|
|
"journal_current_owner_state=present",
|
|
"IFACE=wg-fabric0",
|
|
"vc_fabric_recovery_checkpoint() { :; }",
|
|
"vc_fabric_owner_file_owned() { :; }",
|
|
"mock_wg() { [ \"$1:$2\" = show:wg-fabric0 ]; }",
|
|
"mock_ip() {",
|
|
" case \"$*\" in",
|
|
" 'link show dev wg-fabric0') [ -e \"$VC_TEST_LINK\" ] ;;",
|
|
" 'link del dev wg-fabric0')",
|
|
" [ \"${VC_TEST_DELETE_FAIL:-no}\" = yes ] || "
|
|
"rm -f \"$VC_TEST_LINK\" ;;",
|
|
" *) return 97 ;;",
|
|
" esac",
|
|
"}",
|
|
"WG=mock_wg",
|
|
"IP=mock_ip",
|
|
block,
|
|
"printf '%s\\n' \"$journal_current_iface_state\"",
|
|
))
|
|
env = {
|
|
"PATH": "/usr/bin:/bin",
|
|
"VC_TEST_LINK": str(link_state),
|
|
}
|
|
|
|
removed = subprocess.run(
|
|
["/bin/sh", "-c", harness], env=env, check=False,
|
|
capture_output=True, text=True,
|
|
)
|
|
assert removed.returncode == 0, removed.stderr
|
|
assert removed.stdout == "absent\n"
|
|
assert link_state.exists() is False
|
|
|
|
# A crash after link deletion but before restoring the absent OWNER is
|
|
# repeatable: the next pass observes no live link and leaves it untouched.
|
|
repeated = subprocess.run(
|
|
["/bin/sh", "-c", harness.replace(
|
|
"journal_current_iface_state=present",
|
|
"journal_current_iface_state=absent",
|
|
)], env=env, check=False, capture_output=True, text=True,
|
|
)
|
|
assert repeated.returncode == 0, repeated.stderr
|
|
assert repeated.stdout == "absent\n"
|
|
|
|
link_state.touch()
|
|
failed_env = dict(env)
|
|
failed_env["VC_TEST_DELETE_FAIL"] = "yes"
|
|
failed = subprocess.run(
|
|
["/bin/sh", "-c", harness], env=failed_env, check=False,
|
|
capture_output=True, text=True,
|
|
)
|
|
assert failed.returncode == 79
|
|
assert link_state.exists() is True
|
|
|
|
|
|
def test_first_deploy_recovery_exits_without_absent_helper_or_timer_stop():
|
|
recover_start = _BOOT_HELPER.index("recover_journal() {")
|
|
recover_end = _BOOT_HELPER.index(
|
|
'\n}\nif [ "${VC_FABRIC_APPLYING:-0}" != 1 ]; then', recover_start)
|
|
recovery = _BOOT_HELPER[recover_start:recover_end]
|
|
|
|
helper_snapshot = recovery.index(
|
|
'if [ -f "$JOURNAL/helper.present" ]; then')
|
|
timer_snapshot = recovery.index(
|
|
'if [ -f "$JOURNAL/timer.present" ]; then', helper_snapshot)
|
|
stop_transaction_timer = recovery.index(
|
|
"vc_stop_transaction_timer_if_loaded || exit 79",
|
|
timer_snapshot,
|
|
)
|
|
restore_timer_file = recovery.index(
|
|
'restore_journal_file "/etc/systemd/system/'
|
|
'virt-cloud-wg-fabric-reconcile.timer" timer',
|
|
stop_transaction_timer,
|
|
)
|
|
daemon_reload = recovery.index(
|
|
'"$SYSTEMCTL" daemon-reload', restore_timer_file)
|
|
timer_present_branch = recovery.index(
|
|
'if [ "$journal_timer_before" = present ]; then', daemon_reload)
|
|
timer_restore = recovery.index(
|
|
'vc_restore_timer_activity "$journal_timer_active"',
|
|
timer_present_branch,
|
|
)
|
|
timer_not_found = recovery.index(
|
|
'[ "$journal_timer_load_state" = not-found ]', timer_restore)
|
|
timer_inactive = recovery.index(
|
|
'[ "$journal_timer_active_state" = inactive ]', timer_not_found)
|
|
absent_helper = recovery.index(
|
|
'if [ "$journal_helper_before" = absent ]; then', timer_inactive)
|
|
cleanup_state = recovery.index(
|
|
"journal_publish_cleanup_helper_state || exit 79", absent_helper)
|
|
restore_absent_helper = recovery.index(
|
|
'restore_journal_file "$HELPER" helper || exit 79', cleanup_state)
|
|
remove_journal = recovery.index(
|
|
'rm -rf "$JOURNAL" || exit 79', restore_absent_helper)
|
|
absent_exit = recovery.index("exit 0", remove_journal)
|
|
exec_helper = recovery.index('exec /bin/sh "$HELPER" "$@"', absent_exit)
|
|
|
|
assert helper_snapshot < timer_snapshot < stop_transaction_timer
|
|
assert stop_transaction_timer < restore_timer_file
|
|
assert restore_timer_file < daemon_reload
|
|
assert daemon_reload < timer_present_branch < timer_restore < timer_not_found
|
|
assert timer_not_found < timer_inactive < absent_helper < cleanup_state
|
|
assert cleanup_state < restore_absent_helper < remove_journal
|
|
assert remove_journal < absent_exit < exec_helper
|
|
present_helper_branch = recovery.index(
|
|
'if [ "$journal_helper_before" = present ]; then', timer_snapshot)
|
|
present_helper_restore = recovery.index(
|
|
'restore_journal_file "$HELPER" helper || exit 79',
|
|
present_helper_branch,
|
|
)
|
|
assert present_helper_branch < present_helper_restore < absent_helper
|
|
timer_stop_start = _BOOT_HELPER.index(
|
|
"vc_stop_transaction_timer_if_loaded() {")
|
|
timer_stop_end = _BOOT_HELPER.index("nft_panel_owned() {", timer_stop_start)
|
|
timer_stop = _BOOT_HELPER[timer_stop_start:timer_stop_end]
|
|
assert "not-found:inactive|loaded:inactive" in timer_stop
|
|
assert "loaded:active|loaded:activating|loaded:deactivating" in timer_stop
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("load_state", "active_state", "sticky", "expected_rc", "stops"),
|
|
(
|
|
("not-found", "inactive", False, 0, False),
|
|
("loaded", "inactive", False, 0, False),
|
|
("loaded", "active", False, 0, True),
|
|
("loaded", "active", True, 1, True),
|
|
("not-found", "active", False, 1, False),
|
|
("loaded", "failed", False, 1, False),
|
|
),
|
|
)
|
|
def test_absent_timer_recovery_stops_only_loaded_transaction_unit(
|
|
tmp_path, load_state, active_state, sticky, expected_rc, stops):
|
|
function_start = _BOOT_HELPER.index(
|
|
"vc_stop_transaction_timer_if_loaded() {")
|
|
function_end = _BOOT_HELPER.index("nft_panel_owned() {", function_start)
|
|
function = _BOOT_HELPER[function_start:function_end]
|
|
load = tmp_path / "load"
|
|
active = tmp_path / "active"
|
|
log = tmp_path / "systemctl.log"
|
|
load.write_text(f"{load_state}\n", encoding="utf-8")
|
|
active.write_text(f"{active_state}\n", encoding="utf-8")
|
|
systemctl = tmp_path / "systemctl"
|
|
systemctl.write_text(
|
|
"#!/bin/sh\n"
|
|
"case \"$1:$2:$3\" in\n"
|
|
" 'show:-p:LoadState') cat \"$VC_TIMER_LOAD\" ;;\n"
|
|
" 'show:-p:ActiveState') cat \"$VC_TIMER_ACTIVE\" ;;\n"
|
|
" 'stop:virt-cloud-wg-fabric-reconcile.timer:')\n"
|
|
" printf '%s\\n' stop >> \"$VC_TIMER_LOG\"\n"
|
|
" [ \"$VC_TIMER_STICKY\" = yes ] || "
|
|
"printf '%s\\n' inactive > \"$VC_TIMER_ACTIVE\" ;;\n"
|
|
" *) exit 2 ;;\n"
|
|
"esac\n",
|
|
encoding="utf-8",
|
|
)
|
|
systemctl.chmod(0o700)
|
|
result = subprocess.run(
|
|
["/bin/sh", "-c", "\n".join((
|
|
"set -eu",
|
|
f"SYSTEMCTL='{systemctl}'",
|
|
function,
|
|
"vc_stop_transaction_timer_if_loaded",
|
|
))],
|
|
env={
|
|
"PATH": "/usr/bin:/bin",
|
|
"VC_TIMER_LOAD": str(load),
|
|
"VC_TIMER_ACTIVE": str(active),
|
|
"VC_TIMER_LOG": str(log),
|
|
"VC_TIMER_STICKY": "yes" if sticky else "no",
|
|
},
|
|
check=False, capture_output=True, text=True,
|
|
)
|
|
assert result.returncode == expected_rc, result.stderr
|
|
assert log.exists() is stops
|
|
|
|
|
|
def test_cleanup_helper_terminal_phase_is_repeatable_and_rejects_foreign_file(
|
|
tmp_path):
|
|
digest = "a" * 64
|
|
journal = tmp_path / "journal"
|
|
helper = tmp_path / "helper"
|
|
stat = tmp_path / "stat"
|
|
stat.write_text(
|
|
"#!/bin/sh\n"
|
|
"[ \"$1:$2\" = '-c:%u:%a' ] || exit 2\n"
|
|
"case \"$3\" in\n"
|
|
f" '{helper}') printf '%s\\n' 0:700 ;;\n"
|
|
" *) printf '%s\\n' 0:600 ;;\n"
|
|
"esac\n",
|
|
encoding="utf-8",
|
|
)
|
|
stat.chmod(0o700)
|
|
validator_start = _BOOT_HELPER.index("journal_cleanup_helper_owned() {")
|
|
validator_end = _BOOT_HELPER.index(
|
|
"journal_publish_cleanup_helper_state() {", validator_start)
|
|
validator = _BOOT_HELPER[validator_start:validator_end]
|
|
recover_start = _BOOT_HELPER.index("recover_journal() {")
|
|
branch_start = _BOOT_HELPER.index(" cleanup_helper)", recover_start)
|
|
branch_end = _BOOT_HELPER.index(" guarded)", branch_start)
|
|
cleanup_branch = _BOOT_HELPER[branch_start:branch_end]
|
|
harness = "\n".join((
|
|
"set -eu",
|
|
f"JOURNAL='{journal}'",
|
|
f"HELPER='{helper}'",
|
|
f"STAT='{stat}'",
|
|
"journal_file_secure() {",
|
|
" [ -f \"$JOURNAL/$1\" ] && [ ! -L \"$JOURNAL/$1\" ]",
|
|
"}",
|
|
"journal_file_matches() {",
|
|
" [ -f \"$1\" ] && [ ! -L \"$1\" ] &&",
|
|
" /usr/bin/cmp -s \"$1\" \"$JOURNAL/$2\"",
|
|
"}",
|
|
"journal_value() {",
|
|
" awk -F= -v wanted=\"$1\" '$1 == wanted {print substr($0, "
|
|
"index($0, \"=\") + 1); found=1} END {exit found ? 0 : 1}' "
|
|
"\"$JOURNAL/meta\"",
|
|
"}",
|
|
validator,
|
|
"case cleanup_helper in",
|
|
cleanup_branch,
|
|
"esac",
|
|
"exit 99",
|
|
))
|
|
|
|
def prepare(*, helper_state: str) -> None:
|
|
journal.mkdir()
|
|
(journal / "state").write_text("cleanup_helper\n", encoding="utf-8")
|
|
(journal / "meta").write_text(
|
|
f"schema=1\nplan_digest={digest}\n", encoding="utf-8")
|
|
(journal / "helper.absent").touch()
|
|
for path in (journal / "state", journal / "meta",
|
|
journal / "helper.absent"):
|
|
path.chmod(0o600)
|
|
(journal / "helper.desired").write_text(
|
|
"#!/bin/sh\n"
|
|
"# virt-cloud-wg-fabric-v1\n"
|
|
f"# fabric-plan-digest={digest}\n"
|
|
f'PLAN_DIGEST="{digest}"\n',
|
|
encoding="utf-8",
|
|
)
|
|
(journal / "helper.desired").chmod(0o700)
|
|
if helper_state != "absent":
|
|
actual_digest = digest if helper_state == "owned" else "b" * 64
|
|
helper.write_text(
|
|
"#!/bin/sh\n"
|
|
"# virt-cloud-wg-fabric-v1\n"
|
|
f"# fabric-plan-digest={actual_digest}\n"
|
|
f'PLAN_DIGEST="{actual_digest}"\n',
|
|
encoding="utf-8",
|
|
)
|
|
helper.chmod(0o700)
|
|
|
|
prepare(helper_state="owned")
|
|
owned = subprocess.run(
|
|
["/bin/sh", "-c", harness], check=False,
|
|
capture_output=True, text=True,
|
|
)
|
|
assert owned.returncode == 0, owned.stderr
|
|
assert helper.exists() is False
|
|
assert journal.exists() is False
|
|
|
|
prepare(helper_state="absent")
|
|
absent = subprocess.run(
|
|
["/bin/sh", "-c", harness], check=False,
|
|
capture_output=True, text=True,
|
|
)
|
|
assert absent.returncode == 0, absent.stderr
|
|
assert journal.exists() is False
|
|
|
|
prepare(helper_state="foreign")
|
|
foreign = subprocess.run(
|
|
["/bin/sh", "-c", harness], check=False,
|
|
capture_output=True, text=True,
|
|
)
|
|
assert foreign.returncode == 79
|
|
assert helper.exists() is True
|
|
assert journal.exists() is True
|
|
|
|
current = plan()
|
|
apply = _apply_script(
|
|
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
|
|
)
|
|
existing = apply.index('case "$journal_existing_state" in')
|
|
cleanup = apply.index("cleanup_helper)", existing)
|
|
direct_absent = apply.index(
|
|
'[ ! -e "$HELPER" ] && [ ! -L "$HELPER" ] || exit 79', cleanup)
|
|
ready = apply.index("ready)", direct_absent)
|
|
first_deploy_phase = apply.index(
|
|
'transaction_missing_installing_snapshot=no', ready)
|
|
special_first = apply.index(
|
|
'if [ "$transaction_missing_installing_snapshot" = yes ]; then',
|
|
first_deploy_phase,
|
|
)
|
|
temporary_recovery = apply.index(
|
|
'VC_FABRIC_LOCK_HELD=1 /bin/sh "$_tmp_helper" '
|
|
'--recover-installing || exit 79',
|
|
special_first,
|
|
)
|
|
fixed_helper_owned = apply.index(
|
|
'transaction_recovery_helper_owned || exit 79',
|
|
temporary_recovery,
|
|
)
|
|
fixed_recovery = apply.index(
|
|
'VC_FABRIC_LOCK_HELD=1 /bin/sh "$HELPER" --recover-only || exit 79',
|
|
fixed_helper_owned,
|
|
)
|
|
assert (
|
|
existing < cleanup < direct_absent < ready < first_deploy_phase
|
|
< special_first < temporary_recovery < fixed_helper_owned
|
|
< fixed_recovery
|
|
)
|
|
|
|
|
|
def test_transaction_journal_preparing_state_is_atomically_published():
|
|
current = plan()
|
|
apply = _apply_script(
|
|
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
|
|
)
|
|
|
|
existing = apply.index('if [ -e "$BASE/.wg-fabric-txn" ]; then')
|
|
state_required = apply.index(
|
|
'[ -f "$BASE/.wg-fabric-txn/state" ] &&', existing)
|
|
existing_case = apply.index('case "$journal_existing_state" in', state_required)
|
|
publish_start = apply.index(
|
|
'_tmp_txn_dir=$(mktemp -d '
|
|
'"$BASE/.wg-fabric-txn-preparing.XXXXXX")', existing_case)
|
|
preparing_state = apply.index(
|
|
'printf \'%s\\n\' preparing > "$_tmp_txn_dir/state"', publish_start)
|
|
preparing_meta = apply.index(
|
|
'cat > "$_tmp_txn_dir/meta"', preparing_state)
|
|
absence_proof = apply.index(
|
|
'[ ! -e "$TXN" ] && [ ! -L "$TXN" ] || exit 79', preparing_meta)
|
|
publish = apply.index('mv "$_tmp_txn_dir" "$TXN"', absence_proof)
|
|
readback = apply.index(
|
|
'[ "$(cat "$TXN/state" 2>/dev/null)" = preparing ]', publish)
|
|
first_snapshot = apply.index('snapshot_file "$CONF" conf', readback)
|
|
|
|
assert existing < state_required < existing_case < publish_start
|
|
assert publish_start < preparing_state < preparing_meta < absence_proof
|
|
assert absence_proof < publish < readback < first_snapshot
|
|
assert "firewall_mode=$FIREWALL_MODE" in apply[
|
|
preparing_meta:absence_proof
|
|
]
|
|
assert 'mkdir "$TXN"' not in apply
|
|
|
|
|
|
def test_observe_marks_any_fabric_transaction_journal_as_mismatch():
|
|
current = plan()
|
|
observe = _observe_script(
|
|
("100.64.1.2",), "100.64.1.1", ("203.0.113.2",),
|
|
firewall_mode=FIREWALL_MODE_NATIVE,
|
|
connection_uuid="12345678-1234-5678-1234-567812345678",
|
|
host_id="host-a", fabric_id=current.fabric_id,
|
|
overlay_cidr=current.overlay_cidr,
|
|
expected_digest=current.digest,
|
|
)
|
|
state = observe.index("firewall_mode_state=owned")
|
|
journal_path = observe.index(
|
|
"[ ! -e /etc/virt-cloud/.wg-fabric-txn ]", state)
|
|
journal_link = observe.index(
|
|
"[ ! -L /etc/virt-cloud/.wg-fabric-txn ]", journal_path)
|
|
mismatch = observe.index(
|
|
"|| firewall_mode_state=mismatch", journal_link)
|
|
marker = observe.index("__VC_FABRIC_FIREWALL_STATE__", mismatch)
|
|
assert state < journal_path < journal_link < mismatch < marker
|
|
|
|
|
|
def test_networkmanager_filename_resolution_uses_supported_unique_list_rows():
|
|
assert "GENERAL.FILENAME" not in _NMCLI_FILENAME_RESOLVER_SH
|
|
assert "GENERAL.FILENAME" not in _FIREWALL_MODE_GUARD_SH
|
|
assert "GENERAL.FILENAME" not in _BOOT_HELPER
|
|
assert (
|
|
'LC_ALL=C "$NMCLI" -t --escape no -f UUID,FILENAME'
|
|
in _NMCLI_FILENAME_RESOLVER_SH
|
|
)
|
|
# Resolve before and after semantic ownership checks to reject a filename
|
|
# switch during the readback transaction.
|
|
assert (
|
|
_FIREWALL_MODE_GUARD_SH.count(
|
|
'vc_nm_filename_for_uuid "$CONNECTION_UUID"') == 2
|
|
)
|
|
assert (
|
|
_BOOT_HELPER.count('vc_nm_filename_for_uuid "$vpc_nm_uuid"') == 1
|
|
)
|
|
current = plan()
|
|
apply = _apply_script(
|
|
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
|
|
firewall_mode=FIREWALL_MODE_NATIVE,
|
|
)
|
|
observe = _observe_script(
|
|
("100.64.1.2",), "100.64.1.1", ("203.0.113.2",),
|
|
firewall_mode=FIREWALL_MODE_NATIVE,
|
|
connection_uuid="12345678-1234-5678-1234-567812345678",
|
|
host_id="host-a", fabric_id=current.fabric_id,
|
|
overlay_cidr=current.overlay_cidr,
|
|
expected_digest=current.digest,
|
|
)
|
|
assert "__VC_NMCLI_FILENAME_RESOLVER__" not in apply
|
|
assert "GENERAL.FILENAME" not in apply
|
|
assert "GENERAL.FILENAME" not in observe
|
|
# Apply embeds one resolver for its own transaction and another in the
|
|
# separately installed boot helper; observe is one shell process.
|
|
assert apply.count("vc_nm_filename_for_uuid() {") == 2
|
|
assert observe.count("vc_nm_filename_for_uuid() {") == 1
|
|
assert "nm_vpc_field_raw=$(" in _BOOT_HELPER
|
|
assert (
|
|
'nm_vpc_value=$(nm_vpc_field "$nm_vpc_empty" uuid '
|
|
'"$vpc_nm_uuid") || exit 71' in _BOOT_HELPER
|
|
)
|
|
|
|
target = "11111111-1111-4111-8111-111111111111"
|
|
expected = (
|
|
"/etc/NetworkManager/system-connections/"
|
|
"bridge profile:primary.nmconnection"
|
|
)
|
|
harness = "\n".join((
|
|
"set -eu",
|
|
r'''
|
|
vc_test_nmcli() {
|
|
case "$*" in
|
|
"-t --escape no -f UUID,FILENAME connection show")
|
|
[ "${VC_NMCLI_FAIL:-no}" = no ] || return 3
|
|
printf '%s' "$VC_NMCLI_ROWS"
|
|
;;
|
|
*GENERAL.FILENAME*) return 97 ;;
|
|
*) return 98 ;;
|
|
esac
|
|
}
|
|
NMCLI=vc_test_nmcli
|
|
'''.strip(),
|
|
_NMCLI_FILENAME_RESOLVER_SH,
|
|
'vc_nm_filename_for_uuid "$VC_TARGET_UUID"',
|
|
))
|
|
|
|
def run(rows: str, *, uuid: str = target, fail: bool = False):
|
|
return subprocess.run(
|
|
["/bin/sh", "-c", harness],
|
|
env={
|
|
"PATH": "/usr/bin:/bin",
|
|
"VC_TARGET_UUID": uuid,
|
|
"VC_NMCLI_ROWS": rows,
|
|
"VC_NMCLI_FAIL": "yes" if fail else "no",
|
|
},
|
|
check=False, capture_output=True, text=True,
|
|
)
|
|
|
|
accepted = run(
|
|
"22222222-2222-4222-8222-222222222222:"
|
|
"/run/NetworkManager/system-connections/other.nmconnection\n"
|
|
f"{target}:{expected}\n"
|
|
)
|
|
assert accepted.returncode == 0, accepted.stderr
|
|
assert accepted.stdout == f"{expected}\n"
|
|
|
|
duplicate = run(f"{target}:{expected}\n{target}:{expected}\n")
|
|
assert duplicate.returncode != 0
|
|
missing = run(
|
|
"22222222-2222-4222-8222-222222222222:"
|
|
"/etc/NetworkManager/system-connections/other.nmconnection\n"
|
|
)
|
|
assert missing.returncode != 0
|
|
malformed_inventory = run(
|
|
f"{target}:{expected}\ncontinuation-without-a-uuid\n"
|
|
)
|
|
assert malformed_inventory.returncode != 0
|
|
oversized_row = run(f"{target}:{'x' * 4097}\n")
|
|
assert oversized_row.returncode != 0
|
|
malformed = run(f"{target}:{expected}\n", uuid="../not-a-uuid")
|
|
assert malformed.returncode != 0
|
|
command_failure = run(f"{target}:{expected}\n", fail=True)
|
|
assert command_failure.returncode != 0
|
|
|
|
|
|
def test_networkmanager_quarantine_keeps_owned_device_until_hard_link_down():
|
|
current = plan()
|
|
helper = _apply_script(
|
|
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
|
|
firewall_mode=FIREWALL_MODE_NATIVE,
|
|
)
|
|
start = helper.index("vc_fabric_nm_quarantine() {")
|
|
end = helper.index("\n}", start)
|
|
block = helper[start:end]
|
|
profile_read = block.index("vc_fabric_nm_profile_owned any || return 1")
|
|
unmanaged = block.index(
|
|
'"$NMCLI" device set "$IFACE" managed no', profile_read)
|
|
hard_down = block.index(
|
|
'"$IP" link set dev "$IFACE" down', unmanaged)
|
|
assert profile_read < unmanaged < hard_down
|
|
assert "firewall-cmd" not in block
|
|
assert "connection.zone" not in block
|
|
assert 'device reapply "$IFACE"' not in block
|
|
assert 'connection down uuid "$CONNECTION_UUID"' not in block
|
|
assert 'device disconnect "$IFACE"' not in block
|
|
|
|
|
|
|
|
|
|
def _write_firewall_guard_tools(tmp_path: Path) -> Path:
|
|
systemctl = tmp_path / "systemctl"
|
|
systemctl.write_text("""#!/bin/sh
|
|
case "$1:$2" in
|
|
is-active:nftables) printf '%s\\n' "$VC_NFT_ACTIVE" ;;
|
|
is-enabled:nftables) printf '%s\\n' "$VC_NFT_ENABLED" ;;
|
|
is-active:firewalld) printf '%s\\n' "$VC_FIREWALLD_ACTIVE" ;;
|
|
is-enabled:firewalld) printf '%s\\n' "$VC_FIREWALLD_ENABLED" ;;
|
|
show:-p)
|
|
case "$5" in
|
|
nftables) printf '%s\\n' "$VC_NFT_LOAD" ;;
|
|
firewalld) printf '%s\\n' "$VC_FIREWALLD_LOAD" ;;
|
|
*) exit 2 ;;
|
|
esac ;;
|
|
*) exit 2 ;;
|
|
esac
|
|
""", encoding="utf-8")
|
|
systemctl.chmod(0o700)
|
|
return systemctl
|
|
|
|
|
|
def _run_firewall_guard(
|
|
tmp_path: Path, mode: str, *, nft_active: str, nft_enabled: str,
|
|
nft_load: str, firewalld_active: str, firewalld_enabled: str,
|
|
firewalld_load: str, nft_readable: bool = True,
|
|
) -> subprocess.CompletedProcess[str]:
|
|
systemctl = _write_firewall_guard_tools(tmp_path)
|
|
script = "\n".join((
|
|
"set -eu",
|
|
f"SYSTEMCTL='{systemctl}'",
|
|
f"NFT={'/usr/bin/true' if nft_readable else '/usr/bin/false'}",
|
|
f"FIREWALL_MODE='{mode}'",
|
|
"find_tool() { for p in \"$@\"; do [ -x \"$p\" ] && { printf '%s' \"$p\"; return 0; }; done; return 1; }",
|
|
_FIREWALL_MODE_GUARD_SH,
|
|
"vc_firewall_mode_ready",
|
|
))
|
|
env = {
|
|
"PATH": "/usr/bin:/bin",
|
|
"VC_NFT_ACTIVE": nft_active,
|
|
"VC_NFT_ENABLED": nft_enabled,
|
|
"VC_NFT_LOAD": nft_load,
|
|
"VC_FIREWALLD_ACTIVE": firewalld_active,
|
|
"VC_FIREWALLD_ENABLED": firewalld_enabled,
|
|
"VC_FIREWALLD_LOAD": firewalld_load,
|
|
}
|
|
return subprocess.run(
|
|
["/bin/sh", "-c", script], env=env,
|
|
check=False, capture_output=True, text=True,
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.mark.parametrize(("active", "enabled", "load"), (
|
|
("active", "enabled-runtime", "loaded"),
|
|
("inactive", "disabled", "loaded"),
|
|
("inactive", "masked", "masked"),
|
|
("unknown", "unknown", "not-found"),
|
|
))
|
|
def test_native_runtime_guard_ignores_stock_nftables_service_state(
|
|
tmp_path, active, enabled, load,
|
|
):
|
|
native = _run_firewall_guard(
|
|
tmp_path, FIREWALL_MODE_NATIVE,
|
|
nft_active=active, nft_enabled=enabled, nft_load=load,
|
|
firewalld_active="inactive", firewalld_enabled="masked",
|
|
firewalld_load="masked",
|
|
)
|
|
assert native.returncode == 0, native.stderr
|
|
|
|
|
|
def test_native_runtime_guard_requires_readable_nft_runtime(tmp_path):
|
|
native = _run_firewall_guard(
|
|
tmp_path, FIREWALL_MODE_NATIVE,
|
|
nft_active="inactive", nft_enabled="disabled", nft_load="loaded",
|
|
firewalld_active="inactive", firewalld_enabled="masked",
|
|
firewalld_load="masked", nft_readable=False,
|
|
)
|
|
assert native.returncode != 0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_native_runtime_guard_accepts_a_not_found_firewalld_unit(tmp_path):
|
|
result = _run_firewall_guard(
|
|
tmp_path, FIREWALL_MODE_NATIVE,
|
|
nft_active="active", nft_enabled="enabled", nft_load="loaded",
|
|
firewalld_active="unknown", firewalld_enabled="unknown",
|
|
firewalld_load="not-found",
|
|
)
|
|
assert result.returncode == 0, result.stderr
|
|
|
|
|
|
def test_native_runtime_guard_rejects_inactive_unmasked_firewalld(tmp_path):
|
|
result = _run_firewall_guard(
|
|
tmp_path, FIREWALL_MODE_NATIVE,
|
|
nft_active="active", nft_enabled="enabled", nft_load="loaded",
|
|
firewalld_active="inactive", firewalld_enabled="disabled",
|
|
firewalld_load="loaded",
|
|
)
|
|
assert result.returncode != 0
|
|
|
|
|
|
@pytest.mark.parametrize(("distribution", "target", "source"), [
|
|
("debian-13-iproute2-6.15", "203.0.113.123", "203.0.113.126"),
|
|
("almalinux-10-iproute-6.17", "198.51.100.126", "198.51.100.123"),
|
|
])
|
|
def test_underlay_route_path_accepts_real_explicit_from_output(
|
|
tmp_path, distribution, target, source,
|
|
):
|
|
fake_ip = tmp_path / "ip"
|
|
fake_ip.write_text(
|
|
"#!/bin/sh\n"
|
|
"if [ \"${5:-}\" = from ]; then\n"
|
|
" printf '%s from %s dev %s uid 0\\n cache\\n' "
|
|
"\"$4\" \"$VC_FROM_SRC\" \"$VC_FROM_DEV\"\n"
|
|
"else\n"
|
|
" printf '%s dev %s src %s uid 0\\n cache\\n' "
|
|
"\"$4\" \"$VC_PLAIN_DEV\" \"$VC_PLAIN_SRC\"\n"
|
|
"fi\n",
|
|
encoding="utf-8",
|
|
)
|
|
fake_ip.chmod(0o700)
|
|
script = "\n".join((
|
|
"set -eu",
|
|
f"IP='{fake_ip}'",
|
|
"find_tool() { for p in \"$@\"; do [ -x \"$p\" ] && { printf '%s' \"$p\"; return 0; }; done; return 1; }",
|
|
_FIREWALL_MODE_GUARD_SH,
|
|
f"vc_underlay_route_path {target} {source}",
|
|
))
|
|
base_env = {
|
|
"PATH": "/usr/bin:/bin",
|
|
"VC_PLAIN_DEV": "br0",
|
|
"VC_FROM_DEV": "br0",
|
|
"VC_FROM_SRC": source,
|
|
}
|
|
|
|
exact = subprocess.run(
|
|
["/bin/sh", "-c", script],
|
|
env={**base_env, "VC_PLAIN_SRC": source},
|
|
check=False, capture_output=True, text=True,
|
|
)
|
|
assert exact.returncode == 0, f"{distribution}: {exact.stderr}"
|
|
assert exact.stdout == f"br0|{source}"
|
|
|
|
|
|
@pytest.mark.parametrize(("plain_source", "explicit_source"), [
|
|
("10.20.30.40", "203.0.113.1"),
|
|
("203.0.113.1", "10.20.30.40"),
|
|
])
|
|
def test_underlay_route_path_rejects_inexact_plain_or_explicit_source(
|
|
tmp_path, plain_source, explicit_source,
|
|
):
|
|
fake_ip = tmp_path / "ip"
|
|
fake_ip.write_text(
|
|
"#!/bin/sh\n"
|
|
"if [ \"${5:-}\" = from ]; then\n"
|
|
" printf '%s from %s dev eth0 uid 0\\n cache\\n' "
|
|
"\"$4\" \"$VC_FROM_SRC\"\n"
|
|
"else\n"
|
|
" printf '%s dev eth0 src %s uid 0\\n cache\\n' "
|
|
"\"$4\" \"$VC_PLAIN_SRC\"\n"
|
|
"fi\n",
|
|
encoding="utf-8",
|
|
)
|
|
fake_ip.chmod(0o700)
|
|
script = "\n".join((
|
|
"set -eu",
|
|
f"IP='{fake_ip}'",
|
|
"find_tool() { for p in \"$@\"; do [ -x \"$p\" ] && { printf '%s' \"$p\"; return 0; }; done; return 1; }",
|
|
_FIREWALL_MODE_GUARD_SH,
|
|
"vc_underlay_route_path 203.0.113.2 203.0.113.1",
|
|
))
|
|
|
|
mismatch = subprocess.run(
|
|
["/bin/sh", "-c", script],
|
|
env={
|
|
"PATH": "/usr/bin:/bin",
|
|
"VC_PLAIN_SRC": plain_source,
|
|
"VC_FROM_SRC": explicit_source,
|
|
},
|
|
check=False, capture_output=True, text=True,
|
|
)
|
|
assert mismatch.returncode != 0
|
|
|
|
|
|
def _run_underlay_explicit_evidence(
|
|
tmp_path: Path, explicit_output: str,
|
|
) -> subprocess.CompletedProcess[str]:
|
|
fake_ip = tmp_path / "ip"
|
|
fake_ip.write_text(
|
|
"#!/bin/sh\n"
|
|
"if [ \"${5:-}\" = from ]; then\n"
|
|
" printf '%s\\n' \"$VC_EXPLICIT_OUTPUT\"\n"
|
|
"else\n"
|
|
" printf '%s\\n' '203.0.113.2 dev br0 src 203.0.113.1 uid 0'\n"
|
|
"fi\n",
|
|
encoding="utf-8",
|
|
)
|
|
fake_ip.chmod(0o700)
|
|
script = "\n".join((
|
|
"set -eu",
|
|
f"IP='{fake_ip}'",
|
|
"find_tool() { for p in \"$@\"; do [ -x \"$p\" ] && { printf '%s' \"$p\"; return 0; }; done; return 1; }",
|
|
_FIREWALL_MODE_GUARD_SH,
|
|
"vc_underlay_route_path 203.0.113.2 203.0.113.1",
|
|
))
|
|
return subprocess.run(
|
|
["/bin/sh", "-c", script],
|
|
env={
|
|
"PATH": "/usr/bin:/bin",
|
|
"VC_EXPLICIT_OUTPUT": explicit_output,
|
|
},
|
|
check=False, capture_output=True, text=True,
|
|
)
|
|
|
|
|
|
def test_underlay_route_path_accepts_matching_optional_selected_source(
|
|
tmp_path,
|
|
):
|
|
result = _run_underlay_explicit_evidence(
|
|
tmp_path,
|
|
"203.0.113.2 from 203.0.113.1 dev br0 "
|
|
"src 203.0.113.1 uid 0",
|
|
)
|
|
assert result.returncode == 0, result.stderr
|
|
assert result.stdout == "br0|203.0.113.1"
|
|
|
|
|
|
@pytest.mark.parametrize("explicit_output", [
|
|
"203.0.113.2 dev br0 uid 0",
|
|
"203.0.113.2 dev br0 src 203.0.113.1 uid 0",
|
|
"203.0.113.2 from 203.0.113.1 from 203.0.113.1 dev br0 uid 0",
|
|
(
|
|
"203.0.113.2 from 203.0.113.1 dev br0 "
|
|
"src 10.20.30.40 uid 0"
|
|
),
|
|
"203.0.113.2 from 203.0.113.1 dev br0 dev br0 uid 0",
|
|
"203.0.113.2 dev br0 src 203.0.113.1 from",
|
|
])
|
|
def test_underlay_route_path_rejects_missing_or_ambiguous_explicit_evidence(
|
|
tmp_path, explicit_output,
|
|
):
|
|
result = _run_underlay_explicit_evidence(tmp_path, explicit_output)
|
|
assert result.returncode != 0
|
|
|
|
|
|
def _run_native_apply_underlay_preflight(
|
|
tmp_path: Path, *, local_address: str = "203.0.113.1",
|
|
local_device: str = "eth0",
|
|
plain_device: str = "eth0", plain_source: str = "203.0.113.1",
|
|
from_device: str = "eth0", from_source: str = "203.0.113.1",
|
|
) -> subprocess.CompletedProcess[str]:
|
|
current = plan()
|
|
rendered = _apply_script(
|
|
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
|
|
firewall_mode=FIREWALL_MODE_NATIVE,
|
|
)
|
|
preflight_start = rendered.index("LOCAL_ENDPOINT=203.0.113.1")
|
|
preflight_end = rendered.index(
|
|
'if [ "$DEPLOYMENT_MODE" = networkmanager_nmcli ]; then',
|
|
preflight_start,
|
|
)
|
|
endpoint_start = rendered.index(
|
|
"# The UDP endpoint must leave through the host underlay",
|
|
preflight_end,
|
|
)
|
|
endpoint_end = rendered.index(
|
|
"# A desired overlay /32 may only be replaced", endpoint_start,
|
|
)
|
|
|
|
fake_ip = tmp_path / "ip"
|
|
fake_ip.write_text(
|
|
"#!/bin/sh\n"
|
|
"case \"$*\" in\n"
|
|
" '-o -4 address show up scope global')\n"
|
|
" printf '%s\\n' \"2: $VC_LOCAL_DEVICE inet $VC_LOCAL_ADDRESS/24 "
|
|
"scope global $VC_LOCAL_DEVICE\" ;;\n"
|
|
" '-4 route get 203.0.113.2')\n"
|
|
" printf '%s\\n' \"203.0.113.2 dev $VC_PLAIN_DEV "
|
|
"src $VC_PLAIN_SRC\" ;;\n"
|
|
" '-4 route get 203.0.113.2 from 203.0.113.1')\n"
|
|
" printf '%s\\n' \"203.0.113.2 from $VC_FROM_SRC "
|
|
"dev $VC_FROM_DEV uid 0\" \" cache\" ;;\n"
|
|
" *) exit 91 ;;\n"
|
|
"esac\n",
|
|
encoding="utf-8",
|
|
)
|
|
fake_ip.chmod(0o700)
|
|
shell = "\n".join((
|
|
"set -eu",
|
|
f"IP='{fake_ip}'",
|
|
"FIREWALL_MODE=native_nftables",
|
|
"IFACE=wg-fabric0",
|
|
"CONF=/nonexistent/virt-cloud-wg-fabric.conf",
|
|
"OLD_UNDERLAY=",
|
|
"OLD_FIREWALL_RULE_OWNER=",
|
|
_FIREWALL_MODE_GUARD_SH,
|
|
rendered[preflight_start:preflight_end],
|
|
rendered[endpoint_start:endpoint_end],
|
|
))
|
|
return subprocess.run(
|
|
["/bin/sh", "-c", shell],
|
|
env={
|
|
"PATH": "/usr/bin:/bin",
|
|
"VC_LOCAL_ADDRESS": local_address,
|
|
"VC_LOCAL_DEVICE": local_device,
|
|
"VC_PLAIN_DEV": plain_device,
|
|
"VC_PLAIN_SRC": plain_source,
|
|
"VC_FROM_DEV": from_device,
|
|
"VC_FROM_SRC": from_source,
|
|
},
|
|
check=False, capture_output=True, text=True,
|
|
)
|
|
|
|
|
|
def test_native_apply_uses_local_endpoint_without_firewalld_metadata(tmp_path):
|
|
result = _run_native_apply_underlay_preflight(tmp_path)
|
|
assert result.returncode == 0, result.stderr
|
|
|
|
current = plan()
|
|
rendered = _apply_script(
|
|
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
|
|
firewall_mode=FIREWALL_MODE_NATIVE,
|
|
)
|
|
assert "LOCAL_ENDPOINT=203.0.113.1" in rendered
|
|
assert 'vc_underlay_route_path 203.0.113.2 "$LOCAL_ENDPOINT"' in rendered
|
|
assert "underlay|local|" not in rendered
|
|
assert "firewall_rule|" not in rendered
|
|
|
|
|
|
@pytest.mark.parametrize(("kwargs", "expected_status"), [
|
|
({"local_address": "192.0.2.99"}, 71),
|
|
({"plain_source": "192.0.2.99"}, 77),
|
|
({"from_source": "192.0.2.99"}, 77),
|
|
({"from_device": "eth1"}, 77),
|
|
])
|
|
def test_native_apply_rejects_nonlocal_or_inexact_underlay_path(
|
|
tmp_path, kwargs, expected_status,
|
|
):
|
|
result = _run_native_apply_underlay_preflight(tmp_path, **kwargs)
|
|
assert result.returncode == expected_status, result.stderr
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"interface", ["vnet3", "vcv-123", "tap3", "macvtap3", "fwpr3", "fwln3"],
|
|
)
|
|
def test_native_apply_rejects_guest_or_tenant_underlay_interface(
|
|
tmp_path, interface):
|
|
result = _run_native_apply_underlay_preflight(
|
|
tmp_path,
|
|
local_device=interface,
|
|
plain_device=interface,
|
|
from_device=interface,
|
|
)
|
|
assert result.returncode == 71, result.stderr
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_fabric_nft_validator_rejects_unowned_rules_and_extra_objects():
|
|
digest = "a" * 64
|
|
rules = _nft_rules_text(digest)
|
|
|
|
def validate(candidate: str) -> subprocess.CompletedProcess[str]:
|
|
return subprocess.run(
|
|
["/bin/sh", "-c", "\n".join((
|
|
"set -eu",
|
|
_FABRIC_NFT_VALIDATION_SH,
|
|
'vc_fabric_nft_strict_shape "$VC_NFT_TEXT"',
|
|
))],
|
|
env={"PATH": "/usr/bin:/bin", "VC_NFT_TEXT": candidate},
|
|
check=False, capture_output=True, text=True,
|
|
)
|
|
|
|
accepted = validate(rules)
|
|
assert accepted.returncode == 0, accepted.stderr
|
|
with_tenants = rules.replace(
|
|
" flags interval;",
|
|
" flags interval;\n"
|
|
" elements = { 10.10.0.2, 172.16.4.9/32, 192.168.1.8 }",
|
|
1,
|
|
)
|
|
accepted_tenants = validate(with_tenants)
|
|
assert accepted_tenants.returncode == 0, accepted_tenants.stderr
|
|
|
|
early_accept = rules.replace(
|
|
' iifname "wg-fabric0" ip protocol icmp accept',
|
|
' meta iifname "wg-fabric0" accept\n'
|
|
' iifname "wg-fabric0" ip protocol icmp accept',
|
|
1,
|
|
)
|
|
impossible_drop = rules.replace(
|
|
' iifname "wg-fabric0" counter drop',
|
|
' iifname "foreign0" iifname "wg-fabric0" counter drop',
|
|
1,
|
|
)
|
|
extra_chain = rules.replace(
|
|
" chain input {",
|
|
" chain bypass {\n type filter hook input priority -101; "
|
|
"policy accept;\n }\n chain input {",
|
|
1,
|
|
)
|
|
extra_set = rules.replace(
|
|
" set vpc_tenant_ips {",
|
|
" set foreign_ips { type ipv4_addr; }\n"
|
|
" set vpc_tenant_ips {",
|
|
1,
|
|
)
|
|
wrong_set_type = rules.replace("type ipv4_addr;", "type inet_service;", 1)
|
|
wrong_set_flags = rules.replace("flags interval;", "flags timeout;", 1)
|
|
extra_object = rules.replace(
|
|
" chain input {",
|
|
" counter foreign_counter { }\n chain input {",
|
|
1,
|
|
)
|
|
non_private_element = rules.replace(
|
|
" flags interval;",
|
|
" flags interval;\n elements = { 0.0.0.0/0 }",
|
|
1,
|
|
)
|
|
for tampered in (
|
|
early_accept, impossible_drop, extra_chain, extra_set,
|
|
wrong_set_type, wrong_set_flags, extra_object, non_private_element,
|
|
):
|
|
assert validate(tampered).returncode != 0
|
|
|
|
|
|
def test_observe_nft_guard_matches_exact_rendered_owner_comments():
|
|
current = plan()
|
|
observe = _observe_script(
|
|
(),
|
|
deployment_mode="ifupdown",
|
|
firewall_mode=FIREWALL_MODE_NATIVE,
|
|
expected_digest=current.digest,
|
|
)
|
|
owner_regex = (
|
|
f"""grep -Ec 'comment "{FABRIC_OWNER_MARKER}:[0-9a-f]{{64}}"'"""
|
|
)
|
|
digest_literal = (
|
|
f'grep -Foc "comment \\"{FABRIC_OWNER_MARKER}:$config_digest\\""'
|
|
)
|
|
assert owner_regex in observe
|
|
assert digest_literal in observe
|
|
|
|
guard_start = observe.index(
|
|
'if [ -n "$NFT" ]; then', observe.index("nft_guard=missing")
|
|
)
|
|
guard_end = observe.index("emergency_state=unknown", guard_start)
|
|
guard = observe[guard_start:guard_end]
|
|
program = "\n".join((
|
|
"set -eu",
|
|
_FABRIC_NFT_VALIDATION_SH,
|
|
"vc_test_nft() {",
|
|
' [ "$#" = 5 ] || return 1',
|
|
' [ "$1" = -s ] && [ "$2" = list ] && [ "$3" = table ] &&',
|
|
' [ "$4" = inet ] && [ "$5" = virt_cloud_wg_fabric ] || return 1',
|
|
' printf \'%s\\n\' "$VC_NFT_TEXT"',
|
|
"}",
|
|
"NFT=vc_test_nft",
|
|
f"config_digest={shlex.quote(current.digest)}",
|
|
"nft_guard=missing",
|
|
guard,
|
|
'printf \'%s\\n\' "$nft_guard"',
|
|
))
|
|
|
|
def readback(candidate):
|
|
return subprocess.run(
|
|
["/bin/sh", "-c", program],
|
|
env={"PATH": "/usr/bin:/bin", "VC_NFT_TEXT": candidate},
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
|
|
owned = readback(_nft_rules_text(current.digest))
|
|
assert owned.returncode == 0, owned.stderr
|
|
assert owned.stdout == "owned\n"
|
|
stale = readback(_nft_rules_text("0" * 64))
|
|
assert stale.returncode == 0, stale.stderr
|
|
assert stale.stdout == "missing\n"
|
|
|
|
|
|
def test_fabric_nft_validator_accepts_alma_output_dstnat_priority_only():
|
|
digest = "a" * 64
|
|
rules = _nft_rules_text(digest)
|
|
|
|
def validate(candidate: str) -> subprocess.CompletedProcess[str]:
|
|
return subprocess.run(
|
|
["/bin/sh", "-c", "\n".join((
|
|
"set -eu",
|
|
_FABRIC_NFT_VALIDATION_SH,
|
|
'vc_fabric_nft_strict_shape "$VC_NFT_TEXT"',
|
|
))],
|
|
env={"PATH": "/usr/bin:/bin", "VC_NFT_TEXT": candidate},
|
|
check=False, capture_output=True, text=True,
|
|
)
|
|
|
|
alma_canonical = rules.replace(
|
|
"type filter hook output priority -100; policy accept;",
|
|
"type filter hook output priority dstnat; policy accept;",
|
|
1,
|
|
)
|
|
accepted = validate(alma_canonical)
|
|
assert accepted.returncode == 0, accepted.stderr
|
|
|
|
for hook in ("input", "forward"):
|
|
invalid = rules.replace(
|
|
f"type filter hook {hook} priority -100; policy accept;",
|
|
f"type filter hook {hook} priority dstnat; policy accept;",
|
|
1,
|
|
)
|
|
assert validate(invalid).returncode != 0
|
|
|
|
|
|
def test_boot_recovery_accepts_alma_output_dstnat_priority():
|
|
digest = "a" * 64
|
|
alma_canonical = _nft_rules_text(digest).replace(
|
|
"type filter hook output priority -100; policy accept;",
|
|
"type filter hook output priority dstnat; policy accept;",
|
|
1,
|
|
)
|
|
helper = _BOOT_HELPER.replace(
|
|
"__VC_FABRIC_NFT_VALIDATION__", _FABRIC_NFT_VALIDATION_SH,
|
|
)
|
|
function_start = helper.index("vc_fabric_nft_strict_shape() {")
|
|
function_end = helper.index("\n}", function_start) + 2
|
|
validator = helper[function_start:function_end]
|
|
|
|
result = subprocess.run(
|
|
["/bin/sh", "-c", "\n".join((
|
|
"set -eu",
|
|
validator,
|
|
'vc_fabric_nft_strict_shape "$VC_NFT_TEXT"',
|
|
))],
|
|
env={"PATH": "/usr/bin:/bin", "VC_NFT_TEXT": alma_canonical},
|
|
check=False, capture_output=True, text=True,
|
|
)
|
|
|
|
assert result.returncode == 0, result.stderr
|
|
|
|
|
|
|
|
|
|
def _bridge_port_snapshot_runner(tmp_path, mode):
|
|
start = _BOOT_HELPER.index("vc_bridge_port_names() {")
|
|
end = _BOOT_HELPER.index("\n}\nvc_vpc_emergency_down()", start) + 2
|
|
runtime = tmp_path / "run"
|
|
runtime.mkdir()
|
|
function = _BOOT_HELPER[start:end].replace(
|
|
"/run/virt-cloud", str(runtime),
|
|
)
|
|
bridge = tmp_path / "bridge"
|
|
bridge.write_text(
|
|
"""#!/bin/sh
|
|
case "$VC_BRIDGE_MODE" in
|
|
failed) exit 1 ;;
|
|
malformed) printf '%s\n' 'not-a-bridge-link-record' ;;
|
|
duplicate)
|
|
printf '%s\n' '2: tap0@if3: <BROADCAST> master vcv-0123456789a'
|
|
printf '%s\n' '4: tap0@if5: <BROADCAST> master vcv-0123456789a' ;;
|
|
excessive)
|
|
i=0
|
|
while [ "$i" -lt 4097 ]; do
|
|
printf '%s: tap%s: <BROADCAST> master vcv-0123456789a\n' "$i" "$i"
|
|
i=$((i + 1))
|
|
done ;;
|
|
valid)
|
|
printf '%s\n' '4: vnet2@if5: <BROADCAST> master vcv-0123456789a'
|
|
printf '%s\n' '2: tap1@if3: <BROADCAST> master vcv-0123456789a' ;;
|
|
*) exit 2 ;;
|
|
esac
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
bridge.chmod(0o700)
|
|
stat = tmp_path / "stat"
|
|
stat.write_text(
|
|
"#!/bin/sh\n/usr/bin/stat -f '%z' \"$3\"\n",
|
|
encoding="utf-8",
|
|
)
|
|
stat.chmod(0o700)
|
|
script = "\n".join((
|
|
"set -eu",
|
|
f"BRIDGE_CMD='{bridge}'",
|
|
f"STAT='{stat}'",
|
|
"AWK=/usr/bin/awk",
|
|
function,
|
|
"vc_bridge_port_names vcv-0123456789a",
|
|
))
|
|
result = subprocess.run(
|
|
["/bin/sh", "-c", script],
|
|
env={"PATH": "/usr/bin:/bin", "VC_BRIDGE_MODE": mode},
|
|
check=False, capture_output=True, text=True,
|
|
)
|
|
return result, runtime, function
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"mode", ("failed", "malformed", "duplicate", "excessive"),
|
|
)
|
|
def test_bridge_port_snapshot_fails_closed_on_command_or_parse_errors(
|
|
tmp_path, mode,
|
|
):
|
|
result, runtime, _ = _bridge_port_snapshot_runner(tmp_path, mode)
|
|
|
|
assert result.returncode == 1, result.stderr
|
|
assert list(runtime.iterdir()) == []
|
|
|
|
|
|
def test_bridge_port_snapshot_sorts_valid_unique_names(tmp_path):
|
|
result, runtime, function = _bridge_port_snapshot_runner(
|
|
tmp_path, "valid",
|
|
)
|
|
|
|
assert result.returncode == 0, result.stderr
|
|
assert result.stdout == "tap1\nvnet2\n"
|
|
assert list(runtime.iterdir()) == []
|
|
assert '"$BRIDGE_CMD" link show master' in function
|
|
assert '"$BRIDGE_CMD" link show master "$vc_bridge_ports_bridge" |' \
|
|
not in _BOOT_HELPER
|
|
|
|
|
|
def _vpc_force_down_runner(tmp_path, *, link_up, nm_fails=False):
|
|
start = _BOOT_HELPER.index("vc_link_down_proven() {")
|
|
end = _BOOT_HELPER.index("\n}\nvc_vpc_emergency_down()", start) + 2
|
|
functions = _BOOT_HELPER[start:end]
|
|
ip = tmp_path / "ip"
|
|
ip.write_text(
|
|
"""#!/bin/sh
|
|
printf '%s\n' "$*" >> "$VC_IP_LOG"
|
|
case "$*" in
|
|
'link set dev vcv-0123456789a down') exit 0 ;;
|
|
'-o link show dev vcv-0123456789a')
|
|
if [ "$VC_LINK_UP" = yes ]; then
|
|
printf '%s\n' '2: vcv-0123456789a: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500'
|
|
else
|
|
printf '%s\n' '2: vcv-0123456789a: <BROADCAST,MULTICAST> mtu 1500'
|
|
fi
|
|
exit 0 ;;
|
|
*) exit 2 ;;
|
|
esac
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
ip.chmod(0o700)
|
|
nmcli = tmp_path / "nmcli"
|
|
nmcli.write_text("#!/bin/sh\nexit 1\n", encoding="utf-8")
|
|
nmcli.chmod(0o700)
|
|
log = tmp_path / "ip.log"
|
|
body = [
|
|
"set -eu",
|
|
f"IP='{ip}'",
|
|
f"NMCLI='{nmcli}'",
|
|
"AWK=/usr/bin/awk",
|
|
functions,
|
|
]
|
|
if nm_fails:
|
|
body.extend((
|
|
"nm_failed=no",
|
|
'"$NMCLI" device show vcv-0123456789a >/dev/null 2>&1 || '
|
|
"nm_failed=yes",
|
|
"vc_vpc_bridge_force_down vcv-0123456789a",
|
|
'[ "$nm_failed" = yes ]',
|
|
))
|
|
else:
|
|
body.append("vc_vpc_bridge_force_down vcv-0123456789a")
|
|
result = subprocess.run(
|
|
["/bin/sh", "-c", "\n".join(body)],
|
|
env={
|
|
"PATH": "/usr/bin:/bin",
|
|
"VC_IP_LOG": str(log),
|
|
"VC_LINK_UP": "yes" if link_up else "no",
|
|
},
|
|
check=False, capture_output=True, text=True,
|
|
)
|
|
return result, log
|
|
|
|
|
|
def test_vpc_force_down_rejects_nm_race_that_restores_up_flag(tmp_path):
|
|
result, log = _vpc_force_down_runner(tmp_path, link_up=True)
|
|
|
|
assert result.returncode == 1, result.stderr
|
|
assert log.read_text(encoding="utf-8").splitlines() == [
|
|
"link set dev vcv-0123456789a down",
|
|
"-o link show dev vcv-0123456789a",
|
|
]
|
|
|
|
|
|
def test_vpc_force_down_still_runs_and_proves_isolation_after_nm_failure(
|
|
tmp_path,
|
|
):
|
|
result, log = _vpc_force_down_runner(
|
|
tmp_path, link_up=False, nm_fails=True,
|
|
)
|
|
|
|
assert result.returncode == 0, result.stderr
|
|
assert log.read_text(encoding="utf-8").splitlines() == [
|
|
"link set dev vcv-0123456789a down",
|
|
"-o link show dev vcv-0123456789a",
|
|
]
|
|
early = _BOOT_HELPER.index("vc_vpc_emergency_managed=")
|
|
early_down = _BOOT_HELPER.index(
|
|
'vc_vpc_bridge_force_down "$vc_vpc_emergency_bridge"', early,
|
|
)
|
|
early_after = _BOOT_HELPER.index(
|
|
"vc_vpc_emergency_ports_after=", early_down,
|
|
)
|
|
assert early < early_down < early_after
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_apply_queues_reconcile_dispatcher_only_after_durable_commit():
|
|
current = plan()
|
|
script = _apply_script(
|
|
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
|
|
firewall_mode=FIREWALL_MODE_NATIVE,
|
|
)
|
|
committed_marker = script.rindex('committed > "$_txn_state_tmp"')
|
|
committed_flag = script.rindex("COMMITTED=yes")
|
|
unlock = script.rindex("exec 9>&-")
|
|
queue = script.rindex(
|
|
'"$SYSTEMCTL" start '
|
|
'virt-cloud-wg-fabric-reconcile.service'
|
|
)
|
|
relock = script.rindex('exec 9>>"$LOCK"')
|
|
primary_readback = script.rindex(
|
|
'is-active virt-cloud-wg-fabric.service 2>/dev/null')
|
|
emergency_readback = script.rindex(
|
|
'final_emergency_tables_verify=$($NFT list tables')
|
|
success = script.rindex("__VC_FABRIC_APPLIED__")
|
|
|
|
assert (
|
|
committed_marker < committed_flag < unlock < queue < relock
|
|
< primary_readback < emergency_readback < success
|
|
)
|
|
assert "Type=oneshot" in _UNIT
|
|
assert "RemainAfterExit=yes" in _UNIT
|
|
assert (
|
|
'"$SYSTEMCTL" start virt-cloud-wg-fabric.service'
|
|
not in script[committed_flag:]
|
|
)
|
|
assert "vc_primary_inactive_retry_ready()" in _BOOT_HELPER
|
|
assert "retry_primary_inactive=yes" in _BOOT_HELPER
|
|
primary = _BOOT_HELPER.index(
|
|
'"$SYSTEMCTL" start virt-cloud-wg-fabric.service')
|
|
active = _BOOT_HELPER.index(
|
|
'is-active virt-cloud-wg-fabric.service', primary)
|
|
daemon = _BOOT_HELPER.index(
|
|
'vc_gate_retry_start_claimed "$retry_gate_unit"', active)
|
|
assert primary < active < daemon
|
|
|
|
|
|
def _run_apply_convergence_tail(
|
|
tmp_path: Path, *, start_fails: bool = False,
|
|
timer_fails: bool = False,
|
|
stale_config: bool = False, emergency_present: bool = False,
|
|
committed_trap_state: str | None = None,
|
|
) -> subprocess.CompletedProcess[str]:
|
|
current = plan()
|
|
rendered = _apply_script(
|
|
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
|
|
firewall_mode=FIREWALL_MODE_NATIVE,
|
|
)
|
|
tail_start = rendered.rindex("COMMITTED=yes")
|
|
marker = "printf '%s\\n' '__VC_FABRIC_APPLIED__'"
|
|
tail_end = rendered.rindex(marker) + len(marker)
|
|
rollback_start = rendered.index("rollback() {")
|
|
rollback_end = rendered.index(
|
|
"\n}\ntrap rollback EXIT HUP INT TERM", rollback_start,
|
|
) + 2
|
|
rollback_function = rendered[rollback_start:rollback_end]
|
|
logger_start = rendered.index("vc_fabric_apply_failure_log() {")
|
|
logger_end = rendered.index("\n}\n", logger_start) + 2
|
|
failure_logger = rendered[logger_start:logger_end]
|
|
emergency_owner = tmp_path / "emergency.owner"
|
|
sysctl_file = tmp_path / "fabric-sysctl.conf"
|
|
sysctl_file.write_text(_sysctl_text(current), encoding="utf-8")
|
|
sysctl_file.chmod(0o644)
|
|
rp_filter_file = tmp_path / "rp_filter"
|
|
rp_filter_file.write_text("2\n", encoding="utf-8")
|
|
tail = rendered[tail_start:tail_end].replace(
|
|
"/run/virt-cloud/wg-fabric-emergency.owner",
|
|
str(emergency_owner),
|
|
).replace(
|
|
'"/proc/sys/net/ipv4/conf/$IFACE/rp_filter"',
|
|
f"'{rp_filter_file}'",
|
|
)
|
|
|
|
lock = tmp_path / "fabric.lock"
|
|
lock.touch(mode=0o600)
|
|
conf = tmp_path / "fabric.conf"
|
|
conf.write_text(
|
|
f"digest|{'0' * 64 if stale_config else current.digest}\n"
|
|
f"firewall|{FIREWALL_MODE_NATIVE}\n",
|
|
encoding="utf-8",
|
|
)
|
|
conf.chmod(0o600)
|
|
key = tmp_path / "fabric.key"
|
|
key.write_text("private\n", encoding="utf-8")
|
|
converged = tmp_path / "converged"
|
|
|
|
systemctl = tmp_path / "systemctl"
|
|
systemctl.write_text(
|
|
"""#!/bin/sh
|
|
case "$1:$2" in
|
|
start:virt-cloud-wg-fabric-reconcile.timer)
|
|
[ "${VC_TEST_TIMER_FAIL:-no}" = no ] || exit 1
|
|
exit 0 ;;
|
|
start:virt-cloud-wg-fabric-reconcile.service)
|
|
[ "${VC_TEST_START_FAIL:-no}" = no ] || exit 1
|
|
: > "$VC_TEST_CONVERGED"; exit 0 ;;
|
|
is-active:virt-cloud-wg-fabric.service|\
|
|
is-active:virt-cloud-wg-fabric-reconcile.timer)
|
|
[ -f "$VC_TEST_CONVERGED" ] || exit 1
|
|
printf '%s\n' active; exit 0 ;;
|
|
*) exit 2 ;;
|
|
esac
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
systemctl.chmod(0o700)
|
|
nft = tmp_path / "nft"
|
|
nft.write_text(
|
|
"""#!/bin/sh
|
|
case "$*" in
|
|
'-s list table inet virt_cloud_wg_fabric')
|
|
printf '%s\n' managed-fabric-table ;;
|
|
'list tables')
|
|
if [ "${VC_TEST_EMERGENCY:-no}" = yes ]; then
|
|
printf '%s\n' 'table inet virt_cloud_fabric_emergency'
|
|
printf '%s\n' 'table bridge virt_cloud_fabric_emergency'
|
|
else
|
|
printf '%s\n' 'table inet virt_cloud_wg_fabric'
|
|
fi ;;
|
|
*) exit 2 ;;
|
|
esac
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
nft.chmod(0o700)
|
|
wg = tmp_path / "wg"
|
|
wg.write_text(
|
|
"""#!/bin/sh
|
|
case "$*" in
|
|
'show wg-fabric0') exit 0 ;;
|
|
'pubkey') printf '%s\n' mock-public-key ;;
|
|
'show wg-fabric0 public-key') printf '%s\n' mock-public-key ;;
|
|
'show wg-fabric0 listen-port') printf '%s\n' 51820 ;;
|
|
*) exit 2 ;;
|
|
esac
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
wg.chmod(0o700)
|
|
ip = tmp_path / "ip"
|
|
ip.write_text(
|
|
"#!/bin/sh\n"
|
|
"[ \"$*\" = '-o link show dev wg-fabric0' ] || exit 2\n"
|
|
"printf '%s\\n' '1: wg-fabric0: <UP,LOWER_UP> mtu 1420'\n",
|
|
encoding="utf-8",
|
|
)
|
|
ip.chmod(0o700)
|
|
stat = tmp_path / "stat"
|
|
stat.write_text("#!/bin/sh\nprintf '%s\\n' '0:600'\n", encoding="utf-8")
|
|
stat.chmod(0o700)
|
|
flock = tmp_path / "flock"
|
|
flock.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8")
|
|
flock.chmod(0o700)
|
|
|
|
trap_setup: tuple[str, ...] = ()
|
|
if committed_trap_state is not None:
|
|
journal = tmp_path / "txn"
|
|
journal.mkdir()
|
|
(journal / "state").write_text(
|
|
f"{committed_trap_state}\n", encoding="utf-8",
|
|
)
|
|
trap_setup = (
|
|
f"TXN='{journal}'",
|
|
"JOURNAL_READY=yes",
|
|
"cleanup() { :; }",
|
|
rollback_function,
|
|
"trap rollback EXIT HUP INT TERM",
|
|
)
|
|
|
|
harness = "\n".join((
|
|
"set -eu",
|
|
f"LOCK='{lock}'",
|
|
f"CONF='{conf}'",
|
|
f"KEY='{key}'",
|
|
f"SYSCTL_FILE='{sysctl_file}'",
|
|
"IFACE=wg-fabric0",
|
|
f"EXPECTED_DIGEST='{current.digest}'",
|
|
f"FIREWALL_MODE='{FIREWALL_MODE_NATIVE}'",
|
|
f"SYSTEMCTL='{systemctl}'",
|
|
f"NFT='{nft}'",
|
|
f"WG='{wg}'",
|
|
f"IP='{ip}'",
|
|
f"STAT='{stat}'",
|
|
f"FLOCK='{flock}'",
|
|
"nft_plan_owned() { [ \"$2\" = \"$EXPECTED_DIGEST\" ]; }",
|
|
"vpc_state_nft_set_matches() { return 0; }",
|
|
"vc_fabric_sysctl_file_owned() { return 0; }",
|
|
"read_sysctl() { cat \"$1\"; }",
|
|
"vc_fabric_operation_checkpoint() { :; }",
|
|
failure_logger,
|
|
*trap_setup,
|
|
"exec 9>>\"$LOCK\"",
|
|
tail,
|
|
))
|
|
return subprocess.run(
|
|
["/bin/sh", "-c", harness], check=False,
|
|
capture_output=True, text=True,
|
|
env={
|
|
"PATH": "/usr/bin:/bin",
|
|
"VC_TEST_START_FAIL": "yes" if start_fails else "no",
|
|
"VC_TEST_TIMER_FAIL": "yes" if timer_fails else "no",
|
|
"VC_TEST_EMERGENCY": "yes" if emergency_present else "no",
|
|
"VC_TEST_CONVERGED": str(converged),
|
|
},
|
|
)
|
|
|
|
|
|
def test_apply_convergence_tail_requires_sync_success_and_final_clear(tmp_path):
|
|
success = _run_apply_convergence_tail(tmp_path)
|
|
assert success.returncode == 0, success.stderr
|
|
assert "__VC_FABRIC_APPLIED__" in success.stdout
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"failure",
|
|
["start", "stale_config", "emergency"],
|
|
)
|
|
def test_apply_convergence_tail_refuses_unconfirmed_state(tmp_path, failure):
|
|
result = _run_apply_convergence_tail(
|
|
tmp_path,
|
|
start_fails=failure == "start",
|
|
stale_config=failure == "stale_config",
|
|
emergency_present=failure == "emergency",
|
|
)
|
|
assert result.returncode == 79, result.stderr
|
|
assert "__VC_FABRIC_APPLIED__" not in result.stdout
|
|
|
|
|
|
def test_post_commit_dispatcher_failure_preserves_committed_journal(tmp_path):
|
|
result = _run_apply_convergence_tail(
|
|
tmp_path, start_fails=True, committed_trap_state="committed",
|
|
)
|
|
|
|
assert result.returncode == 79, result.stderr
|
|
assert (tmp_path / "txn" / "state").read_text(
|
|
encoding="utf-8",
|
|
) == "committed\n"
|
|
|
|
|
|
def test_post_commit_timer_failure_preserves_committed_journal(tmp_path):
|
|
result = _run_apply_convergence_tail(
|
|
tmp_path, timer_fails=True, committed_trap_state="committed",
|
|
)
|
|
assert result.returncode == 79, result.stderr
|
|
assert (tmp_path / "txn" / "state").read_text(
|
|
encoding="utf-8",
|
|
) == "committed\n"
|
|
|
|
|
|
def test_old_committed_trap_never_deletes_concurrent_ready_journal(tmp_path):
|
|
result = _run_apply_convergence_tail(
|
|
tmp_path, start_fails=True, committed_trap_state="ready",
|
|
)
|
|
|
|
assert result.returncode == 79, result.stderr
|
|
assert (tmp_path / "txn" / "state").read_text(
|
|
encoding="utf-8",
|
|
) == "ready\n"
|
|
|
|
|
|
def test_inactive_primary_retry_requires_complete_managed_evidence(tmp_path):
|
|
start = _BOOT_HELPER.index("vc_primary_inactive_retry_ready() {")
|
|
end = _BOOT_HELPER.index("\n}\nvc_fabric_rpf_repair_needed()", start) + 2
|
|
function = _BOOT_HELPER[start:end]
|
|
digest = "a" * 64
|
|
conf = tmp_path / "wg-fabric0.conf"
|
|
conf.write_text(
|
|
f"digest|{digest}\nfirewall|native_nftables\n", encoding="utf-8",
|
|
)
|
|
owner = tmp_path / "wg-fabric0.owner"
|
|
owner.write_text("virt-cloud-wg-fabric-v1\n", encoding="utf-8")
|
|
helper = tmp_path / "helper"
|
|
helper.write_text(
|
|
f"#!/bin/sh\n# virt-cloud-wg-fabric-v1\n"
|
|
f"# fabric-plan-digest={digest}\n",
|
|
encoding="utf-8",
|
|
)
|
|
unit = tmp_path / "unit"
|
|
unit.write_text(
|
|
"\n".join((
|
|
"# virt-cloud-wg-fabric-v1",
|
|
f"# fabric-plan-digest={digest}",
|
|
"Type=oneshot",
|
|
"RemainAfterExit=yes",
|
|
"ExecStart=/bin/sh /usr/local/libexec/virt-cloud-wg-fabric",
|
|
)) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
stat = tmp_path / "stat"
|
|
stat.write_text(
|
|
"""#!/bin/sh
|
|
case "$3" in
|
|
*wg-fabric0.conf|*wg-fabric0.owner) printf '%s\n' 0:600 ;;
|
|
*helper) printf '%s\n' 0:700 ;;
|
|
*unit) printf '%s\n' 0:644 ;;
|
|
*) exit 1 ;;
|
|
esac
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
stat.chmod(0o700)
|
|
systemctl = tmp_path / "systemctl"
|
|
systemctl.write_text(
|
|
"""#!/bin/sh
|
|
case "$1" in
|
|
is-active) printf '%s\n' inactive ;;
|
|
is-enabled) printf '%s\n' enabled ;;
|
|
*) exit 2 ;;
|
|
esac
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
systemctl.chmod(0o700)
|
|
wg = tmp_path / "wg"
|
|
wg.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8")
|
|
wg.chmod(0o700)
|
|
base_script = "\n".join((
|
|
"set -eu",
|
|
f"SYSTEMCTL='{systemctl}'",
|
|
f"STAT='{stat}'",
|
|
f"WG='{wg}'",
|
|
f"CONF='{conf}'",
|
|
f"OWNER='{owner}'",
|
|
f"HELPER='{helper}'",
|
|
f"UNIT='{unit}'",
|
|
f"PLAN_DIGEST='{digest}'",
|
|
"FIREWALL_MODE=native_nftables",
|
|
"IFACE=wg-fabric0",
|
|
function,
|
|
"vc_primary_inactive_retry_ready",
|
|
))
|
|
accepted = subprocess.run(
|
|
["/bin/sh", "-c", base_script], check=False,
|
|
capture_output=True, text=True,
|
|
)
|
|
assert accepted.returncode == 0, accepted.stderr
|
|
|
|
unit.write_text(unit.read_text(encoding="utf-8").replace(
|
|
"RemainAfterExit=yes", "RemainAfterExit=no",
|
|
), encoding="utf-8")
|
|
rejected = subprocess.run(
|
|
["/bin/sh", "-c", base_script], check=False,
|
|
capture_output=True, text=True,
|
|
)
|
|
assert rejected.returncode == 1
|
|
|
|
|
|
|
|
|
|
def test_fabric_nft_journal_records_only_post_refresh_as_applied():
|
|
current = plan()
|
|
script = _apply_script(
|
|
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
|
|
firewall_mode=FIREWALL_MODE_NATIVE,
|
|
)
|
|
installing = script.index("write_nft_phase installing")
|
|
installed_snapshot = script.index(
|
|
'"$nft_installed" > "$TXN/nft.installed.tmp"',
|
|
installing,
|
|
)
|
|
installed = script.index("write_nft_phase installed", installed_snapshot)
|
|
refreshing = script.index("write_nft_phase refreshing", installed)
|
|
refresh = script.index("vpc_state_refresh_nft || exit 71", refreshing)
|
|
exact_set = script.index("vpc_state_nft_set_matches || exit 71", refresh)
|
|
applied_snapshot = script.index(
|
|
'"$fabric_acl_applied" > "$TXN/nft.applied.tmp"',
|
|
exact_set,
|
|
)
|
|
refreshed = script.index("write_nft_phase refreshed", applied_snapshot)
|
|
|
|
assert (
|
|
installing < installed_snapshot < installed < refreshing < refresh
|
|
< exact_set < applied_snapshot < refreshed
|
|
)
|
|
assert "before_mutation|installing|installed|refreshing|refreshed" in script
|
|
assert 'case "$rollback_nft_phase" in' in script
|
|
assert 'vpc_state_nft_set_matches || ROLLBACK_FAILED=yes' in script
|
|
|
|
|
|
def test_boot_recovery_handles_both_sides_of_nft_refresh_fail_closed():
|
|
helper = _BOOT_HELPER
|
|
phase_read = helper.index(
|
|
'journal_nft_phase=$(cat "$JOURNAL/nft.phase"')
|
|
pre_refresh_snapshot = helper.index(
|
|
'journal_file_secure nft.installed || exit 79', phase_read)
|
|
refreshing_case = helper.index("refreshing)", pre_refresh_snapshot)
|
|
full_shape = helper.index(
|
|
'nft_plan_owned "$journal_refresh_text" "$journal_plan_digest"',
|
|
refreshing_case,
|
|
)
|
|
exact_set = helper.index("vpc_state_nft_set_matches || exit 79", full_shape)
|
|
stable_recheck = helper.index(
|
|
'"$CMP" -s "$JOURNAL/nft.current" '
|
|
'"$JOURNAL/nft.refresh-verify"',
|
|
exact_set,
|
|
)
|
|
final_snapshot = helper.index(
|
|
'"$CMP" -s "$JOURNAL/nft.current" "$JOURNAL/nft.applied"',
|
|
stable_recheck,
|
|
)
|
|
destructive_batch = helper.index(
|
|
'delete table inet virt_cloud_wg_fabric', final_snapshot)
|
|
|
|
assert (
|
|
phase_read < pre_refresh_snapshot < refreshing_case < full_shape
|
|
< exact_set < stable_recheck < final_snapshot < destructive_batch
|
|
)
|
|
assert (
|
|
"Crash between nft commit and its complete snapshot is\n"
|
|
" # intentionally ambiguous"
|
|
) in helper
|
|
assert (
|
|
'elif [ "$journal_nft_phase" = refreshing ] &&\n'
|
|
' { [ -e "$JOURNAL/nft.applied" ] || '
|
|
'[ -L "$JOURNAL/nft.applied" ]; }; then'
|
|
) in helper
|
|
assert (
|
|
'"$CMP" -s "$JOURNAL/nft.current" '
|
|
'"$JOURNAL/nft.applied" || exit 79'
|
|
) in helper
|
|
|
|
|
|
def test_first_deploy_recovery_uses_only_fenced_temporary_helper():
|
|
current = plan()
|
|
script = _apply_script(
|
|
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
|
|
firewall_mode=FIREWALL_MODE_NATIVE,
|
|
)
|
|
|
|
helper_created = script.index(
|
|
'_tmp_helper=$(mktemp "$BASE/.wg-fabric0.helper.XXXXXX")')
|
|
transaction = script.index("COMMITTED=no", helper_created)
|
|
ready_case = script.index(" ready)", transaction)
|
|
ready_end = script.index(" *) exit 79 ;;", ready_case)
|
|
helper_published = script.index(
|
|
'publish_desired_file "$_tmp_helper" "$HELPER" helper', ready_end)
|
|
ready_contract = script[ready_case:ready_end]
|
|
first_deploy = ready_contract.index(
|
|
'transaction_first_deploy_helper_absent=no')
|
|
missing_snapshot = ready_contract.index(
|
|
'transaction_missing_installing_snapshot=no')
|
|
special_first = ready_contract.index(
|
|
'if [ "$transaction_first_deploy_helper_absent" = yes ]; then',
|
|
missing_snapshot,
|
|
)
|
|
temporary_installing = ready_contract.index(
|
|
'VC_FABRIC_LOCK_HELD=1 /bin/sh "$_tmp_helper" '
|
|
'--recover-installing || exit 79',
|
|
special_first,
|
|
)
|
|
temporary_current = ready_contract.index(
|
|
'VC_FABRIC_LOCK_HELD=1 /bin/sh "$_tmp_helper" '
|
|
'--recover-current || exit 79',
|
|
temporary_installing,
|
|
)
|
|
fixed_helper_owned = ready_contract.index(
|
|
'transaction_recovery_helper_owned || exit 79',
|
|
temporary_current,
|
|
)
|
|
fixed_recovery = ready_contract.index(
|
|
'VC_FABRIC_LOCK_HELD=1 /bin/sh "$HELPER" --recover-only || exit 79',
|
|
fixed_helper_owned,
|
|
)
|
|
|
|
assert helper_created < transaction < ready_case < helper_published
|
|
assert (
|
|
first_deploy < missing_snapshot < special_first < temporary_installing
|
|
< temporary_current
|
|
< fixed_helper_owned < fixed_recovery
|
|
)
|
|
assert (
|
|
'VC_FABRIC_LOCK_HELD=1 /bin/sh "$HELPER" --recover-only || exit 79'
|
|
in ready_contract
|
|
)
|
|
assert (
|
|
'VC_FABRIC_LOCK_HELD=1 /bin/sh "$_tmp_helper" '
|
|
'--recover-installing || exit 79'
|
|
) in ready_contract
|
|
assert (
|
|
'VC_FABRIC_LOCK_HELD=1 /bin/sh "$_tmp_helper" '
|
|
'--recover-current || exit 79'
|
|
) in ready_contract
|
|
assert '"$_tmp_helper" --recover-only' not in ready_contract
|
|
assert '"$HELPER" --recover-installing' not in ready_contract
|
|
assert '"$HELPER" --recover-current' not in ready_contract
|
|
|
|
|
|
def _run_transaction_recovery_helper_owned_harness(
|
|
tmp_path, *, scenario,
|
|
):
|
|
current = plan()
|
|
apply = _apply_script(
|
|
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
|
|
firewall_mode=FIREWALL_MODE_NATIVE,
|
|
)
|
|
function_start = apply.index("transaction_recovery_helper_owned() {")
|
|
function_end = apply.index(
|
|
'\n}\nif [ -L "$BASE" ]', function_start,
|
|
) + 2
|
|
function = apply[function_start:function_end]
|
|
|
|
base = tmp_path / "base"
|
|
journal = base / ".wg-fabric-txn"
|
|
journal.mkdir(parents=True)
|
|
helper = tmp_path / "fixed-helper"
|
|
owned = b"#!/bin/sh\n# owned helper\n"
|
|
fixed = owned if scenario != "foreign_fixed_helper" else (
|
|
b"#!/bin/sh\n# foreign helper\n"
|
|
)
|
|
helper.write_bytes(fixed)
|
|
helper.chmod(0o700)
|
|
|
|
if scenario in ("baseline_exact", "foreign_fixed_helper",
|
|
"baseline_marker_drift"):
|
|
marker = journal / "helper.present"
|
|
marker.write_bytes(
|
|
b"drift\n" if scenario == "baseline_marker_drift" else b"",
|
|
)
|
|
marker.chmod(0o600)
|
|
baseline = journal / "helper"
|
|
baseline.write_bytes(owned)
|
|
baseline.chmod(0o700)
|
|
elif scenario in ("desired_exact", "desired_metadata_drift"):
|
|
desired = journal / "helper.desired"
|
|
desired.write_bytes(owned)
|
|
desired.chmod(0o700)
|
|
if scenario == "desired_metadata_drift":
|
|
marker = journal / "helper.desired.absent"
|
|
marker.write_bytes(b"")
|
|
marker.chmod(0o600)
|
|
else:
|
|
raise AssertionError(f"unknown helper scenario: {scenario}")
|
|
|
|
fake_stat = tmp_path / "stat"
|
|
fake_stat.write_text(
|
|
"""#!/bin/sh
|
|
case "$3" in
|
|
*/helper.present) printf '0:600\n' ;;
|
|
*/helper|*/helper.desired|*/fixed-helper) printf '0:700\n' ;;
|
|
*) exit 2 ;;
|
|
esac
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
fake_stat.chmod(0o700)
|
|
script = "\n".join((
|
|
"set -eu",
|
|
f"BASE={shlex.quote(str(base))}",
|
|
f"HELPER={shlex.quote(str(helper))}",
|
|
f"STAT={shlex.quote(str(fake_stat))}",
|
|
"CMP=/usr/bin/cmp",
|
|
function,
|
|
"if transaction_recovery_helper_owned; then",
|
|
" exit 0",
|
|
"fi",
|
|
"exit 17",
|
|
))
|
|
return subprocess.run(
|
|
["/bin/sh", "-c", script], check=False,
|
|
capture_output=True, text=True,
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize("scenario", ("baseline_exact", "desired_exact"))
|
|
def test_transaction_recovery_fixed_helper_accepts_exact_journal_image(
|
|
tmp_path, scenario,
|
|
):
|
|
result = _run_transaction_recovery_helper_owned_harness(
|
|
tmp_path, scenario=scenario,
|
|
)
|
|
assert result.returncode == 0, result.stderr
|
|
|
|
|
|
@pytest.mark.parametrize("scenario", (
|
|
"foreign_fixed_helper",
|
|
"baseline_marker_drift",
|
|
"desired_metadata_drift",
|
|
))
|
|
def test_transaction_recovery_fixed_helper_rejects_unproven_image(
|
|
tmp_path, scenario,
|
|
):
|
|
result = _run_transaction_recovery_helper_owned_harness(
|
|
tmp_path, scenario=scenario,
|
|
)
|
|
assert result.returncode == 17
|
|
|
|
|
|
def _run_ready_recovery_dispatch_harness(
|
|
tmp_path, *, first_deploy, phase, installed, deny_direct_exec=False,
|
|
):
|
|
current = plan()
|
|
apply = _apply_script(
|
|
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
|
|
firewall_mode=FIREWALL_MODE_NATIVE,
|
|
)
|
|
function_start = apply.index("transaction_recovery_helper_owned() {")
|
|
function_end = apply.index(
|
|
'\n}\nif [ -L "$BASE" ]', function_start,
|
|
) + 2
|
|
function = apply[function_start:function_end]
|
|
ready = apply.index(" ready)", function_end)
|
|
ready_body_start = apply.index("\n", ready) + 1
|
|
ready_body_end = apply.index(
|
|
" ;;\n *) exit 79 ;;", ready_body_start,
|
|
)
|
|
ready_body = apply[ready_body_start:ready_body_end]
|
|
|
|
base = tmp_path / "base"
|
|
journal = base / ".wg-fabric-txn"
|
|
journal.mkdir(parents=True)
|
|
phase_file = journal / "nft.phase"
|
|
phase_file.write_text(f"{phase}\n", encoding="utf-8")
|
|
phase_file.chmod(0o600)
|
|
if installed:
|
|
installed_file = journal / "nft.installed"
|
|
installed_file.write_bytes(b"complete nft snapshot\n")
|
|
installed_file.chmod(0o600)
|
|
|
|
log = tmp_path / "dispatch.log"
|
|
digest = current.digest
|
|
|
|
def helper_text(label, options):
|
|
cases = "|".join(options)
|
|
return (
|
|
"#!/bin/sh\n"
|
|
"# virt-cloud-wg-fabric-v1\n"
|
|
f"# fabric-plan-digest={digest}\n"
|
|
f'PLAN_DIGEST="{digest}"\n'
|
|
f"# accepted recovery arguments: {cases}\n"
|
|
f"case \"${{1:-}}\" in {cases}) : ;; *) exit 31 ;; esac\n"
|
|
f"printf '%s:%s\\n' {shlex.quote(label)} \"$1\" > "
|
|
f"{shlex.quote(str(log))}\n"
|
|
f"rm -rf {shlex.quote(str(journal))}\n"
|
|
)
|
|
|
|
temporary = tmp_path / "temporary-helper"
|
|
temporary.write_text(
|
|
helper_text("temporary", ("--recover-current",
|
|
"--recover-installing")),
|
|
encoding="utf-8",
|
|
)
|
|
temporary.chmod(0o600 if deny_direct_exec else 0o700)
|
|
fixed = tmp_path / "fixed-helper"
|
|
if first_deploy:
|
|
marker = journal / "helper.absent"
|
|
marker.write_bytes(b"")
|
|
marker.chmod(0o600)
|
|
else:
|
|
fixed_bytes = helper_text("fixed", ("--recover-only",)).encode()
|
|
fixed.write_bytes(fixed_bytes)
|
|
fixed.chmod(0o600 if deny_direct_exec else 0o700)
|
|
marker = journal / "helper.present"
|
|
marker.write_bytes(b"")
|
|
marker.chmod(0o600)
|
|
baseline = journal / "helper"
|
|
baseline.write_bytes(fixed_bytes)
|
|
baseline.chmod(0o700)
|
|
|
|
fake_stat = tmp_path / "stat"
|
|
fake_stat.write_text(
|
|
f"""#!{sys.executable}
|
|
import os
|
|
import stat
|
|
import sys
|
|
|
|
if len(sys.argv) != 4 or sys.argv[1] != "-c":
|
|
raise SystemExit(2)
|
|
value = os.lstat(sys.argv[3])
|
|
mode = format(stat.S_IMODE(value.st_mode), "o")
|
|
if {deny_direct_exec!r} and os.path.basename(sys.argv[3]) in (
|
|
"temporary-helper", "fixed-helper"):
|
|
# Model an executable root-owned file on a noexec mount: its metadata
|
|
# passes the contract while the OS rejects a direct exec.
|
|
mode = "700"
|
|
if sys.argv[2] == "%a":
|
|
print(mode)
|
|
elif sys.argv[2] == "%u":
|
|
print("0")
|
|
elif sys.argv[2] == "%u:%a":
|
|
print(f"0:{{mode}}")
|
|
else:
|
|
raise SystemExit(2)
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
fake_stat.chmod(0o700)
|
|
|
|
harness = "\n".join((
|
|
"set -eu",
|
|
f"BASE={shlex.quote(str(base))}",
|
|
f"HELPER={shlex.quote(str(fixed))}",
|
|
f"_tmp_helper={shlex.quote(str(temporary))}",
|
|
f"STAT={shlex.quote(str(fake_stat))}",
|
|
"CMP=/usr/bin/cmp",
|
|
f"EXPECTED_DIGEST={shlex.quote(digest)}",
|
|
"TRANSACTION_RECOVERED=no",
|
|
"vc_fabric_operation_checkpoint() { :; }",
|
|
function,
|
|
ready_body,
|
|
'[ "$TRANSACTION_RECOVERED" = yes ]',
|
|
))
|
|
result = subprocess.run(
|
|
["/bin/sh", "-c", harness], check=False,
|
|
capture_output=True, text=True,
|
|
)
|
|
return result, log
|
|
|
|
|
|
@pytest.mark.parametrize(("phase", "installed", "expected"), (
|
|
("before_mutation", False, "--recover-current"),
|
|
("installing", False, "--recover-installing"),
|
|
("installing", True, "--recover-current"),
|
|
("installed", True, "--recover-current"),
|
|
("refreshing", True, "--recover-current"),
|
|
("refreshed", True, "--recover-current"),
|
|
))
|
|
def test_first_deploy_ready_dispatch_uses_current_plan_helper(
|
|
tmp_path, phase, installed, expected,
|
|
):
|
|
result, log = _run_ready_recovery_dispatch_harness(
|
|
tmp_path, first_deploy=True, phase=phase, installed=installed,
|
|
)
|
|
assert result.returncode == 0, result.stderr
|
|
assert log.read_text(encoding="utf-8") == f"temporary:{expected}\n"
|
|
|
|
|
|
@pytest.mark.parametrize(("first_deploy", "phase", "installed", "expected"), (
|
|
(True, "installing", False, "temporary:--recover-installing"),
|
|
(True, "installed", True, "temporary:--recover-current"),
|
|
(False, "installed", True, "fixed:--recover-only"),
|
|
))
|
|
def test_ready_recovery_uses_interpreter_when_direct_execution_is_denied(
|
|
tmp_path, first_deploy, phase, installed, expected):
|
|
result, log = _run_ready_recovery_dispatch_harness(
|
|
tmp_path, first_deploy=first_deploy, phase=phase, installed=installed,
|
|
deny_direct_exec=True)
|
|
|
|
helper = tmp_path / (
|
|
"temporary-helper" if first_deploy else "fixed-helper")
|
|
with pytest.raises(PermissionError):
|
|
subprocess.run([str(helper)], check=False, capture_output=True)
|
|
assert result.returncode == 0, result.stderr
|
|
assert log.read_text(encoding="utf-8") == expected + "\n"
|
|
|
|
|
|
@pytest.mark.parametrize(("phase", "installed"), (
|
|
("before_mutation", False),
|
|
("installing", False),
|
|
("installing", True),
|
|
("installed", True),
|
|
("refreshing", True),
|
|
("refreshed", True),
|
|
))
|
|
def test_existing_ready_dispatch_uses_exact_owned_fixed_helper(
|
|
tmp_path, phase, installed,
|
|
):
|
|
result, log = _run_ready_recovery_dispatch_harness(
|
|
tmp_path, first_deploy=False, phase=phase, installed=installed,
|
|
)
|
|
assert result.returncode == 0, result.stderr
|
|
assert log.read_text(encoding="utf-8") == "fixed:--recover-only\n"
|
|
|
|
|
|
def test_installing_recovery_requires_exact_stable_first_deploy_evidence():
|
|
current = plan()
|
|
script = _apply_script(
|
|
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
|
|
firewall_mode=FIREWALL_MODE_NATIVE,
|
|
)
|
|
recover_start = script.index("recover_journal() {")
|
|
recover_end = script.index(
|
|
'\n}\nif [ "${VC_FABRIC_APPLYING:-0}" != 1 ]; then',
|
|
recover_start,
|
|
) + 2
|
|
recover = script[recover_start:recover_end]
|
|
|
|
digest_fence = recover.index(
|
|
'if [ "${RECOVER_ONLY:-no}" != yes ] ||\n'
|
|
' [ "${RECOVER_CURRENT:-no}" = yes ]; then')
|
|
digest_match = recover.index(
|
|
'[ "$journal_plan_digest" = "$PLAN_DIGEST" ]', digest_fence)
|
|
connection_match = recover.index(
|
|
'[ "$journal_connection_uuid" = "$CONNECTION_UUID" ]',
|
|
digest_match,
|
|
)
|
|
deployment_match = recover.index(
|
|
'[ "$journal_mode" = "$DEPLOYMENT_MODE" ]', connection_match)
|
|
firewall_match = recover.index(
|
|
'[ "$journal_firewall_mode" = "$FIREWALL_MODE" ]',
|
|
deployment_match,
|
|
)
|
|
special = recover.index(
|
|
'if [ "${RECOVER_INSTALLING:-no}" = yes ]; then', firewall_match)
|
|
|
|
phase_gate = recover.index(
|
|
'[ "$journal_nft_phase" = installing ]', special)
|
|
absent_baseline = recover.index(
|
|
'[ "$journal_nft_before" = no ]', phase_gate)
|
|
clean_first = recover.index(
|
|
'for journal_clean_first in conf owner helper unit repair timer '
|
|
'gate_generator nm_profile ifupdown_fragment sysctl nft_rules; do',
|
|
absent_baseline,
|
|
)
|
|
clean_markers = recover.index(
|
|
'journal_marker_secure "$journal_clean_first.absent" || exit 79',
|
|
clean_first,
|
|
)
|
|
desired_rules = recover.index(
|
|
'journal_file_secure nft_rules.desired || exit 79', clean_markers)
|
|
desired_mode = recover.index(
|
|
'$STAT -c \'%u:%a\' "$JOURNAL/nft_rules.desired"',
|
|
desired_rules,
|
|
)
|
|
desired_sha = recover.index(
|
|
'"$SHA256" "$JOURNAL/nft_rules.desired"', desired_mode)
|
|
expected_sha = recover.index(
|
|
'[ "$journal_desired_nft_sha" = '
|
|
'"$EXPECTED_NFT_RULES_SHA256" ] || exit 79',
|
|
desired_sha,
|
|
)
|
|
desired_shape = recover.index(
|
|
'nft_plan_owned "$journal_desired_nft" '
|
|
'"$journal_plan_digest" || exit 79',
|
|
expected_sha,
|
|
)
|
|
desired_empty = recover.index(
|
|
'vc_fabric_nft_empty_tenant_set "$journal_desired_nft" '
|
|
'|| exit 79',
|
|
desired_shape,
|
|
)
|
|
owner_artifact = recover.index(
|
|
'journal_first_deploy_artifact_safe "$OWNER" owner || exit 79',
|
|
desired_empty,
|
|
)
|
|
sysctl_artifact = recover.index(
|
|
'journal_first_deploy_artifact_safe "$SYSCTL_FILE" sysctl '
|
|
'|| exit 79',
|
|
owner_artifact,
|
|
)
|
|
nft_rules_artifact = recover.index(
|
|
'journal_first_deploy_artifact_safe "$NFT_RULES" nft_rules '
|
|
'|| exit 79',
|
|
sysctl_artifact,
|
|
)
|
|
native_baseline = recover.index(
|
|
'journal_native_nft_main_safe || exit 79', nft_rules_artifact,
|
|
)
|
|
current_owner = recover.index(
|
|
'journal_current_owner_state=absent', native_baseline)
|
|
installing_owner_gate = recover.index(
|
|
'case "$journal_current_owner_state:'
|
|
'$journal_current_iface_state" in',
|
|
current_owner,
|
|
)
|
|
|
|
first_read = recover.index(
|
|
'$NFT -s list table inet virt_cloud_wg_fabric > '
|
|
'"$JOURNAL/nft.current"',
|
|
desired_empty,
|
|
)
|
|
live_shape = recover.index(
|
|
'nft_plan_owned "$journal_installing_text" '
|
|
'"$journal_plan_digest" || exit 79',
|
|
first_read,
|
|
)
|
|
live_empty = recover.index(
|
|
'vc_fabric_nft_empty_tenant_set "$journal_installing_text" '
|
|
'|| exit 79',
|
|
live_shape,
|
|
)
|
|
second_read = recover.index(
|
|
'"$NFT" -s list table inet virt_cloud_wg_fabric > '
|
|
'"$_tmp_nft_installed"',
|
|
live_empty,
|
|
)
|
|
stable_compare = recover.index(
|
|
'"$CMP" -s "$JOURNAL/nft.current" '
|
|
'"$_tmp_nft_installed" || exit 79',
|
|
second_read,
|
|
)
|
|
atomic_snapshot = recover.index(
|
|
'mv -f "$_tmp_nft_installed" "$JOURNAL/nft.installed"',
|
|
stable_compare,
|
|
)
|
|
installed_compare = recover.index(
|
|
'"$CMP" -s "$JOURNAL/nft.current" '
|
|
'"$JOURNAL/nft.installed"',
|
|
atomic_snapshot,
|
|
)
|
|
expected_file = recover.index(
|
|
'journal_nft_expected_file="$JOURNAL/nft.installed"',
|
|
installed_compare,
|
|
)
|
|
allow_replace = recover.index(
|
|
"journal_nft_action=replace", expected_file)
|
|
verify_read = recover.index(
|
|
'$NFT -s list table inet virt_cloud_wg_fabric > '
|
|
'"$JOURNAL/nft.verify"',
|
|
allow_replace,
|
|
)
|
|
verify_compare = recover.index(
|
|
'"$CMP" -s "$JOURNAL/nft.verify" '
|
|
'"$journal_nft_expected_file" || exit 79',
|
|
verify_read,
|
|
)
|
|
precommit_read = recover.index(
|
|
'$NFT -s list table inet virt_cloud_wg_fabric > '
|
|
'"$JOURNAL/nft.precommit"',
|
|
verify_compare,
|
|
)
|
|
precommit_compare = recover.index(
|
|
'"$CMP" -s "$JOURNAL/nft.precommit" '
|
|
'"$journal_nft_expected_file" || exit 79',
|
|
precommit_read,
|
|
)
|
|
destructive_apply = recover.index(
|
|
'"$NFT" -f "$_tmp_nft_rules"', precommit_compare)
|
|
|
|
assert (
|
|
digest_fence < digest_match < connection_match < deployment_match
|
|
< firewall_match < special < phase_gate < absent_baseline < clean_first
|
|
< clean_markers < desired_rules < desired_mode < desired_sha < expected_sha
|
|
< desired_shape < desired_empty < owner_artifact < sysctl_artifact
|
|
< nft_rules_artifact < native_baseline < current_owner
|
|
< installing_owner_gate < first_read < live_shape
|
|
< live_empty < second_read < stable_compare < atomic_snapshot
|
|
< installed_compare < expected_file < allow_replace < verify_read
|
|
< verify_compare < precommit_read < precommit_compare
|
|
< destructive_apply
|
|
)
|
|
|
|
|
|
def _installing_recovery_fragments():
|
|
helper = _BOOT_HELPER.replace(
|
|
"__VC_FABRIC_NFT_VALIDATION__", _FABRIC_NFT_VALIDATION_SH,
|
|
)
|
|
ownership_start = helper.index("nft_panel_owned() {")
|
|
ownership_end = helper.index("\nrecover_journal() {", ownership_start)
|
|
ownership = helper[ownership_start:ownership_end]
|
|
|
|
artifact_start = helper.index("journal_file_secure() {")
|
|
artifact_end = helper.index(
|
|
"\njournal_cleanup_helper_owned() {", artifact_start)
|
|
artifacts = helper[artifact_start:artifact_end]
|
|
|
|
special_start = helper.index(
|
|
' if [ "${RECOVER_INSTALLING:-no}" = yes ]; then')
|
|
special_end = helper.index(
|
|
" journal_applied_present=no", special_start)
|
|
special = helper[special_start:special_end]
|
|
|
|
current_owner_start = helper.index(
|
|
" journal_current_owner_state=absent", special_end)
|
|
current_owner_end = helper.index(
|
|
" journal_nmcli=", current_owner_start)
|
|
current_owner = helper[current_owner_start:current_owner_end]
|
|
|
|
snapshot_start = helper.index(
|
|
' if [ "$journal_recover_installing" = yes ]; then')
|
|
snapshot_end = helper.index(
|
|
' if [ -f "$JOURNAL/nft.installed" ] &&',
|
|
snapshot_start,
|
|
)
|
|
snapshot = helper[snapshot_start:snapshot_end]
|
|
proof_end = helper.index("\n installed)", snapshot_end)
|
|
installed_proof = helper[snapshot_end:proof_end]
|
|
assert re.search(r"fi ;;\s*$", installed_proof)
|
|
installed_proof = re.sub(
|
|
r"fi ;;\s*$", "fi", installed_proof,
|
|
)
|
|
return artifacts, ownership, special, current_owner, snapshot, installed_proof
|
|
|
|
|
|
def _run_installing_recovery_harness(
|
|
tmp_path, *, failure="none", artifact_state="post_trap_absent",
|
|
firewall_mode=FIREWALL_MODE_NATIVE,
|
|
native_state="baseline_current_no_desired",
|
|
):
|
|
digest = "a" * 64
|
|
desired = _nft_rules_text(digest)
|
|
if failure == "nonempty_tenant_set":
|
|
desired = desired.replace(
|
|
" flags interval;",
|
|
" flags interval;\n"
|
|
" elements = { 10.10.0.2 }",
|
|
1,
|
|
)
|
|
elif failure == "extra_object":
|
|
desired = desired.replace(
|
|
" set vpc_tenant_ips {",
|
|
" set foreign_ips { type ipv4_addr; }\n"
|
|
" set vpc_tenant_ips {",
|
|
1,
|
|
)
|
|
desired_bytes = desired.encode()
|
|
expected_sha = hashlib.sha256(desired_bytes).hexdigest()
|
|
if failure == "desired_hash":
|
|
expected_sha = "0" * 64
|
|
|
|
alma_live = _nft_rules_text(digest).replace(
|
|
"type filter hook output priority -100; policy accept;",
|
|
"type filter hook output priority dstnat; policy accept;",
|
|
1,
|
|
)
|
|
second_live = alma_live
|
|
if failure == "second_read_drift":
|
|
second_live = alma_live.replace(
|
|
"type filter hook output priority dstnat; policy accept;",
|
|
"type filter hook output priority -100; policy accept;",
|
|
1,
|
|
)
|
|
|
|
journal = tmp_path / "journal"
|
|
journal.mkdir(mode=0o700)
|
|
clean_first = (
|
|
"conf", "owner", "helper", "unit", "repair", "timer",
|
|
"gate_generator", "nm_profile", "ifupdown_fragment", "sysctl",
|
|
"nft_rules",
|
|
)
|
|
for name in clean_first:
|
|
marker = journal / f"{name}.absent"
|
|
marker.write_bytes(b"")
|
|
marker.chmod(0o600)
|
|
nft_main_marker = journal / (
|
|
"nft_main.present"
|
|
if firewall_mode == FIREWALL_MODE_NATIVE
|
|
else "nft_main.absent"
|
|
)
|
|
nft_main_marker.write_bytes(b"")
|
|
nft_main_marker.chmod(0o600)
|
|
nft_before = journal / "nft.table"
|
|
nft_before.write_bytes(b"")
|
|
nft_before.chmod(0o600)
|
|
desired_path = journal / "nft_rules.desired"
|
|
desired_path.write_bytes(desired_bytes)
|
|
desired_path.chmod(0o600)
|
|
evidence = journal / "operator.evidence"
|
|
evidence.write_bytes(b"do-not-overwrite\n")
|
|
evidence.chmod(0o600)
|
|
|
|
targets_dir = tmp_path / "targets"
|
|
targets_dir.mkdir()
|
|
targets = {
|
|
"conf": targets_dir / "wg-fabric0.conf",
|
|
"owner": targets_dir / "wg-fabric0.owner",
|
|
"helper": targets_dir / "virt-cloud-wg-fabric",
|
|
"unit": targets_dir / "virt-cloud-wg-fabric.service",
|
|
"repair": targets_dir / "virt-cloud-wg-fabric-reconcile.service",
|
|
"timer": targets_dir / "virt-cloud-wg-fabric-reconcile.timer",
|
|
"gate_generator": targets_dir / "virt-cloud-wg-fabric-gate",
|
|
"nm_profile": targets_dir / "virt-cloud-wg-fabric0.nmconnection",
|
|
"ifupdown_fragment": targets_dir / "virt-cloud-wg-fabric0",
|
|
"sysctl": targets_dir / "90-virt-cloud-wg-fabric.conf",
|
|
"nft_rules": targets_dir / "virt-cloud-wg-fabric.nft",
|
|
"nft_main": targets_dir / "nftables.conf",
|
|
}
|
|
artifact_values = {
|
|
"owner": b"virt-cloud-wg-fabric-v1\n",
|
|
"sysctl": b"net.ipv4.ip_forward = 1\n",
|
|
"nft_rules": desired_bytes,
|
|
"nft_main": b'include "/etc/nftables.d/*.nft"\n',
|
|
}
|
|
for name, value in artifact_values.items():
|
|
if name in ("nft_rules", "nft_main"):
|
|
continue
|
|
desired_artifact = journal / f"{name}.desired"
|
|
desired_artifact.write_bytes(value)
|
|
desired_artifact.chmod(0o600)
|
|
|
|
if artifact_state == "post_trap_absent":
|
|
present_artifacts = set()
|
|
elif artifact_state == "hard_crash_exact_desired":
|
|
present_artifacts = set(artifact_values)
|
|
elif artifact_state == "mixed":
|
|
present_artifacts = {"owner", "nft_rules"}
|
|
else:
|
|
raise AssertionError(f"unknown artifact state: {artifact_state}")
|
|
if failure == "artifact_foreign_bytes":
|
|
present_artifacts.add("sysctl")
|
|
elif failure == "artifact_wrong_mode":
|
|
present_artifacts.add("nft_rules")
|
|
elif failure == "artifact_symlink":
|
|
present_artifacts.add("nft_main")
|
|
elif failure in ("artifact_desired_absent", "owner_iface_present"):
|
|
present_artifacts.add("owner")
|
|
|
|
artifact_symlink_target = tmp_path / "artifact-symlink-target"
|
|
for name in present_artifacts:
|
|
target = targets[name]
|
|
if failure == "artifact_symlink" and name == "nft_main":
|
|
artifact_symlink_target.write_bytes(artifact_values[name])
|
|
target.symlink_to(artifact_symlink_target)
|
|
continue
|
|
value = artifact_values[name]
|
|
if failure == "artifact_foreign_bytes" and name == "sysctl":
|
|
value = b"foreign sysctl bytes\n"
|
|
target.write_bytes(value)
|
|
target.chmod(
|
|
0o644
|
|
if failure == "artifact_wrong_mode" and name == "nft_rules"
|
|
else 0o600
|
|
)
|
|
if failure == "artifact_desired_absent":
|
|
desired_absent = journal / "owner.desired.absent"
|
|
desired_absent.write_bytes(b"")
|
|
desired_absent.chmod(0o600)
|
|
|
|
if firewall_mode == FIREWALL_MODE_NATIVE:
|
|
native_baseline = (
|
|
b"# baseline\ninclude \"/etc/nftables.d/*.nft\"\n"
|
|
)
|
|
native_desired = (
|
|
b"# desired\n"
|
|
b"include \"/etc/nftables.d/virt-cloud-wg-fabric.nft\"\n"
|
|
)
|
|
baseline = journal / "nft_main"
|
|
native_snapshot_target = tmp_path / "native-snapshot-target"
|
|
if failure == "native_baseline_symlink":
|
|
native_snapshot_target.write_bytes(native_baseline)
|
|
baseline.symlink_to(native_snapshot_target)
|
|
elif failure != "native_baseline_missing":
|
|
baseline.write_bytes(
|
|
b"# invalid baseline\n"
|
|
if failure == "native_baseline_content"
|
|
else native_baseline
|
|
)
|
|
baseline.chmod(
|
|
0o666 if failure == "native_baseline_mode" else 0o600
|
|
)
|
|
|
|
desired_needed = native_state != "baseline_current_no_desired" or (
|
|
failure in {
|
|
"native_desired_symlink",
|
|
"native_desired_mode",
|
|
"native_desired_content",
|
|
"native_desired_absent_conflict",
|
|
}
|
|
)
|
|
desired_main = journal / "nft_main.desired"
|
|
native_desired_target = tmp_path / "native-desired-target"
|
|
if desired_needed:
|
|
if failure == "native_desired_symlink":
|
|
native_desired_target.write_bytes(native_desired)
|
|
desired_main.symlink_to(native_desired_target)
|
|
else:
|
|
desired_main.write_bytes(
|
|
b"# invalid desired\n"
|
|
if failure == "native_desired_content"
|
|
else native_desired
|
|
)
|
|
desired_main.chmod(
|
|
0o666 if failure == "native_desired_mode" else 0o600
|
|
)
|
|
if failure in ("native_marker_conflict",
|
|
"native_desired_absent_conflict"):
|
|
conflict = journal / (
|
|
"nft_main.absent"
|
|
if failure == "native_marker_conflict"
|
|
else "nft_main.desired.absent"
|
|
)
|
|
conflict.write_bytes(b"")
|
|
conflict.chmod(0o600)
|
|
|
|
live = targets["nft_main"]
|
|
native_live_target = tmp_path / "native-live-target"
|
|
if failure == "native_live_symlink":
|
|
native_live_target.write_bytes(native_baseline)
|
|
live.symlink_to(native_live_target)
|
|
elif failure != "native_live_missing":
|
|
live_value = (
|
|
native_desired
|
|
if native_state == "desired_current"
|
|
else native_baseline
|
|
)
|
|
if failure == "native_live_foreign":
|
|
live_value = b"# foreign live\ninclude \"/etc/nftables.d/*.nft\"\n"
|
|
live.write_bytes(live_value)
|
|
live.chmod(
|
|
0o644 if failure == "native_live_mode" else 0o600
|
|
)
|
|
else:
|
|
native_snapshot_target = tmp_path / "unused-native-snapshot"
|
|
native_desired_target = tmp_path / "unused-native-desired"
|
|
native_live_target = tmp_path / "unused-native-live"
|
|
desired_main = journal / "nft_main.desired"
|
|
desired_main.write_bytes(artifact_values["nft_main"])
|
|
desired_main.chmod(0o600)
|
|
|
|
installed_target = tmp_path / "installed-target"
|
|
if failure == "installed_symlink":
|
|
installed_target.write_bytes(b"foreign-snapshot\n")
|
|
(journal / "nft.installed").symlink_to(installed_target)
|
|
|
|
first_output = tmp_path / "nft-first"
|
|
first_output.write_text(alma_live, encoding="utf-8")
|
|
second_output = tmp_path / "nft-second"
|
|
second_output.write_text(second_live, encoding="utf-8")
|
|
call_count = tmp_path / "nft-count"
|
|
nft = tmp_path / "nft"
|
|
nft.write_text(
|
|
"""#!/bin/sh
|
|
[ "$*" = "-s list table inet virt_cloud_wg_fabric" ] || exit 2
|
|
vc_count=0
|
|
if [ -f "$VC_NFT_CALL_COUNT" ]; then
|
|
vc_count=$(cat "$VC_NFT_CALL_COUNT")
|
|
fi
|
|
vc_count=$((vc_count + 1))
|
|
printf '%s\n' "$vc_count" > "$VC_NFT_CALL_COUNT"
|
|
if [ "$vc_count" -eq 1 ]; then
|
|
cat "$VC_NFT_FIRST"
|
|
else
|
|
cat "$VC_NFT_SECOND"
|
|
fi
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
nft.chmod(0o700)
|
|
|
|
sha256 = tmp_path / "sha256sum"
|
|
sha256.write_text(
|
|
f"""#!{sys.executable}
|
|
import hashlib
|
|
import sys
|
|
|
|
for filename in sys.argv[1:]:
|
|
with open(filename, "rb") as stream:
|
|
digest = hashlib.sha256(stream.read()).hexdigest()
|
|
print(f"{{digest}} {{filename}}")
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
sha256.chmod(0o700)
|
|
fake_stat = tmp_path / "stat"
|
|
fake_stat.write_text(
|
|
f"""#!{sys.executable}
|
|
import os
|
|
import stat
|
|
import sys
|
|
|
|
if len(sys.argv) != 4 or sys.argv[1] != "-c":
|
|
raise SystemExit(2)
|
|
value = os.lstat(sys.argv[3])
|
|
mode = format(stat.S_IMODE(value.st_mode), "o")
|
|
if sys.argv[2] == "%a":
|
|
print(mode)
|
|
elif sys.argv[2] == "%u":
|
|
print("0")
|
|
elif sys.argv[2] == "%u:%a":
|
|
print(f"0:{{mode}}")
|
|
else:
|
|
raise SystemExit(2)
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
fake_stat.chmod(0o700)
|
|
|
|
ip = tmp_path / "ip"
|
|
ip.write_text(
|
|
"""#!/bin/sh
|
|
if [ "${VC_IFACE_PRESENT:-no}" = yes ] &&
|
|
[ "$*" = "link show dev wg-fabric0" ]; then
|
|
exit 0
|
|
fi
|
|
exit 1
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
ip.chmod(0o700)
|
|
wg = tmp_path / "wg"
|
|
wg.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8")
|
|
wg.chmod(0o700)
|
|
|
|
artifacts, ownership, special, current_owner, snapshot, installed_proof = (
|
|
_installing_recovery_fragments()
|
|
)
|
|
special = special.replace(
|
|
"/etc/systemd/system/virt-cloud-wg-fabric-reconcile.service",
|
|
str(targets["repair"]),
|
|
).replace(
|
|
"/etc/systemd/system/virt-cloud-wg-fabric-reconcile.timer",
|
|
str(targets["timer"]),
|
|
)
|
|
script = "\n".join((
|
|
"set -eu",
|
|
"umask 077",
|
|
f"JOURNAL={shlex.quote(str(journal))}",
|
|
f"CONF={shlex.quote(str(targets['conf']))}",
|
|
f"OWNER={shlex.quote(str(targets['owner']))}",
|
|
f"HELPER={shlex.quote(str(targets['helper']))}",
|
|
f"UNIT={shlex.quote(str(targets['unit']))}",
|
|
f"GATE_GENERATOR={shlex.quote(str(targets['gate_generator']))}",
|
|
f"NM_PROFILE={shlex.quote(str(targets['nm_profile']))}",
|
|
f"IFUPDOWN_FRAGMENT={shlex.quote(str(targets['ifupdown_fragment']))}",
|
|
f"SYSCTL_FILE={shlex.quote(str(targets['sysctl']))}",
|
|
f"NFT_RULES={shlex.quote(str(targets['nft_rules']))}",
|
|
f"journal_nft_main={shlex.quote(str(targets['nft_main']))}",
|
|
f"IP={shlex.quote(str(ip))}",
|
|
f"WG={shlex.quote(str(wg))}",
|
|
"IFACE=wg-fabric0",
|
|
f"NFT={shlex.quote(str(nft))}",
|
|
f"STAT={shlex.quote(str(fake_stat))}",
|
|
f"SHA256={shlex.quote(str(sha256))}",
|
|
f"CMP={shlex.quote('/usr/bin/cmp')}",
|
|
f"EXPECTED_NFT_RULES_SHA256={shlex.quote(expected_sha)}",
|
|
f"journal_plan_digest={shlex.quote(digest)}",
|
|
f"journal_firewall_mode={firewall_mode}",
|
|
"journal_nft_phase=installing",
|
|
"journal_nft_before=no",
|
|
"journal_iface_before=no",
|
|
"journal_iface_up_before=no",
|
|
"journal_rp_filter_before=missing",
|
|
"journal_helper_before=absent",
|
|
"journal_iface_recovery_phase=none",
|
|
"journal_recover_installing=no",
|
|
"RECOVER_CURRENT=yes",
|
|
"RECOVER_INSTALLING=yes",
|
|
"_tmp_nft_installed=",
|
|
"cleanup() {",
|
|
" cleanup_status=$?",
|
|
" trap - EXIT",
|
|
' [ -z "$_tmp_nft_installed" ] || '
|
|
'rm -f "$_tmp_nft_installed" || true',
|
|
' exit "$cleanup_status"',
|
|
"}",
|
|
"trap cleanup EXIT",
|
|
"sync() { :; }",
|
|
'stat() { "$STAT" "$@"; }',
|
|
artifacts,
|
|
_FABRIC_NFT_VALIDATION_SH,
|
|
ownership,
|
|
special,
|
|
'[ "$journal_recover_installing" = yes ] || exit 79',
|
|
current_owner,
|
|
'"$NFT" -s list table inet virt_cloud_wg_fabric > '
|
|
'"$JOURNAL/nft.current" 2>/dev/null || exit 79',
|
|
'chmod 600 "$JOURNAL/nft.current" || exit 79',
|
|
snapshot,
|
|
installed_proof,
|
|
"journal_nft_action=replace",
|
|
"printf '%s\\n' \"journal_nft_action=$journal_nft_action\"",
|
|
))
|
|
result = subprocess.run(
|
|
["/bin/sh", "-c", script],
|
|
env={
|
|
"PATH": "/usr/bin:/bin",
|
|
"VC_NFT_CALL_COUNT": str(call_count),
|
|
"VC_NFT_FIRST": str(first_output),
|
|
"VC_NFT_SECOND": str(second_output),
|
|
"VC_IFACE_PRESENT": (
|
|
"yes" if failure == "owner_iface_present" else "no"
|
|
),
|
|
},
|
|
check=False, capture_output=True, text=True,
|
|
)
|
|
return result, SimpleNamespace(
|
|
journal=journal,
|
|
desired=desired_bytes,
|
|
evidence=evidence,
|
|
alma_live=alma_live.encode(),
|
|
call_count=call_count,
|
|
installed_target=installed_target,
|
|
artifact_symlink_target=artifact_symlink_target,
|
|
artifact_state=artifact_state,
|
|
artifact_values=artifact_values,
|
|
targets=targets,
|
|
native_snapshot_target=native_snapshot_target,
|
|
native_desired_target=native_desired_target,
|
|
native_live_target=native_live_target,
|
|
native_baseline=native_baseline,
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize("artifact_state", (
|
|
"post_trap_absent",
|
|
"hard_crash_exact_desired",
|
|
"mixed",
|
|
))
|
|
def test_installing_recovery_execution_snapshots_stable_alma_live_table(
|
|
tmp_path, artifact_state,
|
|
):
|
|
result, fixture = _run_installing_recovery_harness(
|
|
tmp_path, artifact_state=artifact_state,
|
|
)
|
|
|
|
assert result.returncode == 0, result.stderr
|
|
assert result.stdout.splitlines() == ["journal_nft_action=replace"]
|
|
installed = fixture.journal / "nft.installed"
|
|
assert installed.read_bytes() == fixture.alma_live
|
|
assert stat.S_IMODE(installed.stat().st_mode) == 0o600
|
|
assert (fixture.journal / "nft_rules.desired").read_bytes() == (
|
|
fixture.desired
|
|
)
|
|
assert fixture.evidence.read_bytes() == b"do-not-overwrite\n"
|
|
assert fixture.call_count.read_text(encoding="utf-8") == "2\n"
|
|
assert not list(fixture.journal.glob(".nft-installed.*"))
|
|
expected_present = {
|
|
"post_trap_absent": {"nft_main"},
|
|
"hard_crash_exact_desired": set(fixture.artifact_values),
|
|
"mixed": {"owner", "nft_rules", "nft_main"},
|
|
}[artifact_state]
|
|
for name, value in fixture.artifact_values.items():
|
|
target = fixture.targets[name]
|
|
if name in expected_present:
|
|
expected = (
|
|
fixture.native_baseline if name == "nft_main" else value
|
|
)
|
|
assert target.read_bytes() == expected
|
|
assert stat.S_IMODE(target.stat().st_mode) == 0o600
|
|
else:
|
|
assert not target.exists()
|
|
assert not target.is_symlink()
|
|
|
|
|
|
@pytest.mark.parametrize("failure", (
|
|
"desired_hash",
|
|
"nonempty_tenant_set",
|
|
"extra_object",
|
|
"installed_symlink",
|
|
"second_read_drift",
|
|
"artifact_foreign_bytes",
|
|
"artifact_wrong_mode",
|
|
"artifact_symlink",
|
|
"artifact_desired_absent",
|
|
"owner_iface_present",
|
|
))
|
|
def test_installing_recovery_execution_rejects_ambiguous_evidence(
|
|
tmp_path, failure,
|
|
):
|
|
result, fixture = _run_installing_recovery_harness(
|
|
tmp_path, failure=failure,
|
|
)
|
|
|
|
assert result.returncode == 79
|
|
assert (fixture.journal / "nft_rules.desired").read_bytes() == (
|
|
fixture.desired
|
|
)
|
|
assert fixture.evidence.read_bytes() == b"do-not-overwrite\n"
|
|
installed = fixture.journal / "nft.installed"
|
|
if failure == "installed_symlink":
|
|
assert installed.is_symlink()
|
|
assert fixture.installed_target.read_bytes() == b"foreign-snapshot\n"
|
|
else:
|
|
assert not installed.exists()
|
|
assert not installed.is_symlink()
|
|
if failure == "second_read_drift":
|
|
assert (fixture.journal / "nft.current").read_bytes() == (
|
|
fixture.alma_live
|
|
)
|
|
assert fixture.call_count.read_text(encoding="utf-8") == "2\n"
|
|
else:
|
|
assert not (fixture.journal / "nft.current").exists()
|
|
assert not fixture.call_count.exists()
|
|
if failure == "artifact_foreign_bytes":
|
|
assert fixture.targets["sysctl"].read_bytes() == (
|
|
b"foreign sysctl bytes\n"
|
|
)
|
|
elif failure == "artifact_wrong_mode":
|
|
target = fixture.targets["nft_rules"]
|
|
assert target.read_bytes() == fixture.artifact_values["nft_rules"]
|
|
assert stat.S_IMODE(target.stat().st_mode) == 0o644
|
|
elif failure == "artifact_symlink":
|
|
assert fixture.targets["nft_main"].is_symlink()
|
|
assert fixture.artifact_symlink_target.read_bytes() == (
|
|
fixture.native_baseline
|
|
)
|
|
elif failure == "artifact_desired_absent":
|
|
assert (fixture.journal / "owner.desired.absent").is_file()
|
|
assert fixture.targets["owner"].read_bytes() == (
|
|
fixture.artifact_values["owner"]
|
|
)
|
|
elif failure == "owner_iface_present":
|
|
assert fixture.targets["owner"].read_bytes() == (
|
|
fixture.artifact_values["owner"]
|
|
)
|
|
assert not list(fixture.journal.glob(".nft-installed.*"))
|
|
|
|
|
|
@pytest.mark.parametrize("native_state", (
|
|
"baseline_current_no_desired",
|
|
"desired_current",
|
|
"baseline_current_with_desired",
|
|
))
|
|
def test_native_installing_recovery_accepts_proven_nft_main_images(
|
|
tmp_path, native_state,
|
|
):
|
|
result, fixture = _run_installing_recovery_harness(
|
|
tmp_path,
|
|
firewall_mode=FIREWALL_MODE_NATIVE,
|
|
native_state=native_state,
|
|
)
|
|
|
|
assert result.returncode == 0, result.stderr
|
|
assert result.stdout.splitlines() == ["journal_nft_action=replace"]
|
|
assert (fixture.journal / "nft.installed").read_bytes() == (
|
|
fixture.alma_live
|
|
)
|
|
assert (fixture.journal / "nft_main.present").is_file()
|
|
assert not (fixture.journal / "nft_main.absent").exists()
|
|
assert (fixture.journal / "nft_main").read_bytes().startswith(
|
|
b"# baseline\n",
|
|
)
|
|
desired = fixture.journal / "nft_main.desired"
|
|
if native_state == "baseline_current_no_desired":
|
|
assert not desired.exists()
|
|
assert fixture.targets["nft_main"].read_bytes().startswith(
|
|
b"# baseline\n",
|
|
)
|
|
elif native_state == "desired_current":
|
|
assert desired.read_bytes().startswith(b"# desired\n")
|
|
assert fixture.targets["nft_main"].read_bytes() == desired.read_bytes()
|
|
else:
|
|
assert desired.read_bytes().startswith(b"# desired\n")
|
|
assert fixture.targets["nft_main"].read_bytes() == (
|
|
fixture.journal / "nft_main"
|
|
).read_bytes()
|
|
assert fixture.call_count.read_text(encoding="utf-8") == "2\n"
|
|
|
|
|
|
@pytest.mark.parametrize("failure", (
|
|
"native_baseline_missing",
|
|
"native_baseline_symlink",
|
|
"native_baseline_mode",
|
|
"native_baseline_content",
|
|
"native_marker_conflict",
|
|
"native_live_missing",
|
|
"native_live_symlink",
|
|
"native_live_mode",
|
|
"native_live_foreign",
|
|
"native_desired_symlink",
|
|
"native_desired_mode",
|
|
"native_desired_content",
|
|
"native_desired_absent_conflict",
|
|
))
|
|
def test_native_installing_recovery_rejects_unproven_nft_main_images(
|
|
tmp_path, failure,
|
|
):
|
|
result, fixture = _run_installing_recovery_harness(
|
|
tmp_path,
|
|
failure=failure,
|
|
firewall_mode=FIREWALL_MODE_NATIVE,
|
|
)
|
|
|
|
assert result.returncode == 79
|
|
assert fixture.evidence.read_bytes() == b"do-not-overwrite\n"
|
|
assert not (fixture.journal / "nft.current").exists()
|
|
assert not fixture.call_count.exists()
|
|
assert not (fixture.journal / "nft.installed").exists()
|
|
assert not list(fixture.journal.glob(".nft-installed.*"))
|
|
if failure == "native_baseline_symlink":
|
|
assert (fixture.journal / "nft_main").is_symlink()
|
|
assert fixture.native_snapshot_target.read_bytes().startswith(
|
|
b"# baseline\n",
|
|
)
|
|
elif failure == "native_desired_symlink":
|
|
assert (fixture.journal / "nft_main.desired").is_symlink()
|
|
assert fixture.native_desired_target.read_bytes().startswith(
|
|
b"# desired\n",
|
|
)
|
|
elif failure == "native_live_symlink":
|
|
assert fixture.targets["nft_main"].is_symlink()
|
|
assert fixture.native_live_target.read_bytes().startswith(
|
|
b"# baseline\n",
|
|
)
|
|
|
|
|
|
|
|
|
|
def _timer_activity_recovery_runner(
|
|
tmp_path, *, wanted, initial, failure="none",
|
|
):
|
|
start = _BOOT_HELPER.index("vc_restore_timer_activity() {")
|
|
end = _BOOT_HELPER.index("\n}\nnft_panel_owned()", start) + 2
|
|
function = _BOOT_HELPER[start:end]
|
|
state = tmp_path / "timer.state"
|
|
state.write_text(f"{initial}\n", encoding="utf-8")
|
|
log = tmp_path / "systemctl.log"
|
|
systemctl = tmp_path / "systemctl"
|
|
systemctl.write_text(
|
|
"""#!/bin/sh
|
|
printf '%s\n' "$*" >> "$VC_SYSTEMCTL_LOG"
|
|
case "$1" in
|
|
start)
|
|
[ "$VC_SYSTEMCTL_FAILURE" != start ] || exit 1
|
|
printf '%s\n' active > "$VC_TIMER_STATE" ;;
|
|
stop)
|
|
[ "$VC_SYSTEMCTL_FAILURE" != stop ] || exit 1
|
|
[ "$VC_SYSTEMCTL_FAILURE" = sticky ] || \
|
|
printf '%s\n' inactive > "$VC_TIMER_STATE" ;;
|
|
is-active)
|
|
state=$(cat "$VC_TIMER_STATE")
|
|
printf '%s\n' "$state"
|
|
[ "$state" = active ] || [ "$state" = activating ] ;;
|
|
*) exit 2 ;;
|
|
esac
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
systemctl.chmod(0o700)
|
|
result = subprocess.run(
|
|
["/bin/sh", "-c", "\n".join((
|
|
"set -eu",
|
|
f"SYSTEMCTL='{systemctl}'",
|
|
function,
|
|
f"vc_restore_timer_activity {wanted}",
|
|
))],
|
|
env={
|
|
"PATH": "/usr/bin:/bin",
|
|
"VC_SYSTEMCTL_LOG": str(log),
|
|
"VC_TIMER_STATE": str(state),
|
|
"VC_SYSTEMCTL_FAILURE": failure,
|
|
},
|
|
check=False, capture_output=True, text=True,
|
|
)
|
|
return result, state, log
|
|
|
|
|
|
def test_crash_recovery_stops_timer_started_after_inactive_snapshot(tmp_path):
|
|
result, state, log = _timer_activity_recovery_runner(
|
|
tmp_path, wanted="inactive", initial="active",
|
|
)
|
|
|
|
assert result.returncode == 0, result.stderr
|
|
assert state.read_text(encoding="utf-8") == "inactive\n"
|
|
assert log.read_text(encoding="utf-8").splitlines() == [
|
|
"stop virt-cloud-wg-fabric-reconcile.timer",
|
|
"is-active virt-cloud-wg-fabric-reconcile.timer",
|
|
]
|
|
helper = _BOOT_HELPER
|
|
restore_unit = helper.index(
|
|
'restore_journal_file "/etc/systemd/system/'
|
|
'virt-cloud-wg-fabric-reconcile.timer" timer'
|
|
)
|
|
daemon_reload = helper.index(
|
|
'"$SYSTEMCTL" daemon-reload', restore_unit,
|
|
)
|
|
restore_activity = helper.index(
|
|
'vc_restore_timer_activity "$journal_timer_active"', daemon_reload,
|
|
)
|
|
remove_journal = helper.index('rm -rf "$JOURNAL"', restore_activity)
|
|
assert restore_unit < daemon_reload < restore_activity < remove_journal
|
|
|
|
|
|
def test_crash_recovery_restarts_previously_active_timer(tmp_path):
|
|
result, state, log = _timer_activity_recovery_runner(
|
|
tmp_path, wanted="active", initial="inactive",
|
|
)
|
|
|
|
assert result.returncode == 0, result.stderr
|
|
assert state.read_text(encoding="utf-8") == "active\n"
|
|
assert log.read_text(encoding="utf-8").splitlines() == [
|
|
"start virt-cloud-wg-fabric-reconcile.timer",
|
|
"is-active virt-cloud-wg-fabric-reconcile.timer",
|
|
]
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("wanted", "initial", "failure"),
|
|
(("active", "inactive", "start"),
|
|
("inactive", "active", "stop"),
|
|
("inactive", "active", "sticky")),
|
|
)
|
|
def test_crash_recovery_keeps_journal_when_timer_state_cannot_be_restored(
|
|
tmp_path, wanted, initial, failure,
|
|
):
|
|
result, _, _ = _timer_activity_recovery_runner(
|
|
tmp_path, wanted=wanted, initial=initial, failure=failure,
|
|
)
|
|
|
|
assert result.returncode == 1
|
|
|
|
|
|
def test_apply_rollback_recovers_refresh_snapshot_before_phase_marker():
|
|
current = plan()
|
|
script = _apply_script(
|
|
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
|
|
firewall_mode=FIREWALL_MODE_NATIVE,
|
|
)
|
|
snapshot_publish = script.index(
|
|
'mv -f "$TXN/nft.applied.tmp" "$TXN/nft.applied"')
|
|
phase_publish = script.index(
|
|
"write_nft_phase refreshed", snapshot_publish)
|
|
rollback_window = script.index(
|
|
'elif [ "$rollback_nft_phase" = refreshing ] &&')
|
|
exact_snapshot = script.index(
|
|
'"$CMP" -s "$TXN/nft.current" "$TXN/nft.applied" '
|
|
'|| ROLLBACK_FAILED=yes',
|
|
rollback_window,
|
|
)
|
|
exact_set = script.index(
|
|
"vpc_state_nft_set_matches || ROLLBACK_FAILED=yes",
|
|
exact_snapshot,
|
|
)
|
|
stable_snapshot = script.index(
|
|
'"$CMP" -s "$TXN/nft.applied" "$TXN/nft.refresh-verify" '
|
|
'|| ROLLBACK_FAILED=yes',
|
|
exact_set,
|
|
)
|
|
|
|
assert snapshot_publish < phase_publish
|
|
assert rollback_window < exact_snapshot < exact_set < stable_snapshot
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_repair_unit_contains_only_the_retry_dispatcher():
|
|
current = plan()
|
|
script = _apply_script(
|
|
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
|
|
firewall_mode=FIREWALL_MODE_NATIVE,
|
|
)
|
|
rendered = _REPAIR_UNIT.replace(
|
|
"__VC_FABRIC_DIGEST__", current.digest)
|
|
assert rendered in script
|
|
assert (
|
|
'ExecCondition=/bin/sh /usr/local/libexec/virt-cloud-wg-fabric '
|
|
'--needs-repair'
|
|
) in rendered
|
|
assert [
|
|
line for line in rendered.splitlines()
|
|
if line.startswith("ExecStart=")
|
|
] == [
|
|
"ExecStart=/bin/sh /usr/local/libexec/virt-cloud-wg-fabric "
|
|
"--retry-gated-units",
|
|
]
|
|
assert "ExecStartPost=" not in rendered
|
|
assert "__VC_FABRIC_EXEC__" not in rendered
|
|
|
|
|
|
def test_repair_dispatcher_queues_primary_without_proven_dependency_jobs(tmp_path):
|
|
tool = tmp_path / "tool"
|
|
tool.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8")
|
|
tool.chmod(0o700)
|
|
|
|
systemctl = tmp_path / "systemctl"
|
|
systemctl.write_text(
|
|
"""#!/bin/sh
|
|
printf '%s\\n' "$*" >> "$VC_TEST_SYSTEMCTL_LOG"
|
|
case "$1:$2:$3:$5" in
|
|
is-failed:--quiet:virt-cloud-wg-fabric.service:) exit 0 ;;
|
|
show:-p:LoadState:libvirtd.service) printf '%s\\n' loaded ;;
|
|
show:-p:Result:libvirtd.service) printf '%s\\n' dependency ;;
|
|
show:-p:LoadState:virtqemud.service) printf '%s\\n' loaded ;;
|
|
show:-p:Result:virtqemud.service) printf '%s\\n' success ;;
|
|
reset-failed:libvirtd.service::) exit 0 ;;
|
|
reset-failed:virt-cloud-wg-fabric.service::) exit 0 ;;
|
|
start:virt-cloud-wg-fabric.service::) exit 0 ;;
|
|
*) exit 2 ;;
|
|
esac
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
systemctl.chmod(0o700)
|
|
|
|
helper = _BOOT_HELPER.replace(
|
|
"__VC_LIBVIRT_GATE_UNITS__",
|
|
"libvirtd.service virtqemud.service",
|
|
).replace(
|
|
"__VC_FABRIC_SYSCTL_VALIDATION__",
|
|
_FABRIC_SYSCTL_VALIDATION_SH,
|
|
)
|
|
tool_variables = {
|
|
"WG", "IP", "NFT", "STAT", "SHA256", "CMP", "FIREWALL_CMD",
|
|
"AWK", "BRIDGE_CMD",
|
|
}
|
|
helper_lines = []
|
|
for line in helper.splitlines():
|
|
if line == "BASE=/etc/virt-cloud":
|
|
line = 'BASE="$VC_TEST_BASE"'
|
|
elif line.startswith("SYSTEMCTL=$(find_tool "):
|
|
line = 'SYSTEMCTL="$VC_TEST_SYSTEMCTL"'
|
|
elif any(
|
|
line.startswith(f"{name}=$(find_tool ")
|
|
for name in tool_variables
|
|
):
|
|
name = line.split("=", 1)[0]
|
|
line = f'{name}="$VC_TEST_TOOL"'
|
|
helper_lines.append(line)
|
|
|
|
helper_path = tmp_path / "virt-cloud-wg-fabric"
|
|
helper_path.write_text(
|
|
"\n".join(helper_lines) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
helper_path.chmod(0o700)
|
|
base = tmp_path / "state"
|
|
base.mkdir()
|
|
log = tmp_path / "systemctl.log"
|
|
env = {
|
|
"VC_TEST_BASE": str(base),
|
|
"VC_TEST_TOOL": str(tool),
|
|
"VC_TEST_SYSTEMCTL": str(systemctl),
|
|
"VC_TEST_SYSTEMCTL_LOG": str(log),
|
|
}
|
|
|
|
condition = subprocess.run(
|
|
["/bin/sh", str(helper_path), "--needs-repair"],
|
|
check=False, capture_output=True, text=True, env=env,
|
|
)
|
|
assert condition.returncode == 0, condition.stderr
|
|
dispatcher = subprocess.run(
|
|
["/bin/sh", str(helper_path), "--retry-gated-units"],
|
|
check=False, capture_output=True, text=True, env=env,
|
|
)
|
|
assert dispatcher.returncode == 0, dispatcher.stderr
|
|
|
|
calls = log.read_text(encoding="utf-8").splitlines()
|
|
start_calls = [call for call in calls if call.startswith("start ")]
|
|
assert start_calls == [
|
|
"start virt-cloud-wg-fabric.service",
|
|
]
|
|
assert not any("show -p Result" in call for call in calls)
|
|
|
|
|
|
def test_repair_retry_dispatches_before_lock_and_only_requeues_failures():
|
|
retry = _BOOT_HELPER.index('if [ "${1:-}" = --retry-gated-units ]')
|
|
lock = _BOOT_HELPER.index('exec 9>>"$LOCK"')
|
|
assert retry < lock
|
|
assert 'vc_gate_retry_evidence' in _BOOT_HELPER
|
|
assert 'value!="dependency"' in _BOOT_HELPER
|
|
assert 'show -p Result' not in _BOOT_HELPER
|
|
assert 'start virt-cloud-wg-fabric.service' in _BOOT_HELPER
|
|
assert 'exec /bin/sh "$HELPER" --ifupdown-service' in _BOOT_HELPER
|
|
assert 'exec /bin/sh "$HELPER"' in _BOOT_HELPER
|
|
assert 'restart --no-block virt-cloud-wg-fabric.service' not in _BOOT_HELPER
|
|
needs_repair = _BOOT_HELPER.index(
|
|
'if [ "${1:-}" = --needs-repair ]; then')
|
|
lock = _BOOT_HELPER.index('exec 9>>"$LOCK"')
|
|
assert needs_repair < lock
|
|
assert 'is-failed --quiet virt-cloud-wg-fabric.service' in _BOOT_HELPER
|
|
assert (
|
|
'ExecCondition=/bin/sh /usr/local/libexec/virt-cloud-wg-fabric '
|
|
'--needs-repair'
|
|
) in _REPAIR_UNIT
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _operation_status_output(
|
|
operation_id, *, status="queued", checkpoint="dispatch",
|
|
reason="none", created_at=100, started_at=0, updated_at=100,
|
|
finished_at=0):
|
|
return "\n".join((
|
|
"__VC_FABRIC_OPERATION_SCHEMA__=1",
|
|
f"__VC_FABRIC_OPERATION_ID__={operation_id}",
|
|
f"__VC_FABRIC_OPERATION_STATUS__={status}",
|
|
"__VC_FABRIC_OPERATION_PHASE__=apply",
|
|
f"__VC_FABRIC_OPERATION_CHECKPOINT__={checkpoint}",
|
|
f"__VC_FABRIC_OPERATION_REASON__={reason}",
|
|
f"__VC_FABRIC_OPERATION_CREATED_AT__={created_at}",
|
|
f"__VC_FABRIC_OPERATION_STARTED_AT__={started_at}",
|
|
f"__VC_FABRIC_OPERATION_UPDATED_AT__={updated_at}",
|
|
f"__VC_FABRIC_OPERATION_FINISHED_AT__={finished_at}",
|
|
"",
|
|
))
|
|
|
|
|
|
def test_fabric_operation_ids_are_opaque_fixed_lowercase_hex():
|
|
operation_ids = {_new_operation_id() for _index in range(128)}
|
|
|
|
assert len(operation_ids) == 128
|
|
assert all(re.fullmatch(r"[0-9a-f]{32}", value)
|
|
for value in operation_ids)
|
|
|
|
|
|
@pytest.mark.parametrize("operation_id", [
|
|
"", "a" * 31, "a" * 33, "A" * 32, "g" * 32,
|
|
"../" + "a" * 29, "a" * 31 + "/", "a" * 31 + "\n",
|
|
"$(id)".ljust(32, "a"),
|
|
])
|
|
def test_fabric_operation_id_is_validated_before_rendering_remote_shell(
|
|
operation_id):
|
|
with pytest.raises(FabricError) as error:
|
|
_fabric_operation_status_script(operation_id)
|
|
|
|
assert error.value.error_code == "fabric_operation_invalid"
|
|
|
|
|
|
@pytest.mark.parametrize(("kwargs", "expected"), [
|
|
({}, {
|
|
"status": "queued", "checkpoint": "dispatch", "reason": "none",
|
|
"started_at": None, "finished_at": None,
|
|
}),
|
|
({
|
|
"status": "running", "checkpoint": "journal_recovery",
|
|
"created_at": 100, "started_at": 101, "updated_at": 102,
|
|
}, {
|
|
"status": "running", "checkpoint": "journal_recovery",
|
|
"reason": "none", "started_at": 101, "finished_at": None,
|
|
}),
|
|
({
|
|
"status": "succeeded", "checkpoint": "complete",
|
|
"created_at": 100, "started_at": 101, "updated_at": 103,
|
|
"finished_at": 103,
|
|
}, {
|
|
"status": "succeeded", "checkpoint": "complete",
|
|
"reason": "none", "started_at": 101, "finished_at": 103,
|
|
}),
|
|
({
|
|
"status": "failed", "checkpoint": "nft_install",
|
|
"reason": "nft_failed", "created_at": 100, "started_at": 101,
|
|
"updated_at": 104, "finished_at": 104,
|
|
}, {
|
|
"status": "failed", "checkpoint": "nft_install",
|
|
"reason": "nft_failed", "started_at": 101, "finished_at": 104,
|
|
}),
|
|
])
|
|
def test_fabric_operation_status_parser_accepts_only_complete_safe_projection(
|
|
kwargs, expected):
|
|
operation_id = "1" * 32
|
|
result = _parse_operation_status(
|
|
_operation_status_output(operation_id, **kwargs), operation_id)
|
|
|
|
assert result["operation_id"] == operation_id
|
|
assert result["phase"] == "apply"
|
|
for field, value in expected.items():
|
|
assert result[field] == value
|
|
|
|
|
|
def _replace_status_line(output, prefix, replacement):
|
|
lines = output.splitlines()
|
|
return "\n".join(
|
|
replacement if line.startswith(prefix) else line
|
|
for line in lines
|
|
) + "\n"
|
|
|
|
|
|
@pytest.mark.parametrize("mutation", [
|
|
lambda value: value + "__VC_FABRIC_OPERATION_EXTRA__=value\n",
|
|
lambda value: value + "__VC_FABRIC_OPERATION_STATUS__=queued\n",
|
|
lambda value: value.replace(
|
|
"__VC_FABRIC_OPERATION_REASON__=none\n", ""),
|
|
lambda value: "attacker-controlled diagnostic\n" + value,
|
|
lambda value: _replace_status_line(
|
|
value, "__VC_FABRIC_OPERATION_STATUS__=",
|
|
"__VC_FABRIC_OPERATION_STATUS__=unknown"),
|
|
lambda value: _replace_status_line(
|
|
value, "__VC_FABRIC_OPERATION_PHASE__=",
|
|
"__VC_FABRIC_OPERATION_PHASE__=identity"),
|
|
lambda value: _replace_status_line(
|
|
value, "__VC_FABRIC_OPERATION_CHECKPOINT__=",
|
|
"__VC_FABRIC_OPERATION_CHECKPOINT__=arbitrary"),
|
|
lambda value: _replace_status_line(
|
|
value, "__VC_FABRIC_OPERATION_REASON__=",
|
|
"__VC_FABRIC_OPERATION_REASON__=/root/private-key"),
|
|
lambda value: _replace_status_line(
|
|
value, "__VC_FABRIC_OPERATION_CREATED_AT__=",
|
|
"__VC_FABRIC_OPERATION_CREATED_AT__=-1"),
|
|
lambda value: _replace_status_line(
|
|
value, "__VC_FABRIC_OPERATION_UPDATED_AT__=",
|
|
"__VC_FABRIC_OPERATION_UPDATED_AT__=99"),
|
|
])
|
|
def test_fabric_operation_status_parser_rejects_extra_duplicate_or_invalid_data(
|
|
mutation):
|
|
operation_id = "2" * 32
|
|
output = mutation(_operation_status_output(operation_id))
|
|
|
|
with pytest.raises(FabricError) as error:
|
|
_parse_operation_status(output, operation_id)
|
|
|
|
assert error.value.error_code == "fabric_operation_status_invalid"
|
|
safe_error = json.dumps({
|
|
"message": str(error.value),
|
|
"code": error.value.error_code,
|
|
"phase": error.value.failure_phase,
|
|
})
|
|
assert "attacker-controlled" not in safe_error
|
|
assert "/root/private-key" not in safe_error
|
|
|
|
|
|
def test_fabric_operation_status_parser_fences_different_attempts():
|
|
first = "3" * 32
|
|
second = "4" * 32
|
|
|
|
with pytest.raises(FabricError) as error:
|
|
_parse_operation_status(_operation_status_output(first), second)
|
|
|
|
assert error.value.error_code == "fabric_operation_status_invalid"
|
|
|
|
|
|
@pytest.mark.parametrize(("status", "checkpoint", "reason", "started", "finished"), [
|
|
("queued", "dispatch", "none", 1, 0),
|
|
("running", "preflight", "none", 0, 0),
|
|
("running", "preflight", "nft_failed", 101, 0),
|
|
("succeeded", "final_readback", "none", 101, 102),
|
|
("succeeded", "complete", "command_failed", 101, 102),
|
|
("failed", "nft_install", "none", 101, 102),
|
|
])
|
|
def test_fabric_operation_status_parser_rejects_impossible_state_combinations(
|
|
status, checkpoint, reason, started, finished):
|
|
operation_id = "5" * 32
|
|
output = _operation_status_output(
|
|
operation_id, status=status, checkpoint=checkpoint, reason=reason,
|
|
started_at=started, updated_at=102, finished_at=finished)
|
|
|
|
with pytest.raises(FabricError) as error:
|
|
_parse_operation_status(output, operation_id)
|
|
|
|
assert error.value.error_code == "fabric_operation_status_invalid"
|
|
|
|
|
|
def test_fabric_operation_protocol_has_fixed_bounded_vocabulary():
|
|
assert FABRIC_OPERATION_STATUSES == {
|
|
"queued", "running", "succeeded", "failed", "uncertain",
|
|
"interrupted",
|
|
}
|
|
assert FABRIC_OPERATION_CHECKPOINTS == {
|
|
"dispatch", "host_lock", "identity", "journal_recovery",
|
|
"journal_dispatch", "preflight_sysctl", "preflight_tools",
|
|
"preflight_artifacts", "preflight_firewall", "preflight_vpc",
|
|
"preflight_underlay", "preflight_routes", "preflight_persistence",
|
|
"journal_artifact_restore", "journal_nft_restore",
|
|
"journal_runtime_restore", "journal_finalize", "preflight",
|
|
"transaction", "nft_install", "network_activate", "persistence",
|
|
"final_readback", "complete",
|
|
}
|
|
assert FABRIC_OPERATION_REASONS == {
|
|
"none", "tool_missing", "ownership_conflict", "journal_conflict",
|
|
"sysctl_conflict", "persistence_conflict",
|
|
"firewall_conflict", "underlay_conflict", "network_manager_failed",
|
|
"nft_failed", "systemd_failed", "readback_failed",
|
|
"command_failed", "transport_failed", "status_invalid",
|
|
}
|
|
|
|
|
|
def test_fabric_operation_status_reader_is_strictly_read_only():
|
|
operation_id = "6" * 32
|
|
script = _fabric_operation_status_script(operation_id)
|
|
|
|
assert f"OPERATION_ID={operation_id}" in script
|
|
assert "__VC_FABRIC_OPERATION_STATUS__=$state" in script
|
|
assert '"$UPDATE" interrupted' not in script
|
|
assert "mv " not in script
|
|
assert "rm " not in script
|
|
assert "systemctl start" not in script
|
|
assert "systemctl restart" not in script
|
|
|
|
|
|
def test_fabric_operation_dispatch_uses_fixed_nonblocking_systemd_command():
|
|
operation_id = "7" * 32
|
|
secret = "private-key-material-must-not-be-logged"
|
|
rendered_apply = f"#!/bin/sh\nprintf '%s' {shlex.quote(secret)} >/dev/null\n"
|
|
script = _fabric_operation_dispatch_script(
|
|
operation_id, rendered_apply, "8" * 64)
|
|
systemd_start = script.index('"$SYSTEMD_RUN" --unit="$UNIT"')
|
|
|
|
assert f"UNIT={FABRIC_OPERATION_UNIT_PREFIX}{operation_id}.service" in script
|
|
assert "--collect --no-block --quiet" in script[systemd_start:]
|
|
assert "--property=Type=oneshot" in script[systemd_start:]
|
|
assert "--property=StandardOutput=journal" in script[systemd_start:]
|
|
assert "--property=StandardError=null" in script[systemd_start:]
|
|
assert script.count(
|
|
f"--property=TimeoutStartSec={FABRIC_OPERATION_START_TIMEOUT_SECONDS}s"
|
|
) == 2
|
|
assert "RuntimeMaxSec=" not in script
|
|
assert script.count('/bin/sh "$OPERATION_DIR/runner"') == 2
|
|
assert not re.search(r'^\s*"\$OPERATION_DIR/runner"', script, re.MULTILINE)
|
|
assert "/bin/sh -c" not in script[systemd_start:]
|
|
assert "exec >/dev/null 2>&1" in script
|
|
assert "operation=%s state=%s checkpoint=%s reason=%s" in script
|
|
assert secret not in script[systemd_start:]
|
|
|
|
|
|
def test_fabric_operation_runner_preserves_apply_exit_status_for_safe_reason():
|
|
runner = _fabric_operation_runner_script("7" * 32, "8" * 64)
|
|
|
|
run_apply = runner.index('if /bin/sh "$APPLY"; then')
|
|
success = runner.index(
|
|
'/bin/sh "$UPDATE" succeeded complete none', run_apply)
|
|
capture = runner.index("else\n apply_rc=$?\nfi", success)
|
|
tool_reason = runner.index("70) reason=tool_missing", capture)
|
|
journal_reason = runner.index("79) reason=journal_conflict", tool_reason)
|
|
failed = runner.index(
|
|
'/bin/sh "$UPDATE" failed "$checkpoint" "$reason"',
|
|
journal_reason)
|
|
assert run_apply < success < capture < tool_reason < journal_reason < failed
|
|
assert runner.count('/bin/sh "$UPDATE"') == 4
|
|
assert not re.search(r'^\s*"\$UPDATE"', runner, re.MULTILINE)
|
|
|
|
|
|
@pytest.mark.parametrize(("apply_exit", "checkpoint", "expected_calls"), [
|
|
(0, "dispatch", ["running dispatch none", "succeeded complete none"]),
|
|
(79, "journal_recovery", [
|
|
"running dispatch none",
|
|
"failed journal_recovery journal_conflict",
|
|
]),
|
|
(79, "journal_artifact_restore", [
|
|
"running dispatch none",
|
|
"failed journal_artifact_restore journal_conflict",
|
|
]),
|
|
(79, "journal_nft_restore", [
|
|
"running dispatch none",
|
|
"failed journal_nft_restore journal_conflict",
|
|
]),
|
|
(79, "journal_runtime_restore", [
|
|
"running dispatch none",
|
|
"failed journal_runtime_restore journal_conflict",
|
|
]),
|
|
(79, "journal_finalize", [
|
|
"running dispatch none",
|
|
"failed journal_finalize journal_conflict",
|
|
]),
|
|
*[(71, checkpoint, [
|
|
"running dispatch none", f"failed {checkpoint} {reason}",
|
|
]) for checkpoint, reason in (
|
|
("preflight", "command_failed"),
|
|
("preflight_sysctl", "sysctl_conflict"),
|
|
("preflight_tools", "tool_missing"),
|
|
("preflight_artifacts", "ownership_conflict"),
|
|
("preflight_firewall", "firewall_conflict"),
|
|
("preflight_vpc", "readback_failed"),
|
|
("preflight_underlay", "underlay_conflict"),
|
|
("preflight_routes", "underlay_conflict"),
|
|
("preflight_persistence", "persistence_conflict"),
|
|
("journal_dispatch", "journal_conflict"),
|
|
)],
|
|
])
|
|
def test_fabric_operation_runner_uses_shell_for_non_executable_spool_scripts(
|
|
tmp_path, monkeypatch, apply_exit, checkpoint, expected_calls):
|
|
operation_id = "7" * 32
|
|
operation_root = tmp_path / "fabric-operations"
|
|
operation_dir = operation_root / operation_id
|
|
operation_dir.mkdir(parents=True)
|
|
calls = operation_dir / "calls"
|
|
monkeypatch.setattr(
|
|
"virsh_panel.fabric.FABRIC_OPERATION_ROOT", str(operation_root))
|
|
|
|
update = operation_dir / "update"
|
|
update.write_text(
|
|
"#!/bin/sh\n"
|
|
f"printf '%s\\n' \"$*\" >> {shlex.quote(str(calls))}\n",
|
|
encoding="utf-8",
|
|
)
|
|
apply = operation_dir / "apply"
|
|
apply.write_text(
|
|
f"#!/bin/sh\nexit {apply_exit}\n",
|
|
encoding="utf-8",
|
|
)
|
|
(operation_dir / "status").write_text(
|
|
"\n".join((
|
|
"schema=1",
|
|
f"operation_id={operation_id}",
|
|
"status=running",
|
|
"phase=apply",
|
|
f"checkpoint={checkpoint}",
|
|
"reason=none",
|
|
"created_at=100",
|
|
"started_at=101",
|
|
"updated_at=101",
|
|
"finished_at=0",
|
|
"",
|
|
)),
|
|
encoding="utf-8",
|
|
)
|
|
runner = operation_dir / "runner"
|
|
runner.write_text(
|
|
_fabric_operation_runner_script(operation_id, "8" * 64),
|
|
encoding="utf-8",
|
|
)
|
|
for spool_script in (update, apply, runner):
|
|
spool_script.chmod(0o600)
|
|
|
|
result = subprocess.run(
|
|
["/bin/sh", str(runner)],
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
|
|
assert result.returncode == (0 if apply_exit == 0 else 1)
|
|
assert calls.read_text(encoding="utf-8").splitlines() == expected_calls
|
|
|
|
|
|
def test_fabric_operation_spool_is_ephemeral_bounded_and_fail_closed():
|
|
operation_id = "8" * 32
|
|
script = _fabric_operation_dispatch_script(
|
|
operation_id, "#!/bin/sh\nexit 0\n", "9" * 64)
|
|
|
|
assert FABRIC_OPERATION_ROOT == "/run/virt-cloud/fabric-operations"
|
|
assert f"ROOT={FABRIC_OPERATION_ROOT}" in script
|
|
assert (
|
|
f'if [ "$operation_count" -ge {MAX_FABRIC_OPERATION_DIRECTORIES} ]; '
|
|
"then"
|
|
) in script
|
|
assert '"__VC_FABRIC_OPERATION_DISPATCHED__=limit"' in script
|
|
assert 'case "$operation_name" in' in script
|
|
assert "*[!0-9a-f]*) exit 71" in script
|
|
assert '[ "${#operation_name}" = 32 ] || exit 71' in script
|
|
assert 'validate_retained_operation "$operation_path" || exit 71' in script
|
|
assert '[ "$operation_hidden" = "$LOCK" ] || exit 71' in script
|
|
limit = script.index(
|
|
f'if [ "$operation_count" -ge {MAX_FABRIC_OPERATION_DIRECTORIES} ];')
|
|
validate = script.index(
|
|
'validate_retained_operation "$oldest_terminal_path"', limit)
|
|
terminal = script.index(
|
|
'succeeded|failed|uncertain|interrupted)', validate)
|
|
remove = script.index(
|
|
'for retained_file in manifest status apply update runner; do',
|
|
terminal)
|
|
rmdir = script.index('rmdir -- "$oldest_terminal_path"', remove)
|
|
limit_result = script.index(
|
|
'"__VC_FABRIC_OPERATION_DISPATCHED__=limit"', rmdir)
|
|
assert limit < validate < terminal < remove < rmdir < limit_result
|
|
assert 'rm -rf -- "$oldest_terminal_path"' not in script
|
|
|
|
|
|
def test_fabric_operation_existing_attempt_recovers_only_orphaned_queue():
|
|
operation_id = "9" * 32
|
|
script = _fabric_operation_dispatch_script(
|
|
operation_id, "#!/bin/sh\nexit 0\n", "a" * 64)
|
|
queued = script.index('if [ "$retained_state" = queued ]; then')
|
|
unit_read = script.index(
|
|
'"$SYSTEMCTL" show "$UNIT" --property=LoadState --value',
|
|
queued)
|
|
missing = script.index("not-found)", unit_read)
|
|
recover = script.index("recover_operation_unit", missing)
|
|
loaded = script.index("loaded)", recover)
|
|
existing = script.index(
|
|
'"__VC_FABRIC_OPERATION_DISPATCHED__=existing"', loaded)
|
|
existing_exit = script.index(" exit 0", existing)
|
|
|
|
assert queued < unit_read < missing < recover < loaded < existing
|
|
assert existing < existing_exit
|
|
assert script.count("recover_operation_unit") == 2
|
|
recover_function = script[script.index("recover_operation_unit()"):
|
|
script.index("validate_retained_operation()")]
|
|
assert recover_function.count('"$SYSTEMD_RUN"') == 1
|
|
assert '/bin/sh "$OPERATION_DIR/runner"' in recover_function
|
|
assert script.count('/bin/sh "$OPERATION_DIR/update"') == 3
|
|
assert not re.search(
|
|
r'^\s*"\$OPERATION_DIR/update"', script, re.MULTILINE)
|
|
assert (
|
|
'[ "$(sed -n \'2p\' "$OPERATION_DIR/manifest")" = '
|
|
'"operation_id=$OPERATION_ID" ]'
|
|
) in script
|
|
assert (
|
|
'[ "$(sed -n \'3p\' "$OPERATION_DIR/manifest")" = '
|
|
'"plan_digest=$PLAN_DIGEST" ]'
|
|
) in script
|
|
assert (
|
|
'[ "$(sed -n \'4p\' "$OPERATION_DIR/manifest")" = '
|
|
'"script_digest=$SCRIPT_DIGEST" ]'
|
|
) in script
|
|
assert (
|
|
'case "${operation_file##*/}" in '
|
|
"manifest|status|apply|update|runner) : ;; *) exit 71 ;; esac"
|
|
) in script
|
|
assert 'for operation_file in manifest status; do' in script
|
|
assert ' = 0:600 ] || exit 71' in script
|
|
assert 'for operation_file in apply update runner; do' in script
|
|
assert ' = 0:700 ] || exit 71' in script
|
|
assert (
|
|
'$SHA256SUM "$OPERATION_DIR/apply"' in script
|
|
and '$SHA256SUM "$OPERATION_DIR/update"' in script
|
|
and '$SHA256SUM "$OPERATION_DIR/runner"' in script
|
|
)
|
|
|
|
|
|
def test_fabric_dispatch_retires_only_proven_unstarted_legacy_spool():
|
|
operation_id = "9" * 32
|
|
script = _fabric_operation_dispatch_script(
|
|
operation_id, "#!/bin/sh\nexit 0\n", "a" * 64)
|
|
retained = script.index(
|
|
'if [ "$retained_apply:$retained_update:$retained_runner" !=')
|
|
queued = script.index(
|
|
'queued:dispatch:none ] || exit 71', retained)
|
|
never_started = script.index(
|
|
'[ "$retained_started:$retained_finished" = 0:0 ] || exit 71',
|
|
queued)
|
|
never_updated = script.index(
|
|
'[ "$retained_updated" = "$retained_created" ] || exit 71',
|
|
never_started)
|
|
absent = script.index('not-found:inactive) : ;;', never_updated)
|
|
exec_failed = script.index(
|
|
'exit-code:exited:203 ] || exit 71', absent)
|
|
retire = script.index(
|
|
'status=failed phase=apply checkpoint=dispatch', exec_failed)
|
|
current_manifest = script.index(
|
|
'"plan_digest=$PLAN_DIGEST" ] || exit 71', retire)
|
|
|
|
assert retained < queued < never_started < never_updated < absent
|
|
assert absent < exec_failed
|
|
assert exec_failed < retire < current_manifest
|
|
assert '/bin/sh "$OPERATION_DIR/update"' not in script[
|
|
retained:retire]
|
|
assert "possibly started operation" in script
|
|
|
|
|
|
def test_fabric_async_apply_records_recovery_before_new_transaction():
|
|
current = plan()
|
|
script = _apply_script(
|
|
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2})
|
|
recovery_checkpoint = script.index(
|
|
"vc_fabric_operation_checkpoint journal_recovery")
|
|
recovery = script.index(
|
|
'case "$journal_existing_state" in', recovery_checkpoint)
|
|
preflight = script.index(
|
|
"vc_fabric_operation_checkpoint preflight", recovery)
|
|
transaction = script.index(
|
|
"vc_fabric_operation_checkpoint transaction", preflight)
|
|
|
|
assert recovery_checkpoint < recovery < preflight < transaction
|
|
for checkpoint in FABRIC_OPERATION_CHECKPOINTS - {"dispatch"}:
|
|
assert (
|
|
f"vc_fabric_operation_checkpoint {checkpoint}" in script
|
|
or f"vc_fabric_recovery_checkpoint {checkpoint}" in script
|
|
)
|
|
|
|
|
|
def test_fabric_journal_recovery_reports_fixed_safe_subcheckpoints():
|
|
helper = _BOOT_HELPER
|
|
artifact = helper.index(
|
|
"vc_fabric_recovery_checkpoint journal_artifact_restore")
|
|
artifact_restore = helper.index("journal_sysctl_restore=no", artifact)
|
|
nft = helper.index(
|
|
"vc_fabric_recovery_checkpoint journal_nft_restore",
|
|
artifact_restore)
|
|
nft_read = helper.index(
|
|
"journal_tables=$($NFT list tables", nft)
|
|
runtime = helper.index(
|
|
"vc_fabric_recovery_checkpoint journal_runtime_restore", nft_read)
|
|
runtime_restore = helper.index(
|
|
'if [ "$journal_iface_before" = yes ]; then', runtime)
|
|
finalize = helper.index(
|
|
"vc_fabric_recovery_checkpoint journal_finalize", runtime_restore)
|
|
helper_cleanup = helper.index(
|
|
'if [ "$journal_helper_before" = absent ]; then', finalize)
|
|
|
|
assert artifact < artifact_restore < nft < nft_read
|
|
assert nft_read < runtime < runtime_restore < finalize < helper_cleanup
|
|
checkpoint_function = helper[
|
|
helper.index("vc_fabric_recovery_checkpoint()"):
|
|
helper.index("vc_legacy_firewalld_state_detected()")
|
|
]
|
|
assert "/run/virt-cloud/fabric-operations/$VC_FABRIC_OPERATION_ID" in (
|
|
checkpoint_function)
|
|
assert "0:0:700" in checkpoint_function
|
|
assert '/bin/sh "$vc_recovery_update"' in checkpoint_function
|
|
assert "|| true" in checkpoint_function
|
|
|
|
|
|
def test_fabric_apply_operation_checkpoint_uses_shell_on_noexec_run():
|
|
current = plan()
|
|
script = _apply_script(
|
|
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2})
|
|
|
|
assert '/bin/sh "$vc_operation_update"' in script
|
|
assert not re.search(
|
|
r'^\s*"\$vc_operation_update"', script, re.MULTILINE)
|
|
|
|
|
|
class FakeManager:
|
|
ssh_binary = "ssh"
|
|
|
|
def __init__(self, public_key=KEY1, extra_address=False, self_route=False,
|
|
config_digest=None, ping_ok=True, endpoint_device="eth0",
|
|
host=None, fabric_endpoint_ip="",
|
|
firewall_mode=FIREWALL_MODE_NATIVE,
|
|
live_public_key=None):
|
|
self.public_key = public_key
|
|
self.live_public_key = live_public_key or public_key
|
|
self.extra_address = extra_address
|
|
self.self_route = self_route
|
|
self.config_digest = config_digest
|
|
self.ping_ok = ping_ok
|
|
self.endpoint_device = endpoint_device
|
|
self.scripts = []
|
|
self.timeouts = []
|
|
self.reprobe_calls = 0
|
|
self.admin_state = "up"
|
|
self.ip_forward = "1"
|
|
self.rp_filter = "2"
|
|
self.rp_filter_default = "2"
|
|
self.rp_filter_iface = "2"
|
|
self.rp_filter_effective = "2"
|
|
self.sysctl_persistence = "owned"
|
|
self.emergency_state = "clear"
|
|
self.firewall_mode = firewall_mode
|
|
effective_host = host or (
|
|
"203.0.113.1" if public_key == KEY1 else "203.0.113.2")
|
|
self.effective_host = effective_host
|
|
self.underlay_endpoint = fabric_endpoint_ip or effective_host
|
|
self.profile = {
|
|
"host": effective_host,
|
|
"fabric_endpoint_ip": fabric_endpoint_ip,
|
|
"ssh_config_file": "/tmp/fake-ssh-config",
|
|
"ssh_target": "vp-managed",
|
|
"key_locked": False,
|
|
"capabilities": {
|
|
"vpc_ready": True,
|
|
"wireguard": {
|
|
"interface": FABRIC_INTERFACE,
|
|
"deployment_mode": "networkmanager_nmcli",
|
|
"ready": True,
|
|
},
|
|
"network": {
|
|
"network_manager": "networkmanager",
|
|
"network_manager_supported": True,
|
|
"firewall_manager": "nftables",
|
|
"firewall_mode": firewall_mode,
|
|
"ip_probe_valid": True,
|
|
"local_ipv4_addresses": [
|
|
fabric_endpoint_ip or effective_host,
|
|
],
|
|
"local_ipv4_interfaces": {
|
|
fabric_endpoint_ip or effective_host:
|
|
[endpoint_device],
|
|
},
|
|
"rp_filter": 2,
|
|
"rp_filter_default": 2,
|
|
"rp_filter_default_effective": 2,
|
|
"rp_filter_interfaces": {
|
|
endpoint_device: 1,
|
|
},
|
|
"networkmanager_persistent_interface_zones": {
|
|
endpoint_device: "",
|
|
},
|
|
"ifcfg_zone_interfaces": [],
|
|
},
|
|
"nftables": {
|
|
"ready": True,
|
|
"firewall_mode": firewall_mode,
|
|
},
|
|
},
|
|
}
|
|
|
|
def current(self):
|
|
return self.profile
|
|
|
|
def reprobe_current(self):
|
|
self.reprobe_calls += 1
|
|
return self.profile["capabilities"]
|
|
|
|
def _exec(self, argv, *, timeout, input_text=None):
|
|
assert argv[-1] == "/bin/sh -s"
|
|
self.timeouts.append(timeout)
|
|
self.scripts.append(input_text)
|
|
if input_text.startswith("#!/bin/sh\nset -eu") and "__VC_FABRIC_PUBLIC_KEY__=" in input_text:
|
|
output = f"__VC_FABRIC_PUBLIC_KEY__={self.public_key}\n"
|
|
elif "__VC_FABRIC_APPLIED__" in input_text:
|
|
self.live_public_key = self.public_key
|
|
output = "__VC_FABRIC_APPLIED__\n"
|
|
else:
|
|
now = int(time.time())
|
|
peer = KEY2 if self.public_key == KEY1 else KEY1
|
|
address = "100.64.1.1" if self.public_key == KEY1 else "100.64.1.2"
|
|
target = "100.64.1.2" if self.public_key == KEY1 else "100.64.1.1"
|
|
endpoint = "203.0.113.2:51820" if self.public_key == KEY1 else "203.0.113.1:51820"
|
|
extra_address = (
|
|
"__VC_FABRIC_ADDRESS__=100.64.1.99\n"
|
|
if self.extra_address else ""
|
|
)
|
|
self_route = (
|
|
f"__VC_FABRIC_ROUTE__={address}|local|{FABRIC_INTERFACE}\n"
|
|
if self.self_route else ""
|
|
)
|
|
output = (
|
|
f"__VC_FABRIC_OWNER__={FABRIC_OWNER_MARKER}\n"
|
|
"__VC_FABRIC_CONFIG_FABRIC__=hk-fabric\n"
|
|
"__VC_FABRIC_CONFIG_REVISION__=1\n"
|
|
f"__VC_FABRIC_CONFIG_DIGEST__={self.config_digest or plan().digest}\n"
|
|
f"__VC_FABRIC_CONFIG_FIREWALL__={self.firewall_mode}\n"
|
|
"__VC_FABRIC_CONFIG_NETWORK__=100.64.1.0/24\n"
|
|
f"__VC_FABRIC_CONFIG_OVERLAY__={address}\n"
|
|
"__VC_FABRIC_CONFIG_OK__=yes\n"
|
|
f"__VC_FABRIC_CONFIG_PEER__={peer}|{endpoint}|{target}/32\n"
|
|
f"__VC_FABRIC_CONFIG_ROUTE__={target}/32\n"
|
|
f"__VC_FABRIC_PUBLIC_KEY__={self.public_key}\n"
|
|
"__VC_FABRIC_INTERFACE__=wireguard\n"
|
|
f"__VC_FABRIC_LIVE_PUBLIC_KEY__={self.live_public_key}\n"
|
|
"__VC_FABRIC_LISTEN__=51820\n"
|
|
f"__VC_FABRIC_ADDRESS__={address}\n"
|
|
f"{extra_address}"
|
|
f"__VC_FABRIC_PEER__={peer}|{endpoint}|{target}/32\n"
|
|
f"__VC_FABRIC_HANDSHAKE__={peer}|{now}\n"
|
|
f"{self_route}"
|
|
f"__VC_FABRIC_ROUTE__={target}/32|main|{FABRIC_INTERFACE}\n"
|
|
f"__VC_FABRIC_PATH__={target}|{FABRIC_INTERFACE}\n"
|
|
f"__VC_FABRIC_ENDPOINT_PATH__={endpoint.split(':', 1)[0]}|{self.endpoint_device}|{self.underlay_endpoint}\n"
|
|
f"__VC_FABRIC_PING__={target}|{'ok' if self.ping_ok else 'failed'}\n"
|
|
f"__VC_FABRIC_ROUTE_OVERLAP__={target}|clean\n"
|
|
"__VC_FABRIC_MANAGER_MODE__=networkmanager_nmcli\n"
|
|
"__VC_FABRIC_MANAGER_OWNER__=owned\n"
|
|
"__VC_FABRIC_MANAGER_ACTIVE__=active\n"
|
|
f"__VC_FABRIC_FIREWALL_MODE__={self.firewall_mode}\n"
|
|
"__VC_FABRIC_FIREWALL_STATE__=owned\n"
|
|
f"__VC_FABRIC_ADMIN_STATE__={self.admin_state}\n"
|
|
f"__VC_FABRIC_SYSCTL__ip_forward={self.ip_forward}\n"
|
|
f"__VC_FABRIC_SYSCTL__rp_filter={self.rp_filter}\n"
|
|
f"__VC_FABRIC_SYSCTL__rp_filter_default={self.rp_filter_default}\n"
|
|
f"__VC_FABRIC_SYSCTL__rp_filter_effective={self.rp_filter_effective}\n"
|
|
f"__VC_FABRIC_RPF__wg-fabric0={self.rp_filter_iface}\n"
|
|
f"__VC_FABRIC_SYSCTL_PERSISTENCE__={self.sysctl_persistence}\n"
|
|
"__VC_FABRIC_ENABLED__=enabled\n"
|
|
"__VC_FABRIC_ACTIVE__=active\n"
|
|
"__VC_FABRIC_TIMER_ENABLED__=enabled\n"
|
|
"__VC_FABRIC_TIMER_ACTIVE__=active\n"
|
|
"__VC_FABRIC_NFT_GUARD__=owned\n"
|
|
"__VC_FABRIC_NFT_PERSISTENCE__=owned\n"
|
|
f"__VC_FABRIC_EMERGENCY__={self.emergency_state}\n"
|
|
"__VC_FABRIC_VPC_NFT_STATE__=owned\n"
|
|
"__VC_FABRIC_KEY_SECURE__=yes\n"
|
|
"__VC_FABRIC_CONFIG_SECURE__=yes\n"
|
|
"__VC_FABRIC_HELPER__=owned\n"
|
|
"__VC_FABRIC_UNIT__=owned\n"
|
|
"__VC_FABRIC_REPAIR__=owned\n"
|
|
"__VC_FABRIC_TIMER__=owned\n"
|
|
"__VC_FABRIC_LIBVIRT_GATE__=owned\n"
|
|
"__VC_FABRIC_LIBVIRT_DEPENDENCIES__=owned\n"
|
|
)
|
|
return subprocess.CompletedProcess(argv, 0, output, "")
|
|
|
|
|
|
class AsyncOperationManager(FakeManager):
|
|
def __init__(self, *args, operation_id, status="queued",
|
|
checkpoint="dispatch", reason="none", **kwargs):
|
|
super().__init__(*args, **kwargs)
|
|
self.operation_id = operation_id
|
|
self.operation_status = status
|
|
self.operation_checkpoint = checkpoint
|
|
self.operation_reason = reason
|
|
self.dispatch_result = "yes"
|
|
self.disconnect_dispatch = False
|
|
|
|
def _status_output(self):
|
|
started_at = 0
|
|
finished_at = 0
|
|
if self.operation_status != "queued":
|
|
started_at = 101
|
|
if self.operation_status in {
|
|
"succeeded", "failed", "uncertain", "interrupted"}:
|
|
finished_at = 103
|
|
return _operation_status_output(
|
|
self.operation_id,
|
|
status=self.operation_status,
|
|
checkpoint=self.operation_checkpoint,
|
|
reason=self.operation_reason,
|
|
started_at=started_at,
|
|
updated_at=103 if finished_at else 102,
|
|
finished_at=finished_at,
|
|
)
|
|
|
|
def _exec(self, argv, *, timeout, input_text=None):
|
|
assert argv[-1] == "/bin/sh -s"
|
|
self.timeouts.append(timeout)
|
|
self.scripts.append(input_text)
|
|
if 'SYSTEMD_RUN=$(find_tool ' in input_text:
|
|
if self.disconnect_dispatch:
|
|
raise RemoteHostError(
|
|
"ambiguous disconnect with /secret/path", 502,
|
|
"host_remote_command_failed")
|
|
output = (
|
|
f"__VC_FABRIC_OPERATION_DISPATCHED__={self.dispatch_result}\n"
|
|
f"__VC_FABRIC_OPERATION_ID__={self.operation_id}\n"
|
|
)
|
|
return subprocess.CompletedProcess(argv, 0, output, "")
|
|
if "__VC_FABRIC_OPERATION_SCHEMA__=$schema" in input_text:
|
|
return subprocess.CompletedProcess(
|
|
argv, 0, self._status_output(), "")
|
|
raise AssertionError("unexpected asynchronous Fabric command")
|
|
|
|
|
|
def test_adapter_dispatches_apply_and_reads_status_without_waiting_for_job():
|
|
current = plan()
|
|
operation_id = "b" * 32
|
|
manager = AsyncOperationManager(KEY1, operation_id=operation_id)
|
|
adapter = FabricHostAdapter(manager)
|
|
|
|
result = adapter.dispatch_apply(
|
|
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
|
|
operation_id)
|
|
|
|
assert result == {
|
|
"operation_id": operation_id,
|
|
"status": "queued",
|
|
"phase": "apply",
|
|
"checkpoint": "dispatch",
|
|
"reason": "none",
|
|
"created_at": 100,
|
|
"started_at": None,
|
|
"updated_at": 102,
|
|
"finished_at": None,
|
|
}
|
|
serialized = json.dumps(result)
|
|
assert KEY1 not in serialized
|
|
assert KEY2 not in serialized
|
|
assert "/run/virt-cloud" not in serialized
|
|
assert manager.timeouts == [
|
|
FABRIC_REMOTE_READ_TIMEOUT_SECONDS,
|
|
FABRIC_REMOTE_READ_TIMEOUT_SECONDS,
|
|
]
|
|
assert len(manager.scripts) == 2
|
|
assert "--no-block" in manager.scripts[0]
|
|
assert "__VC_FABRIC_APPLIED__" in manager.scripts[0]
|
|
assert "__VC_FABRIC_OPERATION_STATUS__=$state" in manager.scripts[1]
|
|
|
|
|
|
@pytest.mark.parametrize(("status", "checkpoint", "reason"), [
|
|
("running", "journal_recovery", "none"),
|
|
("succeeded", "complete", "none"),
|
|
("failed", "nft_install", "nft_failed"),
|
|
])
|
|
def test_adapter_can_read_exact_attempt_after_dispatch_connection_loss(
|
|
status, checkpoint, reason):
|
|
current = plan()
|
|
operation_id = "c" * 32
|
|
manager = AsyncOperationManager(
|
|
KEY1, operation_id=operation_id, status=status,
|
|
checkpoint=checkpoint, reason=reason)
|
|
manager.disconnect_dispatch = True
|
|
adapter = FabricHostAdapter(manager)
|
|
|
|
with pytest.raises(FabricError) as error:
|
|
adapter.dispatch_apply(
|
|
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
|
|
operation_id)
|
|
|
|
assert error.value.error_code == "fabric_host_connection_failed"
|
|
assert error.value.failure_phase == "apply"
|
|
assert len(manager.scripts) == 1
|
|
|
|
# A lost dispatch response is reconciled only by its exact operation id;
|
|
# the adapter does not submit the mutation a second time.
|
|
manager.disconnect_dispatch = False
|
|
observed = adapter.apply_status(operation_id)
|
|
assert observed["operation_id"] == operation_id
|
|
assert observed["status"] == status
|
|
assert observed["checkpoint"] == checkpoint
|
|
assert observed["reason"] == reason
|
|
assert len(manager.scripts) == 2
|
|
assert 'SYSTEMD_RUN=$(find_tool ' not in manager.scripts[-1]
|
|
|
|
|
|
def test_adapter_rejects_cross_attempt_status_without_redispatch():
|
|
operation_id = "d" * 32
|
|
manager = AsyncOperationManager(KEY1, operation_id="e" * 32)
|
|
adapter = FabricHostAdapter(manager)
|
|
|
|
with pytest.raises(FabricError) as error:
|
|
adapter.apply_status(operation_id)
|
|
|
|
assert error.value.error_code == "fabric_operation_status_invalid"
|
|
assert len(manager.scripts) == 1
|
|
assert 'SYSTEMD_RUN=$(find_tool ' not in manager.scripts[0]
|
|
|
|
|
|
def test_adapter_reports_exact_missing_operation_without_leaking_paths():
|
|
operation_id = "a" * 32
|
|
manager = AsyncOperationManager(KEY1, operation_id=operation_id)
|
|
manager._status_output = lambda: (
|
|
"__VC_FABRIC_OPERATION_MISSING__=yes\n"
|
|
f"__VC_FABRIC_OPERATION_ID__={operation_id}\n"
|
|
)
|
|
adapter = FabricHostAdapter(manager)
|
|
|
|
with pytest.raises(FabricError) as error:
|
|
adapter.apply_status(operation_id)
|
|
|
|
assert error.value.status == 409
|
|
assert error.value.error_code == "fabric_operation_not_found"
|
|
assert error.value.failure_phase == "observe"
|
|
assert "/run/virt-cloud" not in str(error.value)
|
|
assert KEY1 not in str(error.value)
|
|
assert len(manager.scripts) == 1
|
|
assert 'SYSTEMD_RUN=$(find_tool ' not in manager.scripts[0]
|
|
|
|
|
|
def test_adapter_validates_operation_id_before_remote_dispatch():
|
|
current = plan()
|
|
manager = AsyncOperationManager(KEY1, operation_id="f" * 32)
|
|
adapter = FabricHostAdapter(manager)
|
|
|
|
with pytest.raises(FabricError) as error:
|
|
adapter.dispatch_apply(
|
|
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
|
|
"../not-an-operation")
|
|
|
|
assert error.value.error_code == "fabric_operation_invalid"
|
|
assert manager.scripts == []
|
|
|
|
|
|
def test_adapter_readback_requires_exact_peer_route_and_persistence():
|
|
current = plan()
|
|
manager = FakeManager(KEY1)
|
|
adapter = FabricHostAdapter(manager)
|
|
assert adapter.identity() == KEY1
|
|
assert adapter.observe(
|
|
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}
|
|
)["transport_verified"] is True
|
|
rendered_observe = manager.scripts[-1]
|
|
digest_assignment = f"EXPECTED_DIGEST={shlex.quote(current.digest)}"
|
|
assert rendered_observe.index(digest_assignment) < rendered_observe.index(
|
|
"vc_fabric_nm_profile_owned expected"
|
|
)
|
|
|
|
|
|
def test_adapter_uses_stage_specific_remote_timeouts():
|
|
current = plan()
|
|
manager = FakeManager(KEY1)
|
|
adapter = FabricHostAdapter(manager)
|
|
public_keys = {"host-a": KEY1, "host-b": KEY2}
|
|
|
|
assert adapter.identity() == KEY1
|
|
adapter.apply(current, current.nodes[0], public_keys)
|
|
adapter.observe(current, current.nodes[0], public_keys)
|
|
|
|
assert [
|
|
FABRIC_REMOTE_READ_TIMEOUT_SECONDS,
|
|
FABRIC_REMOTE_APPLY_TIMEOUT_SECONDS,
|
|
FABRIC_REMOTE_READ_TIMEOUT_SECONDS,
|
|
] == [35, 300, 35]
|
|
assert manager.timeouts == [35, 300, 35]
|
|
|
|
|
|
@pytest.mark.parametrize("failure_phase", ["identity", "apply", "observe"])
|
|
def test_adapter_remote_failure_exposes_only_whitelisted_phase(failure_phase):
|
|
class FailedRemoteManager(FakeManager):
|
|
def _exec(self, argv, *, timeout, input_text=None):
|
|
self.scripts.append(input_text)
|
|
return subprocess.CompletedProcess(
|
|
argv, 71, "", "secret /root/path private-key-material")
|
|
|
|
current = plan()
|
|
adapter = FabricHostAdapter(FailedRemoteManager(KEY1))
|
|
with pytest.raises(FabricError) as error:
|
|
if failure_phase == "identity":
|
|
adapter.identity()
|
|
elif failure_phase == "apply":
|
|
adapter.apply(
|
|
current, current.nodes[0],
|
|
{"host-a": KEY1, "host-b": KEY2})
|
|
else:
|
|
adapter.observe(
|
|
current, current.nodes[0],
|
|
{"host-a": KEY1, "host-b": KEY2})
|
|
|
|
assert error.value.error_code == "fabric_remote_command_failed"
|
|
assert error.value.failure_phase == failure_phase
|
|
safe_error = json.dumps({
|
|
"message": str(error.value),
|
|
"error_code": error.value.error_code,
|
|
"failure_phase": error.value.failure_phase,
|
|
})
|
|
assert "secret" not in safe_error
|
|
assert "/root/path" not in safe_error
|
|
assert "private-key-material" not in safe_error
|
|
|
|
|
|
|
|
|
|
def test_profile_firewall_mode_rejects_missing_or_crossed_capability_evidence():
|
|
manager = FakeManager(KEY1)
|
|
assert _profile_firewall_mode(manager.profile) == FIREWALL_MODE_NATIVE
|
|
|
|
manager.profile["capabilities"]["network"].update({
|
|
"firewall_manager": "firewalld",
|
|
"firewall_mode": "firewalld_nft",
|
|
})
|
|
manager.profile["capabilities"]["nftables"]["firewall_mode"] = (
|
|
"firewalld_nft")
|
|
with pytest.raises(FabricError) as error:
|
|
_profile_firewall_mode(manager.profile)
|
|
assert error.value.error_code == "fabric_host_firewall_mode_conflict"
|
|
|
|
manager.profile["capabilities"]["nftables"]["firewall_mode"] = (
|
|
FIREWALL_MODE_NATIVE)
|
|
with pytest.raises(FabricError) as error:
|
|
_profile_firewall_mode(manager.profile)
|
|
assert error.value.error_code == "fabric_host_firewall_mode_conflict"
|
|
|
|
|
|
def test_adapter_rejects_an_unexpected_overlay_address():
|
|
current = plan()
|
|
manager = FakeManager(KEY1, extra_address=True)
|
|
observed = FabricHostAdapter(manager).observe(
|
|
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}
|
|
)
|
|
assert observed["config_ok"] is False
|
|
assert observed["state"] == "degraded"
|
|
|
|
|
|
def test_adapter_ignores_the_kernel_connected_route_for_its_own_address():
|
|
current = plan()
|
|
manager = FakeManager(KEY1, self_route=True)
|
|
observed = FabricHostAdapter(manager).observe(
|
|
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}
|
|
)
|
|
assert observed["config_ok"] is True
|
|
|
|
|
|
def test_adapter_requires_admin_up_and_fabric_sysctl_readback():
|
|
current = plan()
|
|
manager = FakeManager(KEY1)
|
|
manager.admin_state = "down"
|
|
assert FabricHostAdapter(manager).observe(
|
|
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}
|
|
)["config_ok"] is False
|
|
|
|
manager.admin_state = "up"
|
|
manager.rp_filter_iface = "1"
|
|
observed = FabricHostAdapter(manager).observe(
|
|
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}
|
|
)
|
|
assert observed["config_ok"] is False
|
|
assert observed["observed"]["admin_state"] == "up"
|
|
assert observed["observed"]["rp_filters"][FABRIC_INTERFACE] == "1"
|
|
|
|
|
|
def test_adapter_accepts_strict_global_defaults_when_owned_fabric_is_loose():
|
|
current = plan()
|
|
manager = FakeManager(KEY1)
|
|
manager.rp_filter = "0"
|
|
manager.rp_filter_default = "1"
|
|
manager.rp_filter_iface = "2"
|
|
manager.rp_filter_effective = "2"
|
|
|
|
observed = FabricHostAdapter(manager).observe(
|
|
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
|
|
)
|
|
|
|
assert observed["config_ok"] is True
|
|
assert observed["observed"]["sysctls"] == {
|
|
"ip_forward": "1",
|
|
"rp_filter": "0",
|
|
"rp_filter_default": "1",
|
|
"rp_filter_effective": "2",
|
|
}
|
|
|
|
|
|
def test_adapter_requires_exact_owned_sysctl_persistence():
|
|
current = plan()
|
|
manager = FakeManager(KEY1)
|
|
manager.sysctl_persistence = "foreign"
|
|
|
|
observed = FabricHostAdapter(manager).observe(
|
|
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
|
|
)
|
|
|
|
assert observed["config_ok"] is False
|
|
assert observed["observed"]["sysctl_persistence"] == "foreign"
|
|
|
|
|
|
@pytest.mark.parametrize("emergency_state", ["present", "unknown"])
|
|
def test_adapter_never_publishes_active_while_emergency_guard_remains(
|
|
emergency_state):
|
|
current = plan()
|
|
manager = FakeManager(KEY1)
|
|
manager.emergency_state = emergency_state
|
|
|
|
observed = FabricHostAdapter(manager).observe(
|
|
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
|
|
)
|
|
|
|
assert observed["config_ok"] is False
|
|
assert observed["state"] == "degraded"
|
|
assert observed["observed"]["emergency_state"] == emergency_state
|
|
|
|
|
|
def test_adapter_rejects_stale_config_digest_and_failed_probe():
|
|
current = plan()
|
|
stale = FakeManager(KEY1, config_digest="0" * 64)
|
|
observed = FabricHostAdapter(stale).observe(
|
|
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}
|
|
)
|
|
assert observed["config_ok"] is False
|
|
|
|
|
|
def test_adapter_rejects_endpoint_egress_drift_to_fabric_or_missing_device():
|
|
current = plan()
|
|
for device in (FABRIC_INTERFACE, ""):
|
|
observed = FabricHostAdapter(FakeManager(KEY1, endpoint_device=device)).observe(
|
|
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}
|
|
)
|
|
assert observed["config_ok"] is False
|
|
failed_ping = FakeManager(KEY1, ping_ok=False)
|
|
observed = FabricHostAdapter(failed_ping).observe(
|
|
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}
|
|
)
|
|
assert observed["config_ok"] is True
|
|
assert observed["state"] == "active"
|
|
assert observed["transport_verified"] is False
|
|
|
|
|
|
def test_adapter_rejects_live_public_key_drift_from_persistent_key():
|
|
current = plan()
|
|
observed = FabricHostAdapter(FakeManager(
|
|
KEY1, live_public_key=KEY3,
|
|
)).observe(
|
|
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2},
|
|
)
|
|
|
|
assert observed["public_key"] == KEY1
|
|
assert observed["observed"]["live_public_key"] == KEY3
|
|
assert observed["config_ok"] is False
|
|
assert observed["state"] == "degraded"
|
|
|
|
|
|
class MemoryStore:
|
|
def __init__(self):
|
|
self.values = {}
|
|
|
|
def get(self, kind, key):
|
|
return self.values.get((kind, key))
|
|
|
|
def set(self, kind, key, value):
|
|
self.values[(kind, key)] = value
|
|
|
|
def list(self, kind):
|
|
return {key: value for (row_kind, key), value in self.values.items()
|
|
if row_kind == kind}
|
|
|
|
|
|
class FakeAdapter:
|
|
failures = set()
|
|
observe_failures = set()
|
|
|
|
def __init__(self, manager, profile=None):
|
|
self.manager = manager
|
|
|
|
def identity(self):
|
|
return self.manager.public_key
|
|
|
|
def apply(self, *_args):
|
|
if self.manager.host_id in self.failures:
|
|
raise FabricError("offline", 502, "fabric_remote_command_failed")
|
|
|
|
def observe(self, current, host, keys):
|
|
if (self.manager.host_id in self.failures
|
|
or self.manager.host_id in self.observe_failures):
|
|
raise FabricError("offline", 502, "fabric_remote_command_failed")
|
|
return {
|
|
"state": "active", "transport_verified": True,
|
|
"observed": {"interface": FABRIC_INTERFACE},
|
|
}
|
|
|
|
|
|
class Registry:
|
|
def __init__(self, components):
|
|
self.components = components
|
|
|
|
async def get(self, host_id):
|
|
return self.components[host_id]
|
|
|
|
|
|
class UnconfiguredRegistry:
|
|
def __init__(self):
|
|
self.components = {}
|
|
self.get_called = False
|
|
|
|
async def is_configured(self, _host_id):
|
|
return False
|
|
|
|
async def get(self, _host_id):
|
|
self.get_called = True
|
|
raise AssertionError("readiness probe constructed an unknown host")
|
|
|
|
|
|
class ProbeManager:
|
|
def __init__(self, host_id, *, host="203.0.113.10",
|
|
fabric_endpoint_ip="", ready=True, failure=None, counter=None):
|
|
self.host_id = host_id
|
|
self.failure = failure
|
|
self.counter = counter
|
|
self.profile = {
|
|
"host_id": host_id,
|
|
"host": host,
|
|
"fabric_endpoint_ip": fabric_endpoint_ip,
|
|
"region": "hk",
|
|
"ssh_config_file": "/secret/ssh/config",
|
|
"private_key": "must-not-leak",
|
|
}
|
|
effective_endpoint = fabric_endpoint_ip or host
|
|
endpoint_is_ipv4 = bool(re.fullmatch(
|
|
r"(?:[0-9]{1,3}\.){3}[0-9]{1,3}", effective_endpoint))
|
|
self.capabilities = {
|
|
"vpc_ready": ready,
|
|
"reasons": [] if ready else [
|
|
"firewall_owner_not_nftables", "rp_filter_not_compatible",
|
|
],
|
|
"os": {"id": "almalinux", "version_id": "10",
|
|
"name": "must-not-be-projected"},
|
|
"network": {
|
|
"network_manager": "networkmanager",
|
|
"network_manager_expected": "networkmanager",
|
|
"network_manager_supported": True,
|
|
"firewall_manager": "nftables" if ready else "firewalld",
|
|
"firewall_mode": "native_nftables" if ready else "unsupported",
|
|
"ip_forward": 1,
|
|
"rp_filter": 2 if ready else 1,
|
|
"rp_filter_default": 1,
|
|
"rp_filter_default_effective": 2 if ready else 1,
|
|
"rp_filter_interfaces": (
|
|
{"br0": 1} if endpoint_is_ipv4 else {}),
|
|
"local_ipv4_addresses": (
|
|
[effective_endpoint] if endpoint_is_ipv4 else []),
|
|
"local_ipv4_interfaces": (
|
|
{effective_endpoint: ["br0"]}
|
|
if endpoint_is_ipv4 else {}),
|
|
"fabric_rp_filter_raw": None,
|
|
"fabric_rp_filter_effective": None,
|
|
"raw_output": "must-not-leak",
|
|
},
|
|
"wireguard": {
|
|
"deployment_mode": "networkmanager_nmcli",
|
|
"ready": True,
|
|
"owned_by_panel": False,
|
|
"ownership_marker": "must-not-be-projected",
|
|
},
|
|
"nftables": {
|
|
"ready": ready,
|
|
"firewall_mode": "native_nftables" if ready else "unsupported",
|
|
"service_active": ready,
|
|
"service_persistent": ready,
|
|
},
|
|
"network_roles": {
|
|
"public": {"selector": "bridge:br0", "verified": True},
|
|
},
|
|
"tools": {"wg": {"path": "/usr/bin/wg"}},
|
|
"raw_output": "must-not-leak",
|
|
}
|
|
|
|
def current(self):
|
|
return self.profile
|
|
|
|
def reprobe_current_snapshot(self):
|
|
if self.counter is not None:
|
|
with self.counter["lock"]:
|
|
self.counter["active"] += 1
|
|
self.counter["maximum"] = max(
|
|
self.counter["maximum"], self.counter["active"])
|
|
if self.counter["active"] == self.counter.get("target"):
|
|
self.counter["gate"].set()
|
|
try:
|
|
gate = self.counter.get("gate")
|
|
if gate is not None:
|
|
gate.wait(timeout=2)
|
|
else:
|
|
time.sleep(0.02)
|
|
finally:
|
|
with self.counter["lock"]:
|
|
self.counter["active"] -= 1
|
|
if self.failure is not None:
|
|
raise self.failure
|
|
return self.profile, self.capabilities
|
|
|
|
|
|
def test_fabric_components_treat_host_region_as_a_display_label():
|
|
components = {}
|
|
labels = {
|
|
"host-a": "中国香港",
|
|
"host-b": "Hong Kong",
|
|
}
|
|
for host_id, key in (("host-a", KEY1), ("host-b", KEY2)):
|
|
manager = FakeManager(key)
|
|
manager.profile["host_id"] = host_id
|
|
manager.profile["region"] = labels[host_id]
|
|
components[host_id] = SimpleNamespace(manager=manager)
|
|
|
|
resolved = asyncio.run(FabricCoordinator(
|
|
MemoryStore(), Registry(components),
|
|
)._components(plan()))
|
|
|
|
assert set(resolved) == {"host-a", "host-b"}
|
|
assert resolved["host-a"][1]["region"] == "中国香港"
|
|
assert resolved["host-b"][1]["region"] == "Hong Kong"
|
|
assert all(component.manager.reprobe_calls == 1
|
|
for component, _profile in resolved.values())
|
|
|
|
|
|
def test_fabric_components_accept_configured_local_underlay_endpoints():
|
|
components = {}
|
|
for host_id, key, endpoint in (
|
|
("host-a", KEY1, "203.0.113.1"),
|
|
("host-b", KEY2, "203.0.113.2"),
|
|
):
|
|
manager = FakeManager(
|
|
key, host="host.example.test", fabric_endpoint_ip=endpoint)
|
|
manager.profile["host_id"] = host_id
|
|
components[host_id] = SimpleNamespace(manager=manager)
|
|
|
|
resolved = asyncio.run(FabricCoordinator(
|
|
MemoryStore(), Registry(components),
|
|
)._components(plan()))
|
|
|
|
assert set(resolved) == {"host-a", "host-b"}
|
|
|
|
|
|
def test_native_ssh_ipv4_fallback_rejects_nat_without_local_mapping():
|
|
first = FakeManager(KEY1, host="203.0.113.1")
|
|
first.profile["host_id"] = "host-a"
|
|
first.profile["capabilities"]["network"]["local_ipv4_addresses"] = []
|
|
first.profile["capabilities"]["network"]["local_ipv4_interfaces"] = {}
|
|
first.profile["capabilities"]["network"]["rp_filter_interfaces"] = {}
|
|
second = FakeManager(KEY2)
|
|
second.profile["host_id"] = "host-b"
|
|
|
|
with pytest.raises(FabricError) as error:
|
|
asyncio.run(FabricCoordinator(
|
|
MemoryStore(), Registry({
|
|
"host-a": SimpleNamespace(manager=first),
|
|
"host-b": SimpleNamespace(manager=second),
|
|
}),
|
|
)._components(plan()))
|
|
|
|
assert error.value.error_code == "fabric_endpoint_not_local"
|
|
|
|
|
|
@pytest.mark.parametrize(("profile_changes", "error_code"), [
|
|
({"host": "203.0.113.9"}, "fabric_endpoint_not_local"),
|
|
({"fabric_endpoint_ip": "127.0.0.1"}, "fabric_endpoint_invalid"),
|
|
])
|
|
def test_fabric_components_reject_mismatched_or_invalid_effective_endpoint(
|
|
profile_changes, error_code,
|
|
):
|
|
first = FakeManager(KEY1)
|
|
first.profile.update(profile_changes)
|
|
first.profile["host_id"] = "host-a"
|
|
second = FakeManager(KEY2)
|
|
second.profile["host_id"] = "host-b"
|
|
coordinator = FabricCoordinator(MemoryStore(), Registry({
|
|
"host-a": SimpleNamespace(manager=first),
|
|
"host-b": SimpleNamespace(manager=second),
|
|
}))
|
|
|
|
with pytest.raises(FabricError) as error:
|
|
asyncio.run(coordinator._components(plan()))
|
|
assert error.value.status == 409
|
|
assert error.value.error_code == error_code
|
|
|
|
|
|
def test_fabric_components_reject_configured_endpoint_without_local_evidence():
|
|
first = FakeManager(KEY1, fabric_endpoint_ip="203.0.113.1")
|
|
first.profile["host_id"] = "host-a"
|
|
first.profile["capabilities"]["network"]["local_ipv4_addresses"] = []
|
|
second = FakeManager(KEY2)
|
|
second.profile["host_id"] = "host-b"
|
|
coordinator = FabricCoordinator(MemoryStore(), Registry({
|
|
"host-a": SimpleNamespace(manager=first),
|
|
"host-b": SimpleNamespace(manager=second),
|
|
}))
|
|
|
|
with pytest.raises(FabricError) as error:
|
|
asyncio.run(coordinator._components(plan()))
|
|
assert error.value.status == 409
|
|
assert error.value.error_code == "fabric_endpoint_not_local"
|
|
|
|
first.profile["capabilities"]["network"]["local_ipv4_addresses"] = [
|
|
"203.0.113.1",
|
|
]
|
|
first.profile["capabilities"]["network"]["local_ipv4_interfaces"] = {
|
|
"203.0.113.1": ["br0"],
|
|
}
|
|
first.profile["capabilities"]["network"]["rp_filter_interfaces"] = {}
|
|
with pytest.raises(FabricError) as error:
|
|
asyncio.run(coordinator._components(plan()))
|
|
assert error.value.error_code == "fabric_endpoint_rp_filter_unverified"
|
|
|
|
|
|
@pytest.mark.parametrize("value", [
|
|
{},
|
|
{"host_ids": []},
|
|
{"host_ids": ["host-a"], "extra": True},
|
|
{"host_ids": ["host-a", "host-a"]},
|
|
{"host_ids": ["bad.id"]},
|
|
{"host_ids": ["host-a"] * 65},
|
|
])
|
|
def test_readiness_probe_rejects_non_exact_or_unbounded_host_ids(value):
|
|
coordinator = FabricCoordinator(MemoryStore(), Registry({}))
|
|
with pytest.raises(FabricError) as error:
|
|
asyncio.run(coordinator.probe_hosts(value))
|
|
assert error.value.status == 422
|
|
assert error.value.error_code == "fabric_probe_invalid"
|
|
|
|
|
|
def test_readiness_probe_is_bounded_isolated_and_secret_free():
|
|
counter = {
|
|
"active": 0,
|
|
"maximum": 0,
|
|
"lock": threading.Lock(),
|
|
"target": FABRIC_PROBE_CONCURRENCY,
|
|
"gate": threading.Event(),
|
|
}
|
|
components = {}
|
|
host_ids = []
|
|
for index in range(FABRIC_PROBE_CONCURRENCY + 4):
|
|
host_id = f"host-{index}"
|
|
host_ids.append(host_id)
|
|
failure = None
|
|
if index == 1:
|
|
failure = RemoteHostError(
|
|
"remote output /secret/path", 502,
|
|
"host_capability_probe_failed")
|
|
manager = ProbeManager(
|
|
host_id,
|
|
host="host.example.test" if index == 2 else f"203.0.113.{index + 1}",
|
|
ready=index != 3,
|
|
failure=failure,
|
|
counter=counter,
|
|
)
|
|
components[host_id] = SimpleNamespace(manager=manager)
|
|
|
|
coordinator = FabricCoordinator(MemoryStore(), Registry(components))
|
|
|
|
async def run_with_constrained_default_executor():
|
|
# Prove readiness uses its own 16-worker executor rather than merely
|
|
# inheriting whatever size asyncio chose for the process-wide pool.
|
|
asyncio.get_running_loop().set_default_executor(ThreadPoolExecutor(
|
|
max_workers=1, thread_name_prefix="test-default",
|
|
))
|
|
return await coordinator.probe_hosts({"host_ids": host_ids})
|
|
|
|
result = asyncio.run(run_with_constrained_default_executor())
|
|
coordinator.close()
|
|
|
|
assert [item["host_id"] for item in result["hosts"]] == host_ids
|
|
assert counter["maximum"] == FABRIC_PROBE_CONCURRENCY
|
|
assert result["hosts"][0] == {
|
|
"host_id": "host-0",
|
|
"configured": True,
|
|
"reachable": True,
|
|
"region": "hk",
|
|
"endpoint_ip": "203.0.113.1",
|
|
"endpoint_source": "ssh_host_ipv4",
|
|
"probed_at": result["hosts"][0]["probed_at"],
|
|
"vpc_ready": True,
|
|
"reasons": [],
|
|
"capabilities": {
|
|
"os_id": "almalinux",
|
|
"os_version": "10",
|
|
"network_manager": "networkmanager",
|
|
"network_manager_expected": "networkmanager",
|
|
"network_manager_supported": True,
|
|
"deployment_mode": "networkmanager_nmcli",
|
|
"firewall_manager": "nftables",
|
|
"firewall_mode": "native_nftables",
|
|
"underlay_interface": "br0",
|
|
"ip_forward": 1,
|
|
"rp_filter": 2,
|
|
"rp_filter_all": 2,
|
|
"rp_filter_default": 1,
|
|
"rp_filter_default_effective": 2,
|
|
"rp_filter_underlay_raw": 1,
|
|
"rp_filter_underlay_effective": 2,
|
|
"rp_filter_fabric_raw": None,
|
|
"rp_filter_fabric_effective": None,
|
|
"wireguard_ready": True,
|
|
"wireguard_owned": False,
|
|
"nftables_ready": True,
|
|
"nftables_active": True,
|
|
"nftables_persistent": True,
|
|
"public_selector": "bridge:br0",
|
|
"public_verified": True,
|
|
},
|
|
}
|
|
failed = result["hosts"][1]
|
|
assert failed["configured"] is True
|
|
assert failed["reachable"] is False
|
|
assert failed["error_code"] == "fabric_host_capability_probe_failed"
|
|
assert failed["reasons"] == ["fabric_host_capability_probe_failed"]
|
|
assert result["hosts"][2]["endpoint_ip"] is None
|
|
assert result["hosts"][2]["endpoint_source"] is None
|
|
assert result["hosts"][3]["vpc_ready"] is False
|
|
assert result["hosts"][3]["reasons"] == [
|
|
"firewall_owner_not_nftables", "rp_filter_not_compatible",
|
|
]
|
|
|
|
|
|
@pytest.mark.parametrize(("raw", "projected"), [
|
|
(FIREWALL_MODE_NATIVE, FIREWALL_MODE_NATIVE),
|
|
("firewalld_nft", "unsupported"),
|
|
("../../operator-data", "unsupported"),
|
|
(None, "unsupported"),
|
|
])
|
|
def test_readiness_projects_only_supported_firewall_modes(raw, projected):
|
|
result = FabricCoordinator._readiness_capabilities({
|
|
"network": {"firewall_mode": raw},
|
|
})
|
|
assert result["firewall_mode"] == projected
|
|
serialized = json.dumps(result)
|
|
assert "must-not-leak" not in serialized
|
|
assert "/secret/" not in serialized
|
|
assert "/usr/bin/wg" not in serialized
|
|
|
|
|
|
def test_readiness_probe_does_not_construct_an_unconfigured_host():
|
|
registry = UnconfiguredRegistry()
|
|
result = asyncio.run(FabricCoordinator(
|
|
MemoryStore(), registry,
|
|
).probe_hosts({"host_ids": ["host-unknown"]}))
|
|
|
|
assert registry.get_called is False
|
|
item = result["hosts"][0]
|
|
assert item == {
|
|
"host_id": "host-unknown",
|
|
"configured": False,
|
|
"reachable": False,
|
|
"region": "",
|
|
"endpoint_ip": None,
|
|
"endpoint_source": None,
|
|
"probed_at": item["probed_at"],
|
|
"vpc_ready": False,
|
|
"reasons": ["fabric_host_not_configured"],
|
|
"capabilities": {},
|
|
"error_code": "fabric_host_not_configured",
|
|
}
|
|
|
|
|
|
@pytest.mark.parametrize("address", [
|
|
"0.0.0.0",
|
|
"0.1.2.3",
|
|
"127.23.45.67",
|
|
"169.254.12.34",
|
|
"224.0.0.1",
|
|
"240.0.0.1",
|
|
"255.255.255.255",
|
|
])
|
|
def test_readiness_probe_hides_unusable_endpoint_ipv4(address):
|
|
manager = ProbeManager("host-a", host=address)
|
|
coordinator = FabricCoordinator(
|
|
MemoryStore(),
|
|
Registry({"host-a": SimpleNamespace(manager=manager)}),
|
|
)
|
|
try:
|
|
result = asyncio.run(coordinator.probe_hosts({
|
|
"host_ids": ["host-a"],
|
|
}))
|
|
finally:
|
|
coordinator.close()
|
|
|
|
readiness = result["hosts"][0]
|
|
assert readiness["configured"] is True
|
|
assert readiness["reachable"] is True
|
|
assert readiness["vpc_ready"] is False
|
|
assert readiness["endpoint_ip"] is None
|
|
assert readiness["endpoint_source"] is None
|
|
assert readiness["error_code"] == "fabric_endpoint_not_local"
|
|
|
|
|
|
def test_readiness_probe_prefers_configured_fabric_endpoint():
|
|
manager = ProbeManager(
|
|
"host-a", host="host.example.test",
|
|
fabric_endpoint_ip="10.20.30.40",
|
|
)
|
|
coordinator = FabricCoordinator(
|
|
MemoryStore(),
|
|
Registry({"host-a": SimpleNamespace(manager=manager)}),
|
|
)
|
|
try:
|
|
result = asyncio.run(coordinator.probe_hosts({
|
|
"host_ids": ["host-a"],
|
|
}))
|
|
finally:
|
|
coordinator.close()
|
|
|
|
readiness = result["hosts"][0]
|
|
assert readiness["endpoint_ip"] == "10.20.30.40"
|
|
assert readiness["endpoint_source"] == "configured"
|
|
|
|
|
|
def test_readiness_probe_rejects_native_fallback_without_local_evidence():
|
|
manager = ProbeManager("host-a", host="203.0.113.10")
|
|
manager.capabilities["network"]["local_ipv4_addresses"] = []
|
|
manager.capabilities["network"]["local_ipv4_interfaces"] = {}
|
|
manager.capabilities["network"]["rp_filter_interfaces"] = {}
|
|
coordinator = FabricCoordinator(
|
|
MemoryStore(),
|
|
Registry({"host-a": SimpleNamespace(manager=manager)}),
|
|
)
|
|
try:
|
|
result = asyncio.run(coordinator.probe_hosts({
|
|
"host_ids": ["host-a"],
|
|
}))
|
|
finally:
|
|
coordinator.close()
|
|
|
|
readiness = result["hosts"][0]
|
|
assert readiness["endpoint_ip"] == "203.0.113.10"
|
|
assert readiness["endpoint_source"] == "ssh_host_ipv4"
|
|
assert readiness["error_code"] == "fabric_endpoint_not_local"
|
|
assert readiness["reasons"] == ["fabric_endpoint_not_local"]
|
|
|
|
manager.capabilities["network"]["local_ipv4_addresses"] = [
|
|
"203.0.113.10",
|
|
]
|
|
manager.capabilities["network"]["local_ipv4_interfaces"] = {
|
|
"203.0.113.10": ["br0"],
|
|
}
|
|
coordinator = FabricCoordinator(
|
|
MemoryStore(),
|
|
Registry({"host-a": SimpleNamespace(manager=manager)}),
|
|
)
|
|
try:
|
|
result = asyncio.run(coordinator.probe_hosts({
|
|
"host_ids": ["host-a"],
|
|
}))
|
|
finally:
|
|
coordinator.close()
|
|
assert result["hosts"][0]["error_code"] == (
|
|
"fabric_endpoint_rp_filter_unverified"
|
|
)
|
|
|
|
|
|
def test_readiness_probe_rejects_overlapping_batch_without_queueing():
|
|
started = threading.Event()
|
|
release = threading.Event()
|
|
manager = ProbeManager("host-a")
|
|
|
|
def blocking_reprobe():
|
|
started.set()
|
|
release.wait(timeout=2)
|
|
return manager.profile, manager.capabilities
|
|
|
|
manager.reprobe_current_snapshot = blocking_reprobe
|
|
coordinator = FabricCoordinator(
|
|
MemoryStore(),
|
|
Registry({"host-a": SimpleNamespace(manager=manager)}),
|
|
)
|
|
|
|
async def exercise():
|
|
first = asyncio.create_task(coordinator.probe_hosts({
|
|
"host_ids": ["host-a"],
|
|
}))
|
|
try:
|
|
for _ in range(100):
|
|
if started.is_set():
|
|
break
|
|
await asyncio.sleep(0.001)
|
|
assert started.is_set()
|
|
with pytest.raises(FabricError) as error:
|
|
await coordinator.probe_hosts({"host_ids": ["host-a"]})
|
|
assert error.value.status == 409
|
|
assert error.value.error_code == "fabric_probe_busy"
|
|
finally:
|
|
release.set()
|
|
return await first
|
|
|
|
try:
|
|
result = asyncio.run(exercise())
|
|
finally:
|
|
release.set()
|
|
coordinator.close()
|
|
assert result["hosts"][0]["reachable"] is True
|
|
|
|
|
|
def test_coordinator_keeps_successful_hosts_and_blocks_next_stage(monkeypatch):
|
|
store = MemoryStore()
|
|
managers = {}
|
|
components = {}
|
|
for host_id, key in (("host-a", KEY1), ("host-b", KEY2)):
|
|
manager = FakeManager(key)
|
|
manager.host_id = host_id
|
|
manager.profile["host_id"] = host_id
|
|
manager.profile["region"] = "hk"
|
|
manager.profile["capabilities"] = {
|
|
"vpc_ready": True,
|
|
"wireguard": {
|
|
"interface": FABRIC_INTERFACE,
|
|
"deployment_mode": "networkmanager_nmcli",
|
|
"ready": True,
|
|
},
|
|
"network": {
|
|
"network_manager": "networkmanager",
|
|
"network_manager_supported": True,
|
|
"ip_probe_valid": True,
|
|
"local_ipv4_addresses": [manager.effective_host],
|
|
"local_ipv4_interfaces": {
|
|
manager.effective_host: ["eth0"],
|
|
},
|
|
"rp_filter": 2,
|
|
"rp_filter_default": 2,
|
|
"rp_filter_interfaces": {"eth0": 1},
|
|
},
|
|
}
|
|
managers[host_id] = manager
|
|
components[host_id] = SimpleNamespace(manager=manager)
|
|
monkeypatch.setattr("virsh_panel.fabric.FabricHostAdapter", FakeAdapter)
|
|
FakeAdapter.failures = {"host-b"}
|
|
result = asyncio.run(FabricCoordinator(store, Registry(components)).apply({
|
|
"fabric_id": "hk-fabric", "region": "hk",
|
|
"overlay_cidr": "100.64.1.0/24", "revision": 1,
|
|
"nodes": [
|
|
{"host_id": "host-a", "overlay_ip": "100.64.1.1",
|
|
"endpoint": "203.0.113.1:51820"},
|
|
{"host_id": "host-b", "overlay_ip": "100.64.1.2",
|
|
"endpoint": "203.0.113.2:51820"},
|
|
],
|
|
}))
|
|
assert result["state"] == "uncertain"
|
|
assert result["next_stage_allowed"] is False
|
|
states = {node["host_id"]: node["state"] for node in result["nodes"]}
|
|
assert states == {"host-a": "active", "host-b": "uncertain"}
|
|
failed = next(
|
|
node for node in result["nodes"] if node["host_id"] == "host-b")
|
|
assert failed["last_error_phase"] == "apply"
|
|
persisted = json.loads(store.get(
|
|
"fabric_node", f"{result['fabric_id']}.host-b"))
|
|
assert persisted["last_error_phase"] == "apply"
|
|
assert managers["host-a"].reprobe_calls == 1
|
|
assert managers["host-b"].reprobe_calls == 1
|
|
serialized = "\n".join(store.values.values())
|
|
assert "PRIVATE KEY" not in serialized
|
|
assert KEY1 in serialized and KEY2 in serialized
|
|
FakeAdapter.failures = set()
|
|
|
|
|
|
def test_initial_apply_identity_failure_carries_safe_phase_without_record(
|
|
monkeypatch):
|
|
current = plan()
|
|
store = MemoryStore()
|
|
components = _fabric_components(current)
|
|
|
|
class IdentityFailureAdapter(FakeAdapter):
|
|
def identity(self):
|
|
raise FabricError(
|
|
"safe", 502, "fabric_remote_command_failed")
|
|
|
|
monkeypatch.setattr(
|
|
"virsh_panel.fabric.FabricHostAdapter", IdentityFailureAdapter)
|
|
|
|
with pytest.raises(FabricError) as error:
|
|
asyncio.run(FabricCoordinator(
|
|
store, Registry(components)).apply(plan_payload(current)))
|
|
|
|
assert error.value.error_code == "fabric_remote_command_failed"
|
|
assert error.value.failure_phase == "identity"
|
|
assert store.get("fabric_desired", current.fabric_id) is None
|
|
|
|
|
|
def test_apply_records_post_apply_observe_failure_phase(monkeypatch):
|
|
current = plan()
|
|
store = MemoryStore()
|
|
components = _fabric_components(current)
|
|
monkeypatch.setattr("virsh_panel.fabric.FabricHostAdapter", FakeAdapter)
|
|
FakeAdapter.failures = set()
|
|
FakeAdapter.observe_failures = {"host-a"}
|
|
try:
|
|
result = asyncio.run(FabricCoordinator(
|
|
store, Registry(components)).apply(plan_payload(current)))
|
|
finally:
|
|
FakeAdapter.observe_failures = set()
|
|
|
|
nodes = {node["host_id"]: node for node in result["nodes"]}
|
|
assert nodes["host-a"]["last_error_code"] == (
|
|
"fabric_remote_command_failed")
|
|
assert nodes["host-a"]["last_error_phase"] == "observe"
|
|
assert "last_error_phase" not in nodes["host-b"]
|
|
persisted = json.loads(store.get(
|
|
"fabric_node", f"{current.fabric_id}.host-a"))
|
|
assert persisted["last_error_phase"] == "observe"
|
|
|
|
|
|
def test_recover_does_not_replay_an_interrupted_write():
|
|
store = MemoryStore()
|
|
desired = {"fabric_id": "hk-fabric", "state": "applying",
|
|
"next_stage_allowed": False,
|
|
"revision": 1, "plan_digest": "x",
|
|
"nodes": [{"host_id": "host-a", "state": "active",
|
|
"transport_verified": True}]}
|
|
store.set("fabric_desired", "hk-fabric", json.dumps(desired))
|
|
coordinator = FabricCoordinator(store, Registry({}))
|
|
coordinator.recover()
|
|
recovered = json.loads(store.get("fabric_desired", "hk-fabric"))
|
|
assert recovered["state"] == "uncertain"
|
|
assert recovered["next_stage_allowed"] is False
|
|
assert recovered["nodes"][0]["transport_verified"] is False
|
|
|
|
desired["state"] = "active"
|
|
desired["next_stage_allowed"] = True
|
|
desired["nodes"][0]["state"] = "active"
|
|
desired["nodes"][0]["transport_verified"] = True
|
|
store.set("fabric_desired", "hk-fabric", json.dumps(desired))
|
|
coordinator.recover()
|
|
restarted = json.loads(store.get("fabric_desired", "hk-fabric"))
|
|
assert restarted["state"] == "uncertain"
|
|
assert restarted["next_stage_allowed"] is False
|
|
|
|
|
|
def test_public_status_does_not_echo_nested_secrets_or_commands():
|
|
public = FabricCoordinator._public({
|
|
"fabric_id": "f", "state": "uncertain",
|
|
"listen_port": 51820,
|
|
"private_key": "secret",
|
|
"command": "rm -rf /",
|
|
"nodes": [{
|
|
"host_id": "host-a", "public_key": KEY1,
|
|
"last_error_code": "fabric_remote_command_failed",
|
|
"last_error_phase": "apply",
|
|
"private_key": "secret",
|
|
"observed": {
|
|
"interface": FABRIC_INTERFACE,
|
|
"command": "wg show",
|
|
"peers": [(KEY2, "203.0.113.2:51820", "100.64.1.2/32")],
|
|
"handshakes": {KEY2: 123},
|
|
},
|
|
}, {
|
|
"host_id": "host-b", "public_key": KEY2,
|
|
"last_error_phase": "observe",
|
|
}, {
|
|
"host_id": "host-c", "public_key": KEY3,
|
|
"last_error_code": "fabric_remote_command_failed",
|
|
"last_error_phase": "stderr:/secret/path",
|
|
}],
|
|
})
|
|
serialized = json.dumps(public)
|
|
assert public["listen_port"] == 51820
|
|
assert "private_key" not in serialized
|
|
assert "command" not in public
|
|
assert "command" not in public["nodes"][0]["observed"]
|
|
assert KEY1 in serialized and KEY2 in serialized
|
|
assert public["nodes"][0]["last_error_phase"] == "apply"
|
|
assert "last_error_phase" not in public["nodes"][1]
|
|
assert "last_error_phase" not in public["nodes"][2]
|
|
assert FabricError(
|
|
"safe", failure_phase="stderr:/secret/path").failure_phase is None
|
|
|
|
|
|
def test_reconcile_rejects_a_tampered_plan_digest():
|
|
store = MemoryStore()
|
|
current = plan()
|
|
store.set("fabric_desired", current.fabric_id, json.dumps({
|
|
"fabric_id": current.fabric_id,
|
|
"region": current.region,
|
|
"overlay_cidr": current.overlay_cidr,
|
|
"revision": current.revision,
|
|
"listen_port": current.listen_port,
|
|
"plan_digest": "0" * 64,
|
|
"public_keys": {node.host_id: key for node, key in zip(
|
|
current.nodes, (KEY1, KEY2))},
|
|
"nodes": [
|
|
{"host_id": node.host_id, "overlay_ip": node.overlay_ip,
|
|
"endpoint": node.endpoint, "listen_port": node.listen_port}
|
|
for node in current.nodes
|
|
],
|
|
}))
|
|
coordinator = FabricCoordinator(store, Registry({}))
|
|
with pytest.raises(FabricError) as error:
|
|
asyncio.run(coordinator.reconcile(current.fabric_id))
|
|
assert error.value.error_code == "fabric_record_corrupt"
|
|
|
|
|
|
def _active_fabric_record(current):
|
|
keys = {node.host_id: key for node, key in zip(
|
|
current.nodes, (KEY1, KEY2))}
|
|
return {
|
|
"schema_version": 1,
|
|
"fabric_id": current.fabric_id,
|
|
"region": current.region,
|
|
"overlay_cidr": current.overlay_cidr,
|
|
"revision": current.revision,
|
|
"listen_port": current.listen_port,
|
|
"plan_digest": current.digest,
|
|
"state": "active",
|
|
"next_stage_allowed": True,
|
|
"updated_at": int(time.time()),
|
|
"public_keys": keys,
|
|
"nodes": [
|
|
{
|
|
"host_id": node.host_id,
|
|
"overlay_ip": node.overlay_ip,
|
|
"endpoint": node.endpoint,
|
|
"listen_port": node.listen_port,
|
|
"state": "active",
|
|
"transport_verified": True,
|
|
"observed_revision": current.revision,
|
|
"observed_plan_digest": current.digest,
|
|
}
|
|
for node in current.nodes
|
|
],
|
|
}
|
|
|
|
|
|
def _fabric_components(current):
|
|
components = {}
|
|
for node, key in zip(current.nodes, (KEY1, KEY2)):
|
|
manager = FakeManager(key)
|
|
manager.host_id = node.host_id
|
|
manager.profile["host_id"] = node.host_id
|
|
manager.profile["region"] = current.region
|
|
components[node.host_id] = SimpleNamespace(manager=manager)
|
|
return components
|
|
|
|
|
|
def _remote_operation_status(
|
|
operation_id, status, *, checkpoint=None, reason=None):
|
|
checkpoint = checkpoint or (
|
|
"complete" if status == "succeeded" else "preflight")
|
|
reason = reason or (
|
|
"none" if status in {"queued", "running", "succeeded"}
|
|
else "command_failed")
|
|
started_at = None if status == "queued" else 101
|
|
finished_at = (
|
|
103 if status in {"succeeded", "failed", "uncertain", "interrupted"}
|
|
else None)
|
|
return {
|
|
"operation_id": operation_id,
|
|
"status": status,
|
|
"phase": "apply",
|
|
"checkpoint": checkpoint,
|
|
"reason": reason,
|
|
"created_at": 100,
|
|
"started_at": started_at,
|
|
"updated_at": 103 if finished_at else 102,
|
|
"finished_at": finished_at,
|
|
}
|
|
|
|
|
|
class AsyncCoordinatorAdapter:
|
|
"""Host stub that separates one dispatch from read-only status polling."""
|
|
|
|
def __init__(self, manager, _profile=None):
|
|
self.manager = manager
|
|
|
|
def identity(self, *, create_key=True):
|
|
self.manager.async_identity_calls += 1
|
|
return self.manager.public_key
|
|
|
|
def dispatch_apply(self, _plan, _host, _keys, operation_id=None):
|
|
self.manager.async_dispatch_calls += 1
|
|
self.manager.async_operation_id = operation_id
|
|
if self.manager.async_dispatch_entered is not None:
|
|
self.manager.async_dispatch_entered.set()
|
|
if self.manager.async_dispatch_release is not None:
|
|
assert self.manager.async_dispatch_release.wait(timeout=2)
|
|
if self.manager.async_dispatch_error is not None:
|
|
error = self.manager.async_dispatch_error
|
|
self.manager.async_dispatch_error = None
|
|
raise error
|
|
return _remote_operation_status(operation_id, "queued")
|
|
|
|
def apply_status(self, operation_id):
|
|
self.manager.async_status_calls += 1
|
|
assert operation_id == self.manager.async_operation_id
|
|
statuses = self.manager.async_statuses
|
|
if len(statuses) > 1:
|
|
status = statuses.pop(0)
|
|
else:
|
|
status = statuses[0]
|
|
if isinstance(status, Exception):
|
|
raise status
|
|
if isinstance(status, dict):
|
|
return dict(status)
|
|
return _remote_operation_status(
|
|
operation_id, status,
|
|
checkpoint=("nft_install" if status == "failed" else None),
|
|
reason=("nft_failed" if status == "failed" else None),
|
|
)
|
|
|
|
def observe(self, current, host, _keys):
|
|
self.manager.async_observe_calls += 1
|
|
return _reconcile_observation(current, host)
|
|
|
|
|
|
class PublishedQueuedGapAdapter(AsyncCoordinatorAdapter):
|
|
"""First dispatch publishes the directory but dies before systemd-run."""
|
|
|
|
def dispatch_apply(self, _plan, _host, _keys, operation_id=None):
|
|
self.manager.async_dispatch_calls += 1
|
|
self.manager.async_operation_id = operation_id
|
|
if self.manager.async_dispatch_calls == 1:
|
|
raise FabricError(
|
|
"ambiguous transport /root/private", 502,
|
|
"fabric_host_connection_failed", "apply")
|
|
self.manager.async_runner_starts += 1
|
|
self.manager.async_statuses = ["succeeded"]
|
|
return _remote_operation_status(operation_id, "queued")
|
|
|
|
|
|
class RecoveredQueuedGapAdapter(AsyncCoordinatorAdapter):
|
|
"""Restart joins a published queue and starts its original runner once."""
|
|
|
|
def dispatch_apply(self, _plan, _host, _keys, operation_id=None):
|
|
self.manager.async_dispatch_calls += 1
|
|
assert operation_id == self.manager.async_operation_id
|
|
self.manager.async_runner_starts += 1
|
|
self.manager.async_statuses = ["succeeded"]
|
|
return _remote_operation_status(operation_id, "queued")
|
|
|
|
|
|
class PartialHostCrashAdapter(AsyncCoordinatorAdapter):
|
|
"""Host A was admitted; host B still needs same-id continuation."""
|
|
|
|
def identity(self, *, create_key=True):
|
|
self.manager.async_identity_calls += 1
|
|
self.manager.partial_identity_create_flags.append(create_key)
|
|
return self.manager.public_key
|
|
|
|
def dispatch_apply(self, _plan, _host, _keys, operation_id=None):
|
|
assert self.manager.partial_configured is False
|
|
self.manager.async_dispatch_calls += 1
|
|
self.manager.async_operation_id = operation_id
|
|
self.manager.partial_configured = True
|
|
return _remote_operation_status(operation_id, "succeeded")
|
|
|
|
def observe(self, current, host, _keys):
|
|
self.manager.async_observe_calls += 1
|
|
if self.manager.partial_configured:
|
|
return _reconcile_observation(current, host)
|
|
return {
|
|
"state": "degraded",
|
|
"config_ok": False,
|
|
"transport_verified": False,
|
|
"public_key": self.manager.public_key,
|
|
"observed": {
|
|
"interface": None,
|
|
"config_revision": None,
|
|
"config_digest": None,
|
|
},
|
|
}
|
|
|
|
|
|
def _async_components(current):
|
|
components = _fabric_components(current)
|
|
for component in components.values():
|
|
manager = component.manager
|
|
manager.async_identity_calls = 0
|
|
manager.async_dispatch_calls = 0
|
|
manager.async_status_calls = 0
|
|
manager.async_observe_calls = 0
|
|
manager.async_operation_id = None
|
|
manager.async_dispatch_entered = None
|
|
manager.async_dispatch_release = None
|
|
manager.async_dispatch_error = None
|
|
manager.async_runner_starts = 0
|
|
manager.partial_configured = False
|
|
manager.partial_identity_create_flags = []
|
|
manager.async_statuses = ["succeeded"]
|
|
return components
|
|
|
|
|
|
async def _await_fabric_operation(coordinator, fabric_id):
|
|
for _attempt in range(500):
|
|
task = coordinator._operation_tasks_by_fabric.get(fabric_id)
|
|
if task is None:
|
|
return
|
|
await asyncio.sleep(0.002)
|
|
raise AssertionError("Fabric background operation did not finish")
|
|
|
|
|
|
def test_enqueue_apply_returns_before_host_systemd_job_finishes(monkeypatch):
|
|
current = plan()
|
|
store = MemoryStore()
|
|
components = _async_components(current)
|
|
entered = threading.Event()
|
|
release = threading.Event()
|
|
components["host-a"].manager.async_dispatch_entered = entered
|
|
components["host-a"].manager.async_dispatch_release = release
|
|
coordinator = FabricCoordinator(store, Registry(components))
|
|
monkeypatch.setattr(
|
|
"virsh_panel.fabric.FabricHostAdapter", AsyncCoordinatorAdapter)
|
|
monkeypatch.setattr(
|
|
"virsh_panel.fabric.FABRIC_OPERATION_POLL_INTERVAL_SECONDS", 0)
|
|
|
|
async def exercise():
|
|
admitted = await coordinator.enqueue_apply(plan_payload(current))
|
|
assert admitted["operation"]["status"] == "queued"
|
|
assert re.fullmatch(r"[0-9a-f]{32}", admitted["operation"]["id"])
|
|
assert not entered.is_set()
|
|
|
|
for _attempt in range(100):
|
|
if entered.is_set():
|
|
break
|
|
await asyncio.sleep(0.002)
|
|
assert entered.is_set()
|
|
live = coordinator.get_status(current.fabric_id)
|
|
assert live["operation"]["status"] == "running"
|
|
release.set()
|
|
await _await_fabric_operation(coordinator, current.fabric_id)
|
|
return admitted, coordinator.get_status(current.fabric_id)
|
|
|
|
admitted, final = asyncio.run(exercise())
|
|
|
|
assert final["operation"]["id"] == admitted["operation"]["id"]
|
|
assert final["operation"]["status"] == "succeeded"
|
|
assert all(node["operation"]["status"] == "succeeded"
|
|
for node in final["nodes"])
|
|
assert all(component.manager.async_dispatch_calls == 1
|
|
for component in components.values())
|
|
|
|
|
|
def test_cancelled_async_dispatch_drains_then_recovers_without_redispatch(
|
|
monkeypatch):
|
|
current = plan()
|
|
store = MemoryStore()
|
|
components = _async_components(current)
|
|
entered = threading.Event()
|
|
release = threading.Event()
|
|
first = components["host-a"].manager
|
|
first.async_dispatch_entered = entered
|
|
first.async_dispatch_release = release
|
|
coordinator = FabricCoordinator(store, Registry(components))
|
|
monkeypatch.setattr(
|
|
"virsh_panel.fabric.FabricHostAdapter", AsyncCoordinatorAdapter)
|
|
|
|
async def exercise():
|
|
admitted = await coordinator.enqueue_apply(plan_payload(current))
|
|
for _attempt in range(100):
|
|
if entered.is_set():
|
|
break
|
|
await asyncio.sleep(0.002)
|
|
assert entered.is_set()
|
|
operation_task = coordinator._operation_tasks_by_fabric[
|
|
current.fabric_id]
|
|
|
|
lock_entered = asyncio.Event()
|
|
|
|
async def wait_for_fabric_lock():
|
|
async with coordinator._locks[current.fabric_id]:
|
|
lock_entered.set()
|
|
|
|
waiter = asyncio.create_task(wait_for_fabric_lock())
|
|
operation_task.cancel()
|
|
await asyncio.sleep(0)
|
|
assert operation_task.done() is False
|
|
assert lock_entered.is_set() is False
|
|
release.set()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await operation_task
|
|
await asyncio.wait_for(waiter, timeout=1)
|
|
assert lock_entered.is_set() is True
|
|
|
|
coordinator.recover()
|
|
first.async_operation_id = admitted["operation"]["id"]
|
|
first.async_statuses = ["succeeded"]
|
|
resumed = await coordinator.enqueue_reconcile(current.fabric_id)
|
|
assert resumed["operation"]["id"] == admitted["operation"]["id"]
|
|
await _await_fabric_operation(coordinator, current.fabric_id)
|
|
return admitted, coordinator.get_status(current.fabric_id)
|
|
|
|
try:
|
|
admitted, final = asyncio.run(exercise())
|
|
finally:
|
|
release.set()
|
|
|
|
assert final["operation"]["id"] == admitted["operation"]["id"]
|
|
assert first.async_dispatch_calls == 1
|
|
assert components["host-b"].manager.async_dispatch_calls == 0
|
|
|
|
|
|
@pytest.mark.parametrize(("terminal", "expected_root"), [
|
|
("succeeded", "succeeded"),
|
|
("failed", "failed"),
|
|
])
|
|
def test_dispatch_disconnect_is_reconciled_by_status_without_replay(
|
|
monkeypatch, terminal, expected_root):
|
|
current = plan()
|
|
store = MemoryStore()
|
|
components = _async_components(current)
|
|
disconnected = components["host-a"].manager
|
|
disconnected.async_dispatch_error = FabricError(
|
|
"safe disconnect", 502, "fabric_host_connection_failed", "apply")
|
|
disconnected.async_statuses = ["running", terminal]
|
|
coordinator = FabricCoordinator(store, Registry(components))
|
|
monkeypatch.setattr(
|
|
"virsh_panel.fabric.FabricHostAdapter", AsyncCoordinatorAdapter)
|
|
monkeypatch.setattr(
|
|
"virsh_panel.fabric.FABRIC_OPERATION_POLL_INTERVAL_SECONDS", 0)
|
|
|
|
async def exercise():
|
|
admitted = await coordinator.enqueue_apply(plan_payload(current))
|
|
await _await_fabric_operation(coordinator, current.fabric_id)
|
|
return admitted, coordinator.get_status(current.fabric_id)
|
|
|
|
admitted, final = asyncio.run(exercise())
|
|
|
|
assert final["operation"]["id"] == admitted["operation"]["id"]
|
|
assert final["operation"]["status"] == expected_root
|
|
failed_or_recovered = next(
|
|
node for node in final["nodes"] if node["host_id"] == "host-a")
|
|
assert failed_or_recovered["operation"]["status"] == terminal
|
|
assert disconnected.async_dispatch_calls == 1
|
|
assert disconnected.async_status_calls >= 2
|
|
if terminal == "succeeded":
|
|
assert disconnected.async_observe_calls == 1
|
|
else:
|
|
assert disconnected.async_observe_calls == 0
|
|
|
|
|
|
def test_async_apply_timeout_is_uncertain_and_never_redispatched(monkeypatch):
|
|
current = plan()
|
|
store = MemoryStore()
|
|
components = _async_components(current)
|
|
stalled = components["host-a"].manager
|
|
stalled.async_statuses = ["running"]
|
|
coordinator = FabricCoordinator(store, Registry(components))
|
|
monkeypatch.setattr(
|
|
"virsh_panel.fabric.FabricHostAdapter", AsyncCoordinatorAdapter)
|
|
monkeypatch.setattr(
|
|
"virsh_panel.fabric.FABRIC_OPERATION_POLL_INTERVAL_SECONDS", 0.002)
|
|
monkeypatch.setattr(
|
|
"virsh_panel.fabric.FABRIC_OPERATION_POLL_TIMEOUT_SECONDS", 0.01)
|
|
|
|
async def exercise():
|
|
await coordinator.enqueue_apply(plan_payload(current))
|
|
await _await_fabric_operation(coordinator, current.fabric_id)
|
|
return coordinator.get_status(current.fabric_id)
|
|
|
|
final = asyncio.run(exercise())
|
|
|
|
stalled_node = next(
|
|
node for node in final["nodes"] if node["host_id"] == "host-a")
|
|
assert final["operation"]["status"] == "uncertain"
|
|
assert stalled_node["operation"]["status"] == "uncertain"
|
|
assert stalled_node["operation"]["reason"] == "transport_failed"
|
|
assert stalled.async_dispatch_calls == 1
|
|
assert stalled.async_status_calls >= 1
|
|
|
|
|
|
@pytest.mark.parametrize("error_code", [
|
|
"fabric_operation_status_invalid",
|
|
"fabric_operation_not_found",
|
|
"fabric_remote_command_failed",
|
|
])
|
|
def test_confirmed_dispatch_status_error_stays_uncertain_and_resumes_same_id(
|
|
monkeypatch, error_code):
|
|
current = plan()
|
|
store = MemoryStore()
|
|
components = _async_components(current)
|
|
for component in components.values():
|
|
component.manager.async_statuses = [
|
|
FabricError("unsafe /root/private", 502, error_code, "observe"),
|
|
"succeeded",
|
|
]
|
|
coordinator = FabricCoordinator(store, Registry(components))
|
|
monkeypatch.setattr(
|
|
"virsh_panel.fabric.FabricHostAdapter", AsyncCoordinatorAdapter)
|
|
monkeypatch.setattr(
|
|
"virsh_panel.fabric.FABRIC_OPERATION_POLL_INTERVAL_SECONDS", 0)
|
|
|
|
async def exercise():
|
|
admitted = await coordinator.enqueue_apply(plan_payload(current))
|
|
await _await_fabric_operation(coordinator, current.fabric_id)
|
|
uncertain = coordinator.get_status(current.fabric_id)
|
|
assert uncertain["operation"]["status"] == "uncertain"
|
|
assert uncertain["next_stage_allowed"] is False
|
|
|
|
resumed = await coordinator.enqueue_reconcile(current.fabric_id)
|
|
assert resumed["operation"]["id"] == admitted["operation"]["id"]
|
|
await _await_fabric_operation(coordinator, current.fabric_id)
|
|
return admitted, coordinator.get_status(current.fabric_id)
|
|
|
|
admitted, final = asyncio.run(exercise())
|
|
|
|
assert final["operation"]["id"] == admitted["operation"]["id"]
|
|
assert final["operation"]["status"] == "succeeded"
|
|
assert all(component.manager.async_dispatch_calls == 1
|
|
for component in components.values())
|
|
assert all(component.manager.async_status_calls >= 2
|
|
for component in components.values())
|
|
assert "/root/private" not in json.dumps(final)
|
|
|
|
|
|
def test_published_queue_gap_recovers_original_runner_once_with_same_id(
|
|
monkeypatch):
|
|
current = plan()
|
|
store = MemoryStore()
|
|
components = _async_components(current)
|
|
for component in components.values():
|
|
component.manager.async_statuses = ["queued"]
|
|
coordinator = FabricCoordinator(store, Registry(components))
|
|
monkeypatch.setattr(
|
|
"virsh_panel.fabric.FabricHostAdapter", PublishedQueuedGapAdapter)
|
|
monkeypatch.setattr(
|
|
"virsh_panel.fabric.FABRIC_OPERATION_POLL_INTERVAL_SECONDS", 0.001)
|
|
monkeypatch.setattr(
|
|
"virsh_panel.fabric.FABRIC_OPERATION_POLL_TIMEOUT_SECONDS", 0.004)
|
|
|
|
async def exercise():
|
|
admitted = await coordinator.enqueue_apply(plan_payload(current))
|
|
await _await_fabric_operation(coordinator, current.fabric_id)
|
|
stalled = coordinator.get_status(current.fabric_id)
|
|
assert stalled["operation"]["status"] == "uncertain"
|
|
assert stalled["next_stage_allowed"] is False
|
|
|
|
resumed = await coordinator.enqueue_reconcile(current.fabric_id)
|
|
assert resumed["operation"]["id"] == admitted["operation"]["id"]
|
|
await _await_fabric_operation(coordinator, current.fabric_id)
|
|
restarted = coordinator.get_status(current.fabric_id)
|
|
assert restarted["operation"]["status"] == "running"
|
|
assert restarted["next_stage_allowed"] is False
|
|
await coordinator.reconcile_all()
|
|
return admitted, coordinator.get_status(current.fabric_id)
|
|
|
|
admitted, final = asyncio.run(exercise())
|
|
|
|
assert final["operation"]["id"] == admitted["operation"]["id"]
|
|
assert final["operation"]["status"] == "succeeded"
|
|
assert final["next_stage_allowed"] is True
|
|
assert all(component.manager.async_dispatch_calls == 2
|
|
for component in components.values())
|
|
assert all(component.manager.async_runner_starts == 1
|
|
for component in components.values())
|
|
assert all(component.manager.async_operation_id
|
|
== admitted["operation"]["id"]
|
|
for component in components.values())
|
|
|
|
|
|
def test_restart_periodic_queued_status_still_allows_same_id_runner_recovery(
|
|
monkeypatch):
|
|
current = plan()
|
|
operation_id = "a" * 32
|
|
store = MemoryStore()
|
|
components = _async_components(current)
|
|
coordinator = FabricCoordinator(store, Registry(components))
|
|
admitted = coordinator._admit_apply_record(current, operation_id)
|
|
admitted["operation"].update(
|
|
status="running", started_at=101, updated_at=101)
|
|
admitted["public_keys"] = {"host-a": KEY1, "host-b": KEY2}
|
|
for node, key in zip(admitted["nodes"], (KEY1, KEY2)):
|
|
node["public_key"] = key
|
|
node["operation"].update(
|
|
status="running", phase="apply", checkpoint="dispatch",
|
|
started_at=101, updated_at=101)
|
|
store.set(
|
|
"fabric_desired", current.fabric_id,
|
|
json.dumps(admitted))
|
|
coordinator.recover()
|
|
for component in components.values():
|
|
manager = component.manager
|
|
manager.async_operation_id = operation_id
|
|
manager.async_statuses = ["queued"]
|
|
monkeypatch.setattr(
|
|
"virsh_panel.fabric.FabricHostAdapter", RecoveredQueuedGapAdapter)
|
|
|
|
async def exercise():
|
|
await coordinator.reconcile_all()
|
|
observed = coordinator.get_status(current.fabric_id)
|
|
assert observed["operation"]["id"] == operation_id
|
|
assert observed["operation"]["status"] == "running"
|
|
assert observed["next_stage_allowed"] is False
|
|
assert all(component.manager.async_dispatch_calls == 0
|
|
for component in components.values())
|
|
|
|
resumed = await coordinator.enqueue_reconcile(current.fabric_id)
|
|
assert resumed["operation"]["id"] == operation_id
|
|
await _await_fabric_operation(coordinator, current.fabric_id)
|
|
restarted = coordinator.get_status(current.fabric_id)
|
|
assert restarted["operation"]["status"] == "running"
|
|
assert restarted["next_stage_allowed"] is False
|
|
await coordinator.reconcile_all()
|
|
return coordinator.get_status(current.fabric_id)
|
|
|
|
final = asyncio.run(exercise())
|
|
|
|
assert final["operation"]["id"] == operation_id
|
|
assert final["operation"]["status"] == "succeeded"
|
|
assert final["next_stage_allowed"] is True
|
|
assert all(component.manager.async_dispatch_calls == 1
|
|
for component in components.values())
|
|
assert all(component.manager.async_runner_starts == 1
|
|
for component in components.values())
|
|
assert all(component.manager.async_operation_id == operation_id
|
|
for component in components.values())
|
|
|
|
|
|
def _interrupted_async_record(current, operation_id):
|
|
record = _active_fabric_record(current)
|
|
record.update(state="uncertain", next_stage_allowed=False)
|
|
record["operation"] = {
|
|
"id": operation_id,
|
|
"kind": "apply",
|
|
"status": "interrupted",
|
|
"created_at": 100,
|
|
"started_at": 101,
|
|
"updated_at": 102,
|
|
"finished_at": 102,
|
|
}
|
|
for node in record["nodes"]:
|
|
node.update(
|
|
state="uncertain", transport_verified=False,
|
|
observed_revision=None, observed_plan_digest=None,
|
|
operation={
|
|
"id": operation_id,
|
|
"status": "uncertain",
|
|
"phase": "apply",
|
|
"checkpoint": "preflight",
|
|
"reason": "transport_failed",
|
|
"created_at": 100,
|
|
"started_at": 101,
|
|
"updated_at": 102,
|
|
"finished_at": 102,
|
|
},
|
|
)
|
|
return record
|
|
|
|
|
|
@pytest.mark.parametrize("crash_stage", [
|
|
"before_identity",
|
|
"mid_identity",
|
|
"identity_fenced",
|
|
])
|
|
def test_pre_dispatch_crash_resumes_same_operation_id_without_replay(
|
|
monkeypatch, crash_stage):
|
|
current = plan()
|
|
operation_id = "5" * 32
|
|
store = MemoryStore()
|
|
components = _async_components(current)
|
|
coordinator = FabricCoordinator(store, Registry(components))
|
|
admitted = coordinator._admit_apply_record(current, operation_id)
|
|
if crash_stage in {"mid_identity", "identity_fenced"}:
|
|
admitted["operation"].update(
|
|
status="running", started_at=101, updated_at=101)
|
|
admitted["nodes"][0]["operation"].update(
|
|
status="running", phase="identity", checkpoint="identity",
|
|
started_at=101, updated_at=101)
|
|
if crash_stage == "identity_fenced":
|
|
admitted["public_keys"] = {"host-a": KEY1, "host-b": KEY2}
|
|
for node, key in zip(admitted["nodes"], (KEY1, KEY2)):
|
|
node["public_key"] = key
|
|
node["operation"].update(
|
|
status="running", phase="identity",
|
|
checkpoint="identity", started_at=101, updated_at=101)
|
|
store.set(
|
|
"fabric_desired", current.fabric_id,
|
|
json.dumps(admitted))
|
|
coordinator.recover()
|
|
for component in components.values():
|
|
component.manager.async_operation_id = operation_id
|
|
component.manager.async_statuses = ["succeeded"]
|
|
monkeypatch.setattr(
|
|
"virsh_panel.fabric.FabricHostAdapter", AsyncCoordinatorAdapter)
|
|
monkeypatch.setattr(
|
|
"virsh_panel.fabric.FABRIC_OPERATION_POLL_INTERVAL_SECONDS", 0)
|
|
|
|
async def exercise():
|
|
resumed = await coordinator.enqueue_reconcile(current.fabric_id)
|
|
assert resumed["operation"]["id"] == operation_id
|
|
await _await_fabric_operation(coordinator, current.fabric_id)
|
|
return coordinator.get_status(current.fabric_id)
|
|
|
|
final = asyncio.run(exercise())
|
|
|
|
assert final["operation"]["id"] == operation_id
|
|
assert final["operation"]["status"] == "succeeded"
|
|
assert all(component.manager.async_dispatch_calls == 1
|
|
for component in components.values())
|
|
|
|
|
|
def _partial_host_crash_record(coordinator, current, operation_id):
|
|
admitted = coordinator._admit_apply_record(current, operation_id)
|
|
admitted["operation"].update(
|
|
status="running", started_at=101, updated_at=101)
|
|
admitted["public_keys"] = {"host-a": KEY1, "host-b": KEY2}
|
|
for node, key in zip(admitted["nodes"], (KEY1, KEY2)):
|
|
node["public_key"] = key
|
|
node["operation"].update(
|
|
status="running", phase="identity",
|
|
checkpoint="identity", started_at=101, updated_at=101)
|
|
admitted["nodes"][0]["operation"].update(
|
|
phase="apply", checkpoint="dispatch")
|
|
return admitted
|
|
|
|
|
|
def test_crash_between_hosts_continues_only_undispatched_host_with_same_id(
|
|
monkeypatch):
|
|
current = plan()
|
|
operation_id = "6" * 32
|
|
store = MemoryStore()
|
|
components = _async_components(current)
|
|
coordinator = FabricCoordinator(store, Registry(components))
|
|
admitted = _partial_host_crash_record(
|
|
coordinator, current, operation_id)
|
|
store.set(
|
|
"fabric_desired", current.fabric_id,
|
|
json.dumps(admitted))
|
|
coordinator.recover()
|
|
|
|
first = components["host-a"].manager
|
|
second = components["host-b"].manager
|
|
first.async_operation_id = operation_id
|
|
first.async_statuses = ["succeeded"]
|
|
first.partial_configured = True
|
|
second.async_statuses = ["succeeded"]
|
|
monkeypatch.setattr(
|
|
"virsh_panel.fabric.FabricHostAdapter", PartialHostCrashAdapter)
|
|
|
|
async def exercise():
|
|
resumed = await coordinator.enqueue_reconcile(current.fabric_id)
|
|
assert resumed["operation"]["id"] == operation_id
|
|
await _await_fabric_operation(coordinator, current.fabric_id)
|
|
continuing = coordinator.get_status(current.fabric_id)
|
|
assert continuing["operation"]["status"] == "running"
|
|
assert continuing["next_stage_allowed"] is False
|
|
await coordinator.reconcile_all()
|
|
return coordinator.get_status(current.fabric_id)
|
|
|
|
final = asyncio.run(exercise())
|
|
|
|
assert final["operation"]["id"] == operation_id
|
|
assert final["operation"]["status"] == "succeeded"
|
|
assert final["next_stage_allowed"] is True
|
|
assert first.async_dispatch_calls == 0
|
|
assert second.async_dispatch_calls == 1
|
|
assert first.async_identity_calls == 0
|
|
assert second.async_identity_calls == 1
|
|
assert second.partial_identity_create_flags == [False]
|
|
assert second.async_operation_id == operation_id
|
|
|
|
|
|
def test_partial_remote_failed_but_exact_continues_only_undispatched_host(
|
|
monkeypatch):
|
|
current = plan()
|
|
operation_id = "7" * 32
|
|
store = MemoryStore()
|
|
components = _async_components(current)
|
|
coordinator = FabricCoordinator(store, Registry(components))
|
|
store.set(
|
|
"fabric_desired", current.fabric_id,
|
|
json.dumps(_partial_host_crash_record(
|
|
coordinator, current, operation_id)))
|
|
coordinator.recover()
|
|
|
|
first = components["host-a"].manager
|
|
second = components["host-b"].manager
|
|
first.async_operation_id = operation_id
|
|
first.async_statuses = ["failed"]
|
|
first.partial_configured = True
|
|
second.async_statuses = ["succeeded"]
|
|
monkeypatch.setattr(
|
|
"virsh_panel.fabric.FabricHostAdapter", PartialHostCrashAdapter)
|
|
|
|
async def exercise():
|
|
resumed = await coordinator.enqueue_reconcile(current.fabric_id)
|
|
assert resumed["operation"]["id"] == operation_id
|
|
await _await_fabric_operation(coordinator, current.fabric_id)
|
|
continuing = coordinator.get_status(current.fabric_id)
|
|
assert continuing["operation"]["status"] == "running"
|
|
assert continuing["next_stage_allowed"] is False
|
|
await coordinator.reconcile_all()
|
|
return coordinator.get_status(current.fabric_id)
|
|
|
|
final = asyncio.run(exercise())
|
|
|
|
assert final["operation"]["id"] == operation_id
|
|
assert final["operation"]["status"] == "succeeded"
|
|
assert final["next_stage_allowed"] is True
|
|
assert first.async_dispatch_calls == 0
|
|
assert second.async_dispatch_calls == 1
|
|
assert first.async_identity_calls == 0
|
|
assert second.async_identity_calls == 1
|
|
assert second.partial_identity_create_flags == [False]
|
|
assert second.async_operation_id == operation_id
|
|
|
|
|
|
def test_partial_remote_failed_nonexact_stays_terminal_then_uses_new_id(
|
|
monkeypatch):
|
|
current = plan()
|
|
operation_id = "8" * 32
|
|
store = MemoryStore()
|
|
components = _async_components(current)
|
|
coordinator = FabricCoordinator(store, Registry(components))
|
|
store.set(
|
|
"fabric_desired", current.fabric_id,
|
|
json.dumps(_partial_host_crash_record(
|
|
coordinator, current, operation_id)))
|
|
coordinator.recover()
|
|
|
|
first = components["host-a"].manager
|
|
second = components["host-b"].manager
|
|
first.async_operation_id = operation_id
|
|
first.async_statuses = ["failed"]
|
|
first.partial_configured = False
|
|
second.partial_configured = False
|
|
monkeypatch.setattr(
|
|
"virsh_panel.fabric.FabricHostAdapter", PartialHostCrashAdapter)
|
|
|
|
async def exercise():
|
|
resumed = await coordinator.enqueue_reconcile(current.fabric_id)
|
|
assert resumed["operation"]["id"] == operation_id
|
|
await _await_fabric_operation(coordinator, current.fabric_id)
|
|
failed = coordinator.get_status(current.fabric_id)
|
|
assert failed["operation"]["status"] == "failed"
|
|
|
|
first.async_statuses = ["succeeded"]
|
|
retried = await coordinator.enqueue_reconcile(current.fabric_id)
|
|
assert retried["operation"]["id"] != operation_id
|
|
await _await_fabric_operation(coordinator, current.fabric_id)
|
|
return failed, coordinator.get_status(current.fabric_id)
|
|
|
|
failed, final = asyncio.run(exercise())
|
|
|
|
assert failed["operation"]["id"] == operation_id
|
|
assert failed["state"] == "uncertain"
|
|
assert failed["next_stage_allowed"] is False
|
|
assert final["operation"]["id"] != operation_id
|
|
assert final["operation"]["status"] == "succeeded"
|
|
assert final["next_stage_allowed"] is True
|
|
assert first.async_dispatch_calls == 1
|
|
assert second.async_dispatch_calls == 1
|
|
assert first.async_identity_calls == 1
|
|
assert second.async_identity_calls == 1
|
|
|
|
|
|
class NonExactAsyncCoordinatorAdapter(AsyncCoordinatorAdapter):
|
|
def observe(self, current, host, keys):
|
|
result = super().observe(current, host, keys)
|
|
result.update(
|
|
state="degraded", config_ok=False,
|
|
transport_verified=False)
|
|
result["observed"]["config_digest"] = "0" * 64
|
|
return result
|
|
|
|
|
|
def test_remote_success_with_non_exact_readback_is_terminal_uncertain(
|
|
monkeypatch):
|
|
current = plan()
|
|
store = MemoryStore()
|
|
components = _async_components(current)
|
|
coordinator = FabricCoordinator(store, Registry(components))
|
|
monkeypatch.setattr(
|
|
"virsh_panel.fabric.FabricHostAdapter",
|
|
NonExactAsyncCoordinatorAdapter)
|
|
monkeypatch.setattr(
|
|
"virsh_panel.fabric.FABRIC_OPERATION_POLL_INTERVAL_SECONDS", 0)
|
|
|
|
async def exercise():
|
|
admitted = await coordinator.enqueue_apply(plan_payload(current))
|
|
await _await_fabric_operation(coordinator, current.fabric_id)
|
|
degraded = coordinator.get_status(current.fabric_id)
|
|
assert degraded["operation"]["status"] == "uncertain"
|
|
assert degraded["next_stage_allowed"] is False
|
|
durable = json.loads(store.get(
|
|
"fabric_desired", current.fabric_id))
|
|
assert all(
|
|
node["operation"]["phase"] == "observe"
|
|
and node["operation"]["checkpoint"] == "final_readback"
|
|
for node in durable["nodes"])
|
|
assert coordinator._operation_may_have_dispatched(
|
|
durable, admitted["operation"]["id"]) is True
|
|
|
|
resumed = await coordinator.enqueue_reconcile(current.fabric_id)
|
|
assert resumed["operation"]["id"] == admitted["operation"]["id"]
|
|
await _await_fabric_operation(coordinator, current.fabric_id)
|
|
return admitted, coordinator.get_status(current.fabric_id)
|
|
|
|
admitted, final = asyncio.run(exercise())
|
|
|
|
assert final["operation"]["id"] == admitted["operation"]["id"]
|
|
assert final["operation"]["status"] == "uncertain"
|
|
assert final["state"] == "uncertain"
|
|
assert final["next_stage_allowed"] is False
|
|
assert all(node["operation"]["status"] == "uncertain"
|
|
for node in final["nodes"])
|
|
assert all(component.manager.async_dispatch_calls == 1
|
|
for component in components.values())
|
|
|
|
|
|
def test_restart_resume_reads_exact_operation_without_redispatch(monkeypatch):
|
|
current = plan()
|
|
operation_id = "1" * 32
|
|
store = MemoryStore()
|
|
store.set(
|
|
"fabric_desired", current.fabric_id,
|
|
json.dumps(_interrupted_async_record(current, operation_id)))
|
|
components = _async_components(current)
|
|
for component in components.values():
|
|
component.manager.async_operation_id = operation_id
|
|
component.manager.async_statuses = ["succeeded"]
|
|
coordinator = FabricCoordinator(store, Registry(components))
|
|
monkeypatch.setattr(
|
|
"virsh_panel.fabric.FabricHostAdapter", AsyncCoordinatorAdapter)
|
|
|
|
async def exercise():
|
|
admitted = await coordinator.enqueue_reconcile(current.fabric_id)
|
|
assert admitted["operation"]["id"] == operation_id
|
|
await _await_fabric_operation(coordinator, current.fabric_id)
|
|
return coordinator.get_status(current.fabric_id)
|
|
|
|
final = asyncio.run(exercise())
|
|
|
|
assert final["operation"]["id"] == operation_id
|
|
assert final["operation"]["status"] == "succeeded"
|
|
assert final["next_stage_allowed"] is True
|
|
assert all(component.manager.async_dispatch_calls == 0
|
|
for component in components.values())
|
|
assert all(component.manager.async_status_calls == 1
|
|
for component in components.values())
|
|
assert all(component.manager.async_observe_calls == 1
|
|
for component in components.values())
|
|
|
|
|
|
def test_restart_resume_does_not_publish_running_job_as_succeeded(monkeypatch):
|
|
current = plan()
|
|
operation_id = "2" * 32
|
|
store = MemoryStore()
|
|
store.set(
|
|
"fabric_desired", current.fabric_id,
|
|
json.dumps(_interrupted_async_record(current, operation_id)))
|
|
components = _async_components(current)
|
|
for component in components.values():
|
|
component.manager.async_operation_id = operation_id
|
|
component.manager.async_statuses = ["running"]
|
|
coordinator = FabricCoordinator(store, Registry(components))
|
|
monkeypatch.setattr(
|
|
"virsh_panel.fabric.FabricHostAdapter", AsyncCoordinatorAdapter)
|
|
|
|
async def exercise():
|
|
await coordinator.enqueue_reconcile(current.fabric_id)
|
|
await _await_fabric_operation(coordinator, current.fabric_id)
|
|
return coordinator.get_status(current.fabric_id)
|
|
|
|
final = asyncio.run(exercise())
|
|
|
|
assert final["operation"]["id"] == operation_id
|
|
assert final["operation"]["status"] == "running"
|
|
assert final["next_stage_allowed"] is False
|
|
assert all(node["operation"]["status"] == "running"
|
|
for node in final["nodes"])
|
|
assert all(component.manager.async_dispatch_calls == 0
|
|
for component in components.values())
|
|
assert all(component.manager.async_observe_calls == 0
|
|
for component in components.values())
|
|
|
|
|
|
def test_restart_missing_runtime_operation_can_converge_by_exact_observation(
|
|
monkeypatch):
|
|
current = plan()
|
|
operation_id = "3" * 32
|
|
store = MemoryStore()
|
|
store.set(
|
|
"fabric_desired", current.fabric_id,
|
|
json.dumps(_interrupted_async_record(current, operation_id)))
|
|
components = _async_components(current)
|
|
for component in components.values():
|
|
component.manager.async_operation_id = operation_id
|
|
component.manager.async_statuses = [FabricError(
|
|
"safe missing", 409, "fabric_operation_not_found", "observe")]
|
|
coordinator = FabricCoordinator(store, Registry(components))
|
|
monkeypatch.setattr(
|
|
"virsh_panel.fabric.FabricHostAdapter", AsyncCoordinatorAdapter)
|
|
|
|
asyncio.run(coordinator.reconcile_all())
|
|
final = coordinator.get_status(current.fabric_id)
|
|
|
|
assert final["operation"]["id"] == operation_id
|
|
assert final["operation"]["status"] == "succeeded"
|
|
assert final["state"] == "active"
|
|
assert final["next_stage_allowed"] is True
|
|
assert all(component.manager.async_dispatch_calls == 0
|
|
for component in components.values())
|
|
assert all(component.manager.async_observe_calls == 1
|
|
for component in components.values())
|
|
|
|
|
|
@pytest.mark.parametrize("error_code", [
|
|
"fabric_host_connection_failed",
|
|
"fabric_operation_status_invalid",
|
|
"fabric_remote_command_failed",
|
|
])
|
|
def test_restart_status_failure_cannot_use_old_exact_observation(
|
|
monkeypatch, error_code):
|
|
current = plan()
|
|
operation_id = "4" * 32
|
|
store = MemoryStore()
|
|
store.set(
|
|
"fabric_desired", current.fabric_id,
|
|
json.dumps(_interrupted_async_record(current, operation_id)))
|
|
components = _async_components(current)
|
|
for component in components.values():
|
|
component.manager.async_operation_id = operation_id
|
|
component.manager.async_statuses = [FabricError(
|
|
"unsafe transport output /root/private", 502,
|
|
error_code, "observe")]
|
|
coordinator = FabricCoordinator(store, Registry(components))
|
|
monkeypatch.setattr(
|
|
"virsh_panel.fabric.FabricHostAdapter", AsyncCoordinatorAdapter)
|
|
|
|
asyncio.run(coordinator.reconcile_all())
|
|
final = coordinator.get_status(current.fabric_id)
|
|
durable = json.loads(store.get("fabric_desired", current.fabric_id))
|
|
|
|
assert final["operation"]["id"] == operation_id
|
|
assert final["operation"]["status"] == "uncertain"
|
|
assert final["state"] == "uncertain"
|
|
assert final["next_stage_allowed"] is False
|
|
assert all(node["operation"]["status"] == "uncertain"
|
|
for node in final["nodes"])
|
|
assert all(component.manager.async_dispatch_calls == 0
|
|
for component in components.values())
|
|
assert "/root/private" not in json.dumps(durable)
|
|
|
|
|
|
def test_async_identity_is_durably_fenced_before_first_host_dispatch(
|
|
monkeypatch):
|
|
current = plan()
|
|
store = MemoryStore()
|
|
components = _async_components(current)
|
|
entered = threading.Event()
|
|
release = threading.Event()
|
|
components["host-a"].manager.async_dispatch_entered = entered
|
|
components["host-a"].manager.async_dispatch_release = release
|
|
coordinator = FabricCoordinator(store, Registry(components))
|
|
monkeypatch.setattr(
|
|
"virsh_panel.fabric.FabricHostAdapter", AsyncCoordinatorAdapter)
|
|
monkeypatch.setattr(
|
|
"virsh_panel.fabric.FABRIC_OPERATION_POLL_INTERVAL_SECONDS", 0)
|
|
|
|
async def exercise():
|
|
admitted = await coordinator.enqueue_apply(plan_payload(current))
|
|
for _attempt in range(100):
|
|
if entered.is_set():
|
|
break
|
|
await asyncio.sleep(0.002)
|
|
assert entered.is_set()
|
|
durable = json.loads(store.get(
|
|
"fabric_desired", current.fabric_id))
|
|
release.set()
|
|
await _await_fabric_operation(coordinator, current.fabric_id)
|
|
return admitted, durable
|
|
|
|
try:
|
|
admitted, durable = asyncio.run(exercise())
|
|
finally:
|
|
release.set()
|
|
|
|
assert durable["operation"]["id"] == admitted["operation"]["id"]
|
|
assert durable["public_keys"] == {"host-a": KEY1, "host-b": KEY2}
|
|
assert {
|
|
node["host_id"]: node.get("public_key")
|
|
for node in durable["nodes"]
|
|
} == {"host-a": KEY1, "host-b": KEY2}
|
|
|
|
|
|
def test_observe_only_probe_failure_closes_durable_active_gate(monkeypatch):
|
|
current = plan()
|
|
store = MemoryStore()
|
|
store.set(
|
|
"fabric_desired", current.fabric_id,
|
|
json.dumps(_active_fabric_record(current)))
|
|
components = _fabric_components(current)
|
|
|
|
def probe_failed():
|
|
raise RemoteHostError(
|
|
"unsafe remote detail /root/private", 502,
|
|
"host_capability_probe_failed")
|
|
|
|
components["host-a"].manager.reprobe_current = probe_failed
|
|
coordinator = FabricCoordinator(store, Registry(components))
|
|
|
|
asyncio.run(coordinator.reconcile_all())
|
|
|
|
durable = json.loads(store.get("fabric_desired", current.fabric_id))
|
|
assert durable["state"] == "uncertain"
|
|
assert durable["next_stage_allowed"] is False
|
|
assert all(node["transport_verified"] is False
|
|
for node in durable["nodes"])
|
|
assert "/root/private" not in json.dumps(durable)
|
|
|
|
|
|
def test_enqueue_reconcile_admission_uses_shared_catalog_lock(monkeypatch):
|
|
current = plan()
|
|
store = MemoryStore()
|
|
store.set(
|
|
"fabric_desired", current.fabric_id,
|
|
json.dumps(_active_fabric_record(current)))
|
|
components = _async_components(current)
|
|
coordinator = FabricCoordinator(store, Registry(components))
|
|
monkeypatch.setattr(
|
|
"virsh_panel.fabric.FabricHostAdapter", AsyncCoordinatorAdapter)
|
|
|
|
async def exercise():
|
|
shared = coordinator.catalog_mutation_lock()
|
|
async with shared:
|
|
admission = asyncio.create_task(
|
|
coordinator.enqueue_reconcile(current.fabric_id))
|
|
await asyncio.sleep(0.02)
|
|
assert admission.done() is False
|
|
durable = json.loads(store.get(
|
|
"fabric_desired", current.fabric_id))
|
|
assert "operation" not in durable
|
|
receipt = await asyncio.wait_for(admission, timeout=1)
|
|
await coordinator.shutdown_operations()
|
|
return receipt
|
|
|
|
receipt = asyncio.run(exercise())
|
|
assert receipt["operation"]["kind"] == "reconcile"
|
|
assert receipt["operation"]["status"] == "queued"
|
|
|
|
|
|
def test_background_internal_error_cannot_leave_operation_running(monkeypatch):
|
|
current = plan()
|
|
store = MemoryStore()
|
|
components = _async_components(current)
|
|
|
|
class UnexpectedFailureAdapter(AsyncCoordinatorAdapter):
|
|
def dispatch_apply(self, *_args, **_kwargs):
|
|
self.manager.async_dispatch_calls += 1
|
|
raise RuntimeError("unsafe implementation detail /root/private")
|
|
|
|
coordinator = FabricCoordinator(store, Registry(components))
|
|
monkeypatch.setattr(
|
|
"virsh_panel.fabric.FabricHostAdapter", UnexpectedFailureAdapter)
|
|
|
|
async def exercise():
|
|
await coordinator.enqueue_apply(plan_payload(current))
|
|
await _await_fabric_operation(coordinator, current.fabric_id)
|
|
return coordinator.get_status(current.fabric_id)
|
|
|
|
final = asyncio.run(exercise())
|
|
|
|
assert final["operation"]["status"] in {
|
|
"failed", "uncertain", "interrupted",
|
|
}
|
|
assert final["operation"]["finished_at"] is not None
|
|
assert all(node["operation"]["status"] not in {"pending", "running"}
|
|
for node in final["nodes"])
|
|
assert "/root/private" not in json.dumps(final)
|
|
|
|
|
|
def test_periodic_reconcile_is_read_only_and_never_dispatches(monkeypatch):
|
|
current = plan()
|
|
store = MemoryStore()
|
|
store.set(
|
|
"fabric_desired", current.fabric_id,
|
|
json.dumps(_active_fabric_record(current)))
|
|
components = _async_components(current)
|
|
coordinator = FabricCoordinator(store, Registry(components))
|
|
monkeypatch.setattr(
|
|
"virsh_panel.fabric.FabricHostAdapter", AsyncCoordinatorAdapter)
|
|
|
|
asyncio.run(coordinator.reconcile_all())
|
|
|
|
final = coordinator.get_status(current.fabric_id)
|
|
assert final["state"] == "active"
|
|
assert final["next_stage_allowed"] is True
|
|
assert all(component.manager.async_dispatch_calls == 0
|
|
for component in components.values())
|
|
assert all(component.manager.async_status_calls == 0
|
|
for component in components.values())
|
|
assert all(component.manager.async_observe_calls == 1
|
|
for component in components.values())
|
|
|
|
|
|
def _reconcile_observation(current, host):
|
|
public_key = KEY1 if host.host_id == "host-a" else KEY2
|
|
return {
|
|
"state": "active",
|
|
"config_ok": True,
|
|
"transport_verified": True,
|
|
"public_key": public_key,
|
|
"observed": {
|
|
"interface": FABRIC_INTERFACE,
|
|
"config_revision": current.revision,
|
|
"config_digest": current.digest,
|
|
},
|
|
}
|
|
|
|
|
|
class ReconcileRecoveryAdapter:
|
|
"""Stateful host stub for fixed-plan reconcile recovery tests."""
|
|
|
|
def __init__(self, manager, _profile=None):
|
|
self.manager = manager
|
|
|
|
def observe(self, current, host, _keys):
|
|
self.manager.reconcile_observe_calls += 1
|
|
if self.manager.reconcile_observe_error is not None:
|
|
error = self.manager.reconcile_observe_error
|
|
if self.manager.reconcile_observe_error_once:
|
|
self.manager.reconcile_observe_error = None
|
|
raise error
|
|
if self.manager.reconcile_configured:
|
|
result = _reconcile_observation(current, host)
|
|
result["public_key"] = self.manager.public_key
|
|
result["transport_verified"] = (
|
|
self.manager.reconcile_transport_verified)
|
|
return result
|
|
return {
|
|
"state": "degraded",
|
|
"config_ok": False,
|
|
"transport_verified": False,
|
|
"public_key": self.manager.public_key,
|
|
"observed": {
|
|
"interface": None,
|
|
"config_revision": None,
|
|
"config_digest": None,
|
|
"emergency_state": "owned",
|
|
},
|
|
}
|
|
|
|
def identity(self, *, create_key=True):
|
|
self.manager.reconcile_identity_calls += 1
|
|
self.manager.reconcile_identity_create_flags.append(create_key)
|
|
entered = self.manager.reconcile_identity_entered
|
|
release = self.manager.reconcile_identity_release
|
|
if entered is not None:
|
|
entered.set()
|
|
if release is not None:
|
|
assert release.wait(timeout=2)
|
|
if self.manager.reconcile_identity_error is not None:
|
|
raise self.manager.reconcile_identity_error
|
|
return self.manager.public_key
|
|
|
|
def apply(self, *_args):
|
|
self.manager.reconcile_apply_calls += 1
|
|
entered = self.manager.reconcile_apply_entered
|
|
release = self.manager.reconcile_apply_release
|
|
if entered is not None:
|
|
entered.set()
|
|
if release is not None:
|
|
assert release.wait(timeout=2)
|
|
if self.manager.reconcile_apply_error is not None:
|
|
raise self.manager.reconcile_apply_error
|
|
self.manager.reconcile_configured = True
|
|
|
|
|
|
def _recovery_components(current, *, configured=False):
|
|
components = _fabric_components(current)
|
|
for component in components.values():
|
|
manager = component.manager
|
|
manager.reconcile_configured = configured
|
|
manager.reconcile_transport_verified = True
|
|
manager.reconcile_observe_calls = 0
|
|
manager.reconcile_observe_error = None
|
|
manager.reconcile_observe_error_once = False
|
|
manager.reconcile_identity_calls = 0
|
|
manager.reconcile_identity_create_flags = []
|
|
manager.reconcile_identity_error = None
|
|
manager.reconcile_identity_entered = None
|
|
manager.reconcile_identity_release = None
|
|
manager.reconcile_apply_calls = 0
|
|
manager.reconcile_apply_error = None
|
|
manager.reconcile_apply_entered = None
|
|
manager.reconcile_apply_release = None
|
|
return components
|
|
|
|
|
|
def test_successful_apply_clears_stale_failure_phase():
|
|
current = plan()
|
|
store = MemoryStore()
|
|
record = _active_fabric_record(current)
|
|
for node in record["nodes"]:
|
|
node["last_error_code"] = "fabric_remote_command_failed"
|
|
node["last_error_phase"] = "observe"
|
|
store.set("fabric_desired", current.fabric_id, json.dumps(record))
|
|
coordinator = FabricCoordinator(
|
|
store, Registry(_fabric_components(current)))
|
|
|
|
result = asyncio.run(coordinator.apply(plan_payload(current)))
|
|
|
|
assert result["state"] == "active"
|
|
assert all("last_error_phase" not in node for node in result["nodes"])
|
|
durable = json.loads(store.get("fabric_desired", current.fabric_id))
|
|
assert all("last_error_phase" not in node for node in durable["nodes"])
|
|
for host in current.nodes:
|
|
persisted = json.loads(store.get(
|
|
"fabric_node", f"{current.fabric_id}.{host.host_id}"))
|
|
assert "last_error_phase" not in persisted
|
|
assert persisted["last_error_code"] is None
|
|
|
|
|
|
def test_reconcile_recovers_when_incomplete_host_observe_initially_fails(
|
|
monkeypatch):
|
|
current = plan()
|
|
store = MemoryStore()
|
|
store.set("fabric_desired", current.fabric_id, json.dumps(
|
|
_active_fabric_record(current)))
|
|
components = _recovery_components(current)
|
|
first = components["host-a"].manager
|
|
first.reconcile_observe_error = FabricError(
|
|
"incomplete", 502, "fabric_remote_command_failed")
|
|
first.reconcile_observe_error_once = True
|
|
coordinator = FabricCoordinator(store, Registry(components))
|
|
monkeypatch.setattr(
|
|
"virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter)
|
|
|
|
result = asyncio.run(coordinator.reconcile(current.fabric_id))
|
|
|
|
assert result["state"] == "active"
|
|
assert result["next_stage_allowed"] is True
|
|
assert all("last_error_phase" not in node for node in result["nodes"])
|
|
assert all(component.manager.reconcile_identity_calls == 1
|
|
for component in components.values())
|
|
assert all(component.manager.reconcile_apply_calls == 1
|
|
for component in components.values())
|
|
assert first.reconcile_observe_calls == 2
|
|
|
|
|
|
def test_reconcile_persists_post_apply_observe_failure_phase(monkeypatch):
|
|
current = plan()
|
|
store = MemoryStore()
|
|
store.set("fabric_desired", current.fabric_id, json.dumps(
|
|
_active_fabric_record(current)))
|
|
components = _recovery_components(current)
|
|
failed = components["host-a"].manager
|
|
|
|
class PostApplyObserveFailureAdapter(ReconcileRecoveryAdapter):
|
|
def observe(self, plan_value, host, keys):
|
|
if self.manager is failed and self.manager.reconcile_configured:
|
|
self.manager.reconcile_observe_calls += 1
|
|
raise FabricError(
|
|
"incomplete", 502, "fabric_remote_command_failed")
|
|
return super().observe(plan_value, host, keys)
|
|
|
|
coordinator = FabricCoordinator(store, Registry(components))
|
|
monkeypatch.setattr(
|
|
"virsh_panel.fabric.FabricHostAdapter",
|
|
PostApplyObserveFailureAdapter)
|
|
|
|
result = asyncio.run(coordinator.reconcile(current.fabric_id))
|
|
|
|
nodes = {node["host_id"]: node for node in result["nodes"]}
|
|
assert result["state"] == "uncertain"
|
|
assert nodes["host-a"]["last_error_code"] == (
|
|
"fabric_remote_command_failed")
|
|
assert nodes["host-a"]["last_error_phase"] == "observe"
|
|
persisted = json.loads(store.get(
|
|
"fabric_node", f"{current.fabric_id}.host-a"))
|
|
assert persisted["last_error_phase"] == "observe"
|
|
assert failed.reconcile_observe_calls == 2
|
|
assert failed.reconcile_apply_calls == 1
|
|
|
|
|
|
def test_reconcile_identity_gate_keeps_other_host_observe_phase(monkeypatch):
|
|
current = plan()
|
|
store = MemoryStore()
|
|
store.set("fabric_desired", current.fabric_id, json.dumps(
|
|
_active_fabric_record(current)))
|
|
components = _recovery_components(current)
|
|
observed_failed = components["host-a"].manager
|
|
identity_failed = components["host-b"].manager
|
|
observed_failed.reconcile_observe_error = FabricError(
|
|
"incomplete", 502, "fabric_remote_command_failed")
|
|
identity_failed.reconcile_identity_error = FabricError(
|
|
"offline", 502, "fabric_remote_command_failed")
|
|
coordinator = FabricCoordinator(store, Registry(components))
|
|
monkeypatch.setattr(
|
|
"virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter)
|
|
|
|
result = asyncio.run(coordinator.reconcile(current.fabric_id))
|
|
|
|
nodes = {node["host_id"]: node for node in result["nodes"]}
|
|
assert nodes["host-a"]["last_error_phase"] == "observe"
|
|
assert nodes["host-b"]["last_error_phase"] == "identity"
|
|
assert all(component.manager.reconcile_apply_calls == 0
|
|
for component in components.values())
|
|
|
|
|
|
def test_successful_reconcile_observation_clears_stale_failure_phase(
|
|
monkeypatch):
|
|
current = plan()
|
|
store = MemoryStore()
|
|
record = _active_fabric_record(current)
|
|
for node in record["nodes"]:
|
|
node["last_error_code"] = "fabric_remote_command_failed"
|
|
node["last_error_phase"] = "apply"
|
|
store.set("fabric_desired", current.fabric_id, json.dumps(record))
|
|
components = _recovery_components(current, configured=True)
|
|
coordinator = FabricCoordinator(store, Registry(components))
|
|
monkeypatch.setattr(
|
|
"virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter)
|
|
|
|
result = asyncio.run(coordinator.reconcile(current.fabric_id))
|
|
|
|
assert result["state"] == "active"
|
|
assert all(node["last_error_code"] is None for node in result["nodes"])
|
|
assert all("last_error_phase" not in node for node in result["nodes"])
|
|
durable = json.loads(store.get("fabric_desired", current.fabric_id))
|
|
assert all("last_error_phase" not in node for node in durable["nodes"])
|
|
for host in current.nodes:
|
|
persisted = json.loads(store.get(
|
|
"fabric_node", f"{current.fabric_id}.{host.host_id}"))
|
|
assert "last_error_phase" not in persisted
|
|
assert persisted["last_error_code"] is None
|
|
|
|
|
|
def test_reconcile_recovers_key_and_emergency_only_then_is_idempotent(
|
|
monkeypatch):
|
|
current = plan()
|
|
store = MemoryStore()
|
|
store.set("fabric_desired", current.fabric_id, json.dumps(
|
|
_active_fabric_record(current)))
|
|
components = _recovery_components(current)
|
|
coordinator = FabricCoordinator(store, Registry(components))
|
|
monkeypatch.setattr(
|
|
"virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter)
|
|
|
|
first = asyncio.run(coordinator.reconcile(current.fabric_id))
|
|
second = asyncio.run(coordinator.reconcile(current.fabric_id))
|
|
|
|
assert first["state"] == "active"
|
|
assert first["next_stage_allowed"] is True
|
|
assert second["state"] == "active"
|
|
assert second["next_stage_allowed"] is True
|
|
for component in components.values():
|
|
manager = component.manager
|
|
assert manager.reconcile_apply_calls == 1
|
|
assert manager.reconcile_identity_calls == 1
|
|
assert manager.reconcile_identity_create_flags == [False]
|
|
# initial degraded + post-apply exact + second reconcile exact
|
|
assert manager.reconcile_observe_calls == 3
|
|
|
|
|
|
def test_reconcile_does_not_reapply_exact_config_for_stale_transport(
|
|
monkeypatch):
|
|
current = plan()
|
|
store = MemoryStore()
|
|
store.set("fabric_desired", current.fabric_id, json.dumps(
|
|
_active_fabric_record(current)))
|
|
components = _recovery_components(current, configured=True)
|
|
for component in components.values():
|
|
component.manager.reconcile_transport_verified = False
|
|
coordinator = FabricCoordinator(store, Registry(components))
|
|
monkeypatch.setattr(
|
|
"virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter)
|
|
|
|
result = asyncio.run(coordinator.reconcile(current.fabric_id))
|
|
|
|
assert result["state"] == "degraded"
|
|
assert result["next_stage_allowed"] is False
|
|
for component in components.values():
|
|
manager = component.manager
|
|
assert manager.reconcile_apply_calls == 0
|
|
assert manager.reconcile_identity_calls == 0
|
|
assert manager.reconcile_observe_calls == 1
|
|
|
|
|
|
def test_reconcile_does_not_reapply_exact_config_for_failed_ping():
|
|
current = plan()
|
|
store = MemoryStore()
|
|
store.set("fabric_desired", current.fabric_id, json.dumps(
|
|
_active_fabric_record(current)))
|
|
components = _fabric_components(current)
|
|
for component in components.values():
|
|
component.manager.ping_ok = False
|
|
coordinator = FabricCoordinator(store, Registry(components))
|
|
|
|
result = asyncio.run(coordinator.reconcile(current.fabric_id))
|
|
|
|
assert result["state"] == "degraded"
|
|
assert result["next_stage_allowed"] is False
|
|
assert all(node["state"] == "active" for node in result["nodes"])
|
|
assert all(node["transport_verified"] is False
|
|
for node in result["nodes"])
|
|
# One script is the initial observe. An identity probe or apply would add
|
|
# another script and prove reconcile mistook liveness for config drift.
|
|
assert all(len(component.manager.scripts) == 1
|
|
for component in components.values())
|
|
|
|
|
|
def test_reconcile_repairs_live_public_key_drift():
|
|
current = plan()
|
|
store = MemoryStore()
|
|
store.set("fabric_desired", current.fabric_id, json.dumps(
|
|
_active_fabric_record(current)))
|
|
components = _fabric_components(current)
|
|
drifted = components["host-a"].manager
|
|
drifted.live_public_key = KEY3
|
|
coordinator = FabricCoordinator(store, Registry(components))
|
|
|
|
result = asyncio.run(coordinator.reconcile(current.fabric_id))
|
|
|
|
assert result["state"] == "active"
|
|
assert result["next_stage_allowed"] is True
|
|
assert drifted.live_public_key == KEY1
|
|
# observe -> fixed-key identity -> apply -> exact post-observe
|
|
assert len(drifted.scripts) == 4
|
|
# The already exact peer receives only its initial observation.
|
|
assert len(components["host-b"].manager.scripts) == 1
|
|
|
|
|
|
@pytest.mark.parametrize("identity_failure", ["drift", "unavailable"])
|
|
def test_reconcile_identity_failure_blocks_all_host_writes(
|
|
monkeypatch, identity_failure):
|
|
current = plan()
|
|
store = MemoryStore()
|
|
store.set("fabric_desired", current.fabric_id, json.dumps(
|
|
_active_fabric_record(current)))
|
|
components = _recovery_components(current)
|
|
failed = components["host-b"].manager
|
|
expected_error = "fabric_identity_conflict"
|
|
if identity_failure == "drift":
|
|
failed.public_key = KEY3
|
|
else:
|
|
expected_error = "fabric_remote_command_failed"
|
|
failed.reconcile_identity_error = FabricError(
|
|
"offline", 502, expected_error)
|
|
coordinator = FabricCoordinator(store, Registry(components))
|
|
monkeypatch.setattr(
|
|
"virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter)
|
|
|
|
result = asyncio.run(coordinator.reconcile(current.fabric_id))
|
|
|
|
nodes = {node["host_id"]: node for node in result["nodes"]}
|
|
assert result["state"] == "uncertain"
|
|
assert result["next_stage_allowed"] is False
|
|
assert nodes["host-b"]["last_error_code"] == expected_error
|
|
assert nodes["host-b"]["last_error_phase"] == "identity"
|
|
persisted = json.loads(store.get(
|
|
"fabric_node", f"{current.fabric_id}.host-b"))
|
|
assert persisted["last_error_phase"] == "identity"
|
|
for component in components.values():
|
|
manager = component.manager
|
|
assert manager.reconcile_identity_calls == 1
|
|
assert manager.reconcile_identity_create_flags == [False]
|
|
assert manager.reconcile_apply_calls == 0
|
|
|
|
|
|
def test_reconcile_apply_failure_on_one_node_does_not_stop_other_node(
|
|
monkeypatch):
|
|
current = plan()
|
|
store = MemoryStore()
|
|
store.set("fabric_desired", current.fabric_id, json.dumps(
|
|
_active_fabric_record(current)))
|
|
components = _recovery_components(current)
|
|
components["host-a"].manager.reconcile_apply_error = FabricError(
|
|
"offline", 502, "fabric_remote_command_failed")
|
|
coordinator = FabricCoordinator(store, Registry(components))
|
|
monkeypatch.setattr(
|
|
"virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter)
|
|
|
|
result = asyncio.run(coordinator.reconcile(current.fabric_id))
|
|
|
|
nodes = {node["host_id"]: node for node in result["nodes"]}
|
|
assert result["state"] == "uncertain"
|
|
assert nodes["host-a"]["last_error_code"] == (
|
|
"fabric_remote_command_failed")
|
|
assert nodes["host-a"]["last_error_phase"] == "apply"
|
|
assert nodes["host-b"]["state"] == "active"
|
|
assert nodes["host-b"]["observed_plan_digest"] == current.digest
|
|
assert all(component.manager.reconcile_identity_calls == 1
|
|
for component in components.values())
|
|
assert all(component.manager.reconcile_apply_calls == 1
|
|
for component in components.values())
|
|
|
|
|
|
def test_cancelled_reconcile_drains_admitted_host_apply_before_unlock(
|
|
monkeypatch):
|
|
current = plan()
|
|
store = MemoryStore()
|
|
store.set("fabric_desired", current.fabric_id, json.dumps(
|
|
_active_fabric_record(current)))
|
|
components = _recovery_components(current)
|
|
entered = threading.Event()
|
|
release = threading.Event()
|
|
first = components["host-a"].manager
|
|
first.reconcile_apply_entered = entered
|
|
first.reconcile_apply_release = release
|
|
coordinator = FabricCoordinator(store, Registry(components))
|
|
monkeypatch.setattr(
|
|
"virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter)
|
|
|
|
async def exercise():
|
|
reconcile_task = asyncio.create_task(
|
|
coordinator.reconcile(current.fabric_id))
|
|
for _attempt in range(100):
|
|
if entered.is_set():
|
|
break
|
|
await asyncio.sleep(0.01)
|
|
assert entered.is_set()
|
|
|
|
lock_entered = asyncio.Event()
|
|
|
|
async def wait_for_fabric_lock():
|
|
async with coordinator._locks[current.fabric_id]:
|
|
lock_entered.set()
|
|
|
|
waiter = asyncio.create_task(wait_for_fabric_lock())
|
|
reconcile_task.cancel()
|
|
await asyncio.sleep(0)
|
|
assert reconcile_task.done() is False
|
|
assert lock_entered.is_set() is False
|
|
release.set()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await reconcile_task
|
|
await asyncio.wait_for(waiter, timeout=1)
|
|
assert lock_entered.is_set() is True
|
|
|
|
try:
|
|
asyncio.run(exercise())
|
|
finally:
|
|
release.set()
|
|
assert first.reconcile_apply_calls == 1
|
|
assert first.reconcile_configured is True
|
|
assert components["host-b"].manager.reconcile_apply_calls == 0
|
|
|
|
|
|
def test_cancelled_reconcile_drains_runtime_identity_lock_before_unlock(
|
|
monkeypatch):
|
|
current = plan()
|
|
store = MemoryStore()
|
|
store.set("fabric_desired", current.fabric_id, json.dumps(
|
|
_active_fabric_record(current)))
|
|
components = _recovery_components(current)
|
|
entered = threading.Event()
|
|
release = threading.Event()
|
|
first = components["host-a"].manager
|
|
first.reconcile_identity_entered = entered
|
|
first.reconcile_identity_release = release
|
|
coordinator = FabricCoordinator(store, Registry(components))
|
|
monkeypatch.setattr(
|
|
"virsh_panel.fabric.FabricHostAdapter", ReconcileRecoveryAdapter)
|
|
|
|
async def exercise():
|
|
reconcile_task = asyncio.create_task(
|
|
coordinator.reconcile(current.fabric_id))
|
|
for _attempt in range(100):
|
|
if entered.is_set():
|
|
break
|
|
await asyncio.sleep(0.01)
|
|
assert entered.is_set()
|
|
|
|
lock_entered = asyncio.Event()
|
|
|
|
async def wait_for_fabric_lock():
|
|
async with coordinator._locks[current.fabric_id]:
|
|
lock_entered.set()
|
|
|
|
waiter = asyncio.create_task(wait_for_fabric_lock())
|
|
reconcile_task.cancel()
|
|
await asyncio.sleep(0)
|
|
assert reconcile_task.done() is False
|
|
assert lock_entered.is_set() is False
|
|
release.set()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await reconcile_task
|
|
await asyncio.wait_for(waiter, timeout=1)
|
|
assert lock_entered.is_set() is True
|
|
|
|
try:
|
|
asyncio.run(exercise())
|
|
finally:
|
|
release.set()
|
|
assert first.reconcile_identity_calls == 1
|
|
assert first.reconcile_apply_calls == 0
|
|
assert components["host-b"].manager.reconcile_identity_calls == 0
|
|
assert components["host-b"].manager.reconcile_apply_calls == 0
|
|
|
|
|
|
def _vpc_payload_for_fabric(current, *, vpc_id="vpc-race"):
|
|
return {
|
|
"schema_version": 1,
|
|
"vpc_id": vpc_id,
|
|
"region": current.region,
|
|
"fabric_id": current.fabric_id,
|
|
"fabric_revision": current.revision,
|
|
"cidr": "10.20.0.0/24",
|
|
"desired_revision": 1,
|
|
"operation_id": f"{vpc_id}-operation-0001",
|
|
"subnets": [
|
|
{"subnet_id": "subnet-a", "cidr": "10.20.0.0/25"},
|
|
],
|
|
"hosts": [{
|
|
"host_id": "host-a",
|
|
"members": [{
|
|
"member_id": "member-a",
|
|
"instance_uuid": "11111111-1111-4111-8111-111111111111",
|
|
"private_ip_cidr": "10.20.0.10/32",
|
|
"subnet_id": "subnet-a",
|
|
}],
|
|
}],
|
|
"routes": [],
|
|
"acls": [],
|
|
}
|
|
|
|
|
|
def test_reconcile_waits_for_admitted_vpc_apply_before_invalidating(
|
|
monkeypatch):
|
|
from virsh_panel.vpc import VpcCoordinator
|
|
|
|
current = plan()
|
|
store = MemoryStore()
|
|
store.set("fabric_desired", current.fabric_id, json.dumps(
|
|
_active_fabric_record(current)))
|
|
coordinator = FabricCoordinator(
|
|
store, Registry(_fabric_components(current)))
|
|
vpc = VpcCoordinator(
|
|
store, coordinator,
|
|
catalog_lock=coordinator.catalog_mutation_lock,
|
|
)
|
|
vpc_entered = asyncio.Event()
|
|
vpc_release = asyncio.Event()
|
|
observe_entered = threading.Event()
|
|
observe_release = threading.Event()
|
|
|
|
async def blocked_vpc_apply(_plan):
|
|
vpc_entered.set()
|
|
await vpc_release.wait()
|
|
return {"vpc_id": "vpc-race", "state": "active"}
|
|
|
|
class BlockingObserveAdapter:
|
|
def __init__(self, _manager, _profile=None):
|
|
pass
|
|
|
|
def observe(self, _plan, host, _keys):
|
|
observe_entered.set()
|
|
assert observe_release.wait(timeout=2)
|
|
return _reconcile_observation(current, host)
|
|
|
|
vpc._apply_locked = blocked_vpc_apply
|
|
monkeypatch.setattr(
|
|
"virsh_panel.fabric.FabricHostAdapter", BlockingObserveAdapter)
|
|
vpc_payload = _vpc_payload_for_fabric(current)
|
|
|
|
async def exercise():
|
|
vpc_task = asyncio.create_task(vpc.apply(vpc_payload))
|
|
await vpc_entered.wait()
|
|
reconcile_task = asyncio.create_task(
|
|
coordinator.reconcile(current.fabric_id))
|
|
await asyncio.sleep(0)
|
|
await asyncio.sleep(0)
|
|
before = json.loads(store.get(
|
|
"fabric_desired", current.fabric_id))
|
|
assert before["state"] == "active"
|
|
assert before["next_stage_allowed"] is True
|
|
|
|
vpc_release.set()
|
|
await vpc_task
|
|
for _attempt in range(100):
|
|
if observe_entered.is_set():
|
|
break
|
|
await asyncio.sleep(0.01)
|
|
assert observe_entered.is_set()
|
|
invalidated = json.loads(store.get(
|
|
"fabric_desired", current.fabric_id))
|
|
assert invalidated["state"] == "uncertain"
|
|
assert invalidated["next_stage_allowed"] is False
|
|
observe_release.set()
|
|
return await asyncio.wait_for(reconcile_task, timeout=1)
|
|
|
|
try:
|
|
result = asyncio.run(exercise())
|
|
finally:
|
|
observe_release.set()
|
|
assert result["state"] == "active"
|
|
assert result["next_stage_allowed"] is True
|
|
|
|
|
|
def test_reconcile_first_durable_invalidation_blocks_vpc_host_mutation(
|
|
monkeypatch):
|
|
from virsh_panel.vpc import VpcCoordinator
|
|
|
|
current = plan()
|
|
|
|
class BlockingInvalidationStore(MemoryStore):
|
|
def __init__(self):
|
|
super().__init__()
|
|
self.block = False
|
|
self.entered = threading.Event()
|
|
self.release = threading.Event()
|
|
|
|
def set(self, kind, key, value):
|
|
parsed = json.loads(value)
|
|
if (self.block and kind == "fabric_desired"
|
|
and parsed.get("state") == "uncertain"):
|
|
self.entered.set()
|
|
assert self.release.wait(timeout=2)
|
|
super().set(kind, key, value)
|
|
|
|
observe_entered = threading.Event()
|
|
observe_release = threading.Event()
|
|
|
|
class BlockingObserveAdapter:
|
|
def __init__(self, _manager, _profile=None):
|
|
pass
|
|
|
|
def observe(self, _plan, host, _keys):
|
|
observe_entered.set()
|
|
assert observe_release.wait(timeout=2)
|
|
return _reconcile_observation(current, host)
|
|
|
|
store = BlockingInvalidationStore()
|
|
store.set("fabric_desired", current.fabric_id, json.dumps(
|
|
_active_fabric_record(current)))
|
|
store.block = True
|
|
coordinator = FabricCoordinator(
|
|
store, Registry(_fabric_components(current)))
|
|
vpc = VpcCoordinator(
|
|
store, coordinator,
|
|
registry=SimpleNamespace(components={}),
|
|
catalog_lock=coordinator.catalog_mutation_lock,
|
|
)
|
|
host_mutation_called = False
|
|
|
|
def forbidden_host_validation(_plan):
|
|
nonlocal host_mutation_called
|
|
host_mutation_called = True
|
|
raise AssertionError("inactive Fabric reached host mutation")
|
|
|
|
vpc._validate_dataplane_resources = forbidden_host_validation
|
|
monkeypatch.setattr(
|
|
"virsh_panel.fabric.FabricHostAdapter", BlockingObserveAdapter)
|
|
|
|
async def exercise():
|
|
reconcile_task = asyncio.create_task(
|
|
coordinator.reconcile(current.fabric_id))
|
|
for _attempt in range(100):
|
|
if store.entered.is_set():
|
|
break
|
|
await asyncio.sleep(0.01)
|
|
assert store.entered.is_set()
|
|
|
|
vpc_task = asyncio.create_task(vpc.apply(
|
|
_vpc_payload_for_fabric(current, vpc_id="vpc-after-reconcile")))
|
|
await asyncio.sleep(0)
|
|
assert vpc_task.done() is False
|
|
assert host_mutation_called is False
|
|
still_active = json.loads(store.get(
|
|
"fabric_desired", current.fabric_id))
|
|
assert still_active["state"] == "active"
|
|
|
|
store.release.set()
|
|
for _attempt in range(100):
|
|
if observe_entered.is_set():
|
|
break
|
|
await asyncio.sleep(0.01)
|
|
assert observe_entered.is_set()
|
|
result = await asyncio.wait_for(vpc_task, timeout=1)
|
|
assert result["state"] == "degraded"
|
|
assert result["next_stage_allowed"] is False
|
|
assert result["fabric_gate"]["error_code"] == (
|
|
"vpc_fabric_not_active")
|
|
assert host_mutation_called is False
|
|
durable_vpc = json.loads(store.get(
|
|
"vpc_desired", "vpc-after-reconcile"))
|
|
assert durable_vpc["state"] == "degraded"
|
|
|
|
observe_release.set()
|
|
await asyncio.wait_for(reconcile_task, timeout=1)
|
|
|
|
try:
|
|
asyncio.run(exercise())
|
|
finally:
|
|
store.release.set()
|
|
observe_release.set()
|
|
|
|
|
|
def test_reconcile_releases_catalog_during_remote_observe_and_relocks_publish(
|
|
monkeypatch):
|
|
current = plan()
|
|
store = MemoryStore()
|
|
store.set("fabric_desired", current.fabric_id, json.dumps(
|
|
_active_fabric_record(current)))
|
|
coordinator = FabricCoordinator(
|
|
store, Registry(_fabric_components(current)))
|
|
observe_entered = threading.Event()
|
|
observe_release = threading.Event()
|
|
all_observed = threading.Event()
|
|
count_lock = threading.Lock()
|
|
observation_count = 0
|
|
|
|
class BlockingObserveAdapter:
|
|
def __init__(self, _manager, _profile=None):
|
|
pass
|
|
|
|
def observe(self, _plan, host, _keys):
|
|
nonlocal observation_count
|
|
observe_entered.set()
|
|
assert observe_release.wait(timeout=2)
|
|
with count_lock:
|
|
observation_count += 1
|
|
if observation_count == len(current.nodes):
|
|
all_observed.set()
|
|
return _reconcile_observation(current, host)
|
|
|
|
monkeypatch.setattr(
|
|
"virsh_panel.fabric.FabricHostAdapter", BlockingObserveAdapter)
|
|
|
|
async def exercise():
|
|
reconcile_task = asyncio.create_task(
|
|
coordinator.reconcile(current.fabric_id))
|
|
for _attempt in range(100):
|
|
if observe_entered.is_set():
|
|
break
|
|
await asyncio.sleep(0.01)
|
|
assert observe_entered.is_set()
|
|
shared_lock = coordinator.catalog_mutation_lock()
|
|
await asyncio.wait_for(shared_lock.acquire(), timeout=1)
|
|
try:
|
|
observe_release.set()
|
|
for _attempt in range(100):
|
|
if all_observed.is_set():
|
|
break
|
|
await asyncio.sleep(0.01)
|
|
assert all_observed.is_set()
|
|
assert reconcile_task.done() is False
|
|
durable = json.loads(store.get(
|
|
"fabric_desired", current.fabric_id))
|
|
assert durable["state"] == "uncertain"
|
|
assert durable["next_stage_allowed"] is False
|
|
finally:
|
|
shared_lock.release()
|
|
return await asyncio.wait_for(reconcile_task, timeout=1)
|
|
|
|
try:
|
|
result = asyncio.run(exercise())
|
|
finally:
|
|
observe_release.set()
|
|
assert result["state"] == "active"
|
|
assert result["next_stage_allowed"] is True
|
|
|
|
|
|
@pytest.mark.parametrize("blocked_state", ["uncertain", "active"])
|
|
def test_cancelled_reconcile_drains_root_save_before_catalog_unlock(
|
|
monkeypatch, blocked_state):
|
|
current = plan()
|
|
|
|
class BlockingRootStore(MemoryStore):
|
|
def __init__(self):
|
|
super().__init__()
|
|
self.block = False
|
|
self.entered = threading.Event()
|
|
self.release = threading.Event()
|
|
|
|
def set(self, kind, key, value):
|
|
parsed = json.loads(value)
|
|
if (self.block and kind == "fabric_desired"
|
|
and parsed.get("state") == blocked_state):
|
|
self.entered.set()
|
|
assert self.release.wait(timeout=2)
|
|
super().set(kind, key, value)
|
|
|
|
class ImmediateObserveAdapter:
|
|
def __init__(self, _manager, _profile=None):
|
|
pass
|
|
|
|
def observe(self, _plan, host, _keys):
|
|
return _reconcile_observation(current, host)
|
|
|
|
store = BlockingRootStore()
|
|
store.set("fabric_desired", current.fabric_id, json.dumps(
|
|
_active_fabric_record(current)))
|
|
store.block = True
|
|
coordinator = FabricCoordinator(
|
|
store, Registry(_fabric_components(current)))
|
|
monkeypatch.setattr(
|
|
"virsh_panel.fabric.FabricHostAdapter", ImmediateObserveAdapter)
|
|
|
|
async def exercise():
|
|
reconcile_task = asyncio.create_task(
|
|
coordinator.reconcile(current.fabric_id))
|
|
for _attempt in range(100):
|
|
if store.entered.is_set():
|
|
break
|
|
await asyncio.sleep(0.01)
|
|
assert store.entered.is_set()
|
|
|
|
catalog_entered = asyncio.Event()
|
|
|
|
async def competing_catalog_apply():
|
|
async with coordinator.catalog_mutation_lock():
|
|
catalog_entered.set()
|
|
|
|
competitor = asyncio.create_task(competing_catalog_apply())
|
|
await asyncio.sleep(0)
|
|
reconcile_task.cancel()
|
|
await asyncio.sleep(0)
|
|
assert reconcile_task.done() is False
|
|
assert catalog_entered.is_set() is False
|
|
store.release.set()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await reconcile_task
|
|
await asyncio.wait_for(competitor, timeout=1)
|
|
assert catalog_entered.is_set() is True
|
|
|
|
try:
|
|
asyncio.run(exercise())
|
|
finally:
|
|
store.release.set()
|
|
durable = json.loads(store.get("fabric_desired", current.fabric_id))
|
|
assert durable["state"] == blocked_state
|
|
assert durable["next_stage_allowed"] is (blocked_state == "active")
|
|
|
|
|
|
def test_new_fabric_cannot_reuse_a_host_or_overlay_catalog_entry():
|
|
store = MemoryStore()
|
|
store.set("fabric_desired", "existing", json.dumps({
|
|
"fabric_id": "existing",
|
|
"overlay_cidr": "100.64.2.0/24",
|
|
"nodes": [{"host_id": "host-a"}, {"host_id": "host-c"}],
|
|
}))
|
|
coordinator = FabricCoordinator(store, Registry({}))
|
|
value = {
|
|
"fabric_id": "new-fabric", "region": "hk",
|
|
"overlay_cidr": "100.64.2.0/24", "revision": 1,
|
|
"nodes": [
|
|
{"host_id": "host-a", "overlay_ip": "100.64.2.10",
|
|
"endpoint": "203.0.113.1:51820"},
|
|
{"host_id": "host-b", "overlay_ip": "100.64.2.11",
|
|
"endpoint": "203.0.113.2:51820"},
|
|
],
|
|
}
|
|
with pytest.raises(FabricError) as error:
|
|
asyncio.run(coordinator.apply(value))
|
|
assert error.value.error_code == "fabric_host_interface_conflict"
|
|
|
|
|
|
def test_existing_fabric_region_is_immutable_without_migration():
|
|
store = MemoryStore()
|
|
current = plan()
|
|
store.set("fabric_desired", current.fabric_id, json.dumps({
|
|
"fabric_id": current.fabric_id,
|
|
"region": current.region,
|
|
"revision": current.revision,
|
|
"plan_digest": current.digest,
|
|
"nodes": [{"host_id": node.host_id} for node in current.nodes],
|
|
}))
|
|
coordinator = FabricCoordinator(store, Registry({}))
|
|
value = {
|
|
"fabric_id": current.fabric_id,
|
|
"region": "sg",
|
|
"overlay_cidr": current.overlay_cidr,
|
|
"revision": current.revision + 1,
|
|
"nodes": [
|
|
{"host_id": node.host_id, "overlay_ip": node.overlay_ip,
|
|
"endpoint": node.endpoint}
|
|
for node in current.nodes
|
|
],
|
|
}
|
|
|
|
with pytest.raises(FabricError) as error:
|
|
asyncio.run(coordinator.apply(value))
|
|
|
|
assert error.value.error_code == "fabric_region_conflict"
|
|
|
|
|
|
@pytest.mark.parametrize(("tenant_fabric", "tenant_region", "expected"), [
|
|
("hk-fabric", "sg", "fabric_region_conflict"),
|
|
("other-fabric", "sg", "fabric_endpoint_vpc_conflict"),
|
|
])
|
|
def test_fabric_checks_region_identity_and_all_tenant_cidrs(
|
|
tenant_fabric, tenant_region, expected):
|
|
from virsh_panel.vpc import parse_vpc_plan
|
|
|
|
store = MemoryStore()
|
|
tenant = parse_vpc_plan({
|
|
"schema_version": 1,
|
|
"vpc_id": "vpc-existing",
|
|
"region": tenant_region,
|
|
"fabric_id": tenant_fabric,
|
|
"fabric_revision": 1,
|
|
"cidr": "192.168.50.0/24",
|
|
"desired_revision": 1,
|
|
"operation_id": "vpc-op-region-0001",
|
|
"subnets": [{"subnet_id": "subnet-a",
|
|
"cidr": "192.168.50.0/25"}],
|
|
"hosts": [{"host_id": "host-a", "members": [{
|
|
"member_id": "member-a",
|
|
"instance_uuid": "11111111-1111-4111-8111-111111111111",
|
|
"private_ip_cidr": "192.168.50.10/32",
|
|
"subnet_id": "subnet-a",
|
|
}]}],
|
|
"routes": [],
|
|
"acls": [],
|
|
})
|
|
record = tenant.canonical()
|
|
record.update({
|
|
"operation_id": tenant.operation_id,
|
|
"desired_digest": tenant.desired_digest,
|
|
})
|
|
store.set("vpc_desired", tenant.vpc_id, json.dumps(record))
|
|
coordinator = FabricCoordinator(store, Registry({}))
|
|
candidate = parse_fabric_plan({
|
|
"fabric_id": "hk-fabric", "region": "hk",
|
|
"overlay_cidr": "100.64.1.0/24", "revision": 1,
|
|
"nodes": [
|
|
{"host_id": "host-a", "overlay_ip": "100.64.1.1",
|
|
"endpoint": "192.168.50.20:51820"},
|
|
{"host_id": "host-b", "overlay_ip": "100.64.1.2",
|
|
"endpoint": "203.0.113.2:51820"},
|
|
],
|
|
})
|
|
|
|
with pytest.raises(FabricError) as error:
|
|
coordinator._validate_resource_exclusivity(candidate)
|
|
|
|
assert error.value.error_code == expected
|
|
|
|
|
|
def test_new_fabric_rejects_endpoint_inside_existing_tenant_cidr():
|
|
from virsh_panel.vpc import parse_vpc_plan
|
|
|
|
store = MemoryStore()
|
|
tenant = parse_vpc_plan({
|
|
"schema_version": 1,
|
|
"vpc_id": "vpc-hk-1",
|
|
"region": "hk",
|
|
"fabric_id": "hk-fabric",
|
|
"fabric_revision": 1,
|
|
"cidr": "10.20.0.0/24",
|
|
"desired_revision": 1,
|
|
"operation_id": "vpc-op-00000001",
|
|
"subnets": [{"subnet_id": "subnet-a", "cidr": "10.20.0.0/25"}],
|
|
"hosts": [{"host_id": "host-a", "members": [{
|
|
"member_id": "member-a",
|
|
"instance_uuid": "11111111-1111-4111-8111-111111111111",
|
|
"private_ip_cidr": "10.20.0.10/32",
|
|
"subnet_id": "subnet-a",
|
|
}]}],
|
|
"routes": [],
|
|
"acls": [],
|
|
})
|
|
tenant_record = tenant.canonical()
|
|
tenant_record.update({
|
|
"operation_id": tenant.operation_id,
|
|
"desired_digest": tenant.desired_digest,
|
|
})
|
|
store.set("vpc_desired", tenant.vpc_id, json.dumps(tenant_record))
|
|
coordinator = FabricCoordinator(store, Registry({}))
|
|
value = {
|
|
"fabric_id": "hk-fabric", "region": "hk",
|
|
"overlay_cidr": "100.64.1.0/24", "revision": 1,
|
|
"nodes": [
|
|
{"host_id": "host-a", "overlay_ip": "100.64.1.1",
|
|
"endpoint": "10.20.0.50:51820"},
|
|
{"host_id": "host-b", "overlay_ip": "100.64.1.2",
|
|
"endpoint": "192.168.50.2:51820"},
|
|
],
|
|
}
|
|
|
|
with pytest.raises(FabricError) as error:
|
|
asyncio.run(coordinator.apply(value))
|
|
|
|
assert error.value.error_code == "fabric_endpoint_vpc_conflict"
|
|
|
|
current = parse_fabric_plan(value)
|
|
store.set("fabric_desired", current.fabric_id, json.dumps({
|
|
"schema_version": 1,
|
|
"fabric_id": current.fabric_id,
|
|
"region": current.region,
|
|
"overlay_cidr": current.overlay_cidr,
|
|
"revision": current.revision,
|
|
"listen_port": current.listen_port,
|
|
"plan_digest": current.digest,
|
|
"state": "active",
|
|
"next_stage_allowed": True,
|
|
"updated_at": int(time.time()),
|
|
"public_keys": {"host-a": KEY1, "host-b": KEY2},
|
|
"nodes": [
|
|
{"host_id": node.host_id, "overlay_ip": node.overlay_ip,
|
|
"endpoint": node.endpoint, "listen_port": node.listen_port,
|
|
"state": "active", "transport_verified": True}
|
|
for node in current.nodes
|
|
],
|
|
}))
|
|
with pytest.raises(FabricError) as reconcile_error:
|
|
asyncio.run(coordinator.reconcile(current.fabric_id))
|
|
assert reconcile_error.value.error_code == "fabric_endpoint_vpc_conflict"
|
|
invalidated = json.loads(store.get("fabric_desired", current.fabric_id))
|
|
assert invalidated["state"] == "uncertain"
|
|
assert invalidated["next_stage_allowed"] is False
|
|
|
|
|
|
class TenantReadbackManager(FakeManager):
|
|
"""Synthetic readback with panel-owned VPC tenant state."""
|
|
|
|
def __init__(self, *args, tenant_ip="10.10.0.2/32", **kwargs):
|
|
super().__init__(*args, **kwargs)
|
|
self.tenant_ip = tenant_ip
|
|
|
|
def _exec(self, argv, *, timeout, input_text=None):
|
|
result = super()._exec(argv, timeout=timeout, input_text=input_text)
|
|
if input_text.startswith("#!/bin/sh\nset -eu"):
|
|
return result
|
|
peer = KEY2 if self.public_key == KEY1 else KEY1
|
|
target = "100.64.1.2" if self.public_key == KEY1 else "100.64.1.1"
|
|
endpoint = "203.0.113.2:51820" if self.public_key == KEY1 else "203.0.113.1:51820"
|
|
marker = f"__VC_FABRIC_PEER__={peer}|{endpoint}|{target}/32"
|
|
output = result.stdout.replace(marker, f"{marker},{self.tenant_ip}")
|
|
output += (
|
|
"__VC_FABRIC_VPC_STATE__=owned\n"
|
|
f"__VC_FABRIC_VPC_TENANT__={peer}|{self.tenant_ip}\n"
|
|
"__VC_FABRIC_VPC_NFT_STATE__=owned\n"
|
|
f"__VC_FABRIC_VPC_NFT_IP__={self.tenant_ip}\n"
|
|
)
|
|
return result.__class__(result.args, result.returncode, output, result.stderr)
|
|
|
|
|
|
def test_fabric_merges_and_reads_panel_owned_vpc_allowed_ips():
|
|
current = plan()
|
|
observed = FabricHostAdapter(TenantReadbackManager(KEY1)).observe(
|
|
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}
|
|
)
|
|
assert observed["config_ok"] is True
|
|
assert observed["observed"]["peer_allowed_exact"] is True
|
|
assert f"{KEY2}|10.10.0.2/32" in observed["observed"]["vpc_tenants"]
|
|
assert all("10001" not in route for route in observed["observed"]["routes"])
|
|
|
|
|
|
def test_fabric_rejects_unbacked_vpc_allowed_ip():
|
|
current = plan()
|
|
manager = FakeManager(KEY1)
|
|
original_exec = manager._exec
|
|
|
|
def unbacked(argv, *, timeout, input_text=None):
|
|
result = original_exec(argv, timeout=timeout, input_text=input_text)
|
|
if input_text.startswith("#!/bin/sh\nset -eu"):
|
|
return result
|
|
peer = KEY2
|
|
marker = f"__VC_FABRIC_PEER__={peer}|203.0.113.2:51820|100.64.1.2/32"
|
|
output = result.stdout.replace(marker, f"{marker},10.10.0.2/32")
|
|
return result.__class__(result.args, result.returncode, output, result.stderr)
|
|
|
|
manager._exec = unbacked
|
|
observed = FabricHostAdapter(manager).observe(
|
|
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}
|
|
)
|
|
assert observed["config_ok"] is False
|
|
assert observed["observed"]["peer_allowed_exact"] is False
|
|
|
|
|
|
def test_fabric_rejects_non_rfc1918_vpc_tenant_prefix():
|
|
current = plan()
|
|
observed = FabricHostAdapter(
|
|
TenantReadbackManager(KEY1, tenant_ip="169.254.1.2/32")
|
|
).observe(current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2})
|
|
assert observed["config_ok"] is False
|
|
assert observed["observed"]["peer_allowed_exact"] is False
|
|
|
|
|
|
def test_fabric_rejects_stale_vpc_nft_tenant_set():
|
|
current = plan()
|
|
manager = TenantReadbackManager(KEY1)
|
|
original_exec = manager._exec
|
|
|
|
def stale_nft(argv, *, timeout, input_text=None):
|
|
result = original_exec(argv, timeout=timeout, input_text=input_text)
|
|
if input_text.startswith("#!/bin/sh\nset -eu"):
|
|
return result
|
|
output = result.stdout.replace(
|
|
"__VC_FABRIC_VPC_NFT_IP__=10.10.0.2/32\n", "")
|
|
return result.__class__(result.args, result.returncode, output, result.stderr)
|
|
|
|
manager._exec = stale_nft
|
|
observed = FabricHostAdapter(manager).observe(
|
|
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}
|
|
)
|
|
assert observed["config_ok"] is False
|
|
assert observed["observed"]["vpc_nft_exact"] is False
|
|
|
|
|
|
def test_fabric_does_not_hide_foreign_custom_route():
|
|
current = plan()
|
|
manager = FakeManager(KEY1)
|
|
original_exec = manager._exec
|
|
|
|
def foreign_route(argv, *, timeout, input_text=None):
|
|
result = original_exec(argv, timeout=timeout, input_text=input_text)
|
|
if input_text.startswith("#!/bin/sh\nset -eu"):
|
|
return result
|
|
output = result.stdout + (
|
|
f"__VC_FABRIC_ROUTE__=10.10.0.2/32|10001|{FABRIC_INTERFACE}\n"
|
|
)
|
|
return result.__class__(result.args, result.returncode, output, result.stderr)
|
|
|
|
manager._exec = foreign_route
|
|
observed = FabricHostAdapter(manager).observe(
|
|
current, current.nodes[0], {"host-a": KEY1, "host-b": KEY2}
|
|
)
|
|
assert observed["config_ok"] is False
|
|
assert "10.10.0.2/32|10001|wg-fabric0" in observed["observed"]["routes"]
|