"""Pure VPC private data-plane contract tests. These tests render the host transaction only; real nft/ip/wg and libvirt acceptance remains a host-matrix test because this checkout has no network namespace or libvirt daemon. """ from __future__ import annotations import os import re import subprocess import tempfile import xml.etree.ElementTree as ET from dataclasses import replace from pathlib import Path import pytest from virsh_panel.shell_contracts import IP_RULE_CONVERGENCE_SH from virsh_panel.vpc import VPC_SCHEMA_VERSION from virsh_panel.vpc_dataplane import ( FIREWALL_MODE_NATIVE, VpcDataplaneError, VpcPrivateHostAdapter, _VPC_SYSCTL_VALIDATION_SH, _allowed, _member_nics, _markers, _routes, parse_vpc_dataplane_plan, patch_private_interface_xml, private_guest_network_generation, private_guest_routes, render_apply_script, render_observe_script, render_remove_script, render_vpc_nft_rules, ) WG_KEY = "AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE=" WG_KEY_2 = "AgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgI=" FABRIC_DIGEST = "ab" * 32 _FAKE_IP_RULE_FUNCTION = r''' mock_ip() { printf '%s\n' "$*" >> "$VC_RULE_LOG" while [ "$#" -gt 0 ]; do case "$1" in -*) shift ;; *) break ;; esac done [ "$1" = rule ] || return 90 shift verb=$1 shift case "$verb" in show) [ "${VC_RULE_SHOW_FAIL:-no}" != yes ] || return 97 wanted= if [ "$#" -gt 0 ]; then [ "$1" = pref ] && [ "$#" = 2 ] || return 91 wanted=$2 fi awk -F'|' -v wanted="$wanted" 'wanted == "" || $1 == wanted { printf "%s: from all to %s", $1, $3 if ($2 != "-") printf " iif %s", $2 printf " lookup %s", $4 if (NF >= 5 && $5 != "-") printf " %s", $5 printf "\n" }' "$VC_RULE_STATE" ;; add) [ "$1" = pref ] || return 92 pref=$2 shift 2 iif=- if [ "$1" = iif ]; then iif=$2; shift 2; fi [ "$1" = to ] || return 93 destination=$2 shift 2 [ "$1" = lookup ] || return 94 table=$2 shift 2 [ "$#" = 0 ] || return 95 [ "${VC_RULE_FAIL_PREF:-}" != "$pref" ] || return 97 printf '%s|%s|%s|%s|-\n' "$pref" "$iif" "$destination" "$table" >> "$VC_RULE_STATE" ;; del) [ "$1" = pref ] || return 92 pref=$2 shift 2 iif=- if [ "$1" = iif ]; then iif=$2; shift 2; fi [ "$1" = to ] || return 93 destination=$2 shift 2 [ "$1" = lookup ] || return 94 table=$2 shift 2 [ "$#" = 0 ] || return 95 awk -F'|' -v pref="$pref" -v iif="$iif" -v destination="$destination" -v table="$table" ' !($1 == pref && $2 == iif && $3 == destination && $4 == table) ' "$VC_RULE_STATE" > "$VC_RULE_STATE.tmp" || return 98 mv "$VC_RULE_STATE.tmp" "$VC_RULE_STATE" || return 98 ;; *) return 96 ;; esac } ''' def _run_ip_rule_contract(initial_state: str, commands: str, **extra_env: str): with tempfile.TemporaryDirectory(prefix="virt-cloud-ip-rule-") as directory: root = Path(directory) paths = { "state": root / "state", "log": root / "log", "before": root / "rules.before", "attempted": root / "rules.attempted", "created": root / "rules.created", "rollback": root / "rollback", } for name, path in paths.items(): path.write_text(initial_state if name == "state" else "") script = "\n".join(( _FAKE_IP_RULE_FUNCTION, IP_RULE_CONVERGENCE_SH, "IP=mock_ip", commands, )) env = dict(os.environ) env.update({ "VC_RULE_STATE": str(paths["state"]), "VC_RULE_LOG": str(paths["log"]), "VC_BEFORE": str(paths["before"]), "VC_ATTEMPTED": str(paths["attempted"]), "VC_CREATED": str(paths["created"]), "VC_ROLLBACK": str(paths["rollback"]), "VC_TXN": str(root), }) env.update(extra_env) result = subprocess.run( ["/bin/sh", "-eu", "-c", script], env=env, capture_output=True, text=True, check=False, ) contents = {name: path.read_text().splitlines() for name, path in paths.items()} return result, contents def _shell_block(script: str, start: str, end: str) -> str: """Return one rendered production shell block between stable markers.""" start_at = script.index(start) end_at = script.index(end, start_at) return script[start_at:end_at] def _run_sysctl_artifact_contract( path: Path, *, stat_result: str = "0:600", ) -> subprocess.CompletedProcess[str]: script = "\n".join(( _VPC_SYSCTL_VALIDATION_SH, "vc_fake_stat() { printf '%s\\n' \"$VC_FAKE_STAT\"; }", "STAT=vc_fake_stat", "vc_vpc_sysctl_artifact_owned \"$VC_SYSCTL_FILE\" " "vpc-hk-dp 1 '" + ("ab" * 32) + "' vcv-0123456789a", )) env = dict(os.environ) env.update({ "VC_FAKE_STAT": stat_result, "VC_SYSCTL_FILE": str(path), }) return subprocess.run( ["/bin/sh", "-eu", "-c", script], env=env, capture_output=True, text=True, check=False, ) def _run_sysctl_artifacts_equal( live: Path, desired: Path, *, stat_result: str = "0:600", ) -> subprocess.CompletedProcess[str]: script = "\n".join(( _VPC_SYSCTL_VALIDATION_SH, "vc_fake_stat() { printf '%s\\n' \"$VC_FAKE_STAT\"; }", "STAT=vc_fake_stat", "CMP=$(command -v cmp)", "vc_vpc_sysctl_artifacts_equal \"$VC_SYSCTL_LIVE\" " "\"$VC_SYSCTL_DESIRED\" vpc-hk-dp 1 '" + ("ab" * 32) + "' vcv-0123456789a", )) env = dict(os.environ) env.update({ "VC_FAKE_STAT": stat_result, "VC_SYSCTL_LIVE": str(live), "VC_SYSCTL_DESIRED": str(desired), }) return subprocess.run( ["/bin/sh", "-eu", "-c", script], env=env, capture_output=True, text=True, check=False, ) def _plan(): return parse_vpc_dataplane_plan({ "schema_version": VPC_SCHEMA_VERSION, "vpc_id": "vpc-hk-dp", "region": "hk", "fabric_id": "fabric-hk", "fabric_revision": 3, "cidr": "10.42.0.0/24", "desired_revision": 1, "operation_id": "vpc-op-dp-0001", "subnets": [{"subnet_id": "subnet-a", "cidr": "10.42.0.0/25"}], "hosts": [ {"host_id": "host-a", "members": [{ "member_id": "member-a", "instance_uuid": "11111111-1111-4111-8111-111111111111", "private_ip_cidr": "10.42.0.10/32", "subnet_id": "subnet-a", }]}, {"host_id": "host-b", "members": [{ "member_id": "member-b", "instance_uuid": "22222222-2222-4222-8222-222222222222", "private_ip_cidr": "10.42.0.11/32", "subnet_id": "subnet-a", }]}, ], "routes": [{ "route_id": "service-a", "destination_cidr": "10.42.0.100/32", "next_hop_member_id": "member-b", "metric": 20, }], "acls": [{ "acl_id": "allow-https", "direction": "ingress", "action": "allow", "protocol": "tcp", "source_cidr": "10.42.0.0/24", "destination_cidr": "10.42.0.10/32", "port_from": 443, "port_to": 443, "priority": 10, }], "fabric_peers": [{ "host_id": "host-b", "public_key": WG_KEY, "overlay_ip": "100.64.0.2/32", "tenant_allowed_ips": ["10.42.0.11/32", "10.42.0.100/32"], }], }, "host-a", fabric_plan_digest=FABRIC_DIGEST) def _guest_network_evidence(member, routes): return { "state": "confirmed", "verified": True, "source": "qemu_guest_agent", "private_mac": member.private_mac, "private_ip_cidr": f"{member.private_ip}/32", "link_state": "up", "routes": sorted(routes), "reason_code": None, "observed_at": 1_700_000_000, } def test_fabric_plan_digest_is_engine_only_and_strictly_typed(): plan = _plan() assert plan.fabric_plan_digest == FABRIC_DIGEST payload = plan.vpc.canonical() payload["fabric_peers"] = [{ "host_id": "host-b", "public_key": WG_KEY, "overlay_ip": "100.64.0.2/32", "tenant_allowed_ips": ["10.42.0.11/32", "10.42.0.100/32"], }] for invalid in (None, "", "A" * 64, "0" * 63, "0" * 65): with pytest.raises(VpcDataplaneError) as rejected: parse_vpc_dataplane_plan( payload, "host-a", fabric_plan_digest=invalid) assert rejected.value.error_code == "vpc_fabric_plan_digest_invalid" # A digest embedded by the browser/tenant is an unknown VPC field. Only # the separately typed engine argument can establish Fabric ownership. spoofed = dict(payload, fabric_plan_digest="0" * 64) with pytest.raises(VpcDataplaneError): parse_vpc_dataplane_plan( spoofed, "host-a", fabric_plan_digest=FABRIC_DIGEST) def test_rendered_data_plane_has_exact_routes_proxy_arp_and_atomic_nft_batch(): plan = _plan() apply = render_apply_script(plan) nft = render_vpc_nft_rules(plan) assert "10.42.0.11/32|wg-fabric0" in apply assert "10.42.0.100/32|wg-fabric0" in apply assert 'route replace "$destination" table "$table" via "$via" dev "$device" onlink' in apply assert 'neigh replace proxy "$proxy_ip"' in apply assert "table arp" in nft assert "arp-proxy-output" in nft assert 'proxy_arp = 0' in apply assert 'proxy_arp_pvlan = 1' in apply assert 'NFT_BATCH="$TXN/apply.nft"' in apply assert '"$NFT" -f "$NFT_BATCH"' in apply scope_call = ( f'vc_ip_rule_scope_check {plan.vpc.cidr} "$ROUTE_TABLE"') assert apply.index(scope_call) < apply.index( "trap rollback EXIT HUP INT TERM") assert '"$NFT" delete table inet "$TABLE"' not in apply assert "anti-spoof-public-to-bridge" in nft assert "anti-spoof-public-to-fabric" in nft assert "anti-spoof-bridge-to-foreign" in nft assert "anti-spoof-fabric-to-foreign" in nft assert 'oifname "wg-fabric0" ip daddr @vpc_dest_ips drop' in nft assert ( "arp operation request arp daddr ip != @local_dest_ips drop" in nft ) assert "bridge-unknown-arp-target-drop" in nft @pytest.mark.parametrize("mode", ("networkmanager_nmcli", "ifupdown")) def test_rp_filter_is_set_before_first_up_and_reasserted_after_manager(mode): script = render_apply_script( _plan(), deployment_mode=mode, firewall_mode="native_nftables") guard = script.index( "# Install and exactly read back the raw tenant guard") calls = [match.start() for match in re.finditer( r"^[ ]*vc_vpc_rp_filter_set \|\| exit 72$", script, re.MULTILINE)] assert len(calls) == 3 first_up = script.index('"$IP" link set dev "$BRIDGE" up', guard) assert guard < calls[0] < first_up if mode == "networkmanager_nmcli": activation = script.index( '"$NMCLI" connection up uuid "$NM_PROFILE_UUID"', first_up) else: activation = script.index( 'VC_FABRIC_APPLYING=1 "$IFUP" --force "$BRIDGE"', first_up) reassert = script.index("vc_vpc_rp_filter_set || exit 72", activation) assert first_up < activation < reassert def test_rp_filter_failure_and_rollback_remain_fail_closed(): script = render_apply_script( _plan(), firewall_mode="native_nftables") helper = _shell_block( script, "vc_vpc_rp_filter_set() {", "vc_vpc_native_ifupdown_rollback_reopen() {") assert helper.index("RP_FILTER_CHANGED=yes") < helper.index( "printf '%s\\n' 2 > \"$vc_rpf_path\" || return 1") assert 'chmod 600 "$TXN/rp_filter.before" || return 1' in helper guard = script.index( "# Install and exactly read back the raw tenant guard") first_set = script.index("vc_vpc_rp_filter_set || exit 72", guard) first_up = script.index('"$IP" link set dev "$BRIDGE" up', guard) assert first_set < first_up rollback = _shell_block(script, "rollback() {", "trap rollback EXIT HUP INT TERM") quarantine = rollback.index('"$IP" link set dev "$BRIDGE" down') restore_raw = rollback.index( 'cat "$TXN/rp_filter.before" > ' '"/proc/sys/net/ipv4/conf/$BRIDGE/rp_filter"') restore_nft = rollback.index('NFT_ROLLBACK="$TXN/rollback.nft"') assert quarantine < restore_raw < restore_nft def test_sysctl_artifact_contract_binds_fixed_shape_and_identity(tmp_path): artifact = tmp_path / "vpc.conf" valid = "\n".join(( "# virt-cloud-vpc-dataplane-v1", f"# vpc=vpc-hk-dp revision=1 digest={FABRIC_DIGEST}", "net.ipv4.conf.vcv-0123456789a.rp_filter = 2", "net.ipv4.conf.vcv-0123456789a.proxy_arp = 0", "net.ipv4.conf.vcv-0123456789a.proxy_arp_pvlan = 1", "", )) artifact.write_text(valid, encoding="utf-8") assert _run_sysctl_artifact_contract(artifact).returncode == 0 for invalid in ( valid.replace("revision=1", "revision=2"), valid.replace("rp_filter = 2", "rp_filter = 1"), valid + "net.ipv4.conf.all.rp_filter = 0\n", ): artifact.write_text(invalid, encoding="utf-8") assert _run_sysctl_artifact_contract(artifact).returncode != 0 artifact.write_text(valid, encoding="utf-8") assert _run_sysctl_artifact_contract( artifact, stat_result="0:644").returncode != 0 assert _run_sysctl_artifact_contract( artifact, stat_result="1:600").returncode != 0 symlink = tmp_path / "vpc-link.conf" symlink.symlink_to(artifact) assert _run_sysctl_artifact_contract(symlink).returncode != 0 desired = tmp_path / "vpc-desired.conf" live = tmp_path / "vpc-live.conf" desired.write_text(valid, encoding="utf-8") live.write_text(valid, encoding="utf-8") assert _run_sysctl_artifacts_equal(live, desired).returncode == 0 # Rollback must refuse a same-name file edited after this transaction # installed its desired artifact, even when the owner line is retained. live.write_text(valid.replace("proxy_arp = 0", "proxy_arp = 1"), encoding="utf-8") assert _run_sysctl_artifacts_equal(live, desired).returncode != 0 live.write_text(valid, encoding="utf-8") desired.write_text(valid.replace("revision=1", "revision=2"), encoding="utf-8") assert _run_sysctl_artifacts_equal(live, desired).returncode != 0 def test_sysctl_artifact_requires_state_and_is_shared_by_all_lifecycle_paths(): plan = _plan() apply = render_apply_script(plan) observe = render_observe_script(plan) empty_host = replace( plan, members=tuple(member for member in plan.members if member.host_id != plan.host_id), ) remove = render_remove_script(empty_host) existing_artifact = apply.index( 'if [ -e "$SYSCTL_FILE" ] || [ -L "$SYSCTL_FILE" ]; then') state_required = apply.index( '[ -f "$STATE_FILE" ] || exit 71', existing_artifact) ownership = apply.index( 'vc_vpc_sysctl_artifact_owned "$SYSCTL_FILE"', state_required) assert existing_artifact < state_required < ownership assert re.search( r'vc_vpc_sysctl_artifact_owned "\$SYSCTL_FILE" ' + re.escape(plan.vpc.vpc_id) + r'\s+' + str(plan.desired_revision) + r'\s+' + re.escape(plan.desired_digest) + r' "\$BRIDGE"', observe, ) assert remove.count( 'vc_vpc_sysctl_artifact_owned "$SYSCTL_FILE"') == 2 assert remove.count('[ -f "$STATE_FILE" ] || exit 71') >= 2 def test_sysctl_apply_precommit_and_rollback_are_transaction_bound(): apply = render_apply_script(_plan()) snapshot = apply.index( 'cp -p "$SYSCTL_FILE" "$TXN/sysctl.before"') desired = apply.index('cat > "$TXN/sysctl.desired"', snapshot) desired_owned = apply.index( 'vc_vpc_sysctl_artifact_owned "$TXN/sysctl.desired"', desired) precommit = apply.index( 'if [ -f "$TXN/sysctl.before" ]; then', desired_owned) previous_owned = apply.index( 'vc_vpc_sysctl_artifact_owned "$SYSCTL_FILE"', precommit) previous_bytes = apply.index( '"$CMP" -s "$SYSCTL_FILE" "$TXN/sysctl.before"', previous_owned) absence = apply.index( '[ ! -e "$SYSCTL_FILE" ] && [ ! -L "$SYSCTL_FILE" ]', previous_bytes, ) install = apply.index('mv -f "$SYSCTL_TMP" "$SYSCTL_FILE"', absence) desired_readback = apply.index( 'vc_vpc_sysctl_artifacts_equal "$SYSCTL_FILE" ' '"$TXN/sysctl.desired"', install, ) assert snapshot < desired < desired_owned < precommit assert precommit < previous_owned < previous_bytes < absence < install assert install < desired_readback rollback = _shell_block( apply, "rollback() {", "trap rollback EXIT HUP INT TERM") rollback_proof = rollback.index( 'vc_vpc_sysctl_artifacts_equal "$SYSCTL_FILE"') restore_previous = rollback.index( 'cp -p "$TXN/sysctl.before" "$SYSCTL_FILE"', rollback_proof) delete_new = rollback.index('rm -f "$SYSCTL_FILE"', restore_previous) assert rollback_proof < restore_previous < delete_new def test_rendered_scripts_are_posix_shell(): plan = _plan() for script in (render_apply_script(plan), render_observe_script(plan)): result = subprocess.run( ["/bin/sh", "-n"], input=script, text=True, capture_output=True, check=False, ) assert result.returncode == 0, result.stderr def test_generated_ip_rule_verbs_are_supported_by_iproute2(): from virsh_panel.fabric import _BOOT_HELPER plan = _plan() empty_host = replace( plan, members=tuple(member for member in plan.members if member.host_id != plan.host_id), ) scripts = ( render_apply_script(plan), render_observe_script(plan), render_remove_script(empty_host), _BOOT_HELPER, ) command = re.compile( r'"?\$IP"?\s+(?:(?:-[A-Za-z0-9]+)\s+)*rule\s+([a-z]+)') for script in scripts: verbs = command.findall(script) assert verbs assert set(verbs) <= {"show", "add", "del"} assert "rule replace" not in script def test_ip_rule_convergence_adds_only_missing_rules_and_replays_without_writes(): commands = r''' vc_ip_rule_preflight 20000 - 10.42.0.0/24 10000 71 72 "$VC_BEFORE" || exit $? vc_ip_rule_preflight 20001 wg-fabric0 10.42.0.0/24 10000 71 72 "$VC_BEFORE" || exit $? vc_ip_rule_ensure 20000 - 10.42.0.0/24 10000 71 72 "$VC_ATTEMPTED" "$VC_CREATED" || exit $? vc_ip_rule_ensure 20001 wg-fabric0 10.42.0.0/24 10000 71 72 "$VC_ATTEMPTED" "$VC_CREATED" || exit $? vc_ip_rule_ensure 20000 - 10.42.0.0/24 10000 71 72 "$VC_ATTEMPTED" "$VC_CREATED" || exit $? vc_ip_rule_ensure 20001 wg-fabric0 10.42.0.0/24 10000 71 72 "$VC_ATTEMPTED" "$VC_CREATED" || exit $? ''' result, contents = _run_ip_rule_contract("", commands) assert result.returncode == 0, result.stderr assert contents["state"] == [ "20000|-|10.42.0.0/24|10000|-", "20001|wg-fabric0|10.42.0.0/24|10000|-", ] assert contents["attempted"] == [ "20000|-|10.42.0.0/24|10000", "20001|wg-fabric0|10.42.0.0/24|10000", ] assert contents["created"] == contents["attempted"] assert sum(" rule add " in f" {line} " for line in contents["log"]) == 2 assert all("rule replace" not in line for line in contents["log"]) existing = "\n".join(contents["state"]) + "\n" result, replay = _run_ip_rule_contract(existing, commands) assert result.returncode == 0, result.stderr assert replay["state"] == contents["state"] assert replay["attempted"] == [] assert replay["created"] == [] assert replay["before"] == [ "20000|-|10.42.0.0/24|10000", "20001|wg-fabric0|10.42.0.0/24|10000", ] assert all(" rule add " not in f" {line} " for line in replay["log"]) def test_ip_rule_preflight_rejects_conflicts_duplicates_and_extra_selectors(): command = ( 'vc_ip_rule_preflight 20000 - 10.42.0.0/24 10000 ' '71 72 "$VC_BEFORE" || exit $?' ) invalid_states = ( "20000|-|10.99.0.0/24|10000|-\n", "20000|-|10.42.0.0/24|10000|-\n" "20000|-|10.42.0.0/24|10000|-\n", "20000|-|10.42.0.0/24|10000|fwmark 0x1\n", ) for initial in invalid_states: result, contents = _run_ip_rule_contract(initial, command) assert result.returncode == 71 assert contents["state"] == initial.splitlines() assert contents["before"] == [] assert all(" rule add " not in f" {line} " for line in contents["log"]) def test_ip_rule_scope_rejects_foreign_rules_and_can_require_empty_scope(): managed = ( "vc_ip_rule_scope_check 10.42.0.0/24 10000 20000 " "wg-fabric0 managed 71 72 || exit $?" ) exact = ( "20000|-|10.42.0.0/24|10000|-\n" "20001|wg-fabric0|10.42.0.0/24|10000|-\n" ) result, contents = _run_ip_rule_contract(exact, managed) assert result.returncode == 0, result.stderr assert contents["state"] == exact.splitlines() foreign_states = ( exact + "19999|-|10.42.0.0/24|10001|-\n", exact + "20100|-|10.99.0.0/24|10000|-\n", exact + "20000|-|10.42.0.0/24|10000|fwmark 0x1\n", ) for initial in foreign_states: result, contents = _run_ip_rule_contract(initial, managed) assert result.returncode == 71 assert contents["state"] == initial.splitlines() assert all(" rule add " not in f" {line} " for line in contents["log"]) unrelated = "19999|-|10.99.0.0/24|10001|-\n" result, _contents = _run_ip_rule_contract(unrelated, managed) assert result.returncode == 0, result.stderr empty = ( "vc_ip_rule_scope_check 10.42.0.0/24 10000 20000 " "wg-fabric0 empty 71 72 || exit $?" ) result, _contents = _run_ip_rule_contract(exact, empty) assert result.returncode == 71 result, contents = _run_ip_rule_contract(unrelated, empty) assert result.returncode == 0, result.stderr assert contents["state"] == unrelated.splitlines() result, contents = _run_ip_rule_contract( exact, managed, VC_RULE_SHOW_FAIL="yes") assert result.returncode == 72 assert contents["state"] == exact.splitlines() def test_ip_rule_failed_add_is_attempted_but_not_claimed_as_created(): commands = r''' vc_ip_rule_preflight 20000 - 10.42.0.0/24 10000 71 72 "$VC_BEFORE" || exit $? vc_ip_rule_preflight 20001 wg-fabric0 10.42.0.0/24 10000 71 72 "$VC_BEFORE" || exit $? vc_ip_rule_ensure 20000 - 10.42.0.0/24 10000 71 72 "$VC_ATTEMPTED" "$VC_CREATED" || exit $? vc_ip_rule_ensure 20001 wg-fabric0 10.42.0.0/24 10000 71 72 "$VC_ATTEMPTED" "$VC_CREATED" || exit $? ''' result, contents = _run_ip_rule_contract( "", commands, VC_RULE_FAIL_PREF="20001") assert result.returncode == 72 assert contents["state"] == ["20000|-|10.42.0.0/24|10000|-"] assert contents["attempted"] == [ "20000|-|10.42.0.0/24|10000", "20001|wg-fabric0|10.42.0.0/24|10000", ] assert contents["created"] == ["20000|-|10.42.0.0/24|10000"] apply = render_apply_script(_plan()) assert 'done < "$TXN/rules.attempted"' in apply assert 'grep -Fqx "$rule_pref|$rule_iif|$rule_to|$rule_table" "$TXN/rules.created"' in apply def test_production_ip_rule_rollback_removes_only_rules_created_by_failed_apply(): plan = _plan() apply = render_apply_script(plan) apply_rules = "\n".join(( _shell_block( apply, "# vc-policy-rule-preflight-begin", "# vc-policy-rule-preflight-end", ), _shell_block( apply, "# vc-policy-rule-ensure-begin", "# vc-policy-rule-ensure-end", ), )) rollback_rules = _shell_block( apply, "# vc-policy-rule-rollback-begin", "# vc-policy-rule-rollback-end", ) priority = plan.resources.rule_priority table = plan.resources.route_table cidr = str(plan.vpc.cidr) commands = f'''\ TXN="$VC_TXN" RULE_PRIORITY={priority} WG_RULE_PRIORITY=$((RULE_PRIORITY + 1)) WG_IFACE=wg-fabric0 ROUTE_TABLE={table} ROLLBACK_FAILED=no production_rule_rollback() {{ {rollback_rules} }} production_rollback_on_exit() {{ status=$? trap - EXIT set +e production_rule_rollback printf '%s\\n' "$ROLLBACK_FAILED" > "$VC_ROLLBACK" exit "$status" }} trap production_rollback_on_exit EXIT {apply_rules} trap - EXIT ''' failed_priority = str(priority + 1) result, fresh = _run_ip_rule_contract( "", commands, VC_RULE_FAIL_PREF=failed_priority) assert result.returncode == 73, result.stderr assert fresh["state"] == [] assert fresh["attempted"] == [ f"{priority}|-|{cidr}|{table}", f"{priority + 1}|wg-fabric0|{cidr}|{table}", ] assert fresh["created"] == [f"{priority}|-|{cidr}|{table}"] assert fresh["rollback"] == ["no"] assert any( f" rule del pref {priority} to {cidr} lookup {table}" in f" {line}" for line in fresh["log"] ) assert all( f" rule del pref {priority + 1} " not in f" {line} " for line in fresh["log"] ) preexisting = f"{priority}|-|{cidr}|{table}|-\n" result, replay = _run_ip_rule_contract( preexisting, commands, VC_RULE_FAIL_PREF=failed_priority) assert result.returncode == 73, result.stderr assert replay["state"] == preexisting.splitlines() assert replay["before"] == [f"{priority}|-|{cidr}|{table}"] assert replay["attempted"] == [ f"{priority + 1}|wg-fabric0|{cidr}|{table}", ] assert replay["created"] == [] assert replay["rollback"] == ["no"] assert all(" rule del " not in f" {line} " for line in replay["log"]) def test_remove_script_checks_ownership_before_destructive_cleanup(): plan = _plan() empty_host = replace( plan, members=tuple(member for member in plan.members if member.host_id != plan.host_id), ) for mode in ("networkmanager_nmcli", "ifupdown"): script = render_remove_script(empty_host, deployment_mode=mode) result = subprocess.run( ["/bin/sh", "-n"], input=script, text=True, capture_output=True, check=False, ) assert result.returncode == 0, result.stderr assert 'connection.stable-id' in script assert 'ipv4.addresses ipv4.routes ipv4.routing-rules' in script assert 'vc_nm_profile_base_validate || exit 71' in script assert "virt-cloud-vpc-dataplane-v1 vpc=" in script assert 'nft_table_count' in script assert '[ "$name" = "$TABLE" ]' in script assert 'BRIDGE_PRESENT=no' in script assert 'if "$IP" link show dev "$BRIDGE"' in script assert 'bridge_routes6' in script assert 'proxy_left' in script for phase in ( "routes_removed", "manager_removed", "bridge_removed", "nft_removed", "files_removed", "wg_merged", "committed", ): assert f"set_phase {phase}" in script assert '[ "$RESUME" = yes ]' in script assert 'committed)' in script assert 'files_removed|wg_merged)' in script assert script.index("exec 8>>") < script.index('exec 9>>"$LOCK"') assert '"$IP" -o -4 -N rule show pref' in script assert '"$IP" -o -4 -N route show table all 2>/dev/null' in script assert 'verify_routes=$(vc_vpc_route_table_read "$ROUTE_TABLE")' in script assert 'orphan_routes=$(vc_vpc_route_table_read "$ROUTE_TABLE")' in script assert 'existing=$(vc_vpc_route_table_read "$table" "$destination")' in script assert 'route show table "$ROUTE_TABLE" 2>/dev/null' not in script assert 'route show table "$table" "$destination" 2>/dev/null' not in script assert "vc_ip_rule_scope_check" in script assert "managed 71 72" in script assert "empty 1 1" in script assert "empty 71 72" in script identity_check = 'vc_vpc_artifact_identity_check "$nft_identity"' assert script.index(identity_check) < script.index( 'mkdir -m 700 "$TXN"') assert script.index(identity_check) < script.index( '"$IP" route del "$destination"') atomic_nft_delete = '"$NFT" -f "$NFT_REMOVE_BATCH"' assert script.index(identity_check) < script.index(atomic_nft_delete) assert script.count(atomic_nft_delete) == 1 assert '"$NFT" delete table "$family" "$name"' not in script def test_remove_route_inventory_accepts_absent_custom_table_and_fails_closed( tmp_path): plan = _plan() delete_plan = replace( plan, members=tuple(member for member in plan.members if member.host_id != plan.host_id), ) script = render_remove_script(delete_plan) function = re.search( r"(?ms)^vc_vpc_route_table_read\(\) \{.*?^\}\n", script, ) assert function is not None fake_ip = tmp_path / "ip" fake_ip.write_text(r'''#!/bin/sh [ "$*" = "-o -4 -N route show table all" ] || exit 96 [ "${VC_ROUTE_STATUS:-0}" = 0 ] || exit "$VC_ROUTE_STATUS" printf '%s\n' "${VC_ROUTE_INVENTORY:-}" ''', encoding="utf-8") fake_ip.chmod(0o700) base_env = os.environ.copy() def run_contract(inventory="", status="0", destination=""): contract = function.group(0) + r''' IP=$1 ROUTE_TABLE=12102 rows=$(vc_vpc_route_table_read "$ROUTE_TABLE" "$2") || exit 81 printf '%s\n' "$rows" ''' return subprocess.run( ["/bin/sh", "-s", str(fake_ip), destination], input=contract, text=True, capture_output=True, check=False, env=base_env | { "VC_ROUTE_INVENTORY": inventory, "VC_ROUTE_STATUS": status, }, ) absent = run_contract( "default via 192.0.2.1 dev eth0 proto 16\n" "local 192.0.2.10 dev eth0 table 255 proto 2 scope host" ) assert absent.returncode == 0, absent.stderr assert absent.stdout == "\n" expected = "10.0.0.2 dev vcv-test proto 186 table 12102 metric 100" present = run_contract( expected + "\n" "10.0.0.3 dev vcv-test proto 186 table 12103 metric 100" ) assert present.returncode == 0, present.stderr assert present.stdout == expected + "\n" selected = run_contract( expected + "\n" "10.0.0.3 dev vcv-test proto 186 table 12102 metric 100", destination="10.0.0.3/32", ) assert selected.returncode == 0, selected.stderr assert selected.stdout.startswith("10.0.0.3 ") assert "10.0.0.2" not in selected.stdout typed = run_contract( "6 10.0.0.4 table 12102 proto 186", destination="10.0.0.4/32", ) assert typed.returncode == 0, typed.stderr assert typed.stdout == "6 10.0.0.4 table 12102 proto 186\n" failed = run_contract(status="2") assert failed.returncode == 81, failed.stderr for malformed in ( "10.0.0.2 dev vcv-test table", "10.0.0.2 dev vcv-test table custom", "10.0.0.2 dev vcv-test table 12102 table 12102", ): result = run_contract(malformed) assert result.returncode == 81, (malformed, result.stderr) def test_remove_artifact_identity_allows_older_revision_only(): plan = _plan() delete_plan = replace( plan, vpc=replace( plan.vpc, desired_revision=2, desired_digest="b" * 64, ), members=tuple(member for member in plan.members if member.host_id != plan.host_id), ) script = render_remove_script(delete_plan) function = re.search( r"(?ms)^vc_vpc_artifact_identity_check\(\) \{.*?^\}\n", script, ) assert function is not None contract = function.group(0) + r''' vc_vpc_artifact_identity_check "vpc=vpc-hk-dp revision=1 digest=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" || exit 81 vc_vpc_artifact_identity_check "vpc=vpc-hk-dp revision=2 digest=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" || exit 82 if vc_vpc_artifact_identity_check "vpc=vpc-hk-dp revision=2 digest=cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"; then exit 83; fi if vc_vpc_artifact_identity_check "vpc=vpc-hk-dp revision=3 digest=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; then exit 84; fi if vc_vpc_artifact_identity_check "vpc=foreign revision=1 digest=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; then exit 85; fi exit 0 ''' result = subprocess.run( ["/bin/sh"], input=contract, text=True, capture_output=True, check=False, ) assert result.returncode == 0, result.stderr @pytest.mark.parametrize("shell", ("/bin/sh", "/bin/bash")) @pytest.mark.parametrize("phase", ("preflight", "before_unlink")) def test_remove_ifupdown_fragment_accepts_owned_apply_output( tmp_path, shell, phase): plan = _plan() applied_plan = replace( plan, vpc=replace(plan.vpc, desired_revision=2, desired_digest="a" * 64)) delete_plan = replace( plan, vpc=replace(plan.vpc, desired_revision=3, desired_digest="b" * 64), members=tuple(member for member in plan.members if member.host_id != plan.host_id), ) apply = render_apply_script(applied_plan, deployment_mode="ifupdown") fragment_match = re.search( r"cat > \"\$IF_TMP\" <<'([^']+)'\n(.*?)\n\1\n", apply, re.DOTALL) assert fragment_match is not None fragment_text = fragment_match.group(2) + "\n" script = render_remove_script(delete_plan, deployment_mode="ifupdown") identity = _shell_block( script, "vc_vpc_artifact_identity_check() {", "vc_remove_capture_nft_precommit() {") if phase == "preflight": check = _shell_block( script, 'if [ -f "$IF_FRAGMENT" ]; then\n', '\nelse\n for artifact in "$NFT_FILE"') else: check = _shell_block( script, 'if [ -e "$IF_FRAGMENT" ] || [ -L "$IF_FRAGMENT" ]; then\n', 'if [ -f "$STATE_FILE" ]; then\n') fragment = tmp_path / "ifupdown-fragment" contract = "\n".join(( identity, "vc_fake_stat() { printf '%s\\n' '0:600'; }", "STAT=vc_fake_stat", check, "printf '%s\\n' fragment_owned", )) env = os.environ | {"IF_FRAGMENT": str(fragment)} def check_fragment(content): fragment.write_text(content, encoding="utf-8") return subprocess.run( [shell, "-eu", "-c", contract], env=env, capture_output=True, text=True, check=False) # A detach revision removes the preceding apply revision's real fragment. result = check_fragment(fragment_text) assert result.returncode == 0, result.stderr assert result.stdout == "fragment_owned\n" current = fragment_text.replace("revision=2", "revision=3").replace( "a" * 64, "b" * 64) result = check_fragment(current) assert result.returncode == 0, result.stderr for invalid in ( fragment_text.replace("virt-cloud-vpc-dataplane-v1", "foreign-owner"), fragment_text.replace("vpc=vpc-hk-dp", "vpc=foreign"), fragment_text.replace("revision=2", "revision=4"), fragment_text.replace("revision=2", "revision=0"), fragment_text.replace(" revision=2", ""), fragment_text.replace("a" * 64, "invalid-digest"), fragment_text.replace(" digest=" + "a" * 64, ""), current.replace("b" * 64, "c" * 64), fragment_text.replace("digest=" + "a" * 64, "digest=" + "a" * 64 + " extra=field"), fragment_text.replace("# virt-cloud-vpc-dataplane-v1 ", ""), ): result = check_fragment(invalid) assert result.returncode == 71, (invalid.splitlines()[0], result.stderr) assert result.stdout == "" def test_proxy_markers_are_bounded_and_typed(): parsed = _markers( "__VC_VPC_PROXY_ARP__=0\n" "__VC_VPC_PROXY_ARP_PVLAN__=1\n" "__VC_VPC_PROXY_STATE__=owned\n" "__VC_VPC_LOCAL_NEIGHBOR_STATE__=owned\n" "__VC_VPC_PROXY_IP__=10.42.0.11\n" ) assert parsed["proxy_arp"] == "0" assert parsed["proxy_arp_pvlan"] == "1" assert parsed["proxy_state"] == "owned" assert parsed["proxy_ips"] == ["10.42.0.11"] def test_peer_allowed_ips_are_closed_over_vpc_or_trusted_fabric_set(): plan = _plan() payload = plan.vpc.canonical() payload["fabric_peers"] = [{ "host_id": "host-b", "public_key": WG_KEY, "overlay_ip": "100.64.0.2/32", "tenant_allowed_ips": ["10.42.0.11/32", "10.42.0.100/32", "10.42.0.200/32"], }] try: parse_vpc_dataplane_plan( payload, "host-a", fabric_plan_digest=FABRIC_DIGEST) except Exception as exc: assert getattr(exc, "error_code", None) == "vpc_tenant_allowed_ip_unowned" else: raise AssertionError("unowned peer AllowedIP was accepted") parsed = parse_vpc_dataplane_plan( payload, "host-a", fabric_plan_digest=FABRIC_DIGEST, trusted_tenant_allowed_ips={ "host-b": ["10.42.0.11/32", "10.42.0.100/32", "10.42.0.200/32"], }) assert parsed.peers[0].tenant_allowed_ips[-1] == "10.42.0.200/32" def test_observe_can_require_the_complete_authoritative_fabric_peer_set(): plan = _plan() strict = parse_vpc_dataplane_plan( plan.vpc.canonical(), "host-a", fabric_plan_digest=FABRIC_DIGEST, fabric_peers=[{ "host_id": "host-b", "public_key": WG_KEY, "overlay_ip": "100.64.0.2/32", "tenant_allowed_ips": ["10.42.0.11/32", "10.42.0.100/32"], }], trusted_tenant_allowed_ips={ "host-b": ["10.42.0.11/32", "10.42.0.100/32"], }, fabric_peer_keys=[WG_KEY, WG_KEY_2], fabric_peer_overlays={ WG_KEY: "100.64.0.2/32", WG_KEY_2: "100.64.0.3/32", }, ) assert strict.fabric_peer_keys == frozenset({WG_KEY, WG_KEY_2}) assert strict.fabric_peer_overlays == frozenset({ (WG_KEY, "100.64.0.2/32"), (WG_KEY_2, "100.64.0.3/32"), }) observe = render_observe_script(strict) assert "__VC_VPC_WG_PEER_SET__" in observe assert "live_fabric_peer_keys" in observe with pytest.raises(VpcDataplaneError) as overlay_error: parse_vpc_dataplane_plan( plan.vpc.canonical(), "host-a", fabric_plan_digest=FABRIC_DIGEST, fabric_peers=[{ "host_id": "host-b", "public_key": WG_KEY, "overlay_ip": "100.64.0.2/32", "tenant_allowed_ips": ["10.42.0.11/32", "10.42.0.100/32"], }], trusted_tenant_allowed_ips={ "host-b": ["10.42.0.11/32", "10.42.0.100/32"], }, fabric_peer_keys=[WG_KEY, WG_KEY_2], fabric_peer_overlays={ WG_KEY: "100.64.0.9/32", WG_KEY_2: "100.64.0.3/32", }, ) assert overlay_error.value.error_code == "vpc_fabric_peer_set_invalid" try: parse_vpc_dataplane_plan( plan.vpc.canonical(), "host-a", fabric_plan_digest=FABRIC_DIGEST, fabric_peers=[{ "host_id": "host-b", "public_key": WG_KEY, "overlay_ip": "100.64.0.2/32", "tenant_allowed_ips": ["10.42.0.11/32", "10.42.0.100/32"], }], trusted_tenant_allowed_ips={ "host-b": ["10.42.0.11/32", "10.42.0.100/32"], }, fabric_peer_keys=[WG_KEY_2], ) except Exception as exc: assert getattr(exc, "error_code", None) == "vpc_fabric_peer_set_invalid" else: raise AssertionError("VPC peer outside authoritative Fabric set was accepted") def test_private_xml_patch_migrates_pre_fix_aliases_and_leaves_private_filter_free(): plan = _plan() member = plan.members[0] xml = """ """ patched = patch_private_interface_xml( xml, member=member, resources=plan.resources, guest_network_generation=private_guest_network_generation( member, private_guest_routes(plan.vpc, member)), vpc_id=plan.vpc.vpc_id, desired_revision=plan.desired_revision, desired_digest=plan.desired_digest, ) assert 'alias name="ua-public0"' in patched assert 'alias name="ua-private0"' in patched assert 'alias name="public0"' not in patched assert 'alias name="private0"' not in patched assert f'bridge="{plan.resources.bridge}"' in patched assert 'filter="clean-traffic"' not in patched assert f'private-network-digest="{plan.desired_digest}"' in patched def test_private_xml_patch_promotes_legacy_public_aliases_idempotently(): plan = _plan() member = plan.members[0] for alias in (None, "net0", "ua-net-0"): alias_xml = "" if alias is None else f"" legacy = f""" {alias_xml} """ patched = patch_private_interface_xml( legacy, member=member, resources=plan.resources, guest_network_generation=private_guest_network_generation( member, private_guest_routes(plan.vpc, member)), vpc_id=plan.vpc.vpc_id, desired_revision=plan.desired_revision, desired_digest=plan.desired_digest, ) assert patched.count('alias name="ua-public0"') == 1 assert patched.count('alias name="ua-private0"') == 1 # A second reconciliation must still identify the original NIC as public. reparsed = patch_private_interface_xml( patched, member=member, resources=plan.resources, guest_network_generation=private_guest_network_generation( member, private_guest_routes(plan.vpc, member)), vpc_id=plan.vpc.vpc_id, desired_revision=plan.desired_revision, desired_digest=plan.desired_digest, ) assert reparsed.count('alias name="ua-public0"') == 1 assert reparsed.count('alias name="ua-private0"') == 1 def test_private_xml_patch_rejects_domain_wide_alias_collision(): plan = _plan() for collision, public_alias in ( ("ua-public0", "net0"), ("ua-private0", "ua-public0"), ): xml = f""" """ try: patch_private_interface_xml( xml, member=plan.members[0], resources=plan.resources, guest_network_generation=private_guest_network_generation( plan.members[0], private_guest_routes(plan.vpc, plan.members[0])), vpc_id=plan.vpc.vpc_id, desired_revision=plan.desired_revision, desired_digest=plan.desired_digest, ) except VpcDataplaneError as exc: assert exc.status == 409 assert exc.error_code == "vpc_private_xml_role_invalid" else: raise AssertionError("domain-wide duplicate libvirt alias was accepted") def test_guest_network_generation_is_stable_across_member_and_route_churn(): plan = _plan() member = plan.local_members[0] routes = private_guest_routes(plan.vpc, member) assert routes == ("10.42.0.0/24",) expected = private_guest_network_generation(member, routes) assert private_guest_routes( replace(plan.vpc, routes=()), member) == routes assert private_guest_routes( replace(plan.vpc, members=(member,)), member) == routes wider = private_guest_routes( replace(plan.vpc, cidr="10.42.0.0/16"), member) assert private_guest_network_generation(member, wider) != expected for invalid in ((), ("10.42.0.11/32",), ("10.42.0.0/24", "10.42.0.11/32")): with pytest.raises(VpcDataplaneError): private_guest_network_generation(member, invalid) def test_running_member_requires_booted_guest_network_generation(): plan = _plan() member = plan.local_members[0] routes = private_guest_routes(plan.vpc, member) generation = private_guest_network_generation(member, routes) original = """ """ persistent = patch_private_interface_xml( original, member=member, resources=plan.resources, guest_network_generation=generation, vpc_id=plan.vpc.vpc_id, desired_revision=plan.desired_revision, desired_digest=plan.desired_digest, ) stale_live_root = ET.fromstring(persistent) stale_managed = stale_live_root.find( "./metadata/{urn:virsh-panel:1}managed") stale_managed.attrib.pop("private-guest-network-generation") stale_live = ET.tostring(stale_live_root, encoding="unicode") pending = _member_nics(plan, { member.instance_uuid: { "inactive": persistent, "live": stale_live, }, })[0] # Device identity is present in both definitions, but the stale live # generation means this running guest still needs a full restart before # a missing guest-agent proof can be treated as potentially curable. # Legacy v1 observations did not carry physical seed proof. Even with # matching XML device identity they must remain non-usable. assert pending["xml_verified"] is False assert pending["config_verified"] is True assert pending["guest_network_verified"] is False assert pending["restart_required"] is True booted = _member_nics(plan, { member.instance_uuid: { "inactive": persistent, "live": persistent, "seed_verified": True, "guest_network": _guest_network_evidence(member, routes), }, })[0] assert booted["xml_verified"] is True assert booted["config_verified"] is True assert booted["domain_running"] is True assert booted["guest_network_verified"] is True assert booted["guest_private_ip_cidr"] == f"{member.private_ip}/32" assert booted["guest_routes"] == sorted(routes) assert booted["restart_required"] is False def test_booted_generation_fails_closed_when_guest_evidence_is_unavailable(): plan = _plan() member = plan.local_members[0] routes = private_guest_routes(plan.vpc, member) persistent = patch_private_interface_xml( """ """, member=member, resources=plan.resources, guest_network_generation=private_guest_network_generation( member, routes), vpc_id=plan.vpc.vpc_id, desired_revision=plan.desired_revision, desired_digest=plan.desired_digest, ) observed = _member_nics(plan, { member.instance_uuid: { "inactive": persistent, "live": persistent, "seed_verified": True, "guest_network": { "state": "unavailable", "verified": False, "reason_code": "vpc_private_guest_agent_unavailable", }, }, })[0] assert observed["config_verified"] is True assert observed["xml_verified"] is True assert observed["domain_running"] is True assert observed["guest_network_verified"] is False assert observed["guest_network_readback"] == "unavailable" assert observed["guest_reason_code"] == ( "vpc_private_guest_agent_unavailable") # The generation is current, so a restart would not cure an unavailable # agent. The host remains non-active for the guest-specific reason. assert observed["restart_required"] is False def test_booted_generation_rejects_guest_address_or_route_mismatch(): plan = _plan() member = plan.local_members[0] routes = private_guest_routes(plan.vpc, member) persistent = patch_private_interface_xml( """ """, member=member, resources=plan.resources, guest_network_generation=private_guest_network_generation( member, routes), vpc_id=plan.vpc.vpc_id, desired_revision=plan.desired_revision, desired_digest=plan.desired_digest, ) mismatch = _guest_network_evidence(member, routes) mismatch.update({ "state": "mismatch", "verified": False, "private_ip_cidr": "10.42.0.99/32", "routes": [], "reason_code": "vpc_private_guest_address_mismatch", }) observed = _member_nics(plan, { member.instance_uuid: { "inactive": persistent, "live": persistent, "seed_verified": True, "guest_network": mismatch, }, })[0] assert observed["config_verified"] is True assert observed["xml_verified"] is True assert observed["guest_network_verified"] is False assert observed["guest_private_ip_cidr"] == "10.42.0.99/32" assert observed["guest_routes"] == [] assert observed["guest_reason_code"] == ( "vpc_private_guest_address_mismatch") assert observed["restart_required"] is False def test_stopped_member_preserves_config_proof_but_is_not_runtime_verified(): plan = _plan() member = plan.local_members[0] routes = private_guest_routes(plan.vpc, member) persistent = patch_private_interface_xml( """ """, member=member, resources=plan.resources, guest_network_generation=private_guest_network_generation( member, routes), vpc_id=plan.vpc.vpc_id, desired_revision=plan.desired_revision, desired_digest=plan.desired_digest, ) observed = _member_nics(plan, { member.instance_uuid: { "inactive": persistent, "live": None, "seed_verified": True, "guest_network": { "state": "stopped", "verified": False, "reason_code": "vpc_private_guest_stopped", }, }, })[0] assert observed["config_verified"] is True assert observed["domain_running"] is False assert observed["xml_verified"] is True assert observed["guest_network_verified"] is False assert observed["restart_required"] is False assert observed["guest_reason_code"] == ( "vpc_private_guest_stopped") def _active_observation_markers(plan): markers = [ f"__VC_VPC_BRIDGE__={plan.resources.bridge}", "__VC_VPC_BRIDGE_OWNER__=virt-cloud-vpc-dataplane-v1", "__VC_VPC_PERSISTENCE__=yes", "__VC_VPC_BRIDGE_BOUNDARY__=yes", "__VC_VPC_PROXY_ARP__=0", "__VC_VPC_PROXY_ARP_PVLAN__=1", "__VC_VPC_PROXY_STATE__=owned", "__VC_VPC_LOCAL_NEIGHBOR_STATE__=owned", "__VC_VPC_RP_FILTER__=2", f"__VC_VPC_NFT_TABLE__={plan.resources.nft_table}", "__VC_VPC_STATE_OWNER__=owned", f"__VC_VPC_STATE_REVISION__={plan.desired_revision}", f"__VC_VPC_STATE_DIGEST__={plan.desired_digest}", f"__VC_VPC_ACL_REVISION__={plan.desired_revision}", f"__VC_VPC_ACL_DIGEST__={plan.desired_digest}", "__VC_VPC_ANTI_SPOOF__=yes", "__VC_VPC_BRIDGE_NFT__=yes", "__VC_VPC_ROUTE_EXTRA__=no", "__VC_VPC_RULE_EXTRA__=no", "__VC_VPC_FABRIC_NFT__=yes", "__VC_VPC_ARP_NFT__=yes", ] for route in _routes(plan): markers.append( "__VC_VPC_ROUTE__=" f"{route['destination_cidr']}|{route['device']}|" f"{route['table']}|{route.get('via') or '-'}|" f"{route.get('metric', 100)}|owned") markers.append( "__VC_VPC_PROXY_IP__=" f"{route['destination_cidr'].removesuffix('/32')}") markers.extend(( "__VC_VPC_RULE__=" f"{plan.resources.rule_priority}|-|{plan.vpc.cidr}|" f"{plan.resources.route_table}|owned", "__VC_VPC_RULE__=" f"{plan.resources.rule_priority + 1}|wg-fabric0|{plan.vpc.cidr}|" f"{plan.resources.route_table}|owned", )) for peer in plan.peers: markers.extend(( f"__VC_VPC_WG_ALLOWED__={peer.host_id}|{peer.public_key}|" f"{_allowed(plan, peer)}", f"__VC_VPC_WG_STATE__={peer.host_id}|owned", )) return markers def _observe_with_route_backed_wg_markers(plan, *, allowed, key=None, wg_state="owned", peer_set="exact"): """Run adapter readback with an explicitly host-derived WG union.""" member = plan.local_members[0] routes = private_guest_routes(plan.vpc, member) persistent = patch_private_interface_xml( """ """, member=member, resources=plan.resources, guest_network_generation=private_guest_network_generation( member, routes), vpc_id=plan.vpc.vpc_id, desired_revision=plan.desired_revision, desired_digest=plan.desired_digest, ) peer = plan.peers[0] markers = [] for item in _active_observation_markers(plan): if item.startswith("__VC_VPC_WG_ALLOWED__="): item = (f"__VC_VPC_WG_ALLOWED__={peer.host_id}|" f"{key or peer.public_key}|{allowed}") elif item.startswith("__VC_VPC_WG_STATE__="): item = f"__VC_VPC_WG_STATE__={peer.host_id}|{wg_state}" markers.append(item) markers.append(f"__VC_VPC_WG_PEER_SET__={peer_set}") adapter = VpcPrivateHostAdapter(object(), {}) adapter._deployment_contract = lambda: ( "networkmanager_nmcli", "", "", "") adapter._run = lambda _script: "\n".join(markers) + "\n" return adapter.observe(plan, xml_by_instance={ member.instance_uuid: { "inactive": persistent, "live": persistent, "seed_verified": True, "guest_network": _guest_network_evidence(member, routes), }, }) def test_adapter_accepts_host_state_union_without_catalog_only_tenant_ip(): base = _plan() route_backed_allowed = _allowed(base, base.peers[0]) peer = replace( base.peers[0], tenant_allowed_ips=tuple(sorted( set(base.peers[0].tenant_allowed_ips) | {'10.45.0.11/32'})), ) plan = replace( base, peers=(peer,), fabric_peer_keys=frozenset({peer.public_key}), fabric_peer_overlays=frozenset({ (peer.public_key, peer.overlay_cidr), })) observed = _observe_with_route_backed_wg_markers( plan, allowed=route_backed_allowed) assert observed["state"] == "active" assert observed["config_verified"] is True assert observed["wg_allowed_ips"] == [{ "peer_host_id": peer.host_id, "public_key": peer.public_key, "tenant_allowed_ips": ["10.42.0.100/32", "10.42.0.11/32"], }] @pytest.mark.parametrize(('mutation', 'kwargs'), [ ('wrong-key', {'key': WG_KEY_2}), ('missing-overlay', { 'allowed': '10.42.0.11/32,10.42.0.100/32', }), ('missing-vpc-route', { 'allowed': '100.64.0.2/32,10.42.0.11/32', }), ('shell-union-mismatch', {'wg_state': 'missing'}), ('fabric-peer-set-mismatch', {'peer_set': 'extra'}), ]) def test_adapter_rejects_incomplete_or_unproven_route_backed_wg_union( mutation, kwargs): plan = replace( _plan(), fabric_peer_keys=frozenset({WG_KEY}), fabric_peer_overlays=frozenset({(WG_KEY, "100.64.0.2/32")})) peer = plan.peers[0] values = {'allowed': _allowed(plan, peer)} values.update(kwargs) observed = _observe_with_route_backed_wg_markers(plan, **values) assert observed["config_verified"] is False, mutation assert observed["state"] == "degraded", mutation @pytest.mark.parametrize("acl_proof", ("exact", "old_revision", "wrong_digest", "missing_nft")) def test_stopped_host_reports_acl_revision_only_after_exact_configuration_proof(acl_proof): plan = _plan() member = plan.local_members[0] routes = private_guest_routes(plan.vpc, member) persistent = patch_private_interface_xml( """ """, member=member, resources=plan.resources, guest_network_generation=private_guest_network_generation(member, routes), vpc_id=plan.vpc.vpc_id, desired_revision=plan.desired_revision, desired_digest=plan.desired_digest, ) markers = _active_observation_markers(plan) invalid_markers = { "old_revision": ("__VC_VPC_ACL_REVISION__=", "0"), "wrong_digest": ("__VC_VPC_ACL_DIGEST__=", "00" * 32), "missing_nft": ("__VC_VPC_NFT_TABLE__=", "missing"), } if acl_proof in invalid_markers: prefix, value = invalid_markers[acl_proof] markers = [prefix + value if item.startswith(prefix) else item for item in markers] adapter = VpcPrivateHostAdapter(object(), {}) adapter._deployment_contract = lambda: ( "networkmanager_nmcli", "", "", "") adapter._run = lambda _script: "\n".join(markers) + "\n" observed = adapter.observe(plan, xml_by_instance={ member.instance_uuid: { "inactive": persistent, "live": None, "seed_verified": True, "guest_network": { "state": "stopped", "verified": False, "reason_code": "vpc_private_guest_stopped", }, }, }) configuration_verified = acl_proof == "exact" assert observed["config_verified"] is configuration_verified assert observed["config_observed_revision"] == ( plan.desired_revision if configuration_verified else 0) assert observed["acl_revision"] == ( plan.desired_revision if configuration_verified else None) assert observed["state"] == "applying" assert observed["observed_revision"] == 0 assert observed["observed_digest"] is None assert observed["readback"] == "pending_guest" assert observed["member_nics"][0]["domain_running"] is False assert observed["member_nics"][0]["config_verified"] is True assert observed["member_nics"][0]["guest_network_verified"] is False def test_stopped_linkdown_host_config_remains_reusable_without_becoming_active(): plan = replace( _plan(), fabric_peer_keys=frozenset({WG_KEY}), fabric_peer_overlays=frozenset({(WG_KEY, "100.64.0.2/32")})) member = plan.local_members[0] routes = private_guest_routes(plan.vpc, member) persistent = patch_private_interface_xml( """ """, member=member, resources=plan.resources, guest_network_generation=private_guest_network_generation( member, routes), vpc_id=plan.vpc.vpc_id, desired_revision=plan.desired_revision, desired_digest=plan.desired_digest, ) markers = [] for item in _active_observation_markers(plan): if (item.startswith("__VC_VPC_ROUTE__=") and f"|{plan.resources.bridge}|" in item): item = item.rsplit("|", 1)[0] + "|linkdown|owned" markers.append(item) markers.extend(( f"__VC_VPC_FIREWALL_MODE__={FIREWALL_MODE_NATIVE}", "__VC_VPC_FIREWALL_STATE__=owned", f"__VC_VPC_STATE_FIREWALL_MODE__={FIREWALL_MODE_NATIVE}", "__VC_VPC_WG_PEER_SET__=exact", "__VC_VPC_HELPER_CONTRACT__=owned", "__VC_VPC_BRIDGE_ADMIN__=up", "__VC_VPC_NFT_FILE_EXACT__=yes", "__VC_VPC_STATE_FILE_EXACT__=yes", "__VC_VPC_PERSISTENCE_EXACT__=yes", )) adapter = VpcPrivateHostAdapter(object(), {}) adapter._deployment_contract = lambda: ( "networkmanager_nmcli", "", "", "", FIREWALL_MODE_NATIVE) adapter._run = lambda _script: "\n".join(markers) + "\n" observed, host_ready, _receipt = adapter._observe( plan, xml_by_instance={ member.instance_uuid: { "inactive": persistent, "live": None, "seed_verified": True, "guest_network": { "state": "stopped", "verified": False, "reason_code": "vpc_private_guest_stopped", }, }, }) assert observed["config_verified"] is True assert observed["host_config_reusable"] is True assert host_ready is False assert observed["state"] == "applying" assert observed["observed_revision"] == 0 def test_topology_mismatch_revokes_reusable_host_config_proof(): plan = replace( _plan(), fabric_peer_keys=frozenset({WG_KEY}), fabric_peer_overlays=frozenset({(WG_KEY, "100.64.0.2/32")})) member = plan.local_members[0] routes = private_guest_routes(plan.vpc, member) persistent = patch_private_interface_xml( """ """, member=member, resources=plan.resources, guest_network_generation=private_guest_network_generation( member, routes), vpc_id=plan.vpc.vpc_id, desired_revision=plan.desired_revision, desired_digest=plan.desired_digest, ) markers = [ "__VC_VPC_BRIDGE_BOUNDARY__=no" if item == "__VC_VPC_BRIDGE_BOUNDARY__=yes" else item for item in _active_observation_markers(plan) ] markers.extend(( f"__VC_VPC_FIREWALL_MODE__={FIREWALL_MODE_NATIVE}", "__VC_VPC_FIREWALL_STATE__=owned", f"__VC_VPC_STATE_FIREWALL_MODE__={FIREWALL_MODE_NATIVE}", "__VC_VPC_WG_PEER_SET__=exact", "__VC_VPC_HELPER_CONTRACT__=owned", "__VC_VPC_BRIDGE_ADMIN__=up", "__VC_VPC_NFT_FILE_EXACT__=yes", "__VC_VPC_STATE_FILE_EXACT__=yes", "__VC_VPC_PERSISTENCE_EXACT__=yes", )) adapter = VpcPrivateHostAdapter(object(), {}) adapter._deployment_contract = lambda: ( "networkmanager_nmcli", "", "", "", FIREWALL_MODE_NATIVE) adapter._run = lambda _script: "\n".join(markers) + "\n" observed, host_ready, _receipt = adapter._observe( plan, xml_by_instance={ member.instance_uuid: { "inactive": persistent, "live": None, "seed_verified": True, "guest_network": { "state": "stopped", "verified": False, "reason_code": "vpc_private_guest_stopped", }, }, }) assert observed["config_verified"] is False assert observed["host_config_reusable"] is False assert host_ready is False assert observed["config_observed_revision"] == 0 assert observed["config_observed_digest"] is None def test_observer_does_not_activate_unavailable_or_mismatched_guest(): plan = _plan() member = plan.local_members[0] routes = private_guest_routes(plan.vpc, member) persistent = patch_private_interface_xml( """ """, member=member, resources=plan.resources, guest_network_generation=private_guest_network_generation( member, routes), vpc_id=plan.vpc.vpc_id, desired_revision=plan.desired_revision, desired_digest=plan.desired_digest, ) adapter = VpcPrivateHostAdapter(object(), {}) adapter._deployment_contract = lambda: ( "networkmanager_nmcli", "", "", "") adapter._run = lambda _script: "\n".join( _active_observation_markers(plan)) + "\n" for state, reason in ( ("unavailable", "vpc_private_guest_agent_unavailable"), ("mismatch", "vpc_private_guest_address_mismatch"), ): observed = adapter.observe(plan, xml_by_instance={ member.instance_uuid: { "inactive": persistent, "live": persistent, "seed_verified": True, "guest_network": { "state": state, "verified": False, "reason_code": reason, }, }, }) assert observed["config_verified"] is True assert observed["acl_revision"] == plan.desired_revision assert observed["state"] == "applying" assert observed["observed_revision"] == 0 assert observed["observed_digest"] is None assert observed["readback"] == "pending_guest" assert observed["reason_code"] == reason assert observed["member_nics"][0]["xml_verified"] is True assert observed["member_nics"][0]["guest_network_verified"] is False seed_mismatch = adapter.observe(plan, xml_by_instance={ member.instance_uuid: { "inactive": persistent, "live": persistent, "seed_verified": False, "guest_network": _guest_network_evidence(member, routes), }, }) assert seed_mismatch["config_verified"] is False assert seed_mismatch["config_observed_revision"] == 0 assert seed_mismatch["config_observed_digest"] is None assert seed_mismatch["member_nics"][0]["seed_verified"] is False assert seed_mismatch["state"] != "active" confirmed = adapter.observe(plan, xml_by_instance={ member.instance_uuid: { "inactive": persistent, "live": persistent, "seed_verified": True, "guest_network": _guest_network_evidence(member, routes), }, }) assert confirmed["state"] == "active" assert confirmed["observed_revision"] == plan.desired_revision assert confirmed["observed_digest"] == plan.desired_digest assert confirmed["readback"] == "confirmed" def test_observer_rejects_shared_fabric_nft_mismatch(): plan = _plan() member = plan.local_members[0] routes = private_guest_routes(plan.vpc, member) persistent = patch_private_interface_xml( """ """, member=member, resources=plan.resources, guest_network_generation=private_guest_network_generation( member, routes), vpc_id=plan.vpc.vpc_id, desired_revision=plan.desired_revision, desired_digest=plan.desired_digest, ) markers = [ ("__VC_VPC_FABRIC_NFT__=no" if item == "__VC_VPC_FABRIC_NFT__=yes" else item) for item in _active_observation_markers(plan) ] adapter = VpcPrivateHostAdapter(object(), {}) adapter._deployment_contract = lambda: ( "networkmanager_nmcli", "", "", "") adapter._run = lambda _script: "\n".join(markers) + "\n" observed = adapter.observe(plan, xml_by_instance={ member.instance_uuid: { "inactive": persistent, "live": persistent, "seed_verified": True, "guest_network": _guest_network_evidence(member, routes), }, }) assert observed["config_verified"] is False assert observed["state"] == "degraded" assert observed["readback"] == "mismatch" def test_observer_reports_pending_restart_instead_of_active(): plan = _plan() member = plan.local_members[0] generation = private_guest_network_generation( member, private_guest_routes(plan.vpc, member)) original = """ """ persistent = patch_private_interface_xml( original, member=member, resources=plan.resources, guest_network_generation=generation, vpc_id=plan.vpc.vpc_id, desired_revision=plan.desired_revision, desired_digest=plan.desired_digest, ) stale_live_root = ET.fromstring(persistent) stale_live_root.find( "./metadata/{urn:virsh-panel:1}managed" ).attrib.pop("private-guest-network-generation") stale_live = ET.tostring(stale_live_root, encoding="unicode") markers = [ f"__VC_VPC_BRIDGE__={plan.resources.bridge}", "__VC_VPC_BRIDGE_OWNER__=virt-cloud-vpc-dataplane-v1", "__VC_VPC_PERSISTENCE__=yes", "__VC_VPC_BRIDGE_BOUNDARY__=yes", "__VC_VPC_PROXY_ARP__=0", "__VC_VPC_PROXY_ARP_PVLAN__=1", "__VC_VPC_PROXY_STATE__=owned", "__VC_VPC_LOCAL_NEIGHBOR_STATE__=owned", "__VC_VPC_RP_FILTER__=2", f"__VC_VPC_NFT_TABLE__={plan.resources.nft_table}", "__VC_VPC_STATE_OWNER__=owned", f"__VC_VPC_STATE_REVISION__={plan.desired_revision}", f"__VC_VPC_STATE_DIGEST__={plan.desired_digest}", f"__VC_VPC_ACL_REVISION__={plan.desired_revision}", f"__VC_VPC_ACL_DIGEST__={plan.desired_digest}", "__VC_VPC_ANTI_SPOOF__=yes", "__VC_VPC_BRIDGE_NFT__=yes", "__VC_VPC_ROUTE_EXTRA__=no", "__VC_VPC_RULE_EXTRA__=no", "__VC_VPC_ARP_NFT__=yes", ] for route in _routes(plan): markers.append( "__VC_VPC_ROUTE__=" f"{route['destination_cidr']}|{route['device']}|" f"{route['table']}|{route.get('via') or '-'}|" f"{route.get('metric', 100)}|owned") markers.append( "__VC_VPC_PROXY_IP__=" f"{route['destination_cidr'].removesuffix('/32')}") markers.extend(( "__VC_VPC_RULE__=" f"{plan.resources.rule_priority}|-|{plan.vpc.cidr}|" f"{plan.resources.route_table}|owned", "__VC_VPC_RULE__=" f"{plan.resources.rule_priority + 1}|wg-fabric0|{plan.vpc.cidr}|" f"{plan.resources.route_table}|owned", )) for peer in plan.peers: markers.extend(( f"__VC_VPC_WG_ALLOWED__={peer.host_id}|{peer.public_key}|" f"{_allowed(plan, peer)}", f"__VC_VPC_WG_STATE__={peer.host_id}|owned", )) adapter = VpcPrivateHostAdapter(object(), {}) adapter._deployment_contract = lambda: ( "networkmanager_nmcli", "", "", "") adapter._run = lambda _script: "\n".join(markers) + "\n" observed = adapter.observe(plan, xml_by_instance={ member.instance_uuid: { "inactive": persistent, "live": stale_live, }, }) assert observed["state"] == "applying" assert observed["readback"] == "pending_restart" assert observed["reason_code"] == "vpc_private_guest_restart_required" assert observed["member_nics"][0]["restart_required"] is True def _run_vpc_nft_observation(tmp_path, shell, family, mutation): """Run the production readback against nft's numeric/named CLI output.""" plan = _plan() rendered = render_vpc_nft_rules(plan) owner = re.search(r'comment "(.+):anti-spoof-local-ip"', rendered).group(1) suffixes = { "inet": "anti-spoof-local-ip", "bridge": "bridge-anti-spoof-arp-mac", "arp": "arp-proxy-output", } for current in ("inet", "bridge", "arp"): table = re.search( rf'^table {current} {plan.resources.nft_table} \{{\n.*?^\}}\n', rendered, re.MULTILINE | re.DOTALL, ).group(0) if current == family: if mutation == "wrong_priority": table = table.replace("priority -190", "priority -189") elif mutation == "missing_owner_comment": table = table.replace( f'comment "{owner}:{suffixes[current]}"', 'comment "foreign-owner"', ) elif mutation == "missing_acl": table = table.replace( f'comment "{owner}:acl:allow-https"', 'comment "foreign-acl"', ) (tmp_path / f"{current}.numeric").write_text(table) # At bridge priority -190, nft's default output is "filter + 10". # inet/arp retain the numeric priority on the affected real hosts. default = (table.replace("priority -190", "priority filter + 10") if current == "bridge" else table) (tmp_path / f"{current}.default").write_text(default) observed = render_observe_script(plan) block = _shell_block(observed, "nft_text=$(", "if route_inventory=") fake_nft = r''' mock_nft() { mode=default if [ "${1:-}" = -y ]; then mode=numeric; shift; fi [ "$#" = 4 ] && [ "$1" = list ] && [ "$2" = table ] && [ "$4" = "$TABLE" ] || return 91 case "$3" in inet|bridge|arp) : ;; *) return 92 ;; esac [ "$3" != "$VC_NFT_FAILED_FAMILY" ] || return 1 cat "$VC_NFT_DIR/$3.$mode" } ''' env = dict(os.environ, TABLE=plan.resources.nft_table, NFT_MARKER=owner, NFT="mock_nft", VC_NFT_DIR=str(tmp_path), VC_NFT_FAILED_FAMILY=family if mutation == "lookup_failed" else "") return subprocess.run( [shell, "-eu", "-c", fake_nft + block], env=env, capture_output=True, text=True, check=False, ) @pytest.mark.parametrize("shell", ["/bin/sh", "/bin/bash"]) @pytest.mark.parametrize("family", ["inet", "bridge", "arp"]) @pytest.mark.parametrize("mutation", [ "valid", "wrong_priority", "missing_owner_comment", "lookup_failed", ]) def test_vpc_nft_observer_requests_numeric_priorities_and_fails_closed( tmp_path, shell, family, mutation, ): result = _run_vpc_nft_observation(tmp_path, shell, family, mutation) assert result.returncode == 0, result.stderr markers = { "inet": "__VC_VPC_ANTI_SPOOF__", "bridge": "__VC_VPC_BRIDGE_NFT__", "arp": "__VC_VPC_ARP_NFT__", } for current, marker in markers.items(): expected = "no" if current == family and mutation != "valid" else "yes" assert f"{marker}={expected}" in result.stdout.splitlines(), result.stdout @pytest.mark.parametrize("family", ["inet", "bridge"]) def test_vpc_nft_numeric_observation_still_requires_configured_acl(tmp_path, family): result = _run_vpc_nft_observation(tmp_path, "/bin/sh", family, "missing_acl") assert result.returncode == 0, result.stderr marker = "__VC_VPC_ANTI_SPOOF__" if family == "inet" else "__VC_VPC_BRIDGE_NFT__" assert f"{marker}=no" in result.stdout.splitlines(), result.stdout def test_vpc_nft_required_observation_comments_are_emitted_by_renderer(): from virsh_panel.vpc_dataplane import ( _NFT_BRIDGE_REQUIRED_SUFFIXES, _NFT_INET_REQUIRED_SUFFIXES, ) plan = _plan() rendered = render_vpc_nft_rules(plan) for family, suffixes in (("inet", _NFT_INET_REQUIRED_SUFFIXES), ("bridge", _NFT_BRIDGE_REQUIRED_SUFFIXES)): table = re.search( rf'^table {family} {plan.resources.nft_table} \{{\n.*?^\}}\n', rendered, re.MULTILINE | re.DOTALL, ).group(0) for suffix in (*suffixes, "acl:allow-https"): assert re.search(rf'comment "[^"\n]+:{re.escape(suffix)}"', table), suffix