You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/engine/virsh_panel/vpc_dataplane.py

6072 lines
290 KiB
Python

"""Panel-owned VPC private L3 data-plane primitives.
This module is deliberately independent from :mod:`virsh_panel.fabric`.
Fabric owns the host overlay (``wg-fabric0``); this boundary owns only a
VPC's bridge, policy route table, tenant AllowedIPs and nftables table. The
pure builders are usable in unit tests and the host adapter runs one fixed
shell transaction through a pinned remote manager.
"""
from __future__ import annotations
import base64
import hashlib
import ipaddress
import re
import shlex
import time
import xml.etree.ElementTree as ET
from collections import OrderedDict
from dataclasses import dataclass
from threading import Lock
from typing import Any, Mapping, Sequence
from .coordination import mark_execution_unknown, mark_remote_write_started
from .network_roles import (
PRIVATE_INTERFACE_ROLE,
PRIVATE_LIBVIRT_ALIAS,
PUBLIC_LIBVIRT_ALIAS,
select_private_interface,
select_public_interface,
set_libvirt_interface_alias,
)
from .shell_contracts import (
BRIDGE_PORT_RECORDS_SH,
IP_RULE_CONVERGENCE_SH,
LOCAL_NEIGHBOR_SH,
WG_ALLOWED_IPS_SH,
VPC_SYSCTL_VALIDATION_SH as _VPC_SYSCTL_VALIDATION_SH,
)
from .vpc import VpcError, VpcMember, VpcPlan, parse_vpc_plan
DATAPLANE_SCHEMA_VERSION = 1
DATAPLANE_OWNER_MARKER = "virt-cloud-vpc-dataplane-v1"
FABRIC_OWNER_MARKER = "virt-cloud-wg-fabric-v1"
FABRIC_INTERFACE = "wg-fabric0"
FABRIC_LOCK = "/run/virt-cloud/wg-fabric0.lock"
LEGACY_FIREWALLD_NFT_MANIFEST = "/etc/virt-cloud/nftables.conf"
FIREWALL_MODE_NATIVE = "native_nftables"
FIREWALL_MODES = frozenset({FIREWALL_MODE_NATIVE})
PRIVATE_INTERFACE = PRIVATE_INTERFACE_ROLE
PANEL_NS = "urn:virsh-panel:1"
BRIDGE_PREFIX = "vcv-"
NFT_PREFIX = "virt_cloud_vpc_"
ROUTE_TABLE_MIN, ROUTE_TABLE_MAX = 10000, 19999
RULE_PRIORITY_MIN, RULE_PRIORITY_MAX = 20000, 29999
MAX_OUTPUT = 262144
_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,63}\Z")
_HOST = re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]{0,63}\Z")
_MAC = re.compile(r"[0-9a-fA-F]{2}(?::[0-9a-fA-F]{2}){5}\Z")
_KEY = re.compile(r"[A-Za-z0-9+/]{42}==\Z|[A-Za-z0-9+/]{43}=\Z")
_DIGEST = re.compile(r"[0-9a-f]{64}\Z")
# Optimization evidence only: never an active/guest receipt. Adapters are
# recreated for each reconcile, so retain bounded successful-apply hashes
# across instances. Restart/eviction deliberately requires protected apply.
_APPLY_RECEIPT_LIMIT = 1024
_APPLY_RECEIPTS: OrderedDict[tuple[str, ...], tuple[str, str]] = OrderedDict()
_APPLY_RECEIPT_LOCK = Lock()
# The synchronous VPC SSH boundary intentionally discards arbitrary remote
# stderr because host tools may print paths or configuration. Keep one
# closed, host-local journal record instead so an apply failure can be placed
# before or after network activation without exposing command output.
VPC_APPLY_SAFE_LOG_SH = r'''
LOGGER=
for vc_vpc_logger_path in /usr/bin/logger /bin/logger /usr/sbin/logger /sbin/logger; do
if [ -x "$vc_vpc_logger_path" ]; then LOGGER=$vc_vpc_logger_path; break; fi
done
VC_VPC_APPLY_PHASE=entry
vc_vpc_apply_failure_log() {
vc_vpc_log_status=$1
case "$vc_vpc_log_status" in ''|*[!0-9]*) vc_vpc_log_status=1 ;; esac
[ "$vc_vpc_log_status" -le 255 ] 2>/dev/null || vc_vpc_log_status=1
[ "$vc_vpc_log_status" != 0 ] || return 0
case "${VC_VPC_APPLY_PHASE:-}" in
entry|tools|preflight|transaction_snapshot|quarantine|firewall|bridge|persistence|activation|post_activation_sysctl|runtime|proxy|neighbors|routes|policy_rules|fabric_acl|wireguard|final_readback|commit|rollback)
vc_vpc_log_phase=$VC_VPC_APPLY_PHASE ;;
*) vc_vpc_log_phase=unknown ;;
esac
[ -n "${LOGGER:-}" ] || return 0
"$LOGGER" -t virt-cloud-vpc -- \
"apply_state=failed apply_phase=$vc_vpc_log_phase exit_status=$vc_vpc_log_status" \
>/dev/null 2>&1 || true
return 0
}
vc_vpc_apply_log_exit() {
vc_vpc_log_exit_status=$?
trap - EXIT
vc_vpc_apply_failure_log "$vc_vpc_log_exit_status" || true
exit "$vc_vpc_log_exit_status"
}
vc_vpc_apply_signal_exit() {
vc_vpc_signal_status=$1
trap - HUP INT TERM
exit "$vc_vpc_signal_status"
}
trap vc_vpc_apply_log_exit EXIT
trap 'vc_vpc_apply_signal_exit 129' HUP
trap 'vc_vpc_apply_signal_exit 130' INT
trap 'vc_vpc_apply_signal_exit 143' TERM
'''.strip()
# iproute2 prints host routes without /32 and suppresses dev when its query
# already selected that device. Infer only the explicitly supplied filter;
# unfiltered inventories must still contain a unique output device. A private
# bridge without a running tap legitimately has no carrier. Only that explicit
# device may retain linkdown as configuration evidence, never runtime readiness.
VPC_ROUTE_RECORDS_SH = r'''
vc_vpc_route_records() {
awk -v selected_device="${1-}" -v selected_table="${2-main}" -v private_bridge="${3-}" '
function ipv4(value, octets, pos) {
if (split(value, octets, ".") != 4) return 0
for (pos=1; pos<=4; pos++)
if (octets[pos] !~ /^(0|[1-9][0-9]*)$/ || octets[pos]+0 > 255) return 0
return 1
}
NF {
destination=$1
sub(/\/32$/, "", destination)
if (!ipv4(destination)) {bad=1; next}
destination=destination "/32"
device=selected_device; table=selected_table; via="-"; metric="0"; proto=""
row_bad=0; dev_count=0; table_count=0; via_count=0; metric_count=0
proto_count=0; scope_count=0; onlink_count=0; linkdown_count=0
for (i=2; i<=NF; i++) {
field=$i
if (field=="onlink") {if (++onlink_count>1) row_bad=1; continue}
if (field=="linkdown") {if (++linkdown_count>1) row_bad=1; continue}
if (i==NF) {row_bad=1; break}
value=$(++i)
if (field=="dev") {
if (++dev_count>1 || value !~ /^[A-Za-z0-9_.:-]+$/ ||
(selected_device!="" && value!=selected_device)) row_bad=1
device=value
} else if (field=="table") {
if (++table_count>1 || (value!="main" && value !~ /^[1-9][0-9]*$/)) row_bad=1
table=value
} else if (field=="via") {
if (++via_count>1 || !ipv4(value)) row_bad=1
via=value
} else if (field=="metric") {
if (++metric_count>1 || value !~ /^(0|[1-9][0-9]*)$/) row_bad=1
metric=value
} else if (field=="proto") {
if (++proto_count>1 || value!="186") row_bad=1
proto=value
} else if (field=="scope") {
if (++scope_count>1 || (value!="link" && value!="global" && value!="253" && value!="0")) row_bad=1
} else row_bad=1
}
if (table=="254") table="main"
if (device=="" || proto_count!=1 || (onlink_count && via=="-") ||
(linkdown_count && (private_bridge=="" || device!=private_bridge)) ||
(selected_device=="" && table!=selected_table) ||
++seen[destination,table]>1) row_bad=1
if (row_bad) {bad=1; next}
printf "%s|%s|%s|%s|%s|%s|%s\n", destination, device, table, via, metric, proto, linkdown_count ? "linkdown" : "-"
}
END {exit bad ? 1 : 0}
'
}
'''.strip()
VPC_APPLY_RUNTIME_SH = r'''
vc_vpc_proxy_snapshot() {
vc_proxy_path=$1
vc_proxy_raw=$("$IP" neigh show proxy dev "$BRIDGE" 2>/dev/null) || return 1
vc_proxy_values=$(printf '%s\n' "$vc_proxy_raw" | awk -v wanted="$BRIDGE" '
NF == 0 {next}
{
if (NF != 1 && !(NF == 2 && $2 == "proxy") && !(NF == 4 && $2 == "dev" && $3 == wanted && $4 == "proxy")) {bad=1; next}
count=split($1, octets, ".")
if (count != 4) {bad=1; next}
valid=1
for (pos=1; pos<=4; pos++)
if (octets[pos] !~ /^(0|[1-9][0-9]*)$/ || octets[pos]+0 > 255) valid=0
if (!valid || seen[$1]++) {bad=1; next}
print $1
}
END {if (bad) exit 1}
') || return 1
if [ -n "$vc_proxy_values" ]; then
printf '%s\n' "$vc_proxy_values" | LC_ALL=C sort -u > "$vc_proxy_path" || return 1
else
: > "$vc_proxy_path" || return 1
fi
}
vc_vpc_routes_snapshot() {
vc_routes_path=$1
vc_routes_all=$("$IP" -o -4 -N route show table all 2>/dev/null) || return 1
vc_routes_raw=$(printf '%s\n' "$vc_routes_all" | awk -v wanted="$ROUTE_TABLE" '
{for (i=1; i<NF; i++) if ($i=="table" && $(i+1)==wanted) {print; break}}
') || return 1
vc_routes_records=$(printf '%s\n' "$vc_routes_raw" |
vc_vpc_route_records "" "$ROUTE_TABLE" "$BRIDGE") || return 1
printf '%s\n' "$vc_routes_records" | awk -F'|' 'NF {print $1 "|" $2 "|" $3 "|" $4 "|" $5}' |
LC_ALL=C sort -u > "$vc_routes_path"
}
vc_vpc_runtime_subset() {
# Missing owned entries are repairable; extra or changed entries are not.
# Both files are fixed paths in this transaction, populated by strict readers.
vc_subset_actual=$1; vc_subset_expected=$2
while IFS= read -r vc_subset_record; do
[ -n "$vc_subset_record" ] || continue
grep -Fqx -- "$vc_subset_record" "$vc_subset_expected" || return 1
done < "$vc_subset_actual"
}
vc_vpc_runtime_baseline() {
: > "$TXN/routes.expected.before"
: > "$TXN/proxy.expected.before"
if [ -f "$TXN/state.before" ]; then
awk -F'|' '$1 == "route" {print $2 "|" $3 "|" $4 "|" $5 "|" $6}' \
"$TXN/state.before" | LC_ALL=C sort -u > "$TXN/routes.expected.before" || return 1
awk -F'|' '$1 == "route" {sub(/\/32$/, "", $2); print $2}' \
"$TXN/state.before" | LC_ALL=C sort -u > "$TXN/proxy.expected.before" || return 1
fi
: > "$TXN/proxy.file.before"
if [ -f "$PROXY_FILE" ]; then
sed '1d' "$PROXY_FILE" | LC_ALL=C sort -u > "$TXN/proxy.file.before" || return 1
fi
"$CMP" -s "$TXN/proxy.file.before" "$TXN/proxy.expected.before" || return 1
vc_vpc_routes_snapshot "$TXN/routes.before" || return 1
vc_vpc_runtime_subset "$TXN/routes.before" "$TXN/routes.expected.before" || return 1
: > "$TXN/proxy.before"
if [ "$BRIDGE_TOPOLOGY_CAPTURED" = yes ]; then
vc_vpc_proxy_snapshot "$TXN/proxy.before" || return 1
vc_vpc_runtime_subset "$TXN/proxy.before" "$TXN/proxy.expected.before" || return 1
for vc_runtime_key in rp_filter proxy_arp proxy_arp_pvlan; do
vc_runtime_path="/proc/sys/net/ipv4/conf/$BRIDGE/$vc_runtime_key"
[ -f "$vc_runtime_path" ] && [ ! -L "$vc_runtime_path" ] || return 1
vc_runtime_value=$(cat "$vc_runtime_path" 2>/dev/null) || return 1
case "$vc_runtime_key:$vc_runtime_value" in
rp_filter:0|rp_filter:1|rp_filter:2|proxy_arp:0|proxy_arp:1|proxy_arp_pvlan:0|proxy_arp_pvlan:1) : ;;
*) return 1 ;;
esac
printf '%s\n' "$vc_runtime_value" > "$TXN/$vc_runtime_key.before" || return 1
done
fi
chmod 600 "$TXN/routes.before" "$TXN/proxy.before" "$TXN/proxy.file.before" || return 1
}
vc_vpc_runtime_restore() {
# NetworkManager activation can clear routes and proxy neighbours. Restore
# only after that activation, under the original ACL and unchanged topology.
vc_vpc_routes_snapshot "$TXN/routes.rollback.current" || return 1
vc_vpc_runtime_subset "$TXN/routes.rollback.current" "$TXN/routes.before" || return 1
vc_vpc_proxy_snapshot "$TXN/proxy.rollback.reopen" || return 1
vc_vpc_runtime_subset "$TXN/proxy.rollback.reopen" "$TXN/proxy.before" || return 1
while IFS='|' read -r old_destination old_device old_table old_via old_metric; do
[ -n "$old_destination" ] || continue
if [ "$old_via" = - ]; then
"$IP" route replace "$old_destination" table "$old_table" dev "$old_device" proto 186 metric "$old_metric" >/dev/null 2>&1 || return 1
else
"$IP" route replace "$old_destination" table "$old_table" via "$old_via" dev "$old_device" onlink proto 186 metric "$old_metric" >/dev/null 2>&1 || return 1
fi
done < "$TXN/routes.before"
for vc_runtime_key in rp_filter proxy_arp proxy_arp_pvlan; do
[ -f "$TXN/$vc_runtime_key.before" ] || continue
cat "$TXN/$vc_runtime_key.before" > "/proc/sys/net/ipv4/conf/$BRIDGE/$vc_runtime_key" || return 1
done
while IFS= read -r rollback_proxy; do
[ -n "$rollback_proxy" ] || continue
"$IP" neigh replace proxy "$rollback_proxy" dev "$BRIDGE" >/dev/null 2>&1 || return 1
done < "$TXN/proxy.before"
vc_vpc_routes_snapshot "$TXN/routes.rollback.restored" || return 1
"$CMP" -s "$TXN/routes.rollback.restored" "$TXN/routes.before" || return 1
vc_vpc_proxy_snapshot "$TXN/proxy.rollback.restored" || return 1
"$CMP" -s "$TXN/proxy.rollback.restored" "$TXN/proxy.before"
}
'''.strip()
VPC_LOCAL_NEIGHBOR_TRANSACTION_SH = r'''
vc_vpc_neighbor_helper_ready() {
vc_ln_helper=/usr/local/libexec/virt-cloud-wg-fabric
[ -f "$vc_ln_helper" ] && [ ! -L "$vc_ln_helper" ] &&
[ -x "$vc_ln_helper" ] || return 1
[ "$("$STAT" -c '%u:%g:%a' "$vc_ln_helper" 2>/dev/null)" = 0:0:700 ] || return 1
[ "$(sed -n '2p' "$vc_ln_helper")" = '# virt-cloud-wg-fabric-v1' ] &&
[ "$(sed -n '3p' "$vc_ln_helper")" = "# fabric-plan-digest=$EXPECTED_DIGEST" ] &&
[ "$(sed -n '4p' "$vc_ln_helper")" = "PLAN_DIGEST=\"$EXPECTED_DIGEST\"" ] &&
[ "$(sed -n '6p' "$vc_ln_helper")" = '# vpc-local-neighbor-contract-v1' ] &&
[ "$(sed -n '7p' "$vc_ln_helper")" = '# vpc-state-route-scope-contract-v4' ]
}
vc_vpc_neighbors_snapshot() {
vc_ln_snapshot=$(vc_local_neighbor_records "$BRIDGE") || return 1
printf '%s\n' "$vc_ln_snapshot" | awk 'NF' > "$1" || return 1
chmod 600 "$1"
}
vc_vpc_neighbors_subset() {
# Readers have already validated the grammar. A missing hardware address
# is repairable only for an explicitly owned, unresolved IP.
awk -F'|' 'FILENAME==ARGV[1] {expected[$1]=$2; next}
NF {if (!($1 in expected) || ($2!="-" && $2!=expected[$1])) bad=1}
END {exit bad ? 1 : 0}' "$2" "$1"
}
vc_vpc_neighbors_baseline() {
: > "$TXN/neighbors.expected.before"
if [ -f "$TXN/state.before" ]; then
vc_local_neighbor_expected "$TXN/state.before" > "$TXN/neighbors.expected.before" || return 1
fi
# The desired authenticated plan supplies MACs during the one-time upgrade
# from states which predate local_neighbor records. Conflicting old/new
# identities are never silently replaced.
vc_ln_known=$(cat "$TXN/neighbors.expected.before" "$TXN/neighbors.desired") || return 1
printf '%s\n' "$vc_ln_known" | sed '/^$/d' |
LC_ALL=C sort -u > "$TXN/neighbors.known" || return 1
awk -F'|' 'NF {if (seen[$1]++) bad=1} END {exit bad ? 1 : 0}' \
"$TXN/neighbors.known" || return 1
: > "$TXN/neighbors.before"
if [ "$BRIDGE_TOPOLOGY_CAPTURED" = yes ]; then
vc_vpc_neighbors_snapshot "$TXN/neighbors.before" || return 1
vc_vpc_neighbors_subset "$TXN/neighbors.before" "$TXN/neighbors.known" || return 1
fi
chmod 600 "$TXN/neighbors.expected.before" "$TXN/neighbors.desired" \
"$TXN/neighbors.known" "$TXN/neighbors.before"
}
vc_vpc_neighbors_reconcile() {
vc_vpc_neighbors_snapshot "$TXN/neighbors.activation" || return 1
vc_vpc_neighbors_subset "$TXN/neighbors.activation" "$TXN/neighbors.known" || return 1
LOCAL_NEIGHBORS_CHANGED=yes
while IFS='|' read -r vc_ln_ip vc_ln_mac vc_ln_state; do
[ -n "$vc_ln_ip" ] || continue
vc_ln_present=$(awk -F'|' -v ip="$vc_ln_ip" '$1==ip {found=1} END {print found ? "yes" : "no"}' "$TXN/neighbors.desired") || return 1
if [ "$vc_ln_present" = no ]; then
"$IP" -4 neigh del "$vc_ln_ip" dev "$BRIDGE" >/dev/null 2>&1 || return 1
fi
done < "$TXN/neighbors.activation"
vc_ln_desired=$(cat "$TXN/neighbors.desired") || return 1
vc_local_neighbor_ensure "$BRIDGE" "$vc_ln_desired"
}
vc_vpc_neighbors_rollback_guard() {
vc_vpc_neighbors_snapshot "$TXN/neighbors.rollback.current" || return 1
vc_vpc_neighbors_subset "$TXN/neighbors.rollback.current" "$TXN/neighbors.known"
}
vc_vpc_neighbors_restore() {
# Administrative DOWN and manager activation can clear permanent entries.
# Restore only after the original ACL and unchanged bridge are verified.
vc_vpc_neighbors_rollback_guard || return 1
while IFS='|' read -r vc_ln_ip vc_ln_mac vc_ln_state; do
[ -n "$vc_ln_ip" ] || continue
"$IP" -4 neigh del "$vc_ln_ip" dev "$BRIDGE" >/dev/null 2>&1 || return 1
done < "$TXN/neighbors.rollback.current"
: > "$TXN/neighbors.rollback.expected"
while IFS='|' read -r vc_ln_ip vc_ln_mac vc_ln_state; do
[ -n "$vc_ln_ip" ] && [ "$vc_ln_mac" != - ] || continue
# Dynamic timer state cannot be replayed. Restore the sampled binding as
# STALE so the kernel can validate it; permanent bindings stay permanent.
vc_ln_restore_state=stale
case "$vc_ln_state" in
PERMANENT) vc_ln_restore_state=permanent ;;
NOARP) vc_ln_restore_state=noarp ;;
esac
"$IP" -4 neigh replace "$vc_ln_ip" lladdr "$vc_ln_mac" nud "$vc_ln_restore_state" dev "$BRIDGE" >/dev/null 2>&1 || return 1
printf '%s|%s\n' "$vc_ln_ip" "$vc_ln_mac" >> "$TXN/neighbors.rollback.expected" || return 1
done < "$TXN/neighbors.before"
vc_vpc_neighbors_snapshot "$TXN/neighbors.rollback.restored" || return 1
vc_vpc_neighbors_subset "$TXN/neighbors.rollback.restored" "$TXN/neighbors.rollback.expected" || return 1
awk -F'|' 'NF {print $1 "|" $2}' "$TXN/neighbors.rollback.restored" | LC_ALL=C sort > "$TXN/neighbors.rollback.pairs" || return 1
LC_ALL=C sort "$TXN/neighbors.rollback.expected" > "$TXN/neighbors.rollback.expected.sorted" || return 1
"$CMP" -s "$TXN/neighbors.rollback.pairs" "$TXN/neighbors.rollback.expected.sorted" || return 1
awk -F'|' 'FILENAME==ARGV[1] {if ($3=="PERMANENT" || $3=="NOARP") stable[$1]=$3; next}
NF && ($1 in stable) && $3!=stable[$1] {bad=1}
END {exit bad ? 1 : 0}' "$TXN/neighbors.before" "$TXN/neighbors.rollback.restored"
}
'''.strip()
VPC_NATIVE_TRANSACTION_GUARD_SH = r'''
vc_vpc_firewall_boundary_clean() {
[ -n "${NFT:-}" ] || return 1
[ ! -e /etc/virt-cloud/nftables.conf ] &&
[ ! -L /etc/virt-cloud/nftables.conf ] || return 1
[ ! -e /etc/virt-cloud/.wg-fabric-txn ] &&
[ ! -L /etc/virt-cloud/.wg-fabric-txn ] || return 1
[ ! -e /run/virt-cloud/wg-fabric-emergency.owner ] &&
[ ! -L /run/virt-cloud/wg-fabric-emergency.owner ] || return 1
vc_vpc_fabric_config=/etc/virt-cloud/wg-fabric0.conf
if [ -e "$vc_vpc_fabric_config" ] || [ -L "$vc_vpc_fabric_config" ]; then
[ -f "$vc_vpc_fabric_config" ] &&
[ ! -L "$vc_vpc_fabric_config" ] || return 1
[ "$($STAT -c '%u:%g:%a' "$vc_vpc_fabric_config" 2>/dev/null)" = 0:0:600 ] || return 1
vc_vpc_fabric_config_size=$($STAT -c '%s' "$vc_vpc_fabric_config" 2>/dev/null) || return 1
case "$vc_vpc_fabric_config_size" in ''|*[!0-9]*) return 1 ;; esac
[ "$vc_vpc_fabric_config_size" -le 65536 ] || return 1
awk -F'|' '
length($0)>4096 {bad=1}
$1=="firewall" {firewall++; if (NF!=2 || $2!="native_nftables") bad=1; next}
$1=="underlay" || $1=="firewall_rule" {bad=1; next}
END {exit (bad || firewall>1) ? 1 : 0}
' "$vc_vpc_fabric_config" || return 1
fi
vc_vpc_transaction_count=0
for vc_vpc_other_transaction in /etc/virt-cloud/.vpc-dataplane-*.txn; do
[ -e "$vc_vpc_other_transaction" ] ||
[ -L "$vc_vpc_other_transaction" ] || continue
vc_vpc_transaction_count=$((vc_vpc_transaction_count + 1))
[ "$vc_vpc_transaction_count" -le 256 ] || return 1
[ "$vc_vpc_other_transaction" = "$TXN" ] && continue
return 1
done
vc_vpc_legacy_snapshot_count=0
for vc_vpc_legacy_snapshot in \
/etc/virt-cloud/vpc-dataplane/*.nft.inet.live \
/etc/virt-cloud/vpc-dataplane/*.nft.bridge.live \
/etc/virt-cloud/vpc-dataplane/*.nft.arp.live; do
[ -e "$vc_vpc_legacy_snapshot" ] ||
[ -L "$vc_vpc_legacy_snapshot" ] || continue
vc_vpc_legacy_snapshot_count=$((vc_vpc_legacy_snapshot_count + 1))
[ "$vc_vpc_legacy_snapshot_count" -le 256 ] || return 1
return 1
done
vc_vpc_state_count=0
for vc_vpc_state_file in /etc/virt-cloud/vpc-dataplane/*.state; do
[ -e "$vc_vpc_state_file" ] || [ -L "$vc_vpc_state_file" ] || continue
vc_vpc_state_count=$((vc_vpc_state_count + 1))
[ "$vc_vpc_state_count" -le 256 ] || return 1
[ -f "$vc_vpc_state_file" ] && [ ! -L "$vc_vpc_state_file" ] || return 1
[ "$($STAT -c '%u:%g:%a' "$vc_vpc_state_file" 2>/dev/null)" = 0:0:600 ] || return 1
vc_vpc_state_size=$($STAT -c '%s' "$vc_vpc_state_file" 2>/dev/null) || return 1
case "$vc_vpc_state_size" in ''|*[!0-9]*) return 1 ;; esac
[ "$vc_vpc_state_size" -le 65536 ] || return 1
awk -F'|' '
$1=="firewall_mode" {firewall++; if (NF!=2 || $2!="native_nftables") bad=1; next}
$1=="nft_snapshot" {bad=1; next}
END {exit (bad || firewall>1) ? 1 : 0}
' "$vc_vpc_state_file" || return 1
done
vc_vpc_firewall_tables=$($NFT list tables 2>/dev/null) || return 1
vc_vpc_firewall_tables_size=$(printf '%s' "$vc_vpc_firewall_tables" |
wc -c | tr -d ' ') || return 1
case "$vc_vpc_firewall_tables_size" in ''|*[!0-9]*) return 1 ;; esac
[ "$vc_vpc_firewall_tables_size" -le 65536 ] || return 1
! printf '%s\n' "$vc_vpc_firewall_tables" |
grep -Eq 'table (inet firewalld|(inet|bridge) virt_cloud_fabric_emergency)([[:space:]]|$)'
}
vc_vpc_native_transaction_owned() {
[ -n "${TXN:-}" ] && [ -n "${STAT:-}" ] || return 1
[ -d "$TXN" ] && [ ! -L "$TXN" ] || return 1
[ "$($STAT -c '%u:%g:%a' "$TXN" 2>/dev/null)" = 0:0:700 ] || return 1
[ -f "$TXN/firewall_mode" ] && [ ! -L "$TXN/firewall_mode" ] || return 1
[ "$($STAT -c '%u:%g:%a' "$TXN/firewall_mode" 2>/dev/null)" = 0:0:600 ] || return 1
vc_vpc_txn_mode_size=$($STAT -c '%s' "$TXN/firewall_mode" 2>/dev/null) || return 1
case "$vc_vpc_txn_mode_size" in ''|*[!0-9]*) return 1 ;; esac
[ "$vc_vpc_txn_mode_size" -le 64 ] || return 1
vc_vpc_txn_mode=$(awk '
NR==1 && $0=="native_nftables" {ok=1}
{rows++}
END {if (ok && rows==1) print "native_nftables"; else exit 1}
' "$TXN/firewall_mode") || return 1
[ "$vc_vpc_txn_mode" = native_nftables ] || return 1
for vc_vpc_legacy_txn_file in \
"$TXN"/nft.persist.before.inet \
"$TXN"/nft.persist.before.bridge \
"$TXN"/nft.persist.before.arp \
"$TXN"/reassert.nft \
"$TXN"/nft.desired.verify.*; do
[ -e "$vc_vpc_legacy_txn_file" ] || [ -L "$vc_vpc_legacy_txn_file" ] || continue
return 1
done
if [ -e "$TXN/state.before" ] || [ -L "$TXN/state.before" ]; then
[ -f "$TXN/state.before" ] && [ ! -L "$TXN/state.before" ] || return 1
[ "$($STAT -c '%u:%g:%a' "$TXN/state.before" 2>/dev/null)" = 0:0:600 ] || return 1
vc_vpc_txn_state_size=$($STAT -c '%s' "$TXN/state.before" 2>/dev/null) || return 1
case "$vc_vpc_txn_state_size" in ''|*[!0-9]*) return 1 ;; esac
[ "$vc_vpc_txn_state_size" -le 65536 ] || return 1
awk -F'|' '
$1=="firewall_mode" {firewall++; if (NF!=2 || $2!="native_nftables") bad=1; next}
$1=="nft_snapshot" {bad=1; next}
END {exit (bad || firewall>1) ? 1 : 0}
' "$TXN/state.before" || return 1
fi
return 0
}
vc_vpc_transaction_file_secure() {
vc_vpc_transaction_file=$1
vc_vpc_transaction_file_limit=$2
[ -f "$vc_vpc_transaction_file" ] &&
[ ! -L "$vc_vpc_transaction_file" ] || return 1
[ "$($STAT -c '%u:%g:%a' "$vc_vpc_transaction_file" 2>/dev/null)" = 0:0:600 ] || return 1
vc_vpc_transaction_file_size=$($STAT -c '%s' "$vc_vpc_transaction_file" 2>/dev/null) || return 1
case "$vc_vpc_transaction_file_size:$vc_vpc_transaction_file_limit" in
*[!0-9:]*) return 1 ;;
esac
[ "$vc_vpc_transaction_file_size" -le "$vc_vpc_transaction_file_limit" ] || return 1
}
vc_vpc_transaction_record_owned() {
vc_vpc_transaction_record_file=$1
vc_vpc_transaction_record_expected=$2
vc_vpc_transaction_file_secure "$vc_vpc_transaction_record_file" 4096 || return 1
awk -v wanted="$vc_vpc_transaction_record_expected" '
{rows++; if (rows!=1 || $0!=wanted) bad=1}
END {exit (bad || rows!=1) ? 1 : 0}
' "$vc_vpc_transaction_record_file"
}
vc_vpc_write_native_transaction_mode() {
[ -d "$TXN" ] && [ ! -L "$TXN" ] || return 1
[ ! -e "$TXN/firewall_mode" ] && [ ! -L "$TXN/firewall_mode" ] || return 1
printf '%s\n' native_nftables > "$TXN/firewall_mode" || return 1
chmod 600 "$TXN/firewall_mode" || return 1
vc_vpc_native_transaction_owned
}
'''.strip()
_NFT_INET_REQUIRED_SUFFIXES = (
"anti-spoof-local-ip", "anti-spoof-egress",
"anti-spoof-public-to-bridge", "anti-spoof-public-to-fabric",
"anti-spoof-bridge-to-foreign", "anti-spoof-fabric-to-foreign",
"anti-spoof-public-to-vpc-bridge", "anti-spoof-public-to-vpc-fabric",
"anti-spoof-bridge-vpc-to-foreign", "anti-spoof-fabric-vpc-to-foreign",
"anti-spoof-wg-ingress", "anti-spoof-wg-egress",
"anti-spoof-wg-source", "member-egress-isolation",
"member-fabric-isolation", "default-deny", "wg-transit-deny",
"ipv6-drop-bridge", "ipv6-drop-fabric", "ipv6-input-drop-bridge",
"ipv6-input-drop-fabric", "ipv6-output-drop-bridge",
"host-isolation", "host-isolation-wg", "host-isolation-out",
"host-isolation-wg-source", "host-isolation-out-wg",
"host-isolation-vpc-out", "host-isolation-vpc-out-wg",
"scope-bridge-in", "scope-bridge-out", "scope-fabric-in",
"scope-fabric-out",
)
_NFT_BRIDGE_REQUIRED_SUFFIXES = (
"bridge-anti-spoof-ingress", "bridge-anti-spoof-arp",
"bridge-anti-spoof-arp-mac", "bridge-anti-spoof-ip",
"bridge-anti-spoof-arp-ip", "bridge-multicast-drop",
"bridge-limited-broadcast-drop", "bridge-vpc-broadcast-drop",
"bridge-unknown-destination-drop", "bridge-default-deny",
"bridge-scope-accept", "bridge-arp-accept", "bridge-non-ip-drop",
"bridge-anti-spoof-arp-input-mac", "bridge-anti-spoof-input",
"bridge-anti-spoof-input-ip", "bridge-ipv6-input-drop",
"bridge-arp-input-accept", "bridge-ip-input-accept",
"bridge-input-non-ip-drop",
)
NFT_LIVE_SNAPSHOT_SH = r'''
vc_nft_table_absent() {
vc_nft_absent_family=$1
vc_nft_absent_name=$2
vc_nft_absent_inventory=$3
case "$vc_nft_absent_family" in inet|bridge|arp) : ;; *) return 1 ;; esac
case "$vc_nft_absent_name" in ''|*[!A-Za-z0-9_]*) return 1 ;; esac
# A failed point lookup is ambiguous (permissions, netlink failure, parse
# error, or actual absence). Only a successful, completely parsed family
# inventory may prove that a table does not exist.
"$NFT" -s list tables "$vc_nft_absent_family" \
> "$vc_nft_absent_inventory" 2>/dev/null || return 1
awk -v family="$vc_nft_absent_family" -v wanted="$vc_nft_absent_name" '
NF == 0 {next}
NF != 3 || $1 != "table" || $2 != family ||
$3 !~ /^[A-Za-z0-9_.-]+$/ {bad=1; next}
$3 == wanted {found++}
END {exit (!bad && found == 0) ? 0 : 1}
' "$vc_nft_absent_inventory"
}
vc_nft_table_snapshot() {
vc_nft_snapshot_family=$1
vc_nft_snapshot_name=$2
vc_nft_snapshot_path=$3
case "$vc_nft_snapshot_family" in inet|bridge|arp) : ;; *) return 1 ;; esac
case "$vc_nft_snapshot_name" in ''|*[!A-Za-z0-9_]*) return 1 ;; esac
rm -f "$vc_nft_snapshot_path" "$vc_nft_snapshot_path.absent" \
"$vc_nft_snapshot_path.raw"
# `nft --stateless list table` is itself valid nft input and retains the
# complete table semantics (chains, expressions, verdicts and set
# elements) while omitting only changing state such as packet counters.
# Do not reduce this to owner comments: a foreign rule can retain a panel
# comment while changing `drop` to `accept`.
if "$NFT" -s list table "$vc_nft_snapshot_family" "$vc_nft_snapshot_name" \
> "$vc_nft_snapshot_path.raw" 2>/dev/null; then
[ -s "$vc_nft_snapshot_path.raw" ] || return 1
mv -f "$vc_nft_snapshot_path.raw" "$vc_nft_snapshot_path" || return 1
chmod 600 "$vc_nft_snapshot_path" || return 1
else
rm -f "$vc_nft_snapshot_path.raw"
vc_nft_table_absent "$vc_nft_snapshot_family" "$vc_nft_snapshot_name" \
"$vc_nft_snapshot_path.inventory" || return 1
rm -f "$vc_nft_snapshot_path.inventory"
: > "$vc_nft_snapshot_path.absent" || return 1
chmod 600 "$vc_nft_snapshot_path.absent" || return 1
fi
}
vc_nft_table_matches_snapshot() {
vc_nft_match_family=$1
vc_nft_match_name=$2
vc_nft_match_path=$3
case "$vc_nft_match_family" in inet|bridge|arp) : ;; *) return 1 ;; esac
case "$vc_nft_match_name" in ''|*[!A-Za-z0-9_]*) return 1 ;; esac
if [ -f "$vc_nft_match_path" ] && [ ! -L "$vc_nft_match_path" ] &&
[ ! -e "$vc_nft_match_path.absent" ] &&
[ ! -L "$vc_nft_match_path.absent" ]; then
"$NFT" -s list table "$vc_nft_match_family" "$vc_nft_match_name" \
> "$vc_nft_match_path.current" 2>/dev/null || return 1
[ -s "$vc_nft_match_path.current" ] || return 1
if "$CMP" -s "$vc_nft_match_path" "$vc_nft_match_path.current"; then
vc_nft_match_status=0
else
vc_nft_match_status=$?
fi
rm -f "$vc_nft_match_path.current"
return "$vc_nft_match_status"
fi
if [ -f "$vc_nft_match_path.absent" ] &&
[ ! -L "$vc_nft_match_path.absent" ] &&
[ ! -e "$vc_nft_match_path" ] && [ ! -L "$vc_nft_match_path" ]; then
if vc_nft_table_absent "$vc_nft_match_family" "$vc_nft_match_name" \
"$vc_nft_match_path.inventory"; then
vc_nft_match_status=0
else
vc_nft_match_status=$?
fi
rm -f "$vc_nft_match_path.inventory"
return "$vc_nft_match_status"
fi
return 1
}
'''.strip()
VPC_APPLY_RECEIPT_SH = r'''
vc_vpc_sha256() (
[ -n "${SHA256SUM:-}" ] || return 1
vc_receipt_hash=$("$SHA256SUM" "$1") || return 1
vc_receipt_hash=${vc_receipt_hash%% *}
printf '%s\n' "$vc_receipt_hash" | grep -Eq '^[0-9a-f]{64}$' || return 1
printf '%s\n' "$vc_receipt_hash"
)
vc_vpc_nft_receipt() (
[ -n "${SHA256SUM:-}" ] || { printf '%s\n' unavailable; exit 0; }
vc_receipt_all=
for vc_receipt_family in inet bridge arp; do
# Failed reads are not absence. Only a successful strict inventory may
# report an absent table, which never matches a successful apply receipt.
vc_receipt_inventory=$("$NFT" -s -n -y list tables "$vc_receipt_family") || exit 1
vc_receipt_present=$(printf '%s\n' "$vc_receipt_inventory" |
awk -v family="$vc_receipt_family" -v wanted="$TABLE" '
NF==0 {next}
NF!=3 || $1!="table" || $2!=family || $3!~/^[A-Za-z0-9_.-]+$/ {bad=1}
$3==wanted {found++}
END {if (bad || found>1) exit 1; print found+0}') || exit 1
if [ "$vc_receipt_present" = 0 ]; then
printf '%s\n' missing
exit 0
fi
vc_receipt_text=$("$NFT" -s -n -y list table "$vc_receipt_family" "$TABLE") || exit 1
[ -n "$vc_receipt_text" ] || exit 1
vc_receipt_hash=$(printf '%s\n' "$vc_receipt_text" | "$SHA256SUM") || exit 1
vc_receipt_hash=${vc_receipt_hash%% *}
printf '%s\n' "$vc_receipt_hash" | grep -Eq '^[0-9a-f]{64}$' || exit 1
vc_receipt_all="${vc_receipt_all}${vc_receipt_all:+,}$vc_receipt_hash"
done
printf '%s\n' "$vc_receipt_all"
)
vc_vpc_bridge_admin_state() (
vc_receipt_link=$("$IP" -o link show dev "$BRIDGE") || exit 1
printf '%s\n' "$vc_receipt_link" | awk '
NF {
rows++
if ($1!~/^[0-9]+:$/ || $3!~/^<[^>]+>$/) {bad=1; next}
flags=$3; sub(/^</,"",flags); sub(/>$/,"",flags)
n=split(flags,items,","); for (i=1;i<=n;i++) if (items[i]=="UP") up++
}
END {if (bad || rows!=1 || up>1) exit 1; print (up==1 ? "up" : "down")}'
)
'''.strip()
NETWORKMANAGER_VPC_PROFILE_SH = r'''
vc_nm_profile_disk_validate() {
vc_nm_disk_file=$1
vc_nm_disk_uuid=$2
vc_nm_disk_auto=${3:-false}
case "$vc_nm_disk_file" in
/etc/NetworkManager/system-connections/*.nmconnection)
vc_nm_disk_name=${vc_nm_disk_file#/etc/NetworkManager/system-connections/}
case "$vc_nm_disk_name" in ''|*/*) return 1 ;; esac
;;
*) return 1 ;;
esac
[ -f "$vc_nm_disk_file" ] && [ ! -L "$vc_nm_disk_file" ] || return 1
[ "$($STAT -c '%u:%a' "$vc_nm_disk_file" 2>/dev/null)" = 0:600 ] || return 1
awk -F= -v wanted_id="$BRIDGE" -v wanted_uuid="$vc_nm_disk_uuid" \
-v wanted_marker="$NM_PROFILE_MARKER" \
-v expected_auto="$vc_nm_disk_auto" '
/^\[/ { in_connection=($0 == "[connection]"); next }
in_connection && $1 == "id" {
id_count++; if (substr($0, 4) != wanted_id) bad=1; next
}
in_connection && $1 == "uuid" {
uuid_count++; if (substr($0, 6) != wanted_uuid) bad=1; next
}
in_connection && $1 == "type" {
type_count++; if (substr($0, 6) != "bridge") bad=1; next
}
in_connection && $1 == "interface-name" {
iface_count++; if (substr($0, 16) != wanted_id) bad=1; next
}
in_connection && $1 == "stable-id" {
marker_count++; if (substr($0, 11) != wanted_marker) bad=1; next
}
in_connection && $1 == "autoconnect" {
auto_count++; if (substr($0, 13) != expected_auto) bad=1; next
}
in_connection && $1 == "autoconnect-retries" {
retry_count++; if (substr($0, 21) != "0") bad=1; next
}
in_connection && $1 == "zone" { zone_count++; next }
END {
exit (bad || id_count != 1 || uuid_count != 1 || type_count != 1 ||
iface_count != 1 || marker_count != 1 || auto_count != 1 ||
retry_count != 1 ||
zone_count != 0) ? 1 : 0
}
' "$vc_nm_disk_file"
}
vc_nm_profile_field() {
vc_nm_field_raw=$("$NMCLI" -t --escape no -g "$1" \
connection show uuid "$NM_PROFILE_UUID" 2>/dev/null) || return 1
printf '%s\n' "$vc_nm_field_raw" | sed -n '1p'
}
vc_nm_profile_empty() {
vc_nm_empty_value=$(vc_nm_profile_field "$1") || return 1
[ -z "$vc_nm_empty_value" ] || [ "$vc_nm_empty_value" = "--" ]
}
vc_nm_profile_resolve() {
vc_nm_resolve_auto=${1:-false}
vc_nm_names_raw=$("$NMCLI" -t --escape no -f NAME \
connection show 2>/dev/null) || return 1
vc_nm_count=$(printf '%s\n' "$vc_nm_names_raw" |
awk -v wanted="$BRIDGE" '$0 == wanted { count++ } END { print count + 0 }') || return 1
case "$vc_nm_count" in 0) return 2 ;; 1) : ;; *) return 1 ;; esac
vc_nm_uuid_raw=$("$NMCLI" -t --escape no -g connection.uuid \
connection show "$BRIDGE" 2>/dev/null) || return 1
NM_PROFILE_UUID=$(printf '%s\n' "$vc_nm_uuid_raw" | sed -n '1p') || return 1
printf '%s\n' "$NM_PROFILE_UUID" |
grep -Eq '^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$' || return 1
NM_PROFILE_FILE=$(vc_nm_filename_for_uuid "$NM_PROFILE_UUID") || return 1
vc_nm_profile_disk_validate "$NM_PROFILE_FILE" "$NM_PROFILE_UUID" \
"$vc_nm_resolve_auto" || return 1
}
vc_nm_profile_base_validate_mode() {
vc_nm_expected_auto=$1
vc_nm_expected_disk_auto=$2
[ "$(vc_nm_profile_field connection.type)" = bridge ] || return 1
[ "$(vc_nm_profile_field connection.interface-name)" = "$BRIDGE" ] || return 1
[ "$(vc_nm_profile_field connection.stable-id)" = "$NM_PROFILE_MARKER" ] || return 1
[ "$(vc_nm_profile_field connection.autoconnect)" = "$vc_nm_expected_auto" ] || return 1
if [ "$vc_nm_expected_auto" = no ]; then
[ "$(vc_nm_profile_field connection.autoconnect-retries)" = 0 ] || return 1
fi
[ "$(vc_nm_profile_field ipv4.method)" = disabled ] || return 1
[ "$(vc_nm_profile_field ipv6.method)" = disabled ] || return 1
for vc_nm_empty_field in \
ipv4.addresses ipv4.routes ipv4.routing-rules ipv4.gateway ipv4.dns \
ipv4.dns-search ipv6.addresses ipv6.routes ipv6.routing-rules \
ipv6.gateway ipv6.dns ipv6.dns-search connection.master \
connection.controller connection.slave-type connection.port-type \
connection.secondaries; do
vc_nm_profile_empty "$vc_nm_empty_field" || return 1
done
case "$(vc_nm_profile_field bridge.stp)" in no|false|off|0|--) : ;; *) return 1 ;; esac
vc_nm_profile_disk_validate "$NM_PROFILE_FILE" "$NM_PROFILE_UUID" \
"$vc_nm_expected_disk_auto" || return 1
}
vc_nm_profile_base_validate() {
vc_nm_profile_base_validate_mode no false
}
vc_nm_profile_active_count() {
vc_nm_active_raw=$("$NMCLI" -t --escape no -f UUID \
connection show --active 2>/dev/null) || return 1
printf '%s\n' "$vc_nm_active_raw" | awk -v wanted="$NM_PROFILE_UUID" '
NF == 0 {next}
length($0) != 36 || $0 !~ /^[0-9A-Fa-f-]+$/ {bad=1; next}
$0 == wanted {count++}
END {
if (bad) exit 1
print count + 0
}
'
}
vc_nm_profile_active() {
[ "$(vc_nm_profile_active_count)" = 1 ]
}
vc_nm_profile_inactive() {
[ "$(vc_nm_profile_active_count)" = 0 ]
}
vc_nm_device_managed_state() {
vc_nm_managed_raw=$(
"$NMCLI" -t --escape no -g GENERAL.NM-MANAGED \
device show "$BRIDGE" 2>/dev/null
) || return 1
vc_nm_managed=$(printf '%s\n' "$vc_nm_managed_raw" | sed -n '1p') || return 1
case "$vc_nm_managed" in yes|no) printf '%s' "$vc_nm_managed" ;; *) return 1 ;; esac
}
vc_nm_device_expect_managed() {
[ "$(vc_nm_device_managed_state)" = "$1" ]
}
vc_nm_link_expect_down() {
vc_nm_link_raw=$("$IP" -o link show dev "$BRIDGE" 2>/dev/null) || return 1
vc_nm_link_flags=$(printf '%s\n' "$vc_nm_link_raw" | awk -v wanted="$BRIDGE" '
NF == 0 {next}
{
count++
name=$2
sub(/:$/, "", name)
sub(/@.*/, "", name)
if (name != wanted || $3 !~ /^<[^>]+>$/) bad=1
flags=$3
}
END {
if (bad || count != 1) exit 1
print flags
}
') || return 1
printf '%s\n' "$vc_nm_link_flags" |
grep -Eq '(^|<|,)UP(,|>)' && return 1 || true
}
vc_nm_preserve_bridge_ready() {
# Re-entering an active controller must not ask NM to deactivate it: that
# operation detaches externally-created libvirt taps during cleanup.
vc_nm_profile_resolve && vc_nm_profile_base_validate || return 1
[ -n "${NM_PRESERVED_PROFILE_UUID:-}" ] &&
[ "$NM_PROFILE_UUID" = "$NM_PRESERVED_PROFILE_UUID" ] || return 1
vc_nm_device_expect_managed yes && vc_nm_profile_active || return 1
vc_nm_preserved_uuid=$(
LC_ALL=C "$NMCLI" -t --escape no -g GENERAL.CON-UUID \
device show "$BRIDGE" 2>/dev/null
) || return 1
[ "$vc_nm_preserved_uuid" = "$NM_PROFILE_UUID" ] || return 1
vc_vpc_bridge_topology_matches
}
vc_nm_device_quarantine() {
if [ "${NM_PRESERVE_ACTIVE_BRIDGE:-no}" = yes ]; then
vc_nm_preserve_bridge_ready || return 1
"$IP" link set dev "$BRIDGE" down >/dev/null 2>&1 || return 1
vc_nm_link_expect_down && vc_nm_preserve_bridge_ready
return $?
fi
"$IP" link show dev "$BRIDGE" >/dev/null 2>&1 || return 1
vc_nm_quarantine_uuid_raw=$(
"$NMCLI" -t --escape no -g GENERAL.CON-UUID \
device show "$BRIDGE" 2>/dev/null
) || return 1
vc_nm_quarantine_uuid=$(printf '%s\n' "$vc_nm_quarantine_uuid_raw" |
sed -n '1p') || return 1
[ -z "$vc_nm_quarantine_uuid" ] ||
[ "$vc_nm_quarantine_uuid" = "$NM_PROFILE_UUID" ] || return 1
vc_nm_quarantine_zone=$(vc_nm_profile_field connection.zone) || return 1
[ -z "$vc_nm_quarantine_zone" ] || [ "$vc_nm_quarantine_zone" = -- ] ||
return 1
"$NMCLI" device set "$BRIDGE" managed no >/dev/null 2>&1 || return 1
vc_nm_device_expect_managed no || return 1
vc_nm_quarantine_zone=$(vc_nm_profile_field connection.zone) || return 1
[ -z "$vc_nm_quarantine_zone" ] || [ "$vc_nm_quarantine_zone" = -- ] || return 1
"$IP" link set dev "$BRIDGE" down >/dev/null 2>&1 || return 1
vc_nm_link_expect_down || return 1
if [ "${BRIDGE_TOPOLOGY_CAPTURED:-no}" = yes ]; then
vc_vpc_bridge_topology_matches || return 1
fi
}
vc_nm_device_release() {
if [ "${NM_PRESERVE_ACTIVE_BRIDGE:-no}" = yes ]; then
vc_nm_preserve_bridge_ready
return $?
fi
"$IP" link show dev "$BRIDGE" >/dev/null 2>&1 || return 1
"$NMCLI" device set "$BRIDGE" managed yes >/dev/null 2>&1 || return 1
vc_nm_device_expect_managed yes
}
vc_nm_created_profile_remove() {
[ -n "$NM_CREATED_UUID" ] || return 1
vc_nm_profile_resolve || return 1
[ "$NM_PROFILE_UUID" = "$NM_CREATED_UUID" ] || return 1
vc_nm_profile_base_validate || return 1
"$NMCLI" connection delete uuid "$NM_CREATED_UUID" >/dev/null 2>&1 ||
return 1
if vc_nm_profile_resolve; then
return 1
else
vc_nm_delete_readback_status=$?
fi
[ "$vc_nm_delete_readback_status" = 2 ]
}
vc_nm_profile_expect() {
vc_nm_expected=$1
vc_nm_profile_base_validate || return 1
vc_nm_memory_zone=$(vc_nm_profile_field connection.zone) || return 1
case "$vc_nm_expected" in
blocked)
[ -z "$vc_nm_memory_zone" ] || [ "$vc_nm_memory_zone" = -- ] || return 1
;;
blocked-active)
[ -z "$vc_nm_memory_zone" ] || [ "$vc_nm_memory_zone" = -- ] || return 1
vc_nm_profile_active
;;
active)
[ -z "$vc_nm_memory_zone" ] || [ "$vc_nm_memory_zone" = -- ] || return 1
vc_nm_device_expect_managed yes || return 1
vc_nm_profile_active
;;
quarantined)
[ -z "$vc_nm_memory_zone" ] || [ "$vc_nm_memory_zone" = -- ] || return 1
vc_nm_device_expect_managed no
;;
preapply)
[ -z "$vc_nm_memory_zone" ] || [ "$vc_nm_memory_zone" = -- ]
;;
*) return 1 ;;
esac
}
'''.strip()
NETWORKMANAGER_VPC_TARGET_INVENTORY_SH = r'''
vc_nm_inventory_capture() {
vc_nm_capture_mode=$1
vc_nm_capture_field=${2-}
vc_nm_capture_uuid=${3-}
vc_nm_capture_nl='
'
vc_nm_capture_marker=__VC_NM_INVENTORY_CAPTURE_END__
if [ "$vc_nm_capture_mode" = list ]; then
vc_nm_capture_raw=$(
if "$NMCLI" -t --escape no -f UUID connection show 2>/dev/null; then
vc_nm_capture_status=0
else
vc_nm_capture_status=$?
fi
printf '\n%s:%s' "$vc_nm_capture_marker" "$vc_nm_capture_status"
)
else
vc_nm_capture_raw=$(
if "$NMCLI" -t --escape no -g "$vc_nm_capture_field" \
connection show uuid "$vc_nm_capture_uuid" 2>/dev/null; then
vc_nm_capture_status=0
else
vc_nm_capture_status=$?
fi
printf '\n%s:%s' "$vc_nm_capture_marker" "$vc_nm_capture_status"
)
fi
vc_nm_capture_suffix=${vc_nm_capture_raw##*"$vc_nm_capture_nl$vc_nm_capture_marker:"}
case "$vc_nm_capture_suffix" in
''|*[!0-9]*) VC_NM_CAPTURE_ERROR=invalid; return 1 ;;
0) : ;;
*) VC_NM_CAPTURE_ERROR=failed; return 1 ;;
esac
VC_NM_CAPTURE=${vc_nm_capture_raw%"$vc_nm_capture_nl$vc_nm_capture_marker:$vc_nm_capture_suffix"}
VC_NM_CAPTURE_ERROR=invalid
if [ -n "$VC_NM_CAPTURE" ]; then
case "$VC_NM_CAPTURE" in
*"$vc_nm_capture_nl")
VC_NM_CAPTURE=${VC_NM_CAPTURE%"$vc_nm_capture_nl"}
;;
*) return 1 ;;
esac
elif [ "$vc_nm_capture_mode" != list ]; then
# A scalar query must emit exactly one row. nmcli represents an unset
# field as an empty row or "--"; no output is an ambiguous response.
return 1
fi
if [ "$vc_nm_capture_mode" != list ]; then
case "$VC_NM_CAPTURE" in *"$vc_nm_capture_nl"*) return 1 ;; esac
fi
VC_NM_CAPTURE_ERROR=none
}
vc_nm_inventory_uuid_set() {
vc_nm_uuid_rows=$1
VC_NM_UUID_SET=
[ -n "$vc_nm_uuid_rows" ] || return 0
vc_nm_uuid_rows=$(printf '%s\n' "$vc_nm_uuid_rows" | awk '
NF != 1 || $0 !~ /^[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}$/ {bad=1; next}
{key=tolower($0); if (seen[key]++) bad=1; print}
END {if (bad) exit 1}
') || return 1
VC_NM_UUID_SET=$(printf '%s\n' "$vc_nm_uuid_rows" | LC_ALL=C sort) ||
return 1
}
vc_nm_target_inventory() {
NM_TARGET_COUNT=invalid
NM_TARGET_QUERY=invalid
NM_TARGET_UUID=
NM_TARGET_ID=
NM_TARGET_IFACE=
NM_TARGET_STABLE=
NM_TARGET_TYPE=
NM_TARGET_EXACT=no
VC_NM_CAPTURE_ERROR=invalid
if ! vc_nm_inventory_capture list; then
NM_TARGET_QUERY=$VC_NM_CAPTURE_ERROR
return 1
fi
vc_nm_inventory_uuid_set "$VC_NM_CAPTURE" || {
NM_TARGET_QUERY=invalid
return 1
}
vc_nm_inventory_uuids=$VC_NM_UUID_SET
vc_nm_target_count=0
while IFS= read -r vc_nm_inventory_uuid; do
[ -n "$vc_nm_inventory_uuid" ] || continue
vc_nm_inventory_capture scalar connection.id "$vc_nm_inventory_uuid" || {
NM_TARGET_QUERY=$VC_NM_CAPTURE_ERROR
return 1
}
vc_nm_inventory_id=$VC_NM_CAPTURE
[ -n "$vc_nm_inventory_id" ] && [ "$vc_nm_inventory_id" != -- ] || {
NM_TARGET_QUERY=invalid
return 1
}
vc_nm_inventory_capture scalar connection.interface-name \
"$vc_nm_inventory_uuid" || {
NM_TARGET_QUERY=$VC_NM_CAPTURE_ERROR
return 1
}
vc_nm_inventory_iface=$VC_NM_CAPTURE
vc_nm_inventory_capture scalar connection.stable-id \
"$vc_nm_inventory_uuid" || {
NM_TARGET_QUERY=$VC_NM_CAPTURE_ERROR
return 1
}
vc_nm_inventory_stable=$VC_NM_CAPTURE
vc_nm_inventory_capture scalar connection.type "$vc_nm_inventory_uuid" || {
NM_TARGET_QUERY=$VC_NM_CAPTURE_ERROR
return 1
}
vc_nm_inventory_type=$VC_NM_CAPTURE
if [ "$vc_nm_inventory_id" = "$BRIDGE" ] ||
[ "$vc_nm_inventory_iface" = "$BRIDGE" ] ||
[ "$vc_nm_inventory_stable" = "$NM_PROFILE_MARKER" ]; then
vc_nm_target_count=$((vc_nm_target_count + 1))
if [ "$vc_nm_target_count" = 1 ]; then
NM_TARGET_UUID=$vc_nm_inventory_uuid
NM_TARGET_ID=$vc_nm_inventory_id
NM_TARGET_IFACE=$vc_nm_inventory_iface
NM_TARGET_STABLE=$vc_nm_inventory_stable
NM_TARGET_TYPE=$vc_nm_inventory_type
fi
fi
done <<__VC_NM_INVENTORY_UUIDS__
$vc_nm_inventory_uuids
__VC_NM_INVENTORY_UUIDS__
# Re-list after reading every profile. A concurrent add/remove/reload must
# not turn a partial inventory into an absence receipt.
if ! vc_nm_inventory_capture list; then
NM_TARGET_QUERY=$VC_NM_CAPTURE_ERROR
return 1
fi
vc_nm_inventory_uuid_set "$VC_NM_CAPTURE" || {
NM_TARGET_QUERY=invalid
return 1
}
[ "$VC_NM_UUID_SET" = "$vc_nm_inventory_uuids" ] || {
NM_TARGET_QUERY=invalid
return 1
}
NM_TARGET_COUNT=$vc_nm_target_count
NM_TARGET_QUERY=ok
if [ "$NM_TARGET_COUNT" = 1 ] &&
[ "$NM_TARGET_ID" = "$BRIDGE" ] &&
[ "$NM_TARGET_IFACE" = "$BRIDGE" ] &&
[ "$NM_TARGET_STABLE" = "$NM_PROFILE_MARKER" ] &&
[ "$NM_TARGET_TYPE" = bridge ]; then
NM_TARGET_EXACT=yes
fi
}
'''.strip()
IFUPDOWN_VPC_TARGET_INVENTORY_SH = r'''
vc_ifupdown_scan_file() {
vc_ifup_scan_file=$1
[ -f "$vc_ifup_scan_file" ] && [ ! -L "$vc_ifup_scan_file" ] &&
[ -r "$vc_ifup_scan_file" ] || return 1
VC_IFUP_SCAN_COUNT=$(awk -v wanted="$BRIDGE" '
/^[[:space:]]*#/ || NF == 0 {next}
$1 == "iface" {
if (NF < 4) {bad=1; next}
if ($2 == wanted) found++
next
}
$1 == "auto" || $1 ~ /^allow-[^[:space:]]+$/ {
if (NF < 2) {bad=1; next}
for (i=2; i<=NF; i++) {
if ($i ~ /^#/) break
if ($i == wanted) found++
}
}
END {if (bad) exit 1; print found + 0}
' "$vc_ifup_scan_file") || return 1
case "$VC_IFUP_SCAN_COUNT" in ''|*[!0-9]*) return 1 ;; esac
}
vc_ifupdown_inventory_once() {
vc_ifup_root=${IFUP_ROOT:-/etc/network}
vc_ifup_main=${IFUP_MAIN:-$vc_ifup_root/interfaces}
vc_ifup_dir=${IFUP_DIR:-$vc_ifup_root/interfaces.d}
if [ -e "$vc_ifup_root" ] || [ -L "$vc_ifup_root" ]; then
[ -d "$vc_ifup_root" ] && [ ! -L "$vc_ifup_root" ] &&
[ -r "$vc_ifup_root" ] && [ -x "$vc_ifup_root" ] || return 1
fi
vc_ifup_total=0
if [ -e "$vc_ifup_main" ] || [ -L "$vc_ifup_main" ]; then
vc_ifupdown_scan_file "$vc_ifup_main" || return 1
vc_ifup_total=$((vc_ifup_total + VC_IFUP_SCAN_COUNT))
fi
if [ -e "$vc_ifup_dir" ] || [ -L "$vc_ifup_dir" ]; then
[ -d "$vc_ifup_dir" ] && [ ! -L "$vc_ifup_dir" ] &&
[ -r "$vc_ifup_dir" ] && [ -x "$vc_ifup_dir" ] || return 1
for vc_ifup_candidate in "$vc_ifup_dir"/*; do
if [ ! -e "$vc_ifup_candidate" ] && [ ! -L "$vc_ifup_candidate" ]; then
continue
fi
vc_ifupdown_scan_file "$vc_ifup_candidate" || return 1
vc_ifup_total=$((vc_ifup_total + VC_IFUP_SCAN_COUNT))
done
fi
VC_IFUP_TARGET_COUNT=$vc_ifup_total
}
vc_ifupdown_target_inventory() {
IFUP_TARGET_COUNT=invalid
IFUP_TARGET_QUERY=invalid
vc_ifupdown_inventory_once || return 1
vc_ifup_first_count=$VC_IFUP_TARGET_COUNT
vc_ifupdown_inventory_once || return 1
[ "$VC_IFUP_TARGET_COUNT" = "$vc_ifup_first_count" ] || return 1
IFUP_TARGET_COUNT=$VC_IFUP_TARGET_COUNT
IFUP_TARGET_QUERY=ok
}
'''.strip()
class VpcDataplaneError(Exception):
"""Safe, typed error at the VPC host boundary."""
def __init__(self, message: str, status: int = 400,
error_code: str = "vpc_dataplane_invalid") -> None:
super().__init__(message)
self.status = status
self.error_code = error_code
def _fabric_plan_digest(value: Any) -> str:
if not isinstance(value, str) or _DIGEST.fullmatch(value) is None:
raise VpcDataplaneError(
"Fabric 计划摘要无效", 409, "vpc_fabric_plan_digest_invalid")
return value
@dataclass(frozen=True)
class VpcFabricPeer:
host_id: str
public_key: str
overlay_ip: str
# Tenant /32s from every panel VPC sharing this Fabric peer. WireGuard
# has one AllowedIPs list per peer, so a VPC must merge this typed set
# instead of replacing another active VPC's entries.
tenant_allowed_ips: tuple[str, ...] = ()
@property
def overlay_cidr(self) -> str:
return f"{self.overlay_ip}/32"
@dataclass(frozen=True)
class VpcDataplaneResources:
token: str
bridge: str
nft_table: str
route_table: int
rule_priority: int
owner_file: str
state_file: str
lock_file: str
transaction_dir: str
@dataclass(frozen=True)
class VpcDataplanePlan:
vpc: VpcPlan
host_id: str
members: tuple[VpcMember, ...]
peers: tuple[VpcFabricPeer, ...]
resources: VpcDataplaneResources
# Engine-authenticated digest of the complete Fabric plan. This is not
# part of the tenant VPC DTO: host transactions use it to prove that the
# shared Fabric quarantine table still belongs to the exact Fabric
# revision admitted by the coordinator.
fabric_plan_digest: str
# The authenticated full Fabric peer set, excluding this host. It is
# separate from ``peers`` because one VPC can use only a subset of the
# peers present on a shared Fabric.
fabric_peer_keys: frozenset[str] | None = None
# Authoritative public-key -> overlay /32 closure from the complete
# engine-validated Fabric plan. The key-only field remains for legacy
# callers, but only this mapping can prove and converge cryptokey routing
# for peers outside this VPC's participant set.
fabric_peer_overlays: frozenset[tuple[str, str]] | None = None
def __post_init__(self) -> None:
_fabric_plan_digest(self.fabric_plan_digest)
@property
def desired_revision(self) -> int:
return self.vpc.desired_revision
@property
def desired_digest(self) -> str:
return self.vpc.desired_digest
@property
def local_members(self) -> tuple[VpcMember, ...]:
return tuple(m for m in self.members if m.host_id == self.host_id)
@property
def remote_members(self) -> tuple[VpcMember, ...]:
return tuple(m for m in self.members if m.host_id != self.host_id)
@dataclass(frozen=True)
class VpcDataplaneObservation:
vpc_id: str
host_id: str
desired_revision: int
observed_revision: int
desired_digest: str
observed_digest: str | None
state: str
bridge: str
bridge_owner: str | None
member_nics: tuple[dict[str, Any], ...]
routes: tuple[dict[str, Any], ...]
wg_allowed_ips: tuple[dict[str, Any], ...]
nft_table: str | None
acl_revision: int | None
anti_spoof: bool
rp_filter: int | None
proxy_arp: int | None
proxy_arp_pvlan: int | None
proxy_ips: tuple[str, ...]
config_verified: bool
config_observed_revision: int
config_observed_digest: str | None
readback: str
rollback: str
reason_code: str | None = None
removed: bool = False
def as_dict(self) -> dict[str, Any]:
return {
"vpc_id": self.vpc_id, "host_id": self.host_id,
"desired_revision": self.desired_revision,
"observed_revision": self.observed_revision,
"desired_digest": self.desired_digest,
"observed_digest": self.observed_digest,
"state": self.state, "bridge": self.bridge,
"bridge_owner": self.bridge_owner,
"member_nics": [dict(x) for x in self.member_nics],
"routes": [dict(x) for x in self.routes],
"wg_allowed_ips": [dict(x) for x in self.wg_allowed_ips],
"nft_table": self.nft_table, "acl_revision": self.acl_revision,
"anti_spoof": self.anti_spoof, "rp_filter": self.rp_filter,
"proxy_arp": self.proxy_arp,
"proxy_arp_pvlan": self.proxy_arp_pvlan,
"proxy_ips": list(self.proxy_ips),
"config_verified": self.config_verified,
"config_observed_revision": self.config_observed_revision,
"config_observed_digest": self.config_observed_digest,
"readback": self.readback,
"rollback": self.rollback, "reason_code": self.reason_code,
"removed": self.removed,
}
def _id(value: Any, label: str = "标识") -> str:
if not isinstance(value, str) or _ID.fullmatch(value) is None:
raise VpcDataplaneError(f"{label}无效")
return value
def _host(value: Any) -> str:
if not isinstance(value, str) or _HOST.fullmatch(value) is None:
raise VpcDataplaneError("宿主机标识无效")
return value
def _key(value: Any) -> str:
if not isinstance(value, str) or _KEY.fullmatch(value) is None:
raise VpcDataplaneError("WireGuard 公钥无效")
try:
raw = base64.b64decode(value, validate=True)
except Exception as exc:
raise VpcDataplaneError("WireGuard 公钥无效") from exc
if len(raw) != 32 or not any(raw):
raise VpcDataplaneError("WireGuard 公钥无效")
return value
def _fabric_overlay_cidr(value: Any) -> str:
try:
interface = ipaddress.ip_interface(value)
except (TypeError, ValueError) as exc:
raise VpcDataplaneError("Fabric peer overlay 地址无效", 409,
"vpc_fabric_peer_set_invalid") from exc
fabric_overlay = ipaddress.IPv4Network("100.64.0.0/10")
if (not isinstance(interface, ipaddress.IPv4Interface)
or interface.network.prefixlen != 32
or interface.ip not in fabric_overlay
or interface.ip.is_loopback
or interface.ip.is_link_local):
raise VpcDataplaneError("Fabric peer overlay 地址无效", 409,
"vpc_fabric_peer_set_invalid")
return f"{interface.ip}/32"
def _mac(value: Any, label: str = "私网 MAC") -> str:
if not isinstance(value, str) or _MAC.fullmatch(value) is None:
raise VpcDataplaneError(f"{label}无效")
octets = [int(x, 16) for x in value.split(":")]
if octets[0] & 1 or not any(octets):
raise VpcDataplaneError(f"{label}无效")
return value.lower()
def derive_private_mac(vpc_id: str, member_id: str) -> str:
"""Stable locally-administered unicast MAC for legacy plans without MAC."""
digest = hashlib.sha256(f"virt-cloud:vpc-private0:{vpc_id}:{member_id}".encode()).digest()
first = (digest[0] & 0xfc) | 0x02
return ":".join(f"{x:02x}" for x in bytes([first]) + digest[1:6])
def private_guest_routes(plan: VpcPlan, member: VpcMember) -> tuple[str, ...]:
"""Return the stable on-link VPC prefix installed inside one guest.
Host ACL, nftables and Fabric changes deliberately do not participate in
this projection. The host keeps exact member/service /32 routes, proxy
neighbours and destination sets; the guest needs only one link-scope VPC
prefix. Member and service-route churn therefore cannot force every
running sibling to consume a new boot-only NoCloud generation.
"""
try:
network = ipaddress.ip_network(plan.cidr, strict=True)
private_ip = ipaddress.ip_address(member.private_ip)
except (TypeError, ValueError) as exc:
raise VpcDataplaneError("VPC 来宾私网路由无效") from exc
if (not isinstance(network, ipaddress.IPv4Network)
or not 16 <= network.prefixlen <= 24
or not isinstance(private_ip, ipaddress.IPv4Address)
or private_ip not in network
or not any(network.subnet_of(block) for block in (
ipaddress.IPv4Network("10.0.0.0/8"),
ipaddress.IPv4Network("172.16.0.0/12"),
ipaddress.IPv4Network("192.168.0.0/16"),
))):
raise VpcDataplaneError("VPC 来宾私网路由无效")
return (str(network),)
def private_guest_network_generation(
member: VpcMember,
private_routes: Sequence[str],
) -> str:
"""Hash only the private network state that a guest consumes at boot."""
private_mac = _mac(member.private_mac)
try:
private_ip = ipaddress.ip_interface(f"{member.private_ip}/32")
except (TypeError, ValueError) as exc:
raise VpcDataplaneError("VPC 成员私网 IP 无效") from exc
if not isinstance(private_ip, ipaddress.IPv4Interface):
raise VpcDataplaneError("VPC 成员私网 IP 无效")
routes: list[ipaddress.IPv4Network] = []
if (not isinstance(private_routes, Sequence)
or isinstance(private_routes, (str, bytes))
or len(private_routes) != 1):
raise VpcDataplaneError("VPC 来宾私网路由无效")
for value in private_routes:
if not isinstance(value, str):
raise VpcDataplaneError("VPC 来宾私网路由无效")
try:
route = ipaddress.ip_network(value, strict=True)
except ValueError as exc:
raise VpcDataplaneError("VPC 来宾私网路由无效") from exc
if (not isinstance(route, ipaddress.IPv4Network)
or not 16 <= route.prefixlen <= 24
or private_ip.ip not in route
or not any(route.subnet_of(block) for block in (
ipaddress.IPv4Network("10.0.0.0/8"),
ipaddress.IPv4Network("172.16.0.0/12"),
ipaddress.IPv4Network("192.168.0.0/16"),
))):
raise VpcDataplaneError(
"VPC 来宾私网路由必须是包含成员地址的 IPv4 /16 到 /24")
routes.append(route)
payload = "\n".join((
"virt-cloud-private-guest-network-v2",
private_mac,
str(private_ip),
str(routes[0]),
)).encode("ascii")
return hashlib.sha256(payload).hexdigest()
def resource_names(vpc_id: str) -> VpcDataplaneResources:
vpc_id = _id(vpc_id, "VPC")
digest = hashlib.sha256(vpc_id.encode()).hexdigest()
token = digest[:11]
span = ROUTE_TABLE_MAX - ROUTE_TABLE_MIN + 1
# Each VPC owns two policy rules (private bridge and Fabric ingress).
# Allocate an aligned pair so ``base + 1`` can never be another VPC's
# base slot. The coordinator still checks collisions across records and
# the host transaction rejects a foreign rule already occupying either
# slot, but a single-slot hash would make those checks order-dependent.
rule_span = (RULE_PRIORITY_MAX - RULE_PRIORITY_MIN + 1) // 2
rule_base = RULE_PRIORITY_MIN + 2 * (int(digest[20:28], 16) % rule_span)
return VpcDataplaneResources(
token=token, bridge=BRIDGE_PREFIX + token,
nft_table=NFT_PREFIX + digest[:12],
route_table=ROUTE_TABLE_MIN + int(digest[12:20], 16) % span,
rule_priority=rule_base,
owner_file=f"/etc/virt-cloud/vpc-dataplane/{token}.owner",
state_file=f"/etc/virt-cloud/vpc-dataplane/{token}.state",
lock_file=f"/run/virt-cloud/vpc-{token}.lock",
transaction_dir=f"/etc/virt-cloud/.vpc-dataplane-{token}.txn",
)
def _expected_peer_tenant_ips(vpc: VpcPlan) -> dict[str, frozenset[str]]:
"""Return the exact /32 set a peer may route for this VPC.
The WireGuard peer list is a cryptokey-routing boundary. RFC1918 alone
is not an ownership check: an otherwise valid private /32 from another
VPC would create a cross-tenant route. The coordinator may pass a
trusted aggregate for a Fabric shared by several VPCs; the default here
covers a standalone plan and keeps direct parser callers fail-closed.
"""
members = {item.member_id: item for item in vpc.members}
allowed: dict[str, set[str]] = {host: set() for host in vpc.host_ids}
for item in vpc.members:
allowed.setdefault(item.host_id, set()).add(f"{item.private_ip}/32")
for route in vpc.routes:
target = (members.get(route.next_hop_member_id)
if route.next_hop_member_id is not None else None)
target_host = target.host_id if target is not None else route.next_hop_host_id
if target_host is not None:
allowed.setdefault(target_host, set()).add(route.destination_cidr)
return {host: frozenset(values) for host, values in allowed.items()}
def _parse_peers(
raw: Any,
host_ids: frozenset[str],
local: str,
expected_tenant_ips: Mapping[str, frozenset[str]] | None = None,
) -> tuple[VpcFabricPeer, ...]:
if raw is None:
raw = []
if not isinstance(raw, (list, tuple)) or len(raw) > 64:
raise VpcDataplaneError("Fabric peer 列表无效")
peers: list[VpcFabricPeer] = []
seen_hosts: set[str] = set()
seen_keys: set[str] = set()
seen_ips: set[str] = set()
seen_tenant_ips: set[str] = set()
for item in raw:
if not isinstance(item, Mapping) or set(item) - {"host_id", "public_key", "overlay_ip", "overlay_ip_cidr", "tenant_allowed_ips"}:
raise VpcDataplaneError("Fabric peer 字段无效")
host_id = _host(item.get("host_id"))
if host_id == local or host_id not in host_ids or host_id in seen_hosts:
raise VpcDataplaneError("Fabric peer 宿主机无效或重复")
public_key = _key(item.get("public_key"))
if public_key in seen_keys:
raise VpcDataplaneError("Fabric peer 公钥重复")
raw_ip = item.get("overlay_ip", item.get("overlay_ip_cidr"))
try:
interface = ipaddress.ip_interface(raw_ip)
except (ValueError, TypeError) as exc:
raise VpcDataplaneError("Fabric peer overlay 地址无效") from exc
fabric_overlay = ipaddress.IPv4Network("100.64.0.0/10")
if (not isinstance(interface, ipaddress.IPv4Interface)
or interface.network.prefixlen != 32
or interface.ip not in fabric_overlay
or interface.ip.is_loopback
or interface.ip.is_link_local):
raise VpcDataplaneError("Fabric peer overlay 地址无效")
overlay_ip = str(interface.ip)
if overlay_ip in seen_ips:
raise VpcDataplaneError("Fabric peer overlay 地址重复")
raw_tenant = item.get("tenant_allowed_ips", ())
if not isinstance(raw_tenant, (list, tuple)) or len(raw_tenant) > 4096:
raise VpcDataplaneError("Fabric peer tenant AllowedIPs 无效")
tenant: set[str] = set()
for raw_ip in raw_tenant:
if not isinstance(raw_ip, str) or not raw_ip.endswith("/32"):
raise VpcDataplaneError("Fabric peer tenant AllowedIPs 必须是 /32")
try:
network = ipaddress.ip_network(raw_ip, strict=True)
except ValueError as exc:
raise VpcDataplaneError("Fabric peer tenant AllowedIPs 无效") from exc
if network.prefixlen != 32:
raise VpcDataplaneError("Fabric peer tenant AllowedIPs 必须是 /32")
# Tenant routes are panel-private RFC1918 /32s. In particular,
# never accept a public prefix or a Fabric overlay address from
# an otherwise authenticated payload: that would turn a peer's
# cryptokey routing table into a host-wide egress route.
private_ranges = (
ipaddress.IPv4Network("10.0.0.0/8"),
ipaddress.IPv4Network("172.16.0.0/12"),
ipaddress.IPv4Network("192.168.0.0/16"),
)
if not any(network.subnet_of(item) for item in private_ranges):
raise VpcDataplaneError("Fabric peer tenant AllowedIPs 必须位于 RFC1918 私网")
tenant_ip = str(network)
if tenant_ip in seen_tenant_ips:
raise VpcDataplaneError("Fabric peer tenant AllowedIPs 重复", 409,
"vpc_tenant_allowed_ip_conflict")
seen_tenant_ips.add(tenant_ip)
tenant.add(tenant_ip)
if expected_tenant_ips is not None:
expected = set(expected_tenant_ips.get(host_id, frozenset()))
if tenant != expected:
raise VpcDataplaneError(
"Fabric peer tenant AllowedIPs 超出 VPC 所有权闭集", 409,
"vpc_tenant_allowed_ip_unowned")
seen_hosts.add(host_id); seen_keys.add(public_key); seen_ips.add(overlay_ip)
peers.append(VpcFabricPeer(host_id, public_key, overlay_ip,
tuple(sorted(tenant))))
return tuple(sorted(peers, key=lambda p: p.host_id))
def parse_vpc_dataplane_plan(
value: Mapping[str, Any], host_id: str, *,
fabric_peers: Sequence[Mapping[str, Any]] | None = None,
trusted_tenant_allowed_ips: Mapping[str, Sequence[str]] | None = None,
fabric_peer_keys: Sequence[str] | None = None,
fabric_peer_overlays: Mapping[str, str] | None = None,
fabric_plan_digest: str | None = None,
) -> VpcDataplanePlan:
"""Parse existing VPC DTO plus an optional engine-only Fabric snapshot."""
if not isinstance(value, Mapping):
raise VpcDataplaneError("VPC 数据面计划必须是对象")
host_id = _host(host_id)
fabric_plan_digest = _fabric_plan_digest(fabric_plan_digest)
payload = dict(value)
embedded = payload.pop("fabric_peers", None)
if embedded is None:
embedded = payload.pop("peers", None)
else:
payload.pop("peers", None)
try:
vpc = parse_vpc_plan(payload)
except VpcError as exc:
raise VpcDataplaneError(str(exc), exc.status, exc.error_code) from exc
if host_id not in vpc.host_ids:
raise VpcDataplaneError("VPC 宿主机不在参与列表中", 409, "vpc_host_not_member")
members: list[VpcMember] = []
seen_macs: set[str] = set()
for member in vpc.members:
private_mac = _mac(member.private_mac) if member.private_mac else derive_private_mac(vpc.vpc_id, member.member_id)
if private_mac in seen_macs:
raise VpcDataplaneError("VPC 成员私网 MAC 重复", 409, "vpc_private_mac_conflict")
seen_macs.add(private_mac)
members.append(VpcMember(member.member_id, member.instance_uuid,
member.host_id, member.private_ip,
member.subnet_id, private_mac))
if trusted_tenant_allowed_ips is None:
expected_tenant_ips = _expected_peer_tenant_ips(vpc)
else:
if not isinstance(trusted_tenant_allowed_ips, Mapping):
raise VpcDataplaneError("Fabric peer tenant 所有权闭集无效")
expected_tenant_ips = {}
remote_hosts = vpc.host_ids - {host_id}
if set(trusted_tenant_allowed_ips) - remote_hosts:
raise VpcDataplaneError("Fabric peer tenant 所有权宿主机无效")
for peer_host, values in trusted_tenant_allowed_ips.items():
if not isinstance(peer_host, str) or peer_host not in remote_hosts:
raise VpcDataplaneError("Fabric peer tenant 所有权宿主机无效")
if not isinstance(values, (list, tuple, set, frozenset)):
raise VpcDataplaneError("Fabric peer tenant 所有权闭集无效")
canonical: set[str] = set()
for value in values:
if not isinstance(value, str):
raise VpcDataplaneError("Fabric peer tenant 所有权闭集无效")
try:
network = ipaddress.ip_network(value, strict=True)
except ValueError as exc:
raise VpcDataplaneError("Fabric peer tenant 所有权闭集无效") from exc
if (not isinstance(network, ipaddress.IPv4Network)
or network.prefixlen != 32
or not any(network.subnet_of(block) for block in (
ipaddress.IPv4Network("10.0.0.0/8"),
ipaddress.IPv4Network("172.16.0.0/12"),
ipaddress.IPv4Network("192.168.0.0/16")))):
raise VpcDataplaneError("Fabric peer tenant 所有权闭集无效")
canonical.add(str(network))
expected_tenant_ips[peer_host] = frozenset(canonical)
peers = _parse_peers(
embedded if fabric_peers is None else fabric_peers,
vpc.host_ids, host_id, expected_tenant_ips)
remote_hosts = {m.host_id for m in members if m.host_id != host_id}
if peers and remote_hosts - {p.host_id for p in peers}:
raise VpcDataplaneError("跨宿主机成员缺少 Fabric peer", 409, "vpc_fabric_peer_missing")
authoritative_keys: frozenset[str] | None = None
if fabric_peer_keys is not None:
if not isinstance(fabric_peer_keys, (list, tuple, set, frozenset)):
raise VpcDataplaneError("Fabric peer 公钥闭集无效", 409,
"vpc_fabric_peer_set_invalid")
if len(fabric_peer_keys) > 64:
raise VpcDataplaneError("Fabric peer 公钥闭集过大", 409,
"vpc_fabric_peer_set_invalid")
canonical_keys: set[str] = set()
for raw_key in fabric_peer_keys:
key = _key(raw_key)
if key in canonical_keys:
raise VpcDataplaneError("Fabric peer 公钥重复", 409,
"vpc_fabric_peer_set_invalid")
canonical_keys.add(key)
if not {peer.public_key for peer in peers}.issubset(canonical_keys):
raise VpcDataplaneError("VPC peer 不属于 Fabric 公钥闭集", 409,
"vpc_fabric_peer_set_invalid")
authoritative_keys = frozenset(canonical_keys)
authoritative_overlays: frozenset[tuple[str, str]] | None = None
if fabric_peer_overlays is not None:
if (not isinstance(fabric_peer_overlays, Mapping)
or len(fabric_peer_overlays) > 64):
raise VpcDataplaneError("Fabric peer overlay 闭集无效", 409,
"vpc_fabric_peer_set_invalid")
canonical_overlays: dict[str, str] = {}
seen_overlay_cidrs: set[str] = set()
for raw_key, raw_overlay in fabric_peer_overlays.items():
key = _key(raw_key)
overlay_cidr = _fabric_overlay_cidr(raw_overlay)
if key in canonical_overlays or overlay_cidr in seen_overlay_cidrs:
raise VpcDataplaneError("Fabric peer overlay 闭集重复", 409,
"vpc_fabric_peer_set_invalid")
canonical_overlays[key] = overlay_cidr
seen_overlay_cidrs.add(overlay_cidr)
overlay_keys = frozenset(canonical_overlays)
if authoritative_keys is not None and overlay_keys != authoritative_keys:
raise VpcDataplaneError("Fabric peer 公钥与 overlay 闭集不一致", 409,
"vpc_fabric_peer_set_invalid")
for peer in peers:
if canonical_overlays.get(peer.public_key) != peer.overlay_cidr:
raise VpcDataplaneError("VPC peer overlay 不属于 Fabric 闭集", 409,
"vpc_fabric_peer_set_invalid")
authoritative_keys = overlay_keys
authoritative_overlays = frozenset(canonical_overlays.items())
return VpcDataplanePlan(
vpc, host_id, tuple(members), peers, resource_names(vpc.vpc_id),
fabric_plan_digest, authoritative_keys, authoritative_overlays)
def _authoritative_fabric_peer_overlays(
plan: VpcDataplanePlan,
) -> dict[str, str] | None:
"""Validate and materialize the engine-only full Fabric peer closure."""
if plan.fabric_peer_overlays is None:
return None
mapping: dict[str, str] = {}
seen_overlays: set[str] = set()
for row in plan.fabric_peer_overlays:
if not isinstance(row, tuple) or len(row) != 2:
raise VpcDataplaneError("Fabric peer overlay 闭集无效", 409,
"vpc_fabric_peer_set_invalid")
key = _key(row[0])
overlay = _fabric_overlay_cidr(row[1])
if key in mapping or overlay in seen_overlays:
raise VpcDataplaneError("Fabric peer overlay 闭集重复", 409,
"vpc_fabric_peer_set_invalid")
mapping[key] = overlay
seen_overlays.add(overlay)
if (plan.fabric_peer_keys is None
or frozenset(mapping) != plan.fabric_peer_keys):
raise VpcDataplaneError("Fabric peer 公钥与 overlay 闭集不一致", 409,
"vpc_fabric_peer_set_invalid")
for peer in plan.peers:
if mapping.get(peer.public_key) != peer.overlay_cidr:
raise VpcDataplaneError("VPC peer overlay 不属于 Fabric 闭集", 409,
"vpc_fabric_peer_set_invalid")
return dict(sorted(mapping.items()))
def _routes(plan: VpcDataplanePlan) -> tuple[dict[str, Any], ...]:
members = {m.member_id: m for m in plan.members}
result: list[dict[str, Any]] = []
seen: set[str] = set()
for member in plan.members:
destination = f"{member.private_ip}/32"
seen.add(destination)
result.append({"destination_cidr": destination,
"device": plan.resources.bridge if member.host_id == plan.host_id else FABRIC_INTERFACE,
"table": plan.resources.route_table,
"owner": DATAPLANE_OWNER_MARKER,
"metric": 100,
"via": None,
"member_id": member.member_id})
for route in plan.vpc.routes:
destination = route.destination_cidr
network = ipaddress.ip_network(destination)
if network.prefixlen != 32:
raise VpcDataplaneError(
"VPC 数据面只支持成员精确 /32 路由", 409,
"vpc_route_not_exact")
if any(ipaddress.IPv4Address(m.private_ip) in network
and destination != f"{m.private_ip}/32" for m in plan.members):
raise VpcDataplaneError("VPC route 不能覆盖成员精确 /32", 409, "vpc_route_member_overlap")
if destination in seen:
raise VpcDataplaneError("VPC route 与成员地址重复", 409, "vpc_route_conflict")
next_host = route.next_hop_host_id
if route.next_hop_member_id is not None:
target = members.get(route.next_hop_member_id)
if target is None:
raise VpcDataplaneError("VPC route next hop member 不存在", 409)
next_host = target.host_id
if next_host is None:
raise VpcDataplaneError("VPC route next hop 缺失")
target_members = [item for item in plan.members
if item.host_id == next_host]
if not target_members:
raise VpcDataplaneError("VPC route next hop 宿主机没有成员", 409,
"vpc_route_next_hop_missing")
# A host-only next hop is ambiguous when several VMs share that host.
# Require an explicit member in that case; using a bare bridge route
# would ARP for the service address and could leak traffic to an
# unrelated guest.
if route.next_hop_member_id is None and len(target_members) != 1:
raise VpcDataplaneError("VPC route 宿主机 next hop 不明确", 409,
"vpc_route_next_hop_ambiguous")
target = (members.get(route.next_hop_member_id)
if route.next_hop_member_id is not None else target_members[0])
if target is None:
raise VpcDataplaneError("VPC route next hop member 不存在", 409,
"vpc_route_next_hop_missing")
is_local = next_host == plan.host_id
if not is_local and next_host not in {peer.host_id for peer in plan.peers}:
raise VpcDataplaneError("VPC route next hop 缺少 Fabric peer", 409,
"vpc_fabric_peer_missing")
seen.add(destination)
result.append({"destination_cidr": destination,
"device": plan.resources.bridge if is_local else FABRIC_INTERFACE,
"table": plan.resources.route_table,
"owner": DATAPLANE_OWNER_MARKER,
# A local host must forward the service /32 through
# the selected VM. Across the overlay the destination
# itself is a WireGuard AllowedIP, so no `via` is used.
"via": target.private_ip if is_local else None,
"metric": route.metric,
"route_id": route.route_id})
return tuple(result)
def _q(value: Any) -> str:
return shlex.quote(str(value))
def _nft_proto(acl: Any) -> str:
if acl.protocol == "any":
return ""
if acl.protocol == "icmp":
return " ip protocol icmp"
suffix = f" ip protocol {acl.protocol}"
if acl.port_from is not None:
ports = str(acl.port_from) if acl.port_from == acl.port_to else f"{acl.port_from}-{acl.port_to}"
suffix += f" {acl.protocol} dport {ports}"
return suffix
def _nft_marker(plan: VpcDataplanePlan) -> str:
"""Return a bounded ownership marker safe for nft comment userdata.
libnftnl limits rule comments to 128 bytes. VPC/host identifiers are
intentionally allowed to be long, so embedding them and the full digest
in every rule would make otherwise valid plans fail at ``nft -c``. The
state file keeps the full identifiers; comments only need a collision
resistant, deterministic token for ownership/readback.
"""
seed = (f"{plan.vpc.vpc_id}:{plan.host_id}:{plan.desired_revision}:"
f"{plan.desired_digest}").encode()
token = hashlib.sha256(seed).hexdigest()[:16]
return f"{DATAPLANE_OWNER_MARKER}:{plan.resources.token}:{token}"
def render_vpc_nft_rules(plan: VpcDataplanePlan) -> str:
"""Render scoped VPC ACL/anti-spoof rules for routed and L2 paths.
The base chains use ``policy accept`` deliberately. A VPC table must not
become a host-wide firewall that drops public bridges or unrelated Fabric
traffic; only packets carrying this VPC's member addresses are subject to
its default deny. A separate bridge-family table covers same-host VM to
VM forwarding and ARP, which do not reliably traverse an inet forward
hook.
"""
r = plan.resources
marker = _nft_marker(plan)
dataplane_routes = _routes(plan)
member_cidrs = {f"{m.private_ip}/32" for m in plan.members}
local_member_cidrs = {f"{m.private_ip}/32" for m in plan.local_members}
# Custom VPC routes are exact /32s too. Keep source sets restricted to
# actual member addresses, while destination sets include routed service
# addresses; otherwise the isolation rules would silently drop a valid
# route before the ACL/default-deny decision.
destination_cidrs = member_cidrs | {
str(item["destination_cidr"]) for item in dataplane_routes
}
# A packet entering a local tap may be routed either to another local
# tap or out through wg-fabric0 to a remote member/service /32. The
# bridge-family unknown-destination guard therefore needs the complete
# VPC destination set; restricting it to local routes would drop every
# same-VPC cross-host flow before the inet routing policy can run.
local_destination_cidrs = set(destination_cidrs)
# Use explicit proxy-neighbour entries for the host's exact /32 routes to
# guest addresses.
# The generic kernel proxy responder stays disabled, so a nftables reload
# or a short ordering window cannot make the host answer for arbitrary
# addresses. proxy_arp_pvlan is enabled only for the same-bridge
# next-hop case and is still bounded by the exact neighbour set and ARP
# family output chain below.
proxy_ips = sorted(
str(ipaddress.ip_network(item["destination_cidr"], strict=True).network_address)
for item in dataplane_routes)
members = ", ".join(sorted(member_cidrs))
local_members = ", ".join(sorted(local_member_cidrs))
destinations = ", ".join(sorted(destination_cidrs))
local_destinations = ", ".join(sorted(local_destination_cidrs))
proxy_destinations = ", ".join(proxy_ips)
local_sources = ", ".join(
f"{m.private_mac} . {m.private_ip}" for m in plan.local_members)
def nft_set(name: str, type_name: str, elements: str, *, interval: bool = False) -> str:
flags = " flags interval;" if interval else ""
body = f" set {name} {{ type {type_name};{flags}"
if elements:
body += f" elements = {{ {elements} }};"
return body + " }"
lines = [
f"# {DATAPLANE_OWNER_MARKER}", f"# vpc-token={r.token}",
f"# vpc={plan.vpc.vpc_id} revision={plan.desired_revision} digest={plan.desired_digest}",
f"table inet {r.nft_table} {{",
nft_set("member_ips", "ipv4_addr", members, interval=True),
nft_set("local_member_ips", "ipv4_addr", local_members, interval=True),
nft_set("vpc_dest_ips", "ipv4_addr", destinations, interval=True),
nft_set("local_member_sources", "ether_addr . ipv4_addr", local_sources),
" chain forward { type filter hook forward priority -190; policy accept;",
f' iifname "{r.bridge}" ip saddr != @local_member_ips drop comment "{marker}:anti-spoof-local-ip"',
f' iifname "{r.bridge}" ip6 saddr ::/0 drop comment "{marker}:ipv6-drop-bridge"',
# A source address is not an ingress identity. A packet arriving on
# the public/management side must never be able to spoof a member /32
# and use the VPC policy route to reach a private destination. Close
# both egress legs by interface before the source-address checks.
# The whole VPC CIDR is guarded, including currently unallocated
# addresses. A missing /32 route must therefore fail closed instead
# of falling through to the host's public/default route.
f' oifname "{r.bridge}" ip daddr {plan.vpc.cidr} iifname != "{r.bridge}" iifname != "{FABRIC_INTERFACE}" drop comment "{marker}:anti-spoof-public-to-vpc-bridge"',
f' oifname "{FABRIC_INTERFACE}" ip daddr {plan.vpc.cidr} iifname != "{r.bridge}" iifname != "{FABRIC_INTERFACE}" drop comment "{marker}:anti-spoof-public-to-vpc-fabric"',
f' iifname "{r.bridge}" ip daddr {plan.vpc.cidr} oifname != "{r.bridge}" oifname != "{FABRIC_INTERFACE}" drop comment "{marker}:anti-spoof-bridge-vpc-to-foreign"',
f' iifname "{FABRIC_INTERFACE}" ip daddr {plan.vpc.cidr} oifname != "{r.bridge}" oifname != "{FABRIC_INTERFACE}" drop comment "{marker}:anti-spoof-fabric-vpc-to-foreign"',
f' oifname "{r.bridge}" ip daddr @vpc_dest_ips iifname != "{r.bridge}" iifname != "{FABRIC_INTERFACE}" drop comment "{marker}:anti-spoof-public-to-bridge"',
f' oifname "{FABRIC_INTERFACE}" ip daddr @vpc_dest_ips iifname != "{r.bridge}" iifname != "{FABRIC_INTERFACE}" drop comment "{marker}:anti-spoof-public-to-fabric"',
# A VPC destination must also leave through one of the two owned legs.
# If a policy route disappears or a foreign/default route wins, the
# packet must be dropped rather than sent to a public or management
# interface. This is deliberately independent of source-IP checks.
f' iifname "{r.bridge}" ip daddr @vpc_dest_ips oifname != "{r.bridge}" oifname != "{FABRIC_INTERFACE}" drop comment "{marker}:anti-spoof-bridge-to-foreign"',
f' iifname "{FABRIC_INTERFACE}" ip daddr @vpc_dest_ips oifname != "{r.bridge}" oifname != "{FABRIC_INTERFACE}" drop comment "{marker}:anti-spoof-fabric-to-foreign"',
f' oifname "{r.bridge}" ip daddr @vpc_dest_ips ip saddr != @member_ips drop comment "{marker}:anti-spoof-egress"',
f' iifname "{FABRIC_INTERFACE}" ip daddr @vpc_dest_ips ip saddr != @member_ips drop comment "{marker}:anti-spoof-wg-ingress"',
f' oifname "{FABRIC_INTERFACE}" ip saddr @member_ips ip daddr != @vpc_dest_ips drop comment "{marker}:anti-spoof-wg-egress"',
f' iifname "{FABRIC_INTERFACE}" ip daddr @vpc_dest_ips ip saddr != @member_ips drop comment "{marker}:anti-spoof-wg-source"',
f' iifname "{r.bridge}" ip saddr @local_member_ips ip daddr != @vpc_dest_ips drop comment "{marker}:member-egress-isolation"',
f' iifname "{FABRIC_INTERFACE}" ip saddr @member_ips ip daddr != @vpc_dest_ips drop comment "{marker}:member-fabric-isolation"',
]
# ACL priority is first-match order. Direction is relative to the VPC
# boundary: ingress is wg -> private bridge; egress is bridge -> wg.
# Same-host members have no wg leg, so their source/destination tuple is
# enforced on the bridge path as well.
for acl in plan.vpc.acls:
action = "accept" if acl.action == "allow" else "drop"
clause = (f" ip saddr {acl.source_cidr} ip daddr {acl.destination_cidr}"
f"{_nft_proto(acl)} {action} comment \"{marker}:acl:{acl.acl_id}\"")
if acl.direction == "ingress":
lines.append(f' iifname "{FABRIC_INTERFACE}" oifname "{r.bridge}"{clause}')
else:
lines.append(f' iifname "{r.bridge}" oifname "{FABRIC_INTERFACE}"{clause}')
lines.append(f' iifname "{r.bridge}" oifname "{r.bridge}"{clause}')
# Scope default-deny to member-to-member VPC traffic. The following
# interface-wide accepts preserve unrelated host/public forwarding.
lines += [
f' iifname "{FABRIC_INTERFACE}" oifname "{r.bridge}" ip saddr @member_ips ip daddr @vpc_dest_ips counter drop comment "{marker}:default-deny"',
f' iifname "{r.bridge}" oifname "{FABRIC_INTERFACE}" ip saddr @member_ips ip daddr @vpc_dest_ips counter drop comment "{marker}:default-deny"',
f' iifname "{r.bridge}" oifname "{r.bridge}" ip saddr @member_ips ip daddr @vpc_dest_ips counter drop comment "{marker}:default-deny"',
f' iifname "{FABRIC_INTERFACE}" oifname "{FABRIC_INTERFACE}" ip saddr @member_ips ip daddr @vpc_dest_ips counter drop comment "{marker}:wg-transit-deny"',
f' iifname "{FABRIC_INTERFACE}" oifname "{r.bridge}" ip6 saddr ::/0 drop comment "{marker}:ipv6-drop-fabric"',
f' iifname "{r.bridge}" accept comment "{marker}:scope-bridge-in"',
f' oifname "{r.bridge}" accept comment "{marker}:scope-bridge-out"',
f' iifname "{FABRIC_INTERFACE}" accept comment "{marker}:scope-fabric-in"',
f' oifname "{FABRIC_INTERFACE}" accept comment "{marker}:scope-fabric-out"',
" }",
" chain input { type filter hook input priority -190; policy accept;",
f' iifname "{r.bridge}" ip saddr @member_ips drop comment "{marker}:host-isolation"',
f' iifname "{FABRIC_INTERFACE}" ip daddr @vpc_dest_ips drop comment "{marker}:host-isolation-wg"',
f' iifname "{FABRIC_INTERFACE}" ip saddr @member_ips ip daddr != @vpc_dest_ips drop comment "{marker}:host-isolation-wg-source"',
f' iifname "{r.bridge}" ip6 saddr ::/0 drop comment "{marker}:ipv6-input-drop-bridge"',
f' iifname "{FABRIC_INTERFACE}" ip6 saddr ::/0 drop comment "{marker}:ipv6-input-drop-fabric"',
" }",
" chain output { type filter hook output priority -190; policy accept;",
f' oifname "{r.bridge}" ip daddr @vpc_dest_ips drop comment "{marker}:host-isolation-out"',
# Host-originated packets must not be able to bind a tenant /32 and
# bypass the private egress boundary. Keep this leg unconditional;
# the source address is not an ingress identity.
f' oifname "{FABRIC_INTERFACE}" ip daddr @vpc_dest_ips drop comment "{marker}:host-isolation-out-wg"',
f' oifname "{r.bridge}" ip daddr {plan.vpc.cidr} drop comment "{marker}:host-isolation-vpc-out"',
f' oifname "{FABRIC_INTERFACE}" ip daddr {plan.vpc.cidr} drop comment "{marker}:host-isolation-vpc-out-wg"',
f' oifname "{r.bridge}" ip6 daddr ::/0 drop comment "{marker}:ipv6-output-drop-bridge"',
" }", "}", "",
]
# Bridge-family rules are emitted only when this host has a local VM. An
# empty VPC still gets the routed table above, while no empty nft set is
# needed for the L2/ARP path.
if plan.local_members:
lines += [
f"table bridge {r.nft_table} {{",
nft_set("local_member_ips", "ipv4_addr", local_members, interval=True),
nft_set("local_dest_ips", "ipv4_addr", local_destinations, interval=True),
nft_set("local_member_sources", "ether_addr . ipv4_addr", local_sources),
" chain forward { type filter hook forward priority -190; policy accept;",
f' meta ibrname "{r.bridge}" ether type ip ether saddr . ip saddr != @local_member_sources drop comment "{marker}:bridge-anti-spoof-ingress"',
f' meta ibrname "{r.bridge}" ether type arp ether saddr . arp saddr ip != @local_member_sources drop comment "{marker}:bridge-anti-spoof-arp-mac"',
f' meta ibrname "{r.bridge}" ether type arp arp saddr ether . arp saddr ip != @local_member_sources drop comment "{marker}:bridge-anti-spoof-arp"',
f' meta ibrname "{r.bridge}" ether type ip ip saddr != @local_member_ips drop comment "{marker}:bridge-anti-spoof-ip"',
f' meta ibrname "{r.bridge}" ether type arp arp saddr ip != @local_member_ips drop comment "{marker}:bridge-anti-spoof-arp-ip"',
# A stable VPC-prefix route lets a guest ask for every address in
# that prefix. Only exact member/service destinations may reach
# another tap or the host proxy responder; unknown ARP targets are
# dropped before they become a bridge-wide scan/flood primitive.
f' meta ibrname "{r.bridge}" ether type arp arp operation request arp daddr ip != @local_dest_ips drop comment "{marker}:bridge-unknown-arp-target-drop"',
# This is an L3 /32 fabric. Do not let broadcast or multicast
# discovery turn the private bridge back into a shared L2 LAN.
f' meta ibrname "{r.bridge}" ip daddr 224.0.0.0/4 drop comment "{marker}:bridge-multicast-drop"',
f' meta ibrname "{r.bridge}" ip daddr 255.255.255.255 drop comment "{marker}:bridge-limited-broadcast-drop"',
f' meta ibrname "{r.bridge}" ip daddr {ipaddress.ip_network(plan.vpc.cidr).broadcast_address} drop comment "{marker}:bridge-vpc-broadcast-drop"',
f' meta ibrname "{r.bridge}" ip saddr @local_member_ips ip daddr != @local_dest_ips drop comment "{marker}:bridge-unknown-destination-drop"',
]
for acl in plan.vpc.acls:
action = "accept" if acl.action == "allow" else "drop"
clause = (f" ip saddr {acl.source_cidr} ip daddr {acl.destination_cidr}"
f"{_nft_proto(acl)} {action} comment \"{marker}:acl:{acl.acl_id}\"")
lines.append(f' meta ibrname "{r.bridge}" meta obrname "{r.bridge}"{clause}')
lines += [
f' meta ibrname "{r.bridge}" meta obrname "{r.bridge}" ip saddr @local_member_ips ip daddr @local_dest_ips counter drop comment "{marker}:bridge-default-deny"',
f' meta ibrname "{r.bridge}" ether type arp accept comment "{marker}:bridge-arp-accept"',
f' meta ibrname "{r.bridge}" ether type ip accept comment "{marker}:bridge-scope-accept"',
f' meta ibrname "{r.bridge}" counter drop comment "{marker}:bridge-non-ip-drop"',
" }",
" chain input { type filter hook input priority -190; policy accept;",
f' meta ibrname "{r.bridge}" ether type arp ether saddr . arp saddr ip != @local_member_sources drop comment "{marker}:bridge-anti-spoof-arp-input-mac"',
f' meta ibrname "{r.bridge}" ether type ip ether saddr . ip saddr != @local_member_sources drop comment "{marker}:bridge-anti-spoof-input-ip"',
f' meta ibrname "{r.bridge}" ether type arp arp saddr ether . arp saddr ip != @local_member_sources drop comment "{marker}:bridge-anti-spoof-input"',
f' meta ibrname "{r.bridge}" ether type ip6 drop comment "{marker}:bridge-ipv6-input-drop"',
f' meta ibrname "{r.bridge}" ether type arp accept comment "{marker}:bridge-arp-input-accept"',
# Proxy ARP makes remote /32 packets arrive at the bridge MAC and
# then enter the host routing path. Source/MAC checks above have
# already authenticated the guest, so IPv4 must be accepted here
# for the inet forward hook to route it to wg-fabric0.
f' meta ibrname "{r.bridge}" ether type ip accept comment "{marker}:bridge-ip-input-accept"',
f' meta ibrname "{r.bridge}" counter drop comment "{marker}:bridge-input-non-ip-drop"',
" }",
"}", "",
]
lines += [
f"table arp {r.nft_table} {{",
nft_set("proxy_ips", "ipv4_addr", proxy_destinations, interval=True),
" chain output { type filter hook output priority -190; policy accept;",
f' oifname "{r.bridge}" arp operation reply arp saddr ip != @proxy_ips drop comment "{marker}:arp-proxy-output"',
" }",
"}", "",
]
return "\n".join(lines)
def _here(path: str, content: str) -> str:
marker = "__VC_VPC_" + hashlib.sha256(content.encode()).hexdigest()[:16]
return f"cat > {_q(path)} <<'{marker}'\n{content}{'' if content.endswith(chr(10)) else chr(10)}{marker}\n"
def _firewall_mode(value: str) -> str:
if value not in FIREWALL_MODES:
raise VpcDataplaneError(
"宿主机防火墙部署模式不支持", 409,
"vpc_dataplane_firewall_mode_conflict")
return value
def _state(
plan: VpcDataplanePlan,
routes: Sequence[Mapping[str, Any]],
firewall_mode: str = FIREWALL_MODE_NATIVE,
) -> str:
r = plan.resources
firewall_mode = _firewall_mode(firewall_mode)
lines = [f"owner|{DATAPLANE_OWNER_MARKER}", f"vpc|{plan.vpc.vpc_id}",
f"cidr|{plan.vpc.cidr}", f"fabric|{plan.vpc.fabric_id}",
f"host|{plan.host_id}", f"revision|{plan.desired_revision}",
f"digest|{plan.desired_digest}", f"bridge|{r.bridge}",
f"table|{r.route_table}", f"priority|{r.rule_priority}",
f"firewall_mode|{firewall_mode}"]
for route in routes:
lines.append("route|{destination}|{device}|{table}|{via}|{metric}".format(
destination=route["destination_cidr"], device=route["device"],
table=route["table"], via=route.get("via") or "-",
metric=route.get("metric", 100)))
for member in sorted(plan.local_members, key=lambda item: item.private_ip):
lines.append(f"local_neighbor|{member.private_ip}|{member.private_mac}")
wg_route_ips = {
str(route["destination_cidr"])
for route in routes if route["device"] == FABRIC_INTERFACE
}
owned_peer_ips = _expected_peer_tenant_ips(plan.vpc)
for peer in plan.peers:
# Persist only this VPC's route-backed contribution. The authenticated
# aggregate remains on ``peer.tenant_allowed_ips`` for live WireGuard
# convergence/readback, while old aggregate state is filtered by the
# loader during a mixed-version rollout.
ips = sorted(set(owned_peer_ips.get(peer.host_id, frozenset()))
& wg_route_ips)
if ips:
lines.append(f"peer|{peer.host_id}|{peer.public_key}|{','.join(ips)}")
return "\n".join(lines) + "\n"
def _allowed(plan: VpcDataplanePlan, peer: VpcFabricPeer) -> str:
members = {m.member_id: m for m in plan.members}
ips = {m.private_ip + "/32" for m in plan.remote_members
if m.host_id == peer.host_id}
for route in plan.vpc.routes:
target = (members.get(route.next_hop_member_id)
if route.next_hop_member_id is not None else None)
target_host = target.host_id if target is not None else route.next_hop_host_id
if target_host == peer.host_id:
ips.add(route.destination_cidr)
return ",".join([peer.overlay_cidr] + sorted(ips | set(peer.tenant_allowed_ips)))
def render_apply_script(plan: VpcDataplanePlan) -> str:
"""Generate a shell transaction with panel ownership and rollback markers."""
routes = _routes(plan)
if plan.remote_members and not plan.peers:
raise VpcDataplaneError("跨宿主机 VPC 缺少 Fabric peer", 409, "vpc_fabric_peer_missing")
r = plan.resources
nft = render_vpc_nft_rules(plan)
state = _state(plan, routes)
route_specs = "\\n".join(
f"{x['destination_cidr']}|{x['device']}|{x['table']}" for x in routes)
peer_specs = "\\n".join(
f"{p.host_id}|{p.public_key}|{_allowed(plan,p)}" for p in plan.peers)
route_checks = "\\n".join(
f'''actual=$("$IP" -o -4 -N route show table {_q(x["table"])} {_q(x["destination_cidr"])} 2>/dev/null || true)
if printf '%s\\n' "$actual" | grep -Fq "dev {_q(x["device"])}" && printf '%s\\n' "$actual" | grep -Eq 'proto 186([[:space:]]|$)'; then
printf '%s\\n' "__VC_VPC_ROUTE__={x["destination_cidr"]}|{x["device"]}|{x["table"]}|owned"
else
printf '%s\\n' "__VC_VPC_ROUTE__={x["destination_cidr"]}|{x["device"]}|{x["table"]}|missing"
fi'''
for x in routes)
peer_checks = "\\n".join(
f'''actual=$("$WG" show "$WG_IFACE" allowed-ips 2>/dev/null | awk -v wanted={_q(p.public_key)} '$1==wanted {{print $2; exit}}' || true)
tenant={_q(','.join(_allowed(plan, p).split(',')[1:]))}
case ",$actual," in *",$tenant,"*) printf '%s\\n' "__VC_VPC_WG_ALLOWED__={p.host_id}|{p.public_key}|$tenant";; *) printf '%s\\n' "__VC_VPC_WG_ALLOWED__={p.host_id}|{p.public_key}|missing";; esac'''
for p in plan.peers)
script = f'''#!/bin/sh
set -eu
umask 077
PATH=/usr/sbin:/usr/bin:/sbin:/bin
export PATH
IP=/usr/sbin/ip; [ -x "$IP" ] || IP=/usr/bin/ip
WG=/usr/bin/wg; [ -x "$WG" ] || WG=/usr/sbin/wg
NFT=/usr/sbin/nft; [ -x "$NFT" ] || NFT=/usr/bin/nft
FLOCK=/usr/bin/flock; [ -x "$FLOCK" ] || FLOCK=/usr/bin/flock
{BRIDGE_PORT_RECORDS_SH}
{LOCAL_NEIGHBOR_SH}
{IP_RULE_CONVERGENCE_SH}
BASE=/etc/virt-cloud/vpc-dataplane
BRIDGE={_q(r.bridge)}; TABLE={_q(r.nft_table)}; ROUTE_TABLE={r.route_table}; RULE_PRIORITY={r.rule_priority}
WG_IFACE={_q(FABRIC_INTERFACE)}; OWNER={_q(r.owner_file)}; STATE={_q(r.state_file)}; LOCK={_q(r.lock_file)}; TXN={_q(r.transaction_dir)}
COMMITTED=no; BRIDGE_CREATED=no; NFT_BEFORE=no; WG_BEFORE=no; ROLLBACK_FAILED=no
mkdir -p /run/virt-cloud "$BASE"
[ ! -L /run/virt-cloud ] && [ ! -L "$BASE" ] || exit 71
: >> "$LOCK"; chmod 600 "$LOCK"; exec 9>>"$LOCK"; "$FLOCK" -x 9
rollback() {{
status=$?; [ "$COMMITTED" = yes ] && return "$status"; trap - EXIT HUP INT TERM
oldIFS=$IFS; IFS=' '
for spec in { _q(route_specs) }; do destination=${{spec%%|*}}; device=${{spec#*|}}; "$IP" route del "$destination" table "$ROUTE_TABLE" dev "$device" >/dev/null 2>&1 || true; done
IFS=$oldIFS
if [ -s "$TXN/rules.attempted" ]; then
if [ -f "$TXN/rules.created" ] &&
grep -Fqx "$RULE_PRIORITY|$BRIDGE|{plan.vpc.cidr}|$ROUTE_TABLE" "$TXN/rules.created"; then
"$IP" -4 rule del pref "$RULE_PRIORITY" iif "$BRIDGE" to {_q(plan.vpc.cidr)} lookup "$ROUTE_TABLE" >/dev/null 2>&1 || ROLLBACK_FAILED=yes
else
ROLLBACK_FAILED=yes
fi
fi
if [ "$WG_BEFORE" = yes ] && [ -f "$TXN/wg.conf" ]; then "$WG" setconf "$WG_IFACE" "$TXN/wg.conf" >/dev/null 2>&1 || ROLLBACK_FAILED=yes; fi
if [ "$NFT_BEFORE" = yes ] && [ -f "$TXN/nft.before" ]; then "$NFT" delete table inet "$TABLE" >/dev/null 2>&1 || true; "$NFT" -f "$TXN/nft.before" >/dev/null 2>&1 || ROLLBACK_FAILED=yes; else "$NFT" delete table inet "$TABLE" >/dev/null 2>&1 || true; fi
if [ "$BRIDGE_CREATED" = yes ]; then "$IP" link del dev "$BRIDGE" >/dev/null 2>&1 || ROLLBACK_FAILED=yes; fi
[ "$ROLLBACK_FAILED" = yes ] && printf '%s\\n' '__VC_VPC_ROLLBACK__=failed' >&2 || printf '%s\\n' '__VC_VPC_ROLLBACK__=ok' >&2
exit "$status"
}}
trap rollback EXIT HUP INT TERM
[ -x "$IP" ] && [ -x "$WG" ] && [ -x "$NFT" ] && [ -x "$FLOCK" ] || exit 70
[ -f /etc/virt-cloud/wg-fabric0.owner ] && grep -Fqx { _q(FABRIC_OWNER_MARKER) } /etc/virt-cloud/wg-fabric0.owner || exit 71
"$IP" link show dev "$WG_IFACE" >/dev/null 2>&1 && "$WG" show "$WG_IFACE" >/dev/null 2>&1 || exit 72
[ ! -e "$TXN" ] && [ ! -L "$TXN" ] || exit 79
mkdir -p "$TXN"; printf '%s\\n' '{DATAPLANE_OWNER_MARKER}:{plan.vpc.vpc_id}:{plan.host_id}:{plan.desired_revision}:{plan.desired_digest}' > "$TXN/inflight"
if [ -f "$STATE" ]; then cp -p "$STATE" "$TXN/state.before"; grep -Fqx 'owner|{DATAPLANE_OWNER_MARKER}' "$STATE" || exit 71; fi
if [ -f "$OWNER" ]; then grep -Fqx { _q(DATAPLANE_OWNER_MARKER) } "$OWNER" || exit 71; else printf '%s\\n' { _q(DATAPLANE_OWNER_MARKER) } > "$OWNER"; chmod 600 "$OWNER"; fi
if "$IP" link show dev "$BRIDGE" >/dev/null 2>&1; then
"$IP" -d link show dev "$BRIDGE" | grep -Eq '(^|[[:space:]])bridge[[:space:]]+(forward_delay|stp_state|hello_time|ageing_time)([[:space:]]|$)' || exit 71
else
"$IP" link add name "$BRIDGE" type bridge; BRIDGE_CREATED=yes
fi
"$IP" link set dev "$BRIDGE" up
if "$WG" showconf "$WG_IFACE" > "$TXN/wg.conf" 2>/dev/null; then WG_BEFORE=yes; fi
if "$NFT" list table inet "$TABLE" > "$TXN/nft.before" 2>/dev/null; then NFT_BEFORE=yes; grep -Fq { _q(DATAPLANE_OWNER_MARKER) } "$TXN/nft.before" || exit 71; fi
'''+_here('$TXN/rules.nft', nft)+f'''"$NFT" delete table inet "$TABLE" >/dev/null 2>&1 || true
"$NFT" -f "$TXN/rules.nft"
oldIFS=$IFS; IFS=' '
for spec in { _q(route_specs) }; do destination=${{spec%%|*}}; device=${{spec#*|}}; "$IP" route replace "$destination" table "$ROUTE_TABLE" dev "$device" metric 100; done
IFS=$oldIFS
vc_ip_rule_preflight "$RULE_PRIORITY" "$BRIDGE" {_q(plan.vpc.cidr)} "$ROUTE_TABLE" 73 73 - || exit $?
: > "$TXN/rules.attempted"; : > "$TXN/rules.created"
vc_ip_rule_ensure "$RULE_PRIORITY" "$BRIDGE" {_q(plan.vpc.cidr)} "$ROUTE_TABLE" 73 73 "$TXN/rules.attempted" "$TXN/rules.created" || exit $?
oldIFS=$IFS; IFS=' '
for spec in { _q(peer_specs) }; do rest=${{spec#*|}}; key=${{rest%%|*}}; allowed=${{rest#*|}}; "$WG" show "$WG_IFACE" peers | grep -Fqx "$key" || exit 74; "$WG" set "$WG_IFACE" peer "$key" allowed-ips "$allowed"; done
IFS=$oldIFS
'''+_here('$STATE', state)+f'''chmod 600 "$STATE"
printf '%s\\n' '__VC_VPC_APPLIED__=yes'
printf '%s\\n' '__VC_VPC_OWNER__={DATAPLANE_OWNER_MARKER}'
printf '%s\\n' '__VC_VPC_REVISION__={plan.desired_revision}'
printf '%s\\n' '__VC_VPC_DIGEST__={plan.desired_digest}'
COMMITTED=yes; trap - EXIT HUP INT TERM; rm -rf "$TXN"; exit 0
'''
if len(script) > MAX_OUTPUT:
raise VpcDataplaneError("VPC 数据面脚本过大", 413, "vpc_script_too_large")
return script
def _render_apply_script_v2(
plan: VpcDataplanePlan,
*,
deployment_mode: str = "networkmanager_nmcli",
firewall_mode: str = FIREWALL_MODE_NATIVE,
ifup_path: str = "/usr/sbin/ifup",
ifdown_path: str = "/usr/sbin/ifdown",
ifquery_path: str = "/usr/sbin/ifquery",
) -> str:
"""Generate the production transaction used by the host adapter.
The original control-plane module predates distribution-specific bridge
persistence. Keep its public builder for compatibility, but route real
applies through this stricter builder: every list is newline-delimited,
every path is fixed/quoted, and rollback only removes exact resources
tagged with the panel route protocol or ownership marker.
"""
if deployment_mode not in {"networkmanager_nmcli", "ifupdown"}:
raise VpcDataplaneError("宿主机网络管理模式不支持", 409,
"vpc_dataplane_network_manager_conflict")
if not plan.local_members:
raise VpcDataplaneError(
"当前宿主机没有 VPC 成员,应执行删除收敛而不是 apply",
409, "vpc_apply_no_local_members")
firewall_mode = _firewall_mode(firewall_mode)
allowed_ifup = {"/usr/sbin/ifup", "/usr/bin/ifup", "/sbin/ifup", "/bin/ifup"}
allowed_ifdown = {"/usr/sbin/ifdown", "/usr/bin/ifdown", "/sbin/ifdown", "/bin/ifdown"}
allowed_ifquery = {"/usr/sbin/ifquery", "/usr/bin/ifquery", "/sbin/ifquery", "/bin/ifquery"}
if ifup_path not in allowed_ifup or ifdown_path not in allowed_ifdown or ifquery_path not in allowed_ifquery:
raise VpcDataplaneError("ifupdown 工具路径不受支持", 409,
"vpc_dataplane_network_manager_conflict")
from .fabric import (
_FABRIC_NFT_VALIDATION_SH,
_FIREWALL_MODE_GUARD_SH,
_NMCLI_FILENAME_RESOLVER_SH,
_VPC_STATE_LOADER,
)
routes = _routes(plan)
if plan.remote_members and not plan.peers:
raise VpcDataplaneError("跨宿主机 VPC 缺少 Fabric peer", 409,
"vpc_fabric_peer_missing")
r = plan.resources
nft = render_vpc_nft_rules(plan)
include = f"/etc/nftables.d/virt-cloud-vpc-{r.token}.nft"
include_line = f'include "{include}"'
fragment = f"/etc/network/interfaces.d/virt-cloud-vpc-{r.token}"
sysctl_file = f"/etc/sysctl.d/99-virt-cloud-vpc-{r.token}.conf"
state_dir = f"/etc/virt-cloud/vpc-dataplane"
proxy_file = f"{state_dir}/{r.token}.proxy"
route_lines = "\n".join(
f"{item['destination_cidr']}|{item['device']}|{item['table']}|"
f"{item.get('via') or '-'}|{item.get('metric', 100)}"
for item in routes)
bridge_route_lines = "\n".join(
f"{item['destination_cidr']}|{item['device']}|{item['table']}|"
f"{item.get('via') or '-'}|{item.get('metric', 100)}|186"
for item in routes if item["device"] == r.bridge)
proxy_lines = "\n".join(
str(ipaddress.ip_network(item["destination_cidr"], strict=True).network_address)
for item in routes)
peers_by_key = {peer.public_key: peer for peer in plan.peers}
authoritative_peer_overlays = _authoritative_fabric_peer_overlays(plan)
if (authoritative_peer_overlays is None
and plan.fabric_peer_keys is not None
and not plan.fabric_peer_keys.issubset(peers_by_key)):
raise VpcDataplaneError(
"完整 Fabric peer 收敛缺少权威 overlay 闭集", 409,
"vpc_fabric_peer_set_invalid")
# A VPC revision can stop using one Fabric peer while the peer itself
# remains part of the host's WireGuard Fabric. Converge that full
# authoritative closure after rewriting durable state, otherwise the
# removed peer retains the withdrawn tenant /32 indefinitely. Direct
# parser callers without the engine-only closure retain the historical
# current-VPC scope; production plans always provide the full key-to-overlay
# closure.
apply_fabric_peer_keys = tuple(sorted(
authoritative_peer_overlays
if authoritative_peer_overlays is not None
else (plan.fabric_peer_keys
if plan.fabric_peer_keys is not None else peers_by_key)
))
peer_lines = "\n".join(
f"{peers_by_key[key].host_id if key in peers_by_key else '-'}|"
f"{key}|{authoritative_peer_overlays[key]}"
if authoritative_peer_overlays is not None
else f"{peers_by_key[key].host_id}|{key}|{peers_by_key[key].overlay_cidr}"
for key in apply_fabric_peer_keys)
apply_fabric_peer_key_words = " ".join(apply_fabric_peer_keys)
apply_fabric_peer_keys_provided = (
"yes" if authoritative_peer_overlays is not None else "no")
state = _state(plan, routes, firewall_mode)
# Local service /32s use a guest address as the next hop on an otherwise
# unnumbered private bridge. Linux therefore needs the explicit `onlink`
# assertion; without it iproute2 rejects the route as an invalid gateway.
# Use a quoted heredoc for data, while keeping the destination as a shell
# variable expanded by the surrounding script. This avoids the literal
# ``$TXN`` path bug in the former builder.
nft_marker = "__VC_VPC_NFT_" + hashlib.sha256(nft.encode()).hexdigest()[:12]
nft_rule_marker = _nft_marker(plan)
nft_families = (("inet", "bridge", "arp")
if plan.local_members else ("inet",))
nft_family_words = " ".join(nft_families)
nft_reassert_deletes = "\n".join(
f"printf 'delete table {family} %s\\n' \"$TABLE\" >> \"$NFT_REASSERT\""
for family in nft_families)
inet_readback_checks = "\n".join(
f''' printf '%s\\n' "$vc_inet_text" | grep -Fq "comment \\"$NFT_RULE_MARKER:{suffix}\\"" || return 1'''
for suffix in (
*_NFT_INET_REQUIRED_SUFFIXES,
*(f"acl:{acl.acl_id}" for acl in plan.vpc.acls),
))
bridge_readback_checks = "\n".join(
f''' printf '%s\\n' "$vc_bridge_text" | grep -Fq "comment \\"$NFT_RULE_MARKER:{suffix}\\"" || return 1'''
for suffix in (
*_NFT_BRIDGE_REQUIRED_SUFFIXES,
*(f"acl:{acl.acl_id}" for acl in plan.vpc.acls),
))
if plan.local_members:
local_nft_readback = f'''
vc_bridge_text=$("$NFT" list table bridge "$TABLE" 2>/dev/null) || return 1
printf '%s\\n' "$vc_bridge_text" | grep -Fq "table bridge $TABLE" || return 1
{bridge_readback_checks}
vc_arp_text=$("$NFT" list table arp "$TABLE" 2>/dev/null) || return 1
printf '%s\\n' "$vc_arp_text" | grep -Fq "table arp $TABLE" || return 1
printf '%s\\n' "$vc_arp_text" | grep -Fq "comment \\"$NFT_RULE_MARKER:arp-proxy-output\\"" || return 1
'''.rstrip()
else:
local_nft_readback = '''
"$NFT" list table bridge "$TABLE" >/dev/null 2>&1 && return 1 || true
"$NFT" list table arp "$TABLE" >/dev/null 2>&1 && return 1 || true
'''.rstrip()
nft_readback_helper = f'''
vc_vpc_desired_nft_ready() {{
vc_inet_text=$("$NFT" list table inet "$TABLE" 2>/dev/null) || return 1
printf '%s\\n' "$vc_inet_text" | grep -Fq "table inet $TABLE" || return 1
{inet_readback_checks}
{local_nft_readback}
return 0
}}
vc_vpc_capture_nft_snapshot() {{
vc_vpc_snapshot_prefix=$1
for vc_vpc_snapshot_family in {nft_family_words}; do
vc_nft_table_snapshot "$vc_vpc_snapshot_family" "$TABLE" \
"$vc_vpc_snapshot_prefix.$vc_vpc_snapshot_family" || return 1
[ -f "$vc_vpc_snapshot_prefix.$vc_vpc_snapshot_family" ] &&
[ ! -L "$vc_vpc_snapshot_prefix.$vc_vpc_snapshot_family" ] &&
[ ! -e "$vc_vpc_snapshot_prefix.$vc_vpc_snapshot_family.absent" ] || return 1
done
}}
vc_vpc_capture_nft_precommit() {{
vc_vpc_precommit_prefix=$1
for vc_vpc_precommit_family in {nft_family_words}; do
vc_nft_table_snapshot "$vc_vpc_precommit_family" "$TABLE" \
"$vc_vpc_precommit_prefix.$vc_vpc_precommit_family" || return 1
done
}}
vc_vpc_nft_matches_snapshot() {{
vc_vpc_snapshot_prefix=$1
for vc_vpc_snapshot_family in {nft_family_words}; do
vc_nft_table_matches_snapshot "$vc_vpc_snapshot_family" "$TABLE" \
"$vc_vpc_snapshot_prefix.$vc_vpc_snapshot_family" || return 1
done
}}
vc_vpc_fabric_nft_shape_file() {{
vc_fabric_shape_file=$1
[ -f "$vc_fabric_shape_file" ] && [ ! -L "$vc_fabric_shape_file" ] || return 1
vc_fabric_shape_text=$(cat "$vc_fabric_shape_file") || return 1
nft_plan_owned "$vc_fabric_shape_text" "$EXPECTED_DIGEST"
}}
vc_vpc_fabric_nft_live_ready() {{
"$NFT" -s list table inet virt_cloud_wg_fabric > "$TXN/fabric.nft.verify" 2>/dev/null || return 1
vc_vpc_fabric_nft_shape_file "$TXN/fabric.nft.verify"
}}
'''.strip()
state_marker = "__VC_VPC_STATE_" + hashlib.sha256(state.encode()).hexdigest()[:12]
frag_marker = "__VC_VPC_IFUP_" + r.token
# Both persistence owners are helper-gated. An `auto` stanza could bring
# the bridge up during boot before the raw tenant ACL has been restored.
ifup_auto_line = ""
route_marker = "__VC_VPC_ROUTES_" + r.token
proxy_marker = "__VC_VPC_PROXY_" + r.token
peer_marker = "__VC_VPC_PEERS_" + r.token
state_loader = _VPC_STATE_LOADER.replace(
"__VC_FABRIC_HOST_ID__", plan.host_id).replace(
"__VC_FABRIC_ID__", plan.vpc.fabric_id).replace(
"__VC_FABRIC_NETWORK__", "100.64.0.0/10").replace(
"__VC_VPC_STATE_OBSERVE__", "no")
script = f'''#!/bin/sh
set -eu
umask 077
PATH=/usr/sbin:/usr/bin:/sbin:/bin
export PATH
{VPC_APPLY_SAFE_LOG_SH}
IP=$(for p in /usr/sbin/ip /usr/bin/ip /sbin/ip /bin/ip; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
WG=$(for p in /usr/bin/wg /usr/sbin/wg /bin/wg /sbin/wg; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
NFT=$(for p in /usr/sbin/nft /usr/bin/nft /sbin/nft /bin/nft; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
FLOCK=$(for p in /usr/bin/flock /usr/sbin/flock /bin/flock /sbin/flock; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
STAT=$(for p in /usr/bin/stat /bin/stat /usr/sbin/stat /sbin/stat; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
CMP=$(for p in /usr/bin/cmp /bin/cmp /usr/sbin/cmp /sbin/cmp; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
SYSCTL=$(for p in /usr/sbin/sysctl /usr/bin/sysctl /sbin/sysctl /bin/sysctl; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
BRIDGE_CMD=$(for p in /usr/sbin/bridge /usr/bin/bridge /sbin/bridge /bin/bridge; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
SYSTEMCTL=$(for p in /usr/bin/systemctl /bin/systemctl /usr/sbin/systemctl /sbin/systemctl; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
{BRIDGE_PORT_RECORDS_SH}
{LOCAL_NEIGHBOR_SH}
{IP_RULE_CONVERGENCE_SH}
{WG_ALLOWED_IPS_SH}
{VPC_ROUTE_RECORDS_SH}
BRIDGE={_q(r.bridge)}; TABLE={_q(r.nft_table)}; ROUTE_TABLE={r.route_table}; RULE_PRIORITY={r.rule_priority}; WG_RULE_PRIORITY=$((RULE_PRIORITY + 1))
WG_IFACE={_q(FABRIC_INTERFACE)}; OWNER_MARKER={_q(DATAPLANE_OWNER_MARKER)}; FABRIC_OWNER={_q(FABRIC_OWNER_MARKER)}
OWNER_FILE={_q(r.owner_file)}; STATE_FILE={_q(r.state_file)}; LOCK={_q(r.lock_file)}; TXN={_q(r.transaction_dir)}; PROXY_FILE={_q(proxy_file)}
NFT_SNAPSHOT_PREFIX={_q(f'{state_dir}/{r.token}.nft')}
NFT_FILE={_q(include)}; NFT_INCLUDE={_q(include_line)}; LEGACY_MANIFEST={_q(LEGACY_FIREWALLD_NFT_MANIFEST)}; IF_FRAGMENT={_q(fragment)}; SYSCTL_FILE={_q(sysctl_file)}; MODE={_q(deployment_mode)}; FIREWALL_MODE={_q(firewall_mode)}; NFT_MAIN=
IFUP={_q(ifup_path)}; IFDOWN={_q(ifdown_path)}; IFQUERY={_q(ifquery_path)}; NMCLI=; NM_PROFILE_UUID=; NM_PROFILE_FILE=
NM_PROFILE_MARKER={_q(DATAPLANE_OWNER_MARKER + " vpc=" + plan.vpc.vpc_id)}
EXPECTED_DIGEST={_q(plan.fabric_plan_digest)}
NFT_RULE_MARKER={_q(nft_rule_marker)}
COMMITTED=no; LOCAL_NEIGHBORS_CHANGED=no; MUTATION_STARTED=no; ROUTES_CHANGED=no; RUNTIME_QUARANTINED=no; BRIDGE_CREATED=no; BRIDGE_WAS_UP=no; BRIDGE_TOPOLOGY_CAPTURED=no; BRIDGE_IFINDEX_BEFORE=; BRIDGE_PORTS_BEFORE=; NM_PRESERVE_ACTIVE_BRIDGE=no; NM_PRESERVED_PROFILE_UUID=; NM_CREATED=no; NM_CREATED_UUID=; NM_PROFILE_WAS_PRESENT=no; NM_WAS_ACTIVE=no; NM_DEVICE_QUARANTINED=no; NM_ACTIVATION_ATTEMPTED=no; IF_FRAGMENT_CHANGED=no; SYSCTL_CHANGED=no; SYSCTL_TMP=; RP_FILTER_CHANGED=no; PROXY_ARP_CHANGED=no; PROXY_ARP_PVLAN_CHANGED=no; NFT_FILE_CHANGED=no; NFT_MAIN_CHANGED=no; NFT_FILE_BEFORE=no; NFT_MAIN_BEFORE=no; BRIDGE_NFT_BEFORE=no; ARP_NFT_BEFORE=no; FABRIC_NFT_BEFORE=no; NFT_APPLIED=no; FABRIC_NFT_APPLIED=no; PROXY_ENTRIES_CHANGED=no; PROXY_FILE_CHANGED=no; FIREWALL_HARD_QUARANTINED=no; ROLLBACK_FAILED=no
{NFT_LIVE_SNAPSHOT_SH}
{_NMCLI_FILENAME_RESOLVER_SH}
{NETWORKMANAGER_VPC_PROFILE_SH}
{nft_readback_helper}
{_FIREWALL_MODE_GUARD_SH}
{_FABRIC_NFT_VALIDATION_SH}
{_VPC_SYSCTL_VALIDATION_SH}
{VPC_NATIVE_TRANSACTION_GUARD_SH}
{VPC_APPLY_RUNTIME_SH}
{VPC_LOCAL_NEIGHBOR_TRANSACTION_SH}
{VPC_APPLY_RECEIPT_SH}
SHA256SUM=$(command -v sha256sum || true)
nft_plan_owned() {{
nft_plan_text=$1
nft_plan_digest=$2
printf '%s\\n' "$nft_plan_digest" | grep -Eq '^[0-9a-f]{{64}}$' || return 1
vc_fabric_nft_strict_shape "$nft_plan_text" || return 1
[ "$(printf '%s\\n' "$nft_plan_text" |
grep -Foc 'comment "virt-cloud-wg-fabric-v1:'"$nft_plan_digest"'"' || true)" = 8 ]
}}
VC_VPC_APPLY_PHASE=tools
[ -n "$IP" ] && [ -n "$WG" ] && [ -n "$NFT" ] && [ -n "$FLOCK" ] && [ -n "$STAT" ] && [ -n "$CMP" ] && [ -n "$SYSCTL" ] && [ -n "$BRIDGE_CMD" ] && [ -n "$SYSTEMCTL" ] || exit 70
if [ "$MODE" = networkmanager_nmcli ]; then
NMCLI=$(for p in /usr/bin/nmcli /usr/sbin/nmcli /bin/nmcli /sbin/nmcli; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
[ -n "$NMCLI" ] || exit 70
fi
[ "$FIREWALL_MODE" = native_nftables ] || exit 71
VC_VPC_APPLY_PHASE=preflight
if [ -L /run/virt-cloud ] || {{ [ -e /run/virt-cloud ] && [ ! -d /run/virt-cloud ]; }}; then exit 71; fi
if [ -L /etc/virt-cloud ] || {{ [ -e /etc/virt-cloud ] && [ ! -d /etc/virt-cloud ]; }}; then exit 71; fi
if [ -L {state_dir} ] || {{ [ -e {state_dir} ] && [ ! -d {state_dir} ]; }}; then exit 71; fi
if [ -e {state_dir} ]; then
[ "$($STAT -c '%u:%a' {state_dir} 2>/dev/null)" = 0:700 ] || exit 71
fi
mkdir -p /run/virt-cloud
chmod 700 /run/virt-cloud
[ ! -L "$LOCK" ] && {{ [ ! -e "$LOCK" ] || [ -f "$LOCK" ]; }} || exit 71
: >> "$LOCK"; chmod 600 "$LOCK"
exec 8>>{_q(FABRIC_LOCK)}; "$FLOCK" -x 8
exec 9>>"$LOCK"; "$FLOCK" -x 9
vc_vpc_bridge_port_names() {{
vc_bridge_ports_raw=$(vc_bridge_port_records "$BRIDGE" 2>/dev/null) || return 1
vc_bridge_ports_parsed=$(printf '%s\\n' "$vc_bridge_ports_raw" | awk '
NF == 0 {{next}}
NF < 2 {{bad=1; next}}
{{name=$2; sub(/:$/, "", name); sub(/@.*/, "", name)}}
name !~ /^[A-Za-z0-9_.-]+$/ {{bad=1; next}}
{{print name}}
END {{if (bad) exit 1}}
') || return 1
printf '%s\\n' "$vc_bridge_ports_parsed" | LC_ALL=C sort -u
}}
vc_vpc_bridge_topology_matches() {{
[ "$BRIDGE_TOPOLOGY_CAPTURED" = yes ] || return 1
if [ -n "$BRIDGE_PORTS_BEFORE" ]; then
vc_bridge_ifindex_now=$(vc_bridge_ifindex "$BRIDGE") || return 1
[ "$vc_bridge_ifindex_now" = "$BRIDGE_IFINDEX_BEFORE" ] || return 1
fi
vc_bridge_ports_now=$(vc_vpc_bridge_port_names) || return 1
[ "$vc_bridge_ports_now" = "$BRIDGE_PORTS_BEFORE" ]
}}
vc_vpc_bridge_link_state() {{
vc_bridge_link_raw=$("$IP" -o link show dev "$BRIDGE" 2>/dev/null) || return 1
vc_bridge_link_flags=$(printf '%s\\n' "$vc_bridge_link_raw" | awk -v wanted="$BRIDGE" '
NF == 0 {{next}}
{{
count++
name=$2
sub(/:$/, "", name)
sub(/@.*/, "", name)
if (name != wanted || $3 !~ /^<[^>]+>$/) bad=1
flags=$3
}}
END {{
if (bad || count != 1) exit 1
print flags
}}
') || return 1
if printf '%s\\n' "$vc_bridge_link_flags" |
grep -Eq '(^|<|,)UP(,|>)'; then
printf '%s' up
else
printf '%s' down
fi
}}
vc_vpc_rp_filter_set() {{
vc_rpf_path="/proc/sys/net/ipv4/conf/$BRIDGE/rp_filter"
[ -f "$vc_rpf_path" ] && [ ! -L "$vc_rpf_path" ] || return 1
if [ ! -f "$TXN/rp_filter.before" ]; then
[ ! -e "$TXN/rp_filter.before" ] && [ ! -L "$TXN/rp_filter.before" ] || return 1
cat "$vc_rpf_path" > "$TXN/rp_filter.before" || return 1
chmod 600 "$TXN/rp_filter.before" || return 1
fi
RP_FILTER_CHANGED=yes
printf '%s\\n' 2 > "$vc_rpf_path" || return 1
[ "$(cat "$vc_rpf_path" 2>/dev/null)" = 2 ]
}}
vc_vpc_native_ifupdown_rollback_reopen() {{
vc_vpc_nft_matches_snapshot "$TXN/nft.precommit.before" || return 1
vc_nft_table_matches_snapshot inet virt_cloud_wg_fabric \
"$TXN/fabric.nft.before" || return 1
vc_vpc_bridge_topology_matches || return 1
vc_bridge_reopen_safe=yes
"$IP" link set dev "$BRIDGE" up >/dev/null 2>&1 ||
vc_bridge_reopen_safe=no
if [ "$vc_bridge_reopen_safe" = yes ]; then
vc_bridge_reopen_state=$(vc_vpc_bridge_link_state) ||
vc_bridge_reopen_safe=no
fi
if [ "$vc_bridge_reopen_safe" = yes ] &&
[ "$vc_bridge_reopen_state" != up ]; then
vc_bridge_reopen_safe=no
fi
if [ "$vc_bridge_reopen_safe" = yes ]; then
vc_vpc_bridge_topology_matches || vc_bridge_reopen_safe=no
fi
if [ "$vc_bridge_reopen_safe" != yes ]; then
# Link-up is not a transactional primitive. If any post-open readback
# fails (including a concurrent tap/port change), immediately re-isolate
# and prove DOWN before reporting the rollback failure.
"$IP" link set dev "$BRIDGE" down >/dev/null 2>&1 || return 1
vc_bridge_reopen_state=$(vc_vpc_bridge_link_state) || return 1
[ "$vc_bridge_reopen_state" = down ] || return 1
return 1
fi
return 0
}}
# The old boot helper rejects the new state record. Require the coordinated
# helper upgrade before writing state or taking the bridge down.
vc_vpc_neighbor_helper_ready || exit 71
# Never mutate even the bridge state while an unmarked, legacy, or malformed
# transaction exists. New native transactions carry an exact backend marker.
vc_vpc_firewall_boundary_clean || exit 79
for vc_vpc_legacy_snapshot in inet bridge arp; do
[ ! -e "$NFT_SNAPSHOT_PREFIX.$vc_vpc_legacy_snapshot.live" ] &&
[ ! -L "$NFT_SNAPSHOT_PREFIX.$vc_vpc_legacy_snapshot.live" ] || exit 79
done
if [ -e "$TXN" ] || [ -L "$TXN" ]; then
vc_vpc_native_transaction_owned || exit 79
fi
if [ ! -e {state_dir} ]; then
mkdir -m 700 {state_dir} || exit 71
fi
[ ! -L {state_dir} ] && [ "$($STAT -c '%u:%a' {state_dir} 2>/dev/null)" = 0:700 ] || exit 71
# Prove existing bridge ownership and topology without changing the link.
# Quarantine follows durable runtime snapshots and still precedes ACL writes.
PREVIOUS_FIREWALL_MODE=absent
if [ -f "$STATE_FILE" ]; then
[ ! -L "$STATE_FILE" ] &&
[ "$($STAT -c '%u:%a' "$STATE_FILE" 2>/dev/null)" = 0:600 ] || exit 71
grep -Fqx 'owner|{DATAPLANE_OWNER_MARKER}' "$STATE_FILE" || exit 71
grep -Fqx 'vpc|{plan.vpc.vpc_id}' "$STATE_FILE" || exit 71
grep -Fqx 'fabric|{plan.vpc.fabric_id}' "$STATE_FILE" || exit 71
grep -Fqx 'host|{plan.host_id}' "$STATE_FILE" || exit 71
grep -Fqx 'bridge|{r.bridge}' "$STATE_FILE" || exit 71
grep -Fqx 'table|{r.route_table}' "$STATE_FILE" || exit 71
state_firewall_quarantine=$(awk -F'|' '$1=="firewall_mode" {{count++; value=$2}} END {{print (count + 0) "|" value}}' "$STATE_FILE")
case "$state_firewall_quarantine" in
'0|'|'1|native_nftables') PREVIOUS_FIREWALL_MODE=native_nftables ;;
*) exit 71 ;;
esac
fi
if [ "$PREVIOUS_FIREWALL_MODE" != absent ]; then
# Only durable state, the owner marker, the kernel bridge type, and the
# exact manager artifact are trusted for this pre-guard safety action.
state_revision=$(awk -F'|' '$1=="revision" {{count++; value=$2}} END {{print (count + 0) "|" value}}' "$STATE_FILE")
state_digest=$(awk -F'|' '$1=="digest" {{count++; value=$2}} END {{print (count + 0) "|" value}}' "$STATE_FILE")
[ "${{state_revision%%|*}}" = 1 ] || exit 71
case "${{state_revision#*|}}" in ''|*[!0-9]*) exit 71 ;; esac
[ "${{state_digest%%|*}}" = 1 ] || exit 71
printf '%s\\n' "${{state_digest#*|}}" | grep -Eq '^[0-9a-f]{{64}}$' || exit 71
[ -f "$OWNER_FILE" ] && [ ! -L "$OWNER_FILE" ] &&
[ "$($STAT -c '%u:%a' "$OWNER_FILE" 2>/dev/null)" = 0:600 ] &&
grep -Fqx "$OWNER_MARKER" "$OWNER_FILE" || exit 71
if [ "$MODE" = networkmanager_nmcli ]; then
vc_nm_profile_resolve || exit 71
vc_nm_profile_base_validate || exit 71
NM_PROFILE_WAS_PRESENT=yes
if vc_nm_profile_active; then NM_WAS_ACTIVE=yes; fi
quarantine_memory_zone=$(vc_nm_profile_field connection.zone) || exit 71
case "$quarantine_memory_zone" in ''|--) : ;; *) exit 71 ;; esac
else
[ -f "$IF_FRAGMENT" ] && [ ! -L "$IF_FRAGMENT" ] &&
[ "$($STAT -c '%u:%a' "$IF_FRAGMENT" 2>/dev/null)" = 0:600 ] || exit 71
awk -v owner="# {DATAPLANE_OWNER_MARKER} vpc={plan.vpc.vpc_id} revision=${{state_revision#*|}} digest=${{state_digest#*|}}" \
-v bridge="$BRIDGE" '
NR==1 && $0==owner {{next}}
NR==2 && $0=="iface " bridge " inet manual" {{next}}
NR==3 && $0==" bridge_ports none" {{next}}
NR==4 && $0==" bridge-stp off" {{next}}
{{bad=1}}
END {{exit (NR==4 && !bad) ? 0 : 1}}
' "$IF_FRAGMENT" || exit 71
fi
if "$IP" link show dev "$BRIDGE" >/dev/null 2>&1; then
"$IP" -d link show dev "$BRIDGE" 2>/dev/null |
grep -Eq '(^|[[:space:]])bridge[[:space:]]+(forward_delay|stp_state|hello_time|ageing_time)([[:space:]]|$)' || exit 71
bridge_initial_link_state=$(vc_vpc_bridge_link_state) || exit 71
if [ "$bridge_initial_link_state" = up ]; then
BRIDGE_WAS_UP=yes
fi
BRIDGE_IFINDEX_BEFORE=$(vc_bridge_ifindex "$BRIDGE") || exit 71
BRIDGE_PORTS_BEFORE=$(vc_vpc_bridge_port_names) || exit 71
BRIDGE_TOPOLOGY_CAPTURED=yes
if [ "$MODE" = networkmanager_nmcli ] && [ -n "$BRIDGE_PORTS_BEFORE" ]; then
NM_PRESERVE_ACTIVE_BRIDGE=yes
NM_PRESERVED_PROFILE_UUID=$NM_PROFILE_UUID
vc_nm_preserve_bridge_ready || exit 71
fi
fi
fi
vc_firewall_mode_ready || exit 71
[ -f /etc/virt-cloud/wg-fabric0.owner ] && [ "$(cat /etc/virt-cloud/wg-fabric0.owner)" = "$FABRIC_OWNER" ] || exit 71
"$WG" show "$WG_IFACE" >/dev/null 2>&1 || exit 71
NFT_MAIN_COUNT=0
for nft_candidate in /etc/nftables.conf /etc/sysconfig/nftables.conf; do
if [ -f "$nft_candidate" ] && [ ! -L "$nft_candidate" ] && [ "$("$STAT" -c '%u' "$nft_candidate" 2>/dev/null)" = 0 ]; then
case "$("$STAT" -c '%a' "$nft_candidate" 2>/dev/null)" in
[0-7][0145][0145]) : ;;
*) continue ;;
esac
NFT_MAIN="$nft_candidate"; NFT_MAIN_COUNT=$((NFT_MAIN_COUNT + 1))
fi
done
[ "$NFT_MAIN_COUNT" -eq 1 ] || exit 71
[ ! -e "$LEGACY_MANIFEST" ] && [ ! -L "$LEGACY_MANIFEST" ] || exit 71
if [ -L /etc/nftables.d ] || {{ [ -e /etc/nftables.d ] && [ ! -d /etc/nftables.d ]; }}; then exit 71; fi
if [ -e /etc/nftables.d ]; then
[ "$("$STAT" -c '%u:%a' /etc/nftables.d 2>/dev/null)" = 0:755 ] || exit 71
else
mkdir /etc/nftables.d; chmod 755 /etc/nftables.d
fi
if [ -L /etc/sysctl.d ] || {{ [ -e /etc/sysctl.d ] && [ ! -d /etc/sysctl.d ]; }}; then exit 71; fi
if [ -e /etc/sysctl.d ]; then
[ "$("$STAT" -c '%u:%a' /etc/sysctl.d 2>/dev/null)" = 0:755 ] || exit 71
else
mkdir /etc/sysctl.d; chmod 755 /etc/sysctl.d
fi
if [ -e "$PROXY_FILE" ] || [ -L "$PROXY_FILE" ]; then
[ -f "$PROXY_FILE" ] && [ ! -L "$PROXY_FILE" ] || exit 71
[ "$($STAT -c '%u:%a' "$PROXY_FILE" 2>/dev/null)" = 0:600 ] || exit 71
sed -n '1p' "$PROXY_FILE" | grep -Fqx '# virt-cloud-vpc-proxy-v1' || exit 71
awk 'NR > 1 && $0 !~ /^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$/ {{ exit 1 }}' "$PROXY_FILE" || exit 71
fi
if [ -e "$NFT_FILE" ] || [ -L "$NFT_FILE" ]; then
[ -f "$NFT_FILE" ] && [ ! -L "$NFT_FILE" ] || exit 71
[ "$("$STAT" -c '%u' "$NFT_FILE" 2>/dev/null)" = 0 ] || exit 71
grep -Fqx '# virt-cloud-vpc-dataplane-v1' "$NFT_FILE" || exit 71
fi
if [ -L "$OWNER_FILE" ] || {{ [ -e "$OWNER_FILE" ] && [ ! -f "$OWNER_FILE" ]; }}; then exit 71; fi
if [ -L "$STATE_FILE" ] || {{ [ -e "$STATE_FILE" ] && [ ! -f "$STATE_FILE" ]; }}; then exit 71; fi
if [ -f "$STATE_FILE" ]; then
[ "$($STAT -c '%u:%a' "$STATE_FILE" 2>/dev/null)" = 0:600 ] || exit 71
grep -Fqx 'owner|{DATAPLANE_OWNER_MARKER}' "$STATE_FILE" || exit 71
grep -Fqx 'vpc|{plan.vpc.vpc_id}' "$STATE_FILE" || exit 71
grep -Fqx 'fabric|{plan.vpc.fabric_id}' "$STATE_FILE" || exit 71
grep -Fqx 'host|{plan.host_id}' "$STATE_FILE" || exit 71
grep -Fqx 'bridge|{r.bridge}' "$STATE_FILE" || exit 71
grep -Fqx 'table|{r.route_table}' "$STATE_FILE" || exit 71
state_firewall_record=$(awk -F'|' '$1=="firewall_mode" {{count++; value=$2}} END {{print (count + 0) "|" value}}' "$STATE_FILE")
case "$state_firewall_record" in '0|'|'1|native_nftables') : ;; *) exit 71 ;; esac
state_revision=$(awk -F'|' '$1=="revision" {{print $2; exit}}' "$STATE_FILE")
state_digest=$(awk -F'|' '$1=="digest" {{print $2; exit}}' "$STATE_FILE")
case "$state_revision" in ''|*[!0-9]*) exit 79;; esac
printf '%s\n' "$state_digest" | grep -Eq '^[0-9a-f]{{64}}$' || exit 79
[ "$state_revision" -le {plan.desired_revision} ] || exit 79
if [ "$state_revision" = {plan.desired_revision} ] && [ "$state_digest" != { _q(plan.desired_digest) } ]; then exit 79; fi
fi
if [ -e "$SYSCTL_FILE" ] || [ -L "$SYSCTL_FILE" ]; then
# A deterministic token and a copied owner header are not authority to
# replace a sysctl file. Bind the complete fixed-shape artifact to the
# durable state that created it; a first apply must see the path absent.
[ -f "$STATE_FILE" ] || exit 71
vc_vpc_sysctl_artifact_owned "$SYSCTL_FILE" { _q(plan.vpc.vpc_id) } \
"$state_revision" "$state_digest" "$BRIDGE" || exit 71
fi
if [ -e "$TXN" ] || [ -L "$TXN" ]; then
# A process can die after writing the committed marker but before removing
# its transaction directory. That state is safe to reap after validating
# ownership; preparing/ready/rollback_failed states remain blocked so a
# later reconcile cannot guess whether runtime resources were partially
# applied.
vc_vpc_native_transaction_owned || exit 79
vc_vpc_transaction_record_owned "$TXN/inflight" \
'{DATAPLANE_OWNER_MARKER}:{plan.vpc.vpc_id}:{plan.host_id}:{plan.desired_revision}:{plan.desired_digest}' || exit 79
vc_vpc_transaction_record_owned "$TXN/state" committed || exit 79
rm -rf "$TXN" || exit 79
fi
# Fail closed on any foreign rule selecting this VPC CIDR or route table
# before installing the destructive rollback trap or mutating host state.
vc_ip_rule_scope_check {_q(plan.vpc.cidr)} "$ROUTE_TABLE" "$RULE_PRIORITY" "$WG_IFACE" managed 73 73 || exit $?
# Read-only preflight leaves the original ACL and link intact. The journal
# must contain the runtime baseline before quarantine can flush kernel state.
VC_VPC_APPLY_PHASE=transaction_snapshot
mkdir -m 700 "$TXN"
vc_vpc_write_native_transaction_mode || exit 79
printf '%s\\n' '{DATAPLANE_OWNER_MARKER}:{plan.vpc.vpc_id}:{plan.host_id}:{plan.desired_revision}:{plan.desired_digest}' > "$TXN/inflight"
chmod 600 "$TXN/inflight"
printf '%s\\n' preparing > "$TXN/state"; chmod 600 "$TXN/state"
rollback() {{
status=$?
trap - EXIT HUP INT TERM
vc_vpc_apply_failure_log "$status" || true
[ "$COMMITTED" = yes ] && exit "$status"
set +e
if [ "$MUTATION_STARTED" = no ]; then
# Snapshot/preflight failure has not touched any dataplane resource.
rm -rf "$TXN" || exit 79
exit "$status"
fi
# Link-down removes kernel proxy-neighbour entries. Prove the exact
# installed set before quarantine, then permit only that same set or the
# empty set produced by our verified DOWN transition. Foreign entries,
# failed reads and every earlier rollback failure remain fail-closed.
PROXY_ROLLBACK_EXPECTED=no; PROXY_ROLLBACK_QUARANTINED=no
vc_vpc_rollback_proxy_snapshot() {{
vc_rb_proxy_path=$1
vc_rb_proxy_raw=$("$IP" neigh show proxy dev "$BRIDGE" 2>/dev/null) || return 1
vc_rb_proxy_values=$(printf '%s\\n' "$vc_rb_proxy_raw" | awk -v wanted="$BRIDGE" '
NF == 0 {{next}}
{{
if (NF != 1 && !(NF == 2 && $2 == "proxy") && !(NF == 4 && $2 == "dev" && $3 == wanted && $4 == "proxy")) {{bad=1; next}}
count=split($1, octets, ".")
if (count != 4) {{bad=1; next}}
valid=1
for (pos=1; pos<=4; pos++)
if (octets[pos] !~ /^(0|[1-9][0-9]*)$/ || octets[pos]+0 > 255) valid=0
if (!valid) {{bad=1; next}}
print $1
}}
END {{if (bad) exit 1}}
') || return 1
if [ -n "$vc_rb_proxy_values" ]; then
printf '%s\\n' "$vc_rb_proxy_values" | LC_ALL=C sort -u > "$vc_rb_proxy_path" || return 1
else
: > "$vc_rb_proxy_path" || return 1
fi
}}
if [ "$PROXY_ENTRIES_CHANGED" = yes ]; then
cat <<'{proxy_marker}' | LC_ALL=C sort -u > "$TXN/proxy.rollback.desired"
{proxy_lines}
{proxy_marker}
elif [ "$RUNTIME_QUARANTINED" = yes ]; then
cp "$TXN/proxy.before" "$TXN/proxy.rollback.desired" || ROLLBACK_FAILED=yes
fi
if [ "$PROXY_ENTRIES_CHANGED" = yes ] || [ "$RUNTIME_QUARANTINED" = yes ]; then
if vc_vpc_rollback_proxy_snapshot "$TXN/proxy.rollback.prequarantine" &&
{{ "$CMP" -s "$TXN/proxy.rollback.prequarantine" "$TXN/proxy.rollback.desired" ||
{{ [ "$PROXY_ENTRIES_CHANGED" = no ] && [ "$RUNTIME_QUARANTINED" = yes ] && [ ! -s "$TXN/proxy.rollback.prequarantine" ]; }}; }}; then
PROXY_ROLLBACK_EXPECTED=yes
else
ROLLBACK_FAILED=yes
fi
fi
if [ "$BRIDGE_TOPOLOGY_CAPTURED" = yes ] || [ "$LOCAL_NEIGHBORS_CHANGED" = yes ]; then
vc_vpc_neighbors_rollback_guard || ROLLBACK_FAILED=yes
fi
# First rollback action: quarantine this already-proven panel bridge before
# touching routes, peers, proxy state, sysctls or raw nft tables. This
# closes the failure window where an ACL readback error could otherwise
# leave an active interface exposed while unrelated rollback work runs.
if "$IP" link show dev "$BRIDGE" >/dev/null 2>&1; then
if [ "$MODE" = networkmanager_nmcli ] && vc_nm_profile_resolve; then
vc_nm_device_quarantine || ROLLBACK_FAILED=yes
[ "$ROLLBACK_FAILED" = yes ] || NM_DEVICE_QUARANTINED=yes
fi
"$IP" link set dev "$BRIDGE" down >/dev/null 2>&1 || ROLLBACK_FAILED=yes
rollback_bridge_link_state=$(vc_vpc_bridge_link_state)
if [ "$?" -ne 0 ] || [ "$rollback_bridge_link_state" != down ]; then
ROLLBACK_FAILED=yes
else
PROXY_ROLLBACK_QUARANTINED=yes
fi
if [ "$BRIDGE_TOPOLOGY_CAPTURED" = yes ]; then
vc_vpc_bridge_topology_matches || ROLLBACK_FAILED=yes
fi
fi
if [ "$ROUTES_CHANGED" = yes ]; then
while IFS='|' read -r destination device table via metric; do
[ -n "$destination" ] || continue
actual=$("$IP" -o -4 -N route show table "$table" "$destination" 2>/dev/null || true)
[ -n "$actual" ] || continue
actual_count=$(printf '%s\n' "$actual" | awk 'NF {{count++}} END {{print count + 0}}')
actual_device=$(printf '%s\n' "$actual" | awk '{{for (i=1;i<=NF;i++) if ($i=="dev" && i<NF) {{print $(i+1); exit}}}}')
actual_via=$(printf '%s\n' "$actual" | awk '{{for (i=1;i<=NF;i++) if ($i=="via" && i<NF) {{print $(i+1); exit}}}}')
actual_metric=$(printf '%s\n' "$actual" | awk '{{for (i=1;i<=NF;i++) if ($i=="metric" && i<NF) {{print $(i+1); exit}}}}')
actual_proto=$(printf '%s\n' "$actual" | awk '{{for (i=1;i<=NF;i++) if ($i=="proto" && i<NF) {{print $(i+1); exit}}}}')
[ -n "$actual_metric" ] || actual_metric=100
if [ "$actual_count" != 1 ] || [ "$actual_device" != "$device" ] \
|| [ "${{actual_via:--}}" != "${{via:--}}" ] \
|| [ "$actual_metric" != "$metric" ] || [ "$actual_proto" != 186 ]; then
# A route with the same destination may have been replaced by an
# operator or another VPC while this transaction was failing. Never
# delete that route merely because its destination matches ours.
ROLLBACK_FAILED=yes
continue
fi
if [ "$via" = - ]; then
"$IP" route del "$destination" table "$table" dev "$device" proto 186 metric "$metric" 2>/dev/null || ROLLBACK_FAILED=yes
else
"$IP" route del "$destination" table "$table" via "$via" dev "$device" proto 186 metric "$metric" 2>/dev/null || ROLLBACK_FAILED=yes
fi
done <<'{route_marker}'
{route_lines}
{route_marker}
fi
# vc-policy-rule-rollback-begin
if [ "$ROLLBACK_FAILED" = no ] && [ -s "$TXN/rules.attempted" ]; then
while IFS='|' read -r rule_pref rule_iif rule_to rule_table; do
case "$rule_pref|$rule_iif|$rule_to|$rule_table" in
"$RULE_PRIORITY|-|{plan.vpc.cidr}|$ROUTE_TABLE"|"$WG_RULE_PRIORITY|$WG_IFACE|{plan.vpc.cidr}|$ROUTE_TABLE") : ;;
*) ROLLBACK_FAILED=yes; continue ;;
esac
if [ ! -f "$TXN/rules.created" ] ||
! grep -Fqx "$rule_pref|$rule_iif|$rule_to|$rule_table" "$TXN/rules.created"; then
if vc_ip_rule_read "$rule_pref" "$rule_iif" "$rule_to" "$rule_table" 79 79 &&
[ "$VC_IP_RULE_PRESENT" = no ]; then
continue
fi
ROLLBACK_FAILED=yes
continue
fi
if ! vc_ip_rule_read "$rule_pref" "$rule_iif" "$rule_to" "$rule_table" 79 79; then
ROLLBACK_FAILED=yes
continue
fi
[ "$VC_IP_RULE_PRESENT" = yes ] || continue
if [ "$rule_iif" = - ]; then
"$IP" -4 rule del pref "$rule_pref" to "$rule_to" lookup "$rule_table" 2>/dev/null || ROLLBACK_FAILED=yes
else
"$IP" -4 rule del pref "$rule_pref" iif "$rule_iif" to "$rule_to" lookup "$rule_table" 2>/dev/null || ROLLBACK_FAILED=yes
fi
if ! vc_ip_rule_read "$rule_pref" "$rule_iif" "$rule_to" "$rule_table" 79 79 ||
[ "$VC_IP_RULE_PRESENT" != no ]; then
ROLLBACK_FAILED=yes
fi
done < "$TXN/rules.attempted"
fi
# vc-policy-rule-rollback-end
if [ -f "$TXN/wg.conf" ]; then "$WG" setconf "$WG_IFACE" "$TXN/wg.conf" >/dev/null 2>&1 || ROLLBACK_FAILED=yes; fi
if [ "$PROXY_ENTRIES_CHANGED" = yes ] || [ "$RUNTIME_QUARANTINED" = yes ]; then
if [ "$ROLLBACK_FAILED" = no ] && [ "$PROXY_ROLLBACK_EXPECTED" = yes ] &&
vc_vpc_rollback_proxy_snapshot "$TXN/proxy.rollback.current" &&
{{ "$CMP" -s "$TXN/proxy.rollback.current" "$TXN/proxy.rollback.desired" ||
{{ [ "$PROXY_ROLLBACK_QUARANTINED" = yes ] && [ ! -s "$TXN/proxy.rollback.current" ]; }}; }}; then
while IFS= read -r rollback_proxy; do
[ -n "$rollback_proxy" ] || continue
"$IP" neigh del proxy "$rollback_proxy" dev "$BRIDGE" >/dev/null 2>&1 || ROLLBACK_FAILED=yes
done < "$TXN/proxy.rollback.current"
else
# Never remove a proxy entry which appeared after quarantine, or
# restore over a pre-quarantine set that did not match this operation.
ROLLBACK_FAILED=yes
fi
fi
if [ "$PROXY_FILE_CHANGED" = yes ]; then
if [ -f "$TXN/proxy_file.before" ]; then
cp -p "$TXN/proxy_file.before" "$PROXY_FILE" || ROLLBACK_FAILED=yes
else
rm -f "$PROXY_FILE" || ROLLBACK_FAILED=yes
fi
fi
# Restore runtime sysctls while the new ARP guard is still installed; this
# avoids a rollback window in which proxy_arp is enabled without its exact
# output filter.
if [ "$PROXY_ARP_CHANGED" = yes ] && [ -e "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp" ]; then
if [ -f "$TXN/proxy_arp.before" ]; then
cat "$TXN/proxy_arp.before" > "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp" || ROLLBACK_FAILED=yes
else
printf '%s\\n' 0 > "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp" || ROLLBACK_FAILED=yes
fi
fi
if [ "$PROXY_ARP_PVLAN_CHANGED" = yes ] && [ -e "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp_pvlan" ]; then
if [ -f "$TXN/proxy_arp_pvlan.before" ]; then
cat "$TXN/proxy_arp_pvlan.before" > "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp_pvlan" || ROLLBACK_FAILED=yes
else
printf '%s\n' 0 > "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp_pvlan" || ROLLBACK_FAILED=yes
fi
fi
if [ "$RP_FILTER_CHANGED" = yes ] && [ -e "/proc/sys/net/ipv4/conf/$BRIDGE/rp_filter" ]; then
if [ -f "$TXN/rp_filter.before" ]; then
cat "$TXN/rp_filter.before" > "/proc/sys/net/ipv4/conf/$BRIDGE/rp_filter" || ROLLBACK_FAILED=yes
fi
fi
NFT_ROLLBACK="$TXN/rollback.nft"
NFT_ROLLBACK_SAFE=yes
if [ "$NFT_APPLIED" = yes ]; then
# A table name or owner comment is not sufficient ownership evidence.
# Refuse the entire atomic restore unless every currently live table is
# byte-for-byte equal to this transaction's post-apply stateless dump.
vc_vpc_nft_matches_snapshot "$TXN/nft.desired" || NFT_ROLLBACK_SAFE=no
fi
if [ "$FABRIC_NFT_APPLIED" = yes ]; then
[ -f "$TXN/fabric.nft.desired" ] &&
[ ! -L "$TXN/fabric.nft.desired" ] &&
vc_nft_table_matches_snapshot inet virt_cloud_wg_fabric \
"$TXN/fabric.nft.desired" || NFT_ROLLBACK_SAFE=no
fi
[ "$NFT_ROLLBACK_SAFE" = yes ] || ROLLBACK_FAILED=yes
if [ "$ROLLBACK_FAILED" = no ]; then
: > "$NFT_ROLLBACK"
if [ "$NFT_APPLIED" = yes ]; then
for rollback_family in {nft_family_words}; do
printf 'delete table %s %s\\n' "$rollback_family" "$TABLE" >> "$NFT_ROLLBACK"
done
if [ -f "$TXN/nft.before" ]; then cat "$TXN/nft.before" >> "$NFT_ROLLBACK"; fi
if [ -f "$TXN/nft.bridge.before" ]; then cat "$TXN/nft.bridge.before" >> "$NFT_ROLLBACK"; fi
if [ -f "$TXN/nft.arp.before" ]; then cat "$TXN/nft.arp.before" >> "$NFT_ROLLBACK"; fi
fi
if [ "$FABRIC_NFT_APPLIED" = yes ]; then
vc_vpc_fabric_nft_shape_file "$TXN/fabric.nft.before" || ROLLBACK_FAILED=yes
if [ "$ROLLBACK_FAILED" = no ]; then
printf 'delete table inet virt_cloud_wg_fabric\\n' >> "$NFT_ROLLBACK"
cat "$TXN/fabric.nft.before" >> "$NFT_ROLLBACK"
fi
fi
fi
if [ "$ROLLBACK_FAILED" = no ] && [ -s "$NFT_ROLLBACK" ]; then
"$NFT" -c -f "$NFT_ROLLBACK" >/dev/null 2>&1 || ROLLBACK_FAILED=yes
# Check mode has no reservation primitive. Repeat the complete desired
# snapshots after dry-run and immediately before the one atomic restore;
# a same-name foreign replacement must never be deleted by rollback.
if [ "$ROLLBACK_FAILED" = no ] && [ "$NFT_APPLIED" = yes ]; then
vc_vpc_nft_matches_snapshot "$TXN/nft.desired" ||
ROLLBACK_FAILED=yes
fi
if [ "$ROLLBACK_FAILED" = no ] &&
[ "$FABRIC_NFT_APPLIED" = yes ]; then
vc_nft_table_matches_snapshot inet virt_cloud_wg_fabric \
"$TXN/fabric.nft.desired" || ROLLBACK_FAILED=yes
fi
if [ "$ROLLBACK_FAILED" = no ]; then
"$NFT" -f "$NFT_ROLLBACK" >/dev/null 2>&1 || ROLLBACK_FAILED=yes
fi
if [ "$ROLLBACK_FAILED" = no ] && [ "$NFT_APPLIED" = yes ]; then
vc_vpc_nft_matches_snapshot "$TXN/nft.precommit.before" ||
ROLLBACK_FAILED=yes
fi
if [ "$ROLLBACK_FAILED" = no ] &&
[ "$FABRIC_NFT_APPLIED" = yes ]; then
vc_nft_table_matches_snapshot inet virt_cloud_wg_fabric \
"$TXN/fabric.nft.before" || ROLLBACK_FAILED=yes
fi
fi
if [ "$NFT_FILE_CHANGED" = yes ]; then
if [ -f "$TXN/nft_file.before" ]; then cp -p "$TXN/nft_file.before" "$NFT_FILE" || ROLLBACK_FAILED=yes; else rm -f "$NFT_FILE" || ROLLBACK_FAILED=yes; fi
fi
if [ "$NFT_MAIN_CHANGED" = yes ] && [ -f "$TXN/nft_main.before" ]; then
cp -p "$TXN/nft_main.before" "$NFT_MAIN" || ROLLBACK_FAILED=yes
fi
if [ "$MODE" = networkmanager_nmcli ] && [ -n "$NMCLI" ]; then
if [ "$NM_ACTIVATION_ATTEMPTED" = yes ] && [ "$NM_WAS_ACTIVE" = no ]; then
rollback_nm_ports=$(vc_bridge_port_records "$BRIDGE" 2>/dev/null)
if [ "$?" -ne 0 ] || [ -n "$rollback_nm_ports" ]; then
# Never destroy a bridge that acquired a concurrent libvirt tap.
ROLLBACK_FAILED=yes
else
vc_nm_device_expect_managed no || ROLLBACK_FAILED=yes
"$IP" link set dev "$BRIDGE" down >/dev/null 2>&1 || ROLLBACK_FAILED=yes
fi
if [ "$ROLLBACK_FAILED" = no ]; then
vc_nm_profile_inactive || ROLLBACK_FAILED=yes
fi
fi
if [ "$NM_CREATED" = yes ] && [ "$ROLLBACK_FAILED" = no ]; then
rollback_nm_ports=$(vc_bridge_port_records "$BRIDGE" 2>/dev/null)
if [ "$?" -ne 0 ] || [ -n "$rollback_nm_ports" ]; then
ROLLBACK_FAILED=yes
else
# The add may have succeeded while exact UUID/profile readback failed.
# A same-name delete is ambiguous and could remove an operator profile.
vc_nm_created_profile_remove || ROLLBACK_FAILED=yes
fi
fi
fi
if [ "$IF_FRAGMENT_CHANGED" = yes ]; then
if [ -f "$TXN/if_fragment.before" ]; then cp -p "$TXN/if_fragment.before" "$IF_FRAGMENT" || ROLLBACK_FAILED=yes; else rm -f "$IF_FRAGMENT" || ROLLBACK_FAILED=yes; fi
fi
if [ "$SYSCTL_CHANGED" = yes ]; then
# The deterministic pathname is not authority to overwrite or delete a
# file that changed after this transaction installed it. Restore the old
# snapshot (or absence) only while the live bytes and full desired
# identity are still exactly the artifact saved in this journal.
vc_vpc_sysctl_artifacts_equal "$SYSCTL_FILE" \
"$TXN/sysctl.desired" {_q(plan.vpc.vpc_id)} \
{plan.desired_revision} {_q(plan.desired_digest)} "$BRIDGE" || \
ROLLBACK_FAILED=yes
if [ "$ROLLBACK_FAILED" = no ]; then
if [ -f "$TXN/sysctl.before" ]; then
cp -p "$TXN/sysctl.before" "$SYSCTL_FILE" || ROLLBACK_FAILED=yes
else
rm -f "$SYSCTL_FILE" || ROLLBACK_FAILED=yes
fi
fi
fi
if [ -n "$SYSCTL_TMP" ]; then
rm -f "$SYSCTL_TMP" || ROLLBACK_FAILED=yes
fi
ROLLBACK_NEW_BRIDGE_RETAINED=no
if [ "$BRIDGE_CREATED" = yes ]; then
# Retain the owner and journal until deletion of this new bridge is
# confirmed. A concurrent tap, an inspection error, or a failed delete
# must not turn the bridge into an unowned resource on the next retry.
ROLLBACK_NEW_BRIDGE_RETAINED=yes
ports=$(vc_bridge_port_records "$BRIDGE" 2>/dev/null)
if [ "$?" -ne 0 ] || [ -n "$ports" ]; then
ROLLBACK_FAILED=yes
elif "$IP" link del dev "$BRIDGE" type bridge 2>/dev/null; then
ROLLBACK_NEW_BRIDGE_RETAINED=no
else
ROLLBACK_FAILED=yes
fi
elif [ "$BRIDGE_WAS_UP" = no ] && "$IP" link show dev "$BRIDGE" >/dev/null 2>&1; then
# Applying a revision may have activated an existing panel bridge through
# NetworkManager/ifupdown. Restore its prior DOWN state only when no tap
# or operator port appeared while the transaction was running.
ports=$(vc_bridge_port_records "$BRIDGE" 2>/dev/null)
if [ "$?" -ne 0 ] || [ -n "$ports" ]; then
ROLLBACK_FAILED=yes
else
"$IP" link set dev "$BRIDGE" down 2>/dev/null || ROLLBACK_FAILED=yes
rollback_bridge_link_state=$(vc_vpc_bridge_link_state)
if [ "$?" -ne 0 ] || [ "$rollback_bridge_link_state" != down ]; then
ROLLBACK_FAILED=yes
fi
fi
fi
if [ "$MODE" = ifupdown ] &&
[ "$BRIDGE_CREATED" = no ] && [ "$BRIDGE_WAS_UP" = yes ] &&
[ "$ROLLBACK_FAILED" = no ]; then
# The existing bridge was hard-quarantined before preflight. Re-open it
# only after the exact pre-transaction nft semantics and unchanged port
# topology have both been proven; a new or sampled-DOWN bridge is never
# brought up by rollback.
vc_vpc_native_ifupdown_rollback_reopen || ROLLBACK_FAILED=yes
fi
# Return a quarantined native NetworkManager profile to its exact prior
# managed/active state only after restoring the original nft ACL.
if [ "$MODE" = networkmanager_nmcli ] &&
[ "$NM_PROFILE_WAS_PRESENT" = yes ] &&
[ "$NM_DEVICE_QUARANTINED" = yes ] &&
[ "$ROLLBACK_FAILED" = no ]; then
if vc_nm_profile_resolve && vc_nm_profile_base_validate &&
{{ {{ [ "$NM_PRESERVE_ACTIVE_BRIDGE" = yes ] && vc_nm_preserve_bridge_ready; }} ||
{{ [ "$NM_PRESERVE_ACTIVE_BRIDGE" = no ] && vc_nm_device_expect_managed no; }}; }} &&
vc_vpc_nft_matches_snapshot "$TXN/nft.precommit.before" &&
vc_nft_table_matches_snapshot inet virt_cloud_wg_fabric "$TXN/fabric.nft.before" &&
vc_vpc_bridge_topology_matches; then
vc_nm_device_release || ROLLBACK_FAILED=yes
if [ "$ROLLBACK_FAILED" = no ] && [ "$NM_WAS_ACTIVE" = yes ]; then
if [ "$NM_PRESERVE_ACTIVE_BRIDGE" = yes ]; then
vc_nm_preserve_bridge_ready || ROLLBACK_FAILED=yes
else
"$NMCLI" connection up uuid "$NM_PROFILE_UUID" >/dev/null 2>&1 ||
ROLLBACK_FAILED=yes
fi
if [ "$ROLLBACK_FAILED" = no ]; then
if [ "$BRIDGE_WAS_UP" = yes ]; then
"$IP" link set dev "$BRIDGE" up >/dev/null 2>&1 ||
ROLLBACK_FAILED=yes
else
"$IP" link set dev "$BRIDGE" down >/dev/null 2>&1 ||
ROLLBACK_FAILED=yes
fi
fi
[ "$ROLLBACK_FAILED" = no ] && vc_nm_profile_active ||
ROLLBACK_FAILED=yes
elif [ "$ROLLBACK_FAILED" = no ]; then
vc_nm_profile_active && ROLLBACK_FAILED=yes || true
"$IP" link set dev "$BRIDGE" down >/dev/null 2>&1 ||
ROLLBACK_FAILED=yes
vc_nm_device_expect_managed yes || ROLLBACK_FAILED=yes
fi
[ "$ROLLBACK_FAILED" = no ] && vc_vpc_bridge_topology_matches ||
ROLLBACK_FAILED=yes
[ "$ROLLBACK_FAILED" = yes ] || NM_DEVICE_QUARANTINED=no
else
ROLLBACK_FAILED=yes
fi
fi
if [ "$ROLLBACK_FAILED" = no ] && [ "$BRIDGE_CREATED" = no ] &&
[ "$BRIDGE_TOPOLOGY_CAPTURED" = yes ]; then
vc_vpc_bridge_topology_matches && vc_vpc_runtime_restore &&
vc_vpc_neighbors_restore && vc_vpc_bridge_topology_matches || ROLLBACK_FAILED=yes
fi
if [ "$ROLLBACK_NEW_BRIDGE_RETAINED" = no ]; then
if [ -f "$TXN/owner.before" ]; then cp -p "$TXN/owner.before" "$OWNER_FILE" || ROLLBACK_FAILED=yes; else rm -f "$OWNER_FILE" || ROLLBACK_FAILED=yes; fi
fi
if [ -f "$TXN/state.before" ]; then cp -p "$TXN/state.before" "$STATE_FILE" || ROLLBACK_FAILED=yes; else rm -f "$STATE_FILE" || ROLLBACK_FAILED=yes; fi
if [ "$ROLLBACK_FAILED" = yes ] && [ "$BRIDGE_TOPOLOGY_CAPTURED" = yes ]; then
# A late manager/readback failure must not leave the bridge reopened.
if [ "$MODE" = networkmanager_nmcli ]; then
vc_nm_device_quarantine || true
fi
"$IP" link set dev "$BRIDGE" down >/dev/null 2>&1 || true
fi
if [ "$ROLLBACK_FAILED" = yes ]; then
VC_VPC_APPLY_PHASE=rollback
vc_vpc_apply_failure_log 79 || true
printf '%s\\n' rollback_failed > "$TXN/state"; chmod 600 "$TXN/state"; exit 79
fi
rm -rf "$TXN"; exit "$status"
}}
trap rollback EXIT HUP INT TERM
trap 'vc_vpc_apply_signal_exit 129' HUP
trap 'vc_vpc_apply_signal_exit 130' INT
trap 'vc_vpc_apply_signal_exit 143' TERM
if [ -f "$OWNER_FILE" ]; then
[ ! -L "$OWNER_FILE" ] && grep -Fqx "$OWNER_MARKER" "$OWNER_FILE" || exit 71
cp -p "$OWNER_FILE" "$TXN/owner.before"
fi
if [ -f "$STATE_FILE" ]; then
[ ! -L "$STATE_FILE" ] || exit 71
grep -Fqx 'owner|{DATAPLANE_OWNER_MARKER}' "$STATE_FILE" || exit 71
cp -p "$STATE_FILE" "$TXN/state.before"
fi
for snapshot_family in inet bridge arp; do
[ ! -e "$NFT_SNAPSHOT_PREFIX.$snapshot_family.live" ] &&
[ ! -L "$NFT_SNAPSHOT_PREFIX.$snapshot_family.live" ] || exit 71
done
"$WG" showconf "$WG_IFACE" > "$TXN/wg.conf"; chmod 600 "$TXN/wg.conf"
if "$NFT" -s list table inet virt_cloud_wg_fabric > "$TXN/fabric.nft.before" 2>/dev/null; then
vc_vpc_fabric_nft_shape_file "$TXN/fabric.nft.before" || exit 71
else
exit 71
fi
if "$NFT" list table inet "$TABLE" > "$TXN/nft.before" 2>/dev/null; then
grep -Fq '{DATAPLANE_OWNER_MARKER}' "$TXN/nft.before" || exit 71
else
rm -f "$TXN/nft.before"
fi
if "$NFT" list table bridge "$TABLE" > "$TXN/nft.bridge.before" 2>/dev/null; then
grep -Fq '{DATAPLANE_OWNER_MARKER}' "$TXN/nft.bridge.before" || exit 71
else
rm -f "$TXN/nft.bridge.before"
fi
if "$NFT" list table arp "$TABLE" > "$TXN/nft.arp.before" 2>/dev/null; then
grep -Fq '{DATAPLANE_OWNER_MARKER}' "$TXN/nft.arp.before" || exit 71
else
rm -f "$TXN/nft.arp.before"
fi
# Preserve a complete stateless/absence baseline before any live nft batch.
# Rollback uses the same evidence before restoring a sampled-UP ifupdown
# bridge, and the apply precommit check detects drift during dry-run.
vc_vpc_capture_nft_precommit "$TXN/nft.precommit.before" || exit 71
cp -p "$NFT_MAIN" "$TXN/nft_main.before"; NFT_MAIN_BEFORE=yes
if [ -f "$NFT_FILE" ]; then cp -p "$NFT_FILE" "$TXN/nft_file.before"; NFT_FILE_BEFORE=yes; fi
if [ -f "$SYSCTL_FILE" ]; then cp -p "$SYSCTL_FILE" "$TXN/sysctl.before"; fi
if [ -f "$PROXY_FILE" ]; then cp -p "$PROXY_FILE" "$TXN/proxy_file.before"; fi
cat > "$TXN/neighbors.desired" <<'__VC_LOCAL_NEIGHBORS__'
{chr(10).join(sorted(f"{m.private_ip}|{m.private_mac}" for m in plan.local_members))}
__VC_LOCAL_NEIGHBORS__
vc_vpc_runtime_baseline || exit 71
vc_vpc_neighbors_baseline || exit 71
if [ "$BRIDGE_TOPOLOGY_CAPTURED" = yes ]; then
# Kernel DOWN and NM unmanaged can flush proxy neighbours and routes. The
# original ACL, owner, manager, topology and runtime are now journaled.
vc_vpc_bridge_topology_matches || exit 72
VC_VPC_APPLY_PHASE=quarantine
MUTATION_STARTED=yes
RUNTIME_QUARANTINED=yes
if [ "$MODE" = networkmanager_nmcli ]; then
NM_DEVICE_QUARANTINED=yes
vc_nm_device_quarantine || exit 72
else
"$IP" link set dev "$BRIDGE" down >/dev/null || exit 72
bridge_quarantine_link_state=$(vc_vpc_bridge_link_state) || exit 72
[ "$bridge_quarantine_link_state" = down ] || exit 72
fi
vc_vpc_bridge_topology_matches || exit 72
FIREWALL_HARD_QUARANTINED=yes
fi
vc_vpc_install_rules() {{
cat > "$TXN/rules.nft" <<'{nft_marker}'
{nft.rstrip()}
{nft_marker}
chmod 600 "$TXN/rules.nft"
"$NFT" -c -f "$TXN/rules.nft" >/dev/null || exit 72
NFT_TMP=$(mktemp "/etc/nftables.d/.virt-cloud-vpc-{r.token}.XXXXXX") || exit 72
cp -p "$TXN/rules.nft" "$NFT_TMP" || exit 72
chmod 600 "$NFT_TMP"
mv -f "$NFT_TMP" "$NFT_FILE"; NFT_FILE_CHANGED=yes
if ! grep -Fqx "$NFT_INCLUDE" "$NFT_MAIN" &&
! grep -Fqx 'include "/etc/nftables.d/*.nft"' "$NFT_MAIN" &&
! grep -Fqx 'include "/etc/nftables.d/*"' "$NFT_MAIN"; then
NFT_MAIN_TMP=$(mktemp "$NFT_MAIN.virt-cloud-vpc.XXXXXX") || exit 72
cat "$NFT_MAIN" > "$NFT_MAIN_TMP"
printf '%s\\n' "$NFT_INCLUDE" >> "$NFT_MAIN_TMP"
chmod "$($STAT -c '%a' "$NFT_MAIN")" "$NFT_MAIN_TMP"
"$NFT" -c -f "$NFT_MAIN_TMP" >/dev/null || exit 72
mv -f "$NFT_MAIN_TMP" "$NFT_MAIN"; NFT_MAIN_CHANGED=yes
fi
"$NFT" -c -f "$NFT_MAIN" >/dev/null || exit 72
NFT_BATCH="$TXN/apply.nft"
: > "$NFT_BATCH"
if [ -f "$TXN/nft.before" ]; then printf 'delete table inet %s\\n' "$TABLE" >> "$NFT_BATCH"; fi
if [ -f "$TXN/nft.bridge.before" ]; then printf 'delete table bridge %s\\n' "$TABLE" >> "$NFT_BATCH"; fi
if [ -f "$TXN/nft.arp.before" ]; then printf 'delete table arp %s\\n' "$TABLE" >> "$NFT_BATCH"; fi
cat "$TXN/rules.nft" >> "$NFT_BATCH"
chmod 600 "$NFT_BATCH"
"$NFT" -c -f "$NFT_BATCH" >/dev/null || exit 72
# nft check mode does not reserve table names or provide compare-and-swap.
# Re-read every full stateless table (or proven absence) after dry-run and
# immediately before the single atomic apply, closing the dry-run mutation
# window for ordinary races.
vc_vpc_nft_matches_snapshot "$TXN/nft.precommit.before" || exit 71
# A single nft batch is atomic in the kernel: the old owned tables remain in
# place if any delete or new-table rule fails validation/application.
"$NFT" -f "$NFT_BATCH" >/dev/null || exit 72
NFT_APPLIED=yes
# Read back all expected ownership markers, then capture the complete
# stateless semantics. The latter includes chain policies, every expression
# and verdict, set elements and rule order; it deliberately excludes only
# changing packet/byte counters.
vc_vpc_desired_nft_ready || exit 72
vc_vpc_capture_nft_snapshot "$TXN/nft.desired" || exit 72
vc_vpc_nft_matches_snapshot "$TXN/nft.desired" || exit 72
}}
# Install and exactly read back the raw tenant guard before restoring link
# administration in either persistence mode.
VC_VPC_APPLY_PHASE=firewall
MUTATION_STARTED=yes
vc_vpc_install_rules
VC_VPC_APPLY_PHASE=bridge
if "$IP" link show dev "$BRIDGE" >/dev/null 2>&1; then
"$IP" -d link show dev "$BRIDGE" | grep -Eq '(^|[[:space:]])bridge[[:space:]]+(forward_delay|stp_state|hello_time|ageing_time)([[:space:]]|$)' || exit 71
[ -f "$OWNER_FILE" ] || exit 71
else
"$IP" link add name "$BRIDGE" type bridge
BRIDGE_CREATED=yes
printf '%s\\n' "$OWNER_MARKER" > "$OWNER_FILE"; chmod 600 "$OWNER_FILE"
fi
bridge_current_link_state=$(vc_vpc_bridge_link_state) || exit 71
if [ "$bridge_current_link_state" = up ]; then BRIDGE_WAS_UP=yes; fi
if [ "$BRIDGE_TOPOLOGY_CAPTURED" = yes ]; then
vc_vpc_bridge_topology_matches || exit 72
fi
# Alma/RHEL may create a new bridge with the distribution's strict
# default=1. Snapshot and converge the panel-owned interface while it is
# still DOWN; the raw nft guard is already installed and exactly read back.
vc_vpc_rp_filter_set || exit 72
cat > "$TXN/sysctl.desired" <<'__VC_VPC_SYSCTL_{r.token}'
# virt-cloud-vpc-dataplane-v1
# vpc={plan.vpc.vpc_id} revision={plan.desired_revision} digest={plan.desired_digest}
net.ipv4.conf.{r.bridge}.rp_filter = 2
net.ipv4.conf.{r.bridge}.proxy_arp = 0
net.ipv4.conf.{r.bridge}.proxy_arp_pvlan = 1
__VC_VPC_SYSCTL_{r.token}
chmod 600 "$TXN/sysctl.desired"
vc_vpc_sysctl_artifact_owned "$TXN/sysctl.desired" {_q(plan.vpc.vpc_id)} \
{plan.desired_revision} {_q(plan.desired_digest)} "$BRIDGE" || exit 72
SYSCTL_TMP=$(mktemp "/etc/sysctl.d/.virt-cloud-vpc-{r.token}.XXXXXX") || exit 72
cp -p "$TXN/sysctl.desired" "$SYSCTL_TMP" || exit 72
chmod 600 "$SYSCTL_TMP" || exit 72
if [ -f "$TXN/sysctl.before" ]; then
vc_vpc_sysctl_artifact_owned "$SYSCTL_FILE" {_q(plan.vpc.vpc_id)} \
"$state_revision" "$state_digest" "$BRIDGE" || exit 71
"$CMP" -s "$SYSCTL_FILE" "$TXN/sysctl.before" || exit 71
else
[ ! -e "$SYSCTL_FILE" ] && [ ! -L "$SYSCTL_FILE" ] || exit 71
fi
mv -f "$SYSCTL_TMP" "$SYSCTL_FILE" || exit 72
SYSCTL_TMP=
SYSCTL_CHANGED=yes
vc_vpc_sysctl_artifacts_equal "$SYSCTL_FILE" "$TXN/sysctl.desired" \
{_q(plan.vpc.vpc_id)} {plan.desired_revision} {_q(plan.desired_digest)} \
"$BRIDGE" || exit 72
"$IP" link set dev "$BRIDGE" up
# A private bridge is an unnumbered, panel-owned L3 boundary. Reject an
# address, a default/connected route, or a physical/foreign bridge port before
# installing any policy. Libvirt tap names are the only accepted members;
# attaching br0, an underlay NIC, or an operator bridge would make VPC traffic
# escape into the public network.
bridge_addrs=$($IP -o addr show dev "$BRIDGE" 2>/dev/null || true)
printf '%s\\n' "$bridge_addrs" | grep -Eq '[[:space:]]inet6?[[:space:]]' && exit 71 || true
bridge_ports=$(vc_bridge_port_records "$BRIDGE" 2>/dev/null) || exit 71
while IFS= read -r bridge_port_line; do
[ -n "$bridge_port_line" ] || continue
bridge_port=$(printf '%s\\n' "$bridge_port_line" | sed -n 's/^[0-9][0-9]*: \\([^:[:space:]]*\\).*/\\1/p')
[ -n "$bridge_port" ] || exit 71
case "$bridge_port" in
vnet[0-9]*|tap[0-9A-Za-z_.-]*|macvtap[0-9]*|fwpr[0-9A-Za-z_.-]*|fwln[0-9A-Za-z_.-]*) : ;;
*) exit 71 ;;
esac
done <<__VC_VPC_BRIDGE_PORTS__
$bridge_ports
__VC_VPC_BRIDGE_PORTS__
cat > "$TXN/bridge.routes.allowed" <<'{route_marker}.bridge'
{bridge_route_lines}
{route_marker}.bridge
if [ -f "$TXN/state.before" ]; then
awk -F'|' '$1 == "route" && $3 == "{r.bridge}" {{print $2 "|" $3 "|" $4 "|" $5 "|" $6 "|186"}}' "$TXN/state.before" >> "$TXN/bridge.routes.allowed"
fi
sort -u "$TXN/bridge.routes.allowed" -o "$TXN/bridge.routes.allowed"
bridge_routes=$($IP -o -4 -N route show table all dev "$BRIDGE" 2>/dev/null) || exit 71
bridge_route_records=$(printf '%s\\n' "$bridge_routes" | vc_vpc_route_records "$BRIDGE" main "$BRIDGE") || exit 71
while IFS='|' read -r destination device table via metric proto flags; do
[ -n "$destination" ] || continue
printf '%s\\n' "$destination|$device|$table|$via|$metric|$proto" | grep -Fqx -f "$TXN/bridge.routes.allowed" || exit 71
done <<EOF
$bridge_route_records
EOF
bridge_routes6=$($IP -o -6 -N route show table all dev "$BRIDGE" 2>/dev/null) || exit 71
[ -z "$bridge_routes6" ] || exit 71
for vc_runtime_key in proxy_arp proxy_arp_pvlan; do
if [ ! -f "$TXN/$vc_runtime_key.before" ]; then
vc_runtime_path="/proc/sys/net/ipv4/conf/$BRIDGE/$vc_runtime_key"
[ -f "$vc_runtime_path" ] && [ ! -L "$vc_runtime_path" ] || exit 72
cat "$vc_runtime_path" > "$TXN/$vc_runtime_key.before" || exit 72
fi
done
VC_VPC_APPLY_PHASE=persistence
if [ "$MODE" = networkmanager_nmcli ]; then
[ -n "$NMCLI" ] || exit 70
NM_PROFILE_UUID=
if vc_nm_profile_resolve; then
vc_nm_profile_base_validate || exit 71
else
nm_resolve_status=$?
[ "$nm_resolve_status" = 2 ] || exit 71
NM_PROFILE_UUID=
"$NMCLI" connection add type bridge ifname "$BRIDGE" con-name "$BRIDGE" \
ipv4.method disabled ipv6.method disabled bridge.stp no \
connection.autoconnect no connection.autoconnect-retries 0 \
connection.stable-id "$NM_PROFILE_MARKER" >/dev/null
NM_CREATED=yes
vc_nm_profile_resolve || exit 72
vc_nm_profile_base_validate || exit 72
NM_CREATED_UUID=$NM_PROFILE_UUID
fi
vc_nm_profile_empty connection.zone || exit 71
vc_nm_profile_resolve || exit 71
vc_nm_profile_base_validate || exit 71
NM_ACTIVATION_ATTEMPTED=yes
if [ "$NM_DEVICE_QUARANTINED" = yes ]; then
vc_nm_device_release || exit 72
NM_DEVICE_QUARANTINED=no
fi
VC_VPC_APPLY_PHASE=activation
if [ "$NM_PRESERVE_ACTIVE_BRIDGE" = yes ]; then
vc_nm_preserve_bridge_ready || exit 72
else
"$NMCLI" connection up uuid "$NM_PROFILE_UUID" >/dev/null
fi
VC_VPC_APPLY_PHASE=post_activation_sysctl
vc_vpc_rp_filter_set || exit 72
else
[ -x "$IFUP" ] && [ -x "$IFQUERY" ] && [ -x "$IFDOWN" ] || exit 70
[ ! -L /etc/network ] && [ ! -L /etc/network/interfaces.d ] || exit 71
# A stanza in the administrator-owned main file wins over an
# interfaces.d fragment on some ifupdown versions. Refuse that ambiguous
# ownership instead of letting a same-name bridge inherit ports, addresses,
# routes, or hooks from /etc/network/interfaces.
if [ -f /etc/network/interfaces ] &&
grep -Eq '^[[:space:]]*(auto|allow-[^[:space:]]+|iface)[[:space:]]+'"$BRIDGE"'([[:space:]]|$)' /etc/network/interfaces; then
exit 71
fi
mkdir -p /etc/network/interfaces.d
if [ -e "$IF_FRAGMENT" ] || [ -L "$IF_FRAGMENT" ]; then
[ -f "$IF_FRAGMENT" ] && [ ! -L "$IF_FRAGMENT" ] || exit 71
grep -Fq '# {DATAPLANE_OWNER_MARKER} vpc=' "$IF_FRAGMENT" || exit 71
cp -p "$IF_FRAGMENT" "$TXN/if_fragment.before"
fi
IF_TMP=$(mktemp "/etc/network/interfaces.d/.virt-cloud-vpc-{r.token}.XXXXXX") || exit 72
cat > "$IF_TMP" <<'{frag_marker}'
# {DATAPLANE_OWNER_MARKER} vpc={plan.vpc.vpc_id} revision={plan.desired_revision} digest={plan.desired_digest}
{ifup_auto_line}iface {r.bridge} inet manual
bridge_ports none
bridge-stp off
{frag_marker}
chmod 600 "$IF_TMP"
mv -f "$IF_TMP" "$IF_FRAGMENT"; IF_FRAGMENT_CHANGED=yes
# Classic Debian ifquery accepts the interface as its positional operand;
# `--list <iface>` is an ifquery2/listing form and fails on ifupdown.
"$IFQUERY" "$BRIDGE" >/dev/null 2>&1 || exit 72
# Rollback never calls ifdown: even a bridge created by this transaction may
# have acquired a libvirt tap before a later step fails. The rollback path
# below inspects ports first, then deletes only an empty newly-created
# bridge or restores an empty existing bridge to its sampled DOWN state.
VC_VPC_APPLY_PHASE=activation
VC_FABRIC_APPLYING=1 "$IFUP" --force "$BRIDGE" >/dev/null 2>&1 || exit 72
VC_VPC_APPLY_PHASE=post_activation_sysctl
vc_vpc_rp_filter_set || exit 72
fi
VC_VPC_APPLY_PHASE=runtime
if [ "$BRIDGE_TOPOLOGY_CAPTURED" = yes ]; then
vc_vpc_bridge_topology_matches || exit 72
fi
PROXY_ARP_CHANGED=yes
printf '%s\\n' 0 > "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp" || exit 72
[ "$(cat "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp" 2>/dev/null)" = 0 ] || exit 72
PROXY_ARP_PVLAN_CHANGED=yes
printf '%s\\n' 1 > "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp_pvlan" || exit 72
[ "$(cat "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp_pvlan" 2>/dev/null)" = 1 ] || exit 72
VC_VPC_APPLY_PHASE=proxy
vc_vpc_proxy_snapshot "$TXN/proxy.activation.current" || exit 72
vc_vpc_runtime_subset "$TXN/proxy.activation.current" "$TXN/proxy.before" || exit 71
PROXY_ENTRIES_CHANGED=yes
while IFS= read -r old_proxy; do
[ -n "$old_proxy" ] || continue
"$IP" neigh del proxy "$old_proxy" dev "$BRIDGE" >/dev/null 2>&1 || exit 72
done < "$TXN/proxy.activation.current"
while IFS= read -r proxy_ip; do
[ -n "$proxy_ip" ] || continue
"$IP" neigh replace proxy "$proxy_ip" dev "$BRIDGE" >/dev/null 2>&1 || exit 72
done <<'{proxy_marker}'
{proxy_lines}
{proxy_marker}
PROXY_TMP=$(mktemp "$PROXY_FILE.XXXXXX") || exit 72
{{
printf '%s\\n' '# virt-cloud-vpc-proxy-v1'
cat <<'{proxy_marker}'
{proxy_lines}
{proxy_marker}
}} > "$PROXY_TMP"
chmod 600 "$PROXY_TMP"
mv -f "$PROXY_TMP" "$PROXY_FILE"; PROXY_FILE_CHANGED=yes
"$IP" neigh show proxy dev "$BRIDGE" 2>/dev/null | awk '$1 ~ /^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$/ {{print $1}}' | LC_ALL=C sort -u > "$TXN/proxy.after"
cat <<'{proxy_marker}' | LC_ALL=C sort -u > "$TXN/proxy.desired"
{proxy_lines}
{proxy_marker}
"$CMP" -s "$TXN/proxy.after" "$TXN/proxy.desired" || exit 72
# Restore exact member bindings after the manager may have cleared them.
VC_VPC_APPLY_PHASE=neighbors
vc_vpc_neighbors_reconcile || exit 72
VC_VPC_APPLY_PHASE=routes
ROUTES_CHANGED=yes
cat > "$TXN/routes.desired" <<'{route_marker}'
{route_lines}
{route_marker}
if [ -f "$STATE_FILE" ]; then
while IFS='|' read -r old_kind old_destination old_device old_table old_via old_metric; do
[ "$old_kind" = route ] || continue
printf '%s\\n' "$old_destination|$old_device|$ROUTE_TABLE|${{old_via:--}}|${{old_metric:-100}}" | grep -Fqx -f "$TXN/routes.desired" && continue
old_existing=$("$IP" -o -4 -N route show table "$ROUTE_TABLE" "$old_destination" 2>/dev/null || true)
[ -n "$old_existing" ] || continue
printf '%s\\n' "$old_existing" | grep -Eq 'proto 186([[:space:]]|$)' || exit 73
old_metric_actual=$(printf '%s\\n' "$old_existing" | awk '{{for (i=1; i<=NF; i++) if ($i == "metric" && i < NF) {{print $(i+1); exit}}}}')
[ -n "$old_metric_actual" ] || old_metric_actual=100
old_actual_device=$(printf '%s\\n' "$old_existing" | awk '{{for (i=1; i<=NF; i++) if ($i == "dev" && i < NF) {{print $(i+1); exit}}}}')
[ "$old_actual_device" = "$old_device" ] || exit 73
old_via_actual=$(printf '%s\\n' "$old_existing" | awk '{{for (i=1; i<=NF; i++) if ($i == "via" && i < NF) {{print $(i+1); exit}}}}')
[ "${{old_via_actual:--}}" = "${{old_via:--}}" ] || exit 73
if [ "${{old_via:--}}" = - ]; then
"$IP" route del "$old_destination" table "$ROUTE_TABLE" dev "$old_device" proto 186 || exit 73
else
"$IP" route del "$old_destination" table "$ROUTE_TABLE" via "$old_via" dev "$old_device" proto 186 || exit 73
fi
done < "$STATE_FILE"
fi
while IFS='|' read -r destination device table via metric; do
[ -n "$destination" ] || continue
existing=$("$IP" -o -4 -N route show table "$table" "$destination" 2>/dev/null || true)
[ -z "$existing" ] || printf '%s\\n' "$existing" | grep -Eq 'proto 186([[:space:]]|$)' || exit 73
if [ -n "$existing" ]; then
old_device=$(printf '%s\\n' "$existing" | awk '{{for (i=1; i<=NF; i++) if ($i == "dev" && i < NF) {{print $(i+1); exit}}}}')
old_metric=$(printf '%s\\n' "$existing" | awk '{{for (i=1; i<=NF; i++) if ($i == "metric" && i < NF) {{print $(i+1); exit}}}}')
[ -n "$old_device" ] || exit 73
[ "$old_device" = "$device" ] || exit 73
old_via=$(printf '%s\\n' "$existing" | awk '{{for (i=1; i<=NF; i++) if ($i == "via" && i < NF) {{print $(i+1); exit}}}}')
[ "${{old_via:--}}" = "$via" ] || exit 73
[ -n "$old_metric" ] || old_metric=100
fi
[ -n "$metric" ] || metric=100
if [ "$via" = - ]; then
"$IP" route replace "$destination" table "$table" dev "$device" proto 186 metric "$metric" || exit 73
else
"$IP" route replace "$destination" table "$table" via "$via" dev "$device" onlink proto 186 metric "$metric" || exit 73
fi
done <<'{route_marker}'
{route_lines}
{route_marker}
# Recheck the complete RPDB scope after route convergence, immediately before
# adding missing rules. An out-of-band rule created during the transaction
# must force rollback rather than make a successful apply read back degraded.
VC_VPC_APPLY_PHASE=policy_rules
# vc-policy-rule-preflight-begin
: > "$TXN/rules.before"
: > "$TXN/rules.attempted"
: > "$TXN/rules.created"
chmod 600 "$TXN/rules.before" "$TXN/rules.attempted" "$TXN/rules.created"
vc_ip_rule_scope_check {_q(plan.vpc.cidr)} "$ROUTE_TABLE" "$RULE_PRIORITY" "$WG_IFACE" managed 73 73 || exit $?
vc_ip_rule_preflight "$RULE_PRIORITY" - {_q(plan.vpc.cidr)} "$ROUTE_TABLE" 73 73 "$TXN/rules.before" || exit $?
vc_ip_rule_preflight "$WG_RULE_PRIORITY" "$WG_IFACE" {_q(plan.vpc.cidr)} "$ROUTE_TABLE" 73 73 "$TXN/rules.before" || exit $?
# vc-policy-rule-preflight-end
# vc-policy-rule-ensure-begin
vc_ip_rule_ensure "$RULE_PRIORITY" - {_q(plan.vpc.cidr)} "$ROUTE_TABLE" 73 73 "$TXN/rules.attempted" "$TXN/rules.created" || exit $?
vc_ip_rule_ensure "$WG_RULE_PRIORITY" "$WG_IFACE" {_q(plan.vpc.cidr)} "$ROUTE_TABLE" 73 73 "$TXN/rules.attempted" "$TXN/rules.created" || exit $?
# vc-policy-rule-ensure-end
VC_VPC_APPLY_PHASE=fabric_acl
cat > "$STATE_FILE" <<'{state_marker}'
{state.rstrip()}
{state_marker}
chmod 600 "$STATE_FILE"
{state_loader}
vpc_state_load || exit 74
FABRIC_NFT_APPLIED=yes
vpc_state_refresh_nft || exit 74
vc_vpc_fabric_nft_live_ready || exit 74
vpc_state_nft_set_matches || exit 74
vc_nft_table_snapshot inet virt_cloud_wg_fabric \
"$TXN/fabric.nft.desired" || exit 74
[ -f "$TXN/fabric.nft.desired" ] &&
[ ! -e "$TXN/fabric.nft.desired.absent" ] || exit 74
vc_nft_table_matches_snapshot inet virt_cloud_wg_fabric \
"$TXN/fabric.nft.desired" || exit 74
VC_VPC_APPLY_PHASE=wireguard
expected_fabric_peer_keys={_q(apply_fabric_peer_key_words)}
if [ "{apply_fabric_peer_keys_provided}" = yes ]; then
live_fabric_peer_keys=$("$WG" show "$WG_IFACE" peers 2>/dev/null) || exit 74
for live_peer_key in $live_fabric_peer_keys; do
case " $expected_fabric_peer_keys " in
*" $live_peer_key "*) : ;;
*) exit 74 ;;
esac
done
for expected_peer_key in $expected_fabric_peer_keys; do
printf '%s\n' "$live_fabric_peer_keys" |
grep -Fqx "$expected_peer_key" || exit 74
done
fi
while IFS='|' read -r peer_host key expected_overlay; do
[ -n "$peer_host" ] || continue
"$WG" show "$WG_IFACE" peers | grep -Fqx "$key" || exit 74
case ",$expected_overlay," in *',0.0.0.0/0,'*|*',0.0.0.0/1,'*|*',128.0.0.0/1,'* ) exit 74 ;; esac
# Fabric has one AllowedIPs list per public key. Merge the complete,
# validated tenant set from every panel-owned VPC state file while keeping
# the peer's single overlay /32. Never replace the list with this VPC's
# partial view: doing so would black-hole another concurrently active VPC.
merge_actual=$(vc_wg_allowed_for_peer "$WG_IFACE" "$key") || exit 74
vpc_state_overlay_from_allowed "$merge_actual" >/dev/null || exit 74
merge_extra=$(vpc_state_allowed_for_peer "$key") || exit 74
merge_allowed="$expected_overlay"
[ -z "$merge_extra" ] || merge_allowed="$merge_allowed,$merge_extra"
"$WG" set "$WG_IFACE" peer "$key" allowed-ips "$merge_allowed" || exit 74
done <<'{peer_marker}'
{peer_lines}
{peer_marker}
# `wg set` success is not an exact-state receipt. Re-read every authoritative
# peer and prove its Fabric overlay and route-backed tenant union exactly.
while IFS='|' read -r final_peer_host final_peer_key final_overlay; do
[ -n "$final_peer_host" ] || continue
final_actual=$(vc_wg_allowed_for_peer "$WG_IFACE" "$final_peer_key") || exit 74
vpc_state_overlay_from_allowed "$final_actual" >/dev/null || exit 74
final_extra=$(vpc_state_allowed_for_peer "$final_peer_key") || exit 74
final_expected="$final_overlay"
[ -z "$final_extra" ] || final_expected="$final_expected,$final_extra"
final_actual_set=$(printf '%s\n' "$final_actual" | tr ',' '\n' | awk 'NF' | LC_ALL=C sort -u)
final_expected_set=$(printf '%s\n' "$final_expected" | tr ',' '\n' | awk 'NF' | LC_ALL=C sort -u)
[ -n "$final_actual_set" ] &&
[ "$final_actual_set" = "$final_expected_set" ] || exit 74
done <<'{peer_marker}'
{peer_lines}
{peer_marker}
# The shared Fabric quarantine table is the last enforcement boundary before
# a VPC bridge is admitted. Reparse the complete stateless grammar,
# bind every owner comment to the engine-authenticated full Fabric digest,
# compare the exact tenant /32 set derived by the state loader, and finally
# prove the whole table is still the post-refresh snapshot. A retained
# comment with altered drop semantics, a range element, or a late mutation
# therefore fails closed.
VC_VPC_APPLY_PHASE=final_readback
vc_vpc_fabric_nft_live_ready || exit 74
vpc_state_nft_set_matches || exit 74
vc_nft_table_matches_snapshot inet virt_cloud_wg_fabric \
"$TXN/fabric.nft.desired" || exit 74
vc_vpc_nft_matches_snapshot "$TXN/nft.desired" || exit 74
# Bind the optimization receipt to this successful atomic apply, not to a
# later observation that might already contain external rule drift.
vc_apply_nft_receipt=$(vc_vpc_nft_receipt) || exit 74
vc_vpc_nft_matches_snapshot "$TXN/nft.desired" || exit 74
vc_ln_final_expected=$(cat "$TXN/neighbors.desired") || exit 74
vc_ln_final_actual=$(vc_local_neighbor_records "$BRIDGE") || exit 74
vc_ln_final_wanted=$(printf '%s\\n' "$vc_ln_final_expected" | awk 'NF {{print $0 "|PERMANENT"}}') || exit 74
[ "$vc_ln_final_actual" = "$vc_ln_final_wanted" ] || exit 74
VC_VPC_APPLY_PHASE=commit
rm -f "$VPC_PEER_FILE" "$VPC_ROUTE_FILE" "$VPC_TENANT_FILE" "$VPC_TENANT_FILE.nft"
printf '%s\\n' ready > "$TXN/state"; chmod 600 "$TXN/state"
printf '%s\\n' '__VC_VPC_APPLIED__=yes'
printf '%s\\n' "__VC_VPC_NFT_RECEIPT__=$vc_apply_nft_receipt"
printf '%s\\n' '__VC_VPC_OWNER__={DATAPLANE_OWNER_MARKER}'
printf '%s\\n' '__VC_VPC_REVISION__={plan.desired_revision}'
printf '%s\\n' '__VC_VPC_DIGEST__={plan.desired_digest}'
COMMITTED=yes
printf '%s\\n' committed > "$TXN/state"; chmod 600 "$TXN/state"
rm -rf "$TXN"
trap - EXIT HUP INT TERM
exit 0
'''
if len(script) > MAX_OUTPUT:
raise VpcDataplaneError("VPC 数据面脚本过大", 413, "vpc_script_too_large")
return script
def render_remove_script(
plan: VpcDataplanePlan,
*,
deployment_mode: str = "networkmanager_nmcli",
firewall_mode: str = FIREWALL_MODE_NATIVE,
ifup_path: str = "/usr/sbin/ifup",
ifdown_path: str = "/usr/sbin/ifdown",
ifquery_path: str = "/usr/sbin/ifquery",
) -> str:
"""Render a fail-closed removal transaction for an empty host."""
if plan.local_members:
raise VpcDataplaneError(
"仍有本机私网成员,不能执行宿主机 VPC 销毁",
409, "vpc_remove_has_local_members")
if deployment_mode not in {"networkmanager_nmcli", "ifupdown"}:
raise VpcDataplaneError("宿主机网络管理模式不支持", 409,
"vpc_dataplane_network_manager_conflict")
firewall_mode = _firewall_mode(firewall_mode)
allowed_ifup = {"/usr/sbin/ifup", "/usr/bin/ifup", "/sbin/ifup", "/bin/ifup"}
allowed_ifdown = {"/usr/sbin/ifdown", "/usr/bin/ifdown", "/sbin/ifdown", "/bin/ifdown"}
allowed_ifquery = {"/usr/sbin/ifquery", "/usr/bin/ifquery", "/sbin/ifquery", "/bin/ifquery"}
if (ifup_path not in allowed_ifup or ifdown_path not in allowed_ifdown
or ifquery_path not in allowed_ifquery):
raise VpcDataplaneError("ifupdown 工具路径不受支持", 409,
"vpc_dataplane_network_manager_conflict")
from .fabric import (
_FABRIC_NFT_VALIDATION_SH,
_FIREWALL_MODE_GUARD_SH,
_NMCLI_FILENAME_RESOLVER_SH,
_VPC_STATE_LOADER,
)
r = plan.resources
include = f"/etc/nftables.d/virt-cloud-vpc-{r.token}.nft"
fragment = f"/etc/network/interfaces.d/virt-cloud-vpc-{r.token}"
sysctl_file = f"/etc/sysctl.d/99-virt-cloud-vpc-{r.token}.conf"
proxy_file = f"/etc/virt-cloud/vpc-dataplane/{r.token}.proxy"
authoritative_peer_overlays = _authoritative_fabric_peer_overlays(plan)
remove_peer_lines = "\n".join(
f"{key}|{overlay}"
for key, overlay in (authoritative_peer_overlays or {}).items())
remove_peer_key_words = " ".join(authoritative_peer_overlays or ())
remove_peer_marker = "__VC_VPC_REMOVE_FABRIC_PEERS_" + hashlib.sha256(
remove_peer_lines.encode()).hexdigest()[:12]
remove_peer_closure_provided = (
"yes" if authoritative_peer_overlays is not None else "no")
state_loader = _VPC_STATE_LOADER.replace(
"__VC_FABRIC_HOST_ID__", plan.host_id).replace(
"__VC_FABRIC_ID__", plan.vpc.fabric_id).replace(
"__VC_FABRIC_NETWORK__", "100.64.0.0/10").replace(
"__VC_VPC_STATE_OBSERVE__", "no")
nm_profile_marker = DATAPLANE_OWNER_MARKER + " vpc=" + plan.vpc.vpc_id
inflight_marker = (f"{DATAPLANE_OWNER_MARKER}:{plan.vpc.vpc_id}:"
f"{plan.host_id}:{plan.desired_revision}:{plan.desired_digest}")
script = f'''#!/bin/sh
set -eu
umask 077
PATH=/usr/sbin:/usr/bin:/sbin:/bin
export PATH
IP=$(for p in /usr/sbin/ip /usr/bin/ip /sbin/ip /bin/ip; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
WG=$(for p in /usr/bin/wg /usr/sbin/wg /bin/wg /sbin/wg; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
NFT=$(for p in /usr/sbin/nft /usr/bin/nft /sbin/nft /bin/nft; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
FLOCK=$(for p in /usr/bin/flock /usr/sbin/flock /bin/flock /sbin/flock; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
STAT=$(for p in /usr/bin/stat /bin/stat /usr/sbin/stat /sbin/stat; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
CMP=$(for p in /usr/bin/cmp /bin/cmp /usr/sbin/cmp /sbin/cmp; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
MKTEMP=$(for p in /usr/bin/mktemp /bin/mktemp /usr/sbin/mktemp /sbin/mktemp; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
BRIDGE_CMD=$(for p in /usr/sbin/bridge /usr/bin/bridge /sbin/bridge /bin/bridge; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
SYSTEMCTL=$(for p in /usr/bin/systemctl /bin/systemctl /usr/sbin/systemctl /sbin/systemctl; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
{BRIDGE_PORT_RECORDS_SH}
{LOCAL_NEIGHBOR_SH}
{IP_RULE_CONVERGENCE_SH}
{WG_ALLOWED_IPS_SH}
BRIDGE={_q(r.bridge)}; TABLE={_q(r.nft_table)}; ROUTE_TABLE={r.route_table}; WG_IFACE={_q(FABRIC_INTERFACE)}
OWNER_FILE={_q(r.owner_file)}; STATE_FILE={_q(r.state_file)}; LOCK={_q(r.lock_file)}; TXN={_q(r.transaction_dir)}
NFT_SNAPSHOT_PREFIX={_q(f'/etc/virt-cloud/vpc-dataplane/{r.token}.nft')}
NFT_FILE={_q(include)}; LEGACY_MANIFEST={_q(LEGACY_FIREWALLD_NFT_MANIFEST)}; IF_FRAGMENT={_q(fragment)}; SYSCTL_FILE={_q(sysctl_file)}; PROXY_FILE={_q(proxy_file)}; STATE_DIR=/etc/virt-cloud/vpc-dataplane
MODE={_q(deployment_mode)}; FIREWALL_MODE={_q(firewall_mode)}; IFUP={_q(ifup_path)}; IFDOWN={_q(ifdown_path)}; IFQUERY={_q(ifquery_path)}; NMCLI=; NM_PROFILE_UUID=; NM_PROFILE_FILE=; NFT_MAIN=; NFT_INCLUDE={_q(f'include "{include}"')}
NM_PROFILE_MARKER={_q(nm_profile_marker)}
EXPECTED_DIGEST={_q(plan.fabric_plan_digest)}
{NFT_LIVE_SNAPSHOT_SH}
{_NMCLI_FILENAME_RESOLVER_SH}
{NETWORKMANAGER_VPC_PROFILE_SH}
{NETWORKMANAGER_VPC_TARGET_INVENTORY_SH}
{IFUPDOWN_VPC_TARGET_INVENTORY_SH}
{_FIREWALL_MODE_GUARD_SH}
{_FABRIC_NFT_VALIDATION_SH}
{_VPC_SYSCTL_VALIDATION_SH}
{VPC_NATIVE_TRANSACTION_GUARD_SH}
nft_plan_owned() {{
nft_plan_text=$1
nft_plan_digest=$2
printf '%s\\n' "$nft_plan_digest" | grep -Eq '^[0-9a-f]{{64}}$' || return 1
vc_fabric_nft_strict_shape "$nft_plan_text" || return 1
[ "$(printf '%s\\n' "$nft_plan_text" |
grep -Foc 'comment "virt-cloud-wg-fabric-v1:'"$nft_plan_digest"'"' || true)" = 8 ]
}}
vc_vpc_fabric_nft_live_ready() {{
vc_remove_fabric_text=$(
"$NFT" -s list table inet virt_cloud_wg_fabric 2>/dev/null
) || return 1
nft_plan_owned "$vc_remove_fabric_text" "$EXPECTED_DIGEST"
}}
[ -n "$IP" ] && [ -n "$WG" ] && [ -n "$NFT" ] && [ -n "$FLOCK" ] && [ -n "$STAT" ] && [ -n "$CMP" ] && [ -n "$MKTEMP" ] && [ -n "$BRIDGE_CMD" ] && [ -n "$SYSTEMCTL" ] || exit 70
if [ "$MODE" = networkmanager_nmcli ]; then
NMCLI=$(for p in /usr/bin/nmcli /usr/sbin/nmcli /bin/nmcli /sbin/nmcli; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
[ -n "$NMCLI" ] || exit 70
fi
[ "$FIREWALL_MODE" = native_nftables ] || exit 71
[ ! -L /run/virt-cloud ] && [ ! -L /etc/virt-cloud ] || exit 71
mkdir -p /run/virt-cloud; chmod 700 /run/virt-cloud
if [ -e "$STATE_DIR" ] || [ -L "$STATE_DIR" ]; then
[ -d "$STATE_DIR" ] && [ ! -L "$STATE_DIR" ] || exit 71
[ "$($STAT -c '%u:%a' "$STATE_DIR" 2>/dev/null)" = 0:700 ] || exit 71
fi
if [ -e /etc/nftables.d ] || [ -L /etc/nftables.d ]; then
[ -d /etc/nftables.d ] && [ ! -L /etc/nftables.d ] || exit 71
[ "$($STAT -c '%u:%a' /etc/nftables.d 2>/dev/null)" = 0:755 ] || exit 71
fi
if [ -e /etc/sysctl.d ] || [ -L /etc/sysctl.d ]; then
[ -d /etc/sysctl.d ] && [ ! -L /etc/sysctl.d ] || exit 71
[ "$($STAT -c '%u:%a' /etc/sysctl.d 2>/dev/null)" = 0:755 ] || exit 71
fi
[ ! -L "$LOCK" ] && {{ [ ! -e "$LOCK" ] || [ -f "$LOCK" ]; }} || exit 71
: >> "$LOCK"; chmod 600 "$LOCK"
exec 8>>{_q(FABRIC_LOCK)}; "$FLOCK" -x 8
exec 9>>"$LOCK"; "$FLOCK" -x 9
vc_vpc_firewall_boundary_clean || exit 79
for vc_vpc_legacy_snapshot in inet bridge arp; do
[ ! -e "$NFT_SNAPSHOT_PREFIX.$vc_vpc_legacy_snapshot.live" ] &&
[ ! -L "$NFT_SNAPSHOT_PREFIX.$vc_vpc_legacy_snapshot.live" ] || exit 79
done
if [ -e "$TXN" ] || [ -L "$TXN" ]; then
vc_vpc_native_transaction_owned || exit 79
fi
REMOVE_NEIGHBOR_EXPECTED=; REMOVE_NEIGHBOR_BEFORE=
REMOVE_BRIDGE_TOPOLOGY_CAPTURED=no; REMOVE_BRIDGE_PORTS_BEFORE=
vc_remove_bridge_port_names() {{
vc_remove_ports_raw=$(vc_bridge_port_records "$BRIDGE" 2>/dev/null) || return 1
vc_remove_ports_parsed=$(printf '%s\\n' "$vc_remove_ports_raw" | awk '
NF == 0 {{next}}
NF < 2 {{bad=1; next}}
{{name=$2; sub(/:$/, "", name); sub(/@.*/, "", name)}}
name !~ /^[A-Za-z0-9_.-]+$/ {{bad=1; next}}
{{print name}}
END {{if (bad) exit 1}}
') || return 1
printf '%s\\n' "$vc_remove_ports_parsed" | LC_ALL=C sort -u
}}
vc_remove_bridge_topology_matches() {{
[ "$REMOVE_BRIDGE_TOPOLOGY_CAPTURED" = yes ] || return 1
vc_remove_ports_now=$(vc_remove_bridge_port_names) || return 1
[ "$vc_remove_ports_now" = "$REMOVE_BRIDGE_PORTS_BEFORE" ]
}}
PREVIOUS_FIREWALL_MODE=absent
if [ -f "$STATE_FILE" ]; then
# Minimum state/manager proof is the only precondition for the hard link
# quarantine. Runtime firewall, Fabric, journal, artifacts and RPDB are
# intentionally checked only after the panel bridge is administratively
# down, so combined drift cannot pass tenant traffic.
[ ! -L "$STATE_FILE" ] &&
[ "$($STAT -c '%u:%a' "$STATE_FILE" 2>/dev/null)" = 0:600 ] || exit 71
grep -Fqx 'owner|{DATAPLANE_OWNER_MARKER}' "$STATE_FILE" || exit 71
grep -Fqx 'vpc|{plan.vpc.vpc_id}' "$STATE_FILE" || exit 71
grep -Fqx 'fabric|{plan.vpc.fabric_id}' "$STATE_FILE" || exit 71
grep -Fqx 'host|{plan.host_id}' "$STATE_FILE" || exit 71
grep -Fqx 'bridge|{r.bridge}' "$STATE_FILE" || exit 71
grep -Fqx 'table|{r.route_table}' "$STATE_FILE" || exit 71
state_firewall_quarantine=$(awk -F'|' '$1=="firewall_mode" {{count++; value=$2}} END {{print (count + 0) "|" value}}' "$STATE_FILE")
case "$state_firewall_quarantine" in
'0|'|'1|native_nftables') PREVIOUS_FIREWALL_MODE=native_nftables ;;
*) exit 71 ;;
esac
quarantine_revision=$(awk -F'|' '$1=="revision" {{count++; value=$2}} END {{print (count + 0) "|" value}}' "$STATE_FILE")
quarantine_digest=$(awk -F'|' '$1=="digest" {{count++; value=$2}} END {{print (count + 0) "|" value}}' "$STATE_FILE")
[ "${{quarantine_revision%%|*}}" = 1 ] || exit 71
case "${{quarantine_revision#*|}}" in ''|*[!0-9]*) exit 71 ;; esac
[ "${{quarantine_digest%%|*}}" = 1 ] || exit 71
printf '%s\\n' "${{quarantine_digest#*|}}" | grep -Eq '^[0-9a-f]{{64}}$' || exit 71
[ -f "$OWNER_FILE" ] && [ ! -L "$OWNER_FILE" ] &&
[ "$($STAT -c '%u:%a' "$OWNER_FILE" 2>/dev/null)" = 0:600 ] &&
grep -Fqx '{DATAPLANE_OWNER_MARKER}' "$OWNER_FILE" || exit 71
if [ "$MODE" = networkmanager_nmcli ]; then
vc_nm_profile_resolve || exit 71
vc_nm_profile_base_validate || exit 71
quarantine_memory_zone=$(vc_nm_profile_field connection.zone) || exit 71
case "$quarantine_memory_zone" in ''|--) : ;; *) exit 71 ;; esac
else
[ -f "$IF_FRAGMENT" ] && [ ! -L "$IF_FRAGMENT" ] &&
[ "$($STAT -c '%u:%a' "$IF_FRAGMENT" 2>/dev/null)" = 0:600 ] || exit 71
awk -v owner="# {DATAPLANE_OWNER_MARKER} vpc={plan.vpc.vpc_id} revision=${{quarantine_revision#*|}} digest=${{quarantine_digest#*|}}" \
-v bridge="$BRIDGE" '
NR==1 && $0==owner {{next}}
NR==2 && $0=="iface " bridge " inet manual" {{next}}
NR==3 && $0==" bridge_ports none" {{next}}
NR==4 && $0==" bridge-stp off" {{next}}
{{bad=1}}
END {{exit (NR==4 && !bad) ? 0 : 1}}
' "$IF_FRAGMENT" || exit 71
fi
if "$IP" link show dev "$BRIDGE" >/dev/null 2>&1; then
"$IP" -d link show dev "$BRIDGE" 2>/dev/null |
grep -Eq '(^|[[:space:]])bridge[[:space:]]+(forward_delay|stp_state|hello_time|ageing_time)([[:space:]]|$)' || exit 71
REMOVE_BRIDGE_PORTS_BEFORE=$(vc_remove_bridge_port_names) || exit 71
REMOVE_BRIDGE_TOPOLOGY_CAPTURED=yes
# vc-local-neighbor-remove-preflight-begin
REMOVE_NEIGHBOR_EXPECTED=$(vc_local_neighbor_expected "$STATE_FILE") || exit 71
vc_local_neighbor_preflight "$BRIDGE" "$REMOVE_NEIGHBOR_EXPECTED" || exit 71
REMOVE_NEIGHBOR_BEFORE=$(vc_local_neighbor_records "$BRIDGE") || exit 71
# vc-local-neighbor-remove-preflight-end
if [ -n "$REMOVE_BRIDGE_PORTS_BEFORE" ]; then
# Removal cannot own or detach a running VM tap. Quarantine the same
# bridge without NM deconfigure, then refuse until its ports are gone.
"$IP" link set dev "$BRIDGE" down >/dev/null || exit 72
vc_nm_link_expect_down || exit 72
vc_remove_bridge_topology_matches || exit 72
exit 71
fi
if [ "$MODE" = networkmanager_nmcli ]; then
vc_nm_device_quarantine || exit 72
else
"$IP" link set dev "$BRIDGE" down >/dev/null || exit 72
vc_nm_link_expect_down || exit 72
fi
vc_remove_bridge_topology_matches || exit 72
fi
fi
vc_firewall_mode_ready || exit 71
[ -f /etc/virt-cloud/wg-fabric0.owner ] && grep -Fqx 'virt-cloud-wg-fabric-v1' /etc/virt-cloud/wg-fabric0.owner || exit 71
"$WG" show "$WG_IFACE" >/dev/null 2>&1 || exit 71
BRIDGE_PRESENT=no
if "$IP" link show dev "$BRIDGE" >/dev/null 2>&1; then BRIDGE_PRESENT=yes; fi
PHASE=preparing; RESUME=no
if [ -e "$TXN" ]; then
vc_vpc_native_transaction_owned || exit 79
vc_vpc_transaction_record_owned "$TXN/inflight" '{inflight_marker}' || exit 79
vc_vpc_transaction_file_secure "$TXN/state" 64 || exit 79
PHASE=$(cat "$TXN/state" 2>/dev/null) || exit 79
case "$PHASE" in
preparing|routes_removed|manager_removed|bridge_removed|nft_removed|files_removed|wg_merged|committed) : ;;
*) exit 79 ;;
esac
RESUME=yes
for snapshot in "$TXN/state.before" "$TXN/routes.before" "$TXN/rules.before" "$TXN/proxy.before"; do
[ -e "$snapshot" ] || continue
[ -f "$snapshot" ] && [ ! -L "$snapshot" ] || exit 79
[ "$($STAT -c '%u:%a' "$snapshot" 2>/dev/null)" = 0:600 ] || exit 79
done
fi
set_phase() {{
phase_tmp="$TXN/.state.$$"
printf '%s\\n' "$1" > "$phase_tmp"
chmod 600 "$phase_tmp"
mv -f "$phase_tmp" "$TXN/state"
PHASE="$1"
}}
vc_vpc_route_table_read() {{
vc_vpc_route_wanted_table=$1
vc_vpc_route_wanted_destination=${{2-}}
case "$vc_vpc_route_wanted_table" in ''|*[!0-9]*) return 1 ;; esac
vc_vpc_route_inventory=$("$IP" -o -4 -N route show table all 2>/dev/null) || return 1
printf '%s\\n' "$vc_vpc_route_inventory" | awk \
-v wanted_table="$vc_vpc_route_wanted_table" \
-v wanted_destination="$vc_vpc_route_wanted_destination" '
NF == 0 {{next}}
{{
table_count=0
table_value="254"
row_bad=0
for (i=1; i<=NF; i++) {{
if ($i == "table") {{
table_count++
if (i == NF || $(i+1) !~ /^[0-9]+$/) row_bad=1
else table_value=$(i+1)
}}
}}
if (table_count > 1 || row_bad) bad=1
route_destination=$1
if ($1 ~ /^[0-9]+$/ && NF > 1) route_destination=$2
wanted_host=wanted_destination
if (length(wanted_host) > 3 &&
substr(wanted_host, length(wanted_host) - 2) == "/32")
wanted_host=substr(wanted_host, 1, length(wanted_host) - 3)
if (!row_bad && table_count <= 1 && table_value == wanted_table &&
(wanted_destination == "" || route_destination == wanted_destination ||
route_destination == wanted_host)) print
}}
END {{exit bad ? 1 : 0}}
'
}}
verify_routes_removed() {{
verify_routes=$(vc_vpc_route_table_read "$ROUTE_TABLE") || return 1
[ -z "$verify_routes" ] || return 1
vc_ip_rule_scope_check {_q(plan.vpc.cidr)} "$ROUTE_TABLE" "{r.rule_priority}" "$WG_IFACE" empty 1 1 || return 1
if $IP link show dev "$BRIDGE" >/dev/null 2>&1; then
verify_proxy=$($IP neigh show proxy dev "$BRIDGE" 2>/dev/null | awk '$1 ~ /^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$/') || return 1
[ -z "$verify_proxy" ] || return 1
fi
return 0
}}
vc_vpc_artifact_identity_check() {{
identity=$1
set -- $identity
[ "$#" -eq 3 ] || return 1
[ "$1" = "vpc={plan.vpc.vpc_id}" ] || return 1
artifact_revision=${{2#revision=}}
[ "$2" = "revision=$artifact_revision" ] || return 1
case "$artifact_revision" in ''|*[!0-9]*) return 1;; esac
[ "$artifact_revision" -ge 1 ] || return 1
artifact_digest=${{3#digest=}}
[ "$3" = "digest=$artifact_digest" ] || return 1
printf '%s\n' "$artifact_digest" | grep -Eq '^[0-9a-f]{{64}}$' || return 1
[ "$artifact_revision" -le {plan.desired_revision} ] || return 1
if [ "$artifact_revision" = {plan.desired_revision} ] &&
[ "$artifact_digest" != {_q(plan.desired_digest)} ]; then
return 1
fi
return 0
}}
vc_remove_capture_nft_precommit() {{
vc_remove_precommit_prefix=$1
for vc_remove_precommit_family in inet bridge arp; do
vc_nft_table_snapshot "$vc_remove_precommit_family" "$TABLE" \
"$vc_remove_precommit_prefix.$vc_remove_precommit_family" || return 1
[ -f "$vc_remove_precommit_prefix.$vc_remove_precommit_family" ] &&
[ ! -L "$vc_remove_precommit_prefix.$vc_remove_precommit_family" ] &&
[ ! -e "$vc_remove_precommit_prefix.$vc_remove_precommit_family.absent" ] ||
return 1
done
}}
vc_remove_nft_precommit_matches() {{
vc_remove_precommit_prefix=$1
for vc_remove_precommit_family in inet bridge arp; do
vc_nft_table_matches_snapshot "$vc_remove_precommit_family" "$TABLE" \
"$vc_remove_precommit_prefix.$vc_remove_precommit_family" || return 1
done
}}
vc_remove_nft_tables_absent() {{
for vc_remove_absent_family in inet bridge arp; do
vc_nft_table_absent "$vc_remove_absent_family" "$TABLE" \
"$TXN/nft.remove.after.$vc_remove_absent_family.inventory" || return 1
rm -f "$TXN/nft.remove.after.$vc_remove_absent_family.inventory"
done
}}
case "$PHASE" in
preparing|routes_removed|manager_removed|bridge_removed|nft_removed) PRE_FILES=yes ;;
files_removed|wg_merged|committed) PRE_FILES=no ;;
*) exit 79 ;;
esac
if [ "$PRE_FILES" = yes ]; then
if [ -f "$OWNER_FILE" ]; then
[ ! -L "$OWNER_FILE" ] || exit 71
[ "$($STAT -c '%u:%a' "$OWNER_FILE" 2>/dev/null)" = 0:600 ] || exit 71
grep -Fqx 'virt-cloud-vpc-dataplane-v1' "$OWNER_FILE" || exit 71
elif [ -e "$STATE_FILE" ] || [ "$BRIDGE_PRESENT" = yes ] || [ -e "$NFT_FILE" ] ||
[ -e "$SYSCTL_FILE" ] || [ -e "$PROXY_FILE" ] || [ -e "$IF_FRAGMENT" ] ||
[ -e "$NFT_SNAPSHOT_PREFIX.inet.live" ] ||
[ -e "$NFT_SNAPSHOT_PREFIX.bridge.live" ] ||
[ -e "$NFT_SNAPSHOT_PREFIX.arp.live" ]; then
# A deterministic bridge/file name is not ownership. Refuse to remove
# anything when the panel's owner marker disappeared or was replaced.
exit 71
fi
if [ -f "$STATE_FILE" ]; then
[ ! -L "$STATE_FILE" ] || exit 71
[ "$($STAT -c '%u:%a' "$STATE_FILE" 2>/dev/null)" = 0:600 ] || exit 71
grep -Fqx 'owner|{DATAPLANE_OWNER_MARKER}' "$STATE_FILE" || exit 71
grep -Fqx 'vpc|{plan.vpc.vpc_id}' "$STATE_FILE" || exit 71
grep -Fqx 'cidr|{plan.vpc.cidr}' "$STATE_FILE" || exit 71
grep -Fqx 'fabric|{plan.vpc.fabric_id}' "$STATE_FILE" || exit 71
grep -Fqx 'host|{plan.host_id}' "$STATE_FILE" || exit 71
grep -Fqx 'bridge|{r.bridge}' "$STATE_FILE" || exit 71
grep -Fqx 'table|{r.route_table}' "$STATE_FILE" || exit 71
state_firewall_record=$(awk -F'|' '$1=="firewall_mode" {{count++; value=$2}} END {{print (count + 0) "|" value}}' "$STATE_FILE")
case "$state_firewall_record" in '0|'|'1|native_nftables') : ;; *) exit 71 ;; esac
state_revision=$(awk -F'|' '$1=="revision" {{print $2; exit}}' "$STATE_FILE")
state_digest=$(awk -F'|' '$1=="digest" {{print $2; exit}}' "$STATE_FILE")
case "$state_revision" in ''|*[!0-9]*) exit 79;; esac
printf '%s\n' "$state_digest" | grep -Eq '^[0-9a-f]{{64}}$' || exit 79
[ "$state_revision" -le {plan.desired_revision} ] || exit 79
if [ "$state_revision" = {plan.desired_revision} ] && [ "$state_digest" != { _q(plan.desired_digest) } ]; then exit 79; fi
for snapshot_family in inet bridge arp; do
[ ! -e "$NFT_SNAPSHOT_PREFIX.$snapshot_family.live" ] &&
[ ! -L "$NFT_SNAPSHOT_PREFIX.$snapshot_family.live" ] || exit 71
done
elif [ -f "$OWNER_FILE" ] && [ "$BRIDGE_PRESENT" = yes ]; then
# A live panel bridge without its state cannot be reconciled safely.
exit 71
else
# Without the state file there is no safe owner proof for deterministic
# policy routes/rules. Refuse to claim a clean removal while any of those
# resources remain; an operator can restore the state and retry.
orphan_routes=$(vc_vpc_route_table_read "$ROUTE_TABLE") || exit 72
[ -z "$orphan_routes" ] || exit 71
vc_ip_rule_scope_check {_q(plan.vpc.cidr)} "$ROUTE_TABLE" "{r.rule_priority}" "$WG_IFACE" empty 71 72 || exit $?
fi
for artifact in "$NFT_FILE" "$SYSCTL_FILE" "$PROXY_FILE" "$IF_FRAGMENT"; do
[ -e "$artifact" ] || continue
[ -f "$artifact" ] && [ ! -L "$artifact" ] || exit 71
done
if [ -f "$NFT_FILE" ]; then
[ "$($STAT -c '%u:%a' "$NFT_FILE" 2>/dev/null)" = 0:600 ] || exit 71
grep -Fqx '# virt-cloud-vpc-dataplane-v1' "$NFT_FILE" || exit 71
nft_identity=$(sed -n '3{{s/^# //;p;}}' "$NFT_FILE")
vc_vpc_artifact_identity_check "$nft_identity" || exit 71
fi
if [ -f "$SYSCTL_FILE" ]; then
[ -f "$STATE_FILE" ] || exit 71
vc_vpc_sysctl_artifact_owned "$SYSCTL_FILE" {_q(plan.vpc.vpc_id)} \
"$state_revision" "$state_digest" "$BRIDGE" || exit 71
fi
if [ -f "$PROXY_FILE" ]; then
[ "$($STAT -c '%u:%a' "$PROXY_FILE" 2>/dev/null)" = 0:600 ] || exit 71
sed -n '1p' "$PROXY_FILE" | grep -Fqx '# virt-cloud-vpc-proxy-v1' || exit 71
awk 'NR > 1 && $0 !~ /^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$/ {{ exit 1 }}' "$PROXY_FILE" || exit 71
fi
if [ -f "$IF_FRAGMENT" ]; then
[ "$($STAT -c '%u:%a' "$IF_FRAGMENT" 2>/dev/null)" = 0:600 ] || exit 71
# The owner prefix is followed by the apply revision and digest. Require
# the prefix, then validate all identity fields before removing the fragment.
if_identity=$(sed -n '1s/^# {DATAPLANE_OWNER_MARKER} //p' "$IF_FRAGMENT")
vc_vpc_artifact_identity_check "$if_identity" || exit 71
grep -Eq '^[[:space:]]*(auto|allow-[^[:space:]]+)[[:space:]]+{r.bridge}([[:space:]]|$)' "$IF_FRAGMENT" && exit 71 || true
grep -Fqx 'iface {r.bridge} inet manual' "$IF_FRAGMENT" || exit 71
if grep -Eq '^[[:space:]]*bridge_ports([[:space:]]|$)' "$IF_FRAGMENT"; then
grep -Eq '^[[:space:]]*bridge_ports[[:space:]]+none[[:space:]]*$' "$IF_FRAGMENT" || exit 71
fi
grep -Eq '^[[:space:]]*(address|gateway|up|down|pre-up|post-up|pre-down|post-down)([[:space:]]|$)' "$IF_FRAGMENT" && exit 71 || true
fi
else
for artifact in "$NFT_FILE" "$SYSCTL_FILE" "$PROXY_FILE" "$IF_FRAGMENT" "$OWNER_FILE" "$STATE_FILE"; do
[ ! -e "$artifact" ] && [ ! -L "$artifact" ] || exit 79
done
for snapshot_family in inet bridge arp; do
[ ! -e "$NFT_SNAPSHOT_PREFIX.$snapshot_family.live" ] &&
[ ! -L "$NFT_SNAPSHOT_PREFIX.$snapshot_family.live" ] || exit 79
done
fi
NM_REMOVE_UUID=
if [ "$MODE" = networkmanager_nmcli ]; then
if [ "$PHASE" = preparing ] || [ "$PHASE" = routes_removed ]; then
vc_nm_target_inventory || exit 71
case "$NM_TARGET_COUNT" in
0)
if [ "$BRIDGE_PRESENT" = yes ]; then
# An unmanaged live bridge is foreign/ambiguous; do not delete it.
exit 71
fi
;;
1)
# Name, interface and owner marker must all identify the same unique
# profile. A renamed or colliding profile remains visible but cannot
# be deleted automatically.
[ "$NM_TARGET_EXACT" = yes ] || exit 71
vc_nm_profile_resolve || exit 71
[ "$NM_PROFILE_UUID" = "$NM_TARGET_UUID" ] || exit 71
vc_nm_profile_base_validate || exit 71
vc_nm_profile_empty connection.zone || exit 71
NM_REMOVE_UUID=$NM_PROFILE_UUID
;;
*) exit 71 ;;
esac
else
vc_nm_target_inventory || exit 79
[ "$NM_TARGET_COUNT" = 0 ] || exit 79
fi
else
if [ "$PHASE" = preparing ] || [ "$PHASE" = routes_removed ]; then
if [ "$BRIDGE_PRESENT" = yes ] || [ -e "$IF_FRAGMENT" ]; then
[ -f "$IF_FRAGMENT" ] || exit 71
"$IFQUERY" "$BRIDGE" >/dev/null 2>&1 || exit 71
fi
fi
fi
vc_ifupdown_target_inventory || exit 71
if [ "$PRE_FILES" = yes ]; then
if [ "$MODE" = ifupdown ] && [ -f "$IF_FRAGMENT" ]; then
[ "$IFUP_TARGET_COUNT" = 1 ] || exit 71
else
[ "$IFUP_TARGET_COUNT" = 0 ] || exit 71
fi
else
[ "$IFUP_TARGET_COUNT" = 0 ] || exit 79
fi
if [ "$PHASE" = preparing ] || [ "$PHASE" = routes_removed ] || [ "$PHASE" = manager_removed ]; then
if "$IP" link show dev "$BRIDGE" >/dev/null 2>&1; then
[ -f "$OWNER_FILE" ] || exit 71
bridge_info=$("$IP" -d link show dev "$BRIDGE" 2>/dev/null) || exit 71
printf '%s\\n' "$bridge_info" | grep -Eq '(^|[[:space:]])bridge[[:space:]]+(forward_delay|stp_state|hello_time|ageing_time)([[:space:]]|$)' || exit 71
bridge_ports=$(vc_bridge_port_records "$BRIDGE" 2>/dev/null) || exit 71
[ -z "$bridge_ports" ] || exit 79
bridge_addrs=$("$IP" -o addr show dev "$BRIDGE" 2>/dev/null) || exit 71
[ -z "$bridge_addrs" ] || exit 79
fi
else
if "$IP" link show dev "$BRIDGE" >/dev/null 2>&1; then exit 79; fi
fi
NFT_MAIN_COUNT=0
for nft_candidate in /etc/nftables.conf /etc/sysconfig/nftables.conf; do
if [ -f "$nft_candidate" ] && [ ! -L "$nft_candidate" ] && [ "$($STAT -c '%u' "$nft_candidate" 2>/dev/null)" = 0 ]; then
case "$($STAT -c '%a' "$nft_candidate" 2>/dev/null)" in
[0-7][0145][0145]) : ;;
*) continue ;;
esac
NFT_MAIN="$nft_candidate"; NFT_MAIN_COUNT=$((NFT_MAIN_COUNT + 1))
fi
done
[ "$NFT_MAIN_COUNT" -eq 1 ] || exit 71
[ ! -e "$LEGACY_MANIFEST" ] && [ ! -L "$LEGACY_MANIFEST" ] || exit 71
if [ "$PHASE" = preparing ] || [ "$PHASE" = routes_removed ] || [ "$PHASE" = manager_removed ] || [ "$PHASE" = bridge_removed ]; then
if [ -f "$NFT_FILE" ]; then
nft_tables=$(awk '$1=="table" {{print $2 "|" $3}}' "$NFT_FILE")
nft_table_count=0
while IFS='|' read -r family name; do
[ -n "$family" ] || continue
case "$family:$name" in inet:*|bridge:*|arp:*) : ;; *) exit 71 ;; esac
[ "$name" = "$TABLE" ] || exit 71
table_dump=$("$NFT" list table "$family" "$name" 2>/dev/null) || exit 71
printf '%s\\n' "$table_dump" | grep -Fq 'virt-cloud-vpc-dataplane-v1' || exit 71
nft_table_count=$((nft_table_count + 1))
done <<EOF
$nft_tables
EOF
[ "$nft_table_count" -gt 0 ] || exit 71
fi
else
if [ -f "$NFT_FILE" ]; then
nft_tables=$(awk '$1=="table" {{print $2 "|" $3}}' "$NFT_FILE")
while IFS='|' read -r family name; do
[ -n "$family" ] || continue
case "$family:$name" in inet:*|bridge:*|arp:*) : ;; *) exit 79 ;; esac
[ "$name" = "$TABLE" ] || exit 79
if "$NFT" list table "$family" "$name" >/dev/null 2>&1; then exit 79; fi
done <<EOF
$nft_tables
EOF
fi
grep -Fqx "$NFT_INCLUDE" "$NFT_MAIN" && exit 79 || true
fi
# A partially completed removal is deliberately fail-closed. The journal is
# retained for an operator/reconciler to inspect; rerunning it must never repeat
# route, bridge, nft, or WireGuard deletions against an unknown intermediate
# state. A fully committed journal only needs the already-verified cleanup.
if [ "$RESUME" = yes ]; then
case "$PHASE" in
committed)
rm -rf "$TXN" || exit 79
printf '%s\\n' '__VC_VPC_REMOVED__=yes'
printf '%s\\n' '__VC_VPC_OWNER__={DATAPLANE_OWNER_MARKER}'
printf '%s\\n' '__VC_VPC_REVISION__={plan.desired_revision}'
printf '%s\\n' '__VC_VPC_DIGEST__={plan.desired_digest}'
printf '%s\\n' '__VC_VPC_ROLLBACK__=none'
exit 0
;;
files_removed|wg_merged)
# All destructive host resources are already absent. Re-running only
# the Fabric state load and peer merge below is idempotent and closes
# the crash window between file removal and AllowedIPs convergence.
:
;;
*) exit 79 ;;
esac
fi
if [ "$RESUME" = no ]; then
mkdir -m 700 "$TXN"
vc_vpc_write_native_transaction_mode || exit 79
printf '%s\\n' '{inflight_marker}' > "$TXN/inflight"; chmod 600 "$TXN/inflight"
set_phase preparing
if [ -f "$STATE_FILE" ]; then cp -p "$STATE_FILE" "$TXN/state.before"; fi
: > "$TXN/routes.before"; : > "$TXN/rules.before"; : > "$TXN/proxy.before"
printf '%s\\n' "$REMOVE_NEIGHBOR_BEFORE" > "$TXN/neighbors.before"
chmod 600 "$TXN/neighbors.before"
chmod 600 "$TXN/routes.before" "$TXN/rules.before" "$TXN/proxy.before"
else
# A resumed transaction must use the original snapshots. Creating an empty
# snapshot after a partial failure would turn an unknown host state into an
# apparently clean state and could make the next destructive step unsafe.
for snapshot in "$TXN/routes.before" "$TXN/rules.before" "$TXN/proxy.before"; do
[ -f "$snapshot" ] && [ ! -L "$snapshot" ] || exit 79
[ "$($STAT -c '%u:%a' "$snapshot" 2>/dev/null)" = 0:600 ] || exit 79
done
fi
if [ -f "$TXN/state.before" ]; then
old_priority=$(awk -F'|' '$1=="priority" {{print $2; exit}}' "$TXN/state.before")
old_cidr=$(awk -F'|' '$1=="cidr" {{print $2; exit}}' "$TXN/state.before")
[ -n "$old_priority" ] && [ -n "$old_cidr" ] || exit 71
case "$old_priority" in ''|*[!0-9]*) exit 71;; esac
[ "$old_priority" = "{r.rule_priority}" ] || exit 71
# Validate both RPDB slots before deleting any route. A foreign or
# ambiguous priority must leave the data plane untouched.
vc_ip_rule_scope_check "$old_cidr" "$ROUTE_TABLE" "$old_priority" "$WG_IFACE" managed 71 72 || exit $?
vc_ip_rule_preflight "$old_priority" - "$old_cidr" "$ROUTE_TABLE" 71 72 "$TXN/rules.before" || exit $?
vc_ip_rule_preflight "$((old_priority + 1))" "$WG_IFACE" "$old_cidr" "$ROUTE_TABLE" 71 72 "$TXN/rules.before" || exit $?
while IFS='|' read -r kind destination device table via metric; do
[ "$kind" = route ] || continue
[ "$table" = "$ROUTE_TABLE" ] || exit 71
[ "$device" = "$BRIDGE" ] || [ "$device" = "$WG_IFACE" ] || exit 71
printf '%s\\n' "$destination" | grep -Eq '^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+/32$' || exit 71
case "$via" in
-|'') : ;;
*) printf '%s\\n' "$via" | grep -Eq '^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$' || exit 71 ;;
esac
case "$metric" in ''|*[!0-9]*) exit 71;; esac
existing=$(vc_vpc_route_table_read "$table" "$destination") || exit 72
[ -n "$existing" ] || continue
existing_count=$(printf '%s\\n' "$existing" | awk 'NF {{ count++ }} END {{ print count + 0 }}')
[ "$existing_count" = 1 ] || exit 71
actual_device=$(printf '%s\\n' "$existing" | awk '{{for (i=1;i<=NF;i++) if ($i=="dev" && i<NF) {{print $(i+1); exit}}}}')
[ "$actual_device" = "$device" ] || exit 71
printf '%s\\n' "$existing" | grep -Eq 'proto 186([[:space:]]|$)' || exit 71
old_via=$(printf '%s\\n' "$existing" | awk '{{for (i=1;i<=NF;i++) if ($i=="via" && i<NF) {{print $(i+1); exit}}}}')
old_metric=$(printf '%s\\n' "$existing" | awk '{{for (i=1;i<=NF;i++) if ($i=="metric" && i<NF) {{print $(i+1); exit}}}}')
[ -n "$old_metric" ] || old_metric=100
[ "$old_metric" = "$metric" ] || exit 71
[ "${{old_via:--}}" = "${{via:--}}" ] || exit 71
printf '%s|%s|%s|%s|%s\\n' "$destination" "$device" "$table" "$old_via" "$old_metric" >> "$TXN/routes.before"
if [ "${{old_via:--}}" != - ]; then
"$IP" route del "$destination" table "$table" via "$old_via" dev "$device" proto 186 metric "$old_metric" || exit 72
else
"$IP" route del "$destination" table "$table" dev "$device" proto 186 metric "$old_metric" || exit 72
fi
done < "$TXN/state.before"
for rule_spec in "$old_priority|-" "$((old_priority + 1))|$WG_IFACE"; do
pref=${{rule_spec%%|*}}; old_iif=${{rule_spec#*|}}
vc_ip_rule_read "$pref" "$old_iif" "$old_cidr" "$ROUTE_TABLE" 71 72 || exit $?
[ "$VC_IP_RULE_PRESENT" = yes ] || continue
if [ "$old_iif" = - ]; then
"$IP" -4 rule del pref "$pref" to "$old_cidr" lookup "$ROUTE_TABLE" || exit 72
else
"$IP" -4 rule del pref "$pref" iif "$old_iif" to "$old_cidr" lookup "$ROUTE_TABLE" || exit 72
fi
vc_ip_rule_read "$pref" "$old_iif" "$old_cidr" "$ROUTE_TABLE" 71 72 || exit $?
[ "$VC_IP_RULE_PRESENT" = no ] || exit 72
done
fi
if [ -f "$PROXY_FILE" ]; then
while IFS= read -r proxy_ip; do
[ -n "$proxy_ip" ] || continue
if "$IP" neigh show proxy dev "$BRIDGE" 2>/dev/null | grep -Fqx "$proxy_ip"; then
printf '%s\\n' "$proxy_ip" >> "$TXN/proxy.before"
"$IP" neigh del proxy "$proxy_ip" dev "$BRIDGE" || exit 72
fi
done <<EOF
$(sed '1d' "$PROXY_FILE")
EOF
fi
verify_routes_removed || exit 72
set_phase routes_removed
# vc-local-neighbor-remove-cleanup-begin
if "$IP" link show dev "$BRIDGE" >/dev/null 2>&1; then
vc_local_neighbor_preflight "$BRIDGE" "$REMOVE_NEIGHBOR_EXPECTED" || exit 71
remove_neighbor_current=$(vc_local_neighbor_records "$BRIDGE") || exit 71
while IFS='|' read -r remove_neighbor_ip remove_neighbor_mac remove_neighbor_state; do
[ -n "$remove_neighbor_ip" ] || continue
"$IP" -4 neigh del "$remove_neighbor_ip" dev "$BRIDGE" || exit 72
done <<__VC_REMOVE_LOCAL_NEIGHBORS__
$remove_neighbor_current
__VC_REMOVE_LOCAL_NEIGHBORS__
remove_neighbor_current=$(vc_local_neighbor_records "$BRIDGE") || exit 79
[ -z "$remove_neighbor_current" ] || exit 79
fi
# vc-local-neighbor-remove-cleanup-end
if [ "$MODE" = networkmanager_nmcli ]; then
vc_nm_target_inventory || exit 71
case "$NM_TARGET_COUNT" in
0) : ;;
1)
[ -n "$NM_REMOVE_UUID" ] && [ "$NM_TARGET_EXACT" = yes ] &&
[ "$NM_TARGET_UUID" = "$NM_REMOVE_UUID" ] || exit 71
vc_nm_profile_resolve || exit 71
[ "$NM_PROFILE_UUID" = "$NM_REMOVE_UUID" ] || exit 71
vc_nm_profile_base_validate || exit 71
vc_nm_profile_empty connection.zone || exit 71
"$NMCLI" connection delete uuid "$NM_REMOVE_UUID" >/dev/null 2>&1 || exit 72
vc_nm_target_inventory || exit 72
[ "$NM_TARGET_COUNT" = 0 ] || exit 72
;;
*) exit 71 ;;
esac
else
if [ -f "$IF_FRAGMENT" ]; then
vc_ifupdown_target_inventory || exit 71
[ "$IFUP_TARGET_COUNT" = 1 ] || exit 71
"$IFQUERY" "$BRIDGE" >/dev/null 2>&1 || exit 71
"$IFDOWN" --force "$BRIDGE" >/dev/null 2>&1 || exit 72
fi
fi
set_phase manager_removed
if "$IP" link show dev "$BRIDGE" >/dev/null 2>&1; then
bridge_info=$("$IP" -d link show dev "$BRIDGE" 2>/dev/null) || exit 71
printf '%s\\n' "$bridge_info" | grep -Eq '(^|[[:space:]])bridge[[:space:]]+(forward_delay|stp_state|hello_time|ageing_time)([[:space:]]|$)' || exit 71
ports=$(vc_bridge_port_records "$BRIDGE" 2>/dev/null) || exit 71
[ -z "$ports" ] || exit 79
bridge_addrs=$("$IP" -o addr show dev "$BRIDGE" 2>/dev/null) || exit 71
[ -z "$bridge_addrs" ] || exit 79
bridge_routes=$("$IP" -o -4 -N route show table all dev "$BRIDGE" 2>/dev/null) || exit 71
[ -z "$bridge_routes" ] || exit 79
bridge_routes6=$("$IP" -o -6 -N route show table all dev "$BRIDGE" 2>/dev/null) || exit 71
[ -z "$bridge_routes6" ] || exit 79
proxy_left=$("$IP" neigh show proxy dev "$BRIDGE" 2>/dev/null | awk '$1 ~ /^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$/') || exit 71
[ -z "$proxy_left" ] || exit 79
remove_neighbor_left=$(vc_local_neighbor_records "$BRIDGE") || exit 79
[ -z "$remove_neighbor_left" ] || exit 79
"$IP" link del dev "$BRIDGE" type bridge || exit 72
fi
if "$IP" link show dev "$BRIDGE" >/dev/null 2>&1; then exit 72; fi
set_phase bridge_removed
if [ -f "$NFT_FILE" ]; then
nft_tables=$(awk '$1=="table" {{print $2 "|" $3}}' "$NFT_FILE")
nft_table_count=0
NFT_REMOVE_BATCH="$TXN/remove.nft"
: > "$NFT_REMOVE_BATCH"
while IFS='|' read -r family name; do
[ -n "$family" ] || continue
case "$family:$name" in inet:*|bridge:*|arp:*) : ;; *) exit 71 ;; esac
[ "$name" = "$TABLE" ] || exit 71
table_dump=$("$NFT" -s list table "$family" "$name" 2>/dev/null) || exit 71
printf '%s\\n' "$table_dump" | grep -Fq 'virt-cloud-vpc-dataplane-v1' || exit 71
printf 'delete table %s %s\\n' "$family" "$name" >> "$NFT_REMOVE_BATCH"
nft_table_count=$((nft_table_count + 1))
done <<EOF
$nft_tables
EOF
[ "$nft_table_count" = 3 ] || exit 71
vc_remove_capture_nft_precommit "$TXN/nft.remove.precommit" || exit 71
"$NFT" -c -f "$NFT_REMOVE_BATCH" >/dev/null || exit 72
# Check mode has no kernel reservation/CAS primitive. Re-read all complete
# stateless tables after dry-run and immediately before the one delete batch.
vc_remove_nft_precommit_matches "$TXN/nft.remove.precommit" || exit 71
"$NFT" -f "$NFT_REMOVE_BATCH" >/dev/null || exit 72
vc_remove_nft_tables_absent || exit 72
fi
if grep -Fqx "$NFT_INCLUDE" "$NFT_MAIN"; then
NFT_MAIN_TMP=$("$MKTEMP" "$NFT_MAIN.virt-cloud-vpc-remove.XXXXXX") || exit 72
awk -v needle="$NFT_INCLUDE" '$0 != needle' "$NFT_MAIN" > "$NFT_MAIN_TMP" || {{ rm -f "$NFT_MAIN_TMP"; exit 72; }}
chmod "$($STAT -c '%a' "$NFT_MAIN")" "$NFT_MAIN_TMP" || {{ rm -f "$NFT_MAIN_TMP"; exit 72; }}
"$NFT" -c -f "$NFT_MAIN_TMP" >/dev/null || {{ rm -f "$NFT_MAIN_TMP"; exit 72; }}
mv -f "$NFT_MAIN_TMP" "$NFT_MAIN" || exit 72
grep -Fqx "$NFT_INCLUDE" "$NFT_MAIN" && exit 72 || true
fi
set_phase nft_removed
if [ -e "$NFT_FILE" ] || [ -L "$NFT_FILE" ]; then
[ -f "$NFT_FILE" ] && [ ! -L "$NFT_FILE" ] || exit 71
grep -Fqx '# virt-cloud-vpc-dataplane-v1' "$NFT_FILE" || exit 71
nft_identity=$(sed -n '3{{s/^# //;p;}}' "$NFT_FILE")
vc_vpc_artifact_identity_check "$nft_identity" || exit 71
fi
if [ -e "$SYSCTL_FILE" ] || [ -L "$SYSCTL_FILE" ]; then
[ -f "$STATE_FILE" ] || exit 71
vc_vpc_sysctl_artifact_owned "$SYSCTL_FILE" {_q(plan.vpc.vpc_id)} \
"$state_revision" "$state_digest" "$BRIDGE" || exit 71
fi
if [ -e "$PROXY_FILE" ] || [ -L "$PROXY_FILE" ]; then
[ -f "$PROXY_FILE" ] && [ ! -L "$PROXY_FILE" ] || exit 71
sed -n '1p' "$PROXY_FILE" | grep -Fqx '# virt-cloud-vpc-proxy-v1' || exit 71
fi
if [ -e "$IF_FRAGMENT" ] || [ -L "$IF_FRAGMENT" ]; then
[ -f "$IF_FRAGMENT" ] && [ ! -L "$IF_FRAGMENT" ] || exit 71
if_identity=$(sed -n '1s/^# {DATAPLANE_OWNER_MARKER} //p' "$IF_FRAGMENT")
vc_vpc_artifact_identity_check "$if_identity" || exit 71
fi
if [ -f "$STATE_FILE" ]; then
[ ! -L "$STATE_FILE" ] || exit 71
grep -Fqx 'owner|{DATAPLANE_OWNER_MARKER}' "$STATE_FILE" || exit 71
grep -Fqx 'vpc|{plan.vpc.vpc_id}' "$STATE_FILE" || exit 71
grep -Fqx 'fabric|{plan.vpc.fabric_id}' "$STATE_FILE" || exit 71
grep -Fqx 'host|{plan.host_id}' "$STATE_FILE" || exit 71
grep -Fqx 'bridge|{r.bridge}' "$STATE_FILE" || exit 71
grep -Fqx 'table|{r.route_table}' "$STATE_FILE" || exit 71
state_firewall_record=$(awk -F'|' '$1=="firewall_mode" {{count++; value=$2}} END {{print (count + 0) "|" value}}' "$STATE_FILE")
case "$state_firewall_record" in '0|'|'1|native_nftables') : ;; *) exit 71 ;; esac
state_revision=$(awk -F'|' '$1=="revision" {{print $2; exit}}' "$STATE_FILE")
state_digest=$(awk -F'|' '$1=="digest" {{print $2; exit}}' "$STATE_FILE")
case "$state_revision" in ''|*[!0-9]*) exit 79;; esac
[ "$state_revision" -le {plan.desired_revision} ] || exit 79
if [ "$state_revision" = {plan.desired_revision} ] && [ "$state_digest" != { _q(plan.desired_digest) } ]; then exit 79; fi
fi
rm -f "$NFT_FILE" "$SYSCTL_FILE" "$PROXY_FILE" "$IF_FRAGMENT" \
"$NFT_SNAPSHOT_PREFIX.inet.live" "$NFT_SNAPSHOT_PREFIX.bridge.live" \
"$NFT_SNAPSHOT_PREFIX.arp.live" "$OWNER_FILE" "$STATE_FILE"
[ ! -e "$NFT_FILE" ] && [ ! -e "$SYSCTL_FILE" ] && [ ! -e "$PROXY_FILE" ] &&
[ ! -e "$IF_FRAGMENT" ] && [ ! -e "$NFT_SNAPSHOT_PREFIX.inet.live" ] &&
[ ! -e "$NFT_SNAPSHOT_PREFIX.bridge.live" ] &&
[ ! -e "$NFT_SNAPSHOT_PREFIX.arp.live" ] &&
[ ! -e "$OWNER_FILE" ] && [ ! -e "$STATE_FILE" ] || exit 72
set_phase files_removed
{state_loader}
vpc_state_load || exit 74
vpc_state_refresh_nft || exit 74
vc_vpc_fabric_nft_live_ready || exit 74
vpc_state_nft_set_matches || exit 74
vc_nft_table_snapshot inet virt_cloud_wg_fabric \
"$TXN/fabric.nft.desired" || exit 74
[ -f "$TXN/fabric.nft.desired" ] &&
[ ! -e "$TXN/fabric.nft.desired.absent" ] || exit 74
vc_nft_table_matches_snapshot inet virt_cloud_wg_fabric \
"$TXN/fabric.nft.desired" || exit 74
expected_fabric_peer_keys={_q(remove_peer_key_words)}
if [ "{remove_peer_closure_provided}" = yes ]; then
live_fabric_peer_keys=$("$WG" show "$WG_IFACE" peers 2>/dev/null) || exit 74
for live_peer_key in $live_fabric_peer_keys; do
case " $expected_fabric_peer_keys " in
*" $live_peer_key "*) : ;;
*) exit 74 ;;
esac
done
for expected_peer_key in $expected_fabric_peer_keys; do
printf '%s\n' "$live_fabric_peer_keys" |
grep -Fqx "$expected_peer_key" || exit 74
done
while IFS='|' read -r key expected_overlay; do
[ -n "$key" ] || continue
actual=$(vc_wg_allowed_for_peer "$WG_IFACE" "$key") || exit 74
vpc_state_overlay_from_allowed "$actual" >/dev/null || exit 74
extra=$(vpc_state_allowed_for_peer "$key") || exit 74
merged="$expected_overlay"; [ -z "$extra" ] || merged="$merged,$extra"
"$WG" set "$WG_IFACE" peer "$key" allowed-ips "$merged" || exit 74
done <<'{remove_peer_marker}'
{remove_peer_lines}
{remove_peer_marker}
else
# Legacy direct callers lack a full authenticated overlay mapping. Keep
# their historical route-withdrawal behavior without treating it as an
# authoritative Fabric closure receipt.
for key in $("$WG" show "$WG_IFACE" peers 2>/dev/null); do
actual=$(vc_wg_allowed_for_peer "$WG_IFACE" "$key") || exit 74
overlay=$(vpc_state_overlay_from_allowed "$actual") || exit 74
extra=$(vpc_state_allowed_for_peer "$key") || exit 74
merged="$overlay"; [ -z "$extra" ] || merged="$merged,$extra"
"$WG" set "$WG_IFACE" peer "$key" allowed-ips "$merged" || exit 74
done
fi
if [ "{remove_peer_closure_provided}" = yes ]; then
while IFS='|' read -r final_peer_key final_overlay; do
[ -n "$final_peer_key" ] || continue
final_actual=$(vc_wg_allowed_for_peer "$WG_IFACE" "$final_peer_key") || exit 74
vpc_state_overlay_from_allowed "$final_actual" >/dev/null || exit 74
final_extra=$(vpc_state_allowed_for_peer "$final_peer_key") || exit 74
final_expected="$final_overlay"
[ -z "$final_extra" ] || final_expected="$final_expected,$final_extra"
final_actual_set=$(printf '%s\n' "$final_actual" | tr ',' '\n' | awk 'NF' | LC_ALL=C sort -u)
final_expected_set=$(printf '%s\n' "$final_expected" | tr ',' '\n' | awk 'NF' | LC_ALL=C sort -u)
[ -n "$final_actual_set" ] &&
[ "$final_actual_set" = "$final_expected_set" ] || exit 74
done <<'{remove_peer_marker}'
{remove_peer_lines}
{remove_peer_marker}
fi
vc_vpc_fabric_nft_live_ready || exit 74
vpc_state_nft_set_matches || exit 74
vc_nft_table_matches_snapshot inet virt_cloud_wg_fabric \
"$TXN/fabric.nft.desired" || exit 74
set_phase wg_merged
verify_routes_removed || exit 79
for family in inet bridge arp; do
"$NFT" list table "$family" "$TABLE" >/dev/null 2>&1 && exit 79 || true
done
grep -Fqx "$NFT_INCLUDE" "$NFT_MAIN" && exit 79 || true
if [ "$MODE" = networkmanager_nmcli ]; then
vc_nm_target_inventory || exit 79
[ "$NM_TARGET_COUNT" = 0 ] || exit 79
fi
vc_ifupdown_target_inventory || exit 79
[ "$IFUP_TARGET_COUNT" = 0 ] || exit 79
[ ! -e "$STATE_FILE" ] && [ ! -e "$OWNER_FILE" ] &&
[ ! -e "$NFT_SNAPSHOT_PREFIX.inet.live" ] &&
[ ! -e "$NFT_SNAPSHOT_PREFIX.bridge.live" ] &&
[ ! -e "$NFT_SNAPSHOT_PREFIX.arp.live" ] || exit 79
if "$IP" link show dev "$BRIDGE" >/dev/null 2>&1; then exit 79; fi
# Keep the final durable commit behind one last full semantic/digest/tenant
# readback and the post-refresh stateless snapshot. The shared Fabric flock
# excludes every panel writer throughout this transaction.
vc_vpc_fabric_nft_live_ready || exit 74
vpc_state_nft_set_matches || exit 74
vc_nft_table_matches_snapshot inet virt_cloud_wg_fabric \
"$TXN/fabric.nft.desired" || exit 74
printf '%s\\n' '__VC_VPC_REMOVED__=yes'
printf '%s\\n' '__VC_VPC_OWNER__={DATAPLANE_OWNER_MARKER}'
printf '%s\\n' '__VC_VPC_REVISION__={plan.desired_revision}'
printf '%s\\n' '__VC_VPC_DIGEST__={plan.desired_digest}'
set_phase committed
rm -rf "$TXN"
printf '%s\\n' '__VC_VPC_ROLLBACK__=none'
exit 0
'''
if len(script) > MAX_OUTPUT:
raise VpcDataplaneError("VPC 销毁脚本过大", 413, "vpc_script_too_large")
return script
# Real host calls use the strict, manager-aware builder. The old positional
# function remains above for compatibility with callers that only render a
# fixture.
render_apply_script = _render_apply_script_v2
def render_observe_script(
plan: VpcDataplanePlan,
*,
deployment_mode: str = "networkmanager_nmcli",
firewall_mode: str = FIREWALL_MODE_NATIVE,
) -> str:
if deployment_mode not in {"networkmanager_nmcli", "ifupdown"}:
raise VpcDataplaneError("宿主机网络管理模式不支持", 409,
"vpc_dataplane_network_manager_conflict")
firewall_mode = _firewall_mode(firewall_mode)
from .fabric import (
_FABRIC_NFT_VALIDATION_SH,
_FIREWALL_MODE_GUARD_SH,
_NMCLI_FILENAME_RESOLVER_SH,
_VPC_STATE_LOADER,
)
routes = _routes(plan)
r = plan.resources
proxy_file = f"/etc/virt-cloud/vpc-dataplane/{r.token}.proxy"
nft_file = f"/etc/nftables.d/virt-cloud-vpc-{r.token}.nft"
nft_include = f'include "{nft_file}"'
sysctl_file = f"/etc/sysctl.d/99-virt-cloud-vpc-{r.token}.conf"
ifup_fragment = f"/etc/network/interfaces.d/virt-cloud-vpc-{r.token}"
nm_profile_marker = DATAPLANE_OWNER_MARKER + " vpc=" + plan.vpc.vpc_id
route_lines = "\n".join(
f"{x['destination_cidr']}|{x['device']}|{x.get('via') or '-'}|{x.get('metric', 100)}"
for x in routes)
peer_lines = "\n".join(
# The host's validated state files, rather than the control-plane
# catalog aggregate, own the tenant half of AllowedIPs. The current
# plan still supplies the authenticated peer identity and overlay /32.
f"{p.host_id}|{p.public_key}|{p.overlay_cidr}"
for p in plan.peers)
authoritative_peer_overlays = _authoritative_fabric_peer_overlays(plan)
fabric_peer_overlay_lines = "\n".join(
f"{key}|{overlay}"
for key, overlay in (authoritative_peer_overlays or {}).items())
fabric_peer_key_words = " ".join(authoritative_peer_overlays or ())
fabric_peer_key_arg = _q(fabric_peer_key_words)
fabric_peer_keys_provided = (
"yes" if authoritative_peer_overlays is not None else "no")
route_marker = "__VC_VPC_OBS_ROUTES_" + hashlib.sha256(
route_lines.encode()).hexdigest()[:12]
peer_marker = "__VC_VPC_OBS_PEERS_" + hashlib.sha256(
peer_lines.encode()).hexdigest()[:12]
fabric_peer_marker = "__VC_VPC_OBS_FABRIC_PEERS_" + hashlib.sha256(
fabric_peer_overlay_lines.encode()).hexdigest()[:12]
state_loader = _VPC_STATE_LOADER.replace(
"__VC_FABRIC_HOST_ID__", plan.host_id).replace(
"__VC_FABRIC_ID__", plan.vpc.fabric_id).replace(
"__VC_FABRIC_NETWORK__", "100.64.0.0/10").replace(
"__VC_VPC_STATE_OBSERVE__", "yes")
expected_state_hash = hashlib.sha256(_state(plan, routes, firewall_mode).encode()).hexdigest()
expected_ifup = (
f"# {DATAPLANE_OWNER_MARKER} vpc={plan.vpc.vpc_id} "
f"revision={plan.desired_revision} digest={plan.desired_digest}\n"
f"iface {r.bridge} inet manual\n"
" bridge_ports none\n bridge-stp off\n")
expected_ifup_hash = hashlib.sha256(expected_ifup.encode()).hexdigest()
nft_marker = _nft_marker(plan)
# These comments are emitted by render_vpc_nft_rules. Requiring the full
# owned shape prevents a stale or foreign table being reported as active
# merely because the deterministic table name exists.
nft_suffixes = (
*_NFT_INET_REQUIRED_SUFFIXES,
*(f"acl:{acl.acl_id}" for acl in plan.vpc.acls),
)
nft_checks = "\n".join(
f''' printf '%s\\n' "$nft_text" | grep -Fq "comment \\\"$NFT_MARKER:{suffix}\\\"" || nft_verified=no'''
for suffix in nft_suffixes)
bridge_suffixes = (
*_NFT_BRIDGE_REQUIRED_SUFFIXES,
*(f"acl:{acl.acl_id}" for acl in plan.vpc.acls),
)
bridge_checks = "\n".join(
f''' printf '%s\\n' "$bridge_text" | grep -Fq "comment \\\"$NFT_MARKER:{suffix}\\\"" || bridge_verified=no'''
for suffix in bridge_suffixes)
if plan.local_members:
bridge_observe = f'''bridge_text=$($NFT -y list table bridge "$TABLE" 2>/dev/null || true)
bridge_verified=yes
printf '%s\\n' "$bridge_text" | grep -Fq "table bridge $TABLE" || bridge_verified=no
printf '%s\\n' "$bridge_text" | grep -Fq "chain forward" || bridge_verified=no
printf '%s\\n' "$bridge_text" | grep -Fq "hook forward priority -190" || bridge_verified=no
printf '%s\\n' "$bridge_text" | grep -Fq "policy accept" || bridge_verified=no
printf '%s\\n' "$bridge_text" | grep -Fq "set local_member_ips" || bridge_verified=no
printf '%s\\n' "$bridge_text" | grep -Fq "set local_dest_ips" || bridge_verified=no
{bridge_checks}
if [ "$bridge_verified" = yes ]; then printf '%s\\n' '__VC_VPC_BRIDGE_NFT__=yes'; else printf '%s\\n' '__VC_VPC_BRIDGE_NFT__=no'; fi'''
else:
# An empty VPC must not silently retain a stale bridge-family table
# from a previous revision or host membership.
bridge_observe = '''
if $NFT list table bridge "$TABLE" >/dev/null 2>&1; then
printf '%s\\n' '__VC_VPC_BRIDGE_NFT__=no'
else
printf '%s\\n' '__VC_VPC_BRIDGE_NFT__=none'
fi'''.strip()
if plan.local_members:
arp_observe = f'''arp_text=$($NFT -y list table arp "$TABLE" 2>/dev/null || true)
arp_verified=yes
printf '%s\\n' "$arp_text" | grep -Fq "table arp $TABLE" || arp_verified=no
printf '%s\\n' "$arp_text" | grep -Fq "set proxy_ips" || arp_verified=no
printf '%s\\n' "$arp_text" | grep -Fq "chain output" || arp_verified=no
printf '%s\\n' "$arp_text" | grep -Fq "hook output priority -190" || arp_verified=no
printf '%s\\n' "$arp_text" | grep -Fq "policy accept" || arp_verified=no
printf '%s\\n' "$arp_text" | grep -Fq "comment \\\"$NFT_MARKER:arp-proxy-output\\\"" || arp_verified=no
if [ "$arp_verified" = yes ]; then printf '%s\\n' '__VC_VPC_ARP_NFT__=yes'; else printf '%s\\n' '__VC_VPC_ARP_NFT__=no'; fi'''
else:
arp_observe = '''
if $NFT list table arp "$TABLE" >/dev/null 2>&1; then
printf '%s\\n' '__VC_VPC_ARP_NFT__=no'
else
printf '%s\\n' '__VC_VPC_ARP_NFT__=none'
fi'''.strip()
route_extra_awk = "\n".join(
f' if ($1 "|" $2 "|" $3 "|" $4 "|" $5 == "{x["destination_cidr"]}|{x["device"]}|{x.get("via") or "-"}|{x.get("metric", 100)}|186" && ($6 == "-" || $6 == "linkdown")) next'
for x in routes)
route_checks = "\n".join(
f'''state=missing
route_flags=$(printf '%s\\n' "$route_records" | awk -F'|' -v wanted={_q(x['destination_cidr'])} -v expected_device={_q(x['device'])} -v expected_via={_q(x.get('via') or '-')} -v expected_metric={_q(str(x.get('metric', 100)))} '$1 == wanted && $2 == expected_device && $3 == expected_via && $4 == expected_metric && $5 == "186" {{print $6}}')
case "$route_flags" in -|linkdown) state=owned ;; *) route_flags=unknown ;; esac
printf '%s\\n' '__VC_VPC_ROUTE__={x['destination_cidr']}|{x['device']}|'"$ROUTE_TABLE"'|{x.get('via') or '-'}|{x.get('metric', 100)}|'"$route_flags|$state"
'''.rstrip()
for x in routes)
if not route_checks:
route_checks = ": # no expected VPC routes"
rule_specs = (
(str(r.rule_priority), "-", plan.vpc.cidr, str(r.route_table)),
(str(r.rule_priority + 1), FABRIC_INTERFACE, plan.vpc.cidr,
str(r.route_table)),
)
rule_checks = "\n".join(
f'''rule_state=missing
if vc_ip_rule_read {_q(pref)} {_q(iif)} {_q(destination)} {_q(table)} 1 1 &&
[ "$VC_IP_RULE_PRESENT" = yes ]; then
rule_state=owned
fi
printf '%s\\n' '__VC_VPC_RULE__={pref}|{iif}|{destination}|{table}|'"$rule_state"
'''.rstrip()
for pref, iif, destination, table in rule_specs)
return f'''#!/bin/sh
set -eu
PATH=/usr/sbin:/usr/bin:/sbin:/bin
IP=/usr/sbin/ip; [ -x "$IP" ] || IP=/usr/bin/ip
{BRIDGE_PORT_RECORDS_SH}
{LOCAL_NEIGHBOR_SH}
{IP_RULE_CONVERGENCE_SH}
{WG_ALLOWED_IPS_SH}
{VPC_ROUTE_RECORDS_SH}
WG=/usr/bin/wg; [ -x "$WG" ] || WG=/usr/sbin/wg
NFT=/usr/sbin/nft; [ -x "$NFT" ] || NFT=/usr/bin/nft
FLOCK=/usr/bin/flock; [ -x "$FLOCK" ] || FLOCK=/usr/sbin/flock
STAT=/usr/bin/stat; [ -x "$STAT" ] || STAT=/bin/stat
CMP=/usr/bin/cmp; [ -x "$CMP" ] || CMP=/bin/cmp
SYSTEMCTL=$(for p in /usr/bin/systemctl /bin/systemctl /usr/sbin/systemctl /sbin/systemctl; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
LOCK=/run/virt-cloud/wg-fabric0.lock
if [ -L /run/virt-cloud ] || {{ [ -e /run/virt-cloud ] && [ ! -d /run/virt-cloud ]; }}; then exit 71; fi
mkdir -p /run/virt-cloud; chmod 700 /run/virt-cloud
[ ! -L "$LOCK" ] && {{ [ ! -e "$LOCK" ] || [ -f "$LOCK" ]; }} || exit 71
: >> "$LOCK"; chmod 600 "$LOCK"; exec 8>>"$LOCK"; "$FLOCK" -s 8
BRIDGE_CMD=
for p in /usr/sbin/bridge /usr/bin/bridge /sbin/bridge /bin/bridge; do
if [ -x "$p" ]; then BRIDGE_CMD=$p; break; fi
done
BRIDGE={_q(r.bridge)}; TABLE={_q(r.nft_table)}; ROUTE_TABLE={r.route_table}; WG_IFACE={_q(FABRIC_INTERFACE)}; OWNER={_q(r.owner_file)}; STATE={_q(r.state_file)}; TXN={_q(r.transaction_dir)}; PROXY_FILE={_q(proxy_file)}
NFT_SNAPSHOT_PREFIX={_q(f'/etc/virt-cloud/vpc-dataplane/{r.token}.nft')}
NFT_FILE={_q(nft_file)}; NFT_INCLUDE={_q(nft_include)}; LEGACY_MANIFEST={_q(LEGACY_FIREWALLD_NFT_MANIFEST)}; SYSCTL_FILE={_q(sysctl_file)}; IF_FRAGMENT={_q(ifup_fragment)}; NM_PROFILE_MARKER={_q(nm_profile_marker)}; MODE={_q(deployment_mode)}; FIREWALL_MODE={_q(firewall_mode)}; NFT_MAIN=; NFT_MAIN_COUNT=0; NM_PROFILE_UUID=; NM_PROFILE_FILE=
NFT_MARKER={_q(nft_marker)}
EXPECTED_DIGEST={_q(plan.fabric_plan_digest)}
{NFT_LIVE_SNAPSHOT_SH}
{VPC_APPLY_RECEIPT_SH}
{VPC_LOCAL_NEIGHBOR_TRANSACTION_SH}
SHA256SUM=$(command -v sha256sum || true)
{_NMCLI_FILENAME_RESOLVER_SH}
{NETWORKMANAGER_VPC_PROFILE_SH}
{NETWORKMANAGER_VPC_TARGET_INVENTORY_SH}
{IFUPDOWN_VPC_TARGET_INVENTORY_SH}
{_FIREWALL_MODE_GUARD_SH}
{_FABRIC_NFT_VALIDATION_SH}
{_VPC_SYSCTL_VALIDATION_SH}
{VPC_NATIVE_TRANSACTION_GUARD_SH}
nft_plan_owned() {{
nft_plan_text=$1
nft_plan_digest=$2
printf '%s\n' "$nft_plan_digest" | grep -Eq '^[0-9a-f]{{64}}$' || return 1
vc_fabric_nft_strict_shape "$nft_plan_text" || return 1
[ "$(printf '%s\n' "$nft_plan_text" |
grep -Foc 'comment "virt-cloud-wg-fabric-v1:'"$nft_plan_digest"'"' || true)" = 8 ]
}}
{state_loader}
FABRIC_NFT_SNAPSHOT=/run/virt-cloud/.virt-cloud-vpc-fabric-observe.$$
vc_vpc_observe_cleanup() {{
rm -f "$VPC_PEER_FILE" "$VPC_ROUTE_FILE" "$VPC_TENANT_FILE" \
"$VPC_TENANT_FILE.nft" "$FABRIC_NFT_SNAPSHOT" \
"$FABRIC_NFT_SNAPSHOT.absent" "$FABRIC_NFT_SNAPSHOT.raw" \
"$FABRIC_NFT_SNAPSHOT.current" "$FABRIC_NFT_SNAPSHOT.inventory" \
2>/dev/null || true
}}
trap 'vc_vpc_observe_cleanup' EXIT
trap 'exit 79' HUP INT TERM
vc_vpc_fabric_closure_ready() {{
case "$VPC_STATE_STATUS" in owned|empty) : ;; *) return 1 ;; esac
vc_nft_table_snapshot inet virt_cloud_wg_fabric \
"$FABRIC_NFT_SNAPSHOT" || return 1
[ -f "$FABRIC_NFT_SNAPSHOT" ] &&
[ ! -L "$FABRIC_NFT_SNAPSHOT" ] &&
[ ! -e "$FABRIC_NFT_SNAPSHOT.absent" ] || return 1
vc_fabric_observe_text=$(cat "$FABRIC_NFT_SNAPSHOT") || return 1
nft_plan_owned "$vc_fabric_observe_text" "$EXPECTED_DIGEST" || return 1
vpc_state_nft_set_matches || return 1
# Re-read the complete stateless table after the exact set comparison.
# This is the final shared-Fabric proof before the observer may emit the
# marker that participates in an active VPC result.
vc_nft_table_matches_snapshot inet virt_cloud_wg_fabric \
"$FABRIC_NFT_SNAPSHOT"
}}
vc_vpc_target_nft_absent() {{
for vc_absent_family in inet bridge arp; do
vc_absent_inventory=$("$NFT" -s -n -y list tables \
"$vc_absent_family") || return 1
vc_absent_count=$(printf '%s\n' "$vc_absent_inventory" |
awk -v family="$vc_absent_family" -v wanted="$TABLE" '
NF==0 {{next}}
NF!=3 || $1!="table" || $2!=family || $3!~/^[A-Za-z0-9_.-]+$/ {{bad=1}}
$3==wanted {{found++}}
END {{if (bad || found>1) exit 1; print found+0}}') || return 1
[ "$vc_absent_count" = 0 ] || return 1
done
}}
firewall_mode_state=owned
vc_firewall_mode_ready || firewall_mode_state=mismatch
[ "$FIREWALL_MODE" = native_nftables ] || firewall_mode_state=mismatch
vc_vpc_firewall_boundary_clean || firewall_mode_state=mismatch
[ ! -e "$TXN" ] && [ ! -L "$TXN" ] || firewall_mode_state=mismatch
owner=missing; [ -f "$OWNER" ] && owner=$(cat "$OWNER" 2>/dev/null || printf missing)
printf '%s\\n' "__VC_VPC_BRIDGE_OWNER__=$owner"
persistence_target=no
opposite_clean=yes
for nft_candidate in /etc/nftables.conf /etc/sysconfig/nftables.conf; do
if [ -f "$nft_candidate" ] && [ ! -L "$nft_candidate" ] && [ "$($STAT -c '%u' "$nft_candidate" 2>/dev/null)" = 0 ]; then
case "$($STAT -c '%a' "$nft_candidate" 2>/dev/null)" in
[0-7][0145][0145]) NFT_MAIN="$nft_candidate"; NFT_MAIN_COUNT=$((NFT_MAIN_COUNT + 1)) ;;
*) : ;;
esac
fi
done
if [ -e "$LEGACY_MANIFEST" ] || [ -L "$LEGACY_MANIFEST" ]; then opposite_clean=no; fi
if [ "$NFT_MAIN_COUNT" = 1 ] && grep -Fqx "$NFT_INCLUDE" "$NFT_MAIN" &&
"$NFT" -c -f "$NFT_MAIN" >/dev/null 2>&1; then
persistence_target=yes
fi
persistence=no
nft_file_ok=no
if [ -f "$NFT_FILE" ] && [ ! -L "$NFT_FILE" ] && [ "$(stat -c '%u:%a' "$NFT_FILE" 2>/dev/null)" = 0:600 ] &&
grep -Fqx '# virt-cloud-vpc-dataplane-v1' "$NFT_FILE" &&
grep -Fqx '# vpc={plan.vpc.vpc_id} revision={plan.desired_revision} digest={plan.desired_digest}' "$NFT_FILE"; then
nft_file_ok=yes
fi
sysctl_file_ok=no
if vc_vpc_sysctl_artifact_owned "$SYSCTL_FILE" {_q(plan.vpc.vpc_id)} \
{plan.desired_revision} {_q(plan.desired_digest)} "$BRIDGE"; then
sysctl_file_ok=yes
fi
nm_owned=no
nm_zone=
nm_present=no
# The pinned deployment mode decides which persistence inventory is
# authoritative. In NetworkManager mode, a missing binary or failed daemon
# query is unknown rather than proof that the profile is absent. Do not put
# nmcli on the left of a pipeline: POSIX sh would report awk's success and
# silently turn a failed query into count=0.
nm_profile_count=invalid
nm_profile_query=unavailable
NMCLI=
if [ "$MODE" = networkmanager_nmcli ]; then
NMCLI=$(for p in /usr/bin/nmcli /usr/sbin/nmcli /bin/nmcli /sbin/nmcli; do
if [ -x "$p" ]; then printf '%s' "$p"; break; fi
done)
if [ -n "$NMCLI" ]; then
if vc_nm_target_inventory; then
nm_profile_query=$NM_TARGET_QUERY
nm_profile_count=$NM_TARGET_COUNT
else
nm_profile_query=$NM_TARGET_QUERY
nm_profile_count=invalid
fi
fi
case "$nm_profile_count" in
''|*[!0-9]*) nm_profile_count=invalid; nm_present=yes ;;
0) : ;;
*) nm_present=yes ;;
esac
else
# ifupdown persistence is proved independently by IF_FRAGMENT. A host
# pinned to this mode is not required to run a NetworkManager daemon.
nm_profile_count=0
nm_profile_query=not_applicable
fi
if [ "$nm_profile_query" = ok ] && [ "$nm_profile_count" = 1 ] &&
[ "$NM_TARGET_EXACT" = yes ]; then
if vc_nm_profile_resolve && [ "$NM_PROFILE_UUID" = "$NM_TARGET_UUID" ] &&
vc_nm_profile_expect active; then nm_owned=yes; fi
fi
printf '%s\\n' "__VC_VPC_NM_PROFILE_COUNT__=$nm_profile_count"
printf '%s\\n' "__VC_VPC_NM_PROFILE_QUERY__=$nm_profile_query"
ifup_profile_count=invalid
ifup_profile_query=invalid
if vc_ifupdown_target_inventory; then
ifup_profile_count=$IFUP_TARGET_COUNT
ifup_profile_query=$IFUP_TARGET_QUERY
fi
printf '%s\\n' "__VC_VPC_IFUP_PROFILE_COUNT__=$ifup_profile_count"
printf '%s\\n' "__VC_VPC_IFUP_PROFILE_QUERY__=$ifup_profile_query"
ifup_owned=no
if [ -f "$IF_FRAGMENT" ] && [ ! -L "$IF_FRAGMENT" ] && [ "$(stat -c '%u:%a' "$IF_FRAGMENT" 2>/dev/null)" = 0:600 ] &&
grep -Fqx '# {DATAPLANE_OWNER_MARKER} vpc={plan.vpc.vpc_id} revision={plan.desired_revision} digest={plan.desired_digest}' "$IF_FRAGMENT" &&
grep -Fqx 'iface {r.bridge} inet manual' "$IF_FRAGMENT"; then
if ! grep -Eq '^[[:space:]]*(auto|allow-[^[:space:]]+)[[:space:]]+{r.bridge}([[:space:]]|$)' "$IF_FRAGMENT"; then
ifup_owned=yes
fi
fi
printf '%s\\n' "__VC_VPC_FIREWALL_MODE__=$FIREWALL_MODE"
printf '%s\\n' "__VC_VPC_FIREWALL_STATE__=$firewall_mode_state"
nft_snapshot_state=yes
for snapshot_family in inet bridge arp; do
snapshot_path="$NFT_SNAPSHOT_PREFIX.$snapshot_family.live"
if [ -e "$snapshot_path" ] || [ -L "$snapshot_path" ]; then
nft_snapshot_state=no
fi
done
printf '%s\\n' "__VC_VPC_NFT_SNAPSHOT__=$nft_snapshot_state"
if [ "$firewall_mode_state" = owned ] && [ "$NFT_MAIN_COUNT" = 1 ] &&
[ "$persistence_target" = yes ] && [ "$opposite_clean" = yes ] &&
[ "$nft_file_ok" = yes ] && [ "$sysctl_file_ok" = yes ] &&
[ "$nft_snapshot_state" = yes ] &&
( {{ [ "$MODE" = networkmanager_nmcli ] &&
[ "$nm_profile_query" = ok ] && [ "$nm_profile_count" = 1 ] &&
[ "$nm_owned" = yes ] && [ "$ifup_owned" = no ] &&
[ "$ifup_profile_query" = ok ] && [ "$ifup_profile_count" = 0 ]; }} ||
{{ [ "$MODE" = ifupdown ] &&
[ "$nm_profile_query" = not_applicable ] && [ "$nm_profile_count" = 0 ] &&
[ "$nm_owned" = no ] && [ "$nm_present" = no ] &&
[ "$ifup_profile_query" = ok ] && [ "$ifup_profile_count" = 1 ] &&
[ "$ifup_owned" = yes ]; }} ); then
persistence=yes
fi
printf '%s\\n' "__VC_VPC_PERSISTENCE__=$persistence"
for artifact_spec in \
"NFT_FILE|$NFT_FILE" "SYSCTL_FILE|$SYSCTL_FILE" \
"PROXY_FILE|$PROXY_FILE" "IF_FRAGMENT|$IF_FRAGMENT" \
"OWNER_FILE|$OWNER" "STATE_FILE|$STATE" "TRANSACTION|$TXN"; do
artifact_name=${{artifact_spec%%|*}}
artifact_path=${{artifact_spec#*|}}
artifact_present=no
if [ -e "$artifact_path" ] || [ -L "$artifact_path" ]; then
artifact_present=yes
fi
printf '%s\\n' "__VC_VPC_${{artifact_name}}_PRESENT__=$artifact_present"
done
printf '%s\\n' "__VC_VPC_NFT_MAIN_COUNT__=$NFT_MAIN_COUNT"
nft_include_present=no
if [ "$NFT_MAIN_COUNT" = 1 ] && grep -Fqx "$NFT_INCLUDE" "$NFT_MAIN"; then
nft_include_present=yes
fi
printf '%s\\n' "__VC_VPC_NFT_INCLUDE_PRESENT__=$nft_include_present"
removal_bridge_absent=no
if removal_link_inventory=$("$IP" -o link show 2>/dev/null) &&
printf '%s\\n' "$removal_link_inventory" | awk -v wanted="$BRIDGE" '
NF<2 || $1!~/^[0-9]+:$/ {{bad=1; next}}
{{name=$2; sub(/:$/, "", name); sub(/@.*/, "", name); if (name==wanted) found++}}
END {{exit (bad || found) ? 1 : 0}}'; then
removal_bridge_absent=yes
fi
printf '%s\\n' "__VC_VPC_REMOVAL_BRIDGE_ABSENT__=$removal_bridge_absent"
if "$IP" -d link show dev "$BRIDGE" >/dev/null 2>&1; then printf '%s\\n' '__VC_VPC_BRIDGE__='"$BRIDGE"; else printf '%s\\n' '__VC_VPC_BRIDGE__=missing'; fi
bridge_boundary=no
bridge_shape=no; bridge_addr_ok=no; bridge_ports_ok=no; bridge_routes_ok=no; bridge_routes6_ok=no
if [ -n "$BRIDGE_CMD" ]; then
if bridge_shape_text=$("$IP" -d link show dev "$BRIDGE" 2>/dev/null) &&
printf '%s\n' "$bridge_shape_text" | grep -Eq '(^|[[:space:]])bridge[[:space:]]+(forward_delay|stp_state|hello_time|ageing_time)([[:space:]]|$)'; then
bridge_shape=yes
fi
if bridge_addr4=$("$IP" -o -4 addr show dev "$BRIDGE" 2>/dev/null) &&
bridge_addr6=$("$IP" -o -6 addr show dev "$BRIDGE" 2>/dev/null) &&
[ -z "$bridge_addr4$bridge_addr6" ]; then
bridge_addr_ok=yes
fi
if bridge_ports=$(vc_bridge_port_records "$BRIDGE" 2>/dev/null); then
bridge_ports_ok=yes
fi
if bridge_routes=$("$IP" -o -4 -N route show table all dev "$BRIDGE" 2>/dev/null) &&
bridge_route_records=$(printf '%s\\n' "$bridge_routes" | vc_vpc_route_records "$BRIDGE" main "$BRIDGE") &&
printf '%s\\n' "$bridge_route_records" | awk -F'|' -v wanted_table="$ROUTE_TABLE" '
NF {{if ($3!=wanted_table) bad=1}}
END {{exit bad ? 1 : 0}}'; then
bridge_routes_ok=yes
fi
if bridge_routes6=$($IP -o -6 -N route show table all dev "$BRIDGE" 2>/dev/null) && [ -z "$bridge_routes6" ]; then
bridge_routes6_ok=yes
fi
fi
if [ "$bridge_shape" = yes ] && [ "$bridge_addr_ok" = yes ] &&
[ "$bridge_ports_ok" = yes ] && [ "$bridge_routes_ok" = yes ] &&
[ "$bridge_routes6_ok" = yes ]; then
bridge_bad=no
while IFS= read -r bridge_port_line; do
[ -n "$bridge_port_line" ] || continue
bridge_port=$(printf '%s\\n' "$bridge_port_line" | sed -n 's/^[0-9][0-9]*: \\([^:[:space:]]*\\).*/\\1/p')
case "$bridge_port" in
vnet[0-9]*|tap[0-9A-Za-z_.-]*|macvtap[0-9]*|fwpr[0-9A-Za-z_.-]*|fwln[0-9A-Za-z_.-]*) : ;;
*) bridge_bad=yes ;;
esac
done <<__VC_VPC_OBS_PORTS__
$bridge_ports
__VC_VPC_OBS_PORTS__
if [ "$bridge_bad" = no ]; then
bridge_boundary=yes
fi
fi
printf '%s\\n' "__VC_VPC_BRIDGE_BOUNDARY__=$bridge_boundary"
if [ -r "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp" ]; then printf '%s\\n' '__VC_VPC_PROXY_ARP__='"$(cat "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp")"; else printf '%s\\n' '__VC_VPC_PROXY_ARP__=missing'; fi
if [ -r "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp_pvlan" ]; then printf '%s\\n' '__VC_VPC_PROXY_ARP_PVLAN__='"$(cat "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp_pvlan")"; else printf '%s\\n' '__VC_VPC_PROXY_ARP_PVLAN__=missing'; fi
if [ -r "/proc/sys/net/ipv4/conf/$BRIDGE/rp_filter" ]; then printf '%s\\n' '__VC_VPC_RP_FILTER__='"$(cat "/proc/sys/net/ipv4/conf/$BRIDGE/rp_filter")"; else printf '%s\\n' '__VC_VPC_RP_FILTER__=missing'; fi
proxy_records=$($IP neigh show proxy dev "$BRIDGE" 2>/dev/null | awk '$1 ~ /^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$/ {{print $1}}' | LC_ALL=C sort -u || true)
printf '%s\\n' "$proxy_records" | while IFS= read -r proxy_ip; do
if [ -n "$proxy_ip" ]; then
printf '%s\\n' "__VC_VPC_PROXY_IP__=$proxy_ip"
fi
done
proxy_state=missing
if [ -f "$PROXY_FILE" ] && sed -n '1p' "$PROXY_FILE" | grep -Fqx '# virt-cloud-vpc-proxy-v1'; then
expected_proxy=$(sed '1d' "$PROXY_FILE" | LC_ALL=C sort -u)
if [ "$proxy_records" = "$expected_proxy" ]; then proxy_state=owned; fi
fi
printf '%s\\n' '__VC_VPC_PROXY_STATE__='"$proxy_state"
# vc-local-neighbor-observe-begin
local_neighbor_state=missing
local_neighbor_expected={_q(chr(10).join(sorted(f"{m.private_ip}|{m.private_mac}" for m in plan.local_members)))}
if local_neighbor_saved=$(vc_local_neighbor_expected "$STATE") &&
local_neighbor_actual=$(vc_local_neighbor_records "$BRIDGE"); then
local_neighbor_wanted=$(printf '%s\\n' "$local_neighbor_expected" | awk 'NF {{print $0 "|PERMANENT"}}')
if [ "$local_neighbor_saved" = "$local_neighbor_expected" ] &&
[ "$local_neighbor_actual" = "$local_neighbor_wanted" ]; then local_neighbor_state=owned; fi
fi
printf '%s\\n' "__VC_VPC_LOCAL_NEIGHBOR_STATE__=$local_neighbor_state"
# vc-local-neighbor-observe-end
# vc-apply-receipt-observe-begin
helper_contract=missing
if vc_vpc_neighbor_helper_ready; then helper_contract=owned; fi
printf '%s\\n' "__VC_VPC_HELPER_CONTRACT__=$helper_contract"
bridge_admin=missing
if [ "$bridge_shape" = yes ]; then
bridge_admin=$(vc_vpc_bridge_admin_state) || exit 74
fi
printf '%s\\n' "__VC_VPC_BRIDGE_ADMIN__=$bridge_admin"
nft_file_exact=no
if [ "$nft_file_ok" = yes ] && [ -n "$SHA256SUM" ]; then
nft_file_hash=$(vc_vpc_sha256 "$NFT_FILE") || exit 74
if [ "$nft_file_hash" = {_q(hashlib.sha256((render_vpc_nft_rules(plan).rstrip() + chr(10)).encode()).hexdigest())} ]; then nft_file_exact=yes; fi
fi
printf '%s\\n' "__VC_VPC_NFT_FILE_EXACT__=$nft_file_exact"
state_file_exact=no
if [ -n "$SHA256SUM" ] && [ -f "$STATE" ] && [ ! -L "$STATE" ] &&
[ "$("$STAT" -c '%u:%g:%a' "$STATE" 2>/dev/null)" = 0:0:600 ]; then
state_file_hash=$(vc_vpc_sha256 "$STATE") || exit 74
if [ "$state_file_hash" = {_q(expected_state_hash)} ]; then state_file_exact=yes; fi
fi
printf '%s\\n' "__VC_VPC_STATE_FILE_EXACT__=$state_file_exact"
persistence_exact=no
if [ "$nm_owned" = yes ] && [ "$ifup_owned" = no ]; then
persistence_exact=yes
elif [ "$ifup_owned" = yes ] && [ "$nm_present" = no ] && [ -n "$SHA256SUM" ]; then
ifup_file_hash=$(vc_vpc_sha256 "$IF_FRAGMENT") || exit 74
if [ "$ifup_file_hash" = {_q(expected_ifup_hash)} ]; then persistence_exact=yes; fi
fi
printf '%s\\n' "__VC_VPC_PERSISTENCE_EXACT__=$persistence_exact"
vc_observe_nft_receipt=$(vc_vpc_nft_receipt) || exit 74
printf '%s\\n' "__VC_VPC_NFT_RECEIPT__=$vc_observe_nft_receipt"
# vc-apply-receipt-observe-end
nft_table_count=0
for nft_family in inet bridge arp; do
if "$NFT" list table "$nft_family" "$TABLE" >/dev/null 2>&1; then
nft_table_count=$((nft_table_count + 1))
fi
done
printf '%s\\n' "__VC_VPC_NFT_TABLE_COUNT__=$nft_table_count"
if "$NFT" list table inet "$TABLE" >/dev/null 2>&1; then printf '%s\\n' '__VC_VPC_NFT_TABLE__='"$TABLE"; else printf '%s\\n' '__VC_VPC_NFT_TABLE__=missing'; fi
if [ -f "$STATE" ] && grep -Fqx 'owner|{DATAPLANE_OWNER_MARKER}' "$STATE"; then
printf '%s\\n' '__VC_VPC_STATE_OWNER__=owned'
printf '%s\\n' "__VC_VPC_STATE_REVISION__=$(awk -F'|' '$1==\"revision\"{{print $2; exit}}' \"$STATE\")"
printf '%s\\n' "__VC_VPC_STATE_DIGEST__=$(awk -F'|' '$1==\"digest\"{{print $2; exit}}' \"$STATE\")"
state_firewall_record=$(awk -F'|' '$1=="firewall_mode" {{count++; value=$2}} END {{print (count + 0) "|" value}}' "$STATE")
state_firewall_observed=invalid
case "$state_firewall_record" in
'0|') state_firewall_observed= ;;
'1|native_nftables') state_firewall_observed=native_nftables ;;
esac
printf '%s\\n' "__VC_VPC_STATE_FIREWALL_MODE__=$state_firewall_observed"
else
printf '%s\\n' '__VC_VPC_STATE_OWNER__=missing'
fi
nft_text=$($NFT -y list table inet "$TABLE" 2>/dev/null || true)
nft_verified=yes
printf '%s\\n' "$nft_text" | grep -Fq "table inet $TABLE" || nft_verified=no
printf '%s\\n' "$nft_text" | grep -Fq "chain forward" || nft_verified=no
printf '%s\\n' "$nft_text" | grep -Fq "hook forward priority -190" || nft_verified=no
printf '%s\\n' "$nft_text" | grep -Fq "chain input" || nft_verified=no
printf '%s\\n' "$nft_text" | grep -Fq "chain output" || nft_verified=no
printf '%s\\n' "$nft_text" | grep -Fq "set member_ips" || nft_verified=no
printf '%s\\n' "$nft_text" | grep -Fq "set vpc_dest_ips" || nft_verified=no
{nft_checks}
if [ "$nft_verified" = yes ]; then
printf '%s\\n' '__VC_VPC_ACL_REVISION__={plan.desired_revision}'
printf '%s\\n' '__VC_VPC_ACL_DIGEST__={plan.desired_digest}'
printf '%s\\n' '__VC_VPC_ANTI_SPOOF__=yes'
else
printf '%s\\n' '__VC_VPC_ACL_REVISION__=missing'
printf '%s\\n' '__VC_VPC_ACL_DIGEST__=missing'
printf '%s\\n' '__VC_VPC_ANTI_SPOOF__=no'
fi
{bridge_observe}
{arp_observe}
if route_inventory=$("$IP" -o -4 -N route show table "$ROUTE_TABLE" 2>/dev/null) &&
route_tuples=$(printf '%s\\n' "$route_inventory" | vc_vpc_route_records "" "$ROUTE_TABLE" "$BRIDGE"); then
route_records=$(printf '%s\\n' "$route_tuples" | awk -F'|' 'NF {{print $1 "|" $2 "|" $4 "|" $5 "|" $6 "|" $7}}')
else
route_records=invalid
fi
route_scope_count=$(printf '%s\\n' "$route_records" | awk 'NF {{count++}} END {{print count + 0}}')
printf '%s\\n' "__VC_VPC_ROUTE_SCOPE_COUNT__=$route_scope_count"
removal_route_absent=no
if [ "$route_scope_count" = 0 ]; then removal_route_absent=yes; fi
printf '%s\\n' "__VC_VPC_REMOVAL_ROUTE_ABSENT__=$removal_route_absent"
rule_inventory_valid=yes
rule_records=$("$IP" -o -4 -N rule show 2>/dev/null) || {{
rule_inventory_valid=no
rule_records=
}}
rule_scope_count=$(printf '%s\\n' "$rule_records" | awk -v expected_to={_q(plan.vpc.cidr)} -v expected_table={_q(str(r.route_table))} '{{
destination=""; table=""
for (i=1;i<=NF;i++) {{
if ($i == "to" && i < NF) destination=$(i+1)
if ($i == "lookup" && i < NF) table=$(i+1)
}}
if (destination == expected_to || table == expected_table) count++
}} END {{print count + 0}}')
printf '%s\\n' "__VC_VPC_RULE_SCOPE_COUNT__=$rule_scope_count"
removal_rule_absent=no
if [ "$rule_inventory_valid" = yes ] && [ "$rule_scope_count" = 0 ]; then
removal_rule_absent=yes
fi
printf '%s\\n' "__VC_VPC_REMOVAL_RULE_ABSENT__=$removal_rule_absent"
rule_extra=yes; [ "$rule_scope_count" = 2 ] && rule_extra=no
printf '%s\\n' "__VC_VPC_RULE_EXTRA__=$rule_extra"
{rule_checks}
route_extra=$(printf '%s\\n' "$route_records" | awk -F'|' '{{
if (length($0) == 0) next
count++
{route_extra_awk}
extra=1
}} END {{ if (count != {len(routes)}) extra=1; print extra ? "yes" : "no" }}')
printf '%s\\n' "__VC_VPC_ROUTE_EXTRA__=$route_extra"
{route_checks}
vpc_state_loaded=no
removal_state_closure=invalid
removal_target_state_absent_before=no
if [ ! -e "$STATE" ] && [ ! -L "$STATE" ]; then
removal_target_state_absent_before=yes
fi
if vpc_state_load; then
if [ "$VPC_STATE_STATUS" = owned ]; then
vpc_state_loaded=yes
fi
case "$VPC_STATE_STATUS" in
owned|empty)
if [ "$removal_target_state_absent_before" = yes ] &&
[ ! -e "$STATE" ] && [ ! -L "$STATE" ]; then
removal_state_closure=$VPC_STATE_STATUS
fi
;;
esac
fi
printf '%s\n' "__VC_VPC_REMOVAL_STATE_CLOSURE__=$removal_state_closure"
while IFS='|' read -r peer_host key expected_overlay; do
[ -n "$peer_host" ] || continue
actual=$(vc_wg_allowed_for_peer "$WG_IFACE" "$key") || actual=invalid
expected_allowed=$expected_overlay
state=missing
if [ "$vpc_state_loaded" = yes ]; then
state_allowed=$(vpc_state_allowed_for_peer "$key") || state_allowed=invalid
if [ "$state_allowed" != invalid ] && [ -n "$state_allowed" ]; then
expected_allowed="$expected_allowed,$state_allowed"
fi
fi
# ``wg show`` does not promise AllowedIPs order. Canonicalize the live set
# and the exact overlay + route-backed union derived by the shared state
# loader. The Python adapter separately proves this VPC's required routes
# are a subset before projecting its own tenant /32s.
expected_set=$(printf '%s\\n' "$expected_allowed" | tr ',' '\\n' | awk 'NF' | LC_ALL=C sort -u)
actual_set=$(printf '%s\\n' "$actual" | tr ',' '\\n' | awk 'NF' | LC_ALL=C sort -u)
allowed=$(printf '%s\\n' "$actual_set" | awk 'BEGIN {{ ORS=""; sep="" }} NF {{ printf "%s%s", sep, $0; sep="," }}')
if [ "$vpc_state_loaded" = yes ] && [ "$state_allowed" != invalid ] &&
[ -n "$actual_set" ] && [ "$actual_set" = "$expected_set" ]; then
state=owned
else
allowed=missing
fi
printf '%s\\n' "__VC_VPC_WG_ALLOWED__=$peer_host|$key|$allowed"
printf '%s\\n' "__VC_VPC_WG_STATE__=$peer_host|$state"
done <<'{peer_marker}'
{peer_lines}
{peer_marker}
expected_fabric_peer_keys={fabric_peer_key_arg}
wg_peer_keys=unverified
if [ "{fabric_peer_keys_provided}" = yes ]; then
wg_peer_keys=exact
if live_fabric_peer_keys=$("$WG" show "$WG_IFACE" peers 2>/dev/null); then
:
else
live_fabric_peer_keys=
wg_peer_keys=missing
fi
for live_peer_key in $live_fabric_peer_keys; do
case " $expected_fabric_peer_keys " in
*" $live_peer_key "*) : ;;
*) wg_peer_keys=extra ;;
esac
done
for expected_peer_key in $expected_fabric_peer_keys; do
printf '%s\n' "$live_fabric_peer_keys" | grep -Fqx "$expected_peer_key" || wg_peer_keys=missing
done
# Key presence alone cannot prove the shared cryptokey-routing boundary.
# A VPC revision may have withdrawn the final route for a peer that remains
# in the Fabric. Require every authoritative key to carry exactly one live
# overlay /32 plus only the route-backed tenant union from durable host
# state, including peers outside this VPC's current participant set.
wg_peer_closure=$wg_peer_keys
if [ "$wg_peer_keys" = exact ] &&
{{ [ "$vpc_state_loaded" = yes ] ||
[ "$removal_state_closure" != invalid ]; }}; then
wg_peer_closure=exact
while IFS='|' read -r expected_peer_key expected_overlay; do
[ -n "$expected_peer_key" ] || continue
closure_actual=$(vc_wg_allowed_for_peer "$WG_IFACE" "$expected_peer_key") || {{
wg_peer_closure=missing
break
}}
vpc_state_overlay_from_allowed "$closure_actual" >/dev/null || {{
wg_peer_closure=missing
break
}}
closure_extra=$(vpc_state_allowed_for_peer "$expected_peer_key") || {{
wg_peer_closure=missing
break
}}
closure_expected="$expected_overlay"
[ -z "$closure_extra" ] || closure_expected="$closure_expected,$closure_extra"
closure_actual_set=$(printf '%s\n' "$closure_actual" | tr ',' '\n' | awk 'NF' | LC_ALL=C sort -u)
closure_expected_set=$(printf '%s\n' "$closure_expected" | tr ',' '\n' | awk 'NF' | LC_ALL=C sort -u)
if [ -z "$closure_actual_set" ] ||
[ "$closure_actual_set" != "$closure_expected_set" ]; then
wg_peer_closure=missing
break
fi
done <<'{fabric_peer_marker}'
{fabric_peer_overlay_lines}
{fabric_peer_marker}
fi
else
wg_peer_closure=unverified
fi
wg_peer_set=$wg_peer_keys
if [ "$wg_peer_keys" = exact ]; then
if [ "$vpc_state_loaded" = yes ] && [ "$wg_peer_closure" = exact ]; then
wg_peer_set=exact
else
wg_peer_set=missing
fi
fi
removal_wg_peer_set=$wg_peer_keys
if [ "$wg_peer_keys" = exact ]; then
if [ "$removal_state_closure" != invalid ] &&
[ "$wg_peer_closure" = exact ]; then
removal_wg_peer_set=exact
else
removal_wg_peer_set=missing
fi
fi
printf '%s\n' "__VC_VPC_WG_PEER_SET__=$wg_peer_set"
printf '%s\n' "__VC_VPC_REMOVAL_WG_PEER_SET__=$removal_wg_peer_set"
fabric_nft_closure=no
if {{ [ "$vpc_state_loaded" = yes ] ||
[ "$removal_state_closure" != invalid ]; }} &&
vc_vpc_fabric_closure_ready; then
fabric_nft_closure=yes
fi
fabric_nft_state=no
if [ "$VPC_STATE_STATUS" = owned ] && [ "$fabric_nft_closure" = yes ]; then
fabric_nft_state=yes
fi
printf '%s\n' "__VC_VPC_FABRIC_NFT__=$fabric_nft_state"
removal_fabric_nft_state=no
if [ "$removal_state_closure" != invalid ] &&
[ "$fabric_nft_closure" = yes ]; then
removal_fabric_nft_state=yes
fi
printf '%s\n' "__VC_VPC_REMOVAL_FABRIC_NFT__=$removal_fabric_nft_state"
removal_nft_absent=no
if vc_vpc_target_nft_absent; then removal_nft_absent=yes; fi
printf '%s\n' "__VC_VPC_REMOVAL_NFT_ABSENT__=$removal_nft_absent"
exit 0
'''
def _markers(output: str) -> dict[str, Any]:
if not isinstance(output, str) or len(output) > MAX_OUTPUT:
raise VpcDataplaneError("VPC 回读输出过大", 502, "vpc_readback_invalid")
result: dict[str, Any] = {"routes": [], "wg_allowed_ips": [],
"wg_states": {}, "rules": [], "proxy_ips": []}
for line in output.splitlines():
if not line.startswith("__VC_VPC_") or "=" not in line:
continue
key, value = line.split("=", 1)
# A peer can legitimately carry many exact /32s from multiple VPCs;
# keep a bounded upper limit without truncating the authenticated set.
if len(value) > 65536:
raise VpcDataplaneError("VPC 回读字段过长", 502, "vpc_readback_invalid")
if key == "__VC_VPC_ROUTE__":
p = value.split("|")
if len(p) in {6, 7}:
result["routes"].append({
"destination_cidr": p[0], "device": p[1], "table": p[2],
"via": None if p[3] == "-" else p[3],
"metric": p[4], "owner": DATAPLANE_OWNER_MARKER,
"state": p[-1],
# Six-field receipts predate the carrier observation.
# Current scripts always retain the explicit flag field.
"flags": ([] if len(p) == 6 or p[5] == "-"
else [p[5]]),
})
elif key == "__VC_VPC_PROXY_IP__":
if re.fullmatch(r"[0-9]{1,3}(?:\.[0-9]{1,3}){3}", value):
try:
address = ipaddress.IPv4Address(value)
except ipaddress.AddressValueError:
continue
result["proxy_ips"].append(str(address))
elif key == "__VC_VPC_RULE__":
p = value.split("|")
if len(p) == 5:
result["rules"].append({"priority": p[0], "iif": p[1],
"destination_cidr": p[2],
"table": p[3], "state": p[4]})
elif key == "__VC_VPC_WG_ALLOWED__":
p = value.split("|")
if len(p) == 3:
result["wg_allowed_ips"].append({"peer_host_id": p[0], "public_key": p[1], "tenant_allowed_ips": [] if p[2] == "missing" else [x for x in p[2].split(",") if x]})
elif key == "__VC_VPC_WG_STATE__":
p = value.split("|")
if len(p) == 2 and p[1] in {"owned", "missing"}:
result["wg_states"][p[0]] = p[1]
else:
result[key.removeprefix("__VC_VPC_").removesuffix("__").lower()] = value
return result
def patch_private_interface_xml(xml: str | bytes, *, member: VpcMember,
resources: VpcDataplaneResources,
guest_network_generation: str,
link_state: str = "up",
vpc_id: str | None = None,
desired_revision: int | None = None,
desired_digest: str | None = None) -> str:
"""Add/validate a private0 NIC on an explicit panel-owned bridge."""
if link_state not in {"up", "down"}:
raise VpcDataplaneError("私网网卡 link 状态无效")
if (not isinstance(guest_network_generation, str)
or _DIGEST.fullmatch(guest_network_generation) is None):
raise VpcDataplaneError("来宾私网 generation 无效")
private_mac = _mac(member.private_mac)
try:
root = ET.fromstring(xml)
except ET.ParseError as exc:
raise VpcDataplaneError("虚拟机 XML 无效", 409, "vpc_private_xml_invalid") from exc
devices = root.find("./devices")
if devices is None:
raise VpcDataplaneError("虚拟机 XML 缺少 devices", 409, "vpc_private_xml_invalid")
interfaces = list(devices.findall("interface"))
try:
public = select_public_interface(interfaces, allow_legacy=True)
# Legacy domains may have a single public NIC without an alias. Give
# it the canonical libvirt alias before adding private0 so the next
# reconciliation does not mistake the public NIC for an unknown role.
set_libvirt_interface_alias(root, public, PUBLIC_LIBVIRT_ALIAS)
private = select_private_interface(interfaces)
except ValueError as exc:
raise VpcDataplaneError(str(exc), 409, "vpc_private_xml_role_invalid") from exc
if private is None:
private = ET.SubElement(devices, "interface", {"type": "bridge"})
else:
if private.get("type") != "bridge":
raise VpcDataplaneError("private0 必须使用 bridge", 409, "vpc_private_xml_source_invalid")
existing_filters = private.findall("filterref")
if existing_filters and any(
item.get("filter") != "clean-traffic"
for item in existing_filters):
raise VpcDataplaneError("private0 nwfilter 不受面板管理", 409,
"vpc_private_xml_filter_invalid")
source = private.find("source")
if source is None or source.get("network") is not None or source.get("bridge") != resources.bridge:
raise VpcDataplaneError("private0 bridge 不匹配", 409, "vpc_private_xml_source_mismatch")
source = private.find("source")
if source is None:
source = ET.Element("source")
private.insert(0, source)
source.attrib.clear(); source.set("bridge", resources.bridge)
mac = private.find("mac")
if mac is None:
mac = ET.Element("mac"); private.insert(0, mac)
mac.set("address", private_mac)
try:
set_libvirt_interface_alias(root, private, PRIVATE_LIBVIRT_ALIAS)
except ValueError as exc:
raise VpcDataplaneError(
str(exc), 409, "vpc_private_xml_role_invalid"
) from exc
model = private.find("model")
if model is None:
model = ET.SubElement(private, "model")
model.set("type", "virtio")
link = private.find("link")
if link is None:
link = ET.SubElement(private, "link")
link.set("state", link_state)
for item in list(private.findall("filterref")):
private.remove(item)
# The private L3 data plane owns anti-spoofing and ACL enforcement in its
# dedicated bridge/inet/arp nft tables. Do not attach the public
# libvirt ``clean-traffic`` nwfilter to private0; the role observer
# deliberately treats any private filter as an unsafe mixed policy.
metadata = root.find("metadata")
if metadata is None:
metadata = ET.Element("metadata"); root.insert(0, metadata)
managed = metadata.find(f"{{{PANEL_NS}}}managed")
if managed is None:
managed = ET.SubElement(metadata, f"{{{PANEL_NS}}}managed")
managed.set("private-member-id", member.member_id)
managed.set("private-ip-cidr", f"{member.private_ip}/32")
managed.set("private-network-bridge", resources.bridge)
managed.set("private-network-verified", "true")
managed.set("private-dataplane-owner", DATAPLANE_OWNER_MARKER)
managed.set("private-guest-network-generation",
guest_network_generation)
if vpc_id is not None:
managed.set("private-vpc-id", vpc_id)
if desired_revision is not None:
managed.set("private-network-revision", str(desired_revision))
if desired_digest is not None:
managed.set("private-network-digest", desired_digest)
ET.register_namespace("panel", PANEL_NS)
return ET.tostring(root, encoding="unicode")
def _member_nics(
plan: VpcDataplanePlan,
xml_by_instance: Mapping[str, Any] | None,
) -> tuple[dict[str, Any], ...]:
def xml_details(
xml: str,
member: VpcMember,
expected_generation: str,
) -> tuple[bool, bool, bool, str | None]:
try:
root = ET.fromstring(xml)
private = select_private_interface(
list(root.findall("./devices/interface")))
source = private.find("source") if private is not None else None
mac = private.find("mac") if private is not None else None
ok = bool(
private and private.get("type") == "bridge"
and source is not None
and source.get("bridge") == plan.resources.bridge
and mac is not None
and mac.get("address", "").lower() == member.private_mac
# VPC isolation is provided by the dedicated bridge/inet/arp
# nft context. The public libvirt nwfilter is intentionally
# absent from private0.
and not private.findall("filterref")
)
link = private.find("link") if private is not None else None
link_state = link.get("state", "up").lower() if link is not None else "up"
if link_state not in {"up", "down"}:
return False, False, False, None
managed = root.find(f"./metadata/{{{PANEL_NS}}}managed")
guest_generation_ok = bool(
managed is not None
and managed.get("private-vpc-id") == plan.vpc.vpc_id
and managed.get("private-member-id") == member.member_id
and managed.get("private-ip-cidr") == f"{member.private_ip}/32"
and managed.get("private-network-bridge")
== plan.resources.bridge
and managed.get("private-dataplane-owner")
== DATAPLANE_OWNER_MARKER
and managed.get("private-guest-network-generation")
== expected_generation
)
desired_metadata_ok = bool(
guest_generation_ok
and managed.get("private-network-revision")
== str(plan.desired_revision)
and managed.get("private-network-digest")
== plan.desired_digest
)
return ok, guest_generation_ok, desired_metadata_ok, link_state
except (ET.ParseError, ValueError):
return False, False, False, None
result = []
for member in plan.local_members:
verified = False
config_verified = False
seed_verified = False
restart_required = False
guest_verified = False
guest_readback = "missing"
guest_reason_code = "vpc_private_guest_agent_unavailable"
guest_private_mac: str | None = None
guest_private_ip_cidr: str | None = None
guest_link_state: str | None = None
guest_routes: list[str] = []
domain_running = False
link_state: str | None = None
expected_routes = private_guest_routes(plan.vpc, member)
expected_generation = private_guest_network_generation(
member, expected_routes)
if xml_by_instance and member.instance_uuid in xml_by_instance:
value = xml_by_instance[member.instance_uuid]
if isinstance(value, Mapping):
seed_verified = value.get("seed_verified") is True
inactive = value.get("inactive")
live = value.get("live")
inactive_ok = False
live_device_ok = False
live_generation_ok = False
inactive_link: str | None = None
live_link: str | None = None
if isinstance(inactive, str):
(inactive_device_ok, inactive_generation_ok,
inactive_metadata_ok, inactive_link) = xml_details(
inactive, member, expected_generation)
inactive_ok = bool(
inactive_device_ok and inactive_generation_ok
and inactive_metadata_ok)
config_verified = inactive_ok
domain_running = live is not None
if domain_running:
if isinstance(live, str):
(live_device_ok, live_generation_ok,
_live_metadata_ok, live_link) = xml_details(
live, member, expected_generation)
guest = value.get("guest_network")
if isinstance(guest, Mapping):
guest_readback = str(guest.get("state", "missing"))[:32]
reason = guest.get("reason_code")
if isinstance(reason, str) and re.fullmatch(
r"[a-z0-9_]{1,64}", reason):
guest_reason_code = reason
observed_routes = guest.get("routes")
if (isinstance(observed_routes, list)
and len(observed_routes) <= 4096
and all(isinstance(route, str)
for route in observed_routes)):
guest_routes = sorted(set(observed_routes))
observed_mac = guest.get("private_mac")
observed_ip = guest.get("private_ip_cidr")
guest_private_mac = (
observed_mac.lower()
if isinstance(observed_mac, str) else None)
guest_private_ip_cidr = (
observed_ip if isinstance(observed_ip, str) else None)
guest_link_state = (
guest.get("link_state")
if guest.get("link_state") in {"up", "down"}
else None)
guest_verified = bool(
guest.get("verified") is True
and guest_readback == "confirmed"
and guest.get("source") == "qemu_guest_agent"
and guest_link_state == "up"
and guest_private_mac == member.private_mac
and guest_private_ip_cidr == f"{member.private_ip}/32"
and guest_routes == list(expected_routes)
)
if guest_verified:
guest_reason_code = None
# Persistent and live libvirt device identity remains a
# prerequisite, but only a bounded guest-agent observation can
# prove that the address and routes exist inside the guest.
stopped_exact = bool(
inactive_ok and seed_verified and not domain_running)
verified = bool(
stopped_exact
or (inactive_ok and seed_verified and domain_running
and live_device_ok
and inactive_link == live_link))
restart_required = bool(
not stopped_exact and (
not domain_running
or not inactive_ok
or not seed_verified
or not live_generation_ok))
if stopped_exact:
guest_readback = "stopped"
guest_reason_code = "vpc_private_guest_stopped"
if restart_required:
guest_reason_code = "vpc_private_guest_restart_required"
link_state = live_link if domain_running else inactive_link
elif isinstance(value, str):
(device_ok, generation_ok, metadata_ok,
link_state) = xml_details(
value, member, expected_generation)
verified = device_ok and generation_ok and metadata_ok
config_verified = verified
result.append({"member_id": member.member_id, "instance_uuid": member.instance_uuid,
"interface": PRIVATE_INTERFACE, "private_ip_cidr": f"{member.private_ip}/32",
"private_mac": member.private_mac, "xml_verified": verified,
"config_verified": config_verified,
"seed_verified": seed_verified,
"restart_required": restart_required,
"domain_running": domain_running,
"guest_network_verified": guest_verified,
"guest_network_readback": guest_readback,
"guest_reason_code": guest_reason_code,
"guest_private_mac": guest_private_mac,
"guest_private_ip_cidr": guest_private_ip_cidr,
"guest_link_state": guest_link_state,
"guest_routes": guest_routes,
"link_state": link_state})
return tuple(result)
class VpcPrivateHostAdapter:
"""Execute generated scripts through a pinned remote host manager."""
def __init__(
self,
manager: Any,
profile: Mapping[str, Any] | None = None,
*,
request_deadline: float | None = None,
):
self.manager = manager
self.profile = dict(profile) if isinstance(profile, Mapping) else None
self.request_deadline = request_deadline
def _run(self, script: str) -> str:
current = self.profile if self.profile is not None else self.manager.current()
if not isinstance(current, Mapping) or not current:
raise VpcDataplaneError("宿主机尚未配置", 409, "vpc_host_not_configured")
if current.get("key_locked"):
raise VpcDataplaneError("宿主机 SSH 私钥仍锁定", 409, "vpc_host_key_locked")
timeout = 35
if self.request_deadline is not None:
remaining = self.request_deadline - time.monotonic()
if remaining < 1:
raise VpcDataplaneError(
"VPC 来宾网络核验超过请求预算", 502,
"vpc_private_guest_probe_deadline_exceeded")
timeout = min(timeout, int(remaining))
argv = [getattr(self.manager, "ssh_binary", "ssh"), "-F", current["ssh_config_file"], "-T", "-o", "ConnectTimeout=10", current["ssh_target"], "/bin/sh -s"]
try:
result = self.manager._exec(
argv, timeout=timeout, input_text=script)
except Exception as exc:
raise VpcDataplaneError("宿主机 VPC SSH 请求失败", 502, "vpc_host_connection_failed") from exc
if result.returncode != 0:
raise VpcDataplaneError("宿主机 VPC 命令失败", 502, "vpc_remote_command_failed")
return result.stdout
def _run_mutation(self, script: str) -> str:
"""Dispatch one host write and retain ambiguity on any lost receipt."""
# Keep _run's long-standing override contract for host/test managers;
# the mutation boundary is an adapter concern, not a transport option.
mark_remote_write_started()
try:
return self._run(script)
except BaseException:
mark_execution_unknown()
raise
def _deployment_contract(self) -> tuple[str, str, str, str, str]:
"""Validate the same immutable host capability gate as Fabric."""
from .fabric import (
FabricError, _profile_ifdown_path, _profile_ifquery_path,
_profile_ifup_path, _profile_network_mode,
_profile_firewall_mode,
)
current = self.profile if self.profile is not None else self.manager.current()
try:
mode = _profile_network_mode(current)
except FabricError as exc:
raise VpcDataplaneError(
"宿主机网络管理器与 VPC 部署模式不匹配", 409,
"vpc_dataplane_network_manager_conflict") from exc
try:
firewall_mode = _profile_firewall_mode(current)
except FabricError as exc:
raise VpcDataplaneError(
"宿主机防火墙与 VPC 部署模式不匹配", 409,
"vpc_dataplane_firewall_mode_conflict") from exc
capabilities = current.get("capabilities") if isinstance(current, Mapping) else None
network = capabilities.get("network") if isinstance(capabilities, Mapping) else None
wireguard = capabilities.get("wireguard") if isinstance(capabilities, Mapping) else None
nftables = capabilities.get("nftables") if isinstance(capabilities, Mapping) else None
if (not isinstance(network, Mapping)
or not isinstance(wireguard, Mapping)
or not isinstance(nftables, Mapping)
or capabilities.get("vpc_ready") is not True
or network.get("firewall_mode") != firewall_mode
or network.get("ip_forward") != 1
or nftables.get("ready") is not True
or wireguard.get("interface") != FABRIC_INTERFACE
or wireguard.get("interface_present") is not True
or wireguard.get("interface_is_wireguard") is not True
or wireguard.get("owned_by_panel") is not True):
raise VpcDataplaneError(
"宿主机 VPC 网络或防火墙能力未通过探测", 409,
"vpc_host_capability_missing")
return (
mode,
_profile_ifup_path(current),
_profile_ifdown_path(current),
_profile_ifquery_path(current),
firewall_mode,
)
def _render_contract(self) -> tuple[str, str, str, str, str]:
"""Normalize legacy test doubles while production uses five fields."""
contract = self._deployment_contract()
if len(contract) == 4:
mode, ifup_path, ifdown_path, ifquery_path = contract
return (mode, ifup_path, ifdown_path, ifquery_path,
FIREWALL_MODE_NATIVE)
mode, ifup_path, ifdown_path, ifquery_path, firewall_mode = contract
return (mode, ifup_path, ifdown_path, ifquery_path,
_firewall_mode(firewall_mode))
def _receipt_scope(self, plan: VpcDataplanePlan) -> tuple[str, ...] | None:
current = self.profile
if current is None and callable(getattr(self.manager, "current", None)):
current = self.manager.current()
if not isinstance(current, Mapping) or any(
not isinstance(current.get(key), str) or not current[key]
for key in ("ssh_target", "ssh_config_file", "host", "fingerprint")):
return None
if type(current.get("port")) is not int or not 1 <= current["port"] <= 65535:
return None
return (plan.host_id, plan.vpc.vpc_id,
str(current.get("ssh_target", "")),
str(current.get("ssh_config_file", "")),
str(current.get("host", "")), str(current.get("port", "")),
str(current.get("fingerprint", "")))
@staticmethod
def _save_receipt(scope: tuple[str, ...] | None, fingerprint: str, receipt: Any) -> None:
if scope is None or not isinstance(receipt, str) or not re.fullmatch(
r"[0-9a-f]{64}(?:,[0-9a-f]{64}){2}", receipt):
return
with _APPLY_RECEIPT_LOCK:
_APPLY_RECEIPTS[scope] = (fingerprint, receipt)
_APPLY_RECEIPTS.move_to_end(scope)
while len(_APPLY_RECEIPTS) > _APPLY_RECEIPT_LIMIT:
_APPLY_RECEIPTS.popitem(last=False)
def apply(self, plan: VpcDataplanePlan) -> dict[str, Any]:
mode, ifup_path, ifdown_path, ifquery_path, firewall_mode = self._render_contract()
script = render_apply_script(
plan, deployment_mode=mode, firewall_mode=firewall_mode,
ifup_path=ifup_path, ifdown_path=ifdown_path,
ifquery_path=ifquery_path,
)
scope = self._receipt_scope(plan)
fingerprint = hashlib.sha256(script.encode("utf-8")).hexdigest()
with _APPLY_RECEIPT_LOCK:
previous = _APPLY_RECEIPTS.pop(scope, None)
if plan.local_members and previous is not None and previous[0] == fingerprint:
# Exceptions propagate; a failed read must never trigger writes.
# Pop first so timeouts/malformed receipts cannot keep stale trust.
_observed, host_ready, receipt = self._observe(plan)
if host_ready and receipt == previous[1]:
self._save_receipt(scope, fingerprint, receipt)
return {"state": "applying", "readback": "pending", "rollback": "none", "owner": DATAPLANE_OWNER_MARKER}
output = self._run_mutation(script)
try:
# The host write has already completed. Parsing and strict
# receipt validation are part of the same uncertainty boundary:
# any exception here means the response cannot authorize replay.
markers = _markers(output)
receipt_fields = {
"applied": "yes", "owner": DATAPLANE_OWNER_MARKER,
"revision": str(plan.desired_revision),
"digest": plan.desired_digest,
}
receipt_keys = (*receipt_fields, "nft_receipt")
receipt_unique = all(sum(
line.startswith(f"__VC_VPC_{key.upper()}__=")
for line in output.splitlines()) == 1 for key in receipt_keys)
receipt = markers.get("nft_receipt")
receipt_valid = bool(isinstance(receipt, str) and re.fullmatch(
r"[0-9a-f]{64}(?:,[0-9a-f]{64}){2}", receipt))
if (not receipt_unique or not receipt_valid
or any(markers.get(key) != value
for key, value in receipt_fields.items())):
raise VpcDataplaneError(
"宿主机未确认 VPC 应用", 502, "vpc_apply_unconfirmed")
except BaseException:
mark_execution_unknown()
raise
if plan.local_members:
self._save_receipt(scope, fingerprint, receipt)
return {"state": "applying", "readback": "pending", "rollback": "none", "owner": DATAPLANE_OWNER_MARKER}
def remove(self, plan: VpcDataplanePlan) -> dict[str, Any]:
"""Remove a panel-owned empty-host VPC and prove the absence.
The removal script is deliberately separate from ``apply``: it may
only run when the parsed plan has no local members and it validates
every persisted artifact/bridge owner before deleting anything.
A positive marker is required before the caller can advance a host
tombstone.
"""
mode, ifup_path, ifdown_path, ifquery_path, firewall_mode = self._render_contract()
scope = self._receipt_scope(plan)
with _APPLY_RECEIPT_LOCK:
_APPLY_RECEIPTS.pop(scope, None)
output = self._run_mutation(render_remove_script(
plan, deployment_mode=mode, firewall_mode=firewall_mode,
ifup_path=ifup_path,
ifdown_path=ifdown_path, ifquery_path=ifquery_path))
try:
# A malformed removal response is just as ambiguous as a lost
# response. Keep the durable claim until an absence observer can
# prove this exact revision instead of allowing a second delete.
markers = _markers(output)
receipt_fields = {
"removed": "yes", "owner": DATAPLANE_OWNER_MARKER,
"revision": str(plan.desired_revision),
"digest": plan.desired_digest, "rollback": "none",
}
receipt_unique = all(sum(
line.startswith(f"__VC_VPC_{key.upper()}__=")
for line in output.splitlines()) == 1 for key in receipt_fields)
if (not receipt_unique
or any(markers.get(key) != value
for key, value in receipt_fields.items())):
raise VpcDataplaneError(
"宿主机未确认 VPC 清理", 502,
"vpc_remove_unconfirmed")
except BaseException:
mark_execution_unknown()
raise
return {
"vpc_id": plan.vpc.vpc_id,
"host_id": plan.host_id,
"desired_revision": plan.desired_revision,
"observed_revision": plan.desired_revision,
"desired_digest": plan.desired_digest,
"observed_digest": plan.desired_digest,
"config_verified": True,
"config_observed_revision": plan.desired_revision,
"config_observed_digest": plan.desired_digest,
"state": "active",
"removed": True,
"bridge": None,
"bridge_owner": None,
"member_nics": [],
"routes": [],
"wg_allowed_ips": [],
"nft_table": None,
"acl_revision": plan.desired_revision,
"anti_spoof": False,
"rp_filter": None,
"proxy_arp": None,
"proxy_arp_pvlan": None,
"proxy_ips": [],
"readback": "confirmed",
"rollback": markers.get("rollback", "none"),
"reason_code": None,
}
def observe(
self, plan: VpcDataplanePlan, *,
xml_by_instance: Mapping[str, Any] | None = None,
) -> dict[str, Any]:
observed, _host_ready, _receipt = self._observe(
plan, xml_by_instance=xml_by_instance)
return observed
def observe_removal(self, plan: VpcDataplanePlan) -> dict[str, Any]:
"""Read back exact host absence without running the removal script."""
if plan.local_members:
raise VpcDataplaneError(
"仍有本机私网成员,不能核对宿主机 VPC 清理", 409,
"vpc_remove_has_local_members")
mode, _ifup, _ifdown, _ifquery, firewall_mode = (
self._render_contract())
output = self._run(
render_observe_script(
plan, deployment_mode=mode, firewall_mode=firewall_mode))
parsed = _markers(output)
absence_fields = (
"nft_file_present", "sysctl_file_present",
"proxy_file_present", "if_fragment_present",
"owner_file_present", "state_file_present",
"transaction_present", "nft_include_present",
)
required_fields = {
"bridge": "missing",
"bridge_owner": "missing",
"nft_table": "missing",
"nft_table_count": "0",
"route_scope_count": "0",
"rule_scope_count": "0",
"nm_profile_count": "0",
"nm_profile_query": (
"ok" if mode == "networkmanager_nmcli"
else "not_applicable"),
"ifup_profile_count": "0",
"ifup_profile_query": "ok",
"nft_main_count": "1",
"nft_snapshot": "yes",
"firewall_mode": "native_nftables",
"firewall_state": "owned",
"removal_bridge_absent": "yes",
"removal_route_absent": "yes",
"removal_rule_absent": "yes",
"removal_nft_absent": "yes",
"removal_fabric_nft": "yes",
"removal_wg_peer_set": "exact",
}
required_fields.update({field: "no" for field in absence_fields})
def unique_scalar(key: str) -> bool:
prefix = f"__VC_VPC_{key.upper()}__="
return sum(line.startswith(prefix)
for line in output.splitlines()) == 1
removed = bool(
all(unique_scalar(key) and parsed.get(key) == expected
for key, expected in required_fields.items())
and unique_scalar("removal_state_closure")
and parsed.get("removal_state_closure") in {"owned", "empty"}
and parsed.get("proxy_ips", []) == []
)
if not removed:
raise VpcDataplaneError(
"宿主机 VPC 清理回读尚未确认", 502,
"vpc_remove_readback_mismatch")
return {
"vpc_id": plan.vpc.vpc_id,
"host_id": plan.host_id,
"desired_revision": plan.desired_revision,
"observed_revision": plan.desired_revision,
"desired_digest": plan.desired_digest,
"observed_digest": plan.desired_digest,
"config_verified": True,
"host_config_reusable": False,
"config_observed_revision": plan.desired_revision,
"config_observed_digest": plan.desired_digest,
"state": "active",
"removed": True,
"bridge": None,
"bridge_owner": None,
"member_nics": [],
"routes": [],
"wg_allowed_ips": [],
"nft_table": None,
"acl_revision": plan.desired_revision,
"anti_spoof": False,
"rp_filter": None,
"proxy_arp": None,
"proxy_arp_pvlan": None,
"proxy_ips": [],
"readback": "confirmed",
"transport_verified": True,
"private_dataplane_verified": True,
"rollback": "none",
"reason_code": None,
}
def _observe(
self, plan: VpcDataplanePlan, *,
xml_by_instance: Mapping[str, Any] | None = None,
) -> tuple[dict[str, Any], bool, Any]:
mode, _ifup, _ifdown, _ifquery, firewall_mode = self._render_contract()
output = self._run(render_observe_script(
plan, deployment_mode=mode, firewall_mode=firewall_mode))
parsed = _markers(output)
receipt_count = sum(line.startswith("__VC_VPC_NFT_RECEIPT__=")
for line in output.splitlines())
receipt = parsed.get("nft_receipt")
if (receipt_count > 1 or (receipt_count == 1
and receipt not in {"missing", "unavailable"}
and not re.fullmatch(r"[0-9a-f]{64}(?:,[0-9a-f]{64}){2}", receipt))):
raise VpcDataplaneError("VPC 回读收据无效", 502, "vpc_readback_invalid")
expected_routes = {
(x["destination_cidr"], x["device"], str(x["table"]),
x.get("via"), str(x.get("metric", 100)))
for x in _routes(plan)
}
expected_proxy_ips = {
str(ipaddress.ip_network(x["destination_cidr"], strict=True).network_address)
for x in _routes(plan)
}
actual_proxy_ips = set(parsed.get("proxy_ips", []))
actual_routes = {
(x.get("destination_cidr"), x.get("device"), str(x.get("table")),
x.get("via"), str(x.get("metric", 100)))
for x in parsed.get("routes", [])
}
wg_allowed_rows = parsed.get("wg_allowed_ips", [])
actual_wg_full = {
x.get("peer_host_id"): (x.get("public_key"),
set(x.get("tenant_allowed_ips", [])))
for x in wg_allowed_rows
}
# Durable host state owns the complete live tenant union. The shell
# proves that union is exact; here retain the plan-level obligation:
# this VPC's peer key, overlay, members and service routes must all be
# present before the result can be accepted or projected to PHP.
required_tenant = _expected_peer_tenant_ips(plan.vpc)
expected_tenant: dict[str, set[str]] = {
p.host_id: set(required_tenant.get(p.host_id, frozenset()))
for p in plan.peers
}
expected_wg = {
p.host_id: (p.public_key, expected_tenant.get(p.host_id, set()))
for p in plan.peers
}
required_wg = {
p.host_id: (p.public_key,
{p.overlay_cidr} | expected_tenant.get(p.host_id, set()))
for p in plan.peers
}
wg_required_verified = (
len(wg_allowed_rows) == len(actual_wg_full) == len(required_wg)
and set(actual_wg_full) == set(required_wg)
and all(
actual_wg_full[host][0] == key
and required_ips.issubset(actual_wg_full[host][1])
for host, (key, required_ips) in required_wg.items()
)
)
members = _member_nics(plan, xml_by_instance)
actual_route_state = {
(x.get("destination_cidr"), x.get("device"), str(x.get("table")),
x.get("via"), str(x.get("metric", 100)))
for x in parsed.get("routes", []) if x.get("state") == "owned"
}
route_flags_valid = all(
x.get("flags", []) == []
or (x.get("flags") == ["linkdown"]
and x.get("device") == plan.resources.bridge)
for x in parsed.get("routes", [])
)
route_links_ready = all(not x.get("flags")
for x in parsed.get("routes", []))
expected_rules = {
(str(plan.resources.rule_priority), "-",
plan.vpc.cidr, str(plan.resources.route_table)),
(str(plan.resources.rule_priority + 1), FABRIC_INTERFACE,
plan.vpc.cidr, str(plan.resources.route_table)),
}
actual_rules = {
(str(x.get("priority")), x.get("iif"),
x.get("destination_cidr"), str(x.get("table")))
for x in parsed.get("rules", []) if x.get("state") == "owned"
}
expected_peer_states = {p.host_id: "owned" for p in plan.peers}
wg_state_marker_count = sum(
line.startswith("__VC_VPC_WG_STATE__=")
for line in output.splitlines()
)
wg_peer_set_marker_count = sum(
line.startswith("__VC_VPC_WG_PEER_SET__=")
for line in output.splitlines()
)
expected_bridge_nft = "yes" if plan.local_members else "none"
host_dataplane_verified = (parsed.get("bridge") == plan.resources.bridge
and parsed.get("bridge_owner") == DATAPLANE_OWNER_MARKER
and (parsed.get("firewall_mode") == firewall_mode
or (firewall_mode == FIREWALL_MODE_NATIVE
and parsed.get("firewall_mode") is None))
and (parsed.get("firewall_state") == "owned"
or (firewall_mode == FIREWALL_MODE_NATIVE
and parsed.get("firewall_state") is None))
and (parsed.get("state_firewall_mode") == firewall_mode
or (firewall_mode == FIREWALL_MODE_NATIVE
and parsed.get("state_firewall_mode") in {None, ""}))
and parsed.get("persistence") == "yes"
and parsed.get("bridge_boundary") == "yes"
and parsed.get("proxy_arp") == "0"
and parsed.get("proxy_arp_pvlan") == "1"
and parsed.get("proxy_state") == "owned"
and parsed.get("local_neighbor_state") == "owned"
and actual_proxy_ips == expected_proxy_ips
and parsed.get("rp_filter") == "2"
and parsed.get("nft_table") == plan.resources.nft_table
and parsed.get("state_owner") == "owned"
and str(parsed.get("state_revision")) == str(plan.desired_revision)
and parsed.get("state_digest") == plan.desired_digest
and str(parsed.get("acl_revision")) == str(plan.desired_revision)
and parsed.get("acl_digest") == plan.desired_digest
and parsed.get("anti_spoof") == "yes"
and parsed.get("bridge_nft") == expected_bridge_nft
and parsed.get("route_extra") == "no"
and parsed.get("rule_extra") == "no"
and parsed.get("fabric_nft") == "yes"
and expected_routes == actual_routes
and actual_routes == actual_route_state
and route_flags_valid
and expected_rules == actual_rules
and wg_required_verified
and ((plan.fabric_peer_keys is None
and plan.fabric_peer_overlays is None)
or (plan.fabric_peer_overlays is not None
and wg_peer_set_marker_count == 1
and parsed.get("wg_peer_set") == "exact"))
and wg_state_marker_count == len(expected_peer_states)
and expected_peer_states == parsed.get("wg_states", {})
and parsed.get("arp_nft") == ("yes" if plan.local_members else "none"))
host_config_verified = host_dataplane_verified and all(
x.get("config_verified") is True
and x.get("seed_verified") is True
for x in members)
host_config_reusable = bool(
host_dataplane_verified
and parsed.get("firewall_mode") == firewall_mode
and parsed.get("firewall_state") == "owned"
and parsed.get("state_firewall_mode") == firewall_mode
and plan.fabric_peer_overlays is not None
and parsed.get("wg_peer_set") == "exact"
and parsed.get("helper_contract") == "owned"
and parsed.get("bridge_admin") == "up"
and parsed.get("nft_file_exact") == "yes"
and parsed.get("state_file_exact") == "yes"
and parsed.get("persistence_exact") == "yes")
# Carrier is runtime evidence. A stopped or just-starting member may
# legitimately leave its owned bridge route marked linkdown while the
# shared host configuration is still exact and safe to re-observe.
host_ready = bool(host_config_reusable and route_links_ready)
active = bool(
host_config_verified
and route_links_ready
and all(
x["xml_verified"]
and x.get("guest_network_verified") is True
for x in members
)
)
restart_required = any(
x.get("restart_required") is True for x in members)
guest_reason_code = next((
x.get("guest_reason_code") for x in members
if x.get("guest_network_verified") is not True
and isinstance(x.get("guest_reason_code"), str)
), None)
projected_wg = tuple(
dict(item, tenant_allowed_ips=sorted(
expected_wg[item.get("peer_host_id")][1]
& set(item.get("tenant_allowed_ips", []))))
for item in parsed.get("wg_allowed_ips", [])
if item.get("peer_host_id") in expected_wg
and expected_wg[item.get("peer_host_id")][1]
)
rp_filter = int(parsed["rp_filter"]) if str(parsed.get("rp_filter", "")).isdigit() else None
proxy_arp = int(parsed["proxy_arp"]) if str(parsed.get("proxy_arp", "")).isdigit() else None
proxy_arp_pvlan = (int(parsed["proxy_arp_pvlan"])
if str(parsed.get("proxy_arp_pvlan", "")).isdigit()
else None)
observed = VpcDataplaneObservation(
plan.vpc.vpc_id, plan.host_id, plan.desired_revision,
plan.desired_revision if active else 0, plan.desired_digest,
plan.desired_digest if active else None,
"active" if active else (
"applying" if restart_required or guest_reason_code else "degraded"),
plan.resources.bridge,
parsed.get("bridge_owner"), members, tuple(parsed.get("routes", [])),
projected_wg, parsed.get("nft_table"),
# ACL installation is host configuration; a stopped or unready
# guest must not erase its exact revision/digest proof.
plan.desired_revision if host_config_verified else None,
parsed.get("anti_spoof") == "yes", rp_filter, proxy_arp,
proxy_arp_pvlan,
tuple(sorted(actual_proxy_ips)),
host_config_verified,
plan.desired_revision if host_config_verified else 0,
plan.desired_digest if host_config_verified else None,
"confirmed" if active else (
"pending_restart" if restart_required
else "pending_guest" if guest_reason_code else "mismatch"),
"none", None if active else (
"vpc_private_guest_restart_required" if restart_required
else guest_reason_code
or "vpc_private_dataplane_readback_mismatch"),
).as_dict()
# This proof deliberately excludes carrier-dependent route readiness.
# It admits a later guest/runtime-only pass for stopped or starting
# members while bridge topology, helpers and persisted host artifacts
# remain exact.
observed["host_config_reusable"] = host_config_reusable
return observed, host_ready, parsed.get("nft_receipt")
__all__ = ["DATAPLANE_OWNER_MARKER", "FABRIC_INTERFACE", "PRIVATE_INTERFACE", "FIREWALL_MODE_NATIVE", "VpcDataplaneError", "VpcFabricPeer", "VpcDataplaneResources", "VpcDataplanePlan", "VpcDataplaneObservation", "derive_private_mac", "private_guest_routes", "private_guest_network_generation", "resource_names", "parse_vpc_dataplane_plan", "render_vpc_nft_rules", "render_apply_script", "render_remove_script", "render_observe_script", "patch_private_interface_xml", "VpcPrivateHostAdapter"]