You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
4538 lines
224 KiB
Python
4538 lines
224 KiB
Python
"""Panel-owned VPC private L3 data-plane primitives.
|
|
|
|
This module is deliberately independent from :mod:`virsh_panel.fabric`.
|
|
Fabric owns the host overlay (``wg-fabric0``); this boundary owns only a
|
|
VPC's bridge, policy route table, tenant AllowedIPs and nftables table. The
|
|
pure builders are usable in unit tests and the host adapter runs one fixed
|
|
shell transaction through a pinned remote manager.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
import hashlib
|
|
import ipaddress
|
|
import re
|
|
import shlex
|
|
import time
|
|
import xml.etree.ElementTree as ET
|
|
from dataclasses import dataclass
|
|
from typing import Any, Mapping, Sequence
|
|
|
|
from .network_roles import (
|
|
PRIVATE_INTERFACE_ROLE,
|
|
PRIVATE_LIBVIRT_ALIAS,
|
|
PUBLIC_LIBVIRT_ALIAS,
|
|
select_private_interface,
|
|
select_public_interface,
|
|
set_libvirt_interface_alias,
|
|
)
|
|
from .shell_contracts import (
|
|
IP_RULE_CONVERGENCE_SH,
|
|
VPC_SYSCTL_VALIDATION_SH as _VPC_SYSCTL_VALIDATION_SH,
|
|
)
|
|
from .vpc import VpcError, VpcMember, VpcPlan, parse_vpc_plan
|
|
|
|
DATAPLANE_SCHEMA_VERSION = 1
|
|
DATAPLANE_OWNER_MARKER = "virt-cloud-vpc-dataplane-v1"
|
|
FABRIC_OWNER_MARKER = "virt-cloud-wg-fabric-v1"
|
|
FABRIC_INTERFACE = "wg-fabric0"
|
|
FABRIC_LOCK = "/run/virt-cloud/wg-fabric0.lock"
|
|
LEGACY_FIREWALLD_NFT_MANIFEST = "/etc/virt-cloud/nftables.conf"
|
|
FIREWALL_MODE_NATIVE = "native_nftables"
|
|
FIREWALL_MODES = frozenset({FIREWALL_MODE_NATIVE})
|
|
PRIVATE_INTERFACE = PRIVATE_INTERFACE_ROLE
|
|
PANEL_NS = "urn:virsh-panel:1"
|
|
BRIDGE_PREFIX = "vcv-"
|
|
NFT_PREFIX = "virt_cloud_vpc_"
|
|
ROUTE_TABLE_MIN, ROUTE_TABLE_MAX = 10000, 19999
|
|
RULE_PRIORITY_MIN, RULE_PRIORITY_MAX = 20000, 29999
|
|
MAX_OUTPUT = 262144
|
|
_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,63}\Z")
|
|
_HOST = re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]{0,63}\Z")
|
|
_MAC = re.compile(r"[0-9a-fA-F]{2}(?::[0-9a-fA-F]{2}){5}\Z")
|
|
_KEY = re.compile(r"[A-Za-z0-9+/]{42}==\Z|[A-Za-z0-9+/]{43}=\Z")
|
|
_DIGEST = re.compile(r"[0-9a-f]{64}\Z")
|
|
|
|
VPC_NATIVE_TRANSACTION_GUARD_SH = r'''
|
|
vc_vpc_firewall_boundary_clean() {
|
|
[ -n "${NFT:-}" ] || return 1
|
|
[ ! -e /etc/virt-cloud/nftables.conf ] &&
|
|
[ ! -L /etc/virt-cloud/nftables.conf ] || return 1
|
|
[ ! -e /etc/virt-cloud/.wg-fabric-txn ] &&
|
|
[ ! -L /etc/virt-cloud/.wg-fabric-txn ] || return 1
|
|
[ ! -e /run/virt-cloud/wg-fabric-emergency.owner ] &&
|
|
[ ! -L /run/virt-cloud/wg-fabric-emergency.owner ] || return 1
|
|
vc_vpc_fabric_config=/etc/virt-cloud/wg-fabric0.conf
|
|
if [ -e "$vc_vpc_fabric_config" ] || [ -L "$vc_vpc_fabric_config" ]; then
|
|
[ -f "$vc_vpc_fabric_config" ] &&
|
|
[ ! -L "$vc_vpc_fabric_config" ] || return 1
|
|
[ "$($STAT -c '%u:%g:%a' "$vc_vpc_fabric_config" 2>/dev/null)" = 0:0:600 ] || return 1
|
|
vc_vpc_fabric_config_size=$($STAT -c '%s' "$vc_vpc_fabric_config" 2>/dev/null) || return 1
|
|
case "$vc_vpc_fabric_config_size" in ''|*[!0-9]*) return 1 ;; esac
|
|
[ "$vc_vpc_fabric_config_size" -le 65536 ] || return 1
|
|
awk -F'|' '
|
|
length($0)>4096 {bad=1}
|
|
$1=="firewall" {firewall++; if (NF!=2 || $2!="native_nftables") bad=1; next}
|
|
$1=="underlay" || $1=="firewall_rule" {bad=1; next}
|
|
END {exit (bad || firewall>1) ? 1 : 0}
|
|
' "$vc_vpc_fabric_config" || return 1
|
|
fi
|
|
vc_vpc_transaction_count=0
|
|
for vc_vpc_other_transaction in /etc/virt-cloud/.vpc-dataplane-*.txn; do
|
|
[ -e "$vc_vpc_other_transaction" ] ||
|
|
[ -L "$vc_vpc_other_transaction" ] || continue
|
|
vc_vpc_transaction_count=$((vc_vpc_transaction_count + 1))
|
|
[ "$vc_vpc_transaction_count" -le 256 ] || return 1
|
|
[ "$vc_vpc_other_transaction" = "$TXN" ] && continue
|
|
return 1
|
|
done
|
|
vc_vpc_legacy_snapshot_count=0
|
|
for vc_vpc_legacy_snapshot in \
|
|
/etc/virt-cloud/vpc-dataplane/*.nft.inet.live \
|
|
/etc/virt-cloud/vpc-dataplane/*.nft.bridge.live \
|
|
/etc/virt-cloud/vpc-dataplane/*.nft.arp.live; do
|
|
[ -e "$vc_vpc_legacy_snapshot" ] ||
|
|
[ -L "$vc_vpc_legacy_snapshot" ] || continue
|
|
vc_vpc_legacy_snapshot_count=$((vc_vpc_legacy_snapshot_count + 1))
|
|
[ "$vc_vpc_legacy_snapshot_count" -le 256 ] || return 1
|
|
return 1
|
|
done
|
|
vc_vpc_state_count=0
|
|
for vc_vpc_state_file in /etc/virt-cloud/vpc-dataplane/*.state; do
|
|
[ -e "$vc_vpc_state_file" ] || [ -L "$vc_vpc_state_file" ] || continue
|
|
vc_vpc_state_count=$((vc_vpc_state_count + 1))
|
|
[ "$vc_vpc_state_count" -le 256 ] || return 1
|
|
[ -f "$vc_vpc_state_file" ] && [ ! -L "$vc_vpc_state_file" ] || return 1
|
|
[ "$($STAT -c '%u:%g:%a' "$vc_vpc_state_file" 2>/dev/null)" = 0:0:600 ] || return 1
|
|
vc_vpc_state_size=$($STAT -c '%s' "$vc_vpc_state_file" 2>/dev/null) || return 1
|
|
case "$vc_vpc_state_size" in ''|*[!0-9]*) return 1 ;; esac
|
|
[ "$vc_vpc_state_size" -le 65536 ] || return 1
|
|
awk -F'|' '
|
|
$1=="firewall_mode" {firewall++; if (NF!=2 || $2!="native_nftables") bad=1; next}
|
|
$1=="nft_snapshot" {bad=1; next}
|
|
END {exit (bad || firewall>1) ? 1 : 0}
|
|
' "$vc_vpc_state_file" || return 1
|
|
done
|
|
vc_vpc_firewall_tables=$($NFT list tables 2>/dev/null) || return 1
|
|
vc_vpc_firewall_tables_size=$(printf '%s' "$vc_vpc_firewall_tables" |
|
|
wc -c | tr -d ' ') || return 1
|
|
case "$vc_vpc_firewall_tables_size" in ''|*[!0-9]*) return 1 ;; esac
|
|
[ "$vc_vpc_firewall_tables_size" -le 65536 ] || return 1
|
|
! printf '%s\n' "$vc_vpc_firewall_tables" |
|
|
grep -Eq 'table (inet firewalld|(inet|bridge) virt_cloud_fabric_emergency)([[:space:]]|$)'
|
|
}
|
|
vc_vpc_native_transaction_owned() {
|
|
[ -n "${TXN:-}" ] && [ -n "${STAT:-}" ] || return 1
|
|
[ -d "$TXN" ] && [ ! -L "$TXN" ] || return 1
|
|
[ "$($STAT -c '%u:%g:%a' "$TXN" 2>/dev/null)" = 0:0:700 ] || return 1
|
|
[ -f "$TXN/firewall_mode" ] && [ ! -L "$TXN/firewall_mode" ] || return 1
|
|
[ "$($STAT -c '%u:%g:%a' "$TXN/firewall_mode" 2>/dev/null)" = 0:0:600 ] || return 1
|
|
vc_vpc_txn_mode_size=$($STAT -c '%s' "$TXN/firewall_mode" 2>/dev/null) || return 1
|
|
case "$vc_vpc_txn_mode_size" in ''|*[!0-9]*) return 1 ;; esac
|
|
[ "$vc_vpc_txn_mode_size" -le 64 ] || return 1
|
|
vc_vpc_txn_mode=$(awk '
|
|
NR==1 && $0=="native_nftables" {ok=1}
|
|
{rows++}
|
|
END {if (ok && rows==1) print "native_nftables"; else exit 1}
|
|
' "$TXN/firewall_mode") || return 1
|
|
[ "$vc_vpc_txn_mode" = native_nftables ] || return 1
|
|
for vc_vpc_legacy_txn_file in \
|
|
"$TXN"/nft.persist.before.inet \
|
|
"$TXN"/nft.persist.before.bridge \
|
|
"$TXN"/nft.persist.before.arp \
|
|
"$TXN"/reassert.nft \
|
|
"$TXN"/nft.desired.verify.*; do
|
|
[ -e "$vc_vpc_legacy_txn_file" ] || [ -L "$vc_vpc_legacy_txn_file" ] || continue
|
|
return 1
|
|
done
|
|
if [ -e "$TXN/state.before" ] || [ -L "$TXN/state.before" ]; then
|
|
[ -f "$TXN/state.before" ] && [ ! -L "$TXN/state.before" ] || return 1
|
|
[ "$($STAT -c '%u:%g:%a' "$TXN/state.before" 2>/dev/null)" = 0:0:600 ] || return 1
|
|
vc_vpc_txn_state_size=$($STAT -c '%s' "$TXN/state.before" 2>/dev/null) || return 1
|
|
case "$vc_vpc_txn_state_size" in ''|*[!0-9]*) return 1 ;; esac
|
|
[ "$vc_vpc_txn_state_size" -le 65536 ] || return 1
|
|
awk -F'|' '
|
|
$1=="firewall_mode" {firewall++; if (NF!=2 || $2!="native_nftables") bad=1; next}
|
|
$1=="nft_snapshot" {bad=1; next}
|
|
END {exit (bad || firewall>1) ? 1 : 0}
|
|
' "$TXN/state.before" || return 1
|
|
fi
|
|
return 0
|
|
}
|
|
vc_vpc_transaction_file_secure() {
|
|
vc_vpc_transaction_file=$1
|
|
vc_vpc_transaction_file_limit=$2
|
|
[ -f "$vc_vpc_transaction_file" ] &&
|
|
[ ! -L "$vc_vpc_transaction_file" ] || return 1
|
|
[ "$($STAT -c '%u:%g:%a' "$vc_vpc_transaction_file" 2>/dev/null)" = 0:0:600 ] || return 1
|
|
vc_vpc_transaction_file_size=$($STAT -c '%s' "$vc_vpc_transaction_file" 2>/dev/null) || return 1
|
|
case "$vc_vpc_transaction_file_size:$vc_vpc_transaction_file_limit" in
|
|
*[!0-9:]*) return 1 ;;
|
|
esac
|
|
[ "$vc_vpc_transaction_file_size" -le "$vc_vpc_transaction_file_limit" ] || return 1
|
|
}
|
|
vc_vpc_transaction_record_owned() {
|
|
vc_vpc_transaction_record_file=$1
|
|
vc_vpc_transaction_record_expected=$2
|
|
vc_vpc_transaction_file_secure "$vc_vpc_transaction_record_file" 4096 || return 1
|
|
awk -v wanted="$vc_vpc_transaction_record_expected" '
|
|
{rows++; if (rows!=1 || $0!=wanted) bad=1}
|
|
END {exit (bad || rows!=1) ? 1 : 0}
|
|
' "$vc_vpc_transaction_record_file"
|
|
}
|
|
vc_vpc_write_native_transaction_mode() {
|
|
[ -d "$TXN" ] && [ ! -L "$TXN" ] || return 1
|
|
[ ! -e "$TXN/firewall_mode" ] && [ ! -L "$TXN/firewall_mode" ] || return 1
|
|
printf '%s\n' native_nftables > "$TXN/firewall_mode" || return 1
|
|
chmod 600 "$TXN/firewall_mode" || return 1
|
|
vc_vpc_native_transaction_owned
|
|
}
|
|
'''.strip()
|
|
|
|
_NFT_INET_REQUIRED_SUFFIXES = (
|
|
"anti-spoof-local-ip", "anti-spoof-egress",
|
|
"anti-spoof-public-to-bridge", "anti-spoof-public-to-fabric",
|
|
"anti-spoof-bridge-to-foreign", "anti-spoof-fabric-to-foreign",
|
|
"anti-spoof-public-to-vpc-bridge", "anti-spoof-public-to-vpc-fabric",
|
|
"anti-spoof-bridge-vpc-to-foreign", "anti-spoof-fabric-vpc-to-foreign",
|
|
"anti-spoof-wg-ingress", "anti-spoof-wg-egress",
|
|
"anti-spoof-wg-source", "member-egress-isolation",
|
|
"member-fabric-isolation", "default-deny", "wg-transit-deny",
|
|
"ipv6-drop-bridge", "ipv6-drop-fabric", "ipv6-input-drop-bridge",
|
|
"ipv6-input-drop-fabric", "ipv6-output-drop-bridge",
|
|
"host-isolation", "host-isolation-wg", "host-isolation-out",
|
|
"host-isolation-wg-source", "host-isolation-out-wg",
|
|
"host-isolation-vpc-out", "host-isolation-vpc-out-wg",
|
|
"scope-bridge-in", "scope-bridge-out", "scope-fabric-in",
|
|
"scope-fabric-out",
|
|
)
|
|
_NFT_BRIDGE_REQUIRED_SUFFIXES = (
|
|
"bridge-anti-spoof-ingress", "bridge-anti-spoof-arp",
|
|
"bridge-anti-spoof-arp-mac", "bridge-anti-spoof-ip",
|
|
"bridge-anti-spoof-arp-ip", "bridge-multicast-drop",
|
|
"bridge-limited-broadcast-drop", "bridge-vpc-broadcast-drop",
|
|
"bridge-unknown-destination-drop", "bridge-default-deny",
|
|
"bridge-scope-accept", "bridge-arp-accept", "bridge-non-ip-drop",
|
|
"bridge-anti-spoof-arp-input-mac", "bridge-anti-spoof-input",
|
|
"bridge-anti-spoof-input-ip", "bridge-ipv6-input-drop",
|
|
"bridge-arp-input-accept", "bridge-ip-input-accept",
|
|
"bridge-input-non-ip-drop",
|
|
)
|
|
|
|
NFT_LIVE_SNAPSHOT_SH = r'''
|
|
vc_nft_table_absent() {
|
|
vc_nft_absent_family=$1
|
|
vc_nft_absent_name=$2
|
|
vc_nft_absent_inventory=$3
|
|
case "$vc_nft_absent_family" in inet|bridge|arp) : ;; *) return 1 ;; esac
|
|
case "$vc_nft_absent_name" in ''|*[!A-Za-z0-9_]*) return 1 ;; esac
|
|
# A failed point lookup is ambiguous (permissions, netlink failure, parse
|
|
# error, or actual absence). Only a successful, completely parsed family
|
|
# inventory may prove that a table does not exist.
|
|
"$NFT" -s list tables "$vc_nft_absent_family" \
|
|
> "$vc_nft_absent_inventory" 2>/dev/null || return 1
|
|
awk -v family="$vc_nft_absent_family" -v wanted="$vc_nft_absent_name" '
|
|
NF == 0 {next}
|
|
NF != 3 || $1 != "table" || $2 != family ||
|
|
$3 !~ /^[A-Za-z0-9_.-]+$/ {bad=1; next}
|
|
$3 == wanted {found++}
|
|
END {exit (!bad && found == 0) ? 0 : 1}
|
|
' "$vc_nft_absent_inventory"
|
|
}
|
|
vc_nft_table_snapshot() {
|
|
vc_nft_snapshot_family=$1
|
|
vc_nft_snapshot_name=$2
|
|
vc_nft_snapshot_path=$3
|
|
case "$vc_nft_snapshot_family" in inet|bridge|arp) : ;; *) return 1 ;; esac
|
|
case "$vc_nft_snapshot_name" in ''|*[!A-Za-z0-9_]*) return 1 ;; esac
|
|
rm -f "$vc_nft_snapshot_path" "$vc_nft_snapshot_path.absent" \
|
|
"$vc_nft_snapshot_path.raw"
|
|
# `nft --stateless list table` is itself valid nft input and retains the
|
|
# complete table semantics (chains, expressions, verdicts and set
|
|
# elements) while omitting only changing state such as packet counters.
|
|
# Do not reduce this to owner comments: a foreign rule can retain a panel
|
|
# comment while changing `drop` to `accept`.
|
|
if "$NFT" -s list table "$vc_nft_snapshot_family" "$vc_nft_snapshot_name" \
|
|
> "$vc_nft_snapshot_path.raw" 2>/dev/null; then
|
|
[ -s "$vc_nft_snapshot_path.raw" ] || return 1
|
|
mv -f "$vc_nft_snapshot_path.raw" "$vc_nft_snapshot_path" || return 1
|
|
chmod 600 "$vc_nft_snapshot_path" || return 1
|
|
else
|
|
rm -f "$vc_nft_snapshot_path.raw"
|
|
vc_nft_table_absent "$vc_nft_snapshot_family" "$vc_nft_snapshot_name" \
|
|
"$vc_nft_snapshot_path.inventory" || return 1
|
|
rm -f "$vc_nft_snapshot_path.inventory"
|
|
: > "$vc_nft_snapshot_path.absent" || return 1
|
|
chmod 600 "$vc_nft_snapshot_path.absent" || return 1
|
|
fi
|
|
}
|
|
vc_nft_table_matches_snapshot() {
|
|
vc_nft_match_family=$1
|
|
vc_nft_match_name=$2
|
|
vc_nft_match_path=$3
|
|
case "$vc_nft_match_family" in inet|bridge|arp) : ;; *) return 1 ;; esac
|
|
case "$vc_nft_match_name" in ''|*[!A-Za-z0-9_]*) return 1 ;; esac
|
|
if [ -f "$vc_nft_match_path" ] && [ ! -L "$vc_nft_match_path" ] &&
|
|
[ ! -e "$vc_nft_match_path.absent" ] &&
|
|
[ ! -L "$vc_nft_match_path.absent" ]; then
|
|
"$NFT" -s list table "$vc_nft_match_family" "$vc_nft_match_name" \
|
|
> "$vc_nft_match_path.current" 2>/dev/null || return 1
|
|
[ -s "$vc_nft_match_path.current" ] || return 1
|
|
if "$CMP" -s "$vc_nft_match_path" "$vc_nft_match_path.current"; then
|
|
vc_nft_match_status=0
|
|
else
|
|
vc_nft_match_status=$?
|
|
fi
|
|
rm -f "$vc_nft_match_path.current"
|
|
return "$vc_nft_match_status"
|
|
fi
|
|
if [ -f "$vc_nft_match_path.absent" ] &&
|
|
[ ! -L "$vc_nft_match_path.absent" ] &&
|
|
[ ! -e "$vc_nft_match_path" ] && [ ! -L "$vc_nft_match_path" ]; then
|
|
if vc_nft_table_absent "$vc_nft_match_family" "$vc_nft_match_name" \
|
|
"$vc_nft_match_path.inventory"; then
|
|
vc_nft_match_status=0
|
|
else
|
|
vc_nft_match_status=$?
|
|
fi
|
|
rm -f "$vc_nft_match_path.inventory"
|
|
return "$vc_nft_match_status"
|
|
fi
|
|
return 1
|
|
}
|
|
'''.strip()
|
|
|
|
NETWORKMANAGER_VPC_PROFILE_SH = r'''
|
|
vc_nm_profile_disk_validate() {
|
|
vc_nm_disk_file=$1
|
|
vc_nm_disk_uuid=$2
|
|
vc_nm_disk_auto=${3:-false}
|
|
case "$vc_nm_disk_file" in
|
|
/etc/NetworkManager/system-connections/*.nmconnection)
|
|
vc_nm_disk_name=${vc_nm_disk_file#/etc/NetworkManager/system-connections/}
|
|
case "$vc_nm_disk_name" in ''|*/*) return 1 ;; esac
|
|
;;
|
|
*) return 1 ;;
|
|
esac
|
|
[ -f "$vc_nm_disk_file" ] && [ ! -L "$vc_nm_disk_file" ] || return 1
|
|
[ "$($STAT -c '%u:%a' "$vc_nm_disk_file" 2>/dev/null)" = 0:600 ] || return 1
|
|
awk -F= -v wanted_id="$BRIDGE" -v wanted_uuid="$vc_nm_disk_uuid" \
|
|
-v wanted_marker="$NM_PROFILE_MARKER" \
|
|
-v expected_auto="$vc_nm_disk_auto" '
|
|
/^\[/ { in_connection=($0 == "[connection]"); next }
|
|
in_connection && $1 == "id" {
|
|
id_count++; if (substr($0, 4) != wanted_id) bad=1; next
|
|
}
|
|
in_connection && $1 == "uuid" {
|
|
uuid_count++; if (substr($0, 6) != wanted_uuid) bad=1; next
|
|
}
|
|
in_connection && $1 == "type" {
|
|
type_count++; if (substr($0, 6) != "bridge") bad=1; next
|
|
}
|
|
in_connection && $1 == "interface-name" {
|
|
iface_count++; if (substr($0, 16) != wanted_id) bad=1; next
|
|
}
|
|
in_connection && $1 == "stable-id" {
|
|
marker_count++; if (substr($0, 11) != wanted_marker) bad=1; next
|
|
}
|
|
in_connection && $1 == "autoconnect" {
|
|
auto_count++; if (substr($0, 13) != expected_auto) bad=1; next
|
|
}
|
|
in_connection && $1 == "autoconnect-retries" {
|
|
retry_count++; if (substr($0, 21) != "0") bad=1; next
|
|
}
|
|
in_connection && $1 == "zone" { zone_count++; next }
|
|
END {
|
|
exit (bad || id_count != 1 || uuid_count != 1 || type_count != 1 ||
|
|
iface_count != 1 || marker_count != 1 || auto_count != 1 ||
|
|
retry_count != 1 ||
|
|
zone_count != 0) ? 1 : 0
|
|
}
|
|
' "$vc_nm_disk_file"
|
|
}
|
|
vc_nm_profile_field() {
|
|
vc_nm_field_raw=$("$NMCLI" -t --escape no -g "$1" \
|
|
connection show uuid "$NM_PROFILE_UUID" 2>/dev/null) || return 1
|
|
printf '%s\n' "$vc_nm_field_raw" | sed -n '1p'
|
|
}
|
|
vc_nm_profile_empty() {
|
|
vc_nm_empty_value=$(vc_nm_profile_field "$1") || return 1
|
|
[ -z "$vc_nm_empty_value" ] || [ "$vc_nm_empty_value" = "--" ]
|
|
}
|
|
vc_nm_profile_resolve() {
|
|
vc_nm_resolve_auto=${1:-false}
|
|
vc_nm_names_raw=$("$NMCLI" -t --escape no -f NAME \
|
|
connection show 2>/dev/null) || return 1
|
|
vc_nm_count=$(printf '%s\n' "$vc_nm_names_raw" |
|
|
awk -v wanted="$BRIDGE" '$0 == wanted { count++ } END { print count + 0 }') || return 1
|
|
case "$vc_nm_count" in 0) return 2 ;; 1) : ;; *) return 1 ;; esac
|
|
vc_nm_uuid_raw=$("$NMCLI" -t --escape no -g connection.uuid \
|
|
connection show "$BRIDGE" 2>/dev/null) || return 1
|
|
NM_PROFILE_UUID=$(printf '%s\n' "$vc_nm_uuid_raw" | sed -n '1p') || return 1
|
|
printf '%s\n' "$NM_PROFILE_UUID" |
|
|
grep -Eq '^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$' || return 1
|
|
NM_PROFILE_FILE=$(vc_nm_filename_for_uuid "$NM_PROFILE_UUID") || return 1
|
|
vc_nm_profile_disk_validate "$NM_PROFILE_FILE" "$NM_PROFILE_UUID" \
|
|
"$vc_nm_resolve_auto" || return 1
|
|
}
|
|
vc_nm_profile_base_validate_mode() {
|
|
vc_nm_expected_auto=$1
|
|
vc_nm_expected_disk_auto=$2
|
|
[ "$(vc_nm_profile_field connection.type)" = bridge ] || return 1
|
|
[ "$(vc_nm_profile_field connection.interface-name)" = "$BRIDGE" ] || return 1
|
|
[ "$(vc_nm_profile_field connection.stable-id)" = "$NM_PROFILE_MARKER" ] || return 1
|
|
[ "$(vc_nm_profile_field connection.autoconnect)" = "$vc_nm_expected_auto" ] || return 1
|
|
if [ "$vc_nm_expected_auto" = no ]; then
|
|
[ "$(vc_nm_profile_field connection.autoconnect-retries)" = 0 ] || return 1
|
|
fi
|
|
[ "$(vc_nm_profile_field ipv4.method)" = disabled ] || return 1
|
|
[ "$(vc_nm_profile_field ipv6.method)" = disabled ] || return 1
|
|
for vc_nm_empty_field in \
|
|
ipv4.addresses ipv4.routes ipv4.routing-rules ipv4.gateway ipv4.dns \
|
|
ipv4.dns-search ipv6.addresses ipv6.routes ipv6.routing-rules \
|
|
ipv6.gateway ipv6.dns ipv6.dns-search connection.master \
|
|
connection.controller connection.slave-type connection.port-type \
|
|
connection.secondaries; do
|
|
vc_nm_profile_empty "$vc_nm_empty_field" || return 1
|
|
done
|
|
case "$(vc_nm_profile_field bridge.stp)" in no|false|off|0|--) : ;; *) return 1 ;; esac
|
|
vc_nm_profile_disk_validate "$NM_PROFILE_FILE" "$NM_PROFILE_UUID" \
|
|
"$vc_nm_expected_disk_auto" || return 1
|
|
}
|
|
vc_nm_profile_base_validate() {
|
|
vc_nm_profile_base_validate_mode no false
|
|
}
|
|
vc_nm_profile_active_count() {
|
|
vc_nm_active_raw=$("$NMCLI" -t --escape no -f UUID \
|
|
connection show --active 2>/dev/null) || return 1
|
|
printf '%s\n' "$vc_nm_active_raw" | awk -v wanted="$NM_PROFILE_UUID" '
|
|
NF == 0 {next}
|
|
length($0) != 36 || $0 !~ /^[0-9A-Fa-f-]+$/ {bad=1; next}
|
|
$0 == wanted {count++}
|
|
END {
|
|
if (bad) exit 1
|
|
print count + 0
|
|
}
|
|
'
|
|
}
|
|
vc_nm_profile_active() {
|
|
[ "$(vc_nm_profile_active_count)" = 1 ]
|
|
}
|
|
vc_nm_profile_inactive() {
|
|
[ "$(vc_nm_profile_active_count)" = 0 ]
|
|
}
|
|
vc_nm_device_managed_state() {
|
|
vc_nm_managed_raw=$(
|
|
"$NMCLI" -t --escape no -g GENERAL.NM-MANAGED \
|
|
device show "$BRIDGE" 2>/dev/null
|
|
) || return 1
|
|
vc_nm_managed=$(printf '%s\n' "$vc_nm_managed_raw" | sed -n '1p') || return 1
|
|
case "$vc_nm_managed" in yes|no) printf '%s' "$vc_nm_managed" ;; *) return 1 ;; esac
|
|
}
|
|
vc_nm_device_expect_managed() {
|
|
[ "$(vc_nm_device_managed_state)" = "$1" ]
|
|
}
|
|
vc_nm_link_expect_down() {
|
|
vc_nm_link_raw=$("$IP" -o link show dev "$BRIDGE" 2>/dev/null) || return 1
|
|
vc_nm_link_flags=$(printf '%s\n' "$vc_nm_link_raw" | awk -v wanted="$BRIDGE" '
|
|
NF == 0 {next}
|
|
{
|
|
count++
|
|
name=$2
|
|
sub(/:$/, "", name)
|
|
sub(/@.*/, "", name)
|
|
if (name != wanted || $3 !~ /^<[^>]+>$/) bad=1
|
|
flags=$3
|
|
}
|
|
END {
|
|
if (bad || count != 1) exit 1
|
|
print flags
|
|
}
|
|
') || return 1
|
|
printf '%s\n' "$vc_nm_link_flags" |
|
|
grep -Eq '(^|<|,)UP(,|>)' && return 1 || true
|
|
}
|
|
vc_nm_device_quarantine() {
|
|
"$IP" link show dev "$BRIDGE" >/dev/null 2>&1 || return 1
|
|
vc_nm_quarantine_uuid_raw=$(
|
|
"$NMCLI" -t --escape no -g GENERAL.CON-UUID \
|
|
device show "$BRIDGE" 2>/dev/null
|
|
) || return 1
|
|
vc_nm_quarantine_uuid=$(printf '%s\n' "$vc_nm_quarantine_uuid_raw" |
|
|
sed -n '1p') || return 1
|
|
[ -z "$vc_nm_quarantine_uuid" ] ||
|
|
[ "$vc_nm_quarantine_uuid" = "$NM_PROFILE_UUID" ] || return 1
|
|
vc_nm_quarantine_zone=$(vc_nm_profile_field connection.zone) || return 1
|
|
[ -z "$vc_nm_quarantine_zone" ] || [ "$vc_nm_quarantine_zone" = -- ] ||
|
|
return 1
|
|
"$NMCLI" device set "$BRIDGE" managed no >/dev/null 2>&1 || return 1
|
|
vc_nm_device_expect_managed no || return 1
|
|
vc_nm_quarantine_zone=$(vc_nm_profile_field connection.zone) || return 1
|
|
[ -z "$vc_nm_quarantine_zone" ] || [ "$vc_nm_quarantine_zone" = -- ] || return 1
|
|
"$IP" link set dev "$BRIDGE" down >/dev/null 2>&1 || return 1
|
|
vc_nm_link_expect_down || return 1
|
|
if [ "${BRIDGE_TOPOLOGY_CAPTURED:-no}" = yes ]; then
|
|
vc_vpc_bridge_topology_matches || return 1
|
|
fi
|
|
}
|
|
vc_nm_device_release() {
|
|
"$IP" link show dev "$BRIDGE" >/dev/null 2>&1 || return 1
|
|
"$NMCLI" device set "$BRIDGE" managed yes >/dev/null 2>&1 || return 1
|
|
vc_nm_device_expect_managed yes
|
|
}
|
|
vc_nm_created_profile_remove() {
|
|
[ -n "$NM_CREATED_UUID" ] || return 1
|
|
vc_nm_profile_resolve || return 1
|
|
[ "$NM_PROFILE_UUID" = "$NM_CREATED_UUID" ] || return 1
|
|
vc_nm_profile_base_validate || return 1
|
|
"$NMCLI" connection delete uuid "$NM_CREATED_UUID" >/dev/null 2>&1 ||
|
|
return 1
|
|
if vc_nm_profile_resolve; then
|
|
return 1
|
|
else
|
|
vc_nm_delete_readback_status=$?
|
|
fi
|
|
[ "$vc_nm_delete_readback_status" = 2 ]
|
|
}
|
|
vc_nm_profile_expect() {
|
|
vc_nm_expected=$1
|
|
vc_nm_profile_base_validate || return 1
|
|
vc_nm_memory_zone=$(vc_nm_profile_field connection.zone) || return 1
|
|
case "$vc_nm_expected" in
|
|
blocked)
|
|
[ -z "$vc_nm_memory_zone" ] || [ "$vc_nm_memory_zone" = -- ] || return 1
|
|
;;
|
|
blocked-active)
|
|
[ -z "$vc_nm_memory_zone" ] || [ "$vc_nm_memory_zone" = -- ] || return 1
|
|
vc_nm_profile_active
|
|
;;
|
|
active)
|
|
[ -z "$vc_nm_memory_zone" ] || [ "$vc_nm_memory_zone" = -- ] || return 1
|
|
vc_nm_device_expect_managed yes || return 1
|
|
vc_nm_profile_active
|
|
;;
|
|
quarantined)
|
|
[ -z "$vc_nm_memory_zone" ] || [ "$vc_nm_memory_zone" = -- ] || return 1
|
|
vc_nm_device_expect_managed no
|
|
;;
|
|
preapply)
|
|
[ -z "$vc_nm_memory_zone" ] || [ "$vc_nm_memory_zone" = -- ]
|
|
;;
|
|
*) return 1 ;;
|
|
esac
|
|
}
|
|
'''.strip()
|
|
|
|
|
|
class VpcDataplaneError(Exception):
|
|
"""Safe, typed error at the VPC host boundary."""
|
|
|
|
def __init__(self, message: str, status: int = 400,
|
|
error_code: str = "vpc_dataplane_invalid") -> None:
|
|
super().__init__(message)
|
|
self.status = status
|
|
self.error_code = error_code
|
|
|
|
|
|
def _fabric_plan_digest(value: Any) -> str:
|
|
if not isinstance(value, str) or _DIGEST.fullmatch(value) is None:
|
|
raise VpcDataplaneError(
|
|
"Fabric 计划摘要无效", 409, "vpc_fabric_plan_digest_invalid")
|
|
return value
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class VpcFabricPeer:
|
|
host_id: str
|
|
public_key: str
|
|
overlay_ip: str
|
|
# Tenant /32s from every panel VPC sharing this Fabric peer. WireGuard
|
|
# has one AllowedIPs list per peer, so a VPC must merge this typed set
|
|
# instead of replacing another active VPC's entries.
|
|
tenant_allowed_ips: tuple[str, ...] = ()
|
|
|
|
@property
|
|
def overlay_cidr(self) -> str:
|
|
return f"{self.overlay_ip}/32"
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class VpcDataplaneResources:
|
|
token: str
|
|
bridge: str
|
|
nft_table: str
|
|
route_table: int
|
|
rule_priority: int
|
|
owner_file: str
|
|
state_file: str
|
|
lock_file: str
|
|
transaction_dir: str
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class VpcDataplanePlan:
|
|
vpc: VpcPlan
|
|
host_id: str
|
|
members: tuple[VpcMember, ...]
|
|
peers: tuple[VpcFabricPeer, ...]
|
|
resources: VpcDataplaneResources
|
|
# Engine-authenticated digest of the complete Fabric plan. This is not
|
|
# part of the tenant VPC DTO: host transactions use it to prove that the
|
|
# shared Fabric quarantine table still belongs to the exact Fabric
|
|
# revision admitted by the coordinator.
|
|
fabric_plan_digest: str
|
|
# The authenticated full Fabric peer set, excluding this host. It is
|
|
# separate from ``peers`` because one VPC can use only a subset of the
|
|
# peers present on a shared Fabric.
|
|
fabric_peer_keys: frozenset[str] | None = None
|
|
|
|
def __post_init__(self) -> None:
|
|
_fabric_plan_digest(self.fabric_plan_digest)
|
|
|
|
@property
|
|
def desired_revision(self) -> int:
|
|
return self.vpc.desired_revision
|
|
|
|
@property
|
|
def desired_digest(self) -> str:
|
|
return self.vpc.desired_digest
|
|
|
|
@property
|
|
def local_members(self) -> tuple[VpcMember, ...]:
|
|
return tuple(m for m in self.members if m.host_id == self.host_id)
|
|
|
|
@property
|
|
def remote_members(self) -> tuple[VpcMember, ...]:
|
|
return tuple(m for m in self.members if m.host_id != self.host_id)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class VpcDataplaneObservation:
|
|
vpc_id: str
|
|
host_id: str
|
|
desired_revision: int
|
|
observed_revision: int
|
|
desired_digest: str
|
|
observed_digest: str | None
|
|
state: str
|
|
bridge: str
|
|
bridge_owner: str | None
|
|
member_nics: tuple[dict[str, Any], ...]
|
|
routes: tuple[dict[str, Any], ...]
|
|
wg_allowed_ips: tuple[dict[str, Any], ...]
|
|
nft_table: str | None
|
|
acl_revision: int | None
|
|
anti_spoof: bool
|
|
rp_filter: int | None
|
|
proxy_arp: int | None
|
|
proxy_arp_pvlan: int | None
|
|
proxy_ips: tuple[str, ...]
|
|
config_verified: bool
|
|
config_observed_revision: int
|
|
config_observed_digest: str | None
|
|
readback: str
|
|
rollback: str
|
|
reason_code: str | None = None
|
|
removed: bool = False
|
|
|
|
def as_dict(self) -> dict[str, Any]:
|
|
return {
|
|
"vpc_id": self.vpc_id, "host_id": self.host_id,
|
|
"desired_revision": self.desired_revision,
|
|
"observed_revision": self.observed_revision,
|
|
"desired_digest": self.desired_digest,
|
|
"observed_digest": self.observed_digest,
|
|
"state": self.state, "bridge": self.bridge,
|
|
"bridge_owner": self.bridge_owner,
|
|
"member_nics": [dict(x) for x in self.member_nics],
|
|
"routes": [dict(x) for x in self.routes],
|
|
"wg_allowed_ips": [dict(x) for x in self.wg_allowed_ips],
|
|
"nft_table": self.nft_table, "acl_revision": self.acl_revision,
|
|
"anti_spoof": self.anti_spoof, "rp_filter": self.rp_filter,
|
|
"proxy_arp": self.proxy_arp,
|
|
"proxy_arp_pvlan": self.proxy_arp_pvlan,
|
|
"proxy_ips": list(self.proxy_ips),
|
|
"config_verified": self.config_verified,
|
|
"config_observed_revision": self.config_observed_revision,
|
|
"config_observed_digest": self.config_observed_digest,
|
|
"readback": self.readback,
|
|
"rollback": self.rollback, "reason_code": self.reason_code,
|
|
"removed": self.removed,
|
|
}
|
|
|
|
|
|
def _id(value: Any, label: str = "标识") -> str:
|
|
if not isinstance(value, str) or _ID.fullmatch(value) is None:
|
|
raise VpcDataplaneError(f"{label}无效")
|
|
return value
|
|
|
|
|
|
def _host(value: Any) -> str:
|
|
if not isinstance(value, str) or _HOST.fullmatch(value) is None:
|
|
raise VpcDataplaneError("宿主机标识无效")
|
|
return value
|
|
|
|
|
|
def _key(value: Any) -> str:
|
|
if not isinstance(value, str) or _KEY.fullmatch(value) is None:
|
|
raise VpcDataplaneError("WireGuard 公钥无效")
|
|
try:
|
|
raw = base64.b64decode(value, validate=True)
|
|
except Exception as exc:
|
|
raise VpcDataplaneError("WireGuard 公钥无效") from exc
|
|
if len(raw) != 32 or not any(raw):
|
|
raise VpcDataplaneError("WireGuard 公钥无效")
|
|
return value
|
|
|
|
|
|
def _mac(value: Any, label: str = "私网 MAC") -> str:
|
|
if not isinstance(value, str) or _MAC.fullmatch(value) is None:
|
|
raise VpcDataplaneError(f"{label}无效")
|
|
octets = [int(x, 16) for x in value.split(":")]
|
|
if octets[0] & 1 or not any(octets):
|
|
raise VpcDataplaneError(f"{label}无效")
|
|
return value.lower()
|
|
|
|
|
|
def derive_private_mac(vpc_id: str, member_id: str) -> str:
|
|
"""Stable locally-administered unicast MAC for legacy plans without MAC."""
|
|
digest = hashlib.sha256(f"virt-cloud:vpc-private0:{vpc_id}:{member_id}".encode()).digest()
|
|
first = (digest[0] & 0xfc) | 0x02
|
|
return ":".join(f"{x:02x}" for x in bytes([first]) + digest[1:6])
|
|
|
|
|
|
def private_guest_routes(plan: VpcPlan, member: VpcMember) -> tuple[str, ...]:
|
|
"""Return the stable on-link VPC prefix installed inside one guest.
|
|
|
|
Host ACL, nftables and Fabric changes deliberately do not participate in
|
|
this projection. The host keeps exact member/service /32 routes, proxy
|
|
neighbours and destination sets; the guest needs only one link-scope VPC
|
|
prefix. Member and service-route churn therefore cannot force every
|
|
running sibling to consume a new boot-only NoCloud generation.
|
|
"""
|
|
try:
|
|
network = ipaddress.ip_network(plan.cidr, strict=True)
|
|
private_ip = ipaddress.ip_address(member.private_ip)
|
|
except (TypeError, ValueError) as exc:
|
|
raise VpcDataplaneError("VPC 来宾私网路由无效") from exc
|
|
if (not isinstance(network, ipaddress.IPv4Network)
|
|
or not 16 <= network.prefixlen <= 24
|
|
or not isinstance(private_ip, ipaddress.IPv4Address)
|
|
or private_ip not in network
|
|
or not any(network.subnet_of(block) for block in (
|
|
ipaddress.IPv4Network("10.0.0.0/8"),
|
|
ipaddress.IPv4Network("172.16.0.0/12"),
|
|
ipaddress.IPv4Network("192.168.0.0/16"),
|
|
))):
|
|
raise VpcDataplaneError("VPC 来宾私网路由无效")
|
|
return (str(network),)
|
|
|
|
|
|
def private_guest_network_generation(
|
|
member: VpcMember,
|
|
private_routes: Sequence[str],
|
|
) -> str:
|
|
"""Hash only the private network state that a guest consumes at boot."""
|
|
private_mac = _mac(member.private_mac)
|
|
try:
|
|
private_ip = ipaddress.ip_interface(f"{member.private_ip}/32")
|
|
except (TypeError, ValueError) as exc:
|
|
raise VpcDataplaneError("VPC 成员私网 IP 无效") from exc
|
|
if not isinstance(private_ip, ipaddress.IPv4Interface):
|
|
raise VpcDataplaneError("VPC 成员私网 IP 无效")
|
|
routes: list[ipaddress.IPv4Network] = []
|
|
if (not isinstance(private_routes, Sequence)
|
|
or isinstance(private_routes, (str, bytes))
|
|
or len(private_routes) != 1):
|
|
raise VpcDataplaneError("VPC 来宾私网路由无效")
|
|
for value in private_routes:
|
|
if not isinstance(value, str):
|
|
raise VpcDataplaneError("VPC 来宾私网路由无效")
|
|
try:
|
|
route = ipaddress.ip_network(value, strict=True)
|
|
except ValueError as exc:
|
|
raise VpcDataplaneError("VPC 来宾私网路由无效") from exc
|
|
if (not isinstance(route, ipaddress.IPv4Network)
|
|
or not 16 <= route.prefixlen <= 24
|
|
or private_ip.ip not in route
|
|
or not any(route.subnet_of(block) for block in (
|
|
ipaddress.IPv4Network("10.0.0.0/8"),
|
|
ipaddress.IPv4Network("172.16.0.0/12"),
|
|
ipaddress.IPv4Network("192.168.0.0/16"),
|
|
))):
|
|
raise VpcDataplaneError(
|
|
"VPC 来宾私网路由必须是包含成员地址的 IPv4 /16 到 /24")
|
|
routes.append(route)
|
|
payload = "\n".join((
|
|
"virt-cloud-private-guest-network-v2",
|
|
private_mac,
|
|
str(private_ip),
|
|
str(routes[0]),
|
|
)).encode("ascii")
|
|
return hashlib.sha256(payload).hexdigest()
|
|
|
|
|
|
def resource_names(vpc_id: str) -> VpcDataplaneResources:
|
|
vpc_id = _id(vpc_id, "VPC")
|
|
digest = hashlib.sha256(vpc_id.encode()).hexdigest()
|
|
token = digest[:11]
|
|
span = ROUTE_TABLE_MAX - ROUTE_TABLE_MIN + 1
|
|
# Each VPC owns two policy rules (private bridge and Fabric ingress).
|
|
# Allocate an aligned pair so ``base + 1`` can never be another VPC's
|
|
# base slot. The coordinator still checks collisions across records and
|
|
# the host transaction rejects a foreign rule already occupying either
|
|
# slot, but a single-slot hash would make those checks order-dependent.
|
|
rule_span = (RULE_PRIORITY_MAX - RULE_PRIORITY_MIN + 1) // 2
|
|
rule_base = RULE_PRIORITY_MIN + 2 * (int(digest[20:28], 16) % rule_span)
|
|
return VpcDataplaneResources(
|
|
token=token, bridge=BRIDGE_PREFIX + token,
|
|
nft_table=NFT_PREFIX + digest[:12],
|
|
route_table=ROUTE_TABLE_MIN + int(digest[12:20], 16) % span,
|
|
rule_priority=rule_base,
|
|
owner_file=f"/etc/virt-cloud/vpc-dataplane/{token}.owner",
|
|
state_file=f"/etc/virt-cloud/vpc-dataplane/{token}.state",
|
|
lock_file=f"/run/virt-cloud/vpc-{token}.lock",
|
|
transaction_dir=f"/etc/virt-cloud/.vpc-dataplane-{token}.txn",
|
|
)
|
|
|
|
|
|
def _expected_peer_tenant_ips(vpc: VpcPlan) -> dict[str, frozenset[str]]:
|
|
"""Return the exact /32 set a peer may route for this VPC.
|
|
|
|
The WireGuard peer list is a cryptokey-routing boundary. RFC1918 alone
|
|
is not an ownership check: an otherwise valid private /32 from another
|
|
VPC would create a cross-tenant route. The coordinator may pass a
|
|
trusted aggregate for a Fabric shared by several VPCs; the default here
|
|
covers a standalone plan and keeps direct parser callers fail-closed.
|
|
"""
|
|
members = {item.member_id: item for item in vpc.members}
|
|
allowed: dict[str, set[str]] = {host: set() for host in vpc.host_ids}
|
|
for item in vpc.members:
|
|
allowed.setdefault(item.host_id, set()).add(f"{item.private_ip}/32")
|
|
for route in vpc.routes:
|
|
target = (members.get(route.next_hop_member_id)
|
|
if route.next_hop_member_id is not None else None)
|
|
target_host = target.host_id if target is not None else route.next_hop_host_id
|
|
if target_host is not None:
|
|
allowed.setdefault(target_host, set()).add(route.destination_cidr)
|
|
return {host: frozenset(values) for host, values in allowed.items()}
|
|
|
|
|
|
def _parse_peers(
|
|
raw: Any,
|
|
host_ids: frozenset[str],
|
|
local: str,
|
|
expected_tenant_ips: Mapping[str, frozenset[str]] | None = None,
|
|
) -> tuple[VpcFabricPeer, ...]:
|
|
if raw is None:
|
|
raw = []
|
|
if not isinstance(raw, (list, tuple)) or len(raw) > 64:
|
|
raise VpcDataplaneError("Fabric peer 列表无效")
|
|
peers: list[VpcFabricPeer] = []
|
|
seen_hosts: set[str] = set()
|
|
seen_keys: set[str] = set()
|
|
seen_ips: set[str] = set()
|
|
seen_tenant_ips: set[str] = set()
|
|
for item in raw:
|
|
if not isinstance(item, Mapping) or set(item) - {"host_id", "public_key", "overlay_ip", "overlay_ip_cidr", "tenant_allowed_ips"}:
|
|
raise VpcDataplaneError("Fabric peer 字段无效")
|
|
host_id = _host(item.get("host_id"))
|
|
if host_id == local or host_id not in host_ids or host_id in seen_hosts:
|
|
raise VpcDataplaneError("Fabric peer 宿主机无效或重复")
|
|
public_key = _key(item.get("public_key"))
|
|
if public_key in seen_keys:
|
|
raise VpcDataplaneError("Fabric peer 公钥重复")
|
|
raw_ip = item.get("overlay_ip", item.get("overlay_ip_cidr"))
|
|
try:
|
|
interface = ipaddress.ip_interface(raw_ip)
|
|
except (ValueError, TypeError) as exc:
|
|
raise VpcDataplaneError("Fabric peer overlay 地址无效") from exc
|
|
fabric_overlay = ipaddress.IPv4Network("100.64.0.0/10")
|
|
if (not isinstance(interface, ipaddress.IPv4Interface)
|
|
or interface.network.prefixlen != 32
|
|
or interface.ip not in fabric_overlay
|
|
or interface.ip.is_loopback
|
|
or interface.ip.is_link_local):
|
|
raise VpcDataplaneError("Fabric peer overlay 地址无效")
|
|
overlay_ip = str(interface.ip)
|
|
if overlay_ip in seen_ips:
|
|
raise VpcDataplaneError("Fabric peer overlay 地址重复")
|
|
raw_tenant = item.get("tenant_allowed_ips", ())
|
|
if not isinstance(raw_tenant, (list, tuple)) or len(raw_tenant) > 4096:
|
|
raise VpcDataplaneError("Fabric peer tenant AllowedIPs 无效")
|
|
tenant: set[str] = set()
|
|
for raw_ip in raw_tenant:
|
|
if not isinstance(raw_ip, str) or not raw_ip.endswith("/32"):
|
|
raise VpcDataplaneError("Fabric peer tenant AllowedIPs 必须是 /32")
|
|
try:
|
|
network = ipaddress.ip_network(raw_ip, strict=True)
|
|
except ValueError as exc:
|
|
raise VpcDataplaneError("Fabric peer tenant AllowedIPs 无效") from exc
|
|
if network.prefixlen != 32:
|
|
raise VpcDataplaneError("Fabric peer tenant AllowedIPs 必须是 /32")
|
|
# Tenant routes are panel-private RFC1918 /32s. In particular,
|
|
# never accept a public prefix or a Fabric overlay address from
|
|
# an otherwise authenticated payload: that would turn a peer's
|
|
# cryptokey routing table into a host-wide egress route.
|
|
private_ranges = (
|
|
ipaddress.IPv4Network("10.0.0.0/8"),
|
|
ipaddress.IPv4Network("172.16.0.0/12"),
|
|
ipaddress.IPv4Network("192.168.0.0/16"),
|
|
)
|
|
if not any(network.subnet_of(item) for item in private_ranges):
|
|
raise VpcDataplaneError("Fabric peer tenant AllowedIPs 必须位于 RFC1918 私网")
|
|
tenant_ip = str(network)
|
|
if tenant_ip in seen_tenant_ips:
|
|
raise VpcDataplaneError("Fabric peer tenant AllowedIPs 重复", 409,
|
|
"vpc_tenant_allowed_ip_conflict")
|
|
seen_tenant_ips.add(tenant_ip)
|
|
tenant.add(tenant_ip)
|
|
if expected_tenant_ips is not None:
|
|
expected = set(expected_tenant_ips.get(host_id, frozenset()))
|
|
if tenant != expected:
|
|
raise VpcDataplaneError(
|
|
"Fabric peer tenant AllowedIPs 超出 VPC 所有权闭集", 409,
|
|
"vpc_tenant_allowed_ip_unowned")
|
|
seen_hosts.add(host_id); seen_keys.add(public_key); seen_ips.add(overlay_ip)
|
|
peers.append(VpcFabricPeer(host_id, public_key, overlay_ip,
|
|
tuple(sorted(tenant))))
|
|
return tuple(sorted(peers, key=lambda p: p.host_id))
|
|
|
|
|
|
def parse_vpc_dataplane_plan(
|
|
value: Mapping[str, Any], host_id: str, *,
|
|
fabric_peers: Sequence[Mapping[str, Any]] | None = None,
|
|
trusted_tenant_allowed_ips: Mapping[str, Sequence[str]] | None = None,
|
|
fabric_peer_keys: Sequence[str] | None = None,
|
|
fabric_plan_digest: str | None = None,
|
|
) -> VpcDataplanePlan:
|
|
"""Parse existing VPC DTO plus an optional engine-only Fabric snapshot."""
|
|
if not isinstance(value, Mapping):
|
|
raise VpcDataplaneError("VPC 数据面计划必须是对象")
|
|
host_id = _host(host_id)
|
|
fabric_plan_digest = _fabric_plan_digest(fabric_plan_digest)
|
|
payload = dict(value)
|
|
embedded = payload.pop("fabric_peers", None)
|
|
if embedded is None:
|
|
embedded = payload.pop("peers", None)
|
|
else:
|
|
payload.pop("peers", None)
|
|
try:
|
|
vpc = parse_vpc_plan(payload)
|
|
except VpcError as exc:
|
|
raise VpcDataplaneError(str(exc), exc.status, exc.error_code) from exc
|
|
if host_id not in vpc.host_ids:
|
|
raise VpcDataplaneError("VPC 宿主机不在参与列表中", 409, "vpc_host_not_member")
|
|
members: list[VpcMember] = []
|
|
seen_macs: set[str] = set()
|
|
for member in vpc.members:
|
|
private_mac = _mac(member.private_mac) if member.private_mac else derive_private_mac(vpc.vpc_id, member.member_id)
|
|
if private_mac in seen_macs:
|
|
raise VpcDataplaneError("VPC 成员私网 MAC 重复", 409, "vpc_private_mac_conflict")
|
|
seen_macs.add(private_mac)
|
|
members.append(VpcMember(member.member_id, member.instance_uuid,
|
|
member.host_id, member.private_ip,
|
|
member.subnet_id, private_mac))
|
|
if trusted_tenant_allowed_ips is None:
|
|
expected_tenant_ips = _expected_peer_tenant_ips(vpc)
|
|
else:
|
|
if not isinstance(trusted_tenant_allowed_ips, Mapping):
|
|
raise VpcDataplaneError("Fabric peer tenant 所有权闭集无效")
|
|
expected_tenant_ips = {}
|
|
remote_hosts = vpc.host_ids - {host_id}
|
|
if set(trusted_tenant_allowed_ips) - remote_hosts:
|
|
raise VpcDataplaneError("Fabric peer tenant 所有权宿主机无效")
|
|
for peer_host, values in trusted_tenant_allowed_ips.items():
|
|
if not isinstance(peer_host, str) or peer_host not in remote_hosts:
|
|
raise VpcDataplaneError("Fabric peer tenant 所有权宿主机无效")
|
|
if not isinstance(values, (list, tuple, set, frozenset)):
|
|
raise VpcDataplaneError("Fabric peer tenant 所有权闭集无效")
|
|
canonical: set[str] = set()
|
|
for value in values:
|
|
if not isinstance(value, str):
|
|
raise VpcDataplaneError("Fabric peer tenant 所有权闭集无效")
|
|
try:
|
|
network = ipaddress.ip_network(value, strict=True)
|
|
except ValueError as exc:
|
|
raise VpcDataplaneError("Fabric peer tenant 所有权闭集无效") from exc
|
|
if (not isinstance(network, ipaddress.IPv4Network)
|
|
or network.prefixlen != 32
|
|
or not any(network.subnet_of(block) for block in (
|
|
ipaddress.IPv4Network("10.0.0.0/8"),
|
|
ipaddress.IPv4Network("172.16.0.0/12"),
|
|
ipaddress.IPv4Network("192.168.0.0/16")))):
|
|
raise VpcDataplaneError("Fabric peer tenant 所有权闭集无效")
|
|
canonical.add(str(network))
|
|
expected_tenant_ips[peer_host] = frozenset(canonical)
|
|
peers = _parse_peers(
|
|
embedded if fabric_peers is None else fabric_peers,
|
|
vpc.host_ids, host_id, expected_tenant_ips)
|
|
remote_hosts = {m.host_id for m in members if m.host_id != host_id}
|
|
if peers and remote_hosts - {p.host_id for p in peers}:
|
|
raise VpcDataplaneError("跨宿主机成员缺少 Fabric peer", 409, "vpc_fabric_peer_missing")
|
|
authoritative_keys: frozenset[str] | None = None
|
|
if fabric_peer_keys is not None:
|
|
if not isinstance(fabric_peer_keys, (list, tuple, set, frozenset)):
|
|
raise VpcDataplaneError("Fabric peer 公钥闭集无效", 409,
|
|
"vpc_fabric_peer_set_invalid")
|
|
if len(fabric_peer_keys) > 64:
|
|
raise VpcDataplaneError("Fabric peer 公钥闭集过大", 409,
|
|
"vpc_fabric_peer_set_invalid")
|
|
canonical_keys: set[str] = set()
|
|
for raw_key in fabric_peer_keys:
|
|
key = _key(raw_key)
|
|
if key in canonical_keys:
|
|
raise VpcDataplaneError("Fabric peer 公钥重复", 409,
|
|
"vpc_fabric_peer_set_invalid")
|
|
canonical_keys.add(key)
|
|
if not {peer.public_key for peer in peers}.issubset(canonical_keys):
|
|
raise VpcDataplaneError("VPC peer 不属于 Fabric 公钥闭集", 409,
|
|
"vpc_fabric_peer_set_invalid")
|
|
authoritative_keys = frozenset(canonical_keys)
|
|
return VpcDataplanePlan(
|
|
vpc, host_id, tuple(members), peers, resource_names(vpc.vpc_id),
|
|
fabric_plan_digest, authoritative_keys)
|
|
|
|
|
|
def _routes(plan: VpcDataplanePlan) -> tuple[dict[str, Any], ...]:
|
|
members = {m.member_id: m for m in plan.members}
|
|
result: list[dict[str, Any]] = []
|
|
seen: set[str] = set()
|
|
for member in plan.members:
|
|
destination = f"{member.private_ip}/32"
|
|
seen.add(destination)
|
|
result.append({"destination_cidr": destination,
|
|
"device": plan.resources.bridge if member.host_id == plan.host_id else FABRIC_INTERFACE,
|
|
"table": plan.resources.route_table,
|
|
"owner": DATAPLANE_OWNER_MARKER,
|
|
"metric": 100,
|
|
"via": None,
|
|
"member_id": member.member_id})
|
|
for route in plan.vpc.routes:
|
|
destination = route.destination_cidr
|
|
network = ipaddress.ip_network(destination)
|
|
if network.prefixlen != 32:
|
|
raise VpcDataplaneError(
|
|
"VPC 数据面只支持成员精确 /32 路由", 409,
|
|
"vpc_route_not_exact")
|
|
if any(ipaddress.IPv4Address(m.private_ip) in network
|
|
and destination != f"{m.private_ip}/32" for m in plan.members):
|
|
raise VpcDataplaneError("VPC route 不能覆盖成员精确 /32", 409, "vpc_route_member_overlap")
|
|
if destination in seen:
|
|
raise VpcDataplaneError("VPC route 与成员地址重复", 409, "vpc_route_conflict")
|
|
next_host = route.next_hop_host_id
|
|
if route.next_hop_member_id is not None:
|
|
target = members.get(route.next_hop_member_id)
|
|
if target is None:
|
|
raise VpcDataplaneError("VPC route next hop member 不存在", 409)
|
|
next_host = target.host_id
|
|
if next_host is None:
|
|
raise VpcDataplaneError("VPC route next hop 缺失")
|
|
target_members = [item for item in plan.members
|
|
if item.host_id == next_host]
|
|
if not target_members:
|
|
raise VpcDataplaneError("VPC route next hop 宿主机没有成员", 409,
|
|
"vpc_route_next_hop_missing")
|
|
# A host-only next hop is ambiguous when several VMs share that host.
|
|
# Require an explicit member in that case; using a bare bridge route
|
|
# would ARP for the service address and could leak traffic to an
|
|
# unrelated guest.
|
|
if route.next_hop_member_id is None and len(target_members) != 1:
|
|
raise VpcDataplaneError("VPC route 宿主机 next hop 不明确", 409,
|
|
"vpc_route_next_hop_ambiguous")
|
|
target = (members.get(route.next_hop_member_id)
|
|
if route.next_hop_member_id is not None else target_members[0])
|
|
if target is None:
|
|
raise VpcDataplaneError("VPC route next hop member 不存在", 409,
|
|
"vpc_route_next_hop_missing")
|
|
is_local = next_host == plan.host_id
|
|
if not is_local and next_host not in {peer.host_id for peer in plan.peers}:
|
|
raise VpcDataplaneError("VPC route next hop 缺少 Fabric peer", 409,
|
|
"vpc_fabric_peer_missing")
|
|
seen.add(destination)
|
|
result.append({"destination_cidr": destination,
|
|
"device": plan.resources.bridge if is_local else FABRIC_INTERFACE,
|
|
"table": plan.resources.route_table,
|
|
"owner": DATAPLANE_OWNER_MARKER,
|
|
# A local host must forward the service /32 through
|
|
# the selected VM. Across the overlay the destination
|
|
# itself is a WireGuard AllowedIP, so no `via` is used.
|
|
"via": target.private_ip if is_local else None,
|
|
"metric": route.metric,
|
|
"route_id": route.route_id})
|
|
return tuple(result)
|
|
|
|
|
|
def _q(value: Any) -> str:
|
|
return shlex.quote(str(value))
|
|
|
|
|
|
def _nft_proto(acl: Any) -> str:
|
|
if acl.protocol == "any":
|
|
return ""
|
|
if acl.protocol == "icmp":
|
|
return " ip protocol icmp"
|
|
suffix = f" ip protocol {acl.protocol}"
|
|
if acl.port_from is not None:
|
|
ports = str(acl.port_from) if acl.port_from == acl.port_to else f"{acl.port_from}-{acl.port_to}"
|
|
suffix += f" {acl.protocol} dport {ports}"
|
|
return suffix
|
|
|
|
|
|
def _nft_marker(plan: VpcDataplanePlan) -> str:
|
|
"""Return a bounded ownership marker safe for nft comment userdata.
|
|
|
|
libnftnl limits rule comments to 128 bytes. VPC/host identifiers are
|
|
intentionally allowed to be long, so embedding them and the full digest
|
|
in every rule would make otherwise valid plans fail at ``nft -c``. The
|
|
state file keeps the full identifiers; comments only need a collision
|
|
resistant, deterministic token for ownership/readback.
|
|
"""
|
|
seed = (f"{plan.vpc.vpc_id}:{plan.host_id}:{plan.desired_revision}:"
|
|
f"{plan.desired_digest}").encode()
|
|
token = hashlib.sha256(seed).hexdigest()[:16]
|
|
return f"{DATAPLANE_OWNER_MARKER}:{plan.resources.token}:{token}"
|
|
|
|
|
|
def render_vpc_nft_rules(plan: VpcDataplanePlan) -> str:
|
|
"""Render scoped VPC ACL/anti-spoof rules for routed and L2 paths.
|
|
|
|
The base chains use ``policy accept`` deliberately. A VPC table must not
|
|
become a host-wide firewall that drops public bridges or unrelated Fabric
|
|
traffic; only packets carrying this VPC's member addresses are subject to
|
|
its default deny. A separate bridge-family table covers same-host VM to
|
|
VM forwarding and ARP, which do not reliably traverse an inet forward
|
|
hook.
|
|
"""
|
|
r = plan.resources
|
|
marker = _nft_marker(plan)
|
|
dataplane_routes = _routes(plan)
|
|
member_cidrs = {f"{m.private_ip}/32" for m in plan.members}
|
|
local_member_cidrs = {f"{m.private_ip}/32" for m in plan.local_members}
|
|
# Custom VPC routes are exact /32s too. Keep source sets restricted to
|
|
# actual member addresses, while destination sets include routed service
|
|
# addresses; otherwise the isolation rules would silently drop a valid
|
|
# route before the ACL/default-deny decision.
|
|
destination_cidrs = member_cidrs | {
|
|
str(item["destination_cidr"]) for item in dataplane_routes
|
|
}
|
|
# A packet entering a local tap may be routed either to another local
|
|
# tap or out through wg-fabric0 to a remote member/service /32. The
|
|
# bridge-family unknown-destination guard therefore needs the complete
|
|
# VPC destination set; restricting it to local routes would drop every
|
|
# same-VPC cross-host flow before the inet routing policy can run.
|
|
local_destination_cidrs = set(destination_cidrs)
|
|
# Use explicit proxy-neighbour entries for the host's exact /32 routes to
|
|
# guest addresses.
|
|
# The generic kernel proxy responder stays disabled, so a nftables reload
|
|
# or a short ordering window cannot make the host answer for arbitrary
|
|
# addresses. proxy_arp_pvlan is enabled only for the same-bridge
|
|
# next-hop case and is still bounded by the exact neighbour set and ARP
|
|
# family output chain below.
|
|
proxy_ips = sorted(
|
|
str(ipaddress.ip_network(item["destination_cidr"], strict=True).network_address)
|
|
for item in dataplane_routes)
|
|
members = ", ".join(sorted(member_cidrs))
|
|
local_members = ", ".join(sorted(local_member_cidrs))
|
|
destinations = ", ".join(sorted(destination_cidrs))
|
|
local_destinations = ", ".join(sorted(local_destination_cidrs))
|
|
proxy_destinations = ", ".join(proxy_ips)
|
|
local_sources = ", ".join(
|
|
f"{m.private_mac} . {m.private_ip}" for m in plan.local_members)
|
|
|
|
def nft_set(name: str, type_name: str, elements: str, *, interval: bool = False) -> str:
|
|
flags = " flags interval;" if interval else ""
|
|
body = f" set {name} {{ type {type_name};{flags}"
|
|
if elements:
|
|
body += f" elements = {{ {elements} }};"
|
|
return body + " }"
|
|
|
|
lines = [
|
|
f"# {DATAPLANE_OWNER_MARKER}", f"# vpc-token={r.token}",
|
|
f"# vpc={plan.vpc.vpc_id} revision={plan.desired_revision} digest={plan.desired_digest}",
|
|
f"table inet {r.nft_table} {{",
|
|
nft_set("member_ips", "ipv4_addr", members, interval=True),
|
|
nft_set("local_member_ips", "ipv4_addr", local_members, interval=True),
|
|
nft_set("vpc_dest_ips", "ipv4_addr", destinations, interval=True),
|
|
nft_set("local_member_sources", "ether_addr . ipv4_addr", local_sources),
|
|
" chain forward { type filter hook forward priority -190; policy accept;",
|
|
f' iifname "{r.bridge}" ip saddr != @local_member_ips drop comment "{marker}:anti-spoof-local-ip"',
|
|
f' iifname "{r.bridge}" ip6 saddr ::/0 drop comment "{marker}:ipv6-drop-bridge"',
|
|
# A source address is not an ingress identity. A packet arriving on
|
|
# the public/management side must never be able to spoof a member /32
|
|
# and use the VPC policy route to reach a private destination. Close
|
|
# both egress legs by interface before the source-address checks.
|
|
# The whole VPC CIDR is guarded, including currently unallocated
|
|
# addresses. A missing /32 route must therefore fail closed instead
|
|
# of falling through to the host's public/default route.
|
|
f' oifname "{r.bridge}" ip daddr {plan.vpc.cidr} iifname != "{r.bridge}" iifname != "{FABRIC_INTERFACE}" drop comment "{marker}:anti-spoof-public-to-vpc-bridge"',
|
|
f' oifname "{FABRIC_INTERFACE}" ip daddr {plan.vpc.cidr} iifname != "{r.bridge}" iifname != "{FABRIC_INTERFACE}" drop comment "{marker}:anti-spoof-public-to-vpc-fabric"',
|
|
f' iifname "{r.bridge}" ip daddr {plan.vpc.cidr} oifname != "{r.bridge}" oifname != "{FABRIC_INTERFACE}" drop comment "{marker}:anti-spoof-bridge-vpc-to-foreign"',
|
|
f' iifname "{FABRIC_INTERFACE}" ip daddr {plan.vpc.cidr} oifname != "{r.bridge}" oifname != "{FABRIC_INTERFACE}" drop comment "{marker}:anti-spoof-fabric-vpc-to-foreign"',
|
|
f' oifname "{r.bridge}" ip daddr @vpc_dest_ips iifname != "{r.bridge}" iifname != "{FABRIC_INTERFACE}" drop comment "{marker}:anti-spoof-public-to-bridge"',
|
|
f' oifname "{FABRIC_INTERFACE}" ip daddr @vpc_dest_ips iifname != "{r.bridge}" iifname != "{FABRIC_INTERFACE}" drop comment "{marker}:anti-spoof-public-to-fabric"',
|
|
# A VPC destination must also leave through one of the two owned legs.
|
|
# If a policy route disappears or a foreign/default route wins, the
|
|
# packet must be dropped rather than sent to a public or management
|
|
# interface. This is deliberately independent of source-IP checks.
|
|
f' iifname "{r.bridge}" ip daddr @vpc_dest_ips oifname != "{r.bridge}" oifname != "{FABRIC_INTERFACE}" drop comment "{marker}:anti-spoof-bridge-to-foreign"',
|
|
f' iifname "{FABRIC_INTERFACE}" ip daddr @vpc_dest_ips oifname != "{r.bridge}" oifname != "{FABRIC_INTERFACE}" drop comment "{marker}:anti-spoof-fabric-to-foreign"',
|
|
f' oifname "{r.bridge}" ip daddr @vpc_dest_ips ip saddr != @member_ips drop comment "{marker}:anti-spoof-egress"',
|
|
f' iifname "{FABRIC_INTERFACE}" ip daddr @vpc_dest_ips ip saddr != @member_ips drop comment "{marker}:anti-spoof-wg-ingress"',
|
|
f' oifname "{FABRIC_INTERFACE}" ip saddr @member_ips ip daddr != @vpc_dest_ips drop comment "{marker}:anti-spoof-wg-egress"',
|
|
f' iifname "{FABRIC_INTERFACE}" ip daddr @vpc_dest_ips ip saddr != @member_ips drop comment "{marker}:anti-spoof-wg-source"',
|
|
f' iifname "{r.bridge}" ip saddr @local_member_ips ip daddr != @vpc_dest_ips drop comment "{marker}:member-egress-isolation"',
|
|
f' iifname "{FABRIC_INTERFACE}" ip saddr @member_ips ip daddr != @vpc_dest_ips drop comment "{marker}:member-fabric-isolation"',
|
|
]
|
|
# ACL priority is first-match order. Direction is relative to the VPC
|
|
# boundary: ingress is wg -> private bridge; egress is bridge -> wg.
|
|
# Same-host members have no wg leg, so their source/destination tuple is
|
|
# enforced on the bridge path as well.
|
|
for acl in plan.vpc.acls:
|
|
action = "accept" if acl.action == "allow" else "drop"
|
|
clause = (f" ip saddr {acl.source_cidr} ip daddr {acl.destination_cidr}"
|
|
f"{_nft_proto(acl)} {action} comment \"{marker}:acl:{acl.acl_id}\"")
|
|
if acl.direction == "ingress":
|
|
lines.append(f' iifname "{FABRIC_INTERFACE}" oifname "{r.bridge}"{clause}')
|
|
else:
|
|
lines.append(f' iifname "{r.bridge}" oifname "{FABRIC_INTERFACE}"{clause}')
|
|
lines.append(f' iifname "{r.bridge}" oifname "{r.bridge}"{clause}')
|
|
# Scope default-deny to member-to-member VPC traffic. The following
|
|
# interface-wide accepts preserve unrelated host/public forwarding.
|
|
lines += [
|
|
f' iifname "{FABRIC_INTERFACE}" oifname "{r.bridge}" ip saddr @member_ips ip daddr @vpc_dest_ips counter drop comment "{marker}:default-deny"',
|
|
f' iifname "{r.bridge}" oifname "{FABRIC_INTERFACE}" ip saddr @member_ips ip daddr @vpc_dest_ips counter drop comment "{marker}:default-deny"',
|
|
f' iifname "{r.bridge}" oifname "{r.bridge}" ip saddr @member_ips ip daddr @vpc_dest_ips counter drop comment "{marker}:default-deny"',
|
|
f' iifname "{FABRIC_INTERFACE}" oifname "{FABRIC_INTERFACE}" ip saddr @member_ips ip daddr @vpc_dest_ips counter drop comment "{marker}:wg-transit-deny"',
|
|
f' iifname "{FABRIC_INTERFACE}" oifname "{r.bridge}" ip6 saddr ::/0 drop comment "{marker}:ipv6-drop-fabric"',
|
|
f' iifname "{r.bridge}" accept comment "{marker}:scope-bridge-in"',
|
|
f' oifname "{r.bridge}" accept comment "{marker}:scope-bridge-out"',
|
|
f' iifname "{FABRIC_INTERFACE}" accept comment "{marker}:scope-fabric-in"',
|
|
f' oifname "{FABRIC_INTERFACE}" accept comment "{marker}:scope-fabric-out"',
|
|
" }",
|
|
" chain input { type filter hook input priority -190; policy accept;",
|
|
f' iifname "{r.bridge}" ip saddr @member_ips drop comment "{marker}:host-isolation"',
|
|
f' iifname "{FABRIC_INTERFACE}" ip daddr @vpc_dest_ips drop comment "{marker}:host-isolation-wg"',
|
|
f' iifname "{FABRIC_INTERFACE}" ip saddr @member_ips ip daddr != @vpc_dest_ips drop comment "{marker}:host-isolation-wg-source"',
|
|
f' iifname "{r.bridge}" ip6 saddr ::/0 drop comment "{marker}:ipv6-input-drop-bridge"',
|
|
f' iifname "{FABRIC_INTERFACE}" ip6 saddr ::/0 drop comment "{marker}:ipv6-input-drop-fabric"',
|
|
" }",
|
|
" chain output { type filter hook output priority -190; policy accept;",
|
|
f' oifname "{r.bridge}" ip daddr @vpc_dest_ips drop comment "{marker}:host-isolation-out"',
|
|
# Host-originated packets must not be able to bind a tenant /32 and
|
|
# bypass the private egress boundary. Keep this leg unconditional;
|
|
# the source address is not an ingress identity.
|
|
f' oifname "{FABRIC_INTERFACE}" ip daddr @vpc_dest_ips drop comment "{marker}:host-isolation-out-wg"',
|
|
f' oifname "{r.bridge}" ip daddr {plan.vpc.cidr} drop comment "{marker}:host-isolation-vpc-out"',
|
|
f' oifname "{FABRIC_INTERFACE}" ip daddr {plan.vpc.cidr} drop comment "{marker}:host-isolation-vpc-out-wg"',
|
|
f' oifname "{r.bridge}" ip6 daddr ::/0 drop comment "{marker}:ipv6-output-drop-bridge"',
|
|
" }", "}", "",
|
|
]
|
|
|
|
# Bridge-family rules are emitted only when this host has a local VM. An
|
|
# empty VPC still gets the routed table above, while no empty nft set is
|
|
# needed for the L2/ARP path.
|
|
if plan.local_members:
|
|
lines += [
|
|
f"table bridge {r.nft_table} {{",
|
|
nft_set("local_member_ips", "ipv4_addr", local_members, interval=True),
|
|
nft_set("local_dest_ips", "ipv4_addr", local_destinations, interval=True),
|
|
nft_set("local_member_sources", "ether_addr . ipv4_addr", local_sources),
|
|
" chain forward { type filter hook forward priority -190; policy accept;",
|
|
f' meta ibrname "{r.bridge}" ether type ip ether saddr . ip saddr != @local_member_sources drop comment "{marker}:bridge-anti-spoof-ingress"',
|
|
f' meta ibrname "{r.bridge}" ether type arp ether saddr != arp saddr ether drop comment "{marker}:bridge-anti-spoof-arp-mac"',
|
|
f' meta ibrname "{r.bridge}" ether type arp arp saddr ether . arp saddr ip != @local_member_sources drop comment "{marker}:bridge-anti-spoof-arp"',
|
|
f' meta ibrname "{r.bridge}" ether type ip ip saddr != @local_member_ips drop comment "{marker}:bridge-anti-spoof-ip"',
|
|
f' meta ibrname "{r.bridge}" ether type arp arp saddr ip != @local_member_ips drop comment "{marker}:bridge-anti-spoof-arp-ip"',
|
|
# A stable VPC-prefix route lets a guest ask for every address in
|
|
# that prefix. Only exact member/service destinations may reach
|
|
# another tap or the host proxy responder; unknown ARP targets are
|
|
# dropped before they become a bridge-wide scan/flood primitive.
|
|
f' meta ibrname "{r.bridge}" ether type arp arp operation request arp daddr ip != @local_dest_ips drop comment "{marker}:bridge-unknown-arp-target-drop"',
|
|
# This is an L3 /32 fabric. Do not let broadcast or multicast
|
|
# discovery turn the private bridge back into a shared L2 LAN.
|
|
f' meta ibrname "{r.bridge}" ip daddr 224.0.0.0/4 drop comment "{marker}:bridge-multicast-drop"',
|
|
f' meta ibrname "{r.bridge}" ip daddr 255.255.255.255 drop comment "{marker}:bridge-limited-broadcast-drop"',
|
|
f' meta ibrname "{r.bridge}" ip daddr {ipaddress.ip_network(plan.vpc.cidr).broadcast_address} drop comment "{marker}:bridge-vpc-broadcast-drop"',
|
|
f' meta ibrname "{r.bridge}" ip saddr @local_member_ips ip daddr != @local_dest_ips drop comment "{marker}:bridge-unknown-destination-drop"',
|
|
]
|
|
for acl in plan.vpc.acls:
|
|
action = "accept" if acl.action == "allow" else "drop"
|
|
clause = (f" ip saddr {acl.source_cidr} ip daddr {acl.destination_cidr}"
|
|
f"{_nft_proto(acl)} {action} comment \"{marker}:acl:{acl.acl_id}\"")
|
|
lines.append(f' meta ibrname "{r.bridge}" meta obrname "{r.bridge}"{clause}')
|
|
lines += [
|
|
f' meta ibrname "{r.bridge}" meta obrname "{r.bridge}" ip saddr @local_member_ips ip daddr @local_dest_ips counter drop comment "{marker}:bridge-default-deny"',
|
|
f' meta ibrname "{r.bridge}" ether type arp accept comment "{marker}:bridge-arp-accept"',
|
|
f' meta ibrname "{r.bridge}" ether type ip accept comment "{marker}:bridge-scope-accept"',
|
|
f' meta ibrname "{r.bridge}" counter drop comment "{marker}:bridge-non-ip-drop"',
|
|
" }",
|
|
" chain input { type filter hook input priority -190; policy accept;",
|
|
f' meta ibrname "{r.bridge}" ether type arp ether saddr != arp saddr ether drop comment "{marker}:bridge-anti-spoof-arp-input-mac"',
|
|
f' meta ibrname "{r.bridge}" ether type ip ether saddr . ip saddr != @local_member_sources drop comment "{marker}:bridge-anti-spoof-input-ip"',
|
|
f' meta ibrname "{r.bridge}" ether type arp arp saddr ether . arp saddr ip != @local_member_sources drop comment "{marker}:bridge-anti-spoof-input"',
|
|
f' meta ibrname "{r.bridge}" ether type ip6 drop comment "{marker}:bridge-ipv6-input-drop"',
|
|
f' meta ibrname "{r.bridge}" ether type arp accept comment "{marker}:bridge-arp-input-accept"',
|
|
# Proxy ARP makes remote /32 packets arrive at the bridge MAC and
|
|
# then enter the host routing path. Source/MAC checks above have
|
|
# already authenticated the guest, so IPv4 must be accepted here
|
|
# for the inet forward hook to route it to wg-fabric0.
|
|
f' meta ibrname "{r.bridge}" ether type ip accept comment "{marker}:bridge-ip-input-accept"',
|
|
f' meta ibrname "{r.bridge}" counter drop comment "{marker}:bridge-input-non-ip-drop"',
|
|
" }",
|
|
"}", "",
|
|
]
|
|
lines += [
|
|
f"table arp {r.nft_table} {{",
|
|
nft_set("proxy_ips", "ipv4_addr", proxy_destinations, interval=True),
|
|
" chain output { type filter hook output priority -190; policy accept;",
|
|
f' oifname "{r.bridge}" arp operation reply arp saddr ip != @proxy_ips drop comment "{marker}:arp-proxy-output"',
|
|
" }",
|
|
"}", "",
|
|
]
|
|
return "\n".join(lines)
|
|
|
|
|
|
def _here(path: str, content: str) -> str:
|
|
marker = "__VC_VPC_" + hashlib.sha256(content.encode()).hexdigest()[:16]
|
|
return f"cat > {_q(path)} <<'{marker}'\n{content}{'' if content.endswith(chr(10)) else chr(10)}{marker}\n"
|
|
|
|
|
|
def _firewall_mode(value: str) -> str:
|
|
if value not in FIREWALL_MODES:
|
|
raise VpcDataplaneError(
|
|
"宿主机防火墙部署模式不支持", 409,
|
|
"vpc_dataplane_firewall_mode_conflict")
|
|
return value
|
|
|
|
|
|
def _state(
|
|
plan: VpcDataplanePlan,
|
|
routes: Sequence[Mapping[str, Any]],
|
|
firewall_mode: str = FIREWALL_MODE_NATIVE,
|
|
) -> str:
|
|
r = plan.resources
|
|
firewall_mode = _firewall_mode(firewall_mode)
|
|
lines = [f"owner|{DATAPLANE_OWNER_MARKER}", f"vpc|{plan.vpc.vpc_id}",
|
|
f"cidr|{plan.vpc.cidr}", f"fabric|{plan.vpc.fabric_id}",
|
|
f"host|{plan.host_id}", f"revision|{plan.desired_revision}",
|
|
f"digest|{plan.desired_digest}", f"bridge|{r.bridge}",
|
|
f"table|{r.route_table}", f"priority|{r.rule_priority}",
|
|
f"firewall_mode|{firewall_mode}"]
|
|
for route in routes:
|
|
lines.append("route|{destination}|{device}|{table}|{via}|{metric}".format(
|
|
destination=route["destination_cidr"], device=route["device"],
|
|
table=route["table"], via=route.get("via") or "-",
|
|
metric=route.get("metric", 100)))
|
|
for peer in plan.peers:
|
|
ips = sorted(set(
|
|
[m.private_ip + "/32" for m in plan.remote_members
|
|
if m.host_id == peer.host_id]
|
|
+ list(peer.tenant_allowed_ips)))
|
|
if ips:
|
|
lines.append(f"peer|{peer.host_id}|{peer.public_key}|{','.join(ips)}")
|
|
return "\n".join(lines) + "\n"
|
|
|
|
|
|
def _allowed(plan: VpcDataplanePlan, peer: VpcFabricPeer) -> str:
|
|
members = {m.member_id: m for m in plan.members}
|
|
ips = {m.private_ip + "/32" for m in plan.remote_members
|
|
if m.host_id == peer.host_id}
|
|
for route in plan.vpc.routes:
|
|
target = (members.get(route.next_hop_member_id)
|
|
if route.next_hop_member_id is not None else None)
|
|
target_host = target.host_id if target is not None else route.next_hop_host_id
|
|
if target_host == peer.host_id:
|
|
ips.add(route.destination_cidr)
|
|
return ",".join([peer.overlay_cidr] + sorted(ips | set(peer.tenant_allowed_ips)))
|
|
|
|
|
|
def render_apply_script(plan: VpcDataplanePlan) -> str:
|
|
"""Generate a shell transaction with panel ownership and rollback markers."""
|
|
routes = _routes(plan)
|
|
if plan.remote_members and not plan.peers:
|
|
raise VpcDataplaneError("跨宿主机 VPC 缺少 Fabric peer", 409, "vpc_fabric_peer_missing")
|
|
r = plan.resources
|
|
nft = render_vpc_nft_rules(plan)
|
|
state = _state(plan, routes)
|
|
route_specs = "\\n".join(
|
|
f"{x['destination_cidr']}|{x['device']}|{x['table']}" for x in routes)
|
|
peer_specs = "\\n".join(
|
|
f"{p.host_id}|{p.public_key}|{_allowed(plan,p)}" for p in plan.peers)
|
|
route_checks = "\\n".join(
|
|
f'''actual=$("$IP" -o -4 route show table {_q(x["table"])} {_q(x["destination_cidr"])} 2>/dev/null || true)
|
|
if printf '%s\\n' "$actual" | grep -Fq "dev {_q(x["device"])}" && printf '%s\\n' "$actual" | grep -Eq 'proto 186([[:space:]]|$)'; then
|
|
printf '%s\\n' "__VC_VPC_ROUTE__={x["destination_cidr"]}|{x["device"]}|{x["table"]}|owned"
|
|
else
|
|
printf '%s\\n' "__VC_VPC_ROUTE__={x["destination_cidr"]}|{x["device"]}|{x["table"]}|missing"
|
|
fi'''
|
|
for x in routes)
|
|
peer_checks = "\\n".join(
|
|
f'''actual=$("$WG" show "$WG_IFACE" allowed-ips 2>/dev/null | awk -v wanted={_q(p.public_key)} '$1==wanted {{print $2; exit}}' || true)
|
|
tenant={_q(','.join(_allowed(plan, p).split(',')[1:]))}
|
|
case ",$actual," in *",$tenant,"*) printf '%s\\n' "__VC_VPC_WG_ALLOWED__={p.host_id}|{p.public_key}|$tenant";; *) printf '%s\\n' "__VC_VPC_WG_ALLOWED__={p.host_id}|{p.public_key}|missing";; esac'''
|
|
for p in plan.peers)
|
|
script = f'''#!/bin/sh
|
|
set -eu
|
|
umask 077
|
|
PATH=/usr/sbin:/usr/bin:/sbin:/bin
|
|
export PATH
|
|
IP=/usr/sbin/ip; [ -x "$IP" ] || IP=/usr/bin/ip
|
|
WG=/usr/bin/wg; [ -x "$WG" ] || WG=/usr/sbin/wg
|
|
NFT=/usr/sbin/nft; [ -x "$NFT" ] || NFT=/usr/bin/nft
|
|
FLOCK=/usr/bin/flock; [ -x "$FLOCK" ] || FLOCK=/usr/bin/flock
|
|
{IP_RULE_CONVERGENCE_SH}
|
|
BASE=/etc/virt-cloud/vpc-dataplane
|
|
BRIDGE={_q(r.bridge)}; TABLE={_q(r.nft_table)}; ROUTE_TABLE={r.route_table}; RULE_PRIORITY={r.rule_priority}
|
|
WG_IFACE={_q(FABRIC_INTERFACE)}; OWNER={_q(r.owner_file)}; STATE={_q(r.state_file)}; LOCK={_q(r.lock_file)}; TXN={_q(r.transaction_dir)}
|
|
COMMITTED=no; BRIDGE_CREATED=no; NFT_BEFORE=no; WG_BEFORE=no; ROLLBACK_FAILED=no
|
|
mkdir -p /run/virt-cloud "$BASE"
|
|
[ ! -L /run/virt-cloud ] && [ ! -L "$BASE" ] || exit 71
|
|
: >> "$LOCK"; chmod 600 "$LOCK"; exec 9>>"$LOCK"; "$FLOCK" -x 9
|
|
rollback() {{
|
|
status=$?; [ "$COMMITTED" = yes ] && return "$status"; trap - EXIT HUP INT TERM
|
|
oldIFS=$IFS; IFS=' '
|
|
for spec in { _q(route_specs) }; do destination=${{spec%%|*}}; device=${{spec#*|}}; "$IP" route del "$destination" table "$ROUTE_TABLE" dev "$device" >/dev/null 2>&1 || true; done
|
|
IFS=$oldIFS
|
|
if [ -s "$TXN/rules.attempted" ]; then
|
|
if [ -f "$TXN/rules.created" ] &&
|
|
grep -Fqx "$RULE_PRIORITY|$BRIDGE|{plan.vpc.cidr}|$ROUTE_TABLE" "$TXN/rules.created"; then
|
|
"$IP" -4 rule del pref "$RULE_PRIORITY" iif "$BRIDGE" to {_q(plan.vpc.cidr)} lookup "$ROUTE_TABLE" >/dev/null 2>&1 || ROLLBACK_FAILED=yes
|
|
else
|
|
ROLLBACK_FAILED=yes
|
|
fi
|
|
fi
|
|
if [ "$WG_BEFORE" = yes ] && [ -f "$TXN/wg.conf" ]; then "$WG" setconf "$WG_IFACE" "$TXN/wg.conf" >/dev/null 2>&1 || ROLLBACK_FAILED=yes; fi
|
|
if [ "$NFT_BEFORE" = yes ] && [ -f "$TXN/nft.before" ]; then "$NFT" delete table inet "$TABLE" >/dev/null 2>&1 || true; "$NFT" -f "$TXN/nft.before" >/dev/null 2>&1 || ROLLBACK_FAILED=yes; else "$NFT" delete table inet "$TABLE" >/dev/null 2>&1 || true; fi
|
|
if [ "$BRIDGE_CREATED" = yes ]; then "$IP" link del dev "$BRIDGE" >/dev/null 2>&1 || ROLLBACK_FAILED=yes; fi
|
|
[ "$ROLLBACK_FAILED" = yes ] && printf '%s\\n' '__VC_VPC_ROLLBACK__=failed' >&2 || printf '%s\\n' '__VC_VPC_ROLLBACK__=ok' >&2
|
|
exit "$status"
|
|
}}
|
|
trap rollback EXIT HUP INT TERM
|
|
[ -x "$IP" ] && [ -x "$WG" ] && [ -x "$NFT" ] && [ -x "$FLOCK" ] || exit 70
|
|
[ -f /etc/virt-cloud/wg-fabric0.owner ] && grep -Fqx { _q(FABRIC_OWNER_MARKER) } /etc/virt-cloud/wg-fabric0.owner || exit 71
|
|
"$IP" link show dev "$WG_IFACE" >/dev/null 2>&1 && "$WG" show "$WG_IFACE" >/dev/null 2>&1 || exit 72
|
|
[ ! -e "$TXN" ] && [ ! -L "$TXN" ] || exit 79
|
|
mkdir -p "$TXN"; printf '%s\\n' '{DATAPLANE_OWNER_MARKER}:{plan.vpc.vpc_id}:{plan.host_id}:{plan.desired_revision}:{plan.desired_digest}' > "$TXN/inflight"
|
|
if [ -f "$STATE" ]; then cp -p "$STATE" "$TXN/state.before"; grep -Fqx 'owner|{DATAPLANE_OWNER_MARKER}' "$STATE" || exit 71; fi
|
|
if [ -f "$OWNER" ]; then grep -Fqx { _q(DATAPLANE_OWNER_MARKER) } "$OWNER" || exit 71; else printf '%s\\n' { _q(DATAPLANE_OWNER_MARKER) } > "$OWNER"; chmod 600 "$OWNER"; fi
|
|
if "$IP" link show dev "$BRIDGE" >/dev/null 2>&1; then
|
|
"$IP" -d link show dev "$BRIDGE" | grep -Eq '(^|[[:space:]])bridge[[:space:]]+(forward_delay|stp_state|hello_time|ageing_time)([[:space:]]|$)' || exit 71
|
|
else
|
|
"$IP" link add name "$BRIDGE" type bridge; BRIDGE_CREATED=yes
|
|
fi
|
|
"$IP" link set dev "$BRIDGE" up
|
|
if "$WG" showconf "$WG_IFACE" > "$TXN/wg.conf" 2>/dev/null; then WG_BEFORE=yes; fi
|
|
if "$NFT" list table inet "$TABLE" > "$TXN/nft.before" 2>/dev/null; then NFT_BEFORE=yes; grep -Fq { _q(DATAPLANE_OWNER_MARKER) } "$TXN/nft.before" || exit 71; fi
|
|
'''+_here('$TXN/rules.nft', nft)+f'''"$NFT" delete table inet "$TABLE" >/dev/null 2>&1 || true
|
|
"$NFT" -f "$TXN/rules.nft"
|
|
oldIFS=$IFS; IFS=' '
|
|
for spec in { _q(route_specs) }; do destination=${{spec%%|*}}; device=${{spec#*|}}; "$IP" route replace "$destination" table "$ROUTE_TABLE" dev "$device" metric 100; done
|
|
IFS=$oldIFS
|
|
vc_ip_rule_preflight "$RULE_PRIORITY" "$BRIDGE" {_q(plan.vpc.cidr)} "$ROUTE_TABLE" 73 73 - || exit $?
|
|
: > "$TXN/rules.attempted"; : > "$TXN/rules.created"
|
|
vc_ip_rule_ensure "$RULE_PRIORITY" "$BRIDGE" {_q(plan.vpc.cidr)} "$ROUTE_TABLE" 73 73 "$TXN/rules.attempted" "$TXN/rules.created" || exit $?
|
|
oldIFS=$IFS; IFS=' '
|
|
for spec in { _q(peer_specs) }; do rest=${{spec#*|}}; key=${{rest%%|*}}; allowed=${{rest#*|}}; "$WG" show "$WG_IFACE" peers | grep -Fqx "$key" || exit 74; "$WG" set "$WG_IFACE" peer "$key" allowed-ips "$allowed"; done
|
|
IFS=$oldIFS
|
|
'''+_here('$STATE', state)+f'''chmod 600 "$STATE"
|
|
printf '%s\\n' '__VC_VPC_APPLIED__=yes'
|
|
printf '%s\\n' '__VC_VPC_OWNER__={DATAPLANE_OWNER_MARKER}'
|
|
printf '%s\\n' '__VC_VPC_REVISION__={plan.desired_revision}'
|
|
printf '%s\\n' '__VC_VPC_DIGEST__={plan.desired_digest}'
|
|
COMMITTED=yes; trap - EXIT HUP INT TERM; rm -rf "$TXN"; exit 0
|
|
'''
|
|
if len(script) > MAX_OUTPUT:
|
|
raise VpcDataplaneError("VPC 数据面脚本过大", 413, "vpc_script_too_large")
|
|
return script
|
|
|
|
|
|
def _render_apply_script_v2(
|
|
plan: VpcDataplanePlan,
|
|
*,
|
|
deployment_mode: str = "networkmanager_nmcli",
|
|
firewall_mode: str = FIREWALL_MODE_NATIVE,
|
|
ifup_path: str = "/usr/sbin/ifup",
|
|
ifdown_path: str = "/usr/sbin/ifdown",
|
|
ifquery_path: str = "/usr/sbin/ifquery",
|
|
) -> str:
|
|
"""Generate the production transaction used by the host adapter.
|
|
|
|
The original control-plane module predates distribution-specific bridge
|
|
persistence. Keep its public builder for compatibility, but route real
|
|
applies through this stricter builder: every list is newline-delimited,
|
|
every path is fixed/quoted, and rollback only removes exact resources
|
|
tagged with the panel route protocol or ownership marker.
|
|
"""
|
|
if deployment_mode not in {"networkmanager_nmcli", "ifupdown"}:
|
|
raise VpcDataplaneError("宿主机网络管理模式不支持", 409,
|
|
"vpc_dataplane_network_manager_conflict")
|
|
if not plan.local_members:
|
|
raise VpcDataplaneError(
|
|
"当前宿主机没有 VPC 成员,应执行删除收敛而不是 apply",
|
|
409, "vpc_apply_no_local_members")
|
|
firewall_mode = _firewall_mode(firewall_mode)
|
|
allowed_ifup = {"/usr/sbin/ifup", "/usr/bin/ifup", "/sbin/ifup", "/bin/ifup"}
|
|
allowed_ifdown = {"/usr/sbin/ifdown", "/usr/bin/ifdown", "/sbin/ifdown", "/bin/ifdown"}
|
|
allowed_ifquery = {"/usr/sbin/ifquery", "/usr/bin/ifquery", "/sbin/ifquery", "/bin/ifquery"}
|
|
if ifup_path not in allowed_ifup or ifdown_path not in allowed_ifdown or ifquery_path not in allowed_ifquery:
|
|
raise VpcDataplaneError("ifupdown 工具路径不受支持", 409,
|
|
"vpc_dataplane_network_manager_conflict")
|
|
from .fabric import (
|
|
_FABRIC_NFT_VALIDATION_SH,
|
|
_FIREWALL_MODE_GUARD_SH,
|
|
_NMCLI_FILENAME_RESOLVER_SH,
|
|
_VPC_STATE_LOADER,
|
|
)
|
|
|
|
routes = _routes(plan)
|
|
if plan.remote_members and not plan.peers:
|
|
raise VpcDataplaneError("跨宿主机 VPC 缺少 Fabric peer", 409,
|
|
"vpc_fabric_peer_missing")
|
|
r = plan.resources
|
|
nft = render_vpc_nft_rules(plan)
|
|
include = f"/etc/nftables.d/virt-cloud-vpc-{r.token}.nft"
|
|
include_line = f'include "{include}"'
|
|
fragment = f"/etc/network/interfaces.d/virt-cloud-vpc-{r.token}"
|
|
sysctl_file = f"/etc/sysctl.d/99-virt-cloud-vpc-{r.token}.conf"
|
|
state_dir = f"/etc/virt-cloud/vpc-dataplane"
|
|
proxy_file = f"{state_dir}/{r.token}.proxy"
|
|
route_lines = "\n".join(
|
|
f"{item['destination_cidr']}|{item['device']}|{item['table']}|"
|
|
f"{item.get('via') or '-'}|{item.get('metric', 100)}"
|
|
for item in routes)
|
|
bridge_route_lines = "\n".join(
|
|
f"{item['destination_cidr']}|{item['device']}|{item['table']}|"
|
|
f"{item.get('via') or '-'}|{item.get('metric', 100)}|186"
|
|
for item in routes if item["device"] == r.bridge)
|
|
proxy_lines = "\n".join(
|
|
str(ipaddress.ip_network(item["destination_cidr"], strict=True).network_address)
|
|
for item in routes)
|
|
peer_lines = "\n".join(
|
|
f"{peer.host_id}|{peer.public_key}|{_allowed(plan, peer)}"
|
|
for peer in plan.peers)
|
|
state = _state(plan, routes, firewall_mode)
|
|
# Local service /32s use a guest address as the next hop on an otherwise
|
|
# unnumbered private bridge. Linux therefore needs the explicit `onlink`
|
|
# assertion; without it iproute2 rejects the route as an invalid gateway.
|
|
# Use a quoted heredoc for data, while keeping the destination as a shell
|
|
# variable expanded by the surrounding script. This avoids the literal
|
|
# ``$TXN`` path bug in the former builder.
|
|
nft_marker = "__VC_VPC_NFT_" + hashlib.sha256(nft.encode()).hexdigest()[:12]
|
|
nft_rule_marker = _nft_marker(plan)
|
|
nft_families = (("inet", "bridge", "arp")
|
|
if plan.local_members else ("inet",))
|
|
nft_family_words = " ".join(nft_families)
|
|
nft_reassert_deletes = "\n".join(
|
|
f"printf 'delete table {family} %s\\n' \"$TABLE\" >> \"$NFT_REASSERT\""
|
|
for family in nft_families)
|
|
inet_readback_checks = "\n".join(
|
|
f''' printf '%s\\n' "$vc_inet_text" | grep -Fq "comment \\"$NFT_RULE_MARKER:{suffix}\\"" || return 1'''
|
|
for suffix in (
|
|
*_NFT_INET_REQUIRED_SUFFIXES,
|
|
*(f"acl:{acl.acl_id}" for acl in plan.vpc.acls),
|
|
))
|
|
bridge_readback_checks = "\n".join(
|
|
f''' printf '%s\\n' "$vc_bridge_text" | grep -Fq "comment \\"$NFT_RULE_MARKER:{suffix}\\"" || return 1'''
|
|
for suffix in (
|
|
*_NFT_BRIDGE_REQUIRED_SUFFIXES,
|
|
*(f"acl:{acl.acl_id}" for acl in plan.vpc.acls),
|
|
))
|
|
if plan.local_members:
|
|
local_nft_readback = f'''
|
|
vc_bridge_text=$("$NFT" list table bridge "$TABLE" 2>/dev/null) || return 1
|
|
printf '%s\\n' "$vc_bridge_text" | grep -Fq "table bridge $TABLE" || return 1
|
|
{bridge_readback_checks}
|
|
vc_arp_text=$("$NFT" list table arp "$TABLE" 2>/dev/null) || return 1
|
|
printf '%s\\n' "$vc_arp_text" | grep -Fq "table arp $TABLE" || return 1
|
|
printf '%s\\n' "$vc_arp_text" | grep -Fq "comment \\"$NFT_RULE_MARKER:arp-proxy-output\\"" || return 1
|
|
'''.rstrip()
|
|
else:
|
|
local_nft_readback = '''
|
|
"$NFT" list table bridge "$TABLE" >/dev/null 2>&1 && return 1 || true
|
|
"$NFT" list table arp "$TABLE" >/dev/null 2>&1 && return 1 || true
|
|
'''.rstrip()
|
|
nft_readback_helper = f'''
|
|
vc_vpc_desired_nft_ready() {{
|
|
vc_inet_text=$("$NFT" list table inet "$TABLE" 2>/dev/null) || return 1
|
|
printf '%s\\n' "$vc_inet_text" | grep -Fq "table inet $TABLE" || return 1
|
|
{inet_readback_checks}
|
|
{local_nft_readback}
|
|
return 0
|
|
}}
|
|
vc_vpc_capture_nft_snapshot() {{
|
|
vc_vpc_snapshot_prefix=$1
|
|
for vc_vpc_snapshot_family in {nft_family_words}; do
|
|
vc_nft_table_snapshot "$vc_vpc_snapshot_family" "$TABLE" \
|
|
"$vc_vpc_snapshot_prefix.$vc_vpc_snapshot_family" || return 1
|
|
[ -f "$vc_vpc_snapshot_prefix.$vc_vpc_snapshot_family" ] &&
|
|
[ ! -L "$vc_vpc_snapshot_prefix.$vc_vpc_snapshot_family" ] &&
|
|
[ ! -e "$vc_vpc_snapshot_prefix.$vc_vpc_snapshot_family.absent" ] || return 1
|
|
done
|
|
}}
|
|
vc_vpc_capture_nft_precommit() {{
|
|
vc_vpc_precommit_prefix=$1
|
|
for vc_vpc_precommit_family in {nft_family_words}; do
|
|
vc_nft_table_snapshot "$vc_vpc_precommit_family" "$TABLE" \
|
|
"$vc_vpc_precommit_prefix.$vc_vpc_precommit_family" || return 1
|
|
done
|
|
}}
|
|
vc_vpc_nft_matches_snapshot() {{
|
|
vc_vpc_snapshot_prefix=$1
|
|
for vc_vpc_snapshot_family in {nft_family_words}; do
|
|
vc_nft_table_matches_snapshot "$vc_vpc_snapshot_family" "$TABLE" \
|
|
"$vc_vpc_snapshot_prefix.$vc_vpc_snapshot_family" || return 1
|
|
done
|
|
}}
|
|
vc_vpc_fabric_nft_shape_file() {{
|
|
vc_fabric_shape_file=$1
|
|
[ -f "$vc_fabric_shape_file" ] && [ ! -L "$vc_fabric_shape_file" ] || return 1
|
|
vc_fabric_shape_text=$(cat "$vc_fabric_shape_file") || return 1
|
|
nft_plan_owned "$vc_fabric_shape_text" "$EXPECTED_DIGEST"
|
|
}}
|
|
vc_vpc_fabric_nft_live_ready() {{
|
|
"$NFT" -s list table inet virt_cloud_wg_fabric > "$TXN/fabric.nft.verify" 2>/dev/null || return 1
|
|
vc_vpc_fabric_nft_shape_file "$TXN/fabric.nft.verify"
|
|
}}
|
|
'''.strip()
|
|
state_marker = "__VC_VPC_STATE_" + hashlib.sha256(state.encode()).hexdigest()[:12]
|
|
frag_marker = "__VC_VPC_IFUP_" + r.token
|
|
# Both persistence owners are helper-gated. An `auto` stanza could bring
|
|
# the bridge up during boot before the raw tenant ACL has been restored.
|
|
ifup_auto_line = ""
|
|
route_marker = "__VC_VPC_ROUTES_" + r.token
|
|
proxy_marker = "__VC_VPC_PROXY_" + r.token
|
|
peer_marker = "__VC_VPC_PEERS_" + r.token
|
|
state_loader = _VPC_STATE_LOADER.replace(
|
|
"__VC_FABRIC_HOST_ID__", plan.host_id).replace(
|
|
"__VC_FABRIC_ID__", plan.vpc.fabric_id).replace(
|
|
"__VC_FABRIC_NETWORK__", "100.64.0.0/10").replace(
|
|
"__VC_VPC_STATE_OBSERVE__", "no")
|
|
script = f'''#!/bin/sh
|
|
set -eu
|
|
umask 077
|
|
PATH=/usr/sbin:/usr/bin:/sbin:/bin
|
|
export PATH
|
|
IP=$(for p in /usr/sbin/ip /usr/bin/ip /sbin/ip /bin/ip; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
|
|
WG=$(for p in /usr/bin/wg /usr/sbin/wg /bin/wg /sbin/wg; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
|
|
NFT=$(for p in /usr/sbin/nft /usr/bin/nft /sbin/nft /bin/nft; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
|
|
FLOCK=$(for p in /usr/bin/flock /usr/sbin/flock /bin/flock /sbin/flock; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
|
|
STAT=$(for p in /usr/bin/stat /bin/stat /usr/sbin/stat /sbin/stat; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
|
|
CMP=$(for p in /usr/bin/cmp /bin/cmp /usr/sbin/cmp /sbin/cmp; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
|
|
SYSCTL=$(for p in /usr/sbin/sysctl /usr/bin/sysctl /sbin/sysctl /bin/sysctl; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
|
|
BRIDGE_CMD=$(for p in /usr/sbin/bridge /usr/bin/bridge /sbin/bridge /bin/bridge; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
|
|
SYSTEMCTL=$(for p in /usr/bin/systemctl /bin/systemctl /usr/sbin/systemctl /sbin/systemctl; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
|
|
{IP_RULE_CONVERGENCE_SH}
|
|
BRIDGE={_q(r.bridge)}; TABLE={_q(r.nft_table)}; ROUTE_TABLE={r.route_table}; RULE_PRIORITY={r.rule_priority}; WG_RULE_PRIORITY=$((RULE_PRIORITY + 1))
|
|
WG_IFACE={_q(FABRIC_INTERFACE)}; OWNER_MARKER={_q(DATAPLANE_OWNER_MARKER)}; FABRIC_OWNER={_q(FABRIC_OWNER_MARKER)}
|
|
OWNER_FILE={_q(r.owner_file)}; STATE_FILE={_q(r.state_file)}; LOCK={_q(r.lock_file)}; TXN={_q(r.transaction_dir)}; PROXY_FILE={_q(proxy_file)}
|
|
NFT_SNAPSHOT_PREFIX={_q(f'{state_dir}/{r.token}.nft')}
|
|
NFT_FILE={_q(include)}; NFT_INCLUDE={_q(include_line)}; LEGACY_MANIFEST={_q(LEGACY_FIREWALLD_NFT_MANIFEST)}; IF_FRAGMENT={_q(fragment)}; SYSCTL_FILE={_q(sysctl_file)}; MODE={_q(deployment_mode)}; FIREWALL_MODE={_q(firewall_mode)}; NFT_MAIN=
|
|
IFUP={_q(ifup_path)}; IFDOWN={_q(ifdown_path)}; IFQUERY={_q(ifquery_path)}; NMCLI=; NM_PROFILE_UUID=; NM_PROFILE_FILE=
|
|
NM_PROFILE_MARKER={_q(DATAPLANE_OWNER_MARKER + " vpc=" + plan.vpc.vpc_id)}
|
|
EXPECTED_DIGEST={_q(plan.fabric_plan_digest)}
|
|
NFT_RULE_MARKER={_q(nft_rule_marker)}
|
|
COMMITTED=no; BRIDGE_CREATED=no; BRIDGE_WAS_UP=no; BRIDGE_TOPOLOGY_CAPTURED=no; BRIDGE_PORTS_BEFORE=; NM_CREATED=no; NM_CREATED_UUID=; NM_PROFILE_WAS_PRESENT=no; NM_WAS_ACTIVE=no; NM_DEVICE_QUARANTINED=no; NM_ACTIVATION_ATTEMPTED=no; IF_FRAGMENT_CHANGED=no; SYSCTL_CHANGED=no; SYSCTL_TMP=; RP_FILTER_CHANGED=no; PROXY_ARP_CHANGED=no; PROXY_ARP_PVLAN_CHANGED=no; NFT_FILE_CHANGED=no; NFT_MAIN_CHANGED=no; NFT_FILE_BEFORE=no; NFT_MAIN_BEFORE=no; BRIDGE_NFT_BEFORE=no; ARP_NFT_BEFORE=no; FABRIC_NFT_BEFORE=no; NFT_APPLIED=no; FABRIC_NFT_APPLIED=no; PROXY_ENTRIES_CHANGED=no; PROXY_FILE_CHANGED=no; FIREWALL_HARD_QUARANTINED=no; ROLLBACK_FAILED=no
|
|
{NFT_LIVE_SNAPSHOT_SH}
|
|
{_NMCLI_FILENAME_RESOLVER_SH}
|
|
{NETWORKMANAGER_VPC_PROFILE_SH}
|
|
{nft_readback_helper}
|
|
{_FIREWALL_MODE_GUARD_SH}
|
|
{_FABRIC_NFT_VALIDATION_SH}
|
|
{_VPC_SYSCTL_VALIDATION_SH}
|
|
{VPC_NATIVE_TRANSACTION_GUARD_SH}
|
|
nft_plan_owned() {{
|
|
nft_plan_text=$1
|
|
nft_plan_digest=$2
|
|
printf '%s\\n' "$nft_plan_digest" | grep -Eq '^[0-9a-f]{{64}}$' || return 1
|
|
vc_fabric_nft_strict_shape "$nft_plan_text" || return 1
|
|
[ "$(printf '%s\\n' "$nft_plan_text" |
|
|
grep -Foc 'comment "virt-cloud-wg-fabric-v1:'"$nft_plan_digest"'"' || true)" = 8 ]
|
|
}}
|
|
[ -n "$IP" ] && [ -n "$WG" ] && [ -n "$NFT" ] && [ -n "$FLOCK" ] && [ -n "$STAT" ] && [ -n "$CMP" ] && [ -n "$SYSCTL" ] && [ -n "$BRIDGE_CMD" ] && [ -n "$SYSTEMCTL" ] || exit 70
|
|
if [ "$MODE" = networkmanager_nmcli ]; then
|
|
NMCLI=$(for p in /usr/bin/nmcli /usr/sbin/nmcli /bin/nmcli /sbin/nmcli; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
|
|
[ -n "$NMCLI" ] || exit 70
|
|
fi
|
|
[ "$FIREWALL_MODE" = native_nftables ] || exit 71
|
|
if [ -L /run/virt-cloud ] || {{ [ -e /run/virt-cloud ] && [ ! -d /run/virt-cloud ]; }}; then exit 71; fi
|
|
if [ -L /etc/virt-cloud ] || {{ [ -e /etc/virt-cloud ] && [ ! -d /etc/virt-cloud ]; }}; then exit 71; fi
|
|
if [ -L {state_dir} ] || {{ [ -e {state_dir} ] && [ ! -d {state_dir} ]; }}; then exit 71; fi
|
|
if [ -e {state_dir} ]; then
|
|
[ "$($STAT -c '%u:%a' {state_dir} 2>/dev/null)" = 0:700 ] || exit 71
|
|
fi
|
|
mkdir -p /run/virt-cloud
|
|
chmod 700 /run/virt-cloud
|
|
[ ! -L "$LOCK" ] && {{ [ ! -e "$LOCK" ] || [ -f "$LOCK" ]; }} || exit 71
|
|
: >> "$LOCK"; chmod 600 "$LOCK"
|
|
exec 8>>{_q(FABRIC_LOCK)}; "$FLOCK" -x 8
|
|
exec 9>>"$LOCK"; "$FLOCK" -x 9
|
|
vc_vpc_bridge_port_names() {{
|
|
vc_bridge_ports_raw=$("$BRIDGE_CMD" link show master "$BRIDGE" 2>/dev/null) || return 1
|
|
vc_bridge_ports_parsed=$(printf '%s\\n' "$vc_bridge_ports_raw" | awk '
|
|
NF == 0 {{next}}
|
|
NF < 2 {{bad=1; next}}
|
|
{{name=$2; sub(/:$/, "", name); sub(/@.*/, "", name)}}
|
|
name !~ /^[A-Za-z0-9_.-]+$/ {{bad=1; next}}
|
|
{{print name}}
|
|
END {{if (bad) exit 1}}
|
|
') || return 1
|
|
printf '%s\\n' "$vc_bridge_ports_parsed" | LC_ALL=C sort -u
|
|
}}
|
|
vc_vpc_bridge_topology_matches() {{
|
|
[ "$BRIDGE_TOPOLOGY_CAPTURED" = yes ] || return 1
|
|
vc_bridge_ports_now=$(vc_vpc_bridge_port_names) || return 1
|
|
[ "$vc_bridge_ports_now" = "$BRIDGE_PORTS_BEFORE" ]
|
|
}}
|
|
vc_vpc_bridge_link_state() {{
|
|
vc_bridge_link_raw=$("$IP" -o link show dev "$BRIDGE" 2>/dev/null) || return 1
|
|
vc_bridge_link_flags=$(printf '%s\\n' "$vc_bridge_link_raw" | awk -v wanted="$BRIDGE" '
|
|
NF == 0 {{next}}
|
|
{{
|
|
count++
|
|
name=$2
|
|
sub(/:$/, "", name)
|
|
sub(/@.*/, "", name)
|
|
if (name != wanted || $3 !~ /^<[^>]+>$/) bad=1
|
|
flags=$3
|
|
}}
|
|
END {{
|
|
if (bad || count != 1) exit 1
|
|
print flags
|
|
}}
|
|
') || return 1
|
|
if printf '%s\\n' "$vc_bridge_link_flags" |
|
|
grep -Eq '(^|<|,)UP(,|>)'; then
|
|
printf '%s' up
|
|
else
|
|
printf '%s' down
|
|
fi
|
|
}}
|
|
vc_vpc_rp_filter_set() {{
|
|
vc_rpf_path="/proc/sys/net/ipv4/conf/$BRIDGE/rp_filter"
|
|
[ -f "$vc_rpf_path" ] && [ ! -L "$vc_rpf_path" ] || return 1
|
|
if [ ! -f "$TXN/rp_filter.before" ]; then
|
|
[ ! -e "$TXN/rp_filter.before" ] && [ ! -L "$TXN/rp_filter.before" ] || return 1
|
|
cat "$vc_rpf_path" > "$TXN/rp_filter.before" || return 1
|
|
chmod 600 "$TXN/rp_filter.before" || return 1
|
|
fi
|
|
RP_FILTER_CHANGED=yes
|
|
printf '%s\\n' 2 > "$vc_rpf_path" || return 1
|
|
[ "$(cat "$vc_rpf_path" 2>/dev/null)" = 2 ]
|
|
}}
|
|
vc_vpc_native_ifupdown_rollback_reopen() {{
|
|
vc_vpc_nft_matches_snapshot "$TXN/nft.precommit.before" || return 1
|
|
vc_nft_table_matches_snapshot inet virt_cloud_wg_fabric \
|
|
"$TXN/fabric.nft.before" || return 1
|
|
vc_vpc_bridge_topology_matches || return 1
|
|
vc_bridge_reopen_safe=yes
|
|
"$IP" link set dev "$BRIDGE" up >/dev/null 2>&1 ||
|
|
vc_bridge_reopen_safe=no
|
|
if [ "$vc_bridge_reopen_safe" = yes ]; then
|
|
vc_bridge_reopen_state=$(vc_vpc_bridge_link_state) ||
|
|
vc_bridge_reopen_safe=no
|
|
fi
|
|
if [ "$vc_bridge_reopen_safe" = yes ] &&
|
|
[ "$vc_bridge_reopen_state" != up ]; then
|
|
vc_bridge_reopen_safe=no
|
|
fi
|
|
if [ "$vc_bridge_reopen_safe" = yes ]; then
|
|
vc_vpc_bridge_topology_matches || vc_bridge_reopen_safe=no
|
|
fi
|
|
if [ "$vc_bridge_reopen_safe" != yes ]; then
|
|
# Link-up is not a transactional primitive. If any post-open readback
|
|
# fails (including a concurrent tap/port change), immediately re-isolate
|
|
# and prove DOWN before reporting the rollback failure.
|
|
"$IP" link set dev "$BRIDGE" down >/dev/null 2>&1 || return 1
|
|
vc_bridge_reopen_state=$(vc_vpc_bridge_link_state) || return 1
|
|
[ "$vc_bridge_reopen_state" = down ] || return 1
|
|
return 1
|
|
fi
|
|
return 0
|
|
}}
|
|
# Never mutate even the bridge state while an unmarked, legacy, or malformed
|
|
# transaction exists. New native transactions carry an exact backend marker.
|
|
vc_vpc_firewall_boundary_clean || exit 79
|
|
for vc_vpc_legacy_snapshot in inet bridge arp; do
|
|
[ ! -e "$NFT_SNAPSHOT_PREFIX.$vc_vpc_legacy_snapshot.live" ] &&
|
|
[ ! -L "$NFT_SNAPSHOT_PREFIX.$vc_vpc_legacy_snapshot.live" ] || exit 79
|
|
done
|
|
if [ -e "$TXN" ] || [ -L "$TXN" ]; then
|
|
vc_vpc_native_transaction_owned || exit 79
|
|
fi
|
|
if [ ! -e {state_dir} ]; then
|
|
mkdir -m 700 {state_dir} || exit 71
|
|
fi
|
|
[ ! -L {state_dir} ] && [ "$($STAT -c '%u:%a' {state_dir} 2>/dev/null)" = 0:700 ] || exit 71
|
|
# Link quarantine deliberately runs before the runtime firewall-mode guard. A
|
|
# saved VPC is hard-blocked before its native nft ACL is replaced.
|
|
PREVIOUS_FIREWALL_MODE=absent
|
|
if [ -f "$STATE_FILE" ]; then
|
|
[ ! -L "$STATE_FILE" ] &&
|
|
[ "$($STAT -c '%u:%a' "$STATE_FILE" 2>/dev/null)" = 0:600 ] || exit 71
|
|
grep -Fqx 'owner|{DATAPLANE_OWNER_MARKER}' "$STATE_FILE" || exit 71
|
|
grep -Fqx 'vpc|{plan.vpc.vpc_id}' "$STATE_FILE" || exit 71
|
|
grep -Fqx 'fabric|{plan.vpc.fabric_id}' "$STATE_FILE" || exit 71
|
|
grep -Fqx 'host|{plan.host_id}' "$STATE_FILE" || exit 71
|
|
grep -Fqx 'bridge|{r.bridge}' "$STATE_FILE" || exit 71
|
|
grep -Fqx 'table|{r.route_table}' "$STATE_FILE" || exit 71
|
|
state_firewall_quarantine=$(awk -F'|' '$1=="firewall_mode" {{count++; value=$2}} END {{print (count + 0) "|" value}}' "$STATE_FILE")
|
|
case "$state_firewall_quarantine" in
|
|
'0|'|'1|native_nftables') PREVIOUS_FIREWALL_MODE=native_nftables ;;
|
|
*) exit 71 ;;
|
|
esac
|
|
fi
|
|
if [ "$PREVIOUS_FIREWALL_MODE" != absent ]; then
|
|
# Only durable state, the owner marker, the kernel bridge type, and the
|
|
# exact manager artifact are trusted for this pre-guard safety action.
|
|
state_revision=$(awk -F'|' '$1=="revision" {{count++; value=$2}} END {{print (count + 0) "|" value}}' "$STATE_FILE")
|
|
state_digest=$(awk -F'|' '$1=="digest" {{count++; value=$2}} END {{print (count + 0) "|" value}}' "$STATE_FILE")
|
|
[ "${{state_revision%%|*}}" = 1 ] || exit 71
|
|
case "${{state_revision#*|}}" in ''|*[!0-9]*) exit 71 ;; esac
|
|
[ "${{state_digest%%|*}}" = 1 ] || exit 71
|
|
printf '%s\\n' "${{state_digest#*|}}" | grep -Eq '^[0-9a-f]{{64}}$' || exit 71
|
|
[ -f "$OWNER_FILE" ] && [ ! -L "$OWNER_FILE" ] &&
|
|
[ "$($STAT -c '%u:%a' "$OWNER_FILE" 2>/dev/null)" = 0:600 ] &&
|
|
grep -Fqx "$OWNER_MARKER" "$OWNER_FILE" || exit 71
|
|
if [ "$MODE" = networkmanager_nmcli ]; then
|
|
vc_nm_profile_resolve || exit 71
|
|
vc_nm_profile_base_validate || exit 71
|
|
NM_PROFILE_WAS_PRESENT=yes
|
|
if vc_nm_profile_active; then NM_WAS_ACTIVE=yes; fi
|
|
quarantine_memory_zone=$(vc_nm_profile_field connection.zone) || exit 71
|
|
case "$quarantine_memory_zone" in ''|--) : ;; *) exit 71 ;; esac
|
|
else
|
|
[ -f "$IF_FRAGMENT" ] && [ ! -L "$IF_FRAGMENT" ] &&
|
|
[ "$($STAT -c '%u:%a' "$IF_FRAGMENT" 2>/dev/null)" = 0:600 ] || exit 71
|
|
awk -v owner="# {DATAPLANE_OWNER_MARKER} vpc={plan.vpc.vpc_id} revision=${{state_revision#*|}} digest=${{state_digest#*|}}" \
|
|
-v bridge="$BRIDGE" '
|
|
NR==1 && $0==owner {{next}}
|
|
NR==2 && $0=="iface " bridge " inet manual" {{next}}
|
|
NR==3 && $0==" bridge_ports none" {{next}}
|
|
NR==4 && $0==" bridge-stp off" {{next}}
|
|
{{bad=1}}
|
|
END {{exit (NR==4 && !bad) ? 0 : 1}}
|
|
' "$IF_FRAGMENT" || exit 71
|
|
fi
|
|
if "$IP" link show dev "$BRIDGE" >/dev/null 2>&1; then
|
|
"$IP" -d link show dev "$BRIDGE" 2>/dev/null |
|
|
grep -Eq '(^|[[:space:]])bridge[[:space:]]+(forward_delay|stp_state|hello_time|ageing_time)([[:space:]]|$)' || exit 71
|
|
bridge_initial_link_state=$(vc_vpc_bridge_link_state) || exit 71
|
|
if [ "$bridge_initial_link_state" = up ]; then
|
|
BRIDGE_WAS_UP=yes
|
|
fi
|
|
BRIDGE_PORTS_BEFORE=$(vc_vpc_bridge_port_names) || exit 71
|
|
BRIDGE_TOPOLOGY_CAPTURED=yes
|
|
if [ "$MODE" = networkmanager_nmcli ]; then
|
|
vc_nm_device_quarantine || exit 72
|
|
NM_DEVICE_QUARANTINED=yes
|
|
else
|
|
"$IP" link set dev "$BRIDGE" down >/dev/null || exit 72
|
|
bridge_quarantine_link_state=$(vc_vpc_bridge_link_state) || exit 72
|
|
[ "$bridge_quarantine_link_state" = down ] || exit 72
|
|
fi
|
|
vc_vpc_bridge_topology_matches || exit 72
|
|
fi
|
|
FIREWALL_HARD_QUARANTINED=yes
|
|
fi
|
|
vc_firewall_mode_ready || exit 71
|
|
[ -f /etc/virt-cloud/wg-fabric0.owner ] && [ "$(cat /etc/virt-cloud/wg-fabric0.owner)" = "$FABRIC_OWNER" ] || exit 71
|
|
"$WG" show "$WG_IFACE" >/dev/null 2>&1 || exit 71
|
|
NFT_MAIN_COUNT=0
|
|
for nft_candidate in /etc/nftables.conf /etc/sysconfig/nftables.conf; do
|
|
if [ -f "$nft_candidate" ] && [ ! -L "$nft_candidate" ] && [ "$("$STAT" -c '%u' "$nft_candidate" 2>/dev/null)" = 0 ]; then
|
|
case "$("$STAT" -c '%a' "$nft_candidate" 2>/dev/null)" in
|
|
[0-7][0145][0145]) : ;;
|
|
*) continue ;;
|
|
esac
|
|
NFT_MAIN="$nft_candidate"; NFT_MAIN_COUNT=$((NFT_MAIN_COUNT + 1))
|
|
fi
|
|
done
|
|
[ "$NFT_MAIN_COUNT" -eq 1 ] || exit 71
|
|
[ ! -e "$LEGACY_MANIFEST" ] && [ ! -L "$LEGACY_MANIFEST" ] || exit 71
|
|
if [ -L /etc/nftables.d ] || {{ [ -e /etc/nftables.d ] && [ ! -d /etc/nftables.d ]; }}; then exit 71; fi
|
|
if [ -e /etc/nftables.d ]; then
|
|
[ "$("$STAT" -c '%u:%a' /etc/nftables.d 2>/dev/null)" = 0:755 ] || exit 71
|
|
else
|
|
mkdir /etc/nftables.d; chmod 755 /etc/nftables.d
|
|
fi
|
|
if [ -L /etc/sysctl.d ] || {{ [ -e /etc/sysctl.d ] && [ ! -d /etc/sysctl.d ]; }}; then exit 71; fi
|
|
if [ -e /etc/sysctl.d ]; then
|
|
[ "$("$STAT" -c '%u:%a' /etc/sysctl.d 2>/dev/null)" = 0:755 ] || exit 71
|
|
else
|
|
mkdir /etc/sysctl.d; chmod 755 /etc/sysctl.d
|
|
fi
|
|
if [ -e "$PROXY_FILE" ] || [ -L "$PROXY_FILE" ]; then
|
|
[ -f "$PROXY_FILE" ] && [ ! -L "$PROXY_FILE" ] || exit 71
|
|
[ "$($STAT -c '%u:%a' "$PROXY_FILE" 2>/dev/null)" = 0:600 ] || exit 71
|
|
sed -n '1p' "$PROXY_FILE" | grep -Fqx '# virt-cloud-vpc-proxy-v1' || exit 71
|
|
awk 'NR > 1 && $0 !~ /^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$/ {{ exit 1 }}' "$PROXY_FILE" || exit 71
|
|
fi
|
|
if [ -e "$NFT_FILE" ] || [ -L "$NFT_FILE" ]; then
|
|
[ -f "$NFT_FILE" ] && [ ! -L "$NFT_FILE" ] || exit 71
|
|
[ "$("$STAT" -c '%u' "$NFT_FILE" 2>/dev/null)" = 0 ] || exit 71
|
|
grep -Fqx '# virt-cloud-vpc-dataplane-v1' "$NFT_FILE" || exit 71
|
|
fi
|
|
if [ -L "$OWNER_FILE" ] || {{ [ -e "$OWNER_FILE" ] && [ ! -f "$OWNER_FILE" ]; }}; then exit 71; fi
|
|
if [ -L "$STATE_FILE" ] || {{ [ -e "$STATE_FILE" ] && [ ! -f "$STATE_FILE" ]; }}; then exit 71; fi
|
|
if [ -f "$STATE_FILE" ]; then
|
|
[ "$($STAT -c '%u:%a' "$STATE_FILE" 2>/dev/null)" = 0:600 ] || exit 71
|
|
grep -Fqx 'owner|{DATAPLANE_OWNER_MARKER}' "$STATE_FILE" || exit 71
|
|
grep -Fqx 'vpc|{plan.vpc.vpc_id}' "$STATE_FILE" || exit 71
|
|
grep -Fqx 'fabric|{plan.vpc.fabric_id}' "$STATE_FILE" || exit 71
|
|
grep -Fqx 'host|{plan.host_id}' "$STATE_FILE" || exit 71
|
|
grep -Fqx 'bridge|{r.bridge}' "$STATE_FILE" || exit 71
|
|
grep -Fqx 'table|{r.route_table}' "$STATE_FILE" || exit 71
|
|
state_firewall_record=$(awk -F'|' '$1=="firewall_mode" {{count++; value=$2}} END {{print (count + 0) "|" value}}' "$STATE_FILE")
|
|
case "$state_firewall_record" in '0|'|'1|native_nftables') : ;; *) exit 71 ;; esac
|
|
state_revision=$(awk -F'|' '$1=="revision" {{print $2; exit}}' "$STATE_FILE")
|
|
state_digest=$(awk -F'|' '$1=="digest" {{print $2; exit}}' "$STATE_FILE")
|
|
case "$state_revision" in ''|*[!0-9]*) exit 79;; esac
|
|
printf '%s\n' "$state_digest" | grep -Eq '^[0-9a-f]{{64}}$' || exit 79
|
|
[ "$state_revision" -le {plan.desired_revision} ] || exit 79
|
|
if [ "$state_revision" = {plan.desired_revision} ] && [ "$state_digest" != { _q(plan.desired_digest) } ]; then exit 79; fi
|
|
fi
|
|
if [ -e "$SYSCTL_FILE" ] || [ -L "$SYSCTL_FILE" ]; then
|
|
# A deterministic token and a copied owner header are not authority to
|
|
# replace a sysctl file. Bind the complete fixed-shape artifact to the
|
|
# durable state that created it; a first apply must see the path absent.
|
|
[ -f "$STATE_FILE" ] || exit 71
|
|
vc_vpc_sysctl_artifact_owned "$SYSCTL_FILE" { _q(plan.vpc.vpc_id) } \
|
|
"$state_revision" "$state_digest" "$BRIDGE" || exit 71
|
|
fi
|
|
if [ -e "$TXN" ] || [ -L "$TXN" ]; then
|
|
# A process can die after writing the committed marker but before removing
|
|
# its transaction directory. That state is safe to reap after validating
|
|
# ownership; preparing/ready/rollback_failed states remain blocked so a
|
|
# later reconcile cannot guess whether runtime resources were partially
|
|
# applied.
|
|
vc_vpc_native_transaction_owned || exit 79
|
|
vc_vpc_transaction_record_owned "$TXN/inflight" \
|
|
'{DATAPLANE_OWNER_MARKER}:{plan.vpc.vpc_id}:{plan.host_id}:{plan.desired_revision}:{plan.desired_digest}' || exit 79
|
|
vc_vpc_transaction_record_owned "$TXN/state" committed || exit 79
|
|
rm -rf "$TXN" || exit 79
|
|
fi
|
|
# Fail closed on any foreign rule selecting this VPC CIDR or route table
|
|
# before installing the destructive rollback trap or mutating host state.
|
|
vc_ip_rule_scope_check {_q(plan.vpc.cidr)} "$ROUTE_TABLE" "$RULE_PRIORITY" "$WG_IFACE" managed 73 73 || exit $?
|
|
# The bridge was quarantined immediately after the per-VPC lock and minimum
|
|
# owner/manager proof, before this RPDB check and all other preflight.
|
|
mkdir -m 700 "$TXN"
|
|
vc_vpc_write_native_transaction_mode || exit 79
|
|
printf '%s\\n' '{DATAPLANE_OWNER_MARKER}:{plan.vpc.vpc_id}:{plan.host_id}:{plan.desired_revision}:{plan.desired_digest}' > "$TXN/inflight"
|
|
chmod 600 "$TXN/inflight"
|
|
printf '%s\\n' preparing > "$TXN/state"; chmod 600 "$TXN/state"
|
|
rollback() {{
|
|
status=$?; [ "$COMMITTED" = yes ] && exit "$status"; trap - EXIT HUP INT TERM
|
|
set +e
|
|
# First rollback action: quarantine this already-proven panel bridge before
|
|
# touching routes, peers, proxy state, sysctls or raw nft tables. This
|
|
# closes the failure window where an ACL readback error could otherwise
|
|
# leave an active interface exposed while unrelated rollback work runs.
|
|
if "$IP" link show dev "$BRIDGE" >/dev/null 2>&1; then
|
|
if [ "$MODE" = networkmanager_nmcli ] && vc_nm_profile_resolve; then
|
|
vc_nm_device_quarantine || ROLLBACK_FAILED=yes
|
|
[ "$ROLLBACK_FAILED" = yes ] || NM_DEVICE_QUARANTINED=yes
|
|
fi
|
|
"$IP" link set dev "$BRIDGE" down >/dev/null 2>&1 || ROLLBACK_FAILED=yes
|
|
rollback_bridge_link_state=$(vc_vpc_bridge_link_state)
|
|
if [ "$?" -ne 0 ] || [ "$rollback_bridge_link_state" != down ]; then
|
|
ROLLBACK_FAILED=yes
|
|
fi
|
|
if [ "$BRIDGE_TOPOLOGY_CAPTURED" = yes ]; then
|
|
vc_vpc_bridge_topology_matches || ROLLBACK_FAILED=yes
|
|
fi
|
|
fi
|
|
while IFS='|' read -r destination device table via metric; do
|
|
[ -n "$destination" ] || continue
|
|
actual=$("$IP" -o -4 route show table "$table" "$destination" 2>/dev/null || true)
|
|
[ -n "$actual" ] || continue
|
|
actual_count=$(printf '%s\n' "$actual" | awk 'NF {{count++}} END {{print count + 0}}')
|
|
actual_device=$(printf '%s\n' "$actual" | awk '{{for (i=1;i<=NF;i++) if ($i=="dev" && i<NF) {{print $(i+1); exit}}}}')
|
|
actual_via=$(printf '%s\n' "$actual" | awk '{{for (i=1;i<=NF;i++) if ($i=="via" && i<NF) {{print $(i+1); exit}}}}')
|
|
actual_metric=$(printf '%s\n' "$actual" | awk '{{for (i=1;i<=NF;i++) if ($i=="metric" && i<NF) {{print $(i+1); exit}}}}')
|
|
actual_proto=$(printf '%s\n' "$actual" | awk '{{for (i=1;i<=NF;i++) if ($i=="proto" && i<NF) {{print $(i+1); exit}}}}')
|
|
[ -n "$actual_metric" ] || actual_metric=100
|
|
if [ "$actual_count" != 1 ] || [ "$actual_device" != "$device" ] \
|
|
|| [ "${{actual_via:--}}" != "${{via:--}}" ] \
|
|
|| [ "$actual_metric" != "$metric" ] || [ "$actual_proto" != 186 ]; then
|
|
# A route with the same destination may have been replaced by an
|
|
# operator or another VPC while this transaction was failing. Never
|
|
# delete that route merely because its destination matches ours.
|
|
ROLLBACK_FAILED=yes
|
|
continue
|
|
fi
|
|
if [ "$via" = - ]; then
|
|
"$IP" route del "$destination" table "$table" dev "$device" proto 186 metric "$metric" 2>/dev/null || ROLLBACK_FAILED=yes
|
|
else
|
|
"$IP" route del "$destination" table "$table" via "$via" dev "$device" proto 186 metric "$metric" 2>/dev/null || ROLLBACK_FAILED=yes
|
|
fi
|
|
done <<'{route_marker}'
|
|
{route_lines}
|
|
{route_marker}
|
|
if [ "$ROLLBACK_FAILED" = no ] && [ -f "$TXN/routes.before" ]; then
|
|
while IFS='|' read -r old_destination old_device old_table old_via old_metric; do
|
|
[ -n "$old_destination" ] || continue
|
|
if [ "$old_via" = - ]; then
|
|
"$IP" route replace "$old_destination" table "$old_table" dev "$old_device" proto 186 metric "$old_metric" 2>/dev/null || ROLLBACK_FAILED=yes
|
|
else
|
|
"$IP" route replace "$old_destination" table "$old_table" via "$old_via" dev "$old_device" proto 186 metric "$old_metric" 2>/dev/null || ROLLBACK_FAILED=yes
|
|
fi
|
|
done < "$TXN/routes.before"
|
|
fi
|
|
# vc-policy-rule-rollback-begin
|
|
if [ "$ROLLBACK_FAILED" = no ] && [ -s "$TXN/rules.attempted" ]; then
|
|
while IFS='|' read -r rule_pref rule_iif rule_to rule_table; do
|
|
case "$rule_pref|$rule_iif|$rule_to|$rule_table" in
|
|
"$RULE_PRIORITY|-|{plan.vpc.cidr}|$ROUTE_TABLE"|"$WG_RULE_PRIORITY|$WG_IFACE|{plan.vpc.cidr}|$ROUTE_TABLE") : ;;
|
|
*) ROLLBACK_FAILED=yes; continue ;;
|
|
esac
|
|
if [ ! -f "$TXN/rules.created" ] ||
|
|
! grep -Fqx "$rule_pref|$rule_iif|$rule_to|$rule_table" "$TXN/rules.created"; then
|
|
if vc_ip_rule_read "$rule_pref" "$rule_iif" "$rule_to" "$rule_table" 79 79 &&
|
|
[ "$VC_IP_RULE_PRESENT" = no ]; then
|
|
continue
|
|
fi
|
|
ROLLBACK_FAILED=yes
|
|
continue
|
|
fi
|
|
if ! vc_ip_rule_read "$rule_pref" "$rule_iif" "$rule_to" "$rule_table" 79 79; then
|
|
ROLLBACK_FAILED=yes
|
|
continue
|
|
fi
|
|
[ "$VC_IP_RULE_PRESENT" = yes ] || continue
|
|
if [ "$rule_iif" = - ]; then
|
|
"$IP" -4 rule del pref "$rule_pref" to "$rule_to" lookup "$rule_table" 2>/dev/null || ROLLBACK_FAILED=yes
|
|
else
|
|
"$IP" -4 rule del pref "$rule_pref" iif "$rule_iif" to "$rule_to" lookup "$rule_table" 2>/dev/null || ROLLBACK_FAILED=yes
|
|
fi
|
|
if ! vc_ip_rule_read "$rule_pref" "$rule_iif" "$rule_to" "$rule_table" 79 79 ||
|
|
[ "$VC_IP_RULE_PRESENT" != no ]; then
|
|
ROLLBACK_FAILED=yes
|
|
fi
|
|
done < "$TXN/rules.attempted"
|
|
fi
|
|
# vc-policy-rule-rollback-end
|
|
if [ -f "$TXN/wg.conf" ]; then "$WG" setconf "$WG_IFACE" "$TXN/wg.conf" >/dev/null 2>&1 || ROLLBACK_FAILED=yes; fi
|
|
if [ "$PROXY_ENTRIES_CHANGED" = yes ]; then
|
|
"$IP" neigh show proxy dev "$BRIDGE" 2>/dev/null | awk '$1 ~ /^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$/ {{print $1}}' | LC_ALL=C sort -u > "$TXN/proxy.rollback.current"
|
|
cat <<'{proxy_marker}' | LC_ALL=C sort -u > "$TXN/proxy.rollback.desired"
|
|
{proxy_lines}
|
|
{proxy_marker}
|
|
if "$CMP" -s "$TXN/proxy.rollback.current" "$TXN/proxy.rollback.desired"; then
|
|
while IFS= read -r rollback_proxy; do
|
|
[ -n "$rollback_proxy" ] || continue
|
|
"$IP" neigh del proxy "$rollback_proxy" dev "$BRIDGE" >/dev/null 2>&1 || ROLLBACK_FAILED=yes
|
|
done < "$TXN/proxy.rollback.desired"
|
|
while IFS= read -r rollback_proxy; do
|
|
[ -n "$rollback_proxy" ] || continue
|
|
"$IP" neigh replace proxy "$rollback_proxy" dev "$BRIDGE" >/dev/null 2>&1 || ROLLBACK_FAILED=yes
|
|
done < "$TXN/proxy.before"
|
|
else
|
|
# Do not delete an entry added concurrently by an operator or another
|
|
# controller. Leave the transaction recoverable and report rollback
|
|
# failure rather than risking an unrelated ARP outage.
|
|
ROLLBACK_FAILED=yes
|
|
fi
|
|
fi
|
|
if [ "$PROXY_FILE_CHANGED" = yes ]; then
|
|
if [ -f "$TXN/proxy_file.before" ]; then
|
|
cp -p "$TXN/proxy_file.before" "$PROXY_FILE" || ROLLBACK_FAILED=yes
|
|
else
|
|
rm -f "$PROXY_FILE" || ROLLBACK_FAILED=yes
|
|
fi
|
|
fi
|
|
# Restore runtime sysctls while the new ARP guard is still installed; this
|
|
# avoids a rollback window in which proxy_arp is enabled without its exact
|
|
# output filter.
|
|
if [ "$PROXY_ARP_CHANGED" = yes ] && [ -e "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp" ]; then
|
|
if [ -f "$TXN/proxy_arp.before" ]; then
|
|
cat "$TXN/proxy_arp.before" > "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp" || ROLLBACK_FAILED=yes
|
|
else
|
|
printf '%s\\n' 0 > "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp" || ROLLBACK_FAILED=yes
|
|
fi
|
|
fi
|
|
if [ "$PROXY_ARP_PVLAN_CHANGED" = yes ] && [ -e "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp_pvlan" ]; then
|
|
if [ -f "$TXN/proxy_arp_pvlan.before" ]; then
|
|
cat "$TXN/proxy_arp_pvlan.before" > "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp_pvlan" || ROLLBACK_FAILED=yes
|
|
else
|
|
printf '%s\n' 0 > "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp_pvlan" || ROLLBACK_FAILED=yes
|
|
fi
|
|
fi
|
|
if [ "$RP_FILTER_CHANGED" = yes ] && [ -e "/proc/sys/net/ipv4/conf/$BRIDGE/rp_filter" ]; then
|
|
if [ -f "$TXN/rp_filter.before" ]; then
|
|
cat "$TXN/rp_filter.before" > "/proc/sys/net/ipv4/conf/$BRIDGE/rp_filter" || ROLLBACK_FAILED=yes
|
|
fi
|
|
fi
|
|
NFT_ROLLBACK="$TXN/rollback.nft"
|
|
NFT_ROLLBACK_SAFE=yes
|
|
if [ "$NFT_APPLIED" = yes ]; then
|
|
# A table name or owner comment is not sufficient ownership evidence.
|
|
# Refuse the entire atomic restore unless every currently live table is
|
|
# byte-for-byte equal to this transaction's post-apply stateless dump.
|
|
vc_vpc_nft_matches_snapshot "$TXN/nft.desired" || NFT_ROLLBACK_SAFE=no
|
|
fi
|
|
if [ "$FABRIC_NFT_APPLIED" = yes ]; then
|
|
[ -f "$TXN/fabric.nft.desired" ] &&
|
|
[ ! -L "$TXN/fabric.nft.desired" ] &&
|
|
vc_nft_table_matches_snapshot inet virt_cloud_wg_fabric \
|
|
"$TXN/fabric.nft.desired" || NFT_ROLLBACK_SAFE=no
|
|
fi
|
|
[ "$NFT_ROLLBACK_SAFE" = yes ] || ROLLBACK_FAILED=yes
|
|
if [ "$ROLLBACK_FAILED" = no ]; then
|
|
: > "$NFT_ROLLBACK"
|
|
if [ "$NFT_APPLIED" = yes ]; then
|
|
for rollback_family in {nft_family_words}; do
|
|
printf 'delete table %s %s\\n' "$rollback_family" "$TABLE" >> "$NFT_ROLLBACK"
|
|
done
|
|
if [ -f "$TXN/nft.before" ]; then cat "$TXN/nft.before" >> "$NFT_ROLLBACK"; fi
|
|
if [ -f "$TXN/nft.bridge.before" ]; then cat "$TXN/nft.bridge.before" >> "$NFT_ROLLBACK"; fi
|
|
if [ -f "$TXN/nft.arp.before" ]; then cat "$TXN/nft.arp.before" >> "$NFT_ROLLBACK"; fi
|
|
fi
|
|
if [ "$FABRIC_NFT_APPLIED" = yes ]; then
|
|
vc_vpc_fabric_nft_shape_file "$TXN/fabric.nft.before" || ROLLBACK_FAILED=yes
|
|
if [ "$ROLLBACK_FAILED" = no ]; then
|
|
printf 'delete table inet virt_cloud_wg_fabric\\n' >> "$NFT_ROLLBACK"
|
|
cat "$TXN/fabric.nft.before" >> "$NFT_ROLLBACK"
|
|
fi
|
|
fi
|
|
fi
|
|
if [ "$ROLLBACK_FAILED" = no ] && [ -s "$NFT_ROLLBACK" ]; then
|
|
"$NFT" -c -f "$NFT_ROLLBACK" >/dev/null 2>&1 || ROLLBACK_FAILED=yes
|
|
# Check mode has no reservation primitive. Repeat the complete desired
|
|
# snapshots after dry-run and immediately before the one atomic restore;
|
|
# a same-name foreign replacement must never be deleted by rollback.
|
|
if [ "$ROLLBACK_FAILED" = no ] && [ "$NFT_APPLIED" = yes ]; then
|
|
vc_vpc_nft_matches_snapshot "$TXN/nft.desired" ||
|
|
ROLLBACK_FAILED=yes
|
|
fi
|
|
if [ "$ROLLBACK_FAILED" = no ] &&
|
|
[ "$FABRIC_NFT_APPLIED" = yes ]; then
|
|
vc_nft_table_matches_snapshot inet virt_cloud_wg_fabric \
|
|
"$TXN/fabric.nft.desired" || ROLLBACK_FAILED=yes
|
|
fi
|
|
if [ "$ROLLBACK_FAILED" = no ]; then
|
|
"$NFT" -f "$NFT_ROLLBACK" >/dev/null 2>&1 || ROLLBACK_FAILED=yes
|
|
fi
|
|
if [ "$ROLLBACK_FAILED" = no ] && [ "$NFT_APPLIED" = yes ]; then
|
|
vc_vpc_nft_matches_snapshot "$TXN/nft.precommit.before" ||
|
|
ROLLBACK_FAILED=yes
|
|
fi
|
|
if [ "$ROLLBACK_FAILED" = no ] &&
|
|
[ "$FABRIC_NFT_APPLIED" = yes ]; then
|
|
vc_nft_table_matches_snapshot inet virt_cloud_wg_fabric \
|
|
"$TXN/fabric.nft.before" || ROLLBACK_FAILED=yes
|
|
fi
|
|
fi
|
|
if [ "$NFT_FILE_CHANGED" = yes ]; then
|
|
if [ -f "$TXN/nft_file.before" ]; then cp -p "$TXN/nft_file.before" "$NFT_FILE" || ROLLBACK_FAILED=yes; else rm -f "$NFT_FILE" || ROLLBACK_FAILED=yes; fi
|
|
fi
|
|
if [ "$NFT_MAIN_CHANGED" = yes ] && [ -f "$TXN/nft_main.before" ]; then
|
|
cp -p "$TXN/nft_main.before" "$NFT_MAIN" || ROLLBACK_FAILED=yes
|
|
fi
|
|
if [ "$MODE" = networkmanager_nmcli ] && [ -n "$NMCLI" ]; then
|
|
if [ "$NM_ACTIVATION_ATTEMPTED" = yes ] && [ "$NM_WAS_ACTIVE" = no ]; then
|
|
rollback_nm_ports=$("$BRIDGE_CMD" link show master "$BRIDGE" 2>/dev/null)
|
|
if [ "$?" -ne 0 ] || [ -n "$rollback_nm_ports" ]; then
|
|
# Never destroy a bridge that acquired a concurrent libvirt tap.
|
|
ROLLBACK_FAILED=yes
|
|
else
|
|
vc_nm_device_expect_managed no || ROLLBACK_FAILED=yes
|
|
"$IP" link set dev "$BRIDGE" down >/dev/null 2>&1 || ROLLBACK_FAILED=yes
|
|
fi
|
|
if [ "$ROLLBACK_FAILED" = no ]; then
|
|
vc_nm_profile_inactive || ROLLBACK_FAILED=yes
|
|
fi
|
|
fi
|
|
if [ "$NM_CREATED" = yes ] && [ "$ROLLBACK_FAILED" = no ]; then
|
|
rollback_nm_ports=$("$BRIDGE_CMD" link show master "$BRIDGE" 2>/dev/null)
|
|
if [ "$?" -ne 0 ] || [ -n "$rollback_nm_ports" ]; then
|
|
ROLLBACK_FAILED=yes
|
|
else
|
|
# The add may have succeeded while exact UUID/profile readback failed.
|
|
# A same-name delete is ambiguous and could remove an operator profile.
|
|
vc_nm_created_profile_remove || ROLLBACK_FAILED=yes
|
|
fi
|
|
fi
|
|
fi
|
|
if [ "$IF_FRAGMENT_CHANGED" = yes ]; then
|
|
if [ -f "$TXN/if_fragment.before" ]; then cp -p "$TXN/if_fragment.before" "$IF_FRAGMENT" || ROLLBACK_FAILED=yes; else rm -f "$IF_FRAGMENT" || ROLLBACK_FAILED=yes; fi
|
|
fi
|
|
if [ "$SYSCTL_CHANGED" = yes ]; then
|
|
# The deterministic pathname is not authority to overwrite or delete a
|
|
# file that changed after this transaction installed it. Restore the old
|
|
# snapshot (or absence) only while the live bytes and full desired
|
|
# identity are still exactly the artifact saved in this journal.
|
|
vc_vpc_sysctl_artifacts_equal "$SYSCTL_FILE" \
|
|
"$TXN/sysctl.desired" {_q(plan.vpc.vpc_id)} \
|
|
{plan.desired_revision} {_q(plan.desired_digest)} "$BRIDGE" || \
|
|
ROLLBACK_FAILED=yes
|
|
if [ "$ROLLBACK_FAILED" = no ]; then
|
|
if [ -f "$TXN/sysctl.before" ]; then
|
|
cp -p "$TXN/sysctl.before" "$SYSCTL_FILE" || ROLLBACK_FAILED=yes
|
|
else
|
|
rm -f "$SYSCTL_FILE" || ROLLBACK_FAILED=yes
|
|
fi
|
|
fi
|
|
fi
|
|
if [ -n "$SYSCTL_TMP" ]; then
|
|
rm -f "$SYSCTL_TMP" || ROLLBACK_FAILED=yes
|
|
fi
|
|
if [ "$BRIDGE_CREATED" = yes ]; then
|
|
# Never delete a bridge that acquired a libvirt tap or an operator port.
|
|
ports=$($BRIDGE_CMD link show master "$BRIDGE" 2>/dev/null)
|
|
if [ "$?" -ne 0 ]; then
|
|
# An inability to inspect ports is a rollback failure. Treating an
|
|
# empty error result as "no ports" could delete a bridge still carrying
|
|
# a libvirt tap and strand the host network.
|
|
ROLLBACK_FAILED=yes
|
|
elif [ -z "$ports" ]; then
|
|
"$IP" link del dev "$BRIDGE" type bridge 2>/dev/null || ROLLBACK_FAILED=yes
|
|
fi
|
|
elif [ "$BRIDGE_WAS_UP" = no ] && "$IP" link show dev "$BRIDGE" >/dev/null 2>&1; then
|
|
# Applying a revision may have activated an existing panel bridge through
|
|
# NetworkManager/ifupdown. Restore its prior DOWN state only when no tap
|
|
# or operator port appeared while the transaction was running.
|
|
ports=$($BRIDGE_CMD link show master "$BRIDGE" 2>/dev/null)
|
|
if [ "$?" -ne 0 ] || [ -n "$ports" ]; then
|
|
ROLLBACK_FAILED=yes
|
|
else
|
|
"$IP" link set dev "$BRIDGE" down 2>/dev/null || ROLLBACK_FAILED=yes
|
|
rollback_bridge_link_state=$(vc_vpc_bridge_link_state)
|
|
if [ "$?" -ne 0 ] || [ "$rollback_bridge_link_state" != down ]; then
|
|
ROLLBACK_FAILED=yes
|
|
fi
|
|
fi
|
|
fi
|
|
if [ "$MODE" = ifupdown ] &&
|
|
[ "$BRIDGE_CREATED" = no ] && [ "$BRIDGE_WAS_UP" = yes ] &&
|
|
[ "$ROLLBACK_FAILED" = no ]; then
|
|
# The existing bridge was hard-quarantined before preflight. Re-open it
|
|
# only after the exact pre-transaction nft semantics and unchanged port
|
|
# topology have both been proven; a new or sampled-DOWN bridge is never
|
|
# brought up by rollback.
|
|
vc_vpc_native_ifupdown_rollback_reopen || ROLLBACK_FAILED=yes
|
|
fi
|
|
# Return a quarantined native NetworkManager profile to its exact prior
|
|
# managed/active state only after restoring the original nft ACL.
|
|
if [ "$MODE" = networkmanager_nmcli ] &&
|
|
[ "$NM_PROFILE_WAS_PRESENT" = yes ] &&
|
|
[ "$NM_DEVICE_QUARANTINED" = yes ] &&
|
|
[ "$ROLLBACK_FAILED" = no ]; then
|
|
if vc_nm_profile_resolve && vc_nm_profile_base_validate &&
|
|
vc_nm_device_expect_managed no &&
|
|
vc_vpc_bridge_topology_matches; then
|
|
vc_nm_device_release || ROLLBACK_FAILED=yes
|
|
if [ "$ROLLBACK_FAILED" = no ] && [ "$NM_WAS_ACTIVE" = yes ]; then
|
|
"$NMCLI" connection up uuid "$NM_PROFILE_UUID" >/dev/null 2>&1 ||
|
|
ROLLBACK_FAILED=yes
|
|
if [ "$ROLLBACK_FAILED" = no ]; then
|
|
if [ "$BRIDGE_WAS_UP" = yes ]; then
|
|
"$IP" link set dev "$BRIDGE" up >/dev/null 2>&1 ||
|
|
ROLLBACK_FAILED=yes
|
|
else
|
|
"$IP" link set dev "$BRIDGE" down >/dev/null 2>&1 ||
|
|
ROLLBACK_FAILED=yes
|
|
fi
|
|
fi
|
|
[ "$ROLLBACK_FAILED" = no ] && vc_nm_profile_active ||
|
|
ROLLBACK_FAILED=yes
|
|
elif [ "$ROLLBACK_FAILED" = no ]; then
|
|
vc_nm_profile_active && ROLLBACK_FAILED=yes || true
|
|
"$IP" link set dev "$BRIDGE" down >/dev/null 2>&1 ||
|
|
ROLLBACK_FAILED=yes
|
|
vc_nm_device_expect_managed yes || ROLLBACK_FAILED=yes
|
|
fi
|
|
[ "$ROLLBACK_FAILED" = no ] && vc_vpc_bridge_topology_matches ||
|
|
ROLLBACK_FAILED=yes
|
|
[ "$ROLLBACK_FAILED" = yes ] || NM_DEVICE_QUARANTINED=no
|
|
else
|
|
ROLLBACK_FAILED=yes
|
|
fi
|
|
fi
|
|
if [ -f "$TXN/owner.before" ]; then cp -p "$TXN/owner.before" "$OWNER_FILE" || ROLLBACK_FAILED=yes; else rm -f "$OWNER_FILE"; fi
|
|
if [ -f "$TXN/state.before" ]; then cp -p "$TXN/state.before" "$STATE_FILE" || ROLLBACK_FAILED=yes; else rm -f "$STATE_FILE"; fi
|
|
if [ "$ROLLBACK_FAILED" = yes ]; then printf '%s\\n' rollback_failed > "$TXN/state"; chmod 600 "$TXN/state"; exit 79; fi
|
|
rm -rf "$TXN"; exit "$status"
|
|
}}
|
|
trap rollback EXIT HUP INT TERM
|
|
if [ -f "$OWNER_FILE" ]; then
|
|
[ ! -L "$OWNER_FILE" ] && grep -Fqx "$OWNER_MARKER" "$OWNER_FILE" || exit 71
|
|
cp -p "$OWNER_FILE" "$TXN/owner.before"
|
|
fi
|
|
if [ -f "$STATE_FILE" ]; then
|
|
[ ! -L "$STATE_FILE" ] || exit 71
|
|
grep -Fqx 'owner|{DATAPLANE_OWNER_MARKER}' "$STATE_FILE" || exit 71
|
|
cp -p "$STATE_FILE" "$TXN/state.before"
|
|
fi
|
|
for snapshot_family in inet bridge arp; do
|
|
[ ! -e "$NFT_SNAPSHOT_PREFIX.$snapshot_family.live" ] &&
|
|
[ ! -L "$NFT_SNAPSHOT_PREFIX.$snapshot_family.live" ] || exit 71
|
|
done
|
|
"$WG" showconf "$WG_IFACE" > "$TXN/wg.conf"; chmod 600 "$TXN/wg.conf"
|
|
if "$NFT" -s list table inet virt_cloud_wg_fabric > "$TXN/fabric.nft.before" 2>/dev/null; then
|
|
vc_vpc_fabric_nft_shape_file "$TXN/fabric.nft.before" || exit 71
|
|
else
|
|
exit 71
|
|
fi
|
|
if "$NFT" list table inet "$TABLE" > "$TXN/nft.before" 2>/dev/null; then
|
|
grep -Fq '{DATAPLANE_OWNER_MARKER}' "$TXN/nft.before" || exit 71
|
|
else
|
|
rm -f "$TXN/nft.before"
|
|
fi
|
|
if "$NFT" list table bridge "$TABLE" > "$TXN/nft.bridge.before" 2>/dev/null; then
|
|
grep -Fq '{DATAPLANE_OWNER_MARKER}' "$TXN/nft.bridge.before" || exit 71
|
|
else
|
|
rm -f "$TXN/nft.bridge.before"
|
|
fi
|
|
if "$NFT" list table arp "$TABLE" > "$TXN/nft.arp.before" 2>/dev/null; then
|
|
grep -Fq '{DATAPLANE_OWNER_MARKER}' "$TXN/nft.arp.before" || exit 71
|
|
else
|
|
rm -f "$TXN/nft.arp.before"
|
|
fi
|
|
# Preserve a complete stateless/absence baseline before any live nft batch.
|
|
# Rollback uses the same evidence before restoring a sampled-UP ifupdown
|
|
# bridge, and the apply precommit check detects drift during dry-run.
|
|
vc_vpc_capture_nft_precommit "$TXN/nft.precommit.before" || exit 71
|
|
cp -p "$NFT_MAIN" "$TXN/nft_main.before"; NFT_MAIN_BEFORE=yes
|
|
if [ -f "$NFT_FILE" ]; then cp -p "$NFT_FILE" "$TXN/nft_file.before"; NFT_FILE_BEFORE=yes; fi
|
|
if [ -f "$SYSCTL_FILE" ]; then cp -p "$SYSCTL_FILE" "$TXN/sysctl.before"; fi
|
|
if [ -f "$PROXY_FILE" ]; then cp -p "$PROXY_FILE" "$TXN/proxy_file.before"; fi
|
|
vc_vpc_install_rules() {{
|
|
cat > "$TXN/rules.nft" <<'{nft_marker}'
|
|
{nft.rstrip()}
|
|
{nft_marker}
|
|
chmod 600 "$TXN/rules.nft"
|
|
"$NFT" -c -f "$TXN/rules.nft" >/dev/null || exit 72
|
|
NFT_TMP=$(mktemp "/etc/nftables.d/.virt-cloud-vpc-{r.token}.XXXXXX") || exit 72
|
|
cp -p "$TXN/rules.nft" "$NFT_TMP" || exit 72
|
|
chmod 600 "$NFT_TMP"
|
|
mv -f "$NFT_TMP" "$NFT_FILE"; NFT_FILE_CHANGED=yes
|
|
if ! grep -Fqx "$NFT_INCLUDE" "$NFT_MAIN" &&
|
|
! grep -Fqx 'include "/etc/nftables.d/*.nft"' "$NFT_MAIN" &&
|
|
! grep -Fqx 'include "/etc/nftables.d/*"' "$NFT_MAIN"; then
|
|
NFT_MAIN_TMP=$(mktemp "$NFT_MAIN.virt-cloud-vpc.XXXXXX") || exit 72
|
|
cat "$NFT_MAIN" > "$NFT_MAIN_TMP"
|
|
printf '%s\\n' "$NFT_INCLUDE" >> "$NFT_MAIN_TMP"
|
|
chmod "$($STAT -c '%a' "$NFT_MAIN")" "$NFT_MAIN_TMP"
|
|
"$NFT" -c -f "$NFT_MAIN_TMP" >/dev/null || exit 72
|
|
mv -f "$NFT_MAIN_TMP" "$NFT_MAIN"; NFT_MAIN_CHANGED=yes
|
|
fi
|
|
"$NFT" -c -f "$NFT_MAIN" >/dev/null || exit 72
|
|
NFT_BATCH="$TXN/apply.nft"
|
|
: > "$NFT_BATCH"
|
|
if [ -f "$TXN/nft.before" ]; then printf 'delete table inet %s\\n' "$TABLE" >> "$NFT_BATCH"; fi
|
|
if [ -f "$TXN/nft.bridge.before" ]; then printf 'delete table bridge %s\\n' "$TABLE" >> "$NFT_BATCH"; fi
|
|
if [ -f "$TXN/nft.arp.before" ]; then printf 'delete table arp %s\\n' "$TABLE" >> "$NFT_BATCH"; fi
|
|
cat "$TXN/rules.nft" >> "$NFT_BATCH"
|
|
chmod 600 "$NFT_BATCH"
|
|
"$NFT" -c -f "$NFT_BATCH" >/dev/null || exit 72
|
|
# nft check mode does not reserve table names or provide compare-and-swap.
|
|
# Re-read every full stateless table (or proven absence) after dry-run and
|
|
# immediately before the single atomic apply, closing the dry-run mutation
|
|
# window for ordinary races.
|
|
vc_vpc_nft_matches_snapshot "$TXN/nft.precommit.before" || exit 71
|
|
# A single nft batch is atomic in the kernel: the old owned tables remain in
|
|
# place if any delete or new-table rule fails validation/application.
|
|
"$NFT" -f "$NFT_BATCH" >/dev/null || exit 72
|
|
NFT_APPLIED=yes
|
|
# Read back all expected ownership markers, then capture the complete
|
|
# stateless semantics. The latter includes chain policies, every expression
|
|
# and verdict, set elements and rule order; it deliberately excludes only
|
|
# changing packet/byte counters.
|
|
vc_vpc_desired_nft_ready || exit 72
|
|
vc_vpc_capture_nft_snapshot "$TXN/nft.desired" || exit 72
|
|
vc_vpc_nft_matches_snapshot "$TXN/nft.desired" || exit 72
|
|
}}
|
|
# Install and exactly read back the raw tenant guard before restoring link
|
|
# administration in either persistence mode.
|
|
vc_vpc_install_rules
|
|
if "$IP" link show dev "$BRIDGE" >/dev/null 2>&1; then
|
|
"$IP" -d link show dev "$BRIDGE" | grep -Eq '(^|[[:space:]])bridge[[:space:]]+(forward_delay|stp_state|hello_time|ageing_time)([[:space:]]|$)' || exit 71
|
|
[ -f "$OWNER_FILE" ] || exit 71
|
|
else
|
|
"$IP" link add name "$BRIDGE" type bridge
|
|
BRIDGE_CREATED=yes
|
|
printf '%s\\n' "$OWNER_MARKER" > "$OWNER_FILE"; chmod 600 "$OWNER_FILE"
|
|
fi
|
|
bridge_current_link_state=$(vc_vpc_bridge_link_state) || exit 71
|
|
if [ "$bridge_current_link_state" = up ]; then BRIDGE_WAS_UP=yes; fi
|
|
if [ "$BRIDGE_TOPOLOGY_CAPTURED" = yes ]; then
|
|
vc_vpc_bridge_topology_matches || exit 72
|
|
fi
|
|
# Alma/RHEL may create a new bridge with the distribution's strict
|
|
# default=1. Snapshot and converge the panel-owned interface while it is
|
|
# still DOWN; the raw nft guard is already installed and exactly read back.
|
|
vc_vpc_rp_filter_set || exit 72
|
|
cat > "$TXN/sysctl.desired" <<'__VC_VPC_SYSCTL_{r.token}'
|
|
# virt-cloud-vpc-dataplane-v1
|
|
# vpc={plan.vpc.vpc_id} revision={plan.desired_revision} digest={plan.desired_digest}
|
|
net.ipv4.conf.{r.bridge}.rp_filter = 2
|
|
net.ipv4.conf.{r.bridge}.proxy_arp = 0
|
|
net.ipv4.conf.{r.bridge}.proxy_arp_pvlan = 1
|
|
__VC_VPC_SYSCTL_{r.token}
|
|
chmod 600 "$TXN/sysctl.desired"
|
|
vc_vpc_sysctl_artifact_owned "$TXN/sysctl.desired" {_q(plan.vpc.vpc_id)} \
|
|
{plan.desired_revision} {_q(plan.desired_digest)} "$BRIDGE" || exit 72
|
|
SYSCTL_TMP=$(mktemp "/etc/sysctl.d/.virt-cloud-vpc-{r.token}.XXXXXX") || exit 72
|
|
cp -p "$TXN/sysctl.desired" "$SYSCTL_TMP" || exit 72
|
|
chmod 600 "$SYSCTL_TMP" || exit 72
|
|
if [ -f "$TXN/sysctl.before" ]; then
|
|
vc_vpc_sysctl_artifact_owned "$SYSCTL_FILE" {_q(plan.vpc.vpc_id)} \
|
|
"$state_revision" "$state_digest" "$BRIDGE" || exit 71
|
|
"$CMP" -s "$SYSCTL_FILE" "$TXN/sysctl.before" || exit 71
|
|
else
|
|
[ ! -e "$SYSCTL_FILE" ] && [ ! -L "$SYSCTL_FILE" ] || exit 71
|
|
fi
|
|
mv -f "$SYSCTL_TMP" "$SYSCTL_FILE" || exit 72
|
|
SYSCTL_TMP=
|
|
SYSCTL_CHANGED=yes
|
|
vc_vpc_sysctl_artifacts_equal "$SYSCTL_FILE" "$TXN/sysctl.desired" \
|
|
{_q(plan.vpc.vpc_id)} {plan.desired_revision} {_q(plan.desired_digest)} \
|
|
"$BRIDGE" || exit 72
|
|
"$IP" link set dev "$BRIDGE" up
|
|
# A private bridge is an unnumbered, panel-owned L3 boundary. Reject an
|
|
# address, a default/connected route, or a physical/foreign bridge port before
|
|
# installing any policy. Libvirt tap names are the only accepted members;
|
|
# attaching br0, an underlay NIC, or an operator bridge would make VPC traffic
|
|
# escape into the public network.
|
|
bridge_addrs=$($IP -o addr show dev "$BRIDGE" 2>/dev/null || true)
|
|
printf '%s\\n' "$bridge_addrs" | grep -Eq '[[:space:]]inet6?[[:space:]]' && exit 71 || true
|
|
bridge_ports=$($BRIDGE_CMD link show master "$BRIDGE" 2>/dev/null) || exit 71
|
|
while IFS= read -r bridge_port_line; do
|
|
[ -n "$bridge_port_line" ] || continue
|
|
bridge_port=$(printf '%s\\n' "$bridge_port_line" | sed -n 's/^[0-9][0-9]*: \\([^:[:space:]]*\\).*/\\1/p')
|
|
[ -n "$bridge_port" ] || exit 71
|
|
case "$bridge_port" in
|
|
vnet[0-9]*|tap[0-9A-Za-z_.-]*|macvtap[0-9]*|fwpr[0-9A-Za-z_.-]*|fwln[0-9A-Za-z_.-]*) : ;;
|
|
*) exit 71 ;;
|
|
esac
|
|
done <<__VC_VPC_BRIDGE_PORTS__
|
|
$bridge_ports
|
|
__VC_VPC_BRIDGE_PORTS__
|
|
cat > "$TXN/bridge.routes.allowed" <<'{route_marker}.bridge'
|
|
{bridge_route_lines}
|
|
{route_marker}.bridge
|
|
if [ -f "$TXN/state.before" ]; then
|
|
awk -F'|' '$1 == "route" && $3 == "{r.bridge}" {{print $2 "|" $3 "|" $4 "|" $5 "|" $6 "|186"}}' "$TXN/state.before" >> "$TXN/bridge.routes.allowed"
|
|
fi
|
|
sort -u "$TXN/bridge.routes.allowed" -o "$TXN/bridge.routes.allowed"
|
|
bridge_routes=$($IP -o -4 route show table all dev "$BRIDGE" 2>/dev/null) || exit 71
|
|
while IFS='|' read -r destination device table via metric proto; do
|
|
[ -n "$destination" ] || continue
|
|
printf '%s\\n' "$destination|$device|$table|$via|$metric|$proto" | grep -Fqx -f "$TXN/bridge.routes.allowed" || exit 71
|
|
done <<EOF
|
|
$(printf '%s\\n' "$bridge_routes" | awk '{{
|
|
destination=$1; device=""; via="-"; metric="100"; table="main"; proto=""
|
|
for (i=1;i<=NF;i++) {{
|
|
if ($i=="dev" && i<NF) device=$(i+1)
|
|
if ($i=="via" && i<NF) via=$(i+1)
|
|
if ($i=="metric" && i<NF) metric=$(i+1)
|
|
if ($i=="table" && i<NF) table=$(i+1)
|
|
if ($i=="proto" && i<NF) proto=$(i+1)
|
|
}}
|
|
if (table=="254") table="main"
|
|
if (destination != "") printf "%s|%s|%s|%s|%s|%s\\n", destination, device, table, via, metric, proto
|
|
}}')
|
|
EOF
|
|
bridge_routes6=$($IP -o -6 route show table all dev "$BRIDGE" 2>/dev/null) || exit 71
|
|
[ -z "$bridge_routes6" ] || exit 71
|
|
: > "$TXN/proxy.before"
|
|
"$IP" neigh show proxy dev "$BRIDGE" 2>/dev/null | awk '$1 ~ /^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$/ {{print $1}}' | LC_ALL=C sort -u > "$TXN/proxy.before"
|
|
if [ -f "$PROXY_FILE" ]; then
|
|
sed '1d' "$PROXY_FILE" | LC_ALL=C sort -u > "$TXN/proxy.file.before"
|
|
else
|
|
: > "$TXN/proxy.file.before"
|
|
fi
|
|
"$CMP" -s "$TXN/proxy.before" "$TXN/proxy.file.before" || exit 71
|
|
if [ -r "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp" ]; then
|
|
cat "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp" > "$TXN/proxy_arp.before"
|
|
else
|
|
exit 72
|
|
fi
|
|
if [ -r "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp_pvlan" ]; then
|
|
cat "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp_pvlan" > "$TXN/proxy_arp_pvlan.before"
|
|
else
|
|
exit 72
|
|
fi
|
|
if [ "$MODE" = networkmanager_nmcli ]; then
|
|
[ -n "$NMCLI" ] || exit 70
|
|
NM_PROFILE_UUID=
|
|
if vc_nm_profile_resolve; then
|
|
vc_nm_profile_base_validate || exit 71
|
|
else
|
|
nm_resolve_status=$?
|
|
[ "$nm_resolve_status" = 2 ] || exit 71
|
|
NM_PROFILE_UUID=
|
|
"$NMCLI" connection add type bridge ifname "$BRIDGE" con-name "$BRIDGE" \
|
|
ipv4.method disabled ipv6.method disabled bridge.stp no \
|
|
connection.autoconnect no connection.autoconnect-retries 0 \
|
|
connection.stable-id "$NM_PROFILE_MARKER" >/dev/null
|
|
NM_CREATED=yes
|
|
vc_nm_profile_resolve || exit 72
|
|
vc_nm_profile_base_validate || exit 72
|
|
NM_CREATED_UUID=$NM_PROFILE_UUID
|
|
fi
|
|
vc_nm_profile_empty connection.zone || exit 71
|
|
vc_nm_profile_resolve || exit 71
|
|
vc_nm_profile_base_validate || exit 71
|
|
NM_ACTIVATION_ATTEMPTED=yes
|
|
if [ "$NM_DEVICE_QUARANTINED" = yes ]; then
|
|
vc_nm_device_release || exit 72
|
|
NM_DEVICE_QUARANTINED=no
|
|
fi
|
|
"$NMCLI" connection up uuid "$NM_PROFILE_UUID" >/dev/null
|
|
vc_vpc_rp_filter_set || exit 72
|
|
else
|
|
[ -x "$IFUP" ] && [ -x "$IFQUERY" ] && [ -x "$IFDOWN" ] || exit 70
|
|
[ ! -L /etc/network ] && [ ! -L /etc/network/interfaces.d ] || exit 71
|
|
# A stanza in the administrator-owned main file wins over an
|
|
# interfaces.d fragment on some ifupdown versions. Refuse that ambiguous
|
|
# ownership instead of letting a same-name bridge inherit ports, addresses,
|
|
# routes, or hooks from /etc/network/interfaces.
|
|
if [ -f /etc/network/interfaces ] &&
|
|
grep -Eq '^[[:space:]]*(auto|allow-[^[:space:]]+|iface)[[:space:]]+'"$BRIDGE"'([[:space:]]|$)' /etc/network/interfaces; then
|
|
exit 71
|
|
fi
|
|
mkdir -p /etc/network/interfaces.d
|
|
if [ -e "$IF_FRAGMENT" ] || [ -L "$IF_FRAGMENT" ]; then
|
|
[ -f "$IF_FRAGMENT" ] && [ ! -L "$IF_FRAGMENT" ] || exit 71
|
|
grep -Fq '# {DATAPLANE_OWNER_MARKER} vpc=' "$IF_FRAGMENT" || exit 71
|
|
cp -p "$IF_FRAGMENT" "$TXN/if_fragment.before"
|
|
fi
|
|
IF_TMP=$(mktemp "/etc/network/interfaces.d/.virt-cloud-vpc-{r.token}.XXXXXX") || exit 72
|
|
cat > "$IF_TMP" <<'{frag_marker}'
|
|
# {DATAPLANE_OWNER_MARKER} vpc={plan.vpc.vpc_id} revision={plan.desired_revision} digest={plan.desired_digest}
|
|
{ifup_auto_line}iface {r.bridge} inet manual
|
|
bridge_ports none
|
|
bridge-stp off
|
|
{frag_marker}
|
|
chmod 600 "$IF_TMP"
|
|
mv -f "$IF_TMP" "$IF_FRAGMENT"; IF_FRAGMENT_CHANGED=yes
|
|
# Classic Debian ifquery accepts the interface as its positional operand;
|
|
# `--list <iface>` is an ifquery2/listing form and fails on ifupdown.
|
|
"$IFQUERY" "$BRIDGE" >/dev/null 2>&1 || exit 72
|
|
# Rollback never calls ifdown: even a bridge created by this transaction may
|
|
# have acquired a libvirt tap before a later step fails. The rollback path
|
|
# below inspects ports first, then deletes only an empty newly-created
|
|
# bridge or restores an empty existing bridge to its sampled DOWN state.
|
|
VC_FABRIC_APPLYING=1 "$IFUP" --force "$BRIDGE" >/dev/null 2>&1 || exit 72
|
|
vc_vpc_rp_filter_set || exit 72
|
|
fi
|
|
PROXY_ARP_CHANGED=yes
|
|
printf '%s\\n' 0 > "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp" || exit 72
|
|
[ "$(cat "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp" 2>/dev/null)" = 0 ] || exit 72
|
|
PROXY_ARP_PVLAN_CHANGED=yes
|
|
printf '%s\\n' 1 > "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp_pvlan" || exit 72
|
|
[ "$(cat "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp_pvlan" 2>/dev/null)" = 1 ] || exit 72
|
|
PROXY_ENTRIES_CHANGED=yes
|
|
while IFS= read -r old_proxy; do
|
|
[ -n "$old_proxy" ] || continue
|
|
"$IP" neigh del proxy "$old_proxy" dev "$BRIDGE" >/dev/null 2>&1 || exit 72
|
|
done < "$TXN/proxy.before"
|
|
while IFS= read -r proxy_ip; do
|
|
[ -n "$proxy_ip" ] || continue
|
|
"$IP" neigh replace proxy "$proxy_ip" dev "$BRIDGE" >/dev/null 2>&1 || exit 72
|
|
done <<'{proxy_marker}'
|
|
{proxy_lines}
|
|
{proxy_marker}
|
|
PROXY_TMP=$(mktemp "$PROXY_FILE.XXXXXX") || exit 72
|
|
{{
|
|
printf '%s\\n' '# virt-cloud-vpc-proxy-v1'
|
|
cat <<'{proxy_marker}'
|
|
{proxy_lines}
|
|
{proxy_marker}
|
|
}} > "$PROXY_TMP"
|
|
chmod 600 "$PROXY_TMP"
|
|
mv -f "$PROXY_TMP" "$PROXY_FILE"; PROXY_FILE_CHANGED=yes
|
|
"$IP" neigh show proxy dev "$BRIDGE" 2>/dev/null | awk '$1 ~ /^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$/ {{print $1}}' | LC_ALL=C sort -u > "$TXN/proxy.after"
|
|
cat <<'{proxy_marker}' | LC_ALL=C sort -u > "$TXN/proxy.desired"
|
|
{proxy_lines}
|
|
{proxy_marker}
|
|
"$CMP" -s "$TXN/proxy.after" "$TXN/proxy.desired" || exit 72
|
|
: > "$TXN/routes.before"
|
|
cat > "$TXN/routes.desired" <<'{route_marker}'
|
|
{route_lines}
|
|
{route_marker}
|
|
if [ -f "$STATE_FILE" ]; then
|
|
while IFS='|' read -r old_kind old_destination old_device old_table old_via old_metric; do
|
|
[ "$old_kind" = route ] || continue
|
|
printf '%s\\n' "$old_destination|$old_device|$ROUTE_TABLE|${{old_via:--}}|${{old_metric:-100}}" | grep -Fqx -f "$TXN/routes.desired" && continue
|
|
old_existing=$("$IP" -o route show table "$ROUTE_TABLE" "$old_destination" 2>/dev/null || true)
|
|
[ -n "$old_existing" ] || continue
|
|
printf '%s\\n' "$old_existing" | grep -Eq 'proto 186([[:space:]]|$)' || exit 73
|
|
old_metric_actual=$(printf '%s\\n' "$old_existing" | awk '{{for (i=1; i<=NF; i++) if ($i == "metric" && i < NF) {{print $(i+1); exit}}}}')
|
|
[ -n "$old_metric_actual" ] || old_metric_actual=100
|
|
old_actual_device=$(printf '%s\\n' "$old_existing" | awk '{{for (i=1; i<=NF; i++) if ($i == "dev" && i < NF) {{print $(i+1); exit}}}}')
|
|
[ "$old_actual_device" = "$old_device" ] || exit 73
|
|
old_via_actual=$(printf '%s\\n' "$old_existing" | awk '{{for (i=1; i<=NF; i++) if ($i == "via" && i < NF) {{print $(i+1); exit}}}}')
|
|
[ "${{old_via_actual:--}}" = "${{old_via:--}}" ] || exit 73
|
|
printf '%s|%s|%s|%s|%s\\n' "$old_destination" "$old_device" "$ROUTE_TABLE" "${{old_via:--}}" "$old_metric_actual" >> "$TXN/routes.before"
|
|
if [ "${{old_via:--}}" = - ]; then
|
|
"$IP" route del "$old_destination" table "$ROUTE_TABLE" dev "$old_device" proto 186 || exit 73
|
|
else
|
|
"$IP" route del "$old_destination" table "$ROUTE_TABLE" via "$old_via" dev "$old_device" proto 186 || exit 73
|
|
fi
|
|
done < "$STATE_FILE"
|
|
fi
|
|
while IFS='|' read -r destination device table via metric; do
|
|
[ -n "$destination" ] || continue
|
|
existing=$("$IP" -o route show table "$table" "$destination" 2>/dev/null || true)
|
|
[ -z "$existing" ] || printf '%s\\n' "$existing" | grep -Eq 'proto 186([[:space:]]|$)' || exit 73
|
|
if [ -n "$existing" ]; then
|
|
old_device=$(printf '%s\\n' "$existing" | awk '{{for (i=1; i<=NF; i++) if ($i == "dev" && i < NF) {{print $(i+1); exit}}}}')
|
|
old_metric=$(printf '%s\\n' "$existing" | awk '{{for (i=1; i<=NF; i++) if ($i == "metric" && i < NF) {{print $(i+1); exit}}}}')
|
|
[ -n "$old_device" ] || exit 73
|
|
[ "$old_device" = "$device" ] || exit 73
|
|
old_via=$(printf '%s\\n' "$existing" | awk '{{for (i=1; i<=NF; i++) if ($i == "via" && i < NF) {{print $(i+1); exit}}}}')
|
|
[ "${{old_via:--}}" = "$via" ] || exit 73
|
|
[ -n "$old_metric" ] || old_metric=100
|
|
printf '%s|%s|%s|%s|%s\\n' "$destination" "$old_device" "$table" "${{old_via:--}}" "$old_metric" >> "$TXN/routes.before"
|
|
fi
|
|
[ -n "$metric" ] || metric=100
|
|
if [ "$via" = - ]; then
|
|
"$IP" route replace "$destination" table "$table" dev "$device" proto 186 metric "$metric" || exit 73
|
|
else
|
|
"$IP" route replace "$destination" table "$table" via "$via" dev "$device" onlink proto 186 metric "$metric" || exit 73
|
|
fi
|
|
done <<'{route_marker}'
|
|
{route_lines}
|
|
{route_marker}
|
|
# Recheck the complete RPDB scope after route convergence, immediately before
|
|
# adding missing rules. An out-of-band rule created during the transaction
|
|
# must force rollback rather than make a successful apply read back degraded.
|
|
# vc-policy-rule-preflight-begin
|
|
: > "$TXN/rules.before"
|
|
: > "$TXN/rules.attempted"
|
|
: > "$TXN/rules.created"
|
|
chmod 600 "$TXN/rules.before" "$TXN/rules.attempted" "$TXN/rules.created"
|
|
vc_ip_rule_scope_check {_q(plan.vpc.cidr)} "$ROUTE_TABLE" "$RULE_PRIORITY" "$WG_IFACE" managed 73 73 || exit $?
|
|
vc_ip_rule_preflight "$RULE_PRIORITY" - {_q(plan.vpc.cidr)} "$ROUTE_TABLE" 73 73 "$TXN/rules.before" || exit $?
|
|
vc_ip_rule_preflight "$WG_RULE_PRIORITY" "$WG_IFACE" {_q(plan.vpc.cidr)} "$ROUTE_TABLE" 73 73 "$TXN/rules.before" || exit $?
|
|
# vc-policy-rule-preflight-end
|
|
# vc-policy-rule-ensure-begin
|
|
vc_ip_rule_ensure "$RULE_PRIORITY" - {_q(plan.vpc.cidr)} "$ROUTE_TABLE" 73 73 "$TXN/rules.attempted" "$TXN/rules.created" || exit $?
|
|
vc_ip_rule_ensure "$WG_RULE_PRIORITY" "$WG_IFACE" {_q(plan.vpc.cidr)} "$ROUTE_TABLE" 73 73 "$TXN/rules.attempted" "$TXN/rules.created" || exit $?
|
|
# vc-policy-rule-ensure-end
|
|
cat > "$STATE_FILE" <<'{state_marker}'
|
|
{state.rstrip()}
|
|
{state_marker}
|
|
chmod 600 "$STATE_FILE"
|
|
{state_loader}
|
|
vpc_state_load || exit 74
|
|
FABRIC_NFT_APPLIED=yes
|
|
vpc_state_refresh_nft || exit 74
|
|
vc_vpc_fabric_nft_live_ready || exit 74
|
|
vpc_state_nft_set_matches || exit 74
|
|
vc_nft_table_snapshot inet virt_cloud_wg_fabric \
|
|
"$TXN/fabric.nft.desired" || exit 74
|
|
[ -f "$TXN/fabric.nft.desired" ] &&
|
|
[ ! -e "$TXN/fabric.nft.desired.absent" ] || exit 74
|
|
vc_nft_table_matches_snapshot inet virt_cloud_wg_fabric \
|
|
"$TXN/fabric.nft.desired" || exit 74
|
|
while IFS='|' read -r peer_host key allowed; do
|
|
[ -n "$peer_host" ] || continue
|
|
"$WG" show "$WG_IFACE" peers | grep -Fqx "$key" || exit 74
|
|
case ",$allowed," in *',0.0.0.0/0,'*|*',0.0.0.0/1,'*|*',128.0.0.0/1,'* ) exit 74 ;; esac
|
|
# Fabric has one AllowedIPs list per public key. Merge the complete,
|
|
# validated tenant set from every panel-owned VPC state file while keeping
|
|
# the peer's single overlay /32. Never replace the list with this VPC's
|
|
# partial view: doing so would black-hole another concurrently active VPC.
|
|
merge_actual=$("$WG" show "$WG_IFACE" allowed-ips 2>/dev/null | awk -v wanted="$key" '$1 == wanted {{print $2; exit}}' || true)
|
|
merge_overlay=$(printf '%s\\n' "$merge_actual" | tr ',' '\\n' | awk '/^100\\.64\\.[0-9]+\\.[0-9]+\\/32$/ {{print; count++}} END {{exit count == 1 ? 0 : 1}}') || exit 74
|
|
merge_extra=$(vpc_state_allowed_for_peer "$key") || exit 74
|
|
merge_allowed="$merge_overlay"
|
|
[ -z "$merge_extra" ] || merge_allowed="$merge_allowed,$merge_extra"
|
|
"$WG" set "$WG_IFACE" peer "$key" allowed-ips "$merge_allowed" || exit 74
|
|
done <<'{peer_marker}'
|
|
{peer_lines}
|
|
{peer_marker}
|
|
# The shared Fabric quarantine table is the last enforcement boundary before
|
|
# a VPC bridge is admitted. Reparse the complete stateless grammar,
|
|
# bind every owner comment to the engine-authenticated full Fabric digest,
|
|
# compare the exact tenant /32 set derived by the state loader, and finally
|
|
# prove the whole table is still the post-refresh snapshot. A retained
|
|
# comment with altered drop semantics, a range element, or a late mutation
|
|
# therefore fails closed.
|
|
vc_vpc_fabric_nft_live_ready || exit 74
|
|
vpc_state_nft_set_matches || exit 74
|
|
vc_nft_table_matches_snapshot inet virt_cloud_wg_fabric \
|
|
"$TXN/fabric.nft.desired" || exit 74
|
|
vc_vpc_nft_matches_snapshot "$TXN/nft.desired" || exit 74
|
|
rm -f "$VPC_PEER_FILE" "$VPC_ROUTE_FILE" "$VPC_TENANT_FILE" "$VPC_TENANT_FILE.nft"
|
|
printf '%s\\n' ready > "$TXN/state"; chmod 600 "$TXN/state"
|
|
printf '%s\\n' '__VC_VPC_APPLIED__=yes'
|
|
printf '%s\\n' '__VC_VPC_OWNER__={DATAPLANE_OWNER_MARKER}'
|
|
printf '%s\\n' '__VC_VPC_REVISION__={plan.desired_revision}'
|
|
printf '%s\\n' '__VC_VPC_DIGEST__={plan.desired_digest}'
|
|
COMMITTED=yes
|
|
printf '%s\\n' committed > "$TXN/state"; chmod 600 "$TXN/state"
|
|
rm -rf "$TXN"
|
|
trap - EXIT HUP INT TERM
|
|
exit 0
|
|
'''
|
|
if len(script) > MAX_OUTPUT:
|
|
raise VpcDataplaneError("VPC 数据面脚本过大", 413, "vpc_script_too_large")
|
|
return script
|
|
|
|
|
|
def render_remove_script(
|
|
plan: VpcDataplanePlan,
|
|
*,
|
|
deployment_mode: str = "networkmanager_nmcli",
|
|
firewall_mode: str = FIREWALL_MODE_NATIVE,
|
|
ifup_path: str = "/usr/sbin/ifup",
|
|
ifdown_path: str = "/usr/sbin/ifdown",
|
|
ifquery_path: str = "/usr/sbin/ifquery",
|
|
) -> str:
|
|
"""Render a fail-closed removal transaction for an empty host."""
|
|
if plan.local_members:
|
|
raise VpcDataplaneError(
|
|
"仍有本机私网成员,不能执行宿主机 VPC 销毁",
|
|
409, "vpc_remove_has_local_members")
|
|
if deployment_mode not in {"networkmanager_nmcli", "ifupdown"}:
|
|
raise VpcDataplaneError("宿主机网络管理模式不支持", 409,
|
|
"vpc_dataplane_network_manager_conflict")
|
|
firewall_mode = _firewall_mode(firewall_mode)
|
|
allowed_ifup = {"/usr/sbin/ifup", "/usr/bin/ifup", "/sbin/ifup", "/bin/ifup"}
|
|
allowed_ifdown = {"/usr/sbin/ifdown", "/usr/bin/ifdown", "/sbin/ifdown", "/bin/ifdown"}
|
|
allowed_ifquery = {"/usr/sbin/ifquery", "/usr/bin/ifquery", "/sbin/ifquery", "/bin/ifquery"}
|
|
if (ifup_path not in allowed_ifup or ifdown_path not in allowed_ifdown
|
|
or ifquery_path not in allowed_ifquery):
|
|
raise VpcDataplaneError("ifupdown 工具路径不受支持", 409,
|
|
"vpc_dataplane_network_manager_conflict")
|
|
from .fabric import (
|
|
_FABRIC_NFT_VALIDATION_SH,
|
|
_FIREWALL_MODE_GUARD_SH,
|
|
_NMCLI_FILENAME_RESOLVER_SH,
|
|
_VPC_STATE_LOADER,
|
|
)
|
|
|
|
r = plan.resources
|
|
include = f"/etc/nftables.d/virt-cloud-vpc-{r.token}.nft"
|
|
fragment = f"/etc/network/interfaces.d/virt-cloud-vpc-{r.token}"
|
|
sysctl_file = f"/etc/sysctl.d/99-virt-cloud-vpc-{r.token}.conf"
|
|
proxy_file = f"/etc/virt-cloud/vpc-dataplane/{r.token}.proxy"
|
|
state_loader = _VPC_STATE_LOADER.replace(
|
|
"__VC_FABRIC_HOST_ID__", plan.host_id).replace(
|
|
"__VC_FABRIC_ID__", plan.vpc.fabric_id).replace(
|
|
"__VC_FABRIC_NETWORK__", "100.64.0.0/10").replace(
|
|
"__VC_VPC_STATE_OBSERVE__", "no")
|
|
nm_profile_marker = DATAPLANE_OWNER_MARKER + " vpc=" + plan.vpc.vpc_id
|
|
ifup_marker = f"# {DATAPLANE_OWNER_MARKER} vpc={plan.vpc.vpc_id}"
|
|
inflight_marker = (f"{DATAPLANE_OWNER_MARKER}:{plan.vpc.vpc_id}:"
|
|
f"{plan.host_id}:{plan.desired_revision}:{plan.desired_digest}")
|
|
script = f'''#!/bin/sh
|
|
set -eu
|
|
umask 077
|
|
PATH=/usr/sbin:/usr/bin:/sbin:/bin
|
|
export PATH
|
|
IP=$(for p in /usr/sbin/ip /usr/bin/ip /sbin/ip /bin/ip; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
|
|
WG=$(for p in /usr/bin/wg /usr/sbin/wg /bin/wg /sbin/wg; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
|
|
NFT=$(for p in /usr/sbin/nft /usr/bin/nft /sbin/nft /bin/nft; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
|
|
FLOCK=$(for p in /usr/bin/flock /usr/sbin/flock /bin/flock /sbin/flock; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
|
|
STAT=$(for p in /usr/bin/stat /bin/stat /usr/sbin/stat /sbin/stat; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
|
|
CMP=$(for p in /usr/bin/cmp /bin/cmp /usr/sbin/cmp /sbin/cmp; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
|
|
MKTEMP=$(for p in /usr/bin/mktemp /bin/mktemp /usr/sbin/mktemp /sbin/mktemp; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
|
|
BRIDGE_CMD=$(for p in /usr/sbin/bridge /usr/bin/bridge /sbin/bridge /bin/bridge; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
|
|
SYSTEMCTL=$(for p in /usr/bin/systemctl /bin/systemctl /usr/sbin/systemctl /sbin/systemctl; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
|
|
{IP_RULE_CONVERGENCE_SH}
|
|
BRIDGE={_q(r.bridge)}; TABLE={_q(r.nft_table)}; ROUTE_TABLE={r.route_table}; WG_IFACE={_q(FABRIC_INTERFACE)}
|
|
OWNER_FILE={_q(r.owner_file)}; STATE_FILE={_q(r.state_file)}; LOCK={_q(r.lock_file)}; TXN={_q(r.transaction_dir)}
|
|
NFT_SNAPSHOT_PREFIX={_q(f'/etc/virt-cloud/vpc-dataplane/{r.token}.nft')}
|
|
NFT_FILE={_q(include)}; LEGACY_MANIFEST={_q(LEGACY_FIREWALLD_NFT_MANIFEST)}; IF_FRAGMENT={_q(fragment)}; SYSCTL_FILE={_q(sysctl_file)}; PROXY_FILE={_q(proxy_file)}; STATE_DIR=/etc/virt-cloud/vpc-dataplane
|
|
MODE={_q(deployment_mode)}; FIREWALL_MODE={_q(firewall_mode)}; IFUP={_q(ifup_path)}; IFDOWN={_q(ifdown_path)}; IFQUERY={_q(ifquery_path)}; NMCLI=; NM_PROFILE_UUID=; NM_PROFILE_FILE=; NFT_MAIN=; NFT_INCLUDE={_q(f'include "{include}"')}
|
|
NM_PROFILE_MARKER={_q(nm_profile_marker)}
|
|
EXPECTED_DIGEST={_q(plan.fabric_plan_digest)}
|
|
{NFT_LIVE_SNAPSHOT_SH}
|
|
{_NMCLI_FILENAME_RESOLVER_SH}
|
|
{NETWORKMANAGER_VPC_PROFILE_SH}
|
|
{_FIREWALL_MODE_GUARD_SH}
|
|
{_FABRIC_NFT_VALIDATION_SH}
|
|
{_VPC_SYSCTL_VALIDATION_SH}
|
|
{VPC_NATIVE_TRANSACTION_GUARD_SH}
|
|
nft_plan_owned() {{
|
|
nft_plan_text=$1
|
|
nft_plan_digest=$2
|
|
printf '%s\\n' "$nft_plan_digest" | grep -Eq '^[0-9a-f]{{64}}$' || return 1
|
|
vc_fabric_nft_strict_shape "$nft_plan_text" || return 1
|
|
[ "$(printf '%s\\n' "$nft_plan_text" |
|
|
grep -Foc 'comment "virt-cloud-wg-fabric-v1:'"$nft_plan_digest"'"' || true)" = 8 ]
|
|
}}
|
|
vc_vpc_fabric_nft_live_ready() {{
|
|
vc_remove_fabric_text=$(
|
|
"$NFT" -s list table inet virt_cloud_wg_fabric 2>/dev/null
|
|
) || return 1
|
|
nft_plan_owned "$vc_remove_fabric_text" "$EXPECTED_DIGEST"
|
|
}}
|
|
[ -n "$IP" ] && [ -n "$WG" ] && [ -n "$NFT" ] && [ -n "$FLOCK" ] && [ -n "$STAT" ] && [ -n "$CMP" ] && [ -n "$MKTEMP" ] && [ -n "$BRIDGE_CMD" ] && [ -n "$SYSTEMCTL" ] || exit 70
|
|
if [ "$MODE" = networkmanager_nmcli ]; then
|
|
NMCLI=$(for p in /usr/bin/nmcli /usr/sbin/nmcli /bin/nmcli /sbin/nmcli; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
|
|
[ -n "$NMCLI" ] || exit 70
|
|
fi
|
|
[ "$FIREWALL_MODE" = native_nftables ] || exit 71
|
|
[ ! -L /run/virt-cloud ] && [ ! -L /etc/virt-cloud ] || exit 71
|
|
mkdir -p /run/virt-cloud; chmod 700 /run/virt-cloud
|
|
if [ -e "$STATE_DIR" ] || [ -L "$STATE_DIR" ]; then
|
|
[ -d "$STATE_DIR" ] && [ ! -L "$STATE_DIR" ] || exit 71
|
|
[ "$($STAT -c '%u:%a' "$STATE_DIR" 2>/dev/null)" = 0:700 ] || exit 71
|
|
fi
|
|
if [ -e /etc/nftables.d ] || [ -L /etc/nftables.d ]; then
|
|
[ -d /etc/nftables.d ] && [ ! -L /etc/nftables.d ] || exit 71
|
|
[ "$($STAT -c '%u:%a' /etc/nftables.d 2>/dev/null)" = 0:755 ] || exit 71
|
|
fi
|
|
if [ -e /etc/sysctl.d ] || [ -L /etc/sysctl.d ]; then
|
|
[ -d /etc/sysctl.d ] && [ ! -L /etc/sysctl.d ] || exit 71
|
|
[ "$($STAT -c '%u:%a' /etc/sysctl.d 2>/dev/null)" = 0:755 ] || exit 71
|
|
fi
|
|
[ ! -L "$LOCK" ] && {{ [ ! -e "$LOCK" ] || [ -f "$LOCK" ]; }} || exit 71
|
|
: >> "$LOCK"; chmod 600 "$LOCK"
|
|
exec 8>>{_q(FABRIC_LOCK)}; "$FLOCK" -x 8
|
|
exec 9>>"$LOCK"; "$FLOCK" -x 9
|
|
vc_vpc_firewall_boundary_clean || exit 79
|
|
for vc_vpc_legacy_snapshot in inet bridge arp; do
|
|
[ ! -e "$NFT_SNAPSHOT_PREFIX.$vc_vpc_legacy_snapshot.live" ] &&
|
|
[ ! -L "$NFT_SNAPSHOT_PREFIX.$vc_vpc_legacy_snapshot.live" ] || exit 79
|
|
done
|
|
if [ -e "$TXN" ] || [ -L "$TXN" ]; then
|
|
vc_vpc_native_transaction_owned || exit 79
|
|
fi
|
|
REMOVE_BRIDGE_TOPOLOGY_CAPTURED=no; REMOVE_BRIDGE_PORTS_BEFORE=
|
|
vc_remove_bridge_port_names() {{
|
|
vc_remove_ports_raw=$("$BRIDGE_CMD" link show master "$BRIDGE" 2>/dev/null) || return 1
|
|
vc_remove_ports_parsed=$(printf '%s\\n' "$vc_remove_ports_raw" | awk '
|
|
NF == 0 {{next}}
|
|
NF < 2 {{bad=1; next}}
|
|
{{name=$2; sub(/:$/, "", name); sub(/@.*/, "", name)}}
|
|
name !~ /^[A-Za-z0-9_.-]+$/ {{bad=1; next}}
|
|
{{print name}}
|
|
END {{if (bad) exit 1}}
|
|
') || return 1
|
|
printf '%s\\n' "$vc_remove_ports_parsed" | LC_ALL=C sort -u
|
|
}}
|
|
vc_remove_bridge_topology_matches() {{
|
|
[ "$REMOVE_BRIDGE_TOPOLOGY_CAPTURED" = yes ] || return 1
|
|
vc_remove_ports_now=$(vc_remove_bridge_port_names) || return 1
|
|
[ "$vc_remove_ports_now" = "$REMOVE_BRIDGE_PORTS_BEFORE" ]
|
|
}}
|
|
PREVIOUS_FIREWALL_MODE=absent
|
|
if [ -f "$STATE_FILE" ]; then
|
|
# Minimum state/manager proof is the only precondition for the hard link
|
|
# quarantine. Runtime firewall, Fabric, journal, artifacts and RPDB are
|
|
# intentionally checked only after the panel bridge is administratively
|
|
# down, so combined drift cannot pass tenant traffic.
|
|
[ ! -L "$STATE_FILE" ] &&
|
|
[ "$($STAT -c '%u:%a' "$STATE_FILE" 2>/dev/null)" = 0:600 ] || exit 71
|
|
grep -Fqx 'owner|{DATAPLANE_OWNER_MARKER}' "$STATE_FILE" || exit 71
|
|
grep -Fqx 'vpc|{plan.vpc.vpc_id}' "$STATE_FILE" || exit 71
|
|
grep -Fqx 'fabric|{plan.vpc.fabric_id}' "$STATE_FILE" || exit 71
|
|
grep -Fqx 'host|{plan.host_id}' "$STATE_FILE" || exit 71
|
|
grep -Fqx 'bridge|{r.bridge}' "$STATE_FILE" || exit 71
|
|
grep -Fqx 'table|{r.route_table}' "$STATE_FILE" || exit 71
|
|
state_firewall_quarantine=$(awk -F'|' '$1=="firewall_mode" {{count++; value=$2}} END {{print (count + 0) "|" value}}' "$STATE_FILE")
|
|
case "$state_firewall_quarantine" in
|
|
'0|'|'1|native_nftables') PREVIOUS_FIREWALL_MODE=native_nftables ;;
|
|
*) exit 71 ;;
|
|
esac
|
|
quarantine_revision=$(awk -F'|' '$1=="revision" {{count++; value=$2}} END {{print (count + 0) "|" value}}' "$STATE_FILE")
|
|
quarantine_digest=$(awk -F'|' '$1=="digest" {{count++; value=$2}} END {{print (count + 0) "|" value}}' "$STATE_FILE")
|
|
[ "${{quarantine_revision%%|*}}" = 1 ] || exit 71
|
|
case "${{quarantine_revision#*|}}" in ''|*[!0-9]*) exit 71 ;; esac
|
|
[ "${{quarantine_digest%%|*}}" = 1 ] || exit 71
|
|
printf '%s\\n' "${{quarantine_digest#*|}}" | grep -Eq '^[0-9a-f]{{64}}$' || exit 71
|
|
[ -f "$OWNER_FILE" ] && [ ! -L "$OWNER_FILE" ] &&
|
|
[ "$($STAT -c '%u:%a' "$OWNER_FILE" 2>/dev/null)" = 0:600 ] &&
|
|
grep -Fqx '{DATAPLANE_OWNER_MARKER}' "$OWNER_FILE" || exit 71
|
|
if [ "$MODE" = networkmanager_nmcli ]; then
|
|
vc_nm_profile_resolve || exit 71
|
|
vc_nm_profile_base_validate || exit 71
|
|
quarantine_memory_zone=$(vc_nm_profile_field connection.zone) || exit 71
|
|
case "$quarantine_memory_zone" in ''|--) : ;; *) exit 71 ;; esac
|
|
else
|
|
[ -f "$IF_FRAGMENT" ] && [ ! -L "$IF_FRAGMENT" ] &&
|
|
[ "$($STAT -c '%u:%a' "$IF_FRAGMENT" 2>/dev/null)" = 0:600 ] || exit 71
|
|
awk -v owner="# {DATAPLANE_OWNER_MARKER} vpc={plan.vpc.vpc_id} revision=${{quarantine_revision#*|}} digest=${{quarantine_digest#*|}}" \
|
|
-v bridge="$BRIDGE" '
|
|
NR==1 && $0==owner {{next}}
|
|
NR==2 && $0=="iface " bridge " inet manual" {{next}}
|
|
NR==3 && $0==" bridge_ports none" {{next}}
|
|
NR==4 && $0==" bridge-stp off" {{next}}
|
|
{{bad=1}}
|
|
END {{exit (NR==4 && !bad) ? 0 : 1}}
|
|
' "$IF_FRAGMENT" || exit 71
|
|
fi
|
|
if "$IP" link show dev "$BRIDGE" >/dev/null 2>&1; then
|
|
"$IP" -d link show dev "$BRIDGE" 2>/dev/null |
|
|
grep -Eq '(^|[[:space:]])bridge[[:space:]]+(forward_delay|stp_state|hello_time|ageing_time)([[:space:]]|$)' || exit 71
|
|
REMOVE_BRIDGE_PORTS_BEFORE=$(vc_remove_bridge_port_names) || exit 71
|
|
REMOVE_BRIDGE_TOPOLOGY_CAPTURED=yes
|
|
if [ "$MODE" = networkmanager_nmcli ]; then
|
|
vc_nm_device_quarantine || exit 72
|
|
else
|
|
"$IP" link set dev "$BRIDGE" down >/dev/null || exit 72
|
|
vc_nm_link_expect_down || exit 72
|
|
fi
|
|
vc_remove_bridge_topology_matches || exit 72
|
|
fi
|
|
fi
|
|
vc_firewall_mode_ready || exit 71
|
|
[ -f /etc/virt-cloud/wg-fabric0.owner ] && grep -Fqx 'virt-cloud-wg-fabric-v1' /etc/virt-cloud/wg-fabric0.owner || exit 71
|
|
"$WG" show "$WG_IFACE" >/dev/null 2>&1 || exit 71
|
|
BRIDGE_PRESENT=no
|
|
if "$IP" link show dev "$BRIDGE" >/dev/null 2>&1; then BRIDGE_PRESENT=yes; fi
|
|
PHASE=preparing; RESUME=no
|
|
if [ -e "$TXN" ]; then
|
|
vc_vpc_native_transaction_owned || exit 79
|
|
vc_vpc_transaction_record_owned "$TXN/inflight" '{inflight_marker}' || exit 79
|
|
vc_vpc_transaction_file_secure "$TXN/state" 64 || exit 79
|
|
PHASE=$(cat "$TXN/state" 2>/dev/null) || exit 79
|
|
case "$PHASE" in
|
|
preparing|routes_removed|manager_removed|bridge_removed|nft_removed|files_removed|wg_merged|committed) : ;;
|
|
*) exit 79 ;;
|
|
esac
|
|
RESUME=yes
|
|
for snapshot in "$TXN/state.before" "$TXN/routes.before" "$TXN/rules.before" "$TXN/proxy.before"; do
|
|
[ -e "$snapshot" ] || continue
|
|
[ -f "$snapshot" ] && [ ! -L "$snapshot" ] || exit 79
|
|
[ "$($STAT -c '%u:%a' "$snapshot" 2>/dev/null)" = 0:600 ] || exit 79
|
|
done
|
|
fi
|
|
set_phase() {{
|
|
phase_tmp="$TXN/.state.$$"
|
|
printf '%s\\n' "$1" > "$phase_tmp"
|
|
chmod 600 "$phase_tmp"
|
|
mv -f "$phase_tmp" "$TXN/state"
|
|
PHASE="$1"
|
|
}}
|
|
vc_vpc_route_table_read() {{
|
|
vc_vpc_route_wanted_table=$1
|
|
vc_vpc_route_wanted_destination=${{2-}}
|
|
case "$vc_vpc_route_wanted_table" in ''|*[!0-9]*) return 1 ;; esac
|
|
vc_vpc_route_inventory=$("$IP" -o -4 -N route show table all 2>/dev/null) || return 1
|
|
printf '%s\\n' "$vc_vpc_route_inventory" | awk \
|
|
-v wanted_table="$vc_vpc_route_wanted_table" \
|
|
-v wanted_destination="$vc_vpc_route_wanted_destination" '
|
|
NF == 0 {{next}}
|
|
{{
|
|
table_count=0
|
|
table_value="254"
|
|
row_bad=0
|
|
for (i=1; i<=NF; i++) {{
|
|
if ($i == "table") {{
|
|
table_count++
|
|
if (i == NF || $(i+1) !~ /^[0-9]+$/) row_bad=1
|
|
else table_value=$(i+1)
|
|
}}
|
|
}}
|
|
if (table_count > 1 || row_bad) bad=1
|
|
route_destination=$1
|
|
if ($1 ~ /^[0-9]+$/ && NF > 1) route_destination=$2
|
|
wanted_host=wanted_destination
|
|
if (length(wanted_host) > 3 &&
|
|
substr(wanted_host, length(wanted_host) - 2) == "/32")
|
|
wanted_host=substr(wanted_host, 1, length(wanted_host) - 3)
|
|
if (!row_bad && table_count <= 1 && table_value == wanted_table &&
|
|
(wanted_destination == "" || route_destination == wanted_destination ||
|
|
route_destination == wanted_host)) print
|
|
}}
|
|
END {{exit bad ? 1 : 0}}
|
|
'
|
|
}}
|
|
verify_routes_removed() {{
|
|
verify_routes=$(vc_vpc_route_table_read "$ROUTE_TABLE") || return 1
|
|
[ -z "$verify_routes" ] || return 1
|
|
vc_ip_rule_scope_check {_q(plan.vpc.cidr)} "$ROUTE_TABLE" "{r.rule_priority}" "$WG_IFACE" empty 1 1 || return 1
|
|
if $IP link show dev "$BRIDGE" >/dev/null 2>&1; then
|
|
verify_proxy=$($IP neigh show proxy dev "$BRIDGE" 2>/dev/null | awk '$1 ~ /^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$/') || return 1
|
|
[ -z "$verify_proxy" ] || return 1
|
|
fi
|
|
return 0
|
|
}}
|
|
vc_vpc_artifact_identity_check() {{
|
|
identity=$1
|
|
set -- $identity
|
|
[ "$#" -eq 3 ] || return 1
|
|
[ "$1" = "vpc={plan.vpc.vpc_id}" ] || return 1
|
|
artifact_revision=${{2#revision=}}
|
|
[ "$2" = "revision=$artifact_revision" ] || return 1
|
|
case "$artifact_revision" in ''|*[!0-9]*) return 1;; esac
|
|
[ "$artifact_revision" -ge 1 ] || return 1
|
|
artifact_digest=${{3#digest=}}
|
|
[ "$3" = "digest=$artifact_digest" ] || return 1
|
|
printf '%s\n' "$artifact_digest" | grep -Eq '^[0-9a-f]{{64}}$' || return 1
|
|
[ "$artifact_revision" -le {plan.desired_revision} ] || return 1
|
|
if [ "$artifact_revision" = {plan.desired_revision} ] &&
|
|
[ "$artifact_digest" != {_q(plan.desired_digest)} ]; then
|
|
return 1
|
|
fi
|
|
return 0
|
|
}}
|
|
vc_remove_capture_nft_precommit() {{
|
|
vc_remove_precommit_prefix=$1
|
|
for vc_remove_precommit_family in inet bridge arp; do
|
|
vc_nft_table_snapshot "$vc_remove_precommit_family" "$TABLE" \
|
|
"$vc_remove_precommit_prefix.$vc_remove_precommit_family" || return 1
|
|
[ -f "$vc_remove_precommit_prefix.$vc_remove_precommit_family" ] &&
|
|
[ ! -L "$vc_remove_precommit_prefix.$vc_remove_precommit_family" ] &&
|
|
[ ! -e "$vc_remove_precommit_prefix.$vc_remove_precommit_family.absent" ] ||
|
|
return 1
|
|
done
|
|
}}
|
|
vc_remove_nft_precommit_matches() {{
|
|
vc_remove_precommit_prefix=$1
|
|
for vc_remove_precommit_family in inet bridge arp; do
|
|
vc_nft_table_matches_snapshot "$vc_remove_precommit_family" "$TABLE" \
|
|
"$vc_remove_precommit_prefix.$vc_remove_precommit_family" || return 1
|
|
done
|
|
}}
|
|
vc_remove_nft_tables_absent() {{
|
|
for vc_remove_absent_family in inet bridge arp; do
|
|
vc_nft_table_absent "$vc_remove_absent_family" "$TABLE" \
|
|
"$TXN/nft.remove.after.$vc_remove_absent_family.inventory" || return 1
|
|
rm -f "$TXN/nft.remove.after.$vc_remove_absent_family.inventory"
|
|
done
|
|
}}
|
|
case "$PHASE" in
|
|
preparing|routes_removed|manager_removed|bridge_removed|nft_removed) PRE_FILES=yes ;;
|
|
files_removed|wg_merged|committed) PRE_FILES=no ;;
|
|
*) exit 79 ;;
|
|
esac
|
|
if [ "$PRE_FILES" = yes ]; then
|
|
if [ -f "$OWNER_FILE" ]; then
|
|
[ ! -L "$OWNER_FILE" ] || exit 71
|
|
[ "$($STAT -c '%u:%a' "$OWNER_FILE" 2>/dev/null)" = 0:600 ] || exit 71
|
|
grep -Fqx 'virt-cloud-vpc-dataplane-v1' "$OWNER_FILE" || exit 71
|
|
elif [ -e "$STATE_FILE" ] || [ "$BRIDGE_PRESENT" = yes ] || [ -e "$NFT_FILE" ] ||
|
|
[ -e "$SYSCTL_FILE" ] || [ -e "$PROXY_FILE" ] || [ -e "$IF_FRAGMENT" ] ||
|
|
[ -e "$NFT_SNAPSHOT_PREFIX.inet.live" ] ||
|
|
[ -e "$NFT_SNAPSHOT_PREFIX.bridge.live" ] ||
|
|
[ -e "$NFT_SNAPSHOT_PREFIX.arp.live" ]; then
|
|
# A deterministic bridge/file name is not ownership. Refuse to remove
|
|
# anything when the panel's owner marker disappeared or was replaced.
|
|
exit 71
|
|
fi
|
|
if [ -f "$STATE_FILE" ]; then
|
|
[ ! -L "$STATE_FILE" ] || exit 71
|
|
[ "$($STAT -c '%u:%a' "$STATE_FILE" 2>/dev/null)" = 0:600 ] || exit 71
|
|
grep -Fqx 'owner|{DATAPLANE_OWNER_MARKER}' "$STATE_FILE" || exit 71
|
|
grep -Fqx 'vpc|{plan.vpc.vpc_id}' "$STATE_FILE" || exit 71
|
|
grep -Fqx 'cidr|{plan.vpc.cidr}' "$STATE_FILE" || exit 71
|
|
grep -Fqx 'fabric|{plan.vpc.fabric_id}' "$STATE_FILE" || exit 71
|
|
grep -Fqx 'host|{plan.host_id}' "$STATE_FILE" || exit 71
|
|
grep -Fqx 'bridge|{r.bridge}' "$STATE_FILE" || exit 71
|
|
grep -Fqx 'table|{r.route_table}' "$STATE_FILE" || exit 71
|
|
state_firewall_record=$(awk -F'|' '$1=="firewall_mode" {{count++; value=$2}} END {{print (count + 0) "|" value}}' "$STATE_FILE")
|
|
case "$state_firewall_record" in '0|'|'1|native_nftables') : ;; *) exit 71 ;; esac
|
|
state_revision=$(awk -F'|' '$1=="revision" {{print $2; exit}}' "$STATE_FILE")
|
|
state_digest=$(awk -F'|' '$1=="digest" {{print $2; exit}}' "$STATE_FILE")
|
|
case "$state_revision" in ''|*[!0-9]*) exit 79;; esac
|
|
printf '%s\n' "$state_digest" | grep -Eq '^[0-9a-f]{{64}}$' || exit 79
|
|
[ "$state_revision" -le {plan.desired_revision} ] || exit 79
|
|
if [ "$state_revision" = {plan.desired_revision} ] && [ "$state_digest" != { _q(plan.desired_digest) } ]; then exit 79; fi
|
|
for snapshot_family in inet bridge arp; do
|
|
[ ! -e "$NFT_SNAPSHOT_PREFIX.$snapshot_family.live" ] &&
|
|
[ ! -L "$NFT_SNAPSHOT_PREFIX.$snapshot_family.live" ] || exit 71
|
|
done
|
|
elif [ -f "$OWNER_FILE" ] && [ "$BRIDGE_PRESENT" = yes ]; then
|
|
# A live panel bridge without its state cannot be reconciled safely.
|
|
exit 71
|
|
else
|
|
# Without the state file there is no safe owner proof for deterministic
|
|
# policy routes/rules. Refuse to claim a clean removal while any of those
|
|
# resources remain; an operator can restore the state and retry.
|
|
orphan_routes=$(vc_vpc_route_table_read "$ROUTE_TABLE") || exit 72
|
|
[ -z "$orphan_routes" ] || exit 71
|
|
vc_ip_rule_scope_check {_q(plan.vpc.cidr)} "$ROUTE_TABLE" "{r.rule_priority}" "$WG_IFACE" empty 71 72 || exit $?
|
|
fi
|
|
for artifact in "$NFT_FILE" "$SYSCTL_FILE" "$PROXY_FILE" "$IF_FRAGMENT"; do
|
|
[ -e "$artifact" ] || continue
|
|
[ -f "$artifact" ] && [ ! -L "$artifact" ] || exit 71
|
|
done
|
|
if [ -f "$NFT_FILE" ]; then
|
|
[ "$($STAT -c '%u:%a' "$NFT_FILE" 2>/dev/null)" = 0:600 ] || exit 71
|
|
grep -Fqx '# virt-cloud-vpc-dataplane-v1' "$NFT_FILE" || exit 71
|
|
nft_identity=$(sed -n '3{{s/^# //;p;}}' "$NFT_FILE")
|
|
vc_vpc_artifact_identity_check "$nft_identity" || exit 71
|
|
fi
|
|
if [ -f "$SYSCTL_FILE" ]; then
|
|
[ -f "$STATE_FILE" ] || exit 71
|
|
vc_vpc_sysctl_artifact_owned "$SYSCTL_FILE" {_q(plan.vpc.vpc_id)} \
|
|
"$state_revision" "$state_digest" "$BRIDGE" || exit 71
|
|
fi
|
|
if [ -f "$PROXY_FILE" ]; then
|
|
[ "$($STAT -c '%u:%a' "$PROXY_FILE" 2>/dev/null)" = 0:600 ] || exit 71
|
|
sed -n '1p' "$PROXY_FILE" | grep -Fqx '# virt-cloud-vpc-proxy-v1' || exit 71
|
|
awk 'NR > 1 && $0 !~ /^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$/ {{ exit 1 }}' "$PROXY_FILE" || exit 71
|
|
fi
|
|
if [ -f "$IF_FRAGMENT" ]; then
|
|
[ "$($STAT -c '%u:%a' "$IF_FRAGMENT" 2>/dev/null)" = 0:600 ] || exit 71
|
|
sed -n '1p' "$IF_FRAGMENT" | grep -Fqx '{ifup_marker}' || exit 71
|
|
if_identity=$(sed -n '1{{s/^# {DATAPLANE_OWNER_MARKER} //;p;}}' "$IF_FRAGMENT")
|
|
vc_vpc_artifact_identity_check "$if_identity" || exit 71
|
|
grep -Eq '^[[:space:]]*(auto|allow-[^[:space:]]+)[[:space:]]+{r.bridge}([[:space:]]|$)' "$IF_FRAGMENT" && exit 71 || true
|
|
grep -Fqx 'iface {r.bridge} inet manual' "$IF_FRAGMENT" || exit 71
|
|
if grep -Eq '^[[:space:]]*bridge_ports([[:space:]]|$)' "$IF_FRAGMENT"; then
|
|
grep -Eq '^[[:space:]]*bridge_ports[[:space:]]+none[[:space:]]*$' "$IF_FRAGMENT" || exit 71
|
|
fi
|
|
grep -Eq '^[[:space:]]*(address|gateway|up|down|pre-up|post-up|pre-down|post-down)([[:space:]]|$)' "$IF_FRAGMENT" && exit 71 || true
|
|
fi
|
|
else
|
|
for artifact in "$NFT_FILE" "$SYSCTL_FILE" "$PROXY_FILE" "$IF_FRAGMENT" "$OWNER_FILE" "$STATE_FILE"; do
|
|
[ ! -e "$artifact" ] && [ ! -L "$artifact" ] || exit 79
|
|
done
|
|
for snapshot_family in inet bridge arp; do
|
|
[ ! -e "$NFT_SNAPSHOT_PREFIX.$snapshot_family.live" ] &&
|
|
[ ! -L "$NFT_SNAPSHOT_PREFIX.$snapshot_family.live" ] || exit 79
|
|
done
|
|
fi
|
|
if [ "$MODE" = networkmanager_nmcli ]; then
|
|
if [ "$PHASE" = preparing ] || [ "$PHASE" = routes_removed ]; then
|
|
if vc_nm_profile_resolve; then
|
|
vc_nm_profile_base_validate || exit 71
|
|
vc_nm_profile_empty connection.zone || exit 71
|
|
else
|
|
nm_resolve_status=$?
|
|
[ "$nm_resolve_status" = 2 ] || exit 71
|
|
if [ "$BRIDGE_PRESENT" = yes ]; then
|
|
# An unmanaged live bridge is foreign/ambiguous; do not delete it.
|
|
exit 71
|
|
fi
|
|
fi
|
|
else
|
|
if "$NMCLI" connection show "$BRIDGE" >/dev/null 2>&1; then exit 79; fi
|
|
fi
|
|
else
|
|
if [ "$PHASE" = preparing ] || [ "$PHASE" = routes_removed ]; then
|
|
if [ "$BRIDGE_PRESENT" = yes ] || [ -e "$IF_FRAGMENT" ]; then
|
|
[ -f "$IF_FRAGMENT" ] || exit 71
|
|
[ ! -L /etc/network ] && [ ! -L /etc/network/interfaces ] && [ ! -L /etc/network/interfaces.d ] || exit 71
|
|
if [ -f /etc/network/interfaces ] &&
|
|
grep -Eq '^[[:space:]]*(auto|allow-[^[:space:]]+|iface)[[:space:]]+'"$BRIDGE"'([[:space:]]|$)' /etc/network/interfaces; then
|
|
exit 71
|
|
fi
|
|
"$IFQUERY" "$BRIDGE" >/dev/null 2>&1 || exit 71
|
|
fi
|
|
fi
|
|
fi
|
|
if [ "$PHASE" = preparing ] || [ "$PHASE" = routes_removed ] || [ "$PHASE" = manager_removed ]; then
|
|
if "$IP" link show dev "$BRIDGE" >/dev/null 2>&1; then
|
|
[ -f "$OWNER_FILE" ] || exit 71
|
|
bridge_info=$("$IP" -d link show dev "$BRIDGE" 2>/dev/null) || exit 71
|
|
printf '%s\\n' "$bridge_info" | grep -Eq '(^|[[:space:]])bridge[[:space:]]+(forward_delay|stp_state|hello_time|ageing_time)([[:space:]]|$)' || exit 71
|
|
bridge_ports=$("$BRIDGE_CMD" link show master "$BRIDGE" 2>/dev/null) || exit 71
|
|
[ -z "$bridge_ports" ] || exit 79
|
|
bridge_addrs=$("$IP" -o addr show dev "$BRIDGE" 2>/dev/null) || exit 71
|
|
[ -z "$bridge_addrs" ] || exit 79
|
|
fi
|
|
else
|
|
if "$IP" link show dev "$BRIDGE" >/dev/null 2>&1; then exit 79; fi
|
|
fi
|
|
NFT_MAIN_COUNT=0
|
|
for nft_candidate in /etc/nftables.conf /etc/sysconfig/nftables.conf; do
|
|
if [ -f "$nft_candidate" ] && [ ! -L "$nft_candidate" ] && [ "$($STAT -c '%u' "$nft_candidate" 2>/dev/null)" = 0 ]; then
|
|
case "$($STAT -c '%a' "$nft_candidate" 2>/dev/null)" in
|
|
[0-7][0145][0145]) : ;;
|
|
*) continue ;;
|
|
esac
|
|
NFT_MAIN="$nft_candidate"; NFT_MAIN_COUNT=$((NFT_MAIN_COUNT + 1))
|
|
fi
|
|
done
|
|
[ "$NFT_MAIN_COUNT" -eq 1 ] || exit 71
|
|
[ ! -e "$LEGACY_MANIFEST" ] && [ ! -L "$LEGACY_MANIFEST" ] || exit 71
|
|
if [ "$PHASE" = preparing ] || [ "$PHASE" = routes_removed ] || [ "$PHASE" = manager_removed ] || [ "$PHASE" = bridge_removed ]; then
|
|
if [ -f "$NFT_FILE" ]; then
|
|
nft_tables=$(awk '$1=="table" {{print $2 "|" $3}}' "$NFT_FILE")
|
|
nft_table_count=0
|
|
while IFS='|' read -r family name; do
|
|
[ -n "$family" ] || continue
|
|
case "$family:$name" in inet:*|bridge:*|arp:*) : ;; *) exit 71 ;; esac
|
|
[ "$name" = "$TABLE" ] || exit 71
|
|
table_dump=$("$NFT" list table "$family" "$name" 2>/dev/null) || exit 71
|
|
printf '%s\\n' "$table_dump" | grep -Fq 'virt-cloud-vpc-dataplane-v1' || exit 71
|
|
nft_table_count=$((nft_table_count + 1))
|
|
done <<EOF
|
|
$nft_tables
|
|
EOF
|
|
[ "$nft_table_count" -gt 0 ] || exit 71
|
|
fi
|
|
else
|
|
if [ -f "$NFT_FILE" ]; then
|
|
nft_tables=$(awk '$1=="table" {{print $2 "|" $3}}' "$NFT_FILE")
|
|
while IFS='|' read -r family name; do
|
|
[ -n "$family" ] || continue
|
|
case "$family:$name" in inet:*|bridge:*|arp:*) : ;; *) exit 79 ;; esac
|
|
[ "$name" = "$TABLE" ] || exit 79
|
|
if "$NFT" list table "$family" "$name" >/dev/null 2>&1; then exit 79; fi
|
|
done <<EOF
|
|
$nft_tables
|
|
EOF
|
|
fi
|
|
grep -Fqx "$NFT_INCLUDE" "$NFT_MAIN" && exit 79 || true
|
|
fi
|
|
# A partially completed removal is deliberately fail-closed. The journal is
|
|
# retained for an operator/reconciler to inspect; rerunning it must never repeat
|
|
# route, bridge, nft, or WireGuard deletions against an unknown intermediate
|
|
# state. A fully committed journal only needs the already-verified cleanup.
|
|
if [ "$RESUME" = yes ]; then
|
|
case "$PHASE" in
|
|
committed)
|
|
rm -rf "$TXN" || exit 79
|
|
printf '%s\\n' '__VC_VPC_REMOVED__=yes'
|
|
printf '%s\\n' '__VC_VPC_OWNER__={DATAPLANE_OWNER_MARKER}'
|
|
printf '%s\\n' '__VC_VPC_REVISION__={plan.desired_revision}'
|
|
printf '%s\\n' '__VC_VPC_DIGEST__={plan.desired_digest}'
|
|
printf '%s\\n' '__VC_VPC_ROLLBACK__=none'
|
|
exit 0
|
|
;;
|
|
files_removed|wg_merged)
|
|
# All destructive host resources are already absent. Re-running only
|
|
# the Fabric state load and peer merge below is idempotent and closes
|
|
# the crash window between file removal and AllowedIPs convergence.
|
|
:
|
|
;;
|
|
*) exit 79 ;;
|
|
esac
|
|
fi
|
|
if [ "$RESUME" = no ]; then
|
|
mkdir -m 700 "$TXN"
|
|
vc_vpc_write_native_transaction_mode || exit 79
|
|
printf '%s\\n' '{inflight_marker}' > "$TXN/inflight"; chmod 600 "$TXN/inflight"
|
|
set_phase preparing
|
|
if [ -f "$STATE_FILE" ]; then cp -p "$STATE_FILE" "$TXN/state.before"; fi
|
|
: > "$TXN/routes.before"; : > "$TXN/rules.before"; : > "$TXN/proxy.before"
|
|
chmod 600 "$TXN/routes.before" "$TXN/rules.before" "$TXN/proxy.before"
|
|
else
|
|
# A resumed transaction must use the original snapshots. Creating an empty
|
|
# snapshot after a partial failure would turn an unknown host state into an
|
|
# apparently clean state and could make the next destructive step unsafe.
|
|
for snapshot in "$TXN/routes.before" "$TXN/rules.before" "$TXN/proxy.before"; do
|
|
[ -f "$snapshot" ] && [ ! -L "$snapshot" ] || exit 79
|
|
[ "$($STAT -c '%u:%a' "$snapshot" 2>/dev/null)" = 0:600 ] || exit 79
|
|
done
|
|
fi
|
|
if [ -f "$TXN/state.before" ]; then
|
|
old_priority=$(awk -F'|' '$1=="priority" {{print $2; exit}}' "$TXN/state.before")
|
|
old_cidr=$(awk -F'|' '$1=="cidr" {{print $2; exit}}' "$TXN/state.before")
|
|
[ -n "$old_priority" ] && [ -n "$old_cidr" ] || exit 71
|
|
case "$old_priority" in ''|*[!0-9]*) exit 71;; esac
|
|
[ "$old_priority" = "{r.rule_priority}" ] || exit 71
|
|
# Validate both RPDB slots before deleting any route. A foreign or
|
|
# ambiguous priority must leave the data plane untouched.
|
|
vc_ip_rule_scope_check "$old_cidr" "$ROUTE_TABLE" "$old_priority" "$WG_IFACE" managed 71 72 || exit $?
|
|
vc_ip_rule_preflight "$old_priority" - "$old_cidr" "$ROUTE_TABLE" 71 72 "$TXN/rules.before" || exit $?
|
|
vc_ip_rule_preflight "$((old_priority + 1))" "$WG_IFACE" "$old_cidr" "$ROUTE_TABLE" 71 72 "$TXN/rules.before" || exit $?
|
|
while IFS='|' read -r kind destination device table via metric; do
|
|
[ "$kind" = route ] || continue
|
|
[ "$table" = "$ROUTE_TABLE" ] || exit 71
|
|
[ "$device" = "$BRIDGE" ] || [ "$device" = "$WG_IFACE" ] || exit 71
|
|
printf '%s\\n' "$destination" | grep -Eq '^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+/32$' || exit 71
|
|
case "$via" in
|
|
-|'') : ;;
|
|
*) printf '%s\\n' "$via" | grep -Eq '^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$' || exit 71 ;;
|
|
esac
|
|
case "$metric" in ''|*[!0-9]*) exit 71;; esac
|
|
existing=$(vc_vpc_route_table_read "$table" "$destination") || exit 72
|
|
[ -n "$existing" ] || continue
|
|
existing_count=$(printf '%s\\n' "$existing" | awk 'NF {{ count++ }} END {{ print count + 0 }}')
|
|
[ "$existing_count" = 1 ] || exit 71
|
|
actual_device=$(printf '%s\\n' "$existing" | awk '{{for (i=1;i<=NF;i++) if ($i=="dev" && i<NF) {{print $(i+1); exit}}}}')
|
|
[ "$actual_device" = "$device" ] || exit 71
|
|
printf '%s\\n' "$existing" | grep -Eq 'proto 186([[:space:]]|$)' || exit 71
|
|
old_via=$(printf '%s\\n' "$existing" | awk '{{for (i=1;i<=NF;i++) if ($i=="via" && i<NF) {{print $(i+1); exit}}}}')
|
|
old_metric=$(printf '%s\\n' "$existing" | awk '{{for (i=1;i<=NF;i++) if ($i=="metric" && i<NF) {{print $(i+1); exit}}}}')
|
|
[ -n "$old_metric" ] || old_metric=100
|
|
[ "$old_metric" = "$metric" ] || exit 71
|
|
[ "${{old_via:--}}" = "${{via:--}}" ] || exit 71
|
|
printf '%s|%s|%s|%s|%s\\n' "$destination" "$device" "$table" "$old_via" "$old_metric" >> "$TXN/routes.before"
|
|
if [ "${{old_via:--}}" != - ]; then
|
|
"$IP" route del "$destination" table "$table" via "$old_via" dev "$device" proto 186 metric "$old_metric" || exit 72
|
|
else
|
|
"$IP" route del "$destination" table "$table" dev "$device" proto 186 metric "$old_metric" || exit 72
|
|
fi
|
|
done < "$TXN/state.before"
|
|
for rule_spec in "$old_priority|-" "$((old_priority + 1))|$WG_IFACE"; do
|
|
pref=${{rule_spec%%|*}}; old_iif=${{rule_spec#*|}}
|
|
vc_ip_rule_read "$pref" "$old_iif" "$old_cidr" "$ROUTE_TABLE" 71 72 || exit $?
|
|
[ "$VC_IP_RULE_PRESENT" = yes ] || continue
|
|
if [ "$old_iif" = - ]; then
|
|
"$IP" -4 rule del pref "$pref" to "$old_cidr" lookup "$ROUTE_TABLE" || exit 72
|
|
else
|
|
"$IP" -4 rule del pref "$pref" iif "$old_iif" to "$old_cidr" lookup "$ROUTE_TABLE" || exit 72
|
|
fi
|
|
vc_ip_rule_read "$pref" "$old_iif" "$old_cidr" "$ROUTE_TABLE" 71 72 || exit $?
|
|
[ "$VC_IP_RULE_PRESENT" = no ] || exit 72
|
|
done
|
|
fi
|
|
if [ -f "$PROXY_FILE" ]; then
|
|
while IFS= read -r proxy_ip; do
|
|
[ -n "$proxy_ip" ] || continue
|
|
if "$IP" neigh show proxy dev "$BRIDGE" 2>/dev/null | grep -Fqx "$proxy_ip"; then
|
|
printf '%s\\n' "$proxy_ip" >> "$TXN/proxy.before"
|
|
"$IP" neigh del proxy "$proxy_ip" dev "$BRIDGE" || exit 72
|
|
fi
|
|
done <<EOF
|
|
$(sed '1d' "$PROXY_FILE")
|
|
EOF
|
|
fi
|
|
verify_routes_removed || exit 72
|
|
set_phase routes_removed
|
|
if [ "$MODE" = networkmanager_nmcli ]; then
|
|
if [ -n "$NMCLI" ] && "$NMCLI" connection show "$BRIDGE" >/dev/null 2>&1; then
|
|
"$NMCLI" connection delete "$BRIDGE" >/dev/null 2>&1 || exit 72
|
|
if "$NMCLI" connection show "$BRIDGE" >/dev/null 2>&1; then exit 72; fi
|
|
fi
|
|
else
|
|
if [ -f "$IF_FRAGMENT" ]; then
|
|
"$IFQUERY" "$BRIDGE" >/dev/null 2>&1 || exit 71
|
|
"$IFDOWN" --force "$BRIDGE" >/dev/null 2>&1 || exit 72
|
|
fi
|
|
fi
|
|
set_phase manager_removed
|
|
if "$IP" link show dev "$BRIDGE" >/dev/null 2>&1; then
|
|
bridge_info=$("$IP" -d link show dev "$BRIDGE" 2>/dev/null) || exit 71
|
|
printf '%s\\n' "$bridge_info" | grep -Eq '(^|[[:space:]])bridge[[:space:]]+(forward_delay|stp_state|hello_time|ageing_time)([[:space:]]|$)' || exit 71
|
|
ports=$("$BRIDGE_CMD" link show master "$BRIDGE" 2>/dev/null) || exit 71
|
|
[ -z "$ports" ] || exit 79
|
|
bridge_addrs=$("$IP" -o addr show dev "$BRIDGE" 2>/dev/null) || exit 71
|
|
[ -z "$bridge_addrs" ] || exit 79
|
|
bridge_routes=$("$IP" -o -4 route show table all dev "$BRIDGE" 2>/dev/null) || exit 71
|
|
[ -z "$bridge_routes" ] || exit 79
|
|
bridge_routes6=$("$IP" -o -6 route show table all dev "$BRIDGE" 2>/dev/null) || exit 71
|
|
[ -z "$bridge_routes6" ] || exit 79
|
|
proxy_left=$("$IP" neigh show proxy dev "$BRIDGE" 2>/dev/null | awk '$1 ~ /^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$/') || exit 71
|
|
[ -z "$proxy_left" ] || exit 79
|
|
"$IP" link del dev "$BRIDGE" type bridge || exit 72
|
|
fi
|
|
if "$IP" link show dev "$BRIDGE" >/dev/null 2>&1; then exit 72; fi
|
|
set_phase bridge_removed
|
|
if [ -f "$NFT_FILE" ]; then
|
|
nft_tables=$(awk '$1=="table" {{print $2 "|" $3}}' "$NFT_FILE")
|
|
nft_table_count=0
|
|
NFT_REMOVE_BATCH="$TXN/remove.nft"
|
|
: > "$NFT_REMOVE_BATCH"
|
|
while IFS='|' read -r family name; do
|
|
[ -n "$family" ] || continue
|
|
case "$family:$name" in inet:*|bridge:*|arp:*) : ;; *) exit 71 ;; esac
|
|
[ "$name" = "$TABLE" ] || exit 71
|
|
table_dump=$("$NFT" -s list table "$family" "$name" 2>/dev/null) || exit 71
|
|
printf '%s\\n' "$table_dump" | grep -Fq 'virt-cloud-vpc-dataplane-v1' || exit 71
|
|
printf 'delete table %s %s\\n' "$family" "$name" >> "$NFT_REMOVE_BATCH"
|
|
nft_table_count=$((nft_table_count + 1))
|
|
done <<EOF
|
|
$nft_tables
|
|
EOF
|
|
[ "$nft_table_count" = 3 ] || exit 71
|
|
vc_remove_capture_nft_precommit "$TXN/nft.remove.precommit" || exit 71
|
|
"$NFT" -c -f "$NFT_REMOVE_BATCH" >/dev/null || exit 72
|
|
# Check mode has no kernel reservation/CAS primitive. Re-read all complete
|
|
# stateless tables after dry-run and immediately before the one delete batch.
|
|
vc_remove_nft_precommit_matches "$TXN/nft.remove.precommit" || exit 71
|
|
"$NFT" -f "$NFT_REMOVE_BATCH" >/dev/null || exit 72
|
|
vc_remove_nft_tables_absent || exit 72
|
|
fi
|
|
if grep -Fqx "$NFT_INCLUDE" "$NFT_MAIN"; then
|
|
NFT_MAIN_TMP=$("$MKTEMP" "$NFT_MAIN.virt-cloud-vpc-remove.XXXXXX") || exit 72
|
|
awk -v needle="$NFT_INCLUDE" '$0 != needle' "$NFT_MAIN" > "$NFT_MAIN_TMP" || {{ rm -f "$NFT_MAIN_TMP"; exit 72; }}
|
|
chmod "$($STAT -c '%a' "$NFT_MAIN")" "$NFT_MAIN_TMP" || {{ rm -f "$NFT_MAIN_TMP"; exit 72; }}
|
|
"$NFT" -c -f "$NFT_MAIN_TMP" >/dev/null || {{ rm -f "$NFT_MAIN_TMP"; exit 72; }}
|
|
mv -f "$NFT_MAIN_TMP" "$NFT_MAIN" || exit 72
|
|
grep -Fqx "$NFT_INCLUDE" "$NFT_MAIN" && exit 72 || true
|
|
fi
|
|
set_phase nft_removed
|
|
if [ -e "$NFT_FILE" ] || [ -L "$NFT_FILE" ]; then
|
|
[ -f "$NFT_FILE" ] && [ ! -L "$NFT_FILE" ] || exit 71
|
|
grep -Fqx '# virt-cloud-vpc-dataplane-v1' "$NFT_FILE" || exit 71
|
|
nft_identity=$(sed -n '3{{s/^# //;p;}}' "$NFT_FILE")
|
|
vc_vpc_artifact_identity_check "$nft_identity" || exit 71
|
|
fi
|
|
if [ -e "$SYSCTL_FILE" ] || [ -L "$SYSCTL_FILE" ]; then
|
|
[ -f "$STATE_FILE" ] || exit 71
|
|
vc_vpc_sysctl_artifact_owned "$SYSCTL_FILE" {_q(plan.vpc.vpc_id)} \
|
|
"$state_revision" "$state_digest" "$BRIDGE" || exit 71
|
|
fi
|
|
if [ -e "$PROXY_FILE" ] || [ -L "$PROXY_FILE" ]; then
|
|
[ -f "$PROXY_FILE" ] && [ ! -L "$PROXY_FILE" ] || exit 71
|
|
sed -n '1p' "$PROXY_FILE" | grep -Fqx '# virt-cloud-vpc-proxy-v1' || exit 71
|
|
fi
|
|
if [ -e "$IF_FRAGMENT" ] || [ -L "$IF_FRAGMENT" ]; then
|
|
[ -f "$IF_FRAGMENT" ] && [ ! -L "$IF_FRAGMENT" ] || exit 71
|
|
sed -n '1p' "$IF_FRAGMENT" | grep -Fqx '{ifup_marker}' || exit 71
|
|
if_identity=$(sed -n '1{{s/^# {DATAPLANE_OWNER_MARKER} //;p;}}' "$IF_FRAGMENT")
|
|
vc_vpc_artifact_identity_check "$if_identity" || exit 71
|
|
fi
|
|
if [ -f "$STATE_FILE" ]; then
|
|
[ ! -L "$STATE_FILE" ] || exit 71
|
|
grep -Fqx 'owner|{DATAPLANE_OWNER_MARKER}' "$STATE_FILE" || exit 71
|
|
grep -Fqx 'vpc|{plan.vpc.vpc_id}' "$STATE_FILE" || exit 71
|
|
grep -Fqx 'fabric|{plan.vpc.fabric_id}' "$STATE_FILE" || exit 71
|
|
grep -Fqx 'host|{plan.host_id}' "$STATE_FILE" || exit 71
|
|
grep -Fqx 'bridge|{r.bridge}' "$STATE_FILE" || exit 71
|
|
grep -Fqx 'table|{r.route_table}' "$STATE_FILE" || exit 71
|
|
state_firewall_record=$(awk -F'|' '$1=="firewall_mode" {{count++; value=$2}} END {{print (count + 0) "|" value}}' "$STATE_FILE")
|
|
case "$state_firewall_record" in '0|'|'1|native_nftables') : ;; *) exit 71 ;; esac
|
|
state_revision=$(awk -F'|' '$1=="revision" {{print $2; exit}}' "$STATE_FILE")
|
|
state_digest=$(awk -F'|' '$1=="digest" {{print $2; exit}}' "$STATE_FILE")
|
|
case "$state_revision" in ''|*[!0-9]*) exit 79;; esac
|
|
[ "$state_revision" -le {plan.desired_revision} ] || exit 79
|
|
if [ "$state_revision" = {plan.desired_revision} ] && [ "$state_digest" != { _q(plan.desired_digest) } ]; then exit 79; fi
|
|
fi
|
|
rm -f "$NFT_FILE" "$SYSCTL_FILE" "$PROXY_FILE" "$IF_FRAGMENT" \
|
|
"$NFT_SNAPSHOT_PREFIX.inet.live" "$NFT_SNAPSHOT_PREFIX.bridge.live" \
|
|
"$NFT_SNAPSHOT_PREFIX.arp.live" "$OWNER_FILE" "$STATE_FILE"
|
|
[ ! -e "$NFT_FILE" ] && [ ! -e "$SYSCTL_FILE" ] && [ ! -e "$PROXY_FILE" ] &&
|
|
[ ! -e "$IF_FRAGMENT" ] && [ ! -e "$NFT_SNAPSHOT_PREFIX.inet.live" ] &&
|
|
[ ! -e "$NFT_SNAPSHOT_PREFIX.bridge.live" ] &&
|
|
[ ! -e "$NFT_SNAPSHOT_PREFIX.arp.live" ] &&
|
|
[ ! -e "$OWNER_FILE" ] && [ ! -e "$STATE_FILE" ] || exit 72
|
|
set_phase files_removed
|
|
{state_loader}
|
|
vpc_state_load || exit 74
|
|
vpc_state_refresh_nft || exit 74
|
|
vc_vpc_fabric_nft_live_ready || exit 74
|
|
vpc_state_nft_set_matches || exit 74
|
|
vc_nft_table_snapshot inet virt_cloud_wg_fabric \
|
|
"$TXN/fabric.nft.desired" || exit 74
|
|
[ -f "$TXN/fabric.nft.desired" ] &&
|
|
[ ! -e "$TXN/fabric.nft.desired.absent" ] || exit 74
|
|
vc_nft_table_matches_snapshot inet virt_cloud_wg_fabric \
|
|
"$TXN/fabric.nft.desired" || exit 74
|
|
for key in $("$WG" show "$WG_IFACE" peers 2>/dev/null); do
|
|
actual=$("$WG" show "$WG_IFACE" allowed-ips 2>/dev/null | awk -v wanted="$key" '$1==wanted {{print $2; exit}}' || true)
|
|
overlay=$(printf '%s\\n' "$actual" | tr ',' '\\n' | awk '/^100\\.64\\.[0-9]+\\.[0-9]+\\/32$/ {{print; count++}} END {{exit count == 1 ? 0 : 1}}') || exit 74
|
|
extra=$(vpc_state_allowed_for_peer "$key") || exit 74
|
|
merged="$overlay"; [ -z "$extra" ] || merged="$merged,$extra"
|
|
"$WG" set "$WG_IFACE" peer "$key" allowed-ips "$merged" || exit 74
|
|
done
|
|
vc_vpc_fabric_nft_live_ready || exit 74
|
|
vpc_state_nft_set_matches || exit 74
|
|
vc_nft_table_matches_snapshot inet virt_cloud_wg_fabric \
|
|
"$TXN/fabric.nft.desired" || exit 74
|
|
set_phase wg_merged
|
|
verify_routes_removed || exit 79
|
|
for family in inet bridge arp; do
|
|
"$NFT" list table "$family" "$TABLE" >/dev/null 2>&1 && exit 79 || true
|
|
done
|
|
grep -Fqx "$NFT_INCLUDE" "$NFT_MAIN" && exit 79 || true
|
|
if [ "$MODE" = networkmanager_nmcli ] && [ -n "$NMCLI" ]; then
|
|
"$NMCLI" connection show "$BRIDGE" >/dev/null 2>&1 && exit 79 || true
|
|
fi
|
|
[ ! -e "$STATE_FILE" ] && [ ! -e "$OWNER_FILE" ] &&
|
|
[ ! -e "$NFT_SNAPSHOT_PREFIX.inet.live" ] &&
|
|
[ ! -e "$NFT_SNAPSHOT_PREFIX.bridge.live" ] &&
|
|
[ ! -e "$NFT_SNAPSHOT_PREFIX.arp.live" ] || exit 79
|
|
if "$IP" link show dev "$BRIDGE" >/dev/null 2>&1; then exit 79; fi
|
|
# Keep the final durable commit behind one last full semantic/digest/tenant
|
|
# readback and the post-refresh stateless snapshot. The shared Fabric flock
|
|
# excludes every panel writer throughout this transaction.
|
|
vc_vpc_fabric_nft_live_ready || exit 74
|
|
vpc_state_nft_set_matches || exit 74
|
|
vc_nft_table_matches_snapshot inet virt_cloud_wg_fabric \
|
|
"$TXN/fabric.nft.desired" || exit 74
|
|
printf '%s\\n' '__VC_VPC_REMOVED__=yes'
|
|
printf '%s\\n' '__VC_VPC_OWNER__={DATAPLANE_OWNER_MARKER}'
|
|
printf '%s\\n' '__VC_VPC_REVISION__={plan.desired_revision}'
|
|
printf '%s\\n' '__VC_VPC_DIGEST__={plan.desired_digest}'
|
|
set_phase committed
|
|
rm -rf "$TXN"
|
|
printf '%s\\n' '__VC_VPC_ROLLBACK__=none'
|
|
exit 0
|
|
'''
|
|
if len(script) > MAX_OUTPUT:
|
|
raise VpcDataplaneError("VPC 销毁脚本过大", 413, "vpc_script_too_large")
|
|
return script
|
|
|
|
|
|
# Real host calls use the strict, manager-aware builder. The old positional
|
|
# function remains above for compatibility with callers that only render a
|
|
# fixture.
|
|
render_apply_script = _render_apply_script_v2
|
|
|
|
|
|
def render_observe_script(
|
|
plan: VpcDataplanePlan,
|
|
*,
|
|
firewall_mode: str = FIREWALL_MODE_NATIVE,
|
|
) -> str:
|
|
firewall_mode = _firewall_mode(firewall_mode)
|
|
from .fabric import (
|
|
_FABRIC_NFT_VALIDATION_SH,
|
|
_FIREWALL_MODE_GUARD_SH,
|
|
_NMCLI_FILENAME_RESOLVER_SH,
|
|
_VPC_STATE_LOADER,
|
|
)
|
|
routes = _routes(plan)
|
|
r = plan.resources
|
|
proxy_file = f"/etc/virt-cloud/vpc-dataplane/{r.token}.proxy"
|
|
nft_file = f"/etc/nftables.d/virt-cloud-vpc-{r.token}.nft"
|
|
nft_include = f'include "{nft_file}"'
|
|
sysctl_file = f"/etc/sysctl.d/99-virt-cloud-vpc-{r.token}.conf"
|
|
ifup_fragment = f"/etc/network/interfaces.d/virt-cloud-vpc-{r.token}"
|
|
nm_profile_marker = DATAPLANE_OWNER_MARKER + " vpc=" + plan.vpc.vpc_id
|
|
route_lines = "\n".join(
|
|
f"{x['destination_cidr']}|{x['device']}|{x.get('via') or '-'}|{x.get('metric', 100)}"
|
|
for x in routes)
|
|
peer_lines = "\n".join(
|
|
f"{p.host_id}|{p.public_key}|{_allowed(plan, p)}"
|
|
for p in plan.peers)
|
|
fabric_peer_key_words = " ".join(sorted(plan.fabric_peer_keys or ()))
|
|
fabric_peer_key_arg = _q(fabric_peer_key_words)
|
|
fabric_peer_keys_provided = "yes" if plan.fabric_peer_keys is not None else "no"
|
|
route_marker = "__VC_VPC_OBS_ROUTES_" + hashlib.sha256(
|
|
route_lines.encode()).hexdigest()[:12]
|
|
peer_marker = "__VC_VPC_OBS_PEERS_" + hashlib.sha256(
|
|
peer_lines.encode()).hexdigest()[:12]
|
|
state_loader = _VPC_STATE_LOADER.replace(
|
|
"__VC_FABRIC_HOST_ID__", plan.host_id).replace(
|
|
"__VC_FABRIC_ID__", plan.vpc.fabric_id).replace(
|
|
"__VC_FABRIC_NETWORK__", "100.64.0.0/10").replace(
|
|
"__VC_VPC_STATE_OBSERVE__", "yes")
|
|
nft_marker = _nft_marker(plan)
|
|
# These comments are emitted by render_vpc_nft_rules. Requiring the full
|
|
# owned shape prevents a stale or foreign table being reported as active
|
|
# merely because the deterministic table name exists.
|
|
nft_suffixes = (
|
|
*_NFT_INET_REQUIRED_SUFFIXES,
|
|
*(f"acl:{acl.acl_id}" for acl in plan.vpc.acls),
|
|
)
|
|
nft_checks = "\n".join(
|
|
f''' printf '%s\\n' "$nft_text" | grep -Fq "comment \\\"$NFT_MARKER:{suffix}\\\"" || nft_verified=no'''
|
|
for suffix in nft_suffixes)
|
|
bridge_suffixes = (
|
|
*_NFT_BRIDGE_REQUIRED_SUFFIXES,
|
|
*(f"acl:{acl.acl_id}" for acl in plan.vpc.acls),
|
|
)
|
|
bridge_checks = "\n".join(
|
|
f''' printf '%s\\n' "$bridge_text" | grep -Fq "comment \\\"$NFT_MARKER:{suffix}\\\"" || bridge_verified=no'''
|
|
for suffix in bridge_suffixes)
|
|
if plan.local_members:
|
|
bridge_observe = f'''bridge_text=$($NFT list table bridge "$TABLE" 2>/dev/null || true)
|
|
bridge_verified=yes
|
|
printf '%s\\n' "$bridge_text" | grep -Fq "table bridge $TABLE" || bridge_verified=no
|
|
printf '%s\\n' "$bridge_text" | grep -Fq "chain forward" || bridge_verified=no
|
|
printf '%s\\n' "$bridge_text" | grep -Fq "hook forward priority -190" || bridge_verified=no
|
|
printf '%s\\n' "$bridge_text" | grep -Fq "policy accept" || bridge_verified=no
|
|
printf '%s\\n' "$bridge_text" | grep -Fq "set local_member_ips" || bridge_verified=no
|
|
printf '%s\\n' "$bridge_text" | grep -Fq "set local_dest_ips" || bridge_verified=no
|
|
{bridge_checks}
|
|
if [ "$bridge_verified" = yes ]; then printf '%s\\n' '__VC_VPC_BRIDGE_NFT__=yes'; else printf '%s\\n' '__VC_VPC_BRIDGE_NFT__=no'; fi'''
|
|
else:
|
|
# An empty VPC must not silently retain a stale bridge-family table
|
|
# from a previous revision or host membership.
|
|
bridge_observe = '''
|
|
if $NFT list table bridge "$TABLE" >/dev/null 2>&1; then
|
|
printf '%s\\n' '__VC_VPC_BRIDGE_NFT__=no'
|
|
else
|
|
printf '%s\\n' '__VC_VPC_BRIDGE_NFT__=none'
|
|
fi'''.strip()
|
|
if plan.local_members:
|
|
arp_observe = f'''arp_text=$($NFT list table arp "$TABLE" 2>/dev/null || true)
|
|
arp_verified=yes
|
|
printf '%s\\n' "$arp_text" | grep -Fq "table arp $TABLE" || arp_verified=no
|
|
printf '%s\\n' "$arp_text" | grep -Fq "set proxy_ips" || arp_verified=no
|
|
printf '%s\\n' "$arp_text" | grep -Fq "chain output" || arp_verified=no
|
|
printf '%s\\n' "$arp_text" | grep -Fq "hook output priority -190" || arp_verified=no
|
|
printf '%s\\n' "$arp_text" | grep -Fq "policy accept" || arp_verified=no
|
|
printf '%s\\n' "$arp_text" | grep -Fq "comment \\\"$NFT_MARKER:arp-proxy-output\\\"" || arp_verified=no
|
|
if [ "$arp_verified" = yes ]; then printf '%s\\n' '__VC_VPC_ARP_NFT__=yes'; else printf '%s\\n' '__VC_VPC_ARP_NFT__=no'; fi'''
|
|
else:
|
|
arp_observe = '''
|
|
if $NFT list table arp "$TABLE" >/dev/null 2>&1; then
|
|
printf '%s\\n' '__VC_VPC_ARP_NFT__=no'
|
|
else
|
|
printf '%s\\n' '__VC_VPC_ARP_NFT__=none'
|
|
fi'''.strip()
|
|
route_extra_awk = "\n".join(
|
|
f' $0 == "{x["destination_cidr"]}|{x["device"]}|{x.get("via") or "-"}|{x.get("metric", 100)}|186" {{ next }}'
|
|
for x in routes)
|
|
route_checks = "\n".join(
|
|
f'''state=missing
|
|
if printf '%s\\n' "$route_records" | awk -F'|' -v wanted={_q(x['destination_cidr'])} -v expected_device={_q(x['device'])} -v expected_via={_q(x.get('via') or '-')} -v expected_metric={_q(str(x.get('metric', 100)))} '$1 == wanted && $2 == expected_device && $3 == expected_via && $4 == expected_metric && $5 == "186" {{found=1}} END {{exit found ? 0 : 1}}'; then
|
|
state=owned
|
|
fi
|
|
printf '%s\\n' '__VC_VPC_ROUTE__={x['destination_cidr']}|{x['device']}|'"$ROUTE_TABLE"'|{x.get('via') or '-'}|{x.get('metric', 100)}|'"$state"
|
|
'''.rstrip()
|
|
for x in routes)
|
|
if not route_checks:
|
|
route_checks = ": # no expected VPC routes"
|
|
rule_specs = (
|
|
(str(r.rule_priority), "-", plan.vpc.cidr, str(r.route_table)),
|
|
(str(r.rule_priority + 1), FABRIC_INTERFACE, plan.vpc.cidr,
|
|
str(r.route_table)),
|
|
)
|
|
rule_checks = "\n".join(
|
|
f'''rule_state=missing
|
|
if vc_ip_rule_read {_q(pref)} {_q(iif)} {_q(destination)} {_q(table)} 1 1 &&
|
|
[ "$VC_IP_RULE_PRESENT" = yes ]; then
|
|
rule_state=owned
|
|
fi
|
|
printf '%s\\n' '__VC_VPC_RULE__={pref}|{iif}|{destination}|{table}|'"$rule_state"
|
|
'''.rstrip()
|
|
for pref, iif, destination, table in rule_specs)
|
|
return f'''#!/bin/sh
|
|
set -eu
|
|
PATH=/usr/sbin:/usr/bin:/sbin:/bin
|
|
IP=/usr/sbin/ip; [ -x "$IP" ] || IP=/usr/bin/ip
|
|
{IP_RULE_CONVERGENCE_SH}
|
|
WG=/usr/bin/wg; [ -x "$WG" ] || WG=/usr/sbin/wg
|
|
NFT=/usr/sbin/nft; [ -x "$NFT" ] || NFT=/usr/bin/nft
|
|
FLOCK=/usr/bin/flock; [ -x "$FLOCK" ] || FLOCK=/usr/sbin/flock
|
|
STAT=/usr/bin/stat; [ -x "$STAT" ] || STAT=/bin/stat
|
|
CMP=/usr/bin/cmp; [ -x "$CMP" ] || CMP=/bin/cmp
|
|
SYSTEMCTL=$(for p in /usr/bin/systemctl /bin/systemctl /usr/sbin/systemctl /sbin/systemctl; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
|
|
LOCK=/run/virt-cloud/wg-fabric0.lock
|
|
if [ -L /run/virt-cloud ] || {{ [ -e /run/virt-cloud ] && [ ! -d /run/virt-cloud ]; }}; then exit 71; fi
|
|
mkdir -p /run/virt-cloud; chmod 700 /run/virt-cloud
|
|
[ ! -L "$LOCK" ] && {{ [ ! -e "$LOCK" ] || [ -f "$LOCK" ]; }} || exit 71
|
|
: >> "$LOCK"; chmod 600 "$LOCK"; exec 8>>"$LOCK"; "$FLOCK" -s 8
|
|
BRIDGE_CMD=
|
|
for p in /usr/sbin/bridge /usr/bin/bridge /sbin/bridge /bin/bridge; do
|
|
if [ -x "$p" ]; then BRIDGE_CMD=$p; break; fi
|
|
done
|
|
BRIDGE={_q(r.bridge)}; TABLE={_q(r.nft_table)}; ROUTE_TABLE={r.route_table}; WG_IFACE={_q(FABRIC_INTERFACE)}; OWNER={_q(r.owner_file)}; STATE={_q(r.state_file)}; TXN={_q(r.transaction_dir)}; PROXY_FILE={_q(proxy_file)}
|
|
NFT_SNAPSHOT_PREFIX={_q(f'/etc/virt-cloud/vpc-dataplane/{r.token}.nft')}
|
|
NFT_FILE={_q(nft_file)}; NFT_INCLUDE={_q(nft_include)}; LEGACY_MANIFEST={_q(LEGACY_FIREWALLD_NFT_MANIFEST)}; SYSCTL_FILE={_q(sysctl_file)}; IF_FRAGMENT={_q(ifup_fragment)}; NM_PROFILE_MARKER={_q(nm_profile_marker)}; FIREWALL_MODE={_q(firewall_mode)}; NFT_MAIN=; NFT_MAIN_COUNT=0; NM_PROFILE_UUID=; NM_PROFILE_FILE=
|
|
NFT_MARKER={_q(nft_marker)}
|
|
EXPECTED_DIGEST={_q(plan.fabric_plan_digest)}
|
|
{NFT_LIVE_SNAPSHOT_SH}
|
|
{_NMCLI_FILENAME_RESOLVER_SH}
|
|
{NETWORKMANAGER_VPC_PROFILE_SH}
|
|
{_FIREWALL_MODE_GUARD_SH}
|
|
{_FABRIC_NFT_VALIDATION_SH}
|
|
{_VPC_SYSCTL_VALIDATION_SH}
|
|
{VPC_NATIVE_TRANSACTION_GUARD_SH}
|
|
nft_plan_owned() {{
|
|
nft_plan_text=$1
|
|
nft_plan_digest=$2
|
|
printf '%s\n' "$nft_plan_digest" | grep -Eq '^[0-9a-f]{{64}}$' || return 1
|
|
vc_fabric_nft_strict_shape "$nft_plan_text" || return 1
|
|
[ "$(printf '%s\n' "$nft_plan_text" |
|
|
grep -Foc 'comment "virt-cloud-wg-fabric-v1:'"$nft_plan_digest"'"' || true)" = 8 ]
|
|
}}
|
|
{state_loader}
|
|
FABRIC_NFT_SNAPSHOT=/run/virt-cloud/.virt-cloud-vpc-fabric-observe.$$
|
|
vc_vpc_observe_cleanup() {{
|
|
rm -f "$VPC_PEER_FILE" "$VPC_ROUTE_FILE" "$VPC_TENANT_FILE" \
|
|
"$VPC_TENANT_FILE.nft" "$FABRIC_NFT_SNAPSHOT" \
|
|
"$FABRIC_NFT_SNAPSHOT.absent" "$FABRIC_NFT_SNAPSHOT.raw" \
|
|
"$FABRIC_NFT_SNAPSHOT.current" "$FABRIC_NFT_SNAPSHOT.inventory" \
|
|
2>/dev/null || true
|
|
}}
|
|
trap 'vc_vpc_observe_cleanup' EXIT
|
|
trap 'exit 79' HUP INT TERM
|
|
vc_vpc_fabric_observe_ready() {{
|
|
vpc_state_load || return 1
|
|
vc_nft_table_snapshot inet virt_cloud_wg_fabric \
|
|
"$FABRIC_NFT_SNAPSHOT" || return 1
|
|
[ -f "$FABRIC_NFT_SNAPSHOT" ] &&
|
|
[ ! -L "$FABRIC_NFT_SNAPSHOT" ] &&
|
|
[ ! -e "$FABRIC_NFT_SNAPSHOT.absent" ] || return 1
|
|
vc_fabric_observe_text=$(cat "$FABRIC_NFT_SNAPSHOT") || return 1
|
|
nft_plan_owned "$vc_fabric_observe_text" "$EXPECTED_DIGEST" || return 1
|
|
vpc_state_nft_set_matches || return 1
|
|
# Re-read the complete stateless table after the exact set comparison.
|
|
# This is the final shared-Fabric proof before the observer may emit the
|
|
# marker that participates in an active VPC result.
|
|
vc_nft_table_matches_snapshot inet virt_cloud_wg_fabric \
|
|
"$FABRIC_NFT_SNAPSHOT"
|
|
}}
|
|
firewall_mode_state=owned
|
|
vc_firewall_mode_ready || firewall_mode_state=mismatch
|
|
[ "$FIREWALL_MODE" = native_nftables ] || firewall_mode_state=mismatch
|
|
vc_vpc_firewall_boundary_clean || firewall_mode_state=mismatch
|
|
[ ! -e "$TXN" ] && [ ! -L "$TXN" ] || firewall_mode_state=mismatch
|
|
owner=missing; [ -f "$OWNER" ] && owner=$(cat "$OWNER" 2>/dev/null || printf missing)
|
|
printf '%s\\n' "__VC_VPC_BRIDGE_OWNER__=$owner"
|
|
persistence_target=no
|
|
opposite_clean=yes
|
|
for nft_candidate in /etc/nftables.conf /etc/sysconfig/nftables.conf; do
|
|
if [ -f "$nft_candidate" ] && [ ! -L "$nft_candidate" ] && [ "$($STAT -c '%u' "$nft_candidate" 2>/dev/null)" = 0 ]; then
|
|
case "$($STAT -c '%a' "$nft_candidate" 2>/dev/null)" in
|
|
[0-7][0145][0145]) NFT_MAIN="$nft_candidate"; NFT_MAIN_COUNT=$((NFT_MAIN_COUNT + 1)) ;;
|
|
*) : ;;
|
|
esac
|
|
fi
|
|
done
|
|
if [ -e "$LEGACY_MANIFEST" ] || [ -L "$LEGACY_MANIFEST" ]; then opposite_clean=no; fi
|
|
if [ "$NFT_MAIN_COUNT" = 1 ] && grep -Fqx "$NFT_INCLUDE" "$NFT_MAIN" &&
|
|
"$NFT" -c -f "$NFT_MAIN" >/dev/null 2>&1; then
|
|
persistence_target=yes
|
|
fi
|
|
persistence=no
|
|
nft_file_ok=no
|
|
if [ -f "$NFT_FILE" ] && [ ! -L "$NFT_FILE" ] && [ "$(stat -c '%u:%a' "$NFT_FILE" 2>/dev/null)" = 0:600 ] &&
|
|
grep -Fqx '# virt-cloud-vpc-dataplane-v1' "$NFT_FILE" &&
|
|
grep -Fqx '# vpc={plan.vpc.vpc_id} revision={plan.desired_revision} digest={plan.desired_digest}' "$NFT_FILE"; then
|
|
nft_file_ok=yes
|
|
fi
|
|
sysctl_file_ok=no
|
|
if vc_vpc_sysctl_artifact_owned "$SYSCTL_FILE" {_q(plan.vpc.vpc_id)} \
|
|
{plan.desired_revision} {_q(plan.desired_digest)} "$BRIDGE"; then
|
|
sysctl_file_ok=yes
|
|
fi
|
|
nm_owned=no
|
|
nm_zone=
|
|
nm_present=no
|
|
NMCLI=$(for p in /usr/bin/nmcli /usr/sbin/nmcli /bin/nmcli /sbin/nmcli; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
|
|
nm_profile_count=0
|
|
if [ -n "$NMCLI" ]; then
|
|
nm_profile_count=$($NMCLI -t --escape no -f NAME connection show 2>/dev/null |
|
|
awk -v wanted="$BRIDGE" '$0 == wanted {{count++}} END {{print count + 0}}') || nm_profile_count=invalid
|
|
case "$nm_profile_count" in ''|*[!0-9]*) nm_profile_count=invalid; nm_present=yes ;; esac
|
|
if [ "$nm_profile_count" != invalid ] && [ "$nm_profile_count" -gt 0 ]; then nm_present=yes; fi
|
|
fi
|
|
if [ -n "$NMCLI" ] && [ "$nm_profile_count" = 1 ]; then
|
|
nm_stable=$($NMCLI -t --escape no -g connection.stable-id connection show "$BRIDGE" 2>/dev/null | sed -n '1p')
|
|
nm_type=$($NMCLI -t --escape no -g connection.type connection show "$BRIDGE" 2>/dev/null | sed -n '1p')
|
|
nm_iface=$($NMCLI -t --escape no -g connection.interface-name connection show "$BRIDGE" 2>/dev/null | sed -n '1p')
|
|
nm_method=$($NMCLI -t --escape no -g ipv4.method connection show "$BRIDGE" 2>/dev/null | sed -n '1p')
|
|
nm6_method=$($NMCLI -t --escape no -g ipv6.method connection show "$BRIDGE" 2>/dev/null | sed -n '1p')
|
|
nm_zone=$($NMCLI -t --escape no -g connection.zone connection show "$BRIDGE" 2>/dev/null | sed -n '1p')
|
|
if [ "$nm_stable" = "$NM_PROFILE_MARKER" ] && [ "$nm_type" = bridge ] && [ "$nm_iface" = "$BRIDGE" ] && [ "$nm_method" = disabled ] && [ "$nm6_method" = disabled ]; then
|
|
if vc_nm_profile_resolve && vc_nm_profile_expect active; then nm_owned=yes; fi
|
|
fi
|
|
fi
|
|
ifup_owned=no
|
|
if [ -f "$IF_FRAGMENT" ] && [ ! -L "$IF_FRAGMENT" ] && [ "$(stat -c '%u:%a' "$IF_FRAGMENT" 2>/dev/null)" = 0:600 ] &&
|
|
grep -Fqx '# {DATAPLANE_OWNER_MARKER} vpc={plan.vpc.vpc_id} revision={plan.desired_revision} digest={plan.desired_digest}' "$IF_FRAGMENT" &&
|
|
grep -Fqx 'iface {r.bridge} inet manual' "$IF_FRAGMENT"; then
|
|
if ! grep -Eq '^[[:space:]]*(auto|allow-[^[:space:]]+)[[:space:]]+{r.bridge}([[:space:]]|$)' "$IF_FRAGMENT"; then
|
|
ifup_owned=yes
|
|
fi
|
|
fi
|
|
printf '%s\\n' "__VC_VPC_FIREWALL_MODE__=$FIREWALL_MODE"
|
|
printf '%s\\n' "__VC_VPC_FIREWALL_STATE__=$firewall_mode_state"
|
|
nft_snapshot_state=yes
|
|
for snapshot_family in inet bridge arp; do
|
|
snapshot_path="$NFT_SNAPSHOT_PREFIX.$snapshot_family.live"
|
|
if [ -e "$snapshot_path" ] || [ -L "$snapshot_path" ]; then
|
|
nft_snapshot_state=no
|
|
fi
|
|
done
|
|
printf '%s\\n' "__VC_VPC_NFT_SNAPSHOT__=$nft_snapshot_state"
|
|
if [ "$firewall_mode_state" = owned ] && [ "$NFT_MAIN_COUNT" = 1 ] &&
|
|
[ "$persistence_target" = yes ] && [ "$opposite_clean" = yes ] &&
|
|
[ "$nft_file_ok" = yes ] && [ "$sysctl_file_ok" = yes ] &&
|
|
[ "$nft_snapshot_state" = yes ] &&
|
|
( [ "$nm_owned" = yes ] && [ "$ifup_owned" = no ] ||
|
|
[ "$nm_owned" = no ] && [ "$nm_present" = no ] && [ "$ifup_owned" = yes ] ); then
|
|
persistence=yes
|
|
fi
|
|
printf '%s\\n' "__VC_VPC_PERSISTENCE__=$persistence"
|
|
if "$IP" -d link show dev "$BRIDGE" >/dev/null 2>&1; then printf '%s\\n' '__VC_VPC_BRIDGE__='"$BRIDGE"; else printf '%s\\n' '__VC_VPC_BRIDGE__=missing'; fi
|
|
bridge_boundary=no
|
|
bridge_shape=no; bridge_addr_ok=no; bridge_ports_ok=no; bridge_routes_ok=no; bridge_routes6_ok=no
|
|
if [ -n "$BRIDGE_CMD" ]; then
|
|
if bridge_shape_text=$("$IP" -d link show dev "$BRIDGE" 2>/dev/null) &&
|
|
printf '%s\n' "$bridge_shape_text" | grep -Eq '(^|[[:space:]])bridge[[:space:]]+(forward_delay|stp_state|hello_time|ageing_time)([[:space:]]|$)'; then
|
|
bridge_shape=yes
|
|
fi
|
|
if bridge_addr4=$("$IP" -o -4 addr show dev "$BRIDGE" 2>/dev/null) &&
|
|
bridge_addr6=$("$IP" -o -6 addr show dev "$BRIDGE" 2>/dev/null) &&
|
|
[ -z "$bridge_addr4$bridge_addr6" ]; then
|
|
bridge_addr_ok=yes
|
|
fi
|
|
if bridge_ports=$("$BRIDGE_CMD" link show master "$BRIDGE" 2>/dev/null); then
|
|
bridge_ports_ok=yes
|
|
fi
|
|
if bridge_routes=$("$IP" -o -4 route show table all dev "$BRIDGE" 2>/dev/null) &&
|
|
printf '%s\n' "$bridge_routes" | awk -v wanted_table="$ROUTE_TABLE" '
|
|
NF {{table="main"; proto=""; prefix=$1
|
|
for (i=1;i<=NF;i++) {{if ($i=="table") table=$(i+1); if ($i=="proto") proto=$(i+1)}}
|
|
if (table=="254") table="main"
|
|
if (prefix=="default" || table!=wanted_table || proto!="186") bad=1}}
|
|
END {{exit bad ? 1 : 0}}'; then
|
|
bridge_routes_ok=yes
|
|
fi
|
|
if bridge_routes6=$($IP -o -6 route show table all dev "$BRIDGE" 2>/dev/null) && [ -z "$bridge_routes6" ]; then
|
|
bridge_routes6_ok=yes
|
|
fi
|
|
fi
|
|
if [ "$bridge_shape" = yes ] && [ "$bridge_addr_ok" = yes ] &&
|
|
[ "$bridge_ports_ok" = yes ] && [ "$bridge_routes_ok" = yes ] &&
|
|
[ "$bridge_routes6_ok" = yes ]; then
|
|
bridge_bad=no
|
|
while IFS= read -r bridge_port_line; do
|
|
[ -n "$bridge_port_line" ] || continue
|
|
bridge_port=$(printf '%s\\n' "$bridge_port_line" | sed -n 's/^[0-9][0-9]*: \\([^:[:space:]]*\\).*/\\1/p')
|
|
case "$bridge_port" in
|
|
vnet[0-9]*|tap[0-9A-Za-z_.-]*|macvtap[0-9]*|fwpr[0-9A-Za-z_.-]*|fwln[0-9A-Za-z_.-]*) : ;;
|
|
*) bridge_bad=yes ;;
|
|
esac
|
|
done <<__VC_VPC_OBS_PORTS__
|
|
$bridge_ports
|
|
__VC_VPC_OBS_PORTS__
|
|
if [ "$bridge_bad" = no ]; then
|
|
bridge_boundary=yes
|
|
fi
|
|
fi
|
|
printf '%s\\n' "__VC_VPC_BRIDGE_BOUNDARY__=$bridge_boundary"
|
|
if [ -r "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp" ]; then printf '%s\\n' '__VC_VPC_PROXY_ARP__='"$(cat "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp")"; else printf '%s\\n' '__VC_VPC_PROXY_ARP__=missing'; fi
|
|
if [ -r "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp_pvlan" ]; then printf '%s\\n' '__VC_VPC_PROXY_ARP_PVLAN__='"$(cat "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp_pvlan")"; else printf '%s\\n' '__VC_VPC_PROXY_ARP_PVLAN__=missing'; fi
|
|
if [ -r "/proc/sys/net/ipv4/conf/$BRIDGE/rp_filter" ]; then printf '%s\\n' '__VC_VPC_RP_FILTER__='"$(cat "/proc/sys/net/ipv4/conf/$BRIDGE/rp_filter")"; else printf '%s\\n' '__VC_VPC_RP_FILTER__=missing'; fi
|
|
proxy_records=$($IP neigh show proxy dev "$BRIDGE" 2>/dev/null | awk '$1 ~ /^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$/ {{print $1}}' | LC_ALL=C sort -u || true)
|
|
printf '%s\\n' "$proxy_records" | while IFS= read -r proxy_ip; do
|
|
[ -n "$proxy_ip" ] && printf '%s\\n' "__VC_VPC_PROXY_IP__=$proxy_ip"
|
|
done
|
|
proxy_state=missing
|
|
if [ -f "$PROXY_FILE" ] && sed -n '1p' "$PROXY_FILE" | grep -Fqx '# virt-cloud-vpc-proxy-v1'; then
|
|
expected_proxy=$(sed '1d' "$PROXY_FILE" | LC_ALL=C sort -u)
|
|
if [ "$proxy_records" = "$expected_proxy" ]; then proxy_state=owned; fi
|
|
fi
|
|
printf '%s\\n' '__VC_VPC_PROXY_STATE__='"$proxy_state"
|
|
if "$NFT" list table inet "$TABLE" >/dev/null 2>&1; then printf '%s\\n' '__VC_VPC_NFT_TABLE__='"$TABLE"; else printf '%s\\n' '__VC_VPC_NFT_TABLE__=missing'; fi
|
|
if [ -f "$STATE" ] && grep -Fqx 'owner|{DATAPLANE_OWNER_MARKER}' "$STATE"; then
|
|
printf '%s\\n' '__VC_VPC_STATE_OWNER__=owned'
|
|
printf '%s\\n' "__VC_VPC_STATE_REVISION__=$(awk -F'|' '$1==\"revision\"{{print $2; exit}}' \"$STATE\")"
|
|
printf '%s\\n' "__VC_VPC_STATE_DIGEST__=$(awk -F'|' '$1==\"digest\"{{print $2; exit}}' \"$STATE\")"
|
|
state_firewall_record=$(awk -F'|' '$1=="firewall_mode" {{count++; value=$2}} END {{print (count + 0) "|" value}}' "$STATE")
|
|
state_firewall_observed=invalid
|
|
case "$state_firewall_record" in
|
|
'0|') state_firewall_observed= ;;
|
|
'1|native_nftables') state_firewall_observed=native_nftables ;;
|
|
esac
|
|
printf '%s\\n' "__VC_VPC_STATE_FIREWALL_MODE__=$state_firewall_observed"
|
|
else
|
|
printf '%s\\n' '__VC_VPC_STATE_OWNER__=missing'
|
|
fi
|
|
nft_text=$($NFT list table inet "$TABLE" 2>/dev/null || true)
|
|
nft_verified=yes
|
|
printf '%s\\n' "$nft_text" | grep -Fq "table inet $TABLE" || nft_verified=no
|
|
printf '%s\\n' "$nft_text" | grep -Fq "chain forward" || nft_verified=no
|
|
printf '%s\\n' "$nft_text" | grep -Fq "hook forward priority -190" || nft_verified=no
|
|
printf '%s\\n' "$nft_text" | grep -Fq "chain input" || nft_verified=no
|
|
printf '%s\\n' "$nft_text" | grep -Fq "chain output" || nft_verified=no
|
|
printf '%s\\n' "$nft_text" | grep -Fq "set member_ips" || nft_verified=no
|
|
printf '%s\\n' "$nft_text" | grep -Fq "set vpc_dest_ips" || nft_verified=no
|
|
{nft_checks}
|
|
if [ "$nft_verified" = yes ]; then
|
|
printf '%s\\n' '__VC_VPC_ACL_REVISION__={plan.desired_revision}'
|
|
printf '%s\\n' '__VC_VPC_ACL_DIGEST__={plan.desired_digest}'
|
|
printf '%s\\n' '__VC_VPC_ANTI_SPOOF__=yes'
|
|
else
|
|
printf '%s\\n' '__VC_VPC_ACL_REVISION__=missing'
|
|
printf '%s\\n' '__VC_VPC_ACL_DIGEST__=missing'
|
|
printf '%s\\n' '__VC_VPC_ANTI_SPOOF__=no'
|
|
fi
|
|
{bridge_observe}
|
|
{arp_observe}
|
|
route_records=$($IP -o -4 route show table "$ROUTE_TABLE" 2>/dev/null | awk '{{
|
|
destination=$1; device=""; via="-"; metric="100"; proto=""
|
|
for (i=1; i<=NF; i++) {{
|
|
if ($i == "dev" && i < NF) device=$(i+1)
|
|
if ($i == "via" && i < NF) via=$(i+1)
|
|
if ($i == "metric" && i < NF) metric=$(i+1)
|
|
if ($i == "proto" && i < NF) proto=$(i+1)
|
|
}}
|
|
if (destination != "") printf "%s|%s|%s|%s|%s\\n", destination, device, via, metric, proto
|
|
}}' || true)
|
|
rule_records=$("$IP" -o -4 -N rule show 2>/dev/null || true)
|
|
rule_scope_count=$(printf '%s\\n' "$rule_records" | awk -v expected_to={_q(plan.vpc.cidr)} -v expected_table={_q(str(r.route_table))} '{{
|
|
destination=""; table=""
|
|
for (i=1;i<=NF;i++) {{
|
|
if ($i == "to" && i < NF) destination=$(i+1)
|
|
if ($i == "lookup" && i < NF) table=$(i+1)
|
|
}}
|
|
if (destination == expected_to || table == expected_table) count++
|
|
}} END {{print count + 0}}')
|
|
rule_extra=yes; [ "$rule_scope_count" = 2 ] && rule_extra=no
|
|
printf '%s\\n' "__VC_VPC_RULE_EXTRA__=$rule_extra"
|
|
{rule_checks}
|
|
route_extra=$(printf '%s\\n' "$route_records" | awk '{{
|
|
if (length($0) == 0) next
|
|
count++
|
|
{route_extra_awk}
|
|
extra=1
|
|
}} END {{ if (count != {len(routes)}) extra=1; print extra ? "yes" : "no" }}')
|
|
printf '%s\\n' "__VC_VPC_ROUTE_EXTRA__=$route_extra"
|
|
{route_checks}
|
|
while IFS='|' read -r peer_host key expected_allowed; do
|
|
[ -n "$peer_host" ] || continue
|
|
actual=$(printf '%s\\n' "$($WG show "$WG_IFACE" allowed-ips 2>/dev/null || true)" | awk -v wanted="$key" '$1 == wanted {{print $2; exit}}')
|
|
# ``wg show`` does not promise AllowedIPs order. Canonicalize both
|
|
# comma-separated lists before comparing, and retain the complete set in
|
|
# the marker. The Python adapter projects this to this VPC's tenant /32s
|
|
# after proving that no extra or missing entry exists.
|
|
expected_set=$(printf '%s\\n' "$expected_allowed" | tr ',' '\\n' | awk 'NF' | LC_ALL=C sort -u)
|
|
actual_set=$(printf '%s\\n' "$actual" | tr ',' '\\n' | awk 'NF' | LC_ALL=C sort -u)
|
|
allowed=$(printf '%s\\n' "$actual_set" | awk 'BEGIN {{ ORS=""; sep="" }} NF {{ printf "%s%s", sep, $0; sep="," }}')
|
|
state=missing
|
|
if [ -n "$actual_set" ] && [ "$actual_set" = "$expected_set" ]; then
|
|
state=owned
|
|
else
|
|
allowed=missing
|
|
fi
|
|
printf '%s\\n' "__VC_VPC_WG_ALLOWED__=$peer_host|$key|$allowed"
|
|
printf '%s\\n' "__VC_VPC_WG_STATE__=$peer_host|$state"
|
|
done <<'{peer_marker}'
|
|
{peer_lines}
|
|
{peer_marker}
|
|
expected_fabric_peer_keys={fabric_peer_key_arg}
|
|
wg_peer_set=unverified
|
|
if [ "{fabric_peer_keys_provided}" = yes ]; then
|
|
live_fabric_peer_keys=$("$WG" show "$WG_IFACE" peers 2>/dev/null || true)
|
|
wg_peer_set=exact
|
|
for live_peer_key in $live_fabric_peer_keys; do
|
|
case " $expected_fabric_peer_keys " in
|
|
*" $live_peer_key "*) : ;;
|
|
*) wg_peer_set=extra ;;
|
|
esac
|
|
done
|
|
for expected_peer_key in $expected_fabric_peer_keys; do
|
|
printf '%s\n' "$live_fabric_peer_keys" | grep -Fqx "$expected_peer_key" || wg_peer_set=missing
|
|
done
|
|
fi
|
|
printf '%s\n' "__VC_VPC_WG_PEER_SET__=$wg_peer_set"
|
|
fabric_nft_state=no
|
|
if vc_vpc_fabric_observe_ready; then fabric_nft_state=yes; fi
|
|
printf '%s\n' "__VC_VPC_FABRIC_NFT__=$fabric_nft_state"
|
|
exit 0
|
|
'''
|
|
|
|
|
|
def _markers(output: str) -> dict[str, Any]:
|
|
if not isinstance(output, str) or len(output) > MAX_OUTPUT:
|
|
raise VpcDataplaneError("VPC 回读输出过大", 502, "vpc_readback_invalid")
|
|
result: dict[str, Any] = {"routes": [], "wg_allowed_ips": [],
|
|
"wg_states": {}, "rules": [], "proxy_ips": []}
|
|
for line in output.splitlines():
|
|
if not line.startswith("__VC_VPC_") or "=" not in line:
|
|
continue
|
|
key, value = line.split("=", 1)
|
|
# A peer can legitimately carry many exact /32s from multiple VPCs;
|
|
# keep a bounded upper limit without truncating the authenticated set.
|
|
if len(value) > 65536:
|
|
raise VpcDataplaneError("VPC 回读字段过长", 502, "vpc_readback_invalid")
|
|
if key == "__VC_VPC_ROUTE__":
|
|
p = value.split("|")
|
|
if len(p) == 6:
|
|
result["routes"].append({
|
|
"destination_cidr": p[0], "device": p[1], "table": p[2],
|
|
"via": None if p[3] == "-" else p[3],
|
|
"metric": p[4], "owner": DATAPLANE_OWNER_MARKER,
|
|
"state": p[5],
|
|
})
|
|
elif key == "__VC_VPC_PROXY_IP__":
|
|
if re.fullmatch(r"[0-9]{1,3}(?:\.[0-9]{1,3}){3}", value):
|
|
try:
|
|
address = ipaddress.IPv4Address(value)
|
|
except ipaddress.AddressValueError:
|
|
continue
|
|
result["proxy_ips"].append(str(address))
|
|
elif key == "__VC_VPC_RULE__":
|
|
p = value.split("|")
|
|
if len(p) == 5:
|
|
result["rules"].append({"priority": p[0], "iif": p[1],
|
|
"destination_cidr": p[2],
|
|
"table": p[3], "state": p[4]})
|
|
elif key == "__VC_VPC_WG_ALLOWED__":
|
|
p = value.split("|")
|
|
if len(p) == 3:
|
|
result["wg_allowed_ips"].append({"peer_host_id": p[0], "public_key": p[1], "tenant_allowed_ips": [] if p[2] == "missing" else [x for x in p[2].split(",") if x]})
|
|
elif key == "__VC_VPC_WG_STATE__":
|
|
p = value.split("|")
|
|
if len(p) == 2 and p[1] in {"owned", "missing"}:
|
|
result["wg_states"][p[0]] = p[1]
|
|
else:
|
|
result[key.removeprefix("__VC_VPC_").removesuffix("__").lower()] = value
|
|
return result
|
|
|
|
|
|
def patch_private_interface_xml(xml: str | bytes, *, member: VpcMember,
|
|
resources: VpcDataplaneResources,
|
|
guest_network_generation: str,
|
|
link_state: str = "up",
|
|
vpc_id: str | None = None,
|
|
desired_revision: int | None = None,
|
|
desired_digest: str | None = None) -> str:
|
|
"""Add/validate a private0 NIC on an explicit panel-owned bridge."""
|
|
if link_state not in {"up", "down"}:
|
|
raise VpcDataplaneError("私网网卡 link 状态无效")
|
|
if (not isinstance(guest_network_generation, str)
|
|
or _DIGEST.fullmatch(guest_network_generation) is None):
|
|
raise VpcDataplaneError("来宾私网 generation 无效")
|
|
private_mac = _mac(member.private_mac)
|
|
try:
|
|
root = ET.fromstring(xml)
|
|
except ET.ParseError as exc:
|
|
raise VpcDataplaneError("虚拟机 XML 无效", 409, "vpc_private_xml_invalid") from exc
|
|
devices = root.find("./devices")
|
|
if devices is None:
|
|
raise VpcDataplaneError("虚拟机 XML 缺少 devices", 409, "vpc_private_xml_invalid")
|
|
interfaces = list(devices.findall("interface"))
|
|
try:
|
|
public = select_public_interface(interfaces, allow_legacy=True)
|
|
# Legacy domains may have a single public NIC without an alias. Give
|
|
# it the canonical libvirt alias before adding private0 so the next
|
|
# reconciliation does not mistake the public NIC for an unknown role.
|
|
set_libvirt_interface_alias(root, public, PUBLIC_LIBVIRT_ALIAS)
|
|
private = select_private_interface(interfaces)
|
|
except ValueError as exc:
|
|
raise VpcDataplaneError(str(exc), 409, "vpc_private_xml_role_invalid") from exc
|
|
if private is None:
|
|
private = ET.SubElement(devices, "interface", {"type": "bridge"})
|
|
else:
|
|
if private.get("type") != "bridge":
|
|
raise VpcDataplaneError("private0 必须使用 bridge", 409, "vpc_private_xml_source_invalid")
|
|
existing_filters = private.findall("filterref")
|
|
if existing_filters and any(
|
|
item.get("filter") != "clean-traffic"
|
|
for item in existing_filters):
|
|
raise VpcDataplaneError("private0 nwfilter 不受面板管理", 409,
|
|
"vpc_private_xml_filter_invalid")
|
|
source = private.find("source")
|
|
if source is None or source.get("network") is not None or source.get("bridge") != resources.bridge:
|
|
raise VpcDataplaneError("private0 bridge 不匹配", 409, "vpc_private_xml_source_mismatch")
|
|
source = private.find("source")
|
|
if source is None:
|
|
source = ET.Element("source")
|
|
private.insert(0, source)
|
|
source.attrib.clear(); source.set("bridge", resources.bridge)
|
|
mac = private.find("mac")
|
|
if mac is None:
|
|
mac = ET.Element("mac"); private.insert(0, mac)
|
|
mac.set("address", private_mac)
|
|
try:
|
|
set_libvirt_interface_alias(root, private, PRIVATE_LIBVIRT_ALIAS)
|
|
except ValueError as exc:
|
|
raise VpcDataplaneError(
|
|
str(exc), 409, "vpc_private_xml_role_invalid"
|
|
) from exc
|
|
model = private.find("model")
|
|
if model is None:
|
|
model = ET.SubElement(private, "model")
|
|
model.set("type", "virtio")
|
|
link = private.find("link")
|
|
if link is None:
|
|
link = ET.SubElement(private, "link")
|
|
link.set("state", link_state)
|
|
for item in list(private.findall("filterref")):
|
|
private.remove(item)
|
|
# The private L3 data plane owns anti-spoofing and ACL enforcement in its
|
|
# dedicated bridge/inet/arp nft tables. Do not attach the public
|
|
# libvirt ``clean-traffic`` nwfilter to private0; the role observer
|
|
# deliberately treats any private filter as an unsafe mixed policy.
|
|
metadata = root.find("metadata")
|
|
if metadata is None:
|
|
metadata = ET.Element("metadata"); root.insert(0, metadata)
|
|
managed = metadata.find(f"{{{PANEL_NS}}}managed")
|
|
if managed is None:
|
|
managed = ET.SubElement(metadata, f"{{{PANEL_NS}}}managed")
|
|
managed.set("private-member-id", member.member_id)
|
|
managed.set("private-ip-cidr", f"{member.private_ip}/32")
|
|
managed.set("private-network-bridge", resources.bridge)
|
|
managed.set("private-network-verified", "true")
|
|
managed.set("private-dataplane-owner", DATAPLANE_OWNER_MARKER)
|
|
managed.set("private-guest-network-generation",
|
|
guest_network_generation)
|
|
if vpc_id is not None:
|
|
managed.set("private-vpc-id", vpc_id)
|
|
if desired_revision is not None:
|
|
managed.set("private-network-revision", str(desired_revision))
|
|
if desired_digest is not None:
|
|
managed.set("private-network-digest", desired_digest)
|
|
ET.register_namespace("panel", PANEL_NS)
|
|
return ET.tostring(root, encoding="unicode")
|
|
|
|
|
|
def _member_nics(
|
|
plan: VpcDataplanePlan,
|
|
xml_by_instance: Mapping[str, Any] | None,
|
|
) -> tuple[dict[str, Any], ...]:
|
|
def xml_details(
|
|
xml: str,
|
|
member: VpcMember,
|
|
expected_generation: str,
|
|
) -> tuple[bool, bool, bool, str | None]:
|
|
try:
|
|
root = ET.fromstring(xml)
|
|
private = select_private_interface(
|
|
list(root.findall("./devices/interface")))
|
|
source = private.find("source") if private is not None else None
|
|
mac = private.find("mac") if private is not None else None
|
|
ok = bool(
|
|
private and private.get("type") == "bridge"
|
|
and source is not None
|
|
and source.get("bridge") == plan.resources.bridge
|
|
and mac is not None
|
|
and mac.get("address", "").lower() == member.private_mac
|
|
# VPC isolation is provided by the dedicated bridge/inet/arp
|
|
# nft context. The public libvirt nwfilter is intentionally
|
|
# absent from private0.
|
|
and not private.findall("filterref")
|
|
)
|
|
link = private.find("link") if private is not None else None
|
|
link_state = link.get("state", "up").lower() if link is not None else "up"
|
|
if link_state not in {"up", "down"}:
|
|
return False, False, False, None
|
|
managed = root.find(f"./metadata/{{{PANEL_NS}}}managed")
|
|
guest_generation_ok = bool(
|
|
managed is not None
|
|
and managed.get("private-vpc-id") == plan.vpc.vpc_id
|
|
and managed.get("private-member-id") == member.member_id
|
|
and managed.get("private-ip-cidr") == f"{member.private_ip}/32"
|
|
and managed.get("private-network-bridge")
|
|
== plan.resources.bridge
|
|
and managed.get("private-dataplane-owner")
|
|
== DATAPLANE_OWNER_MARKER
|
|
and managed.get("private-guest-network-generation")
|
|
== expected_generation
|
|
)
|
|
desired_metadata_ok = bool(
|
|
guest_generation_ok
|
|
and managed.get("private-network-revision")
|
|
== str(plan.desired_revision)
|
|
and managed.get("private-network-digest")
|
|
== plan.desired_digest
|
|
)
|
|
return ok, guest_generation_ok, desired_metadata_ok, link_state
|
|
except (ET.ParseError, ValueError):
|
|
return False, False, False, None
|
|
|
|
result = []
|
|
for member in plan.local_members:
|
|
verified = False
|
|
config_verified = False
|
|
seed_verified = False
|
|
restart_required = False
|
|
guest_verified = False
|
|
guest_readback = "missing"
|
|
guest_reason_code = "vpc_private_guest_agent_unavailable"
|
|
guest_private_mac: str | None = None
|
|
guest_private_ip_cidr: str | None = None
|
|
guest_link_state: str | None = None
|
|
guest_routes: list[str] = []
|
|
domain_running = False
|
|
link_state: str | None = None
|
|
expected_routes = private_guest_routes(plan.vpc, member)
|
|
expected_generation = private_guest_network_generation(
|
|
member, expected_routes)
|
|
if xml_by_instance and member.instance_uuid in xml_by_instance:
|
|
value = xml_by_instance[member.instance_uuid]
|
|
if isinstance(value, Mapping):
|
|
seed_verified = value.get("seed_verified") is True
|
|
inactive = value.get("inactive")
|
|
live = value.get("live")
|
|
inactive_ok = False
|
|
live_device_ok = False
|
|
live_generation_ok = False
|
|
inactive_link: str | None = None
|
|
live_link: str | None = None
|
|
if isinstance(inactive, str):
|
|
(inactive_device_ok, inactive_generation_ok,
|
|
inactive_metadata_ok, inactive_link) = xml_details(
|
|
inactive, member, expected_generation)
|
|
inactive_ok = bool(
|
|
inactive_device_ok and inactive_generation_ok
|
|
and inactive_metadata_ok)
|
|
config_verified = inactive_ok
|
|
domain_running = live is not None
|
|
if domain_running:
|
|
if isinstance(live, str):
|
|
(live_device_ok, live_generation_ok,
|
|
_live_metadata_ok, live_link) = xml_details(
|
|
live, member, expected_generation)
|
|
guest = value.get("guest_network")
|
|
if isinstance(guest, Mapping):
|
|
guest_readback = str(guest.get("state", "missing"))[:32]
|
|
reason = guest.get("reason_code")
|
|
if isinstance(reason, str) and re.fullmatch(
|
|
r"[a-z0-9_]{1,64}", reason):
|
|
guest_reason_code = reason
|
|
observed_routes = guest.get("routes")
|
|
if (isinstance(observed_routes, list)
|
|
and len(observed_routes) <= 4096
|
|
and all(isinstance(route, str)
|
|
for route in observed_routes)):
|
|
guest_routes = sorted(set(observed_routes))
|
|
observed_mac = guest.get("private_mac")
|
|
observed_ip = guest.get("private_ip_cidr")
|
|
guest_private_mac = (
|
|
observed_mac.lower()
|
|
if isinstance(observed_mac, str) else None)
|
|
guest_private_ip_cidr = (
|
|
observed_ip if isinstance(observed_ip, str) else None)
|
|
guest_link_state = (
|
|
guest.get("link_state")
|
|
if guest.get("link_state") in {"up", "down"}
|
|
else None)
|
|
guest_verified = bool(
|
|
guest.get("verified") is True
|
|
and guest_readback == "confirmed"
|
|
and guest.get("source") == "qemu_guest_agent"
|
|
and guest_link_state == "up"
|
|
and guest_private_mac == member.private_mac
|
|
and guest_private_ip_cidr == f"{member.private_ip}/32"
|
|
and guest_routes == list(expected_routes)
|
|
)
|
|
if guest_verified:
|
|
guest_reason_code = None
|
|
# Persistent and live libvirt device identity remains a
|
|
# prerequisite, but only a bounded guest-agent observation can
|
|
# prove that the address and routes exist inside the guest.
|
|
stopped_exact = bool(
|
|
inactive_ok and seed_verified and not domain_running)
|
|
verified = bool(
|
|
stopped_exact
|
|
or (inactive_ok and seed_verified and domain_running
|
|
and live_device_ok
|
|
and inactive_link == live_link))
|
|
restart_required = bool(
|
|
not stopped_exact and (
|
|
not domain_running
|
|
or not inactive_ok
|
|
or not seed_verified
|
|
or not live_generation_ok))
|
|
if stopped_exact:
|
|
guest_readback = "stopped"
|
|
guest_reason_code = "vpc_private_guest_stopped"
|
|
if restart_required:
|
|
guest_reason_code = "vpc_private_guest_restart_required"
|
|
link_state = live_link if domain_running else inactive_link
|
|
elif isinstance(value, str):
|
|
(device_ok, generation_ok, metadata_ok,
|
|
link_state) = xml_details(
|
|
value, member, expected_generation)
|
|
verified = device_ok and generation_ok and metadata_ok
|
|
config_verified = verified
|
|
result.append({"member_id": member.member_id, "instance_uuid": member.instance_uuid,
|
|
"interface": PRIVATE_INTERFACE, "private_ip_cidr": f"{member.private_ip}/32",
|
|
"private_mac": member.private_mac, "xml_verified": verified,
|
|
"config_verified": config_verified,
|
|
"seed_verified": seed_verified,
|
|
"restart_required": restart_required,
|
|
"domain_running": domain_running,
|
|
"guest_network_verified": guest_verified,
|
|
"guest_network_readback": guest_readback,
|
|
"guest_reason_code": guest_reason_code,
|
|
"guest_private_mac": guest_private_mac,
|
|
"guest_private_ip_cidr": guest_private_ip_cidr,
|
|
"guest_link_state": guest_link_state,
|
|
"guest_routes": guest_routes,
|
|
"link_state": link_state})
|
|
return tuple(result)
|
|
|
|
|
|
class VpcPrivateHostAdapter:
|
|
"""Execute generated scripts through a pinned remote host manager."""
|
|
|
|
def __init__(
|
|
self,
|
|
manager: Any,
|
|
profile: Mapping[str, Any] | None = None,
|
|
*,
|
|
request_deadline: float | None = None,
|
|
):
|
|
self.manager = manager
|
|
self.profile = dict(profile) if isinstance(profile, Mapping) else None
|
|
self.request_deadline = request_deadline
|
|
|
|
def _run(self, script: str) -> str:
|
|
current = self.profile if self.profile is not None else self.manager.current()
|
|
if not isinstance(current, Mapping) or not current:
|
|
raise VpcDataplaneError("宿主机尚未配置", 409, "vpc_host_not_configured")
|
|
if current.get("key_locked"):
|
|
raise VpcDataplaneError("宿主机 SSH 私钥仍锁定", 409, "vpc_host_key_locked")
|
|
timeout = 35
|
|
if self.request_deadline is not None:
|
|
remaining = self.request_deadline - time.monotonic()
|
|
if remaining < 1:
|
|
raise VpcDataplaneError(
|
|
"VPC 来宾网络核验超过请求预算", 502,
|
|
"vpc_private_guest_probe_deadline_exceeded")
|
|
timeout = min(timeout, int(remaining))
|
|
argv = [getattr(self.manager, "ssh_binary", "ssh"), "-F", current["ssh_config_file"], "-T", "-o", "ConnectTimeout=10", current["ssh_target"], "/bin/sh -s"]
|
|
try:
|
|
result = self.manager._exec(
|
|
argv, timeout=timeout, input_text=script)
|
|
except Exception as exc:
|
|
raise VpcDataplaneError("宿主机 VPC SSH 请求失败", 502, "vpc_host_connection_failed") from exc
|
|
if result.returncode != 0:
|
|
raise VpcDataplaneError("宿主机 VPC 命令失败", 502, "vpc_remote_command_failed")
|
|
return result.stdout
|
|
|
|
def _deployment_contract(self) -> tuple[str, str, str, str, str]:
|
|
"""Validate the same immutable host capability gate as Fabric."""
|
|
from .fabric import (
|
|
FabricError, _profile_ifdown_path, _profile_ifquery_path,
|
|
_profile_ifup_path, _profile_network_mode,
|
|
_profile_firewall_mode,
|
|
)
|
|
|
|
current = self.profile if self.profile is not None else self.manager.current()
|
|
try:
|
|
mode = _profile_network_mode(current)
|
|
except FabricError as exc:
|
|
raise VpcDataplaneError(
|
|
"宿主机网络管理器与 VPC 部署模式不匹配", 409,
|
|
"vpc_dataplane_network_manager_conflict") from exc
|
|
try:
|
|
firewall_mode = _profile_firewall_mode(current)
|
|
except FabricError as exc:
|
|
raise VpcDataplaneError(
|
|
"宿主机防火墙与 VPC 部署模式不匹配", 409,
|
|
"vpc_dataplane_firewall_mode_conflict") from exc
|
|
capabilities = current.get("capabilities") if isinstance(current, Mapping) else None
|
|
network = capabilities.get("network") if isinstance(capabilities, Mapping) else None
|
|
wireguard = capabilities.get("wireguard") if isinstance(capabilities, Mapping) else None
|
|
nftables = capabilities.get("nftables") if isinstance(capabilities, Mapping) else None
|
|
if (not isinstance(network, Mapping)
|
|
or not isinstance(wireguard, Mapping)
|
|
or not isinstance(nftables, Mapping)
|
|
or capabilities.get("vpc_ready") is not True
|
|
or network.get("firewall_mode") != firewall_mode
|
|
or network.get("ip_forward") != 1
|
|
or nftables.get("ready") is not True
|
|
or wireguard.get("interface") != FABRIC_INTERFACE
|
|
or wireguard.get("interface_present") is not True
|
|
or wireguard.get("interface_is_wireguard") is not True
|
|
or wireguard.get("owned_by_panel") is not True):
|
|
raise VpcDataplaneError(
|
|
"宿主机 VPC 网络或防火墙能力未通过探测", 409,
|
|
"vpc_host_capability_missing")
|
|
return (
|
|
mode,
|
|
_profile_ifup_path(current),
|
|
_profile_ifdown_path(current),
|
|
_profile_ifquery_path(current),
|
|
firewall_mode,
|
|
)
|
|
|
|
def _render_contract(self) -> tuple[str, str, str, str, str]:
|
|
"""Normalize legacy test doubles while production uses five fields."""
|
|
contract = self._deployment_contract()
|
|
if len(contract) == 4:
|
|
mode, ifup_path, ifdown_path, ifquery_path = contract
|
|
return (mode, ifup_path, ifdown_path, ifquery_path,
|
|
FIREWALL_MODE_NATIVE)
|
|
mode, ifup_path, ifdown_path, ifquery_path, firewall_mode = contract
|
|
return (mode, ifup_path, ifdown_path, ifquery_path,
|
|
_firewall_mode(firewall_mode))
|
|
|
|
def apply(self, plan: VpcDataplanePlan) -> dict[str, Any]:
|
|
mode, ifup_path, ifdown_path, ifquery_path, firewall_mode = self._render_contract()
|
|
script = render_apply_script(
|
|
plan, deployment_mode=mode, firewall_mode=firewall_mode,
|
|
ifup_path=ifup_path, ifdown_path=ifdown_path,
|
|
ifquery_path=ifquery_path,
|
|
)
|
|
markers = _markers(self._run(script))
|
|
if markers.get("applied") != "yes":
|
|
raise VpcDataplaneError("宿主机未确认 VPC 应用", 502, "vpc_apply_unconfirmed")
|
|
return {"state": "applying", "readback": "pending", "rollback": "none", "owner": DATAPLANE_OWNER_MARKER}
|
|
|
|
def remove(self, plan: VpcDataplanePlan) -> dict[str, Any]:
|
|
"""Remove a panel-owned empty-host VPC and prove the absence.
|
|
|
|
The removal script is deliberately separate from ``apply``: it may
|
|
only run when the parsed plan has no local members and it validates
|
|
every persisted artifact/bridge owner before deleting anything.
|
|
A positive marker is required before the caller can advance a host
|
|
tombstone.
|
|
"""
|
|
mode, ifup_path, ifdown_path, ifquery_path, firewall_mode = self._render_contract()
|
|
markers = _markers(self._run(render_remove_script(
|
|
plan, deployment_mode=mode, firewall_mode=firewall_mode,
|
|
ifup_path=ifup_path,
|
|
ifdown_path=ifdown_path, ifquery_path=ifquery_path)))
|
|
if markers.get("removed") != "yes":
|
|
raise VpcDataplaneError(
|
|
"宿主机未确认 VPC 清理", 502, "vpc_remove_unconfirmed")
|
|
return {
|
|
"vpc_id": plan.vpc.vpc_id,
|
|
"host_id": plan.host_id,
|
|
"desired_revision": plan.desired_revision,
|
|
"observed_revision": plan.desired_revision,
|
|
"desired_digest": plan.desired_digest,
|
|
"observed_digest": plan.desired_digest,
|
|
"config_verified": True,
|
|
"config_observed_revision": plan.desired_revision,
|
|
"config_observed_digest": plan.desired_digest,
|
|
"state": "active",
|
|
"removed": True,
|
|
"bridge": None,
|
|
"bridge_owner": None,
|
|
"member_nics": [],
|
|
"routes": [],
|
|
"wg_allowed_ips": [],
|
|
"nft_table": None,
|
|
"acl_revision": plan.desired_revision,
|
|
"anti_spoof": False,
|
|
"rp_filter": None,
|
|
"proxy_arp": None,
|
|
"proxy_arp_pvlan": None,
|
|
"proxy_ips": [],
|
|
"readback": "confirmed",
|
|
"rollback": markers.get("rollback", "none"),
|
|
"reason_code": None,
|
|
}
|
|
|
|
def observe(
|
|
self, plan: VpcDataplanePlan, *,
|
|
xml_by_instance: Mapping[str, Any] | None = None,
|
|
) -> dict[str, Any]:
|
|
_mode, _ifup, _ifdown, _ifquery, firewall_mode = self._render_contract()
|
|
parsed = _markers(self._run(render_observe_script(
|
|
plan, firewall_mode=firewall_mode)))
|
|
expected_routes = {
|
|
(x["destination_cidr"], x["device"], str(x["table"]),
|
|
x.get("via"), str(x.get("metric", 100)))
|
|
for x in _routes(plan)
|
|
}
|
|
expected_proxy_ips = {
|
|
str(ipaddress.ip_network(x["destination_cidr"], strict=True).network_address)
|
|
for x in _routes(plan)
|
|
}
|
|
actual_proxy_ips = set(parsed.get("proxy_ips", []))
|
|
actual_routes = {
|
|
(x.get("destination_cidr"), x.get("device"), str(x.get("table")),
|
|
x.get("via"), str(x.get("metric", 100)))
|
|
for x in parsed.get("routes", [])
|
|
}
|
|
expected_wg_full = {
|
|
# The live `wg show` value includes the peer's overlay /32 as the
|
|
# first AllowedIP. Compare the complete cryptokey-routing set;
|
|
# project tenant entries only after that exact check.
|
|
p.host_id: (p.public_key, set(_allowed(plan, p).split(",")))
|
|
for p in plan.peers
|
|
}
|
|
actual_wg_full = {
|
|
x.get("peer_host_id"): (x.get("public_key"),
|
|
set(x.get("tenant_allowed_ips", [])))
|
|
for x in parsed.get("wg_allowed_ips", [])
|
|
}
|
|
# The live peer may carry /32s belonging to other VPCs sharing this
|
|
# Fabric. Verify the complete aggregate here, then expose only this
|
|
# VPC's tenant set to the PHP contract.
|
|
expected_tenant: dict[str, set[str]] = {
|
|
p.host_id: {m.private_ip + "/32" for m in plan.remote_members
|
|
if m.host_id == p.host_id}
|
|
for p in plan.peers
|
|
}
|
|
members_by_id = {m.member_id: m for m in plan.members}
|
|
for route in plan.vpc.routes:
|
|
target = (members_by_id.get(route.next_hop_member_id)
|
|
if route.next_hop_member_id is not None else None)
|
|
target_host = target.host_id if target is not None else route.next_hop_host_id
|
|
if target_host in expected_tenant:
|
|
expected_tenant[target_host].add(route.destination_cidr)
|
|
expected_wg = {
|
|
p.host_id: (p.public_key, expected_tenant.get(p.host_id, set()))
|
|
for p in plan.peers
|
|
}
|
|
actual_wg = {
|
|
host: (key, ips & expected_wg.get(host, (None, set()))[1])
|
|
for host, (key, ips) in actual_wg_full.items()
|
|
}
|
|
members = _member_nics(plan, xml_by_instance)
|
|
actual_route_state = {
|
|
(x.get("destination_cidr"), x.get("device"), str(x.get("table")),
|
|
x.get("via"), str(x.get("metric", 100)))
|
|
for x in parsed.get("routes", []) if x.get("state") == "owned"
|
|
}
|
|
expected_rules = {
|
|
(str(plan.resources.rule_priority), "-",
|
|
plan.vpc.cidr, str(plan.resources.route_table)),
|
|
(str(plan.resources.rule_priority + 1), FABRIC_INTERFACE,
|
|
plan.vpc.cidr, str(plan.resources.route_table)),
|
|
}
|
|
actual_rules = {
|
|
(str(x.get("priority")), x.get("iif"),
|
|
x.get("destination_cidr"), str(x.get("table")))
|
|
for x in parsed.get("rules", []) if x.get("state") == "owned"
|
|
}
|
|
expected_peer_states = {p.host_id: "owned" for p in plan.peers}
|
|
expected_bridge_nft = "yes" if plan.local_members else "none"
|
|
host_config_verified = (parsed.get("bridge") == plan.resources.bridge
|
|
and parsed.get("bridge_owner") == DATAPLANE_OWNER_MARKER
|
|
and (parsed.get("firewall_mode") == firewall_mode
|
|
or (firewall_mode == FIREWALL_MODE_NATIVE
|
|
and parsed.get("firewall_mode") is None))
|
|
and (parsed.get("firewall_state") == "owned"
|
|
or (firewall_mode == FIREWALL_MODE_NATIVE
|
|
and parsed.get("firewall_state") is None))
|
|
and (parsed.get("state_firewall_mode") == firewall_mode
|
|
or (firewall_mode == FIREWALL_MODE_NATIVE
|
|
and parsed.get("state_firewall_mode") in {None, ""}))
|
|
and parsed.get("persistence") == "yes"
|
|
and parsed.get("bridge_boundary") == "yes"
|
|
and parsed.get("proxy_arp") == "0"
|
|
and parsed.get("proxy_arp_pvlan") == "1"
|
|
and parsed.get("proxy_state") == "owned"
|
|
and actual_proxy_ips == expected_proxy_ips
|
|
and parsed.get("rp_filter") == "2"
|
|
and parsed.get("nft_table") == plan.resources.nft_table
|
|
and parsed.get("state_owner") == "owned"
|
|
and str(parsed.get("state_revision")) == str(plan.desired_revision)
|
|
and parsed.get("state_digest") == plan.desired_digest
|
|
and str(parsed.get("acl_revision")) == str(plan.desired_revision)
|
|
and parsed.get("acl_digest") == plan.desired_digest
|
|
and parsed.get("anti_spoof") == "yes"
|
|
and parsed.get("bridge_nft") == expected_bridge_nft
|
|
and parsed.get("route_extra") == "no"
|
|
and parsed.get("rule_extra") == "no"
|
|
and parsed.get("fabric_nft") == "yes"
|
|
and expected_routes == actual_routes
|
|
and actual_routes == actual_route_state
|
|
and expected_rules == actual_rules
|
|
and expected_wg_full == actual_wg_full
|
|
and (plan.fabric_peer_keys is None
|
|
or parsed.get("wg_peer_set") == "exact")
|
|
and expected_peer_states == parsed.get("wg_states", {})
|
|
and parsed.get("arp_nft") == ("yes" if plan.local_members else "none")
|
|
and all(x.get("config_verified") is True for x in members))
|
|
active = bool(
|
|
host_config_verified
|
|
and all(
|
|
x["xml_verified"]
|
|
and x.get("guest_network_verified") is True
|
|
for x in members
|
|
)
|
|
)
|
|
restart_required = any(
|
|
x.get("restart_required") is True for x in members)
|
|
guest_reason_code = next((
|
|
x.get("guest_reason_code") for x in members
|
|
if x.get("guest_network_verified") is not True
|
|
and isinstance(x.get("guest_reason_code"), str)
|
|
), None)
|
|
projected_wg = tuple(
|
|
dict(item, tenant_allowed_ips=sorted(
|
|
expected_wg[item.get("peer_host_id")][1]
|
|
& set(item.get("tenant_allowed_ips", []))))
|
|
for item in parsed.get("wg_allowed_ips", [])
|
|
if item.get("peer_host_id") in expected_wg
|
|
and expected_wg[item.get("peer_host_id")][1]
|
|
)
|
|
rp_filter = int(parsed["rp_filter"]) if str(parsed.get("rp_filter", "")).isdigit() else None
|
|
proxy_arp = int(parsed["proxy_arp"]) if str(parsed.get("proxy_arp", "")).isdigit() else None
|
|
proxy_arp_pvlan = (int(parsed["proxy_arp_pvlan"])
|
|
if str(parsed.get("proxy_arp_pvlan", "")).isdigit()
|
|
else None)
|
|
return VpcDataplaneObservation(
|
|
plan.vpc.vpc_id, plan.host_id, plan.desired_revision,
|
|
plan.desired_revision if active else 0, plan.desired_digest,
|
|
plan.desired_digest if active else None,
|
|
"active" if active else (
|
|
"applying" if restart_required or guest_reason_code else "degraded"),
|
|
plan.resources.bridge,
|
|
parsed.get("bridge_owner"), members, tuple(parsed.get("routes", [])),
|
|
projected_wg, parsed.get("nft_table"),
|
|
plan.desired_revision if active else None,
|
|
parsed.get("anti_spoof") == "yes", rp_filter, proxy_arp,
|
|
proxy_arp_pvlan,
|
|
tuple(sorted(actual_proxy_ips)),
|
|
host_config_verified,
|
|
plan.desired_revision if host_config_verified else 0,
|
|
plan.desired_digest if host_config_verified else None,
|
|
"confirmed" if active else (
|
|
"pending_restart" if restart_required
|
|
else "pending_guest" if guest_reason_code else "mismatch"),
|
|
"none", None if active else (
|
|
"vpc_private_guest_restart_required" if restart_required
|
|
else guest_reason_code
|
|
or "vpc_private_dataplane_readback_mismatch"),
|
|
).as_dict()
|
|
|
|
|
|
__all__ = ["DATAPLANE_OWNER_MARKER", "FABRIC_INTERFACE", "PRIVATE_INTERFACE", "FIREWALL_MODE_NATIVE", "VpcDataplaneError", "VpcFabricPeer", "VpcDataplaneResources", "VpcDataplanePlan", "VpcDataplaneObservation", "derive_private_mac", "private_guest_routes", "private_guest_network_generation", "resource_names", "parse_vpc_dataplane_plan", "render_vpc_nft_rules", "render_apply_script", "render_remove_script", "render_observe_script", "patch_private_interface_xml", "VpcPrivateHostAdapter"]
|