You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/engine/virsh_panel/vpc_dataplane.py

2975 lines
156 KiB
Python

"""Panel-owned VPC private L3 data-plane primitives.
This module is deliberately independent from :mod:`virsh_panel.fabric`.
Fabric owns the host overlay (``wg-fabric0``); this boundary owns only a
VPC's bridge, policy route table, tenant AllowedIPs and nftables table. The
pure builders are usable in unit tests and the host adapter runs one fixed
shell transaction through a pinned remote manager.
"""
from __future__ import annotations
import base64
import hashlib
import ipaddress
import re
import shlex
import xml.etree.ElementTree as ET
from dataclasses import dataclass
from typing import Any, Mapping, Sequence
from .network_roles import (
PRIVATE_INTERFACE_ALIAS,
PUBLIC_INTERFACE_ALIAS,
select_private_interface,
select_public_interface,
)
from .vpc import VpcError, VpcMember, VpcPlan, parse_vpc_plan
DATAPLANE_SCHEMA_VERSION = 1
DATAPLANE_OWNER_MARKER = "virt-cloud-vpc-dataplane-v1"
FABRIC_OWNER_MARKER = "virt-cloud-wg-fabric-v1"
FABRIC_INTERFACE = "wg-fabric0"
FABRIC_LOCK = "/run/virt-cloud/wg-fabric0.lock"
PRIVATE_INTERFACE = PRIVATE_INTERFACE_ALIAS
PANEL_NS = "urn:virsh-panel:1"
BRIDGE_PREFIX = "vcv-"
NFT_PREFIX = "virt_cloud_vpc_"
ROUTE_TABLE_MIN, ROUTE_TABLE_MAX = 10000, 19999
RULE_PRIORITY_MIN, RULE_PRIORITY_MAX = 20000, 29999
MAX_OUTPUT = 262144
_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,63}\Z")
_HOST = re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]{0,63}\Z")
_MAC = re.compile(r"[0-9a-fA-F]{2}(?::[0-9a-fA-F]{2}){5}\Z")
_KEY = re.compile(r"[A-Za-z0-9+/]{42}==\Z|[A-Za-z0-9+/]{43}=\Z")
_DIGEST = re.compile(r"[0-9a-f]{64}\Z")
class VpcDataplaneError(Exception):
"""Safe, typed error at the VPC host boundary."""
def __init__(self, message: str, status: int = 400,
error_code: str = "vpc_dataplane_invalid") -> None:
super().__init__(message)
self.status = status
self.error_code = error_code
@dataclass(frozen=True)
class VpcFabricPeer:
host_id: str
public_key: str
overlay_ip: str
# Tenant /32s from every panel VPC sharing this Fabric peer. WireGuard
# has one AllowedIPs list per peer, so a VPC must merge this typed set
# instead of replacing another active VPC's entries.
tenant_allowed_ips: tuple[str, ...] = ()
@property
def overlay_cidr(self) -> str:
return f"{self.overlay_ip}/32"
@dataclass(frozen=True)
class VpcDataplaneResources:
token: str
bridge: str
nft_table: str
route_table: int
rule_priority: int
owner_file: str
state_file: str
lock_file: str
transaction_dir: str
@dataclass(frozen=True)
class VpcDataplanePlan:
vpc: VpcPlan
host_id: str
members: tuple[VpcMember, ...]
peers: tuple[VpcFabricPeer, ...]
resources: VpcDataplaneResources
# The authenticated full Fabric peer set, excluding this host. It is
# separate from ``peers`` because one VPC can use only a subset of the
# peers present on a shared Fabric.
fabric_peer_keys: frozenset[str] | None = None
@property
def desired_revision(self) -> int:
return self.vpc.desired_revision
@property
def desired_digest(self) -> str:
return self.vpc.desired_digest
@property
def local_members(self) -> tuple[VpcMember, ...]:
return tuple(m for m in self.members if m.host_id == self.host_id)
@property
def remote_members(self) -> tuple[VpcMember, ...]:
return tuple(m for m in self.members if m.host_id != self.host_id)
@dataclass(frozen=True)
class VpcDataplaneObservation:
vpc_id: str
host_id: str
desired_revision: int
observed_revision: int
desired_digest: str
observed_digest: str | None
state: str
bridge: str
bridge_owner: str | None
member_nics: tuple[dict[str, Any], ...]
routes: tuple[dict[str, Any], ...]
wg_allowed_ips: tuple[dict[str, Any], ...]
nft_table: str | None
acl_revision: int | None
anti_spoof: bool
rp_filter: int | None
proxy_arp: int | None
proxy_arp_pvlan: int | None
proxy_ips: tuple[str, ...]
readback: str
rollback: str
reason_code: str | None = None
removed: bool = False
def as_dict(self) -> dict[str, Any]:
return {
"vpc_id": self.vpc_id, "host_id": self.host_id,
"desired_revision": self.desired_revision,
"observed_revision": self.observed_revision,
"desired_digest": self.desired_digest,
"observed_digest": self.observed_digest,
"state": self.state, "bridge": self.bridge,
"bridge_owner": self.bridge_owner,
"member_nics": [dict(x) for x in self.member_nics],
"routes": [dict(x) for x in self.routes],
"wg_allowed_ips": [dict(x) for x in self.wg_allowed_ips],
"nft_table": self.nft_table, "acl_revision": self.acl_revision,
"anti_spoof": self.anti_spoof, "rp_filter": self.rp_filter,
"proxy_arp": self.proxy_arp,
"proxy_arp_pvlan": self.proxy_arp_pvlan,
"proxy_ips": list(self.proxy_ips),
"readback": self.readback,
"rollback": self.rollback, "reason_code": self.reason_code,
"removed": self.removed,
}
def _id(value: Any, label: str = "标识") -> str:
if not isinstance(value, str) or _ID.fullmatch(value) is None:
raise VpcDataplaneError(f"{label}无效")
return value
def _host(value: Any) -> str:
if not isinstance(value, str) or _HOST.fullmatch(value) is None:
raise VpcDataplaneError("宿主机标识无效")
return value
def _key(value: Any) -> str:
if not isinstance(value, str) or _KEY.fullmatch(value) is None:
raise VpcDataplaneError("WireGuard 公钥无效")
try:
raw = base64.b64decode(value, validate=True)
except Exception as exc:
raise VpcDataplaneError("WireGuard 公钥无效") from exc
if len(raw) != 32 or not any(raw):
raise VpcDataplaneError("WireGuard 公钥无效")
return value
def _mac(value: Any, label: str = "私网 MAC") -> str:
if not isinstance(value, str) or _MAC.fullmatch(value) is None:
raise VpcDataplaneError(f"{label}无效")
octets = [int(x, 16) for x in value.split(":")]
if octets[0] & 1 or not any(octets):
raise VpcDataplaneError(f"{label}无效")
return value.lower()
def derive_private_mac(vpc_id: str, member_id: str) -> str:
"""Stable locally-administered unicast MAC for legacy plans without MAC."""
digest = hashlib.sha256(f"virt-cloud:vpc-private0:{vpc_id}:{member_id}".encode()).digest()
first = (digest[0] & 0xfc) | 0x02
return ":".join(f"{x:02x}" for x in bytes([first]) + digest[1:6])
def resource_names(vpc_id: str) -> VpcDataplaneResources:
vpc_id = _id(vpc_id, "VPC")
digest = hashlib.sha256(vpc_id.encode()).hexdigest()
token = digest[:11]
span = ROUTE_TABLE_MAX - ROUTE_TABLE_MIN + 1
# Each VPC owns two policy rules (private bridge and Fabric ingress).
# Allocate an aligned pair so ``base + 1`` can never be another VPC's
# base slot. The coordinator still checks collisions across records and
# the host transaction rejects a foreign rule already occupying either
# slot, but a single-slot hash would make those checks order-dependent.
rule_span = (RULE_PRIORITY_MAX - RULE_PRIORITY_MIN + 1) // 2
rule_base = RULE_PRIORITY_MIN + 2 * (int(digest[20:28], 16) % rule_span)
return VpcDataplaneResources(
token=token, bridge=BRIDGE_PREFIX + token,
nft_table=NFT_PREFIX + digest[:12],
route_table=ROUTE_TABLE_MIN + int(digest[12:20], 16) % span,
rule_priority=rule_base,
owner_file=f"/etc/virt-cloud/vpc-dataplane/{token}.owner",
state_file=f"/etc/virt-cloud/vpc-dataplane/{token}.state",
lock_file=f"/run/virt-cloud/vpc-{token}.lock",
transaction_dir=f"/etc/virt-cloud/.vpc-dataplane-{token}.txn",
)
def _expected_peer_tenant_ips(vpc: VpcPlan) -> dict[str, frozenset[str]]:
"""Return the exact /32 set a peer may route for this VPC.
The WireGuard peer list is a cryptokey-routing boundary. RFC1918 alone
is not an ownership check: an otherwise valid private /32 from another
VPC would create a cross-tenant route. The coordinator may pass a
trusted aggregate for a Fabric shared by several VPCs; the default here
covers a standalone plan and keeps direct parser callers fail-closed.
"""
members = {item.member_id: item for item in vpc.members}
allowed: dict[str, set[str]] = {host: set() for host in vpc.host_ids}
for item in vpc.members:
allowed.setdefault(item.host_id, set()).add(f"{item.private_ip}/32")
for route in vpc.routes:
target = (members.get(route.next_hop_member_id)
if route.next_hop_member_id is not None else None)
target_host = target.host_id if target is not None else route.next_hop_host_id
if target_host is not None:
allowed.setdefault(target_host, set()).add(route.destination_cidr)
return {host: frozenset(values) for host, values in allowed.items()}
def _parse_peers(
raw: Any,
host_ids: frozenset[str],
local: str,
expected_tenant_ips: Mapping[str, frozenset[str]] | None = None,
) -> tuple[VpcFabricPeer, ...]:
if raw is None:
raw = []
if not isinstance(raw, (list, tuple)) or len(raw) > 64:
raise VpcDataplaneError("Fabric peer 列表无效")
peers: list[VpcFabricPeer] = []
seen_hosts: set[str] = set()
seen_keys: set[str] = set()
seen_ips: set[str] = set()
seen_tenant_ips: set[str] = set()
for item in raw:
if not isinstance(item, Mapping) or set(item) - {"host_id", "public_key", "overlay_ip", "overlay_ip_cidr", "tenant_allowed_ips"}:
raise VpcDataplaneError("Fabric peer 字段无效")
host_id = _host(item.get("host_id"))
if host_id == local or host_id not in host_ids or host_id in seen_hosts:
raise VpcDataplaneError("Fabric peer 宿主机无效或重复")
public_key = _key(item.get("public_key"))
if public_key in seen_keys:
raise VpcDataplaneError("Fabric peer 公钥重复")
raw_ip = item.get("overlay_ip", item.get("overlay_ip_cidr"))
try:
interface = ipaddress.ip_interface(raw_ip)
except (ValueError, TypeError) as exc:
raise VpcDataplaneError("Fabric peer overlay 地址无效") from exc
fabric_overlay = ipaddress.IPv4Network("100.64.0.0/10")
if (not isinstance(interface, ipaddress.IPv4Interface)
or interface.network.prefixlen != 32
or interface.ip not in fabric_overlay
or interface.ip.is_loopback
or interface.ip.is_link_local):
raise VpcDataplaneError("Fabric peer overlay 地址无效")
overlay_ip = str(interface.ip)
if overlay_ip in seen_ips:
raise VpcDataplaneError("Fabric peer overlay 地址重复")
raw_tenant = item.get("tenant_allowed_ips", ())
if not isinstance(raw_tenant, (list, tuple)) or len(raw_tenant) > 4096:
raise VpcDataplaneError("Fabric peer tenant AllowedIPs 无效")
tenant: set[str] = set()
for raw_ip in raw_tenant:
if not isinstance(raw_ip, str) or not raw_ip.endswith("/32"):
raise VpcDataplaneError("Fabric peer tenant AllowedIPs 必须是 /32")
try:
network = ipaddress.ip_network(raw_ip, strict=True)
except ValueError as exc:
raise VpcDataplaneError("Fabric peer tenant AllowedIPs 无效") from exc
if network.prefixlen != 32:
raise VpcDataplaneError("Fabric peer tenant AllowedIPs 必须是 /32")
# Tenant routes are panel-private RFC1918 /32s. In particular,
# never accept a public prefix or a Fabric overlay address from
# an otherwise authenticated payload: that would turn a peer's
# cryptokey routing table into a host-wide egress route.
private_ranges = (
ipaddress.IPv4Network("10.0.0.0/8"),
ipaddress.IPv4Network("172.16.0.0/12"),
ipaddress.IPv4Network("192.168.0.0/16"),
)
if not any(network.subnet_of(item) for item in private_ranges):
raise VpcDataplaneError("Fabric peer tenant AllowedIPs 必须位于 RFC1918 私网")
tenant_ip = str(network)
if tenant_ip in seen_tenant_ips:
raise VpcDataplaneError("Fabric peer tenant AllowedIPs 重复", 409,
"vpc_tenant_allowed_ip_conflict")
seen_tenant_ips.add(tenant_ip)
tenant.add(tenant_ip)
if expected_tenant_ips is not None:
expected = set(expected_tenant_ips.get(host_id, frozenset()))
if tenant != expected:
raise VpcDataplaneError(
"Fabric peer tenant AllowedIPs 超出 VPC 所有权闭集", 409,
"vpc_tenant_allowed_ip_unowned")
seen_hosts.add(host_id); seen_keys.add(public_key); seen_ips.add(overlay_ip)
peers.append(VpcFabricPeer(host_id, public_key, overlay_ip,
tuple(sorted(tenant))))
return tuple(sorted(peers, key=lambda p: p.host_id))
def parse_vpc_dataplane_plan(
value: Mapping[str, Any], host_id: str, *,
fabric_peers: Sequence[Mapping[str, Any]] | None = None,
trusted_tenant_allowed_ips: Mapping[str, Sequence[str]] | None = None,
fabric_peer_keys: Sequence[str] | None = None,
) -> VpcDataplanePlan:
"""Parse existing VPC DTO plus an optional engine-only Fabric snapshot."""
if not isinstance(value, Mapping):
raise VpcDataplaneError("VPC 数据面计划必须是对象")
host_id = _host(host_id)
payload = dict(value)
embedded = payload.pop("fabric_peers", None)
if embedded is None:
embedded = payload.pop("peers", None)
else:
payload.pop("peers", None)
try:
vpc = parse_vpc_plan(payload)
except VpcError as exc:
raise VpcDataplaneError(str(exc), exc.status, exc.error_code) from exc
if host_id not in vpc.host_ids:
raise VpcDataplaneError("VPC 宿主机不在参与列表中", 409, "vpc_host_not_member")
members: list[VpcMember] = []
seen_macs: set[str] = set()
for member in vpc.members:
private_mac = _mac(member.private_mac) if member.private_mac else derive_private_mac(vpc.vpc_id, member.member_id)
if private_mac in seen_macs:
raise VpcDataplaneError("VPC 成员私网 MAC 重复", 409, "vpc_private_mac_conflict")
seen_macs.add(private_mac)
members.append(VpcMember(member.member_id, member.instance_uuid,
member.host_id, member.private_ip,
member.subnet_id, private_mac))
if trusted_tenant_allowed_ips is None:
expected_tenant_ips = _expected_peer_tenant_ips(vpc)
else:
if not isinstance(trusted_tenant_allowed_ips, Mapping):
raise VpcDataplaneError("Fabric peer tenant 所有权闭集无效")
expected_tenant_ips = {}
remote_hosts = vpc.host_ids - {host_id}
if set(trusted_tenant_allowed_ips) - remote_hosts:
raise VpcDataplaneError("Fabric peer tenant 所有权宿主机无效")
for peer_host, values in trusted_tenant_allowed_ips.items():
if not isinstance(peer_host, str) or peer_host not in remote_hosts:
raise VpcDataplaneError("Fabric peer tenant 所有权宿主机无效")
if not isinstance(values, (list, tuple, set, frozenset)):
raise VpcDataplaneError("Fabric peer tenant 所有权闭集无效")
canonical: set[str] = set()
for value in values:
if not isinstance(value, str):
raise VpcDataplaneError("Fabric peer tenant 所有权闭集无效")
try:
network = ipaddress.ip_network(value, strict=True)
except ValueError as exc:
raise VpcDataplaneError("Fabric peer tenant 所有权闭集无效") from exc
if (not isinstance(network, ipaddress.IPv4Network)
or network.prefixlen != 32
or not any(network.subnet_of(block) for block in (
ipaddress.IPv4Network("10.0.0.0/8"),
ipaddress.IPv4Network("172.16.0.0/12"),
ipaddress.IPv4Network("192.168.0.0/16")))):
raise VpcDataplaneError("Fabric peer tenant 所有权闭集无效")
canonical.add(str(network))
expected_tenant_ips[peer_host] = frozenset(canonical)
peers = _parse_peers(
embedded if fabric_peers is None else fabric_peers,
vpc.host_ids, host_id, expected_tenant_ips)
remote_hosts = {m.host_id for m in members if m.host_id != host_id}
if peers and remote_hosts - {p.host_id for p in peers}:
raise VpcDataplaneError("跨宿主机成员缺少 Fabric peer", 409, "vpc_fabric_peer_missing")
authoritative_keys: frozenset[str] | None = None
if fabric_peer_keys is not None:
if not isinstance(fabric_peer_keys, (list, tuple, set, frozenset)):
raise VpcDataplaneError("Fabric peer 公钥闭集无效", 409,
"vpc_fabric_peer_set_invalid")
if len(fabric_peer_keys) > 64:
raise VpcDataplaneError("Fabric peer 公钥闭集过大", 409,
"vpc_fabric_peer_set_invalid")
canonical_keys: set[str] = set()
for raw_key in fabric_peer_keys:
key = _key(raw_key)
if key in canonical_keys:
raise VpcDataplaneError("Fabric peer 公钥重复", 409,
"vpc_fabric_peer_set_invalid")
canonical_keys.add(key)
if not {peer.public_key for peer in peers}.issubset(canonical_keys):
raise VpcDataplaneError("VPC peer 不属于 Fabric 公钥闭集", 409,
"vpc_fabric_peer_set_invalid")
authoritative_keys = frozenset(canonical_keys)
return VpcDataplanePlan(
vpc, host_id, tuple(members), peers, resource_names(vpc.vpc_id),
authoritative_keys)
def _routes(plan: VpcDataplanePlan) -> tuple[dict[str, Any], ...]:
members = {m.member_id: m for m in plan.members}
result: list[dict[str, Any]] = []
seen: set[str] = set()
for member in plan.members:
destination = f"{member.private_ip}/32"
seen.add(destination)
result.append({"destination_cidr": destination,
"device": plan.resources.bridge if member.host_id == plan.host_id else FABRIC_INTERFACE,
"table": plan.resources.route_table,
"owner": DATAPLANE_OWNER_MARKER,
"metric": 100,
"via": None,
"member_id": member.member_id})
for route in plan.vpc.routes:
destination = route.destination_cidr
network = ipaddress.ip_network(destination)
if network.prefixlen != 32:
raise VpcDataplaneError(
"VPC 数据面只支持成员精确 /32 路由", 409,
"vpc_route_not_exact")
if any(ipaddress.IPv4Address(m.private_ip) in network
and destination != f"{m.private_ip}/32" for m in plan.members):
raise VpcDataplaneError("VPC route 不能覆盖成员精确 /32", 409, "vpc_route_member_overlap")
if destination in seen:
raise VpcDataplaneError("VPC route 与成员地址重复", 409, "vpc_route_conflict")
next_host = route.next_hop_host_id
if route.next_hop_member_id is not None:
target = members.get(route.next_hop_member_id)
if target is None:
raise VpcDataplaneError("VPC route next hop member 不存在", 409)
next_host = target.host_id
if next_host is None:
raise VpcDataplaneError("VPC route next hop 缺失")
target_members = [item for item in plan.members
if item.host_id == next_host]
if not target_members:
raise VpcDataplaneError("VPC route next hop 宿主机没有成员", 409,
"vpc_route_next_hop_missing")
# A host-only next hop is ambiguous when several VMs share that host.
# Require an explicit member in that case; using a bare bridge route
# would ARP for the service address and could leak traffic to an
# unrelated guest.
if route.next_hop_member_id is None and len(target_members) != 1:
raise VpcDataplaneError("VPC route 宿主机 next hop 不明确", 409,
"vpc_route_next_hop_ambiguous")
target = (members.get(route.next_hop_member_id)
if route.next_hop_member_id is not None else target_members[0])
if target is None:
raise VpcDataplaneError("VPC route next hop member 不存在", 409,
"vpc_route_next_hop_missing")
is_local = next_host == plan.host_id
if not is_local and next_host not in {peer.host_id for peer in plan.peers}:
raise VpcDataplaneError("VPC route next hop 缺少 Fabric peer", 409,
"vpc_fabric_peer_missing")
seen.add(destination)
result.append({"destination_cidr": destination,
"device": plan.resources.bridge if is_local else FABRIC_INTERFACE,
"table": plan.resources.route_table,
"owner": DATAPLANE_OWNER_MARKER,
# A local host must forward the service /32 through
# the selected VM. Across the overlay the destination
# itself is a WireGuard AllowedIP, so no `via` is used.
"via": target.private_ip if is_local else None,
"metric": route.metric,
"route_id": route.route_id})
return tuple(result)
def _q(value: Any) -> str:
return shlex.quote(str(value))
def _nft_proto(acl: Any) -> str:
if acl.protocol == "any":
return ""
if acl.protocol == "icmp":
return " ip protocol icmp"
suffix = f" ip protocol {acl.protocol}"
if acl.port_from is not None:
ports = str(acl.port_from) if acl.port_from == acl.port_to else f"{acl.port_from}-{acl.port_to}"
suffix += f" {acl.protocol} dport {ports}"
return suffix
def _nft_marker(plan: VpcDataplanePlan) -> str:
"""Return a bounded ownership marker safe for nft comment userdata.
libnftnl limits rule comments to 128 bytes. VPC/host identifiers are
intentionally allowed to be long, so embedding them and the full digest
in every rule would make otherwise valid plans fail at ``nft -c``. The
state file keeps the full identifiers; comments only need a collision
resistant, deterministic token for ownership/readback.
"""
seed = (f"{plan.vpc.vpc_id}:{plan.host_id}:{plan.desired_revision}:"
f"{plan.desired_digest}").encode()
token = hashlib.sha256(seed).hexdigest()[:16]
return f"{DATAPLANE_OWNER_MARKER}:{plan.resources.token}:{token}"
def render_vpc_nft_rules(plan: VpcDataplanePlan) -> str:
"""Render scoped VPC ACL/anti-spoof rules for routed and L2 paths.
The base chains use ``policy accept`` deliberately. A VPC table must not
become a host-wide firewall that drops public bridges or unrelated Fabric
traffic; only packets carrying this VPC's member addresses are subject to
its default deny. A separate bridge-family table covers same-host VM to
VM forwarding and ARP, which do not reliably traverse an inet forward
hook.
"""
r = plan.resources
marker = _nft_marker(plan)
dataplane_routes = _routes(plan)
member_cidrs = {f"{m.private_ip}/32" for m in plan.members}
local_member_cidrs = {f"{m.private_ip}/32" for m in plan.local_members}
# Custom VPC routes are exact /32s too. Keep source sets restricted to
# actual member addresses, while destination sets include routed service
# addresses; otherwise the isolation rules would silently drop a valid
# route before the ACL/default-deny decision.
destination_cidrs = member_cidrs | {
str(item["destination_cidr"]) for item in dataplane_routes
}
# A packet entering a local tap may be routed either to another local
# tap or out through wg-fabric0 to a remote member/service /32. The
# bridge-family unknown-destination guard therefore needs the complete
# VPC destination set; restricting it to local routes would drop every
# same-VPC cross-host flow before the inet routing policy can run.
local_destination_cidrs = set(destination_cidrs)
# Use explicit proxy-neighbour entries for the guest's routed /32 model.
# The generic kernel proxy responder stays disabled, so a nftables reload
# or a short ordering window cannot make the host answer for arbitrary
# addresses. proxy_arp_pvlan is enabled only for the same-bridge
# next-hop case and is still bounded by the exact neighbour set and ARP
# family output chain below.
proxy_ips = sorted(
str(ipaddress.ip_network(item["destination_cidr"], strict=True).network_address)
for item in dataplane_routes)
members = ", ".join(sorted(member_cidrs))
local_members = ", ".join(sorted(local_member_cidrs))
destinations = ", ".join(sorted(destination_cidrs))
local_destinations = ", ".join(sorted(local_destination_cidrs))
proxy_destinations = ", ".join(proxy_ips)
local_sources = ", ".join(
f"{m.private_mac} . {m.private_ip}" for m in plan.local_members)
def nft_set(name: str, type_name: str, elements: str, *, interval: bool = False) -> str:
flags = " flags interval;" if interval else ""
body = f" set {name} {{ type {type_name};{flags}"
if elements:
body += f" elements = {{ {elements} }};"
return body + " }"
lines = [
f"# {DATAPLANE_OWNER_MARKER}", f"# vpc-token={r.token}",
f"# vpc={plan.vpc.vpc_id} revision={plan.desired_revision} digest={plan.desired_digest}",
f"table inet {r.nft_table} {{",
nft_set("member_ips", "ipv4_addr", members, interval=True),
nft_set("local_member_ips", "ipv4_addr", local_members, interval=True),
nft_set("vpc_dest_ips", "ipv4_addr", destinations, interval=True),
nft_set("local_member_sources", "ether_addr . ipv4_addr", local_sources),
" chain forward { type filter hook forward priority -190; policy accept;",
f' iifname "{r.bridge}" ip saddr != @local_member_ips drop comment "{marker}:anti-spoof-local-ip"',
f' iifname "{r.bridge}" ip6 saddr ::/0 drop comment "{marker}:ipv6-drop-bridge"',
# A source address is not an ingress identity. A packet arriving on
# the public/management side must never be able to spoof a member /32
# and use the VPC policy route to reach a private destination. Close
# both egress legs by interface before the source-address checks.
# The whole VPC CIDR is guarded, including currently unallocated
# addresses. A missing /32 route must therefore fail closed instead
# of falling through to the host's public/default route.
f' oifname "{r.bridge}" ip daddr {plan.vpc.cidr} iifname != "{r.bridge}" iifname != "{FABRIC_INTERFACE}" drop comment "{marker}:anti-spoof-public-to-vpc-bridge"',
f' oifname "{FABRIC_INTERFACE}" ip daddr {plan.vpc.cidr} iifname != "{r.bridge}" iifname != "{FABRIC_INTERFACE}" drop comment "{marker}:anti-spoof-public-to-vpc-fabric"',
f' iifname "{r.bridge}" ip daddr {plan.vpc.cidr} oifname != "{r.bridge}" oifname != "{FABRIC_INTERFACE}" drop comment "{marker}:anti-spoof-bridge-vpc-to-foreign"',
f' iifname "{FABRIC_INTERFACE}" ip daddr {plan.vpc.cidr} oifname != "{r.bridge}" oifname != "{FABRIC_INTERFACE}" drop comment "{marker}:anti-spoof-fabric-vpc-to-foreign"',
f' oifname "{r.bridge}" ip daddr @vpc_dest_ips iifname != "{r.bridge}" iifname != "{FABRIC_INTERFACE}" drop comment "{marker}:anti-spoof-public-to-bridge"',
f' oifname "{FABRIC_INTERFACE}" ip daddr @vpc_dest_ips iifname != "{r.bridge}" iifname != "{FABRIC_INTERFACE}" drop comment "{marker}:anti-spoof-public-to-fabric"',
# A VPC destination must also leave through one of the two owned legs.
# If a policy route disappears or a foreign/default route wins, the
# packet must be dropped rather than sent to a public or management
# interface. This is deliberately independent of source-IP checks.
f' iifname "{r.bridge}" ip daddr @vpc_dest_ips oifname != "{r.bridge}" oifname != "{FABRIC_INTERFACE}" drop comment "{marker}:anti-spoof-bridge-to-foreign"',
f' iifname "{FABRIC_INTERFACE}" ip daddr @vpc_dest_ips oifname != "{r.bridge}" oifname != "{FABRIC_INTERFACE}" drop comment "{marker}:anti-spoof-fabric-to-foreign"',
f' oifname "{r.bridge}" ip daddr @vpc_dest_ips ip saddr != @member_ips drop comment "{marker}:anti-spoof-egress"',
f' iifname "{FABRIC_INTERFACE}" ip daddr @vpc_dest_ips ip saddr != @member_ips drop comment "{marker}:anti-spoof-wg-ingress"',
f' oifname "{FABRIC_INTERFACE}" ip saddr @member_ips ip daddr != @vpc_dest_ips drop comment "{marker}:anti-spoof-wg-egress"',
f' iifname "{FABRIC_INTERFACE}" ip daddr @vpc_dest_ips ip saddr != @member_ips drop comment "{marker}:anti-spoof-wg-source"',
f' iifname "{r.bridge}" ip saddr @local_member_ips ip daddr != @vpc_dest_ips drop comment "{marker}:member-egress-isolation"',
f' iifname "{FABRIC_INTERFACE}" ip saddr @member_ips ip daddr != @vpc_dest_ips drop comment "{marker}:member-fabric-isolation"',
]
# ACL priority is first-match order. Direction is relative to the VPC
# boundary: ingress is wg -> private bridge; egress is bridge -> wg.
# Same-host members have no wg leg, so their source/destination tuple is
# enforced on the bridge path as well.
for acl in plan.vpc.acls:
action = "accept" if acl.action == "allow" else "drop"
clause = (f" ip saddr {acl.source_cidr} ip daddr {acl.destination_cidr}"
f"{_nft_proto(acl)} {action} comment \"{marker}:acl:{acl.acl_id}\"")
if acl.direction == "ingress":
lines.append(f' iifname "{FABRIC_INTERFACE}" oifname "{r.bridge}"{clause}')
else:
lines.append(f' iifname "{r.bridge}" oifname "{FABRIC_INTERFACE}"{clause}')
lines.append(f' iifname "{r.bridge}" oifname "{r.bridge}"{clause}')
# Scope default-deny to member-to-member VPC traffic. The following
# interface-wide accepts preserve unrelated host/public forwarding.
lines += [
f' iifname "{FABRIC_INTERFACE}" oifname "{r.bridge}" ip saddr @member_ips ip daddr @vpc_dest_ips counter drop comment "{marker}:default-deny"',
f' iifname "{r.bridge}" oifname "{FABRIC_INTERFACE}" ip saddr @member_ips ip daddr @vpc_dest_ips counter drop comment "{marker}:default-deny"',
f' iifname "{r.bridge}" oifname "{r.bridge}" ip saddr @member_ips ip daddr @vpc_dest_ips counter drop comment "{marker}:default-deny"',
f' iifname "{FABRIC_INTERFACE}" oifname "{FABRIC_INTERFACE}" ip saddr @member_ips ip daddr @vpc_dest_ips counter drop comment "{marker}:wg-transit-deny"',
f' iifname "{FABRIC_INTERFACE}" oifname "{r.bridge}" ip6 saddr ::/0 drop comment "{marker}:ipv6-drop-fabric"',
f' iifname "{r.bridge}" accept comment "{marker}:scope-bridge-in"',
f' oifname "{r.bridge}" accept comment "{marker}:scope-bridge-out"',
f' iifname "{FABRIC_INTERFACE}" accept comment "{marker}:scope-fabric-in"',
f' oifname "{FABRIC_INTERFACE}" accept comment "{marker}:scope-fabric-out"',
" }",
" chain input { type filter hook input priority -190; policy accept;",
f' iifname "{r.bridge}" ip saddr @member_ips drop comment "{marker}:host-isolation"',
f' iifname "{FABRIC_INTERFACE}" ip daddr @vpc_dest_ips drop comment "{marker}:host-isolation-wg"',
f' iifname "{FABRIC_INTERFACE}" ip saddr @member_ips ip daddr != @vpc_dest_ips drop comment "{marker}:host-isolation-wg-source"',
f' iifname "{r.bridge}" ip6 saddr ::/0 drop comment "{marker}:ipv6-input-drop-bridge"',
f' iifname "{FABRIC_INTERFACE}" ip6 saddr ::/0 drop comment "{marker}:ipv6-input-drop-fabric"',
" }",
" chain output { type filter hook output priority -190; policy accept;",
f' oifname "{r.bridge}" ip daddr @vpc_dest_ips drop comment "{marker}:host-isolation-out"',
# Host-originated packets must not be able to bind a tenant /32 and
# bypass the private egress boundary. Keep this leg unconditional;
# the source address is not an ingress identity.
f' oifname "{FABRIC_INTERFACE}" ip daddr @vpc_dest_ips drop comment "{marker}:host-isolation-out-wg"',
f' oifname "{r.bridge}" ip daddr {plan.vpc.cidr} drop comment "{marker}:host-isolation-vpc-out"',
f' oifname "{FABRIC_INTERFACE}" ip daddr {plan.vpc.cidr} drop comment "{marker}:host-isolation-vpc-out-wg"',
f' oifname "{r.bridge}" ip6 daddr ::/0 drop comment "{marker}:ipv6-output-drop-bridge"',
" }", "}", "",
]
# Bridge-family rules are emitted only when this host has a local VM. An
# empty VPC still gets the routed table above, while no empty nft set is
# needed for the L2/ARP path.
if plan.local_members:
lines += [
f"table bridge {r.nft_table} {{",
nft_set("local_member_ips", "ipv4_addr", local_members, interval=True),
nft_set("local_dest_ips", "ipv4_addr", local_destinations, interval=True),
nft_set("local_member_sources", "ether_addr . ipv4_addr", local_sources),
" chain forward { type filter hook forward priority -190; policy accept;",
f' meta ibrname "{r.bridge}" ether type ip ether saddr . ip saddr != @local_member_sources drop comment "{marker}:bridge-anti-spoof-ingress"',
f' meta ibrname "{r.bridge}" ether type arp ether saddr != arp saddr ether drop comment "{marker}:bridge-anti-spoof-arp-mac"',
f' meta ibrname "{r.bridge}" ether type arp arp saddr ether . arp saddr ip != @local_member_sources drop comment "{marker}:bridge-anti-spoof-arp"',
f' meta ibrname "{r.bridge}" ether type ip ip saddr != @local_member_ips drop comment "{marker}:bridge-anti-spoof-ip"',
f' meta ibrname "{r.bridge}" ether type arp arp saddr ip != @local_member_ips drop comment "{marker}:bridge-anti-spoof-arp-ip"',
# This is an L3 /32 fabric. Do not let broadcast or multicast
# discovery turn the private bridge back into a shared L2 LAN.
f' meta ibrname "{r.bridge}" ip daddr 224.0.0.0/4 drop comment "{marker}:bridge-multicast-drop"',
f' meta ibrname "{r.bridge}" ip daddr 255.255.255.255 drop comment "{marker}:bridge-limited-broadcast-drop"',
f' meta ibrname "{r.bridge}" ip daddr {ipaddress.ip_network(plan.vpc.cidr).broadcast_address} drop comment "{marker}:bridge-vpc-broadcast-drop"',
f' meta ibrname "{r.bridge}" ip saddr @local_member_ips ip daddr != @local_dest_ips drop comment "{marker}:bridge-unknown-destination-drop"',
]
for acl in plan.vpc.acls:
action = "accept" if acl.action == "allow" else "drop"
clause = (f" ip saddr {acl.source_cidr} ip daddr {acl.destination_cidr}"
f"{_nft_proto(acl)} {action} comment \"{marker}:acl:{acl.acl_id}\"")
lines.append(f' meta ibrname "{r.bridge}" meta obrname "{r.bridge}"{clause}')
lines += [
f' meta ibrname "{r.bridge}" meta obrname "{r.bridge}" ip saddr @local_member_ips ip daddr @local_dest_ips counter drop comment "{marker}:bridge-default-deny"',
f' meta ibrname "{r.bridge}" ether type arp accept comment "{marker}:bridge-arp-accept"',
f' meta ibrname "{r.bridge}" ether type ip accept comment "{marker}:bridge-scope-accept"',
f' meta ibrname "{r.bridge}" counter drop comment "{marker}:bridge-non-ip-drop"',
" }",
" chain input { type filter hook input priority -190; policy accept;",
f' meta ibrname "{r.bridge}" ether type arp ether saddr != arp saddr ether drop comment "{marker}:bridge-anti-spoof-arp-input-mac"',
f' meta ibrname "{r.bridge}" ether type ip ether saddr . ip saddr != @local_member_sources drop comment "{marker}:bridge-anti-spoof-input-ip"',
f' meta ibrname "{r.bridge}" ether type arp arp saddr ether . arp saddr ip != @local_member_sources drop comment "{marker}:bridge-anti-spoof-input"',
f' meta ibrname "{r.bridge}" ether type ip6 drop comment "{marker}:bridge-ipv6-input-drop"',
f' meta ibrname "{r.bridge}" ether type arp accept comment "{marker}:bridge-arp-input-accept"',
# Proxy ARP makes remote /32 packets arrive at the bridge MAC and
# then enter the host routing path. Source/MAC checks above have
# already authenticated the guest, so IPv4 must be accepted here
# for the inet forward hook to route it to wg-fabric0.
f' meta ibrname "{r.bridge}" ether type ip accept comment "{marker}:bridge-ip-input-accept"',
f' meta ibrname "{r.bridge}" counter drop comment "{marker}:bridge-input-non-ip-drop"',
" }",
"}", "",
]
lines += [
f"table arp {r.nft_table} {{",
nft_set("proxy_ips", "ipv4_addr", proxy_destinations, interval=True),
" chain output { type filter hook output priority -190; policy accept;",
f' oifname "{r.bridge}" arp operation reply arp saddr ip != @proxy_ips drop comment "{marker}:arp-proxy-output"',
" }",
"}", "",
]
return "\n".join(lines)
def _here(path: str, content: str) -> str:
marker = "__VC_VPC_" + hashlib.sha256(content.encode()).hexdigest()[:16]
return f"cat > {_q(path)} <<'{marker}'\n{content}{'' if content.endswith(chr(10)) else chr(10)}{marker}\n"
def _state(plan: VpcDataplanePlan, routes: Sequence[Mapping[str, Any]]) -> str:
r = plan.resources
lines = [f"owner|{DATAPLANE_OWNER_MARKER}", f"vpc|{plan.vpc.vpc_id}",
f"cidr|{plan.vpc.cidr}", f"fabric|{plan.vpc.fabric_id}",
f"host|{plan.host_id}", f"revision|{plan.desired_revision}",
f"digest|{plan.desired_digest}", f"bridge|{r.bridge}",
f"table|{r.route_table}", f"priority|{r.rule_priority}"]
for route in routes:
lines.append("route|{destination}|{device}|{table}|{via}|{metric}".format(
destination=route["destination_cidr"], device=route["device"],
table=route["table"], via=route.get("via") or "-",
metric=route.get("metric", 100)))
for peer in plan.peers:
ips = sorted(set(
[m.private_ip + "/32" for m in plan.remote_members
if m.host_id == peer.host_id]
+ list(peer.tenant_allowed_ips)))
if ips:
lines.append(f"peer|{peer.host_id}|{peer.public_key}|{','.join(ips)}")
return "\n".join(lines) + "\n"
def _allowed(plan: VpcDataplanePlan, peer: VpcFabricPeer) -> str:
members = {m.member_id: m for m in plan.members}
ips = {m.private_ip + "/32" for m in plan.remote_members
if m.host_id == peer.host_id}
for route in plan.vpc.routes:
target = (members.get(route.next_hop_member_id)
if route.next_hop_member_id is not None else None)
target_host = target.host_id if target is not None else route.next_hop_host_id
if target_host == peer.host_id:
ips.add(route.destination_cidr)
return ",".join([peer.overlay_cidr] + sorted(ips | set(peer.tenant_allowed_ips)))
def render_apply_script(plan: VpcDataplanePlan) -> str:
"""Generate a shell transaction with panel ownership and rollback markers."""
routes = _routes(plan)
if plan.remote_members and not plan.peers:
raise VpcDataplaneError("跨宿主机 VPC 缺少 Fabric peer", 409, "vpc_fabric_peer_missing")
r = plan.resources
nft = render_vpc_nft_rules(plan)
state = _state(plan, routes)
route_specs = "\\n".join(
f"{x['destination_cidr']}|{x['device']}|{x['table']}" for x in routes)
peer_specs = "\\n".join(
f"{p.host_id}|{p.public_key}|{_allowed(plan,p)}" for p in plan.peers)
route_checks = "\\n".join(
f'''actual=$("$IP" -o -4 route show table {_q(x["table"])} {_q(x["destination_cidr"])} 2>/dev/null || true)
if printf '%s\\n' "$actual" | grep -Fq "dev {_q(x["device"])}" && printf '%s\\n' "$actual" | grep -Eq 'proto 186([[:space:]]|$)'; then
printf '%s\\n' "__VC_VPC_ROUTE__={x["destination_cidr"]}|{x["device"]}|{x["table"]}|owned"
else
printf '%s\\n' "__VC_VPC_ROUTE__={x["destination_cidr"]}|{x["device"]}|{x["table"]}|missing"
fi'''
for x in routes)
peer_checks = "\\n".join(
f'''actual=$("$WG" show "$WG_IFACE" allowed-ips 2>/dev/null | awk -v wanted={_q(p.public_key)} '$1==wanted {{print $2; exit}}' || true)
tenant={_q(','.join(_allowed(plan, p).split(',')[1:]))}
case ",$actual," in *",$tenant,"*) printf '%s\\n' "__VC_VPC_WG_ALLOWED__={p.host_id}|{p.public_key}|$tenant";; *) printf '%s\\n' "__VC_VPC_WG_ALLOWED__={p.host_id}|{p.public_key}|missing";; esac'''
for p in plan.peers)
script = f'''#!/bin/sh
set -eu
umask 077
PATH=/usr/sbin:/usr/bin:/sbin:/bin
export PATH
IP=/usr/sbin/ip; [ -x "$IP" ] || IP=/usr/bin/ip
WG=/usr/bin/wg; [ -x "$WG" ] || WG=/usr/sbin/wg
NFT=/usr/sbin/nft; [ -x "$NFT" ] || NFT=/usr/bin/nft
FLOCK=/usr/bin/flock; [ -x "$FLOCK" ] || FLOCK=/usr/bin/flock
BASE=/etc/virt-cloud/vpc-dataplane
BRIDGE={_q(r.bridge)}; TABLE={_q(r.nft_table)}; ROUTE_TABLE={r.route_table}; RULE_PRIORITY={r.rule_priority}
WG_IFACE={_q(FABRIC_INTERFACE)}; OWNER={_q(r.owner_file)}; STATE={_q(r.state_file)}; LOCK={_q(r.lock_file)}; TXN={_q(r.transaction_dir)}
COMMITTED=no; BRIDGE_CREATED=no; NFT_BEFORE=no; WG_BEFORE=no; ROLLBACK_FAILED=no
mkdir -p /run/virt-cloud "$BASE"
[ ! -L /run/virt-cloud ] && [ ! -L "$BASE" ] || exit 71
: >> "$LOCK"; chmod 600 "$LOCK"; exec 9>>"$LOCK"; "$FLOCK" -x 9
rollback() {{
status=$?; [ "$COMMITTED" = yes ] && return "$status"; trap - EXIT HUP INT TERM
oldIFS=$IFS; IFS=' '
for spec in { _q(route_specs) }; do destination=${{spec%%|*}}; device=${{spec#*|}}; "$IP" route del "$destination" table "$ROUTE_TABLE" dev "$device" >/dev/null 2>&1 || true; done
IFS=$oldIFS
"$IP" rule del pref "$RULE_PRIORITY" iif "$BRIDGE" to {_q(plan.vpc.cidr)} lookup "$ROUTE_TABLE" >/dev/null 2>&1 || true
if [ "$WG_BEFORE" = yes ] && [ -f "$TXN/wg.conf" ]; then "$WG" setconf "$WG_IFACE" "$TXN/wg.conf" >/dev/null 2>&1 || ROLLBACK_FAILED=yes; fi
if [ "$NFT_BEFORE" = yes ] && [ -f "$TXN/nft.before" ]; then "$NFT" delete table inet "$TABLE" >/dev/null 2>&1 || true; "$NFT" -f "$TXN/nft.before" >/dev/null 2>&1 || ROLLBACK_FAILED=yes; else "$NFT" delete table inet "$TABLE" >/dev/null 2>&1 || true; fi
if [ "$BRIDGE_CREATED" = yes ]; then "$IP" link del dev "$BRIDGE" >/dev/null 2>&1 || ROLLBACK_FAILED=yes; fi
[ "$ROLLBACK_FAILED" = yes ] && printf '%s\\n' '__VC_VPC_ROLLBACK__=failed' >&2 || printf '%s\\n' '__VC_VPC_ROLLBACK__=ok' >&2
exit "$status"
}}
trap rollback EXIT HUP INT TERM
[ -x "$IP" ] && [ -x "$WG" ] && [ -x "$NFT" ] && [ -x "$FLOCK" ] || exit 70
[ -f /etc/virt-cloud/wg-fabric0.owner ] && grep -Fqx { _q(FABRIC_OWNER_MARKER) } /etc/virt-cloud/wg-fabric0.owner || exit 71
"$IP" link show dev "$WG_IFACE" >/dev/null 2>&1 && "$WG" show "$WG_IFACE" >/dev/null 2>&1 || exit 72
[ ! -e "$TXN" ] && [ ! -L "$TXN" ] || exit 79
mkdir -p "$TXN"; printf '%s\\n' '{DATAPLANE_OWNER_MARKER}:{plan.vpc.vpc_id}:{plan.host_id}:{plan.desired_revision}:{plan.desired_digest}' > "$TXN/inflight"
if [ -f "$STATE" ]; then cp -p "$STATE" "$TXN/state.before"; grep -Fqx 'owner|{DATAPLANE_OWNER_MARKER}' "$STATE" || exit 71; fi
if [ -f "$OWNER" ]; then grep -Fqx { _q(DATAPLANE_OWNER_MARKER) } "$OWNER" || exit 71; else printf '%s\\n' { _q(DATAPLANE_OWNER_MARKER) } > "$OWNER"; chmod 600 "$OWNER"; fi
if "$IP" link show dev "$BRIDGE" >/dev/null 2>&1; then
"$IP" -d link show dev "$BRIDGE" | grep -Eq '(^|[[:space:]])bridge[[:space:]]+(forward_delay|stp_state|hello_time|ageing_time)([[:space:]]|$)' || exit 71
else
"$IP" link add name "$BRIDGE" type bridge; BRIDGE_CREATED=yes
fi
"$IP" link set dev "$BRIDGE" up
if "$WG" showconf "$WG_IFACE" > "$TXN/wg.conf" 2>/dev/null; then WG_BEFORE=yes; fi
if "$NFT" list table inet "$TABLE" > "$TXN/nft.before" 2>/dev/null; then NFT_BEFORE=yes; grep -Fq { _q(DATAPLANE_OWNER_MARKER) } "$TXN/nft.before" || exit 71; fi
'''+_here('$TXN/rules.nft', nft)+f'''"$NFT" delete table inet "$TABLE" >/dev/null 2>&1 || true
"$NFT" -f "$TXN/rules.nft"
oldIFS=$IFS; IFS=' '
for spec in { _q(route_specs) }; do destination=${{spec%%|*}}; device=${{spec#*|}}; "$IP" route replace "$destination" table "$ROUTE_TABLE" dev "$device" metric 100; done
IFS=$oldIFS
"$IP" rule replace pref "$RULE_PRIORITY" iif "$BRIDGE" to {_q(plan.vpc.cidr)} lookup "$ROUTE_TABLE"
oldIFS=$IFS; IFS=' '
for spec in { _q(peer_specs) }; do rest=${{spec#*|}}; key=${{rest%%|*}}; allowed=${{rest#*|}}; "$WG" show "$WG_IFACE" peers | grep -Fqx "$key" || exit 74; "$WG" set "$WG_IFACE" peer "$key" allowed-ips "$allowed"; done
IFS=$oldIFS
'''+_here('$STATE', state)+f'''chmod 600 "$STATE"
printf '%s\\n' '__VC_VPC_APPLIED__=yes'
printf '%s\\n' '__VC_VPC_OWNER__={DATAPLANE_OWNER_MARKER}'
printf '%s\\n' '__VC_VPC_REVISION__={plan.desired_revision}'
printf '%s\\n' '__VC_VPC_DIGEST__={plan.desired_digest}'
COMMITTED=yes; trap - EXIT HUP INT TERM; rm -rf "$TXN"; exit 0
'''
if len(script) > MAX_OUTPUT:
raise VpcDataplaneError("VPC 数据面脚本过大", 413, "vpc_script_too_large")
return script
def _render_apply_script_v2(
plan: VpcDataplanePlan,
*,
deployment_mode: str = "networkmanager_nmcli",
ifup_path: str = "/usr/sbin/ifup",
ifdown_path: str = "/usr/sbin/ifdown",
ifquery_path: str = "/usr/sbin/ifquery",
) -> str:
"""Generate the production transaction used by the host adapter.
The original control-plane module predates distribution-specific bridge
persistence. Keep its public builder for compatibility, but route real
applies through this stricter builder: every list is newline-delimited,
every path is fixed/quoted, and rollback only removes exact resources
tagged with the panel route protocol or ownership marker.
"""
if deployment_mode not in {"networkmanager_nmcli", "ifupdown"}:
raise VpcDataplaneError("宿主机网络管理模式不支持", 409,
"vpc_dataplane_network_manager_conflict")
allowed_ifup = {"/usr/sbin/ifup", "/usr/bin/ifup", "/sbin/ifup", "/bin/ifup"}
allowed_ifdown = {"/usr/sbin/ifdown", "/usr/bin/ifdown", "/sbin/ifdown", "/bin/ifdown"}
allowed_ifquery = {"/usr/sbin/ifquery", "/usr/bin/ifquery", "/sbin/ifquery", "/bin/ifquery"}
if ifup_path not in allowed_ifup or ifdown_path not in allowed_ifdown or ifquery_path not in allowed_ifquery:
raise VpcDataplaneError("ifupdown 工具路径不受支持", 409,
"vpc_dataplane_network_manager_conflict")
from .fabric import _VPC_STATE_LOADER
routes = _routes(plan)
if plan.remote_members and not plan.peers:
raise VpcDataplaneError("跨宿主机 VPC 缺少 Fabric peer", 409,
"vpc_fabric_peer_missing")
r = plan.resources
nft = render_vpc_nft_rules(plan)
include = f"/etc/nftables.d/virt-cloud-vpc-{r.token}.nft"
include_line = f'include "{include}"'
fragment = f"/etc/network/interfaces.d/virt-cloud-vpc-{r.token}"
sysctl_file = f"/etc/sysctl.d/99-virt-cloud-vpc-{r.token}.conf"
state_dir = f"/etc/virt-cloud/vpc-dataplane"
proxy_file = f"{state_dir}/{r.token}.proxy"
route_lines = "\n".join(
f"{item['destination_cidr']}|{item['device']}|{item['table']}|"
f"{item.get('via') or '-'}|{item.get('metric', 100)}"
for item in routes)
bridge_route_lines = "\n".join(
f"{item['destination_cidr']}|{item['device']}|{item['table']}|"
f"{item.get('via') or '-'}|{item.get('metric', 100)}|186"
for item in routes if item["device"] == r.bridge)
proxy_lines = "\n".join(
str(ipaddress.ip_network(item["destination_cidr"], strict=True).network_address)
for item in routes)
peer_lines = "\n".join(
f"{peer.host_id}|{peer.public_key}|{_allowed(plan, peer)}"
for peer in plan.peers)
state = _state(plan, routes)
# Local service /32s use a guest address as the next hop on an otherwise
# unnumbered private bridge. Linux therefore needs the explicit `onlink`
# assertion; without it iproute2 rejects the route as an invalid gateway.
# Use a quoted heredoc for data, while keeping the destination as a shell
# variable expanded by the surrounding script. This avoids the literal
# ``$TXN`` path bug in the former builder.
nft_marker = "__VC_VPC_NFT_" + hashlib.sha256(nft.encode()).hexdigest()[:12]
state_marker = "__VC_VPC_STATE_" + hashlib.sha256(state.encode()).hexdigest()[:12]
frag_marker = "__VC_VPC_IFUP_" + r.token
route_marker = "__VC_VPC_ROUTES_" + r.token
proxy_marker = "__VC_VPC_PROXY_" + r.token
peer_marker = "__VC_VPC_PEERS_" + r.token
state_loader = _VPC_STATE_LOADER.replace(
"__VC_FABRIC_HOST_ID__", plan.host_id).replace(
"__VC_FABRIC_ID__", plan.vpc.fabric_id).replace(
"__VC_FABRIC_NETWORK__", "100.64.0.0/10").replace(
"__VC_VPC_STATE_OBSERVE__", "no")
script = f'''#!/bin/sh
set -eu
umask 077
PATH=/usr/sbin:/usr/bin:/sbin:/bin
export PATH
IP=$(for p in /usr/sbin/ip /usr/bin/ip /sbin/ip /bin/ip; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
WG=$(for p in /usr/bin/wg /usr/sbin/wg /bin/wg /sbin/wg; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
NFT=$(for p in /usr/sbin/nft /usr/bin/nft /sbin/nft /bin/nft; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
FLOCK=$(for p in /usr/bin/flock /usr/sbin/flock /bin/flock /sbin/flock; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
STAT=$(for p in /usr/bin/stat /bin/stat /usr/sbin/stat /sbin/stat; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
CMP=$(for p in /usr/bin/cmp /bin/cmp /usr/sbin/cmp /sbin/cmp; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
SYSCTL=$(for p in /usr/sbin/sysctl /usr/bin/sysctl /sbin/sysctl /bin/sysctl; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
BRIDGE_CMD=$(for p in /usr/sbin/bridge /usr/bin/bridge /sbin/bridge /bin/bridge; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
BRIDGE={_q(r.bridge)}; TABLE={_q(r.nft_table)}; ROUTE_TABLE={r.route_table}; RULE_PRIORITY={r.rule_priority}; WG_RULE_PRIORITY=$((RULE_PRIORITY + 1))
WG_IFACE={_q(FABRIC_INTERFACE)}; OWNER_MARKER={_q(DATAPLANE_OWNER_MARKER)}; FABRIC_OWNER={_q(FABRIC_OWNER_MARKER)}
OWNER_FILE={_q(r.owner_file)}; STATE_FILE={_q(r.state_file)}; LOCK={_q(r.lock_file)}; TXN={_q(r.transaction_dir)}; PROXY_FILE={_q(proxy_file)}
NFT_FILE={_q(include)}; NFT_INCLUDE={_q(include_line)}; IF_FRAGMENT={_q(fragment)}; SYSCTL_FILE={_q(sysctl_file)}; MODE={_q(deployment_mode)}; NFT_MAIN=
IFUP={_q(ifup_path)}; IFDOWN={_q(ifdown_path)}; IFQUERY={_q(ifquery_path)}; NMCLI=
NM_PROFILE_MARKER={_q(DATAPLANE_OWNER_MARKER + " vpc=" + plan.vpc.vpc_id)}
COMMITTED=no; BRIDGE_CREATED=no; BRIDGE_WAS_UP=no; NM_CREATED=no; NM_WAS_ACTIVE=no; NM_ACTIVATION_ATTEMPTED=no; IF_CREATED=no; IF_ACTIVATION_ATTEMPTED=no; IF_FRAGMENT_CHANGED=no; SYSCTL_CHANGED=no; RP_FILTER_CHANGED=no; PROXY_ARP_CHANGED=no; PROXY_ARP_PVLAN_CHANGED=no; NFT_FILE_CHANGED=no; NFT_MAIN_CHANGED=no; NFT_FILE_BEFORE=no; NFT_MAIN_BEFORE=no; BRIDGE_NFT_BEFORE=no; ARP_NFT_BEFORE=no; FABRIC_NFT_BEFORE=no; PROXY_ENTRIES_CHANGED=no; PROXY_FILE_CHANGED=no; ROLLBACK_FAILED=no
[ -n "$IP" ] && [ -n "$WG" ] && [ -n "$NFT" ] && [ -n "$FLOCK" ] && [ -n "$STAT" ] && [ -n "$CMP" ] && [ -n "$SYSCTL" ] && [ -n "$BRIDGE_CMD" ] || exit 70
if [ "$MODE" = networkmanager_nmcli ]; then
NMCLI=$(for p in /usr/bin/nmcli /usr/sbin/nmcli /bin/nmcli /sbin/nmcli; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
[ -n "$NMCLI" ] || exit 70
fi
if [ -L /run/virt-cloud ] || {{ [ -e /run/virt-cloud ] && [ ! -d /run/virt-cloud ]; }}; then exit 71; fi
if [ -L /etc/virt-cloud ] || {{ [ -e /etc/virt-cloud ] && [ ! -d /etc/virt-cloud ]; }}; then exit 71; fi
if [ -L {state_dir} ] || {{ [ -e {state_dir} ] && [ ! -d {state_dir} ]; }}; then exit 71; fi
mkdir -p /run/virt-cloud {state_dir}
chmod 700 /run/virt-cloud {state_dir}
[ ! -L {state_dir} ] && [ "$($STAT -c '%u:%a' {state_dir} 2>/dev/null)" = 0:700 ] || exit 71
[ ! -L "$LOCK" ] && {{ [ ! -e "$LOCK" ] || [ -f "$LOCK" ]; }} || exit 71
: >> "$LOCK"; chmod 600 "$LOCK"
exec 8>>{_q(FABRIC_LOCK)}; "$FLOCK" -x 8
exec 9>>"$LOCK"; "$FLOCK" -x 9
[ -f /etc/virt-cloud/wg-fabric0.owner ] && [ "$(cat /etc/virt-cloud/wg-fabric0.owner)" = "$FABRIC_OWNER" ] || exit 71
"$WG" show "$WG_IFACE" >/dev/null 2>&1 || exit 71
NFT_MAIN_COUNT=0
for nft_candidate in /etc/nftables.conf /etc/sysconfig/nftables.conf; do
if [ -f "$nft_candidate" ] && [ ! -L "$nft_candidate" ] && [ "$("$STAT" -c '%u' "$nft_candidate" 2>/dev/null)" = 0 ]; then
case "$("$STAT" -c '%a' "$nft_candidate" 2>/dev/null)" in
[0-7][0145][0145]) : ;;
*) continue ;;
esac
NFT_MAIN="$nft_candidate"; NFT_MAIN_COUNT=$((NFT_MAIN_COUNT + 1))
fi
done
[ "$NFT_MAIN_COUNT" -eq 1 ] || exit 71
if [ -L /etc/nftables.d ] || {{ [ -e /etc/nftables.d ] && [ ! -d /etc/nftables.d ]; }}; then exit 71; fi
if [ -e /etc/nftables.d ]; then
[ "$("$STAT" -c '%u:%a' /etc/nftables.d 2>/dev/null)" = 0:755 ] || exit 71
else
mkdir /etc/nftables.d; chmod 755 /etc/nftables.d
fi
if [ -L /etc/sysctl.d ] || {{ [ -e /etc/sysctl.d ] && [ ! -d /etc/sysctl.d ]; }}; then exit 71; fi
if [ -e /etc/sysctl.d ]; then
[ "$("$STAT" -c '%u:%a' /etc/sysctl.d 2>/dev/null)" = 0:755 ] || exit 71
else
mkdir /etc/sysctl.d; chmod 755 /etc/sysctl.d
fi
if [ -e "$SYSCTL_FILE" ] || [ -L "$SYSCTL_FILE" ]; then
[ -f "$SYSCTL_FILE" ] && [ ! -L "$SYSCTL_FILE" ] || exit 71
[ "$("$STAT" -c '%u' "$SYSCTL_FILE" 2>/dev/null)" = 0 ] || exit 71
grep -Fqx '# virt-cloud-vpc-dataplane-v1' "$SYSCTL_FILE" || exit 71
fi
if [ -e "$PROXY_FILE" ] || [ -L "$PROXY_FILE" ]; then
[ -f "$PROXY_FILE" ] && [ ! -L "$PROXY_FILE" ] || exit 71
[ "$($STAT -c '%u:%a' "$PROXY_FILE" 2>/dev/null)" = 0:600 ] || exit 71
sed -n '1p' "$PROXY_FILE" | grep -Fqx '# virt-cloud-vpc-proxy-v1' || exit 71
awk 'NR > 1 && $0 !~ /^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$/ {{ exit 1 }}' "$PROXY_FILE" || exit 71
fi
if [ -e "$NFT_FILE" ] || [ -L "$NFT_FILE" ]; then
[ -f "$NFT_FILE" ] && [ ! -L "$NFT_FILE" ] || exit 71
[ "$("$STAT" -c '%u' "$NFT_FILE" 2>/dev/null)" = 0 ] || exit 71
grep -Fqx '# virt-cloud-vpc-dataplane-v1' "$NFT_FILE" || exit 71
fi
if [ -L "$OWNER_FILE" ] || {{ [ -e "$OWNER_FILE" ] && [ ! -f "$OWNER_FILE" ]; }}; then exit 71; fi
if [ -L "$STATE_FILE" ] || {{ [ -e "$STATE_FILE" ] && [ ! -f "$STATE_FILE" ]; }}; then exit 71; fi
if [ -f "$STATE_FILE" ]; then
[ "$($STAT -c '%u:%a' "$STATE_FILE" 2>/dev/null)" = 0:600 ] || exit 71
grep -Fqx 'owner|{DATAPLANE_OWNER_MARKER}' "$STATE_FILE" || exit 71
grep -Fqx 'vpc|{plan.vpc.vpc_id}' "$STATE_FILE" || exit 71
grep -Fqx 'fabric|{plan.vpc.fabric_id}' "$STATE_FILE" || exit 71
grep -Fqx 'host|{plan.host_id}' "$STATE_FILE" || exit 71
grep -Fqx 'bridge|{r.bridge}' "$STATE_FILE" || exit 71
grep -Fqx 'table|{r.route_table}' "$STATE_FILE" || exit 71
state_revision=$(awk -F'|' '$1=="revision" {{print $2; exit}}' "$STATE_FILE")
state_digest=$(awk -F'|' '$1=="digest" {{print $2; exit}}' "$STATE_FILE")
case "$state_revision" in ''|*[!0-9]*) exit 79;; esac
printf '%s\n' "$state_digest" | grep -Eq '^[0-9a-f]{{64}}$' || exit 79
[ "$state_revision" -le {plan.desired_revision} ] || exit 79
if [ "$state_revision" = {plan.desired_revision} ] && [ "$state_digest" != { _q(plan.desired_digest) } ]; then exit 79; fi
fi
if [ -e "$TXN" ] || [ -L "$TXN" ]; then
# A process can die after writing the committed marker but before removing
# its transaction directory. That state is safe to reap after validating
# ownership; preparing/ready/rollback_failed states remain blocked so a
# later reconcile cannot guess whether runtime resources were partially
# applied.
[ -d "$TXN" ] && [ ! -L "$TXN" ] || exit 79
[ "$("$STAT" -c '%u:%a' "$TXN" 2>/dev/null)" = 0:700 ] || exit 79
[ -f "$TXN/inflight" ] && [ ! -L "$TXN/inflight" ] || exit 79
grep -Fqx '{DATAPLANE_OWNER_MARKER}:{plan.vpc.vpc_id}:{plan.host_id}:{plan.desired_revision}:{plan.desired_digest}' "$TXN/inflight" || exit 79
[ -f "$TXN/state" ] && [ "$(cat "$TXN/state" 2>/dev/null)" = committed ] || exit 79
rm -rf "$TXN" || exit 79
fi
mkdir -m 700 "$TXN"
printf '%s\\n' preparing > "$TXN/state"; chmod 600 "$TXN/state"
rollback() {{
status=$?; [ "$COMMITTED" = yes ] && exit "$status"; trap - EXIT HUP INT TERM
set +e
while IFS='|' read -r destination device table via metric; do
[ -n "$destination" ] || continue
actual=$("$IP" -o -4 route show table "$table" "$destination" 2>/dev/null || true)
[ -n "$actual" ] || continue
actual_count=$(printf '%s\n' "$actual" | awk 'NF {{count++}} END {{print count + 0}}')
actual_device=$(printf '%s\n' "$actual" | awk '{{for (i=1;i<=NF;i++) if ($i=="dev" && i<NF) {{print $(i+1); exit}}}}')
actual_via=$(printf '%s\n' "$actual" | awk '{{for (i=1;i<=NF;i++) if ($i=="via" && i<NF) {{print $(i+1); exit}}}}')
actual_metric=$(printf '%s\n' "$actual" | awk '{{for (i=1;i<=NF;i++) if ($i=="metric" && i<NF) {{print $(i+1); exit}}}}')
actual_proto=$(printf '%s\n' "$actual" | awk '{{for (i=1;i<=NF;i++) if ($i=="proto" && i<NF) {{print $(i+1); exit}}}}')
[ -n "$actual_metric" ] || actual_metric=100
if [ "$actual_count" != 1 ] || [ "$actual_device" != "$device" ] \
|| [ "${{actual_via:--}}" != "${{via:--}}" ] \
|| [ "$actual_metric" != "$metric" ] || [ "$actual_proto" != 186 ]; then
# A route with the same destination may have been replaced by an
# operator or another VPC while this transaction was failing. Never
# delete that route merely because its destination matches ours.
ROLLBACK_FAILED=yes
continue
fi
if [ "$via" = - ]; then
"$IP" route del "$destination" table "$table" dev "$device" proto 186 metric "$metric" 2>/dev/null || ROLLBACK_FAILED=yes
else
"$IP" route del "$destination" table "$table" via "$via" dev "$device" proto 186 metric "$metric" 2>/dev/null || ROLLBACK_FAILED=yes
fi
done <<'{route_marker}'
{route_lines}
{route_marker}
if [ "$ROLLBACK_FAILED" = no ] && [ -f "$TXN/routes.before" ]; then
while IFS='|' read -r old_destination old_device old_table old_via old_metric; do
[ -n "$old_destination" ] || continue
if [ "$old_via" = - ]; then
"$IP" route replace "$old_destination" table "$old_table" dev "$old_device" proto 186 metric "$old_metric" 2>/dev/null || ROLLBACK_FAILED=yes
else
"$IP" route replace "$old_destination" table "$old_table" via "$old_via" dev "$old_device" proto 186 metric "$old_metric" 2>/dev/null || ROLLBACK_FAILED=yes
fi
done < "$TXN/routes.before"
fi
if [ "$ROLLBACK_FAILED" = no ]; then
for rule_spec in "$RULE_PRIORITY|$ROUTE_TABLE|-" "$WG_RULE_PRIORITY|$ROUTE_TABLE|$WG_IFACE"; do
rule_pref=${{rule_spec%%|*}}; rule_rest=${{rule_spec#*|}}; rule_table=${{rule_rest%%|*}}; rule_iif=${{rule_rest#*|}}
actual_rule=$("$IP" -o rule show pref "$rule_pref" 2>/dev/null || true)
[ -n "$actual_rule" ] || continue
actual_rule_count=$(printf '%s\n' "$actual_rule" | awk 'NF {{count++}} END {{print count + 0}}')
if [ "$actual_rule_count" != 1 ] || ! printf '%s\n' "$actual_rule" | grep -Fq "to {_q(plan.vpc.cidr)} lookup $rule_table"; then
ROLLBACK_FAILED=yes
continue
fi
if [ "$rule_iif" = - ]; then
if printf '%s\n' "$actual_rule" | grep -Eq '(^|[[:space:]])iif[[:space:]]'; then
ROLLBACK_FAILED=yes
continue
fi
elif ! printf '%s\n' "$actual_rule" | grep -Fq "iif $rule_iif"; then
ROLLBACK_FAILED=yes
continue
fi
"$IP" rule del pref "$rule_pref" 2>/dev/null || ROLLBACK_FAILED=yes
remaining_rule=$("$IP" -o rule show pref "$rule_pref" 2>/dev/null || true)
[ -z "$remaining_rule" ] || ROLLBACK_FAILED=yes
done
fi
if [ "$ROLLBACK_FAILED" = no ] && [ -f "$TXN/rules.before" ]; then
while IFS='|' read -r old_pref old_iif old_to old_lookup; do
[ -n "$old_pref" ] || continue
if [ "$old_iif" = - ]; then
"$IP" rule add pref "$old_pref" to "$old_to" lookup "$old_lookup" 2>/dev/null || ROLLBACK_FAILED=yes
else
"$IP" rule add pref "$old_pref" iif "$old_iif" to "$old_to" lookup "$old_lookup" 2>/dev/null || ROLLBACK_FAILED=yes
fi
done < "$TXN/rules.before"
fi
if [ -f "$TXN/wg.conf" ]; then "$WG" setconf "$WG_IFACE" "$TXN/wg.conf" >/dev/null 2>&1 || ROLLBACK_FAILED=yes; fi
if [ "$PROXY_ENTRIES_CHANGED" = yes ]; then
"$IP" neigh show proxy dev "$BRIDGE" 2>/dev/null | awk '$1 ~ /^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$/ {{print $1}}' | LC_ALL=C sort -u > "$TXN/proxy.rollback.current"
cat <<'{proxy_marker}' | LC_ALL=C sort -u > "$TXN/proxy.rollback.desired"
{proxy_lines}
{proxy_marker}
if "$CMP" -s "$TXN/proxy.rollback.current" "$TXN/proxy.rollback.desired"; then
while IFS= read -r rollback_proxy; do
[ -n "$rollback_proxy" ] || continue
"$IP" neigh del proxy "$rollback_proxy" dev "$BRIDGE" >/dev/null 2>&1 || ROLLBACK_FAILED=yes
done < "$TXN/proxy.rollback.desired"
while IFS= read -r rollback_proxy; do
[ -n "$rollback_proxy" ] || continue
"$IP" neigh replace proxy "$rollback_proxy" dev "$BRIDGE" >/dev/null 2>&1 || ROLLBACK_FAILED=yes
done < "$TXN/proxy.before"
else
# Do not delete an entry added concurrently by an operator or another
# controller. Leave the transaction recoverable and report rollback
# failure rather than risking an unrelated ARP outage.
ROLLBACK_FAILED=yes
fi
fi
if [ "$PROXY_FILE_CHANGED" = yes ]; then
if [ -f "$TXN/proxy_file.before" ]; then
cp -p "$TXN/proxy_file.before" "$PROXY_FILE" || ROLLBACK_FAILED=yes
else
rm -f "$PROXY_FILE" || ROLLBACK_FAILED=yes
fi
fi
# Restore runtime sysctls while the new ARP guard is still installed; this
# avoids a rollback window in which proxy_arp is enabled without its exact
# output filter.
if [ "$PROXY_ARP_CHANGED" = yes ] && [ -e "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp" ]; then
if [ -f "$TXN/proxy_arp.before" ]; then
cat "$TXN/proxy_arp.before" > "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp" || ROLLBACK_FAILED=yes
else
printf '%s\\n' 0 > "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp" || ROLLBACK_FAILED=yes
fi
fi
if [ "$PROXY_ARP_PVLAN_CHANGED" = yes ] && [ -e "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp_pvlan" ]; then
if [ -f "$TXN/proxy_arp_pvlan.before" ]; then
cat "$TXN/proxy_arp_pvlan.before" > "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp_pvlan" || ROLLBACK_FAILED=yes
else
printf '%s\n' 0 > "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp_pvlan" || ROLLBACK_FAILED=yes
fi
fi
if [ "$RP_FILTER_CHANGED" = yes ] && [ -e "/proc/sys/net/ipv4/conf/$BRIDGE/rp_filter" ]; then
if [ -f "$TXN/rp_filter.before" ]; then
cat "$TXN/rp_filter.before" > "/proc/sys/net/ipv4/conf/$BRIDGE/rp_filter" || ROLLBACK_FAILED=yes
fi
fi
NFT_ROLLBACK="$TXN/rollback.nft"
: > "$NFT_ROLLBACK"
if "$NFT" list table inet "$TABLE" >/dev/null 2>&1; then printf 'delete table inet %s\\n' "$TABLE" >> "$NFT_ROLLBACK"; fi
if "$NFT" list table bridge "$TABLE" >/dev/null 2>&1; then printf 'delete table bridge %s\\n' "$TABLE" >> "$NFT_ROLLBACK"; fi
if "$NFT" list table arp "$TABLE" >/dev/null 2>&1; then printf 'delete table arp %s\\n' "$TABLE" >> "$NFT_ROLLBACK"; fi
if [ -f "$TXN/nft.before" ]; then cat "$TXN/nft.before" >> "$NFT_ROLLBACK"; fi
if [ -f "$TXN/nft.bridge.before" ]; then cat "$TXN/nft.bridge.before" >> "$NFT_ROLLBACK"; fi
if [ -f "$TXN/nft.arp.before" ]; then cat "$TXN/nft.arp.before" >> "$NFT_ROLLBACK"; fi
if [ -s "$NFT_ROLLBACK" ]; then
"$NFT" -c -f "$NFT_ROLLBACK" >/dev/null 2>&1 || ROLLBACK_FAILED=yes
"$NFT" -f "$NFT_ROLLBACK" >/dev/null 2>&1 || ROLLBACK_FAILED=yes
fi
if [ -f "$TXN/fabric.nft.before" ]; then
"$NFT" delete table inet virt_cloud_wg_fabric >/dev/null 2>&1 || true
"$NFT" -f "$TXN/fabric.nft.before" >/dev/null 2>&1 || ROLLBACK_FAILED=yes
fi
if [ "$NFT_FILE_CHANGED" = yes ]; then
if [ -f "$TXN/nft_file.before" ]; then cp -p "$TXN/nft_file.before" "$NFT_FILE" || ROLLBACK_FAILED=yes; else rm -f "$NFT_FILE" || ROLLBACK_FAILED=yes; fi
fi
if [ "$NFT_MAIN_CHANGED" = yes ] && [ -f "$TXN/nft_main.before" ]; then
cp -p "$TXN/nft_main.before" "$NFT_MAIN" || ROLLBACK_FAILED=yes
fi
if [ "$MODE" = networkmanager_nmcli ] && [ -n "$NMCLI" ]; then
if [ "$NM_ACTIVATION_ATTEMPTED" = yes ] && [ "$NM_WAS_ACTIVE" = no ]; then
"$NMCLI" connection down "$BRIDGE" >/dev/null 2>&1 || ROLLBACK_FAILED=yes
if "$NMCLI" -t --escape no -f NAME connection show --active 2>/dev/null | grep -Fqx "$BRIDGE"; then ROLLBACK_FAILED=yes; fi
fi
[ "$NM_CREATED" = yes ] && "$NMCLI" connection delete "$BRIDGE" >/dev/null 2>&1 || true
elif [ "$MODE" = ifupdown ]; then
if [ "$IF_ACTIVATION_ATTEMPTED" = yes ] && [ "$BRIDGE_WAS_UP" = no ]; then
"$IFDOWN" --force "$BRIDGE" >/dev/null 2>&1 || ROLLBACK_FAILED=yes
fi
[ "$IF_CREATED" = yes ] && "$IFDOWN" --force "$BRIDGE" >/dev/null 2>&1 || true
fi
if [ "$IF_FRAGMENT_CHANGED" = yes ]; then
if [ -f "$TXN/if_fragment.before" ]; then cp -p "$TXN/if_fragment.before" "$IF_FRAGMENT" || ROLLBACK_FAILED=yes; else rm -f "$IF_FRAGMENT" || ROLLBACK_FAILED=yes; fi
fi
if [ "$SYSCTL_CHANGED" = yes ]; then
if [ -f "$TXN/sysctl.before" ]; then
cp -p "$TXN/sysctl.before" "$SYSCTL_FILE" || ROLLBACK_FAILED=yes
else
rm -f "$SYSCTL_FILE" || ROLLBACK_FAILED=yes
fi
fi
if [ "$BRIDGE_CREATED" = yes ]; then
# Never delete a bridge that acquired a libvirt tap or an operator port.
ports=$($BRIDGE_CMD link show master "$BRIDGE" 2>/dev/null)
if [ "$?" -ne 0 ]; then
# An inability to inspect ports is a rollback failure. Treating an
# empty error result as "no ports" could delete a bridge still carrying
# a libvirt tap and strand the host network.
ROLLBACK_FAILED=yes
elif [ -z "$ports" ]; then
"$IP" link del dev "$BRIDGE" type bridge 2>/dev/null || ROLLBACK_FAILED=yes
fi
elif [ "$BRIDGE_WAS_UP" = no ] && "$IP" link show dev "$BRIDGE" >/dev/null 2>&1; then
# Applying a revision may have activated an existing panel bridge through
# NetworkManager/ifupdown. Restore its prior DOWN state only when no tap
# or operator port appeared while the transaction was running.
ports=$($BRIDGE_CMD link show master "$BRIDGE" 2>/dev/null)
if [ "$?" -ne 0 ] || [ -n "$ports" ]; then
ROLLBACK_FAILED=yes
else
"$IP" link set dev "$BRIDGE" down 2>/dev/null || ROLLBACK_FAILED=yes
"$IP" link show dev "$BRIDGE" 2>/dev/null | grep -q 'state UP' && ROLLBACK_FAILED=yes
fi
fi
if [ -f "$TXN/owner.before" ]; then cp -p "$TXN/owner.before" "$OWNER_FILE" || ROLLBACK_FAILED=yes; else rm -f "$OWNER_FILE"; fi
if [ -f "$TXN/state.before" ]; then cp -p "$TXN/state.before" "$STATE_FILE" || ROLLBACK_FAILED=yes; else rm -f "$STATE_FILE"; fi
if [ "$ROLLBACK_FAILED" = yes ]; then printf '%s\\n' rollback_failed > "$TXN/state"; chmod 600 "$TXN/state"; exit 79; fi
rm -rf "$TXN"; exit "$status"
}}
trap rollback EXIT HUP INT TERM
if [ -f "$OWNER_FILE" ]; then
[ ! -L "$OWNER_FILE" ] && grep -Fqx "$OWNER_MARKER" "$OWNER_FILE" || exit 71
cp -p "$OWNER_FILE" "$TXN/owner.before"
fi
if [ -f "$STATE_FILE" ]; then
[ ! -L "$STATE_FILE" ] || exit 71
grep -Fqx 'owner|{DATAPLANE_OWNER_MARKER}' "$STATE_FILE" || exit 71
cp -p "$STATE_FILE" "$TXN/state.before"
fi
"$WG" showconf "$WG_IFACE" > "$TXN/wg.conf"; chmod 600 "$TXN/wg.conf"
if "$NFT" list table inet virt_cloud_wg_fabric > "$TXN/fabric.nft.before" 2>/dev/null; then
grep -Fq 'virt-cloud-wg-fabric-v1' "$TXN/fabric.nft.before" || exit 71
else
exit 71
fi
if "$NFT" list table inet "$TABLE" > "$TXN/nft.before" 2>/dev/null; then
grep -Fq '{DATAPLANE_OWNER_MARKER}' "$TXN/nft.before" || exit 71
else
rm -f "$TXN/nft.before"
fi
if "$NFT" list table bridge "$TABLE" > "$TXN/nft.bridge.before" 2>/dev/null; then
grep -Fq '{DATAPLANE_OWNER_MARKER}' "$TXN/nft.bridge.before" || exit 71
else
rm -f "$TXN/nft.bridge.before"
fi
if "$NFT" list table arp "$TABLE" > "$TXN/nft.arp.before" 2>/dev/null; then
grep -Fq '{DATAPLANE_OWNER_MARKER}' "$TXN/nft.arp.before" || exit 71
else
rm -f "$TXN/nft.arp.before"
fi
cp -p "$NFT_MAIN" "$TXN/nft_main.before"; NFT_MAIN_BEFORE=yes
if [ -f "$NFT_FILE" ]; then cp -p "$NFT_FILE" "$TXN/nft_file.before"; NFT_FILE_BEFORE=yes; fi
if [ -f "$SYSCTL_FILE" ]; then cp -p "$SYSCTL_FILE" "$TXN/sysctl.before"; fi
if [ -f "$PROXY_FILE" ]; then cp -p "$PROXY_FILE" "$TXN/proxy_file.before"; fi
if "$IP" link show dev "$BRIDGE" >/dev/null 2>&1; then
"$IP" -d link show dev "$BRIDGE" | grep -Eq '(^|[[:space:]])bridge[[:space:]]+(forward_delay|stp_state|hello_time|ageing_time)([[:space:]]|$)' || exit 71
[ -f "$OWNER_FILE" ] || exit 71
else
"$IP" link add name "$BRIDGE" type bridge
BRIDGE_CREATED=yes
printf '%s\\n' "$OWNER_MARKER" > "$OWNER_FILE"; chmod 600 "$OWNER_FILE"
fi
if "$IP" link show dev "$BRIDGE" 2>/dev/null | grep -q 'state UP'; then BRIDGE_WAS_UP=yes; fi
"$IP" link set dev "$BRIDGE" up
# A private bridge is an unnumbered, panel-owned L3 boundary. Reject an
# address, a default/connected route, or a physical/foreign bridge port before
# installing any policy. Libvirt tap names are the only accepted members;
# attaching br0, an underlay NIC, or an operator bridge would make VPC traffic
# escape into the public network.
bridge_addrs=$($IP -o addr show dev "$BRIDGE" 2>/dev/null || true)
printf '%s\\n' "$bridge_addrs" | grep -Eq '[[:space:]]inet6?[[:space:]]' && exit 71 || true
bridge_ports=$($BRIDGE_CMD link show master "$BRIDGE" 2>/dev/null) || exit 71
while IFS= read -r bridge_port_line; do
[ -n "$bridge_port_line" ] || continue
bridge_port=$(printf '%s\\n' "$bridge_port_line" | sed -n 's/^[0-9][0-9]*: \\([^:[:space:]]*\\).*/\\1/p')
[ -n "$bridge_port" ] || exit 71
case "$bridge_port" in
vnet[0-9]*|tap[0-9A-Za-z_.-]*|macvtap[0-9]*|fwpr[0-9A-Za-z_.-]*|fwln[0-9A-Za-z_.-]*) : ;;
*) exit 71 ;;
esac
done <<__VC_VPC_BRIDGE_PORTS__
$bridge_ports
__VC_VPC_BRIDGE_PORTS__
cat > "$TXN/bridge.routes.allowed" <<'{route_marker}.bridge'
{bridge_route_lines}
{route_marker}.bridge
if [ -f "$TXN/state.before" ]; then
awk -F'|' '$1 == "route" && $3 == "{r.bridge}" {{print $2 "|" $3 "|" $4 "|" $5 "|" $6 "|186"}}' "$TXN/state.before" >> "$TXN/bridge.routes.allowed"
fi
sort -u "$TXN/bridge.routes.allowed" -o "$TXN/bridge.routes.allowed"
bridge_routes=$($IP -o -4 route show table all dev "$BRIDGE" 2>/dev/null) || exit 71
while IFS='|' read -r destination device table via metric proto; do
[ -n "$destination" ] || continue
printf '%s\\n' "$destination|$device|$table|$via|$metric|$proto" | grep -Fqx -f "$TXN/bridge.routes.allowed" || exit 71
done <<EOF
$(printf '%s\\n' "$bridge_routes" | awk '{{
destination=$1; device=""; via="-"; metric="100"; table="main"; proto=""
for (i=1;i<=NF;i++) {{
if ($i=="dev" && i<NF) device=$(i+1)
if ($i=="via" && i<NF) via=$(i+1)
if ($i=="metric" && i<NF) metric=$(i+1)
if ($i=="table" && i<NF) table=$(i+1)
if ($i=="proto" && i<NF) proto=$(i+1)
}}
if (table=="254") table="main"
if (destination != "") printf "%s|%s|%s|%s|%s|%s\\n", destination, device, table, via, metric, proto
}}')
EOF
bridge_routes6=$($IP -o -6 route show table all dev "$BRIDGE" 2>/dev/null) || exit 71
[ -z "$bridge_routes6" ] || exit 71
: > "$TXN/proxy.before"
"$IP" neigh show proxy dev "$BRIDGE" 2>/dev/null | awk '$1 ~ /^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$/ {{print $1}}' | LC_ALL=C sort -u > "$TXN/proxy.before"
if [ -f "$PROXY_FILE" ]; then
sed '1d' "$PROXY_FILE" | LC_ALL=C sort -u > "$TXN/proxy.file.before"
else
: > "$TXN/proxy.file.before"
fi
"$CMP" -s "$TXN/proxy.before" "$TXN/proxy.file.before" || exit 71
if [ -r "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp" ]; then
cat "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp" > "$TXN/proxy_arp.before"
else
exit 72
fi
if [ -r "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp_pvlan" ]; then
cat "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp_pvlan" > "$TXN/proxy_arp_pvlan.before"
else
exit 72
fi
if [ -r "/proc/sys/net/ipv4/conf/$BRIDGE/rp_filter" ]; then
cat "/proc/sys/net/ipv4/conf/$BRIDGE/rp_filter" > "$TXN/rp_filter.before"
else
exit 72
fi
SYSCTL_TMP=$(mktemp "/etc/sysctl.d/.virt-cloud-vpc-{r.token}.XXXXXX") || exit 72
cat > "$SYSCTL_TMP" <<'__VC_VPC_SYSCTL_{r.token}'
# virt-cloud-vpc-dataplane-v1
# vpc={plan.vpc.vpc_id} revision={plan.desired_revision} digest={plan.desired_digest}
net.ipv4.conf.{r.bridge}.rp_filter = 2
net.ipv4.conf.{r.bridge}.proxy_arp = 0
net.ipv4.conf.{r.bridge}.proxy_arp_pvlan = 1
__VC_VPC_SYSCTL_{r.token}
chmod 600 "$SYSCTL_TMP"
mv -f "$SYSCTL_TMP" "$SYSCTL_FILE"; SYSCTL_CHANGED=yes
if [ "$MODE" = networkmanager_nmcli ]; then
[ -n "$NMCLI" ] || exit 70
nm_field() {{ "$NMCLI" -t --escape no -g "$1" connection show "$BRIDGE" 2>/dev/null | sed -n '1p'; }}
nm_empty() {{ value=$(nm_field "$1"); [ -z "$value" ] || [ "$value" = "--" ]; }}
NM_PROFILE_COUNT=$("$NMCLI" -t --escape no -f NAME connection show 2>/dev/null | awk -v wanted="$BRIDGE" '$0 == wanted {{ count++ }} END {{ print count + 0 }}')
if [ "$NM_PROFILE_COUNT" = 0 ]; then
"$NMCLI" connection add type bridge ifname "$BRIDGE" con-name "$BRIDGE" \
ipv4.method disabled ipv6.method disabled bridge.stp no \
connection.autoconnect yes connection.stable-id "$NM_PROFILE_MARKER" >/dev/null
NM_CREATED=yes
elif [ "$NM_PROFILE_COUNT" = 1 ]; then
# A deterministic profile name is not ownership by itself. Reusing an
# operator profile with an address, route, controller, or non-bridge type
# could leak VPC traffic into the public network. Require the exact
# panel marker (stored in NetworkManager's supported stable-id property)
# and validate every field that can add L3 reachability before activating
# an existing profile.
[ "$(nm_field connection.type)" = bridge ] || exit 71
[ "$(nm_field connection.interface-name)" = "$BRIDGE" ] || exit 71
[ "$(nm_field ipv4.method)" = disabled ] || exit 71
[ "$(nm_field ipv6.method)" = disabled ] || exit 71
nm_empty ipv4.addresses || exit 71
nm_empty ipv4.routes || exit 71
nm_empty ipv4.routing-rules || exit 71
nm_empty ipv4.gateway || exit 71
nm_empty ipv4.dns || exit 71
nm_empty ipv4.dns-search || exit 71
nm_empty ipv6.addresses || exit 71
nm_empty ipv6.routes || exit 71
nm_empty ipv6.routing-rules || exit 71
nm_empty ipv6.gateway || exit 71
nm_empty ipv6.dns || exit 71
nm_empty ipv6.dns-search || exit 71
nm_empty connection.master || exit 71
nm_empty connection.controller || exit 71
nm_empty connection.slave-type || exit 71
nm_empty connection.port-type || exit 71
nm_empty connection.secondaries || exit 71
nm_empty connection.zone || exit 71
case "$(nm_field bridge.stp)" in no|false|off|0|--) : ;; *) exit 71 ;; esac
NM_PROFILE_STABLE_ID=$(nm_field connection.stable-id)
[ "$NM_PROFILE_STABLE_ID" = "$NM_PROFILE_MARKER" ] || exit 71
else
# More than one same-name profile makes `connection up <name>` ambiguous;
# fail closed instead of possibly activating a public profile.
exit 71
fi
if "$NMCLI" -t --escape no -f NAME connection show --active 2>/dev/null | grep -Fqx "$BRIDGE"; then NM_WAS_ACTIVE=yes; fi
NM_ACTIVATION_ATTEMPTED=yes
"$NMCLI" connection up "$BRIDGE" >/dev/null
else
[ -x "$IFUP" ] && [ -x "$IFQUERY" ] && [ -x "$IFDOWN" ] || exit 70
[ ! -L /etc/network ] && [ ! -L /etc/network/interfaces.d ] || exit 71
# A stanza in the administrator-owned main file wins over an
# interfaces.d fragment on some ifupdown versions. Refuse that ambiguous
# ownership instead of letting a same-name bridge inherit ports, addresses,
# routes, or hooks from /etc/network/interfaces.
if [ -f /etc/network/interfaces ] &&
grep -Eq '^[[:space:]]*(auto|allow-[^[:space:]]+|iface)[[:space:]]+'"$BRIDGE"'([[:space:]]|$)' /etc/network/interfaces; then
exit 71
fi
mkdir -p /etc/network/interfaces.d
if [ -e "$IF_FRAGMENT" ] || [ -L "$IF_FRAGMENT" ]; then
[ -f "$IF_FRAGMENT" ] && [ ! -L "$IF_FRAGMENT" ] || exit 71
grep -Fq '# {DATAPLANE_OWNER_MARKER} vpc=' "$IF_FRAGMENT" || exit 71
cp -p "$IF_FRAGMENT" "$TXN/if_fragment.before"
fi
IF_TMP=$(mktemp "/etc/network/interfaces.d/.virt-cloud-vpc-{r.token}.XXXXXX") || exit 72
cat > "$IF_TMP" <<'{frag_marker}'
# {DATAPLANE_OWNER_MARKER} vpc={plan.vpc.vpc_id} revision={plan.desired_revision} digest={plan.desired_digest}
auto {r.bridge}
iface {r.bridge} inet manual
bridge_ports none
bridge-stp off
{frag_marker}
chmod 600 "$IF_TMP"
mv -f "$IF_TMP" "$IF_FRAGMENT"; IF_FRAGMENT_CHANGED=yes
# Classic Debian ifquery accepts the interface as its positional operand;
# `--list <iface>` is an ifquery2/listing form and fails on ifupdown.
"$IFQUERY" "$BRIDGE" >/dev/null 2>&1 || exit 72
# An existing bridge is shared with libvirt; never ifdown it during
# rollback. A newly created bridge is the only interface this transaction
# may tear down.
[ "$BRIDGE_CREATED" = yes ] && IF_CREATED=yes
IF_ACTIVATION_ATTEMPTED=yes
VC_FABRIC_APPLYING=1 "$IFUP" --force "$BRIDGE" >/dev/null 2>&1 || exit 72
fi
cat > "$TXN/rules.nft" <<'{nft_marker}'
{nft.rstrip()}
{nft_marker}
chmod 600 "$TXN/rules.nft"
"$NFT" -c -f "$TXN/rules.nft" >/dev/null || exit 72
NFT_TMP=$(mktemp "/etc/nftables.d/.virt-cloud-vpc-{r.token}.XXXXXX") || exit 72
cp -p "$TXN/rules.nft" "$NFT_TMP" || exit 72
chmod 600 "$NFT_TMP"
mv -f "$NFT_TMP" "$NFT_FILE"; NFT_FILE_CHANGED=yes
if ! grep -Fqx "$NFT_INCLUDE" "$NFT_MAIN" &&
! grep -Fqx 'include "/etc/nftables.d/*.nft"' "$NFT_MAIN" &&
! grep -Fqx 'include "/etc/nftables.d/*"' "$NFT_MAIN"; then
NFT_MAIN_TMP=$(mktemp "$NFT_MAIN.virt-cloud-vpc.XXXXXX") || exit 72
cat "$NFT_MAIN" > "$NFT_MAIN_TMP"
printf '%s\\n' "$NFT_INCLUDE" >> "$NFT_MAIN_TMP"
"$NFT" -c -f "$NFT_MAIN_TMP" >/dev/null || exit 72
chmod "$($STAT -c '%a' "$NFT_MAIN")" "$NFT_MAIN_TMP"
mv -f "$NFT_MAIN_TMP" "$NFT_MAIN"; NFT_MAIN_CHANGED=yes
fi
"$NFT" -c -f "$NFT_MAIN" >/dev/null || exit 72
NFT_BATCH="$TXN/apply.nft"
: > "$NFT_BATCH"
if [ -f "$TXN/nft.before" ]; then printf 'delete table inet %s\\n' "$TABLE" >> "$NFT_BATCH"; fi
if [ -f "$TXN/nft.bridge.before" ]; then printf 'delete table bridge %s\\n' "$TABLE" >> "$NFT_BATCH"; fi
if [ -f "$TXN/nft.arp.before" ]; then printf 'delete table arp %s\\n' "$TABLE" >> "$NFT_BATCH"; fi
cat "$TXN/rules.nft" >> "$NFT_BATCH"
chmod 600 "$NFT_BATCH"
"$NFT" -c -f "$NFT_BATCH" >/dev/null || exit 72
# A single nft batch is atomic in the kernel: the old owned tables remain in
# place if any delete or new-table rule fails validation/application.
"$NFT" -f "$NFT_BATCH" >/dev/null || exit 72
RP_FILTER_CHANGED=yes
printf '%s\\n' 2 > "/proc/sys/net/ipv4/conf/$BRIDGE/rp_filter" || exit 72
[ "$(cat "/proc/sys/net/ipv4/conf/$BRIDGE/rp_filter" 2>/dev/null)" = 2 ] || exit 72
PROXY_ARP_CHANGED=yes
printf '%s\\n' 0 > "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp" || exit 72
[ "$(cat "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp" 2>/dev/null)" = 0 ] || exit 72
PROXY_ARP_PVLAN_CHANGED=yes
printf '%s\\n' 1 > "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp_pvlan" || exit 72
[ "$(cat "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp_pvlan" 2>/dev/null)" = 1 ] || exit 72
PROXY_ENTRIES_CHANGED=yes
while IFS= read -r old_proxy; do
[ -n "$old_proxy" ] || continue
"$IP" neigh del proxy "$old_proxy" dev "$BRIDGE" >/dev/null 2>&1 || exit 72
done < "$TXN/proxy.before"
while IFS= read -r proxy_ip; do
[ -n "$proxy_ip" ] || continue
"$IP" neigh replace proxy "$proxy_ip" dev "$BRIDGE" >/dev/null 2>&1 || exit 72
done <<'{proxy_marker}'
{proxy_lines}
{proxy_marker}
PROXY_TMP=$(mktemp "$PROXY_FILE.XXXXXX") || exit 72
{{
printf '%s\\n' '# virt-cloud-vpc-proxy-v1'
cat <<'{proxy_marker}'
{proxy_lines}
{proxy_marker}
}} > "$PROXY_TMP"
chmod 600 "$PROXY_TMP"
mv -f "$PROXY_TMP" "$PROXY_FILE"; PROXY_FILE_CHANGED=yes
"$IP" neigh show proxy dev "$BRIDGE" 2>/dev/null | awk '$1 ~ /^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$/ {{print $1}}' | LC_ALL=C sort -u > "$TXN/proxy.after"
cat <<'{proxy_marker}' | LC_ALL=C sort -u > "$TXN/proxy.desired"
{proxy_lines}
{proxy_marker}
"$CMP" -s "$TXN/proxy.after" "$TXN/proxy.desired" || exit 72
: > "$TXN/routes.before"
cat > "$TXN/routes.desired" <<'{route_marker}'
{route_lines}
{route_marker}
if [ -f "$STATE_FILE" ]; then
while IFS='|' read -r old_kind old_destination old_device old_table old_via old_metric; do
[ "$old_kind" = route ] || continue
printf '%s\\n' "$old_destination|$old_device|$ROUTE_TABLE|${{old_via:--}}|${{old_metric:-100}}" | grep -Fqx -f "$TXN/routes.desired" && continue
old_existing=$("$IP" -o route show table "$ROUTE_TABLE" "$old_destination" 2>/dev/null || true)
[ -n "$old_existing" ] || continue
printf '%s\\n' "$old_existing" | grep -Eq 'proto 186([[:space:]]|$)' || exit 73
old_metric_actual=$(printf '%s\\n' "$old_existing" | awk '{{for (i=1; i<=NF; i++) if ($i == "metric" && i < NF) {{print $(i+1); exit}}}}')
[ -n "$old_metric_actual" ] || old_metric_actual=100
old_actual_device=$(printf '%s\\n' "$old_existing" | awk '{{for (i=1; i<=NF; i++) if ($i == "dev" && i < NF) {{print $(i+1); exit}}}}')
[ "$old_actual_device" = "$old_device" ] || exit 73
old_via_actual=$(printf '%s\\n' "$old_existing" | awk '{{for (i=1; i<=NF; i++) if ($i == "via" && i < NF) {{print $(i+1); exit}}}}')
[ "${{old_via_actual:--}}" = "${{old_via:--}}" ] || exit 73
printf '%s|%s|%s|%s|%s\\n' "$old_destination" "$old_device" "$ROUTE_TABLE" "${{old_via:--}}" "$old_metric_actual" >> "$TXN/routes.before"
if [ "${{old_via:--}}" = - ]; then
"$IP" route del "$old_destination" table "$ROUTE_TABLE" dev "$old_device" proto 186 || exit 73
else
"$IP" route del "$old_destination" table "$ROUTE_TABLE" via "$old_via" dev "$old_device" proto 186 || exit 73
fi
done < "$STATE_FILE"
fi
while IFS='|' read -r destination device table via metric; do
[ -n "$destination" ] || continue
existing=$("$IP" -o route show table "$table" "$destination" 2>/dev/null || true)
[ -z "$existing" ] || printf '%s\\n' "$existing" | grep -Eq 'proto 186([[:space:]]|$)' || exit 73
if [ -n "$existing" ]; then
old_device=$(printf '%s\\n' "$existing" | awk '{{for (i=1; i<=NF; i++) if ($i == "dev" && i < NF) {{print $(i+1); exit}}}}')
old_metric=$(printf '%s\\n' "$existing" | awk '{{for (i=1; i<=NF; i++) if ($i == "metric" && i < NF) {{print $(i+1); exit}}}}')
[ -n "$old_device" ] || exit 73
[ "$old_device" = "$device" ] || exit 73
old_via=$(printf '%s\\n' "$existing" | awk '{{for (i=1; i<=NF; i++) if ($i == "via" && i < NF) {{print $(i+1); exit}}}}')
[ "${{old_via:--}}" = "$via" ] || exit 73
[ -n "$old_metric" ] || old_metric=100
printf '%s|%s|%s|%s|%s\\n' "$destination" "$old_device" "$table" "${{old_via:--}}" "$old_metric" >> "$TXN/routes.before"
fi
[ -n "$metric" ] || metric=100
if [ "$via" = - ]; then
"$IP" route replace "$destination" table "$table" dev "$device" proto 186 metric "$metric" || exit 73
else
"$IP" route replace "$destination" table "$table" via "$via" dev "$device" onlink proto 186 metric "$metric" || exit 73
fi
done <<'{route_marker}'
{route_lines}
{route_marker}
: > "$TXN/rules.before"
for pref in "$RULE_PRIORITY" "$WG_RULE_PRIORITY"; do
existing=$("$IP" -o rule show pref "$pref" 2>/dev/null || true)
if [ -n "$existing" ]; then
printf '%s\\n' "$existing" | grep -Fq "lookup $ROUTE_TABLE" || exit 73
old_iif=$(printf '%s\\n' "$existing" | awk '{{for (i=1; i<=NF; i++) if ($i == "iif" && i < NF) {{print $(i+1); exit}}}}')
old_to=$(printf '%s\\n' "$existing" | awk '{{for (i=1; i<=NF; i++) if ($i == "to" && i < NF) {{print $(i+1); exit}}}}')
old_lookup=$(printf '%s\\n' "$existing" | awk '{{for (i=1; i<=NF; i++) if ($i == "lookup" && i < NF) {{print $(i+1); exit}}}}')
[ -n "$old_to" ] && [ -n "$old_lookup" ] || exit 73
if [ "$pref" = "$RULE_PRIORITY" ]; then
[ -z "$old_iif" ] && [ "$old_to" = {_q(plan.vpc.cidr)} ] || exit 73
else
[ "$old_iif" = "$WG_IFACE" ] && [ "$old_to" = {_q(plan.vpc.cidr)} ] || exit 73
fi
printf '%s|%s|%s|%s\\n' "$pref" "${{old_iif:--}}" "$old_to" "$old_lookup" >> "$TXN/rules.before"
fi
done
"$IP" rule replace pref "$RULE_PRIORITY" to {_q(plan.vpc.cidr)} lookup "$ROUTE_TABLE"
"$IP" rule replace pref "$WG_RULE_PRIORITY" iif "$WG_IFACE" to {_q(plan.vpc.cidr)} lookup "$ROUTE_TABLE"
cat > "$STATE_FILE" <<'{state_marker}'
{state.rstrip()}
{state_marker}
chmod 600 "$STATE_FILE"
{state_loader}
vpc_state_load || exit 74
vpc_state_refresh_nft || exit 74
while IFS='|' read -r peer_host key allowed; do
[ -n "$peer_host" ] || continue
"$WG" show "$WG_IFACE" peers | grep -Fqx "$key" || exit 74
case ",$allowed," in *',0.0.0.0/0,'*|*',0.0.0.0/1,'*|*',128.0.0.0/1,'* ) exit 74 ;; esac
# Fabric has one AllowedIPs list per public key. Merge the complete,
# validated tenant set from every panel-owned VPC state file while keeping
# the peer's single overlay /32. Never replace the list with this VPC's
# partial view: doing so would black-hole another concurrently active VPC.
merge_actual=$("$WG" show "$WG_IFACE" allowed-ips 2>/dev/null | awk -v wanted="$key" '$1 == wanted {{print $2; exit}}' || true)
merge_overlay=$(printf '%s\\n' "$merge_actual" | tr ',' '\\n' | awk '/^100\\.64\\.[0-9]+\\.[0-9]+\\/32$/ {{print; count++}} END {{exit count == 1 ? 0 : 1}}') || exit 74
merge_extra=$(vpc_state_allowed_for_peer "$key") || exit 74
merge_allowed="$merge_overlay"
[ -z "$merge_extra" ] || merge_allowed="$merge_allowed,$merge_extra"
"$WG" set "$WG_IFACE" peer "$key" allowed-ips "$merge_allowed" || exit 74
done <<'{peer_marker}'
{peer_lines}
{peer_marker}
rm -f "$VPC_PEER_FILE" "$VPC_ROUTE_FILE" "$VPC_TENANT_FILE" "$VPC_TENANT_FILE.nft"
printf '%s\\n' ready > "$TXN/state"; chmod 600 "$TXN/state"
printf '%s\\n' '__VC_VPC_APPLIED__=yes'
printf '%s\\n' '__VC_VPC_OWNER__={DATAPLANE_OWNER_MARKER}'
printf '%s\\n' '__VC_VPC_REVISION__={plan.desired_revision}'
printf '%s\\n' '__VC_VPC_DIGEST__={plan.desired_digest}'
COMMITTED=yes
printf '%s\\n' committed > "$TXN/state"; chmod 600 "$TXN/state"
rm -rf "$TXN"
trap - EXIT HUP INT TERM
exit 0
'''
if len(script) > MAX_OUTPUT:
raise VpcDataplaneError("VPC 数据面脚本过大", 413, "vpc_script_too_large")
return script
def render_remove_script(
plan: VpcDataplanePlan,
*,
deployment_mode: str = "networkmanager_nmcli",
ifup_path: str = "/usr/sbin/ifup",
ifdown_path: str = "/usr/sbin/ifdown",
ifquery_path: str = "/usr/sbin/ifquery",
) -> str:
"""Render a fail-closed removal transaction for an empty host."""
if plan.local_members:
raise VpcDataplaneError(
"仍有本机私网成员,不能执行宿主机 VPC 销毁",
409, "vpc_remove_has_local_members")
if deployment_mode not in {"networkmanager_nmcli", "ifupdown"}:
raise VpcDataplaneError("宿主机网络管理模式不支持", 409,
"vpc_dataplane_network_manager_conflict")
allowed_ifup = {"/usr/sbin/ifup", "/usr/bin/ifup", "/sbin/ifup", "/bin/ifup"}
allowed_ifdown = {"/usr/sbin/ifdown", "/usr/bin/ifdown", "/sbin/ifdown", "/bin/ifdown"}
allowed_ifquery = {"/usr/sbin/ifquery", "/usr/bin/ifquery", "/sbin/ifquery", "/bin/ifquery"}
if (ifup_path not in allowed_ifup or ifdown_path not in allowed_ifdown
or ifquery_path not in allowed_ifquery):
raise VpcDataplaneError("ifupdown 工具路径不受支持", 409,
"vpc_dataplane_network_manager_conflict")
from .fabric import _VPC_STATE_LOADER
r = plan.resources
include = f"/etc/nftables.d/virt-cloud-vpc-{r.token}.nft"
fragment = f"/etc/network/interfaces.d/virt-cloud-vpc-{r.token}"
sysctl_file = f"/etc/sysctl.d/99-virt-cloud-vpc-{r.token}.conf"
proxy_file = f"/etc/virt-cloud/vpc-dataplane/{r.token}.proxy"
state_loader = _VPC_STATE_LOADER.replace(
"__VC_FABRIC_HOST_ID__", plan.host_id).replace(
"__VC_FABRIC_ID__", plan.vpc.fabric_id).replace(
"__VC_FABRIC_NETWORK__", "100.64.0.0/10").replace(
"__VC_VPC_STATE_OBSERVE__", "no")
nm_profile_marker = DATAPLANE_OWNER_MARKER + " vpc=" + plan.vpc.vpc_id
ifup_marker = f"# {DATAPLANE_OWNER_MARKER} vpc={plan.vpc.vpc_id}"
inflight_marker = (f"{DATAPLANE_OWNER_MARKER}:{plan.vpc.vpc_id}:"
f"{plan.host_id}:{plan.desired_revision}:{plan.desired_digest}")
script = f'''#!/bin/sh
set -eu
umask 077
PATH=/usr/sbin:/usr/bin:/sbin:/bin
export PATH
IP=$(for p in /usr/sbin/ip /usr/bin/ip /sbin/ip /bin/ip; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
WG=$(for p in /usr/bin/wg /usr/sbin/wg /bin/wg /sbin/wg; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
NFT=$(for p in /usr/sbin/nft /usr/bin/nft /sbin/nft /bin/nft; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
FLOCK=$(for p in /usr/bin/flock /usr/sbin/flock /bin/flock /sbin/flock; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
STAT=$(for p in /usr/bin/stat /bin/stat /usr/sbin/stat /sbin/stat; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
MKTEMP=$(for p in /usr/bin/mktemp /bin/mktemp /usr/sbin/mktemp /sbin/mktemp; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
BRIDGE_CMD=$(for p in /usr/sbin/bridge /usr/bin/bridge /sbin/bridge /bin/bridge; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
BRIDGE={_q(r.bridge)}; TABLE={_q(r.nft_table)}; ROUTE_TABLE={r.route_table}; WG_IFACE={_q(FABRIC_INTERFACE)}
OWNER_FILE={_q(r.owner_file)}; STATE_FILE={_q(r.state_file)}; LOCK={_q(r.lock_file)}; TXN={_q(r.transaction_dir)}
NFT_FILE={_q(include)}; IF_FRAGMENT={_q(fragment)}; SYSCTL_FILE={_q(sysctl_file)}; PROXY_FILE={_q(proxy_file)}; STATE_DIR=/etc/virt-cloud/vpc-dataplane
MODE={_q(deployment_mode)}; IFUP={_q(ifup_path)}; IFDOWN={_q(ifdown_path)}; IFQUERY={_q(ifquery_path)}; NMCLI=; NFT_MAIN=; NFT_INCLUDE={_q(f'include "{include}"')}
[ -n "$IP" ] && [ -n "$WG" ] && [ -n "$NFT" ] && [ -n "$FLOCK" ] && [ -n "$STAT" ] && [ -n "$MKTEMP" ] && [ -n "$BRIDGE_CMD" ] || exit 70
if [ "$MODE" = networkmanager_nmcli ]; then
NMCLI=$(for p in /usr/bin/nmcli /usr/sbin/nmcli /bin/nmcli /sbin/nmcli; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
[ -n "$NMCLI" ] || exit 70
fi
[ ! -L /run/virt-cloud ] && [ ! -L /etc/virt-cloud ] || exit 71
mkdir -p /run/virt-cloud; chmod 700 /run/virt-cloud
if [ -e "$STATE_DIR" ] || [ -L "$STATE_DIR" ]; then
[ -d "$STATE_DIR" ] && [ ! -L "$STATE_DIR" ] || exit 71
[ "$($STAT -c '%u:%a' "$STATE_DIR" 2>/dev/null)" = 0:700 ] || exit 71
fi
if [ -e /etc/nftables.d ] || [ -L /etc/nftables.d ]; then
[ -d /etc/nftables.d ] && [ ! -L /etc/nftables.d ] || exit 71
[ "$($STAT -c '%u:%a' /etc/nftables.d 2>/dev/null)" = 0:755 ] || exit 71
fi
if [ -e /etc/sysctl.d ] || [ -L /etc/sysctl.d ]; then
[ -d /etc/sysctl.d ] && [ ! -L /etc/sysctl.d ] || exit 71
[ "$($STAT -c '%u:%a' /etc/sysctl.d 2>/dev/null)" = 0:755 ] || exit 71
fi
[ ! -L "$LOCK" ] && {{ [ ! -e "$LOCK" ] || [ -f "$LOCK" ]; }} || exit 71
: >> "$LOCK"; chmod 600 "$LOCK"; exec 9>>"$LOCK"; "$FLOCK" -x 9
exec 8>>{_q(FABRIC_LOCK)}; "$FLOCK" -x 8
[ -f /etc/virt-cloud/wg-fabric0.owner ] && grep -Fqx 'virt-cloud-wg-fabric-v1' /etc/virt-cloud/wg-fabric0.owner || exit 71
"$WG" show "$WG_IFACE" >/dev/null 2>&1 || exit 71
BRIDGE_PRESENT=no
if "$IP" link show dev "$BRIDGE" >/dev/null 2>&1; then BRIDGE_PRESENT=yes; fi
PHASE=preparing; RESUME=no
if [ -e "$TXN" ]; then
[ -d "$TXN" ] && [ ! -L "$TXN" ] || exit 79
[ "$($STAT -c '%u:%a' "$TXN" 2>/dev/null)" = 0:700 ] || exit 79
[ -f "$TXN/inflight" ] && [ ! -L "$TXN/inflight" ] || exit 79
[ "$($STAT -c '%u:%a' "$TXN/inflight" 2>/dev/null)" = 0:600 ] || exit 79
grep -Fqx '{inflight_marker}' "$TXN/inflight" || exit 79
[ -f "$TXN/state" ] && [ ! -L "$TXN/state" ] || exit 79
[ "$($STAT -c '%u:%a' "$TXN/state" 2>/dev/null)" = 0:600 ] || exit 79
PHASE=$(cat "$TXN/state" 2>/dev/null) || exit 79
case "$PHASE" in
preparing|routes_removed|manager_removed|bridge_removed|nft_removed|files_removed|wg_merged|committed) : ;;
*) exit 79 ;;
esac
RESUME=yes
for snapshot in "$TXN/state.before" "$TXN/routes.before" "$TXN/rules.before" "$TXN/proxy.before"; do
[ -e "$snapshot" ] || continue
[ -f "$snapshot" ] && [ ! -L "$snapshot" ] || exit 79
[ "$($STAT -c '%u:%a' "$snapshot" 2>/dev/null)" = 0:600 ] || exit 79
done
fi
set_phase() {{
phase_tmp="$TXN/.state.$$"
printf '%s\\n' "$1" > "$phase_tmp"
chmod 600 "$phase_tmp"
mv -f "$phase_tmp" "$TXN/state"
PHASE="$1"
}}
verify_routes_removed() {{
verify_routes=$($IP -o -4 route show table "$ROUTE_TABLE" 2>/dev/null || true)
[ -z "$verify_routes" ] || return 1
verify_rule=$($IP -o rule show pref "{r.rule_priority}" 2>/dev/null || true)
[ -z "$verify_rule" ] || return 1
verify_rule=$($IP -o rule show pref "$(( {r.rule_priority} + 1 ))" 2>/dev/null || true)
[ -z "$verify_rule" ] || return 1
if $IP link show dev "$BRIDGE" >/dev/null 2>&1; then
verify_proxy=$($IP neigh show proxy dev "$BRIDGE" 2>/dev/null | awk '$1 ~ /^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$/') || return 1
[ -z "$verify_proxy" ] || return 1
fi
return 0
}}
case "$PHASE" in
preparing|routes_removed|manager_removed|bridge_removed|nft_removed) PRE_FILES=yes ;;
files_removed|wg_merged|committed) PRE_FILES=no ;;
*) exit 79 ;;
esac
if [ "$PRE_FILES" = yes ]; then
if [ -f "$OWNER_FILE" ]; then
[ ! -L "$OWNER_FILE" ] || exit 71
[ "$($STAT -c '%u:%a' "$OWNER_FILE" 2>/dev/null)" = 0:600 ] || exit 71
grep -Fqx 'virt-cloud-vpc-dataplane-v1' "$OWNER_FILE" || exit 71
elif [ -e "$STATE_FILE" ] || [ "$BRIDGE_PRESENT" = yes ] || [ -e "$NFT_FILE" ] ||
[ -e "$SYSCTL_FILE" ] || [ -e "$PROXY_FILE" ] || [ -e "$IF_FRAGMENT" ]; then
# A deterministic bridge/file name is not ownership. Refuse to remove
# anything when the panel's owner marker disappeared or was replaced.
exit 71
fi
if [ -f "$STATE_FILE" ]; then
[ ! -L "$STATE_FILE" ] || exit 71
[ "$($STAT -c '%u:%a' "$STATE_FILE" 2>/dev/null)" = 0:600 ] || exit 71
grep -Fqx 'owner|{DATAPLANE_OWNER_MARKER}' "$STATE_FILE" || exit 71
grep -Fqx 'vpc|{plan.vpc.vpc_id}' "$STATE_FILE" || exit 71
grep -Fqx 'cidr|{plan.vpc.cidr}' "$STATE_FILE" || exit 71
grep -Fqx 'fabric|{plan.vpc.fabric_id}' "$STATE_FILE" || exit 71
grep -Fqx 'host|{plan.host_id}' "$STATE_FILE" || exit 71
grep -Fqx 'bridge|{r.bridge}' "$STATE_FILE" || exit 71
grep -Fqx 'table|{r.route_table}' "$STATE_FILE" || exit 71
state_revision=$(awk -F'|' '$1=="revision" {{print $2; exit}}' "$STATE_FILE")
state_digest=$(awk -F'|' '$1=="digest" {{print $2; exit}}' "$STATE_FILE")
case "$state_revision" in ''|*[!0-9]*) exit 79;; esac
printf '%s\n' "$state_digest" | grep -Eq '^[0-9a-f]{{64}}$' || exit 79
[ "$state_revision" -le {plan.desired_revision} ] || exit 79
if [ "$state_revision" = {plan.desired_revision} ] && [ "$state_digest" != { _q(plan.desired_digest) } ]; then exit 79; fi
elif [ -f "$OWNER_FILE" ] && [ "$BRIDGE_PRESENT" = yes ]; then
# A live panel bridge without its state cannot be reconciled safely.
exit 71
else
# Without the state file there is no safe owner proof for deterministic
# policy routes/rules. Refuse to claim a clean removal while any of those
# resources remain; an operator can restore the state and retry.
orphan_routes=$("$IP" -o -4 route show table "$ROUTE_TABLE" 2>/dev/null || true)
[ -z "$orphan_routes" ] || exit 71
orphan_rule=$("$IP" -o rule show pref "{r.rule_priority}" 2>/dev/null || true)
[ -z "$orphan_rule" ] || exit 71
orphan_rule=$("$IP" -o rule show pref "$(( {r.rule_priority} + 1 ))" 2>/dev/null || true)
[ -z "$orphan_rule" ] || exit 71
fi
for artifact in "$NFT_FILE" "$SYSCTL_FILE" "$PROXY_FILE" "$IF_FRAGMENT"; do
[ -e "$artifact" ] || continue
[ -f "$artifact" ] && [ ! -L "$artifact" ] || exit 71
done
if [ -f "$NFT_FILE" ]; then
[ "$($STAT -c '%u:%a' "$NFT_FILE" 2>/dev/null)" = 0:600 ] || exit 71
grep -Fqx '# virt-cloud-vpc-dataplane-v1' "$NFT_FILE" || exit 71
fi
if [ -f "$SYSCTL_FILE" ]; then
[ "$($STAT -c '%u:%a' "$SYSCTL_FILE" 2>/dev/null)" = 0:600 ] || exit 71
sed -n '1p' "$SYSCTL_FILE" | grep -Fqx '# virt-cloud-vpc-dataplane-v1' || exit 71
fi
if [ -f "$PROXY_FILE" ]; then
[ "$($STAT -c '%u:%a' "$PROXY_FILE" 2>/dev/null)" = 0:600 ] || exit 71
sed -n '1p' "$PROXY_FILE" | grep -Fqx '# virt-cloud-vpc-proxy-v1' || exit 71
awk 'NR > 1 && $0 !~ /^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$/ {{ exit 1 }}' "$PROXY_FILE" || exit 71
fi
if [ -f "$IF_FRAGMENT" ]; then
[ "$($STAT -c '%u:%a' "$IF_FRAGMENT" 2>/dev/null)" = 0:600 ] || exit 71
sed -n '1p' "$IF_FRAGMENT" | grep -Fqx '{ifup_marker}' || exit 71
grep -Fqx 'auto {r.bridge}' "$IF_FRAGMENT" || exit 71
grep -Fqx 'iface {r.bridge} inet manual' "$IF_FRAGMENT" || exit 71
if grep -Eq '^[[:space:]]*bridge_ports([[:space:]]|$)' "$IF_FRAGMENT"; then
grep -Eq '^[[:space:]]*bridge_ports[[:space:]]+none[[:space:]]*$' "$IF_FRAGMENT" || exit 71
fi
grep -Eq '^[[:space:]]*(address|gateway|up|down|pre-up|post-up|pre-down|post-down)([[:space:]]|$)' "$IF_FRAGMENT" && exit 71 || true
fi
else
for artifact in "$NFT_FILE" "$SYSCTL_FILE" "$PROXY_FILE" "$IF_FRAGMENT" "$OWNER_FILE" "$STATE_FILE"; do
[ ! -e "$artifact" ] && [ ! -L "$artifact" ] || exit 79
done
fi
if [ "$MODE" = networkmanager_nmcli ]; then
if [ "$PHASE" = preparing ] || [ "$PHASE" = routes_removed ]; then
nm_field() {{ "$NMCLI" -t --escape no -g "$1" connection show "$BRIDGE" 2>/dev/null | sed -n '1p'; }}
nm_empty() {{ value=$(nm_field "$1"); [ -z "$value" ] || [ "$value" = "--" ]; }}
NM_PROFILE_COUNT=$("$NMCLI" -t --escape no -f NAME connection show 2>/dev/null | awk -v wanted="$BRIDGE" '$0 == wanted {{ count++ }} END {{ print count + 0 }}')
case "$NM_PROFILE_COUNT" in ''|*[!0-9]*) exit 71;; esac
if [ "$NM_PROFILE_COUNT" = 1 ]; then
# The deterministic profile name is not sufficient ownership evidence.
# Match the apply-side contract before taking a profile down or deleting
# it, so a public/operator bridge can never be removed by this cleanup.
[ "$(nm_field connection.type)" = bridge ] || exit 71
[ "$(nm_field connection.interface-name)" = "$BRIDGE" ] || exit 71
[ "$(nm_field connection.stable-id)" = "{nm_profile_marker}" ] || exit 71
[ "$(nm_field ipv4.method)" = disabled ] || exit 71
[ "$(nm_field ipv6.method)" = disabled ] || exit 71
nm_empty ipv4.addresses || exit 71
nm_empty ipv4.routes || exit 71
nm_empty ipv4.routing-rules || exit 71
nm_empty ipv4.gateway || exit 71
nm_empty ipv4.dns || exit 71
nm_empty ipv4.dns-search || exit 71
nm_empty ipv6.addresses || exit 71
nm_empty ipv6.routes || exit 71
nm_empty ipv6.routing-rules || exit 71
nm_empty ipv6.gateway || exit 71
nm_empty ipv6.dns || exit 71
nm_empty ipv6.dns-search || exit 71
nm_empty connection.master || exit 71
nm_empty connection.controller || exit 71
nm_empty connection.slave-type || exit 71
nm_empty connection.port-type || exit 71
nm_empty connection.secondaries || exit 71
nm_empty connection.zone || exit 71
case "$(nm_field bridge.stp)" in no|false|off|0|--) : ;; *) exit 71 ;; esac
elif [ "$NM_PROFILE_COUNT" -gt 1 ]; then
exit 71
elif [ "$BRIDGE_PRESENT" = yes ]; then
# An unmanaged live bridge is foreign/ambiguous; do not delete it.
exit 71
fi
else
if "$NMCLI" connection show "$BRIDGE" >/dev/null 2>&1; then exit 79; fi
fi
else
if [ "$PHASE" = preparing ] || [ "$PHASE" = routes_removed ]; then
if [ "$BRIDGE_PRESENT" = yes ] || [ -e "$IF_FRAGMENT" ]; then
[ -f "$IF_FRAGMENT" ] || exit 71
[ ! -L /etc/network ] && [ ! -L /etc/network/interfaces ] && [ ! -L /etc/network/interfaces.d ] || exit 71
if [ -f /etc/network/interfaces ] &&
grep -Eq '^[[:space:]]*(auto|allow-[^[:space:]]+|iface)[[:space:]]+'"$BRIDGE"'([[:space:]]|$)' /etc/network/interfaces; then
exit 71
fi
"$IFQUERY" "$BRIDGE" >/dev/null 2>&1 || exit 71
fi
fi
fi
if [ "$PHASE" = preparing ] || [ "$PHASE" = routes_removed ] || [ "$PHASE" = manager_removed ]; then
if "$IP" link show dev "$BRIDGE" >/dev/null 2>&1; then
[ -f "$OWNER_FILE" ] || exit 71
bridge_info=$("$IP" -d link show dev "$BRIDGE" 2>/dev/null) || exit 71
printf '%s\\n' "$bridge_info" | grep -Eq '(^|[[:space:]])bridge[[:space:]]+(forward_delay|stp_state|hello_time|ageing_time)([[:space:]]|$)' || exit 71
bridge_ports=$("$BRIDGE_CMD" link show master "$BRIDGE" 2>/dev/null) || exit 71
[ -z "$bridge_ports" ] || exit 79
bridge_addrs=$("$IP" -o addr show dev "$BRIDGE" 2>/dev/null) || exit 71
[ -z "$bridge_addrs" ] || exit 79
fi
else
if "$IP" link show dev "$BRIDGE" >/dev/null 2>&1; then exit 79; fi
fi
NFT_MAIN_COUNT=0
for nft_candidate in /etc/nftables.conf /etc/sysconfig/nftables.conf; do
if [ -f "$nft_candidate" ] && [ ! -L "$nft_candidate" ] && [ "$($STAT -c '%u' "$nft_candidate" 2>/dev/null)" = 0 ]; then
case "$($STAT -c '%a' "$nft_candidate" 2>/dev/null)" in
[0-7][0145][0145]) : ;;
*) continue ;;
esac
NFT_MAIN="$nft_candidate"; NFT_MAIN_COUNT=$((NFT_MAIN_COUNT + 1))
fi
done
[ "$NFT_MAIN_COUNT" -eq 1 ] || exit 71
if [ "$PHASE" = preparing ] || [ "$PHASE" = routes_removed ] || [ "$PHASE" = manager_removed ] || [ "$PHASE" = bridge_removed ]; then
if [ -f "$NFT_FILE" ]; then
nft_tables=$(awk '$1=="table" {{print $2 "|" $3}}' "$NFT_FILE")
nft_table_count=0
while IFS='|' read -r family name; do
[ -n "$family" ] || continue
case "$family:$name" in inet:*|bridge:*|arp:*) : ;; *) exit 71 ;; esac
[ "$name" = "$TABLE" ] || exit 71
table_dump=$("$NFT" list table "$family" "$name" 2>/dev/null) || exit 71
printf '%s\\n' "$table_dump" | grep -Fq 'virt-cloud-vpc-dataplane-v1' || exit 71
nft_table_count=$((nft_table_count + 1))
done <<EOF
$nft_tables
EOF
[ "$nft_table_count" -gt 0 ] || exit 71
fi
else
if [ -f "$NFT_FILE" ]; then
nft_tables=$(awk '$1=="table" {{print $2 "|" $3}}' "$NFT_FILE")
while IFS='|' read -r family name; do
[ -n "$family" ] || continue
case "$family:$name" in inet:*|bridge:*|arp:*) : ;; *) exit 79 ;; esac
[ "$name" = "$TABLE" ] || exit 79
if "$NFT" list table "$family" "$name" >/dev/null 2>&1; then exit 79; fi
done <<EOF
$nft_tables
EOF
fi
grep -Fqx "$NFT_INCLUDE" "$NFT_MAIN" && exit 79 || true
fi
# A partially completed removal is deliberately fail-closed. The journal is
# retained for an operator/reconciler to inspect; rerunning it must never repeat
# route, bridge, nft, or WireGuard deletions against an unknown intermediate
# state. A fully committed journal only needs the already-verified cleanup.
if [ "$RESUME" = yes ]; then
case "$PHASE" in
committed)
rm -rf "$TXN" || exit 79
printf '%s\\n' '__VC_VPC_REMOVED__=yes'
printf '%s\\n' '__VC_VPC_OWNER__={DATAPLANE_OWNER_MARKER}'
printf '%s\\n' '__VC_VPC_REVISION__={plan.desired_revision}'
printf '%s\\n' '__VC_VPC_DIGEST__={plan.desired_digest}'
printf '%s\\n' '__VC_VPC_ROLLBACK__=none'
exit 0
;;
files_removed|wg_merged)
# All destructive host resources are already absent. Re-running only
# the Fabric state load and peer merge below is idempotent and closes
# the crash window between file removal and AllowedIPs convergence.
:
;;
*) exit 79 ;;
esac
fi
if [ "$RESUME" = no ]; then
mkdir -m 700 "$TXN"
printf '%s\\n' '{inflight_marker}' > "$TXN/inflight"; chmod 600 "$TXN/inflight"
set_phase preparing
if [ -f "$STATE_FILE" ]; then cp -p "$STATE_FILE" "$TXN/state.before"; fi
: > "$TXN/routes.before"; : > "$TXN/rules.before"; : > "$TXN/proxy.before"
chmod 600 "$TXN/routes.before" "$TXN/rules.before" "$TXN/proxy.before"
else
# A resumed transaction must use the original snapshots. Creating an empty
# snapshot after a partial failure would turn an unknown host state into an
# apparently clean state and could make the next destructive step unsafe.
for snapshot in "$TXN/routes.before" "$TXN/rules.before" "$TXN/proxy.before"; do
[ -f "$snapshot" ] && [ ! -L "$snapshot" ] || exit 79
[ "$($STAT -c '%u:%a' "$snapshot" 2>/dev/null)" = 0:600 ] || exit 79
done
fi
if [ -f "$TXN/state.before" ]; then
while IFS='|' read -r kind destination device table via metric; do
[ "$kind" = route ] || continue
[ "$table" = "$ROUTE_TABLE" ] || exit 71
[ "$device" = "$BRIDGE" ] || [ "$device" = "$WG_IFACE" ] || exit 71
printf '%s\\n' "$destination" | grep -Eq '^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+/32$' || exit 71
case "$via" in
-|'') : ;;
*) printf '%s\\n' "$via" | grep -Eq '^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$' || exit 71 ;;
esac
case "$metric" in ''|*[!0-9]*) exit 71;; esac
existing=$("$IP" -o -4 route show table "$table" "$destination" 2>/dev/null || true)
[ -n "$existing" ] || continue
existing_count=$(printf '%s\\n' "$existing" | awk 'NF {{ count++ }} END {{ print count + 0 }}')
[ "$existing_count" = 1 ] || exit 71
actual_device=$(printf '%s\\n' "$existing" | awk '{{for (i=1;i<=NF;i++) if ($i=="dev" && i<NF) {{print $(i+1); exit}}}}')
[ "$actual_device" = "$device" ] || exit 71
printf '%s\\n' "$existing" | grep -Eq 'proto 186([[:space:]]|$)' || exit 71
old_via=$(printf '%s\\n' "$existing" | awk '{{for (i=1;i<=NF;i++) if ($i=="via" && i<NF) {{print $(i+1); exit}}}}')
old_metric=$(printf '%s\\n' "$existing" | awk '{{for (i=1;i<=NF;i++) if ($i=="metric" && i<NF) {{print $(i+1); exit}}}}')
[ -n "$old_metric" ] || old_metric=100
[ "$old_metric" = "$metric" ] || exit 71
[ "${{old_via:--}}" = "${{via:--}}" ] || exit 71
printf '%s|%s|%s|%s|%s\\n' "$destination" "$device" "$table" "$old_via" "$old_metric" >> "$TXN/routes.before"
if [ "${{old_via:--}}" != - ]; then
"$IP" route del "$destination" table "$table" via "$old_via" dev "$device" proto 186 metric "$old_metric" || exit 72
else
"$IP" route del "$destination" table "$table" dev "$device" proto 186 metric "$old_metric" || exit 72
fi
done < "$TXN/state.before"
old_priority=$(awk -F'|' '$1=="priority" {{print $2; exit}}' "$TXN/state.before")
old_cidr=$(awk -F'|' '$1=="cidr" {{print $2; exit}}' "$TXN/state.before")
[ -n "$old_priority" ] && [ -n "$old_cidr" ] || exit 71
case "$old_priority" in ''|*[!0-9]*) exit 71;; esac
[ "$old_priority" = "{r.rule_priority}" ] || exit 71
for pref in "$old_priority" "$((old_priority + 1))"; do
existing=$("$IP" -o rule show pref "$pref" 2>/dev/null || true)
[ -n "$existing" ] || continue
existing_count=$(printf '%s\\n' "$existing" | awk 'NF {{ count++ }} END {{ print count + 0 }}')
[ "$existing_count" = 1 ] || exit 71
printf '%s\\n' "$existing" | grep -Fq "to $old_cidr lookup $ROUTE_TABLE" || exit 71
old_iif=$(printf '%s\\n' "$existing" | awk '{{for (i=1;i<=NF;i++) if ($i=="iif" && i<NF) {{print $(i+1); exit}}}}')
if [ "$pref" = "$old_priority" ]; then
[ -z "$old_iif" ] || exit 71
else
[ "$old_iif" = "$WG_IFACE" ] || exit 71
fi
printf '%s|%s|%s|%s\\n' "$pref" "$old_iif" "$old_cidr" "$ROUTE_TABLE" >> "$TXN/rules.before"
"$IP" rule del pref "$pref" || exit 72
done
fi
if [ -f "$PROXY_FILE" ]; then
while IFS= read -r proxy_ip; do
[ -n "$proxy_ip" ] || continue
if "$IP" neigh show proxy dev "$BRIDGE" 2>/dev/null | grep -Fqx "$proxy_ip"; then
printf '%s\\n' "$proxy_ip" >> "$TXN/proxy.before"
"$IP" neigh del proxy "$proxy_ip" dev "$BRIDGE" || exit 72
fi
done <<EOF
$(sed '1d' "$PROXY_FILE")
EOF
fi
verify_routes_removed || exit 72
set_phase routes_removed
if [ "$MODE" = networkmanager_nmcli ]; then
if [ -n "$NMCLI" ] && "$NMCLI" connection show "$BRIDGE" >/dev/null 2>&1; then
"$NMCLI" connection down "$BRIDGE" >/dev/null 2>&1 || true
"$NMCLI" connection delete "$BRIDGE" >/dev/null 2>&1 || exit 72
if "$NMCLI" connection show "$BRIDGE" >/dev/null 2>&1; then exit 72; fi
fi
else
if [ -f "$IF_FRAGMENT" ]; then
"$IFQUERY" "$BRIDGE" >/dev/null 2>&1 || exit 71
"$IFDOWN" --force "$BRIDGE" >/dev/null 2>&1 || exit 72
fi
fi
set_phase manager_removed
if "$IP" link show dev "$BRIDGE" >/dev/null 2>&1; then
bridge_info=$("$IP" -d link show dev "$BRIDGE" 2>/dev/null) || exit 71
printf '%s\\n' "$bridge_info" | grep -Eq '(^|[[:space:]])bridge[[:space:]]+(forward_delay|stp_state|hello_time|ageing_time)([[:space:]]|$)' || exit 71
ports=$("$BRIDGE_CMD" link show master "$BRIDGE" 2>/dev/null) || exit 71
[ -z "$ports" ] || exit 79
bridge_addrs=$("$IP" -o addr show dev "$BRIDGE" 2>/dev/null) || exit 71
[ -z "$bridge_addrs" ] || exit 79
bridge_routes=$("$IP" -o -4 route show table all dev "$BRIDGE" 2>/dev/null) || exit 71
[ -z "$bridge_routes" ] || exit 79
bridge_routes6=$("$IP" -o -6 route show table all dev "$BRIDGE" 2>/dev/null) || exit 71
[ -z "$bridge_routes6" ] || exit 79
proxy_left=$("$IP" neigh show proxy dev "$BRIDGE" 2>/dev/null | awk '$1 ~ /^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$/') || exit 71
[ -z "$proxy_left" ] || exit 79
"$IP" link del dev "$BRIDGE" type bridge || exit 72
fi
if "$IP" link show dev "$BRIDGE" >/dev/null 2>&1; then exit 72; fi
set_phase bridge_removed
if [ -f "$NFT_FILE" ]; then
nft_tables=$(awk '$1=="table" {{print $2 "|" $3}}' "$NFT_FILE")
nft_table_count=0
while IFS='|' read -r family name; do
[ -n "$family" ] || continue
case "$family:$name" in inet:*|bridge:*|arp:*) : ;; *) exit 71 ;; esac
[ "$name" = "$TABLE" ] || exit 71
table_dump=$("$NFT" list table "$family" "$name" 2>/dev/null) || exit 71
printf '%s\\n' "$table_dump" | grep -Fq 'virt-cloud-vpc-dataplane-v1' || exit 71
"$NFT" delete table "$family" "$name" || exit 72
if "$NFT" list table "$family" "$name" >/dev/null 2>&1; then exit 72; fi
nft_table_count=$((nft_table_count + 1))
done <<EOF
$nft_tables
EOF
[ "$nft_table_count" -gt 0 ] || exit 71
fi
if grep -Fqx "$NFT_INCLUDE" "$NFT_MAIN"; then
NFT_MAIN_TMP=$("$MKTEMP" "$NFT_MAIN.virt-cloud-vpc-remove.XXXXXX") || exit 72
awk -v needle="$NFT_INCLUDE" '$0 != needle' "$NFT_MAIN" > "$NFT_MAIN_TMP" || {{ rm -f "$NFT_MAIN_TMP"; exit 72; }}
"$NFT" -c -f "$NFT_MAIN_TMP" >/dev/null || {{ rm -f "$NFT_MAIN_TMP"; exit 72; }}
chmod "$($STAT -c '%a' "$NFT_MAIN")" "$NFT_MAIN_TMP" || {{ rm -f "$NFT_MAIN_TMP"; exit 72; }}
mv -f "$NFT_MAIN_TMP" "$NFT_MAIN" || exit 72
grep -Fqx "$NFT_INCLUDE" "$NFT_MAIN" && exit 72 || true
fi
set_phase nft_removed
if [ -e "$NFT_FILE" ] || [ -L "$NFT_FILE" ]; then
[ -f "$NFT_FILE" ] && [ ! -L "$NFT_FILE" ] || exit 71
grep -Fqx '# virt-cloud-vpc-dataplane-v1' "$NFT_FILE" || exit 71
grep -Fqx '# vpc={plan.vpc.vpc_id} revision={plan.desired_revision} digest={plan.desired_digest}' "$NFT_FILE" || exit 71
fi
if [ -e "$SYSCTL_FILE" ] || [ -L "$SYSCTL_FILE" ]; then
[ -f "$SYSCTL_FILE" ] && [ ! -L "$SYSCTL_FILE" ] || exit 71
sed -n '1p' "$SYSCTL_FILE" | grep -Fqx '# virt-cloud-vpc-dataplane-v1' || exit 71
fi
if [ -e "$PROXY_FILE" ] || [ -L "$PROXY_FILE" ]; then
[ -f "$PROXY_FILE" ] && [ ! -L "$PROXY_FILE" ] || exit 71
sed -n '1p' "$PROXY_FILE" | grep -Fqx '# virt-cloud-vpc-proxy-v1' || exit 71
fi
if [ -e "$IF_FRAGMENT" ] || [ -L "$IF_FRAGMENT" ]; then
[ -f "$IF_FRAGMENT" ] && [ ! -L "$IF_FRAGMENT" ] || exit 71
sed -n '1p' "$IF_FRAGMENT" | grep -Fqx '{ifup_marker}' || exit 71
fi
if [ -f "$STATE_FILE" ]; then
[ ! -L "$STATE_FILE" ] || exit 71
grep -Fqx 'owner|{DATAPLANE_OWNER_MARKER}' "$STATE_FILE" || exit 71
grep -Fqx 'vpc|{plan.vpc.vpc_id}' "$STATE_FILE" || exit 71
grep -Fqx 'fabric|{plan.vpc.fabric_id}' "$STATE_FILE" || exit 71
grep -Fqx 'host|{plan.host_id}' "$STATE_FILE" || exit 71
grep -Fqx 'bridge|{r.bridge}' "$STATE_FILE" || exit 71
grep -Fqx 'table|{r.route_table}' "$STATE_FILE" || exit 71
state_revision=$(awk -F'|' '$1=="revision" {{print $2; exit}}' "$STATE_FILE")
state_digest=$(awk -F'|' '$1=="digest" {{print $2; exit}}' "$STATE_FILE")
case "$state_revision" in ''|*[!0-9]*) exit 79;; esac
[ "$state_revision" -le {plan.desired_revision} ] || exit 79
if [ "$state_revision" = {plan.desired_revision} ] && [ "$state_digest" != { _q(plan.desired_digest) } ]; then exit 79; fi
fi
rm -f "$NFT_FILE" "$SYSCTL_FILE" "$PROXY_FILE" "$IF_FRAGMENT" "$OWNER_FILE" "$STATE_FILE"
[ ! -e "$NFT_FILE" ] && [ ! -e "$SYSCTL_FILE" ] && [ ! -e "$PROXY_FILE" ] &&
[ ! -e "$IF_FRAGMENT" ] && [ ! -e "$OWNER_FILE" ] && [ ! -e "$STATE_FILE" ] || exit 72
set_phase files_removed
{state_loader}
vpc_state_load || exit 74
vpc_state_refresh_nft || exit 74
for key in $("$WG" show "$WG_IFACE" peers 2>/dev/null); do
actual=$("$WG" show "$WG_IFACE" allowed-ips 2>/dev/null | awk -v wanted="$key" '$1==wanted {{print $2; exit}}' || true)
overlay=$(printf '%s\\n' "$actual" | tr ',' '\\n' | awk '/^100\\.64\\.[0-9]+\\.[0-9]+\\/32$/ {{print; count++}} END {{exit count == 1 ? 0 : 1}}') || exit 74
extra=$(vpc_state_allowed_for_peer "$key") || exit 74
merged="$overlay"; [ -z "$extra" ] || merged="$merged,$extra"
"$WG" set "$WG_IFACE" peer "$key" allowed-ips "$merged" || exit 74
done
set_phase wg_merged
verify_routes_removed || exit 79
for family in inet bridge arp; do
"$NFT" list table "$family" "$TABLE" >/dev/null 2>&1 && exit 79 || true
done
grep -Fqx "$NFT_INCLUDE" "$NFT_MAIN" && exit 79 || true
if [ "$MODE" = networkmanager_nmcli ] && [ -n "$NMCLI" ]; then
"$NMCLI" connection show "$BRIDGE" >/dev/null 2>&1 && exit 79 || true
fi
[ ! -e "$STATE_FILE" ] && [ ! -e "$OWNER_FILE" ] || exit 79
if "$IP" link show dev "$BRIDGE" >/dev/null 2>&1; then exit 79; fi
printf '%s\\n' '__VC_VPC_REMOVED__=yes'
printf '%s\\n' '__VC_VPC_OWNER__={DATAPLANE_OWNER_MARKER}'
printf '%s\\n' '__VC_VPC_REVISION__={plan.desired_revision}'
printf '%s\\n' '__VC_VPC_DIGEST__={plan.desired_digest}'
set_phase committed
rm -rf "$TXN"
printf '%s\\n' '__VC_VPC_ROLLBACK__=none'
exit 0
'''
if len(script) > MAX_OUTPUT:
raise VpcDataplaneError("VPC 销毁脚本过大", 413, "vpc_script_too_large")
return script
# Real host calls use the strict, manager-aware builder. The old positional
# function remains above for compatibility with callers that only render a
# fixture.
render_apply_script = _render_apply_script_v2
def render_observe_script(plan: VpcDataplanePlan) -> str:
routes = _routes(plan)
r = plan.resources
proxy_file = f"/etc/virt-cloud/vpc-dataplane/{r.token}.proxy"
nft_file = f"/etc/nftables.d/virt-cloud-vpc-{r.token}.nft"
nft_include = f'include "{nft_file}"'
sysctl_file = f"/etc/sysctl.d/99-virt-cloud-vpc-{r.token}.conf"
ifup_fragment = f"/etc/network/interfaces.d/virt-cloud-vpc-{r.token}"
nm_profile_marker = DATAPLANE_OWNER_MARKER + " vpc=" + plan.vpc.vpc_id
route_lines = "\n".join(
f"{x['destination_cidr']}|{x['device']}|{x.get('via') or '-'}|{x.get('metric', 100)}"
for x in routes)
peer_lines = "\n".join(
f"{p.host_id}|{p.public_key}|{_allowed(plan, p)}"
for p in plan.peers)
fabric_peer_key_words = " ".join(sorted(plan.fabric_peer_keys or ()))
fabric_peer_key_arg = _q(fabric_peer_key_words)
fabric_peer_keys_provided = "yes" if plan.fabric_peer_keys is not None else "no"
route_marker = "__VC_VPC_OBS_ROUTES_" + hashlib.sha256(
route_lines.encode()).hexdigest()[:12]
peer_marker = "__VC_VPC_OBS_PEERS_" + hashlib.sha256(
peer_lines.encode()).hexdigest()[:12]
nft_marker = _nft_marker(plan)
# These comments are emitted by render_vpc_nft_rules. Requiring the full
# owned shape prevents a stale or foreign table being reported as active
# merely because the deterministic table name exists.
nft_suffixes = (
"anti-spoof-local-ip", "anti-spoof-egress",
"anti-spoof-public-to-bridge", "anti-spoof-public-to-fabric",
"anti-spoof-bridge-to-foreign", "anti-spoof-fabric-to-foreign",
"anti-spoof-public-to-vpc-bridge", "anti-spoof-public-to-vpc-fabric",
"anti-spoof-bridge-vpc-to-foreign", "anti-spoof-fabric-vpc-to-foreign",
"anti-spoof-wg-ingress", "anti-spoof-wg-egress",
"anti-spoof-wg-source", "member-egress-isolation",
"member-fabric-isolation",
"default-deny",
"wg-transit-deny",
"ipv6-drop-bridge", "ipv6-drop-fabric",
"ipv6-input-drop-bridge", "ipv6-input-drop-fabric",
"ipv6-output-drop-bridge",
"host-isolation", "host-isolation-wg", "host-isolation-out",
"host-isolation-wg-source", "host-isolation-out-wg",
"host-isolation-vpc-out", "host-isolation-vpc-out-wg",
"scope-bridge-in", "scope-bridge-out", "scope-fabric-in",
"scope-fabric-out",
*(f"acl:{acl.acl_id}" for acl in plan.vpc.acls),
)
nft_checks = "\n".join(
f''' printf '%s\\n' "$nft_text" | grep -Fq "comment \\\"$NFT_MARKER:{suffix}\\\"" || nft_verified=no'''
for suffix in nft_suffixes)
bridge_suffixes = (
"bridge-anti-spoof-ingress", "bridge-anti-spoof-arp",
"bridge-anti-spoof-arp-mac", "bridge-anti-spoof-ip",
"bridge-anti-spoof-arp-ip",
"bridge-multicast-drop", "bridge-limited-broadcast-drop",
"bridge-vpc-broadcast-drop",
"bridge-unknown-destination-drop",
"bridge-default-deny", "bridge-scope-accept",
"bridge-arp-accept", "bridge-non-ip-drop",
"bridge-anti-spoof-arp-input-mac", "bridge-anti-spoof-input",
"bridge-anti-spoof-input-ip", "bridge-ipv6-input-drop",
"bridge-arp-input-accept", "bridge-ip-input-accept",
"bridge-input-non-ip-drop",
*(f"acl:{acl.acl_id}" for acl in plan.vpc.acls),
)
bridge_checks = "\n".join(
f''' printf '%s\\n' "$bridge_text" | grep -Fq "comment \\\"$NFT_MARKER:{suffix}\\\"" || bridge_verified=no'''
for suffix in bridge_suffixes)
if plan.local_members:
bridge_observe = f'''bridge_text=$($NFT list table bridge "$TABLE" 2>/dev/null || true)
bridge_verified=yes
printf '%s\\n' "$bridge_text" | grep -Fq "table bridge $TABLE" || bridge_verified=no
printf '%s\\n' "$bridge_text" | grep -Fq "chain forward" || bridge_verified=no
printf '%s\\n' "$bridge_text" | grep -Fq "hook forward priority -190" || bridge_verified=no
printf '%s\\n' "$bridge_text" | grep -Fq "policy accept" || bridge_verified=no
printf '%s\\n' "$bridge_text" | grep -Fq "set local_member_ips" || bridge_verified=no
printf '%s\\n' "$bridge_text" | grep -Fq "set local_dest_ips" || bridge_verified=no
{bridge_checks}
if [ "$bridge_verified" = yes ]; then printf '%s\\n' '__VC_VPC_BRIDGE_NFT__=yes'; else printf '%s\\n' '__VC_VPC_BRIDGE_NFT__=no'; fi'''
else:
# An empty VPC must not silently retain a stale bridge-family table
# from a previous revision or host membership.
bridge_observe = '''
if $NFT list table bridge "$TABLE" >/dev/null 2>&1; then
printf '%s\\n' '__VC_VPC_BRIDGE_NFT__=no'
else
printf '%s\\n' '__VC_VPC_BRIDGE_NFT__=none'
fi'''.strip()
if plan.local_members:
arp_observe = f'''arp_text=$($NFT list table arp "$TABLE" 2>/dev/null || true)
arp_verified=yes
printf '%s\\n' "$arp_text" | grep -Fq "table arp $TABLE" || arp_verified=no
printf '%s\\n' "$arp_text" | grep -Fq "set proxy_ips" || arp_verified=no
printf '%s\\n' "$arp_text" | grep -Fq "chain output" || arp_verified=no
printf '%s\\n' "$arp_text" | grep -Fq "hook output priority -190" || arp_verified=no
printf '%s\\n' "$arp_text" | grep -Fq "policy accept" || arp_verified=no
printf '%s\\n' "$arp_text" | grep -Fq "comment \\\"$NFT_MARKER:arp-proxy-output\\\"" || arp_verified=no
if [ "$arp_verified" = yes ]; then printf '%s\\n' '__VC_VPC_ARP_NFT__=yes'; else printf '%s\\n' '__VC_VPC_ARP_NFT__=no'; fi'''
else:
arp_observe = '''
if $NFT list table arp "$TABLE" >/dev/null 2>&1; then
printf '%s\\n' '__VC_VPC_ARP_NFT__=no'
else
printf '%s\\n' '__VC_VPC_ARP_NFT__=none'
fi'''.strip()
route_extra_awk = "\n".join(
f' $0 == "{x["destination_cidr"]}|{x["device"]}|{x.get("via") or "-"}|{x.get("metric", 100)}|186" {{ next }}'
for x in routes)
route_checks = "\n".join(
f'''state=missing
if printf '%s\\n' "$route_records" | awk -F'|' -v wanted={_q(x['destination_cidr'])} -v expected_device={_q(x['device'])} -v expected_via={_q(x.get('via') or '-')} -v expected_metric={_q(str(x.get('metric', 100)))} '$1 == wanted && $2 == expected_device && $3 == expected_via && $4 == expected_metric && $5 == "186" {{found=1}} END {{exit found ? 0 : 1}}'; then
state=owned
fi
printf '%s\\n' '__VC_VPC_ROUTE__={x['destination_cidr']}|{x['device']}|'"$ROUTE_TABLE"'|{x.get('via') or '-'}|{x.get('metric', 100)}|'"$state"
'''.rstrip()
for x in routes)
if not route_checks:
route_checks = ": # no expected VPC routes"
rule_specs = (
(str(r.rule_priority), "-", plan.vpc.cidr, str(r.route_table)),
(str(r.rule_priority + 1), FABRIC_INTERFACE, plan.vpc.cidr,
str(r.route_table)),
)
rule_checks = "\n".join(
f'''rule_state=missing
if printf '%s\\n' "$rule_records" | awk -F'|' -v pref={_q(pref)} -v expected_iif={_q(iif)} -v expected_to={_q(destination)} -v expected_table={_q(table)} '$1 == pref && $2 == expected_iif && $3 == expected_to && $4 == expected_table {{found=1}} END {{exit found ? 0 : 1}}'; then
rule_state=owned
fi
printf '%s\\n' '__VC_VPC_RULE__={pref}|{iif}|{destination}|{table}|'"$rule_state"
'''.rstrip()
for pref, iif, destination, table in rule_specs)
return f'''#!/bin/sh
set -eu
PATH=/usr/sbin:/usr/bin:/sbin:/bin
IP=/usr/sbin/ip; [ -x "$IP" ] || IP=/usr/bin/ip
WG=/usr/bin/wg; [ -x "$WG" ] || WG=/usr/sbin/wg
NFT=/usr/sbin/nft; [ -x "$NFT" ] || NFT=/usr/bin/nft
FLOCK=/usr/bin/flock; [ -x "$FLOCK" ] || FLOCK=/usr/sbin/flock
LOCK=/run/virt-cloud/wg-fabric0.lock
if [ -L /run/virt-cloud ] || {{ [ -e /run/virt-cloud ] && [ ! -d /run/virt-cloud ]; }}; then exit 71; fi
mkdir -p /run/virt-cloud; chmod 700 /run/virt-cloud
[ ! -L "$LOCK" ] && {{ [ ! -e "$LOCK" ] || [ -f "$LOCK" ]; }} || exit 71
: >> "$LOCK"; chmod 600 "$LOCK"; exec 8>>"$LOCK"; "$FLOCK" -s 8
BRIDGE_CMD=
for p in /usr/sbin/bridge /usr/bin/bridge /sbin/bridge /bin/bridge; do
if [ -x "$p" ]; then BRIDGE_CMD=$p; break; fi
done
BRIDGE={_q(r.bridge)}; TABLE={_q(r.nft_table)}; ROUTE_TABLE={r.route_table}; WG_IFACE={_q(FABRIC_INTERFACE)}; OWNER={_q(r.owner_file)}; STATE={_q(r.state_file)}; PROXY_FILE={_q(proxy_file)}
NFT_FILE={_q(nft_file)}; NFT_INCLUDE={_q(nft_include)}; SYSCTL_FILE={_q(sysctl_file)}; IF_FRAGMENT={_q(ifup_fragment)}; NM_PROFILE_MARKER={_q(nm_profile_marker)}; NFT_MAIN=; NFT_MAIN_COUNT=0
NFT_MARKER={_q(nft_marker)}
owner=missing; [ -f "$OWNER" ] && owner=$(cat "$OWNER" 2>/dev/null || printf missing)
printf '%s\\n' "__VC_VPC_BRIDGE_OWNER__=$owner"
for nft_candidate in /etc/nftables.conf /etc/sysconfig/nftables.conf; do
if [ -f "$nft_candidate" ] && [ ! -L "$nft_candidate" ] && [ "$(stat -c '%u' "$nft_candidate" 2>/dev/null)" = 0 ]; then
case "$(stat -c '%a' "$nft_candidate" 2>/dev/null)" in
[0-7][0145][0145]) NFT_MAIN="$nft_candidate"; NFT_MAIN_COUNT=$((NFT_MAIN_COUNT + 1)) ;;
*) : ;;
esac
fi
done
persistence=no
nft_file_ok=no
if [ -f "$NFT_FILE" ] && [ ! -L "$NFT_FILE" ] && [ "$(stat -c '%u:%a' "$NFT_FILE" 2>/dev/null)" = 0:600 ] &&
grep -Fqx '# virt-cloud-vpc-dataplane-v1' "$NFT_FILE" &&
grep -Fqx '# vpc={plan.vpc.vpc_id} revision={plan.desired_revision} digest={plan.desired_digest}' "$NFT_FILE"; then
nft_file_ok=yes
fi
sysctl_file_ok=no
if [ -f "$SYSCTL_FILE" ] && [ ! -L "$SYSCTL_FILE" ] && [ "$(stat -c '%u:%a' "$SYSCTL_FILE" 2>/dev/null)" = 0:600 ] &&
grep -Fqx '# virt-cloud-vpc-dataplane-v1' "$SYSCTL_FILE" &&
grep -Fqx 'net.ipv4.conf.{r.bridge}.rp_filter = 2' "$SYSCTL_FILE" &&
grep -Fqx 'net.ipv4.conf.{r.bridge}.proxy_arp = 0' "$SYSCTL_FILE" &&
grep -Fqx 'net.ipv4.conf.{r.bridge}.proxy_arp_pvlan = 1' "$SYSCTL_FILE"; then
sysctl_file_ok=yes
fi
nm_owned=no
NMCLI=$(for p in /usr/bin/nmcli /usr/sbin/nmcli /bin/nmcli /sbin/nmcli; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
if [ -n "$NMCLI" ] && [ "$($NMCLI -t --escape no -f NAME connection show 2>/dev/null | awk -v wanted="$BRIDGE" '$0 == wanted {{count++}} END {{print count + 0}}')" = 1 ]; then
nm_stable=$($NMCLI -t --escape no -g connection.stable-id connection show "$BRIDGE" 2>/dev/null | sed -n '1p')
nm_type=$($NMCLI -t --escape no -g connection.type connection show "$BRIDGE" 2>/dev/null | sed -n '1p')
nm_iface=$($NMCLI -t --escape no -g connection.interface-name connection show "$BRIDGE" 2>/dev/null | sed -n '1p')
nm_method=$($NMCLI -t --escape no -g ipv4.method connection show "$BRIDGE" 2>/dev/null | sed -n '1p')
nm6_method=$($NMCLI -t --escape no -g ipv6.method connection show "$BRIDGE" 2>/dev/null | sed -n '1p')
if [ "$nm_stable" = "$NM_PROFILE_MARKER" ] && [ "$nm_type" = bridge ] && [ "$nm_iface" = "$BRIDGE" ] && [ "$nm_method" = disabled ] && [ "$nm6_method" = disabled ]; then nm_owned=yes; fi
fi
ifup_owned=no
if [ -f "$IF_FRAGMENT" ] && [ ! -L "$IF_FRAGMENT" ] && [ "$(stat -c '%u:%a' "$IF_FRAGMENT" 2>/dev/null)" = 0:600 ] &&
grep -Fqx '# {DATAPLANE_OWNER_MARKER} vpc={plan.vpc.vpc_id} revision={plan.desired_revision} digest={plan.desired_digest}' "$IF_FRAGMENT" &&
grep -Fqx 'auto {r.bridge}' "$IF_FRAGMENT" && grep -Fqx 'iface {r.bridge} inet manual' "$IF_FRAGMENT"; then
ifup_owned=yes
fi
if [ "$NFT_MAIN_COUNT" = 1 ] && [ "$nft_file_ok" = yes ] && [ "$sysctl_file_ok" = yes ] &&
grep -Fqx "$NFT_INCLUDE" "$NFT_MAIN" && "$NFT" -c -f "$NFT_MAIN" >/dev/null 2>&1 &&
( [ "$nm_owned" = yes ] && [ "$ifup_owned" = no ] ||
[ "$nm_owned" = no ] && [ "$ifup_owned" = yes ] ); then
persistence=yes
fi
printf '%s\\n' "__VC_VPC_PERSISTENCE__=$persistence"
if "$IP" -d link show dev "$BRIDGE" >/dev/null 2>&1; then printf '%s\\n' '__VC_VPC_BRIDGE__='"$BRIDGE"; else printf '%s\\n' '__VC_VPC_BRIDGE__=missing'; fi
bridge_boundary=no
bridge_shape=no; bridge_addr_ok=no; bridge_ports_ok=no; bridge_routes_ok=no; bridge_routes6_ok=no
if [ -n "$BRIDGE_CMD" ]; then
if bridge_shape_text=$("$IP" -d link show dev "$BRIDGE" 2>/dev/null) &&
printf '%s\n' "$bridge_shape_text" | grep -Eq '(^|[[:space:]])bridge[[:space:]]+(forward_delay|stp_state|hello_time|ageing_time)([[:space:]]|$)'; then
bridge_shape=yes
fi
if bridge_addr4=$("$IP" -o -4 addr show dev "$BRIDGE" 2>/dev/null) &&
bridge_addr6=$("$IP" -o -6 addr show dev "$BRIDGE" 2>/dev/null) &&
[ -z "$bridge_addr4$bridge_addr6" ]; then
bridge_addr_ok=yes
fi
if bridge_ports=$("$BRIDGE_CMD" link show master "$BRIDGE" 2>/dev/null); then
bridge_ports_ok=yes
fi
if bridge_routes=$("$IP" -o -4 route show table all dev "$BRIDGE" 2>/dev/null) &&
printf '%s\n' "$bridge_routes" | awk -v wanted_table="$ROUTE_TABLE" '
NF {{table="main"; proto=""; prefix=$1
for (i=1;i<=NF;i++) {{if ($i=="table") table=$(i+1); if ($i=="proto") proto=$(i+1)}}
if (table=="254") table="main"
if (prefix=="default" || table!=wanted_table || proto!="186") bad=1}}
END {{exit bad ? 1 : 0}}'; then
bridge_routes_ok=yes
fi
if bridge_routes6=$($IP -o -6 route show table all dev "$BRIDGE" 2>/dev/null) && [ -z "$bridge_routes6" ]; then
bridge_routes6_ok=yes
fi
fi
if [ "$bridge_shape" = yes ] && [ "$bridge_addr_ok" = yes ] &&
[ "$bridge_ports_ok" = yes ] && [ "$bridge_routes_ok" = yes ] &&
[ "$bridge_routes6_ok" = yes ]; then
bridge_bad=no
while IFS= read -r bridge_port_line; do
[ -n "$bridge_port_line" ] || continue
bridge_port=$(printf '%s\\n' "$bridge_port_line" | sed -n 's/^[0-9][0-9]*: \\([^:[:space:]]*\\).*/\\1/p')
case "$bridge_port" in
vnet[0-9]*|tap[0-9A-Za-z_.-]*|macvtap[0-9]*|fwpr[0-9A-Za-z_.-]*|fwln[0-9A-Za-z_.-]*) : ;;
*) bridge_bad=yes ;;
esac
done <<__VC_VPC_OBS_PORTS__
$bridge_ports
__VC_VPC_OBS_PORTS__
if [ "$bridge_bad" = no ]; then
bridge_boundary=yes
fi
fi
printf '%s\\n' "__VC_VPC_BRIDGE_BOUNDARY__=$bridge_boundary"
if [ -r "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp" ]; then printf '%s\\n' '__VC_VPC_PROXY_ARP__='"$(cat "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp")"; else printf '%s\\n' '__VC_VPC_PROXY_ARP__=missing'; fi
if [ -r "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp_pvlan" ]; then printf '%s\\n' '__VC_VPC_PROXY_ARP_PVLAN__='"$(cat "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp_pvlan")"; else printf '%s\\n' '__VC_VPC_PROXY_ARP_PVLAN__=missing'; fi
if [ -r "/proc/sys/net/ipv4/conf/$BRIDGE/rp_filter" ]; then printf '%s\\n' '__VC_VPC_RP_FILTER__='"$(cat "/proc/sys/net/ipv4/conf/$BRIDGE/rp_filter")"; else printf '%s\\n' '__VC_VPC_RP_FILTER__=missing'; fi
proxy_records=$($IP neigh show proxy dev "$BRIDGE" 2>/dev/null | awk '$1 ~ /^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$/ {{print $1}}' | LC_ALL=C sort -u || true)
printf '%s\\n' "$proxy_records" | while IFS= read -r proxy_ip; do
[ -n "$proxy_ip" ] && printf '%s\\n' "__VC_VPC_PROXY_IP__=$proxy_ip"
done
proxy_state=missing
if [ -f "$PROXY_FILE" ] && sed -n '1p' "$PROXY_FILE" | grep -Fqx '# virt-cloud-vpc-proxy-v1'; then
expected_proxy=$(sed '1d' "$PROXY_FILE" | LC_ALL=C sort -u)
if [ "$proxy_records" = "$expected_proxy" ]; then proxy_state=owned; fi
fi
printf '%s\\n' '__VC_VPC_PROXY_STATE__='"$proxy_state"
if "$NFT" list table inet "$TABLE" >/dev/null 2>&1; then printf '%s\\n' '__VC_VPC_NFT_TABLE__='"$TABLE"; else printf '%s\\n' '__VC_VPC_NFT_TABLE__=missing'; fi
if [ -f "$STATE" ] && grep -Fqx 'owner|{DATAPLANE_OWNER_MARKER}' "$STATE"; then
printf '%s\\n' '__VC_VPC_STATE_OWNER__=owned'
printf '%s\\n' "__VC_VPC_STATE_REVISION__=$(awk -F'|' '$1==\"revision\"{{print $2; exit}}' \"$STATE\")"
printf '%s\\n' "__VC_VPC_STATE_DIGEST__=$(awk -F'|' '$1==\"digest\"{{print $2; exit}}' \"$STATE\")"
else
printf '%s\\n' '__VC_VPC_STATE_OWNER__=missing'
fi
nft_text=$($NFT list table inet "$TABLE" 2>/dev/null || true)
nft_verified=yes
printf '%s\\n' "$nft_text" | grep -Fq "table inet $TABLE" || nft_verified=no
printf '%s\\n' "$nft_text" | grep -Fq "chain forward" || nft_verified=no
printf '%s\\n' "$nft_text" | grep -Fq "hook forward priority -190" || nft_verified=no
printf '%s\\n' "$nft_text" | grep -Fq "chain input" || nft_verified=no
printf '%s\\n' "$nft_text" | grep -Fq "chain output" || nft_verified=no
printf '%s\\n' "$nft_text" | grep -Fq "set member_ips" || nft_verified=no
printf '%s\\n' "$nft_text" | grep -Fq "set vpc_dest_ips" || nft_verified=no
{nft_checks}
if [ "$nft_verified" = yes ]; then
printf '%s\\n' '__VC_VPC_ACL_REVISION__={plan.desired_revision}'
printf '%s\\n' '__VC_VPC_ACL_DIGEST__={plan.desired_digest}'
printf '%s\\n' '__VC_VPC_ANTI_SPOOF__=yes'
else
printf '%s\\n' '__VC_VPC_ACL_REVISION__=missing'
printf '%s\\n' '__VC_VPC_ACL_DIGEST__=missing'
printf '%s\\n' '__VC_VPC_ANTI_SPOOF__=no'
fi
{bridge_observe}
{arp_observe}
route_records=$($IP -o -4 route show table "$ROUTE_TABLE" 2>/dev/null | awk '{{
destination=$1; device=""; via="-"; metric="100"; proto=""
for (i=1; i<=NF; i++) {{
if ($i == "dev" && i < NF) device=$(i+1)
if ($i == "via" && i < NF) via=$(i+1)
if ($i == "metric" && i < NF) metric=$(i+1)
if ($i == "proto" && i < NF) proto=$(i+1)
}}
if (destination != "") printf "%s|%s|%s|%s|%s\\n", destination, device, via, metric, proto
}}' || true)
rule_records=$($IP -o rule show 2>/dev/null | awk '{{
pref=$1; sub(/:$/, "", pref); iif="-"; destination=""; table=""
for (i=1;i<=NF;i++) {{
if ($i == "iif" && i < NF) iif=$(i+1)
if ($i == "to" && i < NF) destination=$(i+1)
if ($i == "lookup" && i < NF) table=$(i+1)
}}
if (pref != "" && iif != "" && destination != "" && table != "") printf "%s|%s|%s|%s\\n", pref, iif, destination, table
}}' || true)
{rule_checks}
route_extra=$(printf '%s\\n' "$route_records" | awk '{{
if (length($0) == 0) next
count++
{route_extra_awk}
extra=1
}} END {{ if (count != {len(routes)}) extra=1; print extra ? "yes" : "no" }}')
printf '%s\\n' "__VC_VPC_ROUTE_EXTRA__=$route_extra"
{route_checks}
while IFS='|' read -r peer_host key expected_allowed; do
[ -n "$peer_host" ] || continue
actual=$(printf '%s\\n' "$($WG show "$WG_IFACE" allowed-ips 2>/dev/null || true)" | awk -v wanted="$key" '$1 == wanted {{print $2; exit}}')
# ``wg show`` does not promise AllowedIPs order. Canonicalize both
# comma-separated lists before comparing, and retain the complete set in
# the marker. The Python adapter projects this to this VPC's tenant /32s
# after proving that no extra or missing entry exists.
expected_set=$(printf '%s\\n' "$expected_allowed" | tr ',' '\\n' | awk 'NF' | LC_ALL=C sort -u)
actual_set=$(printf '%s\\n' "$actual" | tr ',' '\\n' | awk 'NF' | LC_ALL=C sort -u)
allowed=$(printf '%s\\n' "$actual_set" | awk 'BEGIN {{ ORS=""; sep="" }} NF {{ printf "%s%s", sep, $0; sep="," }}')
state=missing
if [ -n "$actual_set" ] && [ "$actual_set" = "$expected_set" ]; then
state=owned
else
allowed=missing
fi
printf '%s\\n' "__VC_VPC_WG_ALLOWED__=$peer_host|$key|$allowed"
printf '%s\\n' "__VC_VPC_WG_STATE__=$peer_host|$state"
done <<'{peer_marker}'
{peer_lines}
{peer_marker}
expected_fabric_peer_keys={fabric_peer_key_arg}
wg_peer_set=unverified
if [ "{fabric_peer_keys_provided}" = yes ]; then
live_fabric_peer_keys=$("$WG" show "$WG_IFACE" peers 2>/dev/null || true)
wg_peer_set=exact
for live_peer_key in $live_fabric_peer_keys; do
case " $expected_fabric_peer_keys " in
*" $live_peer_key "*) : ;;
*) wg_peer_set=extra ;;
esac
done
for expected_peer_key in $expected_fabric_peer_keys; do
printf '%s\n' "$live_fabric_peer_keys" | grep -Fqx "$expected_peer_key" || wg_peer_set=missing
done
fi
printf '%s\n' "__VC_VPC_WG_PEER_SET__=$wg_peer_set"
exit 0
'''
def _markers(output: str) -> dict[str, Any]:
if not isinstance(output, str) or len(output) > MAX_OUTPUT:
raise VpcDataplaneError("VPC 回读输出过大", 502, "vpc_readback_invalid")
result: dict[str, Any] = {"routes": [], "wg_allowed_ips": [],
"wg_states": {}, "rules": [], "proxy_ips": []}
for line in output.splitlines():
if not line.startswith("__VC_VPC_") or "=" not in line:
continue
key, value = line.split("=", 1)
# A peer can legitimately carry many exact /32s from multiple VPCs;
# keep a bounded upper limit without truncating the authenticated set.
if len(value) > 65536:
raise VpcDataplaneError("VPC 回读字段过长", 502, "vpc_readback_invalid")
if key == "__VC_VPC_ROUTE__":
p = value.split("|")
if len(p) == 6:
result["routes"].append({
"destination_cidr": p[0], "device": p[1], "table": p[2],
"via": None if p[3] == "-" else p[3],
"metric": p[4], "owner": DATAPLANE_OWNER_MARKER,
"state": p[5],
})
elif key == "__VC_VPC_PROXY_IP__":
if re.fullmatch(r"[0-9]{1,3}(?:\.[0-9]{1,3}){3}", value):
try:
address = ipaddress.IPv4Address(value)
except ipaddress.AddressValueError:
continue
result["proxy_ips"].append(str(address))
elif key == "__VC_VPC_RULE__":
p = value.split("|")
if len(p) == 5:
result["rules"].append({"priority": p[0], "iif": p[1],
"destination_cidr": p[2],
"table": p[3], "state": p[4]})
elif key == "__VC_VPC_WG_ALLOWED__":
p = value.split("|")
if len(p) == 3:
result["wg_allowed_ips"].append({"peer_host_id": p[0], "public_key": p[1], "tenant_allowed_ips": [] if p[2] == "missing" else [x for x in p[2].split(",") if x]})
elif key == "__VC_VPC_WG_STATE__":
p = value.split("|")
if len(p) == 2 and p[1] in {"owned", "missing"}:
result["wg_states"][p[0]] = p[1]
else:
result[key.removeprefix("__VC_VPC_").removesuffix("__").lower()] = value
return result
def patch_private_interface_xml(xml: str | bytes, *, member: VpcMember,
resources: VpcDataplaneResources,
link_state: str = "up",
vpc_id: str | None = None,
desired_revision: int | None = None,
desired_digest: str | None = None) -> str:
"""Add/validate a private0 NIC on an explicit panel-owned bridge."""
if link_state not in {"up", "down"}:
raise VpcDataplaneError("私网网卡 link 状态无效")
private_mac = _mac(member.private_mac)
try:
root = ET.fromstring(xml)
except ET.ParseError as exc:
raise VpcDataplaneError("虚拟机 XML 无效", 409, "vpc_private_xml_invalid") from exc
devices = root.find("./devices")
if devices is None:
raise VpcDataplaneError("虚拟机 XML 缺少 devices", 409, "vpc_private_xml_invalid")
interfaces = list(devices.findall("interface"))
try:
public = select_public_interface(interfaces, allow_legacy_single=True)
# Legacy domains may have a single public NIC without an alias. Give
# it the stable role name before adding private0 so the next
# reconciliation does not mistake the public NIC for an unknown role.
alias = public.find("alias")
if alias is None:
alias = ET.SubElement(public, "alias")
alias.set("name", PUBLIC_INTERFACE_ALIAS)
private = select_private_interface(interfaces)
except ValueError as exc:
raise VpcDataplaneError(str(exc), 409, "vpc_private_xml_role_invalid") from exc
if private is None:
private = ET.SubElement(devices, "interface", {"type": "bridge"})
else:
if private.get("type") != "bridge":
raise VpcDataplaneError("private0 必须使用 bridge", 409, "vpc_private_xml_source_invalid")
existing_filters = private.findall("filterref")
if existing_filters and any(
item.get("filter") != "clean-traffic"
for item in existing_filters):
raise VpcDataplaneError("private0 nwfilter 不受面板管理", 409,
"vpc_private_xml_filter_invalid")
source = private.find("source")
if source is None or source.get("network") is not None or source.get("bridge") != resources.bridge:
raise VpcDataplaneError("private0 bridge 不匹配", 409, "vpc_private_xml_source_mismatch")
source = private.find("source")
if source is None:
source = ET.Element("source")
private.insert(0, source)
source.attrib.clear(); source.set("bridge", resources.bridge)
mac = private.find("mac")
if mac is None:
mac = ET.Element("mac"); private.insert(0, mac)
mac.set("address", private_mac)
alias = private.find("alias")
if alias is None:
alias = ET.SubElement(private, "alias")
alias.set("name", PRIVATE_INTERFACE)
model = private.find("model")
if model is None:
model = ET.SubElement(private, "model")
model.set("type", "virtio")
link = private.find("link")
if link is None:
link = ET.SubElement(private, "link")
link.set("state", link_state)
for item in list(private.findall("filterref")):
private.remove(item)
# The private L3 data plane owns anti-spoofing and ACL enforcement in its
# dedicated bridge/inet/arp nft tables. Do not attach the public
# libvirt ``clean-traffic`` nwfilter to private0; the role observer
# deliberately treats any private filter as an unsafe mixed policy.
metadata = root.find("metadata")
if metadata is None:
metadata = ET.Element("metadata"); root.insert(0, metadata)
managed = metadata.find(f"{{{PANEL_NS}}}managed")
if managed is None:
managed = ET.SubElement(metadata, f"{{{PANEL_NS}}}managed")
managed.set("private-member-id", member.member_id)
managed.set("private-ip-cidr", f"{member.private_ip}/32")
managed.set("private-network-bridge", resources.bridge)
managed.set("private-network-verified", "true")
managed.set("private-dataplane-owner", DATAPLANE_OWNER_MARKER)
if vpc_id is not None:
managed.set("private-vpc-id", vpc_id)
if desired_revision is not None:
managed.set("private-network-revision", str(desired_revision))
if desired_digest is not None:
managed.set("private-network-digest", desired_digest)
ET.register_namespace("panel", PANEL_NS)
return ET.tostring(root, encoding="unicode")
def _member_nics(
plan: VpcDataplanePlan,
xml_by_instance: Mapping[str, Any] | None,
) -> tuple[dict[str, Any], ...]:
def xml_details(xml: str, member: VpcMember, *, metadata: bool) -> tuple[bool, str | None]:
try:
root = ET.fromstring(xml)
private = select_private_interface(
list(root.findall("./devices/interface")))
source = private.find("source") if private is not None else None
mac = private.find("mac") if private is not None else None
ok = bool(
private and private.get("type") == "bridge"
and source is not None
and source.get("bridge") == plan.resources.bridge
and mac is not None
and mac.get("address", "").lower() == member.private_mac
# VPC isolation is provided by the dedicated bridge/inet/arp
# nft context. The public libvirt nwfilter is intentionally
# absent from private0.
and not private.findall("filterref")
)
link = private.find("link") if private is not None else None
link_state = link.get("state", "up").lower() if link is not None else "up"
if link_state not in {"up", "down"}:
return False, None
if not metadata:
return ok, link_state
managed = root.find(f"./metadata/{{{PANEL_NS}}}managed")
return bool(
ok and managed is not None
and managed.get("private-vpc-id") == plan.vpc.vpc_id
and managed.get("private-member-id") == member.member_id
and managed.get("private-ip-cidr") == f"{member.private_ip}/32"
and managed.get("private-network-revision")
== str(plan.desired_revision)
and managed.get("private-network-digest")
== plan.desired_digest
), link_state
except (ET.ParseError, ValueError):
return False, None
result = []
for member in plan.local_members:
verified = False
link_state: str | None = None
if xml_by_instance and member.instance_uuid in xml_by_instance:
value = xml_by_instance[member.instance_uuid]
if isinstance(value, Mapping):
inactive = value.get("inactive")
live = value.get("live")
inactive_ok = False
live_ok = True
inactive_link: str | None = None
live_link: str | None = None
if isinstance(inactive, str):
inactive_ok, inactive_link = xml_details(
inactive, member, metadata=True)
if live is not None:
if isinstance(live, str):
live_ok, live_link = xml_details(
live, member, metadata=False)
else:
live_ok = False
# A running domain is only healthy when the persistent and
# live role agree. This makes a stale config-only link state
# visible to the lifecycle gate instead of silently choosing
# whichever XML happened to be read first.
verified = inactive_ok and live_ok and (
live is None or inactive_link == live_link)
link_state = live_link if live is not None else inactive_link
elif isinstance(value, str):
verified, link_state = xml_details(value, member, metadata=True)
result.append({"member_id": member.member_id, "instance_uuid": member.instance_uuid,
"interface": PRIVATE_INTERFACE, "private_ip_cidr": f"{member.private_ip}/32",
"private_mac": member.private_mac, "xml_verified": verified,
"link_state": link_state})
return tuple(result)
class VpcPrivateHostAdapter:
"""Execute generated scripts through a pinned remote host manager."""
def __init__(self, manager: Any, profile: Mapping[str, Any] | None = None):
self.manager = manager
self.profile = dict(profile) if isinstance(profile, Mapping) else None
def _run(self, script: str) -> str:
current = self.profile if self.profile is not None else self.manager.current()
if not isinstance(current, Mapping) or not current:
raise VpcDataplaneError("宿主机尚未配置", 409, "vpc_host_not_configured")
if current.get("key_locked"):
raise VpcDataplaneError("宿主机 SSH 私钥仍锁定", 409, "vpc_host_key_locked")
argv = [getattr(self.manager, "ssh_binary", "ssh"), "-F", current["ssh_config_file"], "-T", "-o", "ConnectTimeout=10", current["ssh_target"], "/bin/sh -s"]
try:
result = self.manager._exec(argv, timeout=35, input_text=script)
except Exception as exc:
raise VpcDataplaneError("宿主机 VPC SSH 请求失败", 502, "vpc_host_connection_failed") from exc
if result.returncode != 0:
raise VpcDataplaneError("宿主机 VPC 命令失败", 502, "vpc_remote_command_failed")
return result.stdout
def _deployment_contract(self) -> tuple[str, str, str, str]:
"""Validate the same immutable host capability gate as Fabric."""
from .fabric import (
FabricError, _profile_ifdown_path, _profile_ifquery_path,
_profile_ifup_path, _profile_network_mode,
)
current = self.profile if self.profile is not None else self.manager.current()
try:
mode = _profile_network_mode(current)
except FabricError as exc:
raise VpcDataplaneError(
"宿主机网络管理器与 VPC 部署模式不匹配", 409,
"vpc_dataplane_network_manager_conflict") from exc
capabilities = current.get("capabilities") if isinstance(current, Mapping) else None
network = capabilities.get("network") if isinstance(capabilities, Mapping) else None
wireguard = capabilities.get("wireguard") if isinstance(capabilities, Mapping) else None
nftables = capabilities.get("nftables") if isinstance(capabilities, Mapping) else None
if (not isinstance(network, Mapping)
or not isinstance(wireguard, Mapping)
or not isinstance(nftables, Mapping)
or capabilities.get("vpc_ready") is not True
or network.get("firewall_manager") != "nftables"
or network.get("ip_forward") != 1
or nftables.get("ready") is not True
or wireguard.get("interface") != FABRIC_INTERFACE
or wireguard.get("interface_present") is not True
or wireguard.get("interface_is_wireguard") is not True
or wireguard.get("owned_by_panel") is not True):
raise VpcDataplaneError(
"宿主机 VPC 网络或防火墙能力未通过探测", 409,
"vpc_host_capability_missing")
return (
mode,
_profile_ifup_path(current),
_profile_ifdown_path(current),
_profile_ifquery_path(current),
)
def apply(self, plan: VpcDataplanePlan) -> dict[str, Any]:
mode, ifup_path, ifdown_path, ifquery_path = self._deployment_contract()
script = render_apply_script(
plan, deployment_mode=mode,
ifup_path=ifup_path, ifdown_path=ifdown_path,
ifquery_path=ifquery_path,
)
markers = _markers(self._run(script))
if markers.get("applied") != "yes":
raise VpcDataplaneError("宿主机未确认 VPC 应用", 502, "vpc_apply_unconfirmed")
return {"state": "applying", "readback": "pending", "rollback": "none", "owner": DATAPLANE_OWNER_MARKER}
def remove(self, plan: VpcDataplanePlan) -> dict[str, Any]:
"""Remove a panel-owned empty-host VPC and prove the absence.
The removal script is deliberately separate from ``apply``: it may
only run when the parsed plan has no local members and it validates
every persisted artifact/bridge owner before deleting anything.
A positive marker is required before the caller can advance a host
tombstone.
"""
mode, ifup_path, ifdown_path, ifquery_path = self._deployment_contract()
markers = _markers(self._run(render_remove_script(
plan, deployment_mode=mode, ifup_path=ifup_path,
ifdown_path=ifdown_path, ifquery_path=ifquery_path)))
if markers.get("removed") != "yes":
raise VpcDataplaneError(
"宿主机未确认 VPC 清理", 502, "vpc_remove_unconfirmed")
return {
"vpc_id": plan.vpc.vpc_id,
"host_id": plan.host_id,
"desired_revision": plan.desired_revision,
"observed_revision": plan.desired_revision,
"desired_digest": plan.desired_digest,
"observed_digest": plan.desired_digest,
"state": "active",
"removed": True,
"bridge": None,
"bridge_owner": None,
"member_nics": [],
"routes": [],
"wg_allowed_ips": [],
"nft_table": None,
"acl_revision": plan.desired_revision,
"anti_spoof": False,
"rp_filter": None,
"proxy_arp": None,
"proxy_arp_pvlan": None,
"proxy_ips": [],
"readback": "confirmed",
"rollback": markers.get("rollback", "none"),
"reason_code": None,
}
def observe(
self, plan: VpcDataplanePlan, *,
xml_by_instance: Mapping[str, Any] | None = None,
) -> dict[str, Any]:
self._deployment_contract()
parsed = _markers(self._run(render_observe_script(plan)))
expected_routes = {
(x["destination_cidr"], x["device"], str(x["table"]),
x.get("via"), str(x.get("metric", 100)))
for x in _routes(plan)
}
expected_proxy_ips = {
str(ipaddress.ip_network(x["destination_cidr"], strict=True).network_address)
for x in _routes(plan)
}
actual_proxy_ips = set(parsed.get("proxy_ips", []))
actual_routes = {
(x.get("destination_cidr"), x.get("device"), str(x.get("table")),
x.get("via"), str(x.get("metric", 100)))
for x in parsed.get("routes", [])
}
expected_wg_full = {
# The live `wg show` value includes the peer's overlay /32 as the
# first AllowedIP. Compare the complete cryptokey-routing set;
# project tenant entries only after that exact check.
p.host_id: (p.public_key, set(_allowed(plan, p).split(",")))
for p in plan.peers
}
actual_wg_full = {
x.get("peer_host_id"): (x.get("public_key"),
set(x.get("tenant_allowed_ips", [])))
for x in parsed.get("wg_allowed_ips", [])
}
# The live peer may carry /32s belonging to other VPCs sharing this
# Fabric. Verify the complete aggregate here, then expose only this
# VPC's tenant set to the PHP contract.
expected_tenant: dict[str, set[str]] = {
p.host_id: {m.private_ip + "/32" for m in plan.remote_members
if m.host_id == p.host_id}
for p in plan.peers
}
members_by_id = {m.member_id: m for m in plan.members}
for route in plan.vpc.routes:
target = (members_by_id.get(route.next_hop_member_id)
if route.next_hop_member_id is not None else None)
target_host = target.host_id if target is not None else route.next_hop_host_id
if target_host in expected_tenant:
expected_tenant[target_host].add(route.destination_cidr)
expected_wg = {
p.host_id: (p.public_key, expected_tenant.get(p.host_id, set()))
for p in plan.peers
}
actual_wg = {
host: (key, ips & expected_wg.get(host, (None, set()))[1])
for host, (key, ips) in actual_wg_full.items()
}
members = _member_nics(plan, xml_by_instance)
actual_route_state = {
(x.get("destination_cidr"), x.get("device"), str(x.get("table")),
x.get("via"), str(x.get("metric", 100)))
for x in parsed.get("routes", []) if x.get("state") == "owned"
}
expected_rules = {
(str(plan.resources.rule_priority), "-",
plan.vpc.cidr, str(plan.resources.route_table)),
(str(plan.resources.rule_priority + 1), FABRIC_INTERFACE,
plan.vpc.cidr, str(plan.resources.route_table)),
}
actual_rules = {
(str(x.get("priority")), x.get("iif"),
x.get("destination_cidr"), str(x.get("table")))
for x in parsed.get("rules", []) if x.get("state") == "owned"
}
expected_peer_states = {p.host_id: "owned" for p in plan.peers}
expected_bridge_nft = "yes" if plan.local_members else "none"
active = (parsed.get("bridge") == plan.resources.bridge
and parsed.get("bridge_owner") == DATAPLANE_OWNER_MARKER
and parsed.get("persistence") == "yes"
and parsed.get("bridge_boundary") == "yes"
and parsed.get("proxy_arp") == "0"
and parsed.get("proxy_arp_pvlan") == "1"
and parsed.get("proxy_state") == "owned"
and actual_proxy_ips == expected_proxy_ips
and parsed.get("rp_filter") == "2"
and parsed.get("nft_table") == plan.resources.nft_table
and parsed.get("state_owner") == "owned"
and str(parsed.get("state_revision")) == str(plan.desired_revision)
and parsed.get("state_digest") == plan.desired_digest
and str(parsed.get("acl_revision")) == str(plan.desired_revision)
and parsed.get("acl_digest") == plan.desired_digest
and parsed.get("anti_spoof") == "yes"
and parsed.get("bridge_nft") == expected_bridge_nft
and parsed.get("route_extra") == "no"
and expected_routes == actual_routes
and actual_routes == actual_route_state
and expected_rules == actual_rules
and expected_wg_full == actual_wg_full
and (plan.fabric_peer_keys is None
or parsed.get("wg_peer_set") == "exact")
and expected_peer_states == parsed.get("wg_states", {})
and parsed.get("arp_nft") == ("yes" if plan.local_members else "none")
and all(x["xml_verified"] for x in members))
projected_wg = tuple(
dict(item, tenant_allowed_ips=sorted(
expected_wg[item.get("peer_host_id")][1]
& set(item.get("tenant_allowed_ips", []))))
for item in parsed.get("wg_allowed_ips", [])
if item.get("peer_host_id") in expected_wg
and expected_wg[item.get("peer_host_id")][1]
)
rp_filter = int(parsed["rp_filter"]) if str(parsed.get("rp_filter", "")).isdigit() else None
proxy_arp = int(parsed["proxy_arp"]) if str(parsed.get("proxy_arp", "")).isdigit() else None
proxy_arp_pvlan = (int(parsed["proxy_arp_pvlan"])
if str(parsed.get("proxy_arp_pvlan", "")).isdigit()
else None)
return VpcDataplaneObservation(
plan.vpc.vpc_id, plan.host_id, plan.desired_revision,
plan.desired_revision if active else 0, plan.desired_digest,
plan.desired_digest if active else None,
"active" if active else "degraded", plan.resources.bridge,
parsed.get("bridge_owner"), members, tuple(parsed.get("routes", [])),
projected_wg, parsed.get("nft_table"),
plan.desired_revision if active else None,
parsed.get("anti_spoof") == "yes", rp_filter, proxy_arp,
proxy_arp_pvlan,
tuple(sorted(actual_proxy_ips)), "confirmed" if active else "mismatch",
"none", None if active else "vpc_private_dataplane_readback_mismatch",
).as_dict()
__all__ = ["DATAPLANE_OWNER_MARKER", "FABRIC_INTERFACE", "PRIVATE_INTERFACE", "VpcDataplaneError", "VpcFabricPeer", "VpcDataplaneResources", "VpcDataplanePlan", "VpcDataplaneObservation", "derive_private_mac", "resource_names", "parse_vpc_dataplane_plan", "render_vpc_nft_rules", "render_apply_script", "render_remove_script", "render_observe_script", "patch_private_interface_xml", "VpcPrivateHostAdapter"]