You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
2975 lines
156 KiB
Python
2975 lines
156 KiB
Python
"""Panel-owned VPC private L3 data-plane primitives.
|
|
|
|
This module is deliberately independent from :mod:`virsh_panel.fabric`.
|
|
Fabric owns the host overlay (``wg-fabric0``); this boundary owns only a
|
|
VPC's bridge, policy route table, tenant AllowedIPs and nftables table. The
|
|
pure builders are usable in unit tests and the host adapter runs one fixed
|
|
shell transaction through a pinned remote manager.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
import hashlib
|
|
import ipaddress
|
|
import re
|
|
import shlex
|
|
import xml.etree.ElementTree as ET
|
|
from dataclasses import dataclass
|
|
from typing import Any, Mapping, Sequence
|
|
|
|
from .network_roles import (
|
|
PRIVATE_INTERFACE_ALIAS,
|
|
PUBLIC_INTERFACE_ALIAS,
|
|
select_private_interface,
|
|
select_public_interface,
|
|
)
|
|
from .vpc import VpcError, VpcMember, VpcPlan, parse_vpc_plan
|
|
|
|
DATAPLANE_SCHEMA_VERSION = 1
|
|
DATAPLANE_OWNER_MARKER = "virt-cloud-vpc-dataplane-v1"
|
|
FABRIC_OWNER_MARKER = "virt-cloud-wg-fabric-v1"
|
|
FABRIC_INTERFACE = "wg-fabric0"
|
|
FABRIC_LOCK = "/run/virt-cloud/wg-fabric0.lock"
|
|
PRIVATE_INTERFACE = PRIVATE_INTERFACE_ALIAS
|
|
PANEL_NS = "urn:virsh-panel:1"
|
|
BRIDGE_PREFIX = "vcv-"
|
|
NFT_PREFIX = "virt_cloud_vpc_"
|
|
ROUTE_TABLE_MIN, ROUTE_TABLE_MAX = 10000, 19999
|
|
RULE_PRIORITY_MIN, RULE_PRIORITY_MAX = 20000, 29999
|
|
MAX_OUTPUT = 262144
|
|
_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,63}\Z")
|
|
_HOST = re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]{0,63}\Z")
|
|
_MAC = re.compile(r"[0-9a-fA-F]{2}(?::[0-9a-fA-F]{2}){5}\Z")
|
|
_KEY = re.compile(r"[A-Za-z0-9+/]{42}==\Z|[A-Za-z0-9+/]{43}=\Z")
|
|
_DIGEST = re.compile(r"[0-9a-f]{64}\Z")
|
|
|
|
|
|
class VpcDataplaneError(Exception):
|
|
"""Safe, typed error at the VPC host boundary."""
|
|
|
|
def __init__(self, message: str, status: int = 400,
|
|
error_code: str = "vpc_dataplane_invalid") -> None:
|
|
super().__init__(message)
|
|
self.status = status
|
|
self.error_code = error_code
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class VpcFabricPeer:
|
|
host_id: str
|
|
public_key: str
|
|
overlay_ip: str
|
|
# Tenant /32s from every panel VPC sharing this Fabric peer. WireGuard
|
|
# has one AllowedIPs list per peer, so a VPC must merge this typed set
|
|
# instead of replacing another active VPC's entries.
|
|
tenant_allowed_ips: tuple[str, ...] = ()
|
|
|
|
@property
|
|
def overlay_cidr(self) -> str:
|
|
return f"{self.overlay_ip}/32"
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class VpcDataplaneResources:
|
|
token: str
|
|
bridge: str
|
|
nft_table: str
|
|
route_table: int
|
|
rule_priority: int
|
|
owner_file: str
|
|
state_file: str
|
|
lock_file: str
|
|
transaction_dir: str
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class VpcDataplanePlan:
|
|
vpc: VpcPlan
|
|
host_id: str
|
|
members: tuple[VpcMember, ...]
|
|
peers: tuple[VpcFabricPeer, ...]
|
|
resources: VpcDataplaneResources
|
|
# The authenticated full Fabric peer set, excluding this host. It is
|
|
# separate from ``peers`` because one VPC can use only a subset of the
|
|
# peers present on a shared Fabric.
|
|
fabric_peer_keys: frozenset[str] | None = None
|
|
|
|
@property
|
|
def desired_revision(self) -> int:
|
|
return self.vpc.desired_revision
|
|
|
|
@property
|
|
def desired_digest(self) -> str:
|
|
return self.vpc.desired_digest
|
|
|
|
@property
|
|
def local_members(self) -> tuple[VpcMember, ...]:
|
|
return tuple(m for m in self.members if m.host_id == self.host_id)
|
|
|
|
@property
|
|
def remote_members(self) -> tuple[VpcMember, ...]:
|
|
return tuple(m for m in self.members if m.host_id != self.host_id)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class VpcDataplaneObservation:
|
|
vpc_id: str
|
|
host_id: str
|
|
desired_revision: int
|
|
observed_revision: int
|
|
desired_digest: str
|
|
observed_digest: str | None
|
|
state: str
|
|
bridge: str
|
|
bridge_owner: str | None
|
|
member_nics: tuple[dict[str, Any], ...]
|
|
routes: tuple[dict[str, Any], ...]
|
|
wg_allowed_ips: tuple[dict[str, Any], ...]
|
|
nft_table: str | None
|
|
acl_revision: int | None
|
|
anti_spoof: bool
|
|
rp_filter: int | None
|
|
proxy_arp: int | None
|
|
proxy_arp_pvlan: int | None
|
|
proxy_ips: tuple[str, ...]
|
|
readback: str
|
|
rollback: str
|
|
reason_code: str | None = None
|
|
removed: bool = False
|
|
|
|
def as_dict(self) -> dict[str, Any]:
|
|
return {
|
|
"vpc_id": self.vpc_id, "host_id": self.host_id,
|
|
"desired_revision": self.desired_revision,
|
|
"observed_revision": self.observed_revision,
|
|
"desired_digest": self.desired_digest,
|
|
"observed_digest": self.observed_digest,
|
|
"state": self.state, "bridge": self.bridge,
|
|
"bridge_owner": self.bridge_owner,
|
|
"member_nics": [dict(x) for x in self.member_nics],
|
|
"routes": [dict(x) for x in self.routes],
|
|
"wg_allowed_ips": [dict(x) for x in self.wg_allowed_ips],
|
|
"nft_table": self.nft_table, "acl_revision": self.acl_revision,
|
|
"anti_spoof": self.anti_spoof, "rp_filter": self.rp_filter,
|
|
"proxy_arp": self.proxy_arp,
|
|
"proxy_arp_pvlan": self.proxy_arp_pvlan,
|
|
"proxy_ips": list(self.proxy_ips),
|
|
"readback": self.readback,
|
|
"rollback": self.rollback, "reason_code": self.reason_code,
|
|
"removed": self.removed,
|
|
}
|
|
|
|
|
|
def _id(value: Any, label: str = "标识") -> str:
|
|
if not isinstance(value, str) or _ID.fullmatch(value) is None:
|
|
raise VpcDataplaneError(f"{label}无效")
|
|
return value
|
|
|
|
|
|
def _host(value: Any) -> str:
|
|
if not isinstance(value, str) or _HOST.fullmatch(value) is None:
|
|
raise VpcDataplaneError("宿主机标识无效")
|
|
return value
|
|
|
|
|
|
def _key(value: Any) -> str:
|
|
if not isinstance(value, str) or _KEY.fullmatch(value) is None:
|
|
raise VpcDataplaneError("WireGuard 公钥无效")
|
|
try:
|
|
raw = base64.b64decode(value, validate=True)
|
|
except Exception as exc:
|
|
raise VpcDataplaneError("WireGuard 公钥无效") from exc
|
|
if len(raw) != 32 or not any(raw):
|
|
raise VpcDataplaneError("WireGuard 公钥无效")
|
|
return value
|
|
|
|
|
|
def _mac(value: Any, label: str = "私网 MAC") -> str:
|
|
if not isinstance(value, str) or _MAC.fullmatch(value) is None:
|
|
raise VpcDataplaneError(f"{label}无效")
|
|
octets = [int(x, 16) for x in value.split(":")]
|
|
if octets[0] & 1 or not any(octets):
|
|
raise VpcDataplaneError(f"{label}无效")
|
|
return value.lower()
|
|
|
|
|
|
def derive_private_mac(vpc_id: str, member_id: str) -> str:
|
|
"""Stable locally-administered unicast MAC for legacy plans without MAC."""
|
|
digest = hashlib.sha256(f"virt-cloud:vpc-private0:{vpc_id}:{member_id}".encode()).digest()
|
|
first = (digest[0] & 0xfc) | 0x02
|
|
return ":".join(f"{x:02x}" for x in bytes([first]) + digest[1:6])
|
|
|
|
|
|
def resource_names(vpc_id: str) -> VpcDataplaneResources:
|
|
vpc_id = _id(vpc_id, "VPC")
|
|
digest = hashlib.sha256(vpc_id.encode()).hexdigest()
|
|
token = digest[:11]
|
|
span = ROUTE_TABLE_MAX - ROUTE_TABLE_MIN + 1
|
|
# Each VPC owns two policy rules (private bridge and Fabric ingress).
|
|
# Allocate an aligned pair so ``base + 1`` can never be another VPC's
|
|
# base slot. The coordinator still checks collisions across records and
|
|
# the host transaction rejects a foreign rule already occupying either
|
|
# slot, but a single-slot hash would make those checks order-dependent.
|
|
rule_span = (RULE_PRIORITY_MAX - RULE_PRIORITY_MIN + 1) // 2
|
|
rule_base = RULE_PRIORITY_MIN + 2 * (int(digest[20:28], 16) % rule_span)
|
|
return VpcDataplaneResources(
|
|
token=token, bridge=BRIDGE_PREFIX + token,
|
|
nft_table=NFT_PREFIX + digest[:12],
|
|
route_table=ROUTE_TABLE_MIN + int(digest[12:20], 16) % span,
|
|
rule_priority=rule_base,
|
|
owner_file=f"/etc/virt-cloud/vpc-dataplane/{token}.owner",
|
|
state_file=f"/etc/virt-cloud/vpc-dataplane/{token}.state",
|
|
lock_file=f"/run/virt-cloud/vpc-{token}.lock",
|
|
transaction_dir=f"/etc/virt-cloud/.vpc-dataplane-{token}.txn",
|
|
)
|
|
|
|
|
|
def _expected_peer_tenant_ips(vpc: VpcPlan) -> dict[str, frozenset[str]]:
|
|
"""Return the exact /32 set a peer may route for this VPC.
|
|
|
|
The WireGuard peer list is a cryptokey-routing boundary. RFC1918 alone
|
|
is not an ownership check: an otherwise valid private /32 from another
|
|
VPC would create a cross-tenant route. The coordinator may pass a
|
|
trusted aggregate for a Fabric shared by several VPCs; the default here
|
|
covers a standalone plan and keeps direct parser callers fail-closed.
|
|
"""
|
|
members = {item.member_id: item for item in vpc.members}
|
|
allowed: dict[str, set[str]] = {host: set() for host in vpc.host_ids}
|
|
for item in vpc.members:
|
|
allowed.setdefault(item.host_id, set()).add(f"{item.private_ip}/32")
|
|
for route in vpc.routes:
|
|
target = (members.get(route.next_hop_member_id)
|
|
if route.next_hop_member_id is not None else None)
|
|
target_host = target.host_id if target is not None else route.next_hop_host_id
|
|
if target_host is not None:
|
|
allowed.setdefault(target_host, set()).add(route.destination_cidr)
|
|
return {host: frozenset(values) for host, values in allowed.items()}
|
|
|
|
|
|
def _parse_peers(
|
|
raw: Any,
|
|
host_ids: frozenset[str],
|
|
local: str,
|
|
expected_tenant_ips: Mapping[str, frozenset[str]] | None = None,
|
|
) -> tuple[VpcFabricPeer, ...]:
|
|
if raw is None:
|
|
raw = []
|
|
if not isinstance(raw, (list, tuple)) or len(raw) > 64:
|
|
raise VpcDataplaneError("Fabric peer 列表无效")
|
|
peers: list[VpcFabricPeer] = []
|
|
seen_hosts: set[str] = set()
|
|
seen_keys: set[str] = set()
|
|
seen_ips: set[str] = set()
|
|
seen_tenant_ips: set[str] = set()
|
|
for item in raw:
|
|
if not isinstance(item, Mapping) or set(item) - {"host_id", "public_key", "overlay_ip", "overlay_ip_cidr", "tenant_allowed_ips"}:
|
|
raise VpcDataplaneError("Fabric peer 字段无效")
|
|
host_id = _host(item.get("host_id"))
|
|
if host_id == local or host_id not in host_ids or host_id in seen_hosts:
|
|
raise VpcDataplaneError("Fabric peer 宿主机无效或重复")
|
|
public_key = _key(item.get("public_key"))
|
|
if public_key in seen_keys:
|
|
raise VpcDataplaneError("Fabric peer 公钥重复")
|
|
raw_ip = item.get("overlay_ip", item.get("overlay_ip_cidr"))
|
|
try:
|
|
interface = ipaddress.ip_interface(raw_ip)
|
|
except (ValueError, TypeError) as exc:
|
|
raise VpcDataplaneError("Fabric peer overlay 地址无效") from exc
|
|
fabric_overlay = ipaddress.IPv4Network("100.64.0.0/10")
|
|
if (not isinstance(interface, ipaddress.IPv4Interface)
|
|
or interface.network.prefixlen != 32
|
|
or interface.ip not in fabric_overlay
|
|
or interface.ip.is_loopback
|
|
or interface.ip.is_link_local):
|
|
raise VpcDataplaneError("Fabric peer overlay 地址无效")
|
|
overlay_ip = str(interface.ip)
|
|
if overlay_ip in seen_ips:
|
|
raise VpcDataplaneError("Fabric peer overlay 地址重复")
|
|
raw_tenant = item.get("tenant_allowed_ips", ())
|
|
if not isinstance(raw_tenant, (list, tuple)) or len(raw_tenant) > 4096:
|
|
raise VpcDataplaneError("Fabric peer tenant AllowedIPs 无效")
|
|
tenant: set[str] = set()
|
|
for raw_ip in raw_tenant:
|
|
if not isinstance(raw_ip, str) or not raw_ip.endswith("/32"):
|
|
raise VpcDataplaneError("Fabric peer tenant AllowedIPs 必须是 /32")
|
|
try:
|
|
network = ipaddress.ip_network(raw_ip, strict=True)
|
|
except ValueError as exc:
|
|
raise VpcDataplaneError("Fabric peer tenant AllowedIPs 无效") from exc
|
|
if network.prefixlen != 32:
|
|
raise VpcDataplaneError("Fabric peer tenant AllowedIPs 必须是 /32")
|
|
# Tenant routes are panel-private RFC1918 /32s. In particular,
|
|
# never accept a public prefix or a Fabric overlay address from
|
|
# an otherwise authenticated payload: that would turn a peer's
|
|
# cryptokey routing table into a host-wide egress route.
|
|
private_ranges = (
|
|
ipaddress.IPv4Network("10.0.0.0/8"),
|
|
ipaddress.IPv4Network("172.16.0.0/12"),
|
|
ipaddress.IPv4Network("192.168.0.0/16"),
|
|
)
|
|
if not any(network.subnet_of(item) for item in private_ranges):
|
|
raise VpcDataplaneError("Fabric peer tenant AllowedIPs 必须位于 RFC1918 私网")
|
|
tenant_ip = str(network)
|
|
if tenant_ip in seen_tenant_ips:
|
|
raise VpcDataplaneError("Fabric peer tenant AllowedIPs 重复", 409,
|
|
"vpc_tenant_allowed_ip_conflict")
|
|
seen_tenant_ips.add(tenant_ip)
|
|
tenant.add(tenant_ip)
|
|
if expected_tenant_ips is not None:
|
|
expected = set(expected_tenant_ips.get(host_id, frozenset()))
|
|
if tenant != expected:
|
|
raise VpcDataplaneError(
|
|
"Fabric peer tenant AllowedIPs 超出 VPC 所有权闭集", 409,
|
|
"vpc_tenant_allowed_ip_unowned")
|
|
seen_hosts.add(host_id); seen_keys.add(public_key); seen_ips.add(overlay_ip)
|
|
peers.append(VpcFabricPeer(host_id, public_key, overlay_ip,
|
|
tuple(sorted(tenant))))
|
|
return tuple(sorted(peers, key=lambda p: p.host_id))
|
|
|
|
|
|
def parse_vpc_dataplane_plan(
|
|
value: Mapping[str, Any], host_id: str, *,
|
|
fabric_peers: Sequence[Mapping[str, Any]] | None = None,
|
|
trusted_tenant_allowed_ips: Mapping[str, Sequence[str]] | None = None,
|
|
fabric_peer_keys: Sequence[str] | None = None,
|
|
) -> VpcDataplanePlan:
|
|
"""Parse existing VPC DTO plus an optional engine-only Fabric snapshot."""
|
|
if not isinstance(value, Mapping):
|
|
raise VpcDataplaneError("VPC 数据面计划必须是对象")
|
|
host_id = _host(host_id)
|
|
payload = dict(value)
|
|
embedded = payload.pop("fabric_peers", None)
|
|
if embedded is None:
|
|
embedded = payload.pop("peers", None)
|
|
else:
|
|
payload.pop("peers", None)
|
|
try:
|
|
vpc = parse_vpc_plan(payload)
|
|
except VpcError as exc:
|
|
raise VpcDataplaneError(str(exc), exc.status, exc.error_code) from exc
|
|
if host_id not in vpc.host_ids:
|
|
raise VpcDataplaneError("VPC 宿主机不在参与列表中", 409, "vpc_host_not_member")
|
|
members: list[VpcMember] = []
|
|
seen_macs: set[str] = set()
|
|
for member in vpc.members:
|
|
private_mac = _mac(member.private_mac) if member.private_mac else derive_private_mac(vpc.vpc_id, member.member_id)
|
|
if private_mac in seen_macs:
|
|
raise VpcDataplaneError("VPC 成员私网 MAC 重复", 409, "vpc_private_mac_conflict")
|
|
seen_macs.add(private_mac)
|
|
members.append(VpcMember(member.member_id, member.instance_uuid,
|
|
member.host_id, member.private_ip,
|
|
member.subnet_id, private_mac))
|
|
if trusted_tenant_allowed_ips is None:
|
|
expected_tenant_ips = _expected_peer_tenant_ips(vpc)
|
|
else:
|
|
if not isinstance(trusted_tenant_allowed_ips, Mapping):
|
|
raise VpcDataplaneError("Fabric peer tenant 所有权闭集无效")
|
|
expected_tenant_ips = {}
|
|
remote_hosts = vpc.host_ids - {host_id}
|
|
if set(trusted_tenant_allowed_ips) - remote_hosts:
|
|
raise VpcDataplaneError("Fabric peer tenant 所有权宿主机无效")
|
|
for peer_host, values in trusted_tenant_allowed_ips.items():
|
|
if not isinstance(peer_host, str) or peer_host not in remote_hosts:
|
|
raise VpcDataplaneError("Fabric peer tenant 所有权宿主机无效")
|
|
if not isinstance(values, (list, tuple, set, frozenset)):
|
|
raise VpcDataplaneError("Fabric peer tenant 所有权闭集无效")
|
|
canonical: set[str] = set()
|
|
for value in values:
|
|
if not isinstance(value, str):
|
|
raise VpcDataplaneError("Fabric peer tenant 所有权闭集无效")
|
|
try:
|
|
network = ipaddress.ip_network(value, strict=True)
|
|
except ValueError as exc:
|
|
raise VpcDataplaneError("Fabric peer tenant 所有权闭集无效") from exc
|
|
if (not isinstance(network, ipaddress.IPv4Network)
|
|
or network.prefixlen != 32
|
|
or not any(network.subnet_of(block) for block in (
|
|
ipaddress.IPv4Network("10.0.0.0/8"),
|
|
ipaddress.IPv4Network("172.16.0.0/12"),
|
|
ipaddress.IPv4Network("192.168.0.0/16")))):
|
|
raise VpcDataplaneError("Fabric peer tenant 所有权闭集无效")
|
|
canonical.add(str(network))
|
|
expected_tenant_ips[peer_host] = frozenset(canonical)
|
|
peers = _parse_peers(
|
|
embedded if fabric_peers is None else fabric_peers,
|
|
vpc.host_ids, host_id, expected_tenant_ips)
|
|
remote_hosts = {m.host_id for m in members if m.host_id != host_id}
|
|
if peers and remote_hosts - {p.host_id for p in peers}:
|
|
raise VpcDataplaneError("跨宿主机成员缺少 Fabric peer", 409, "vpc_fabric_peer_missing")
|
|
authoritative_keys: frozenset[str] | None = None
|
|
if fabric_peer_keys is not None:
|
|
if not isinstance(fabric_peer_keys, (list, tuple, set, frozenset)):
|
|
raise VpcDataplaneError("Fabric peer 公钥闭集无效", 409,
|
|
"vpc_fabric_peer_set_invalid")
|
|
if len(fabric_peer_keys) > 64:
|
|
raise VpcDataplaneError("Fabric peer 公钥闭集过大", 409,
|
|
"vpc_fabric_peer_set_invalid")
|
|
canonical_keys: set[str] = set()
|
|
for raw_key in fabric_peer_keys:
|
|
key = _key(raw_key)
|
|
if key in canonical_keys:
|
|
raise VpcDataplaneError("Fabric peer 公钥重复", 409,
|
|
"vpc_fabric_peer_set_invalid")
|
|
canonical_keys.add(key)
|
|
if not {peer.public_key for peer in peers}.issubset(canonical_keys):
|
|
raise VpcDataplaneError("VPC peer 不属于 Fabric 公钥闭集", 409,
|
|
"vpc_fabric_peer_set_invalid")
|
|
authoritative_keys = frozenset(canonical_keys)
|
|
return VpcDataplanePlan(
|
|
vpc, host_id, tuple(members), peers, resource_names(vpc.vpc_id),
|
|
authoritative_keys)
|
|
|
|
|
|
def _routes(plan: VpcDataplanePlan) -> tuple[dict[str, Any], ...]:
|
|
members = {m.member_id: m for m in plan.members}
|
|
result: list[dict[str, Any]] = []
|
|
seen: set[str] = set()
|
|
for member in plan.members:
|
|
destination = f"{member.private_ip}/32"
|
|
seen.add(destination)
|
|
result.append({"destination_cidr": destination,
|
|
"device": plan.resources.bridge if member.host_id == plan.host_id else FABRIC_INTERFACE,
|
|
"table": plan.resources.route_table,
|
|
"owner": DATAPLANE_OWNER_MARKER,
|
|
"metric": 100,
|
|
"via": None,
|
|
"member_id": member.member_id})
|
|
for route in plan.vpc.routes:
|
|
destination = route.destination_cidr
|
|
network = ipaddress.ip_network(destination)
|
|
if network.prefixlen != 32:
|
|
raise VpcDataplaneError(
|
|
"VPC 数据面只支持成员精确 /32 路由", 409,
|
|
"vpc_route_not_exact")
|
|
if any(ipaddress.IPv4Address(m.private_ip) in network
|
|
and destination != f"{m.private_ip}/32" for m in plan.members):
|
|
raise VpcDataplaneError("VPC route 不能覆盖成员精确 /32", 409, "vpc_route_member_overlap")
|
|
if destination in seen:
|
|
raise VpcDataplaneError("VPC route 与成员地址重复", 409, "vpc_route_conflict")
|
|
next_host = route.next_hop_host_id
|
|
if route.next_hop_member_id is not None:
|
|
target = members.get(route.next_hop_member_id)
|
|
if target is None:
|
|
raise VpcDataplaneError("VPC route next hop member 不存在", 409)
|
|
next_host = target.host_id
|
|
if next_host is None:
|
|
raise VpcDataplaneError("VPC route next hop 缺失")
|
|
target_members = [item for item in plan.members
|
|
if item.host_id == next_host]
|
|
if not target_members:
|
|
raise VpcDataplaneError("VPC route next hop 宿主机没有成员", 409,
|
|
"vpc_route_next_hop_missing")
|
|
# A host-only next hop is ambiguous when several VMs share that host.
|
|
# Require an explicit member in that case; using a bare bridge route
|
|
# would ARP for the service address and could leak traffic to an
|
|
# unrelated guest.
|
|
if route.next_hop_member_id is None and len(target_members) != 1:
|
|
raise VpcDataplaneError("VPC route 宿主机 next hop 不明确", 409,
|
|
"vpc_route_next_hop_ambiguous")
|
|
target = (members.get(route.next_hop_member_id)
|
|
if route.next_hop_member_id is not None else target_members[0])
|
|
if target is None:
|
|
raise VpcDataplaneError("VPC route next hop member 不存在", 409,
|
|
"vpc_route_next_hop_missing")
|
|
is_local = next_host == plan.host_id
|
|
if not is_local and next_host not in {peer.host_id for peer in plan.peers}:
|
|
raise VpcDataplaneError("VPC route next hop 缺少 Fabric peer", 409,
|
|
"vpc_fabric_peer_missing")
|
|
seen.add(destination)
|
|
result.append({"destination_cidr": destination,
|
|
"device": plan.resources.bridge if is_local else FABRIC_INTERFACE,
|
|
"table": plan.resources.route_table,
|
|
"owner": DATAPLANE_OWNER_MARKER,
|
|
# A local host must forward the service /32 through
|
|
# the selected VM. Across the overlay the destination
|
|
# itself is a WireGuard AllowedIP, so no `via` is used.
|
|
"via": target.private_ip if is_local else None,
|
|
"metric": route.metric,
|
|
"route_id": route.route_id})
|
|
return tuple(result)
|
|
|
|
|
|
def _q(value: Any) -> str:
|
|
return shlex.quote(str(value))
|
|
|
|
|
|
def _nft_proto(acl: Any) -> str:
|
|
if acl.protocol == "any":
|
|
return ""
|
|
if acl.protocol == "icmp":
|
|
return " ip protocol icmp"
|
|
suffix = f" ip protocol {acl.protocol}"
|
|
if acl.port_from is not None:
|
|
ports = str(acl.port_from) if acl.port_from == acl.port_to else f"{acl.port_from}-{acl.port_to}"
|
|
suffix += f" {acl.protocol} dport {ports}"
|
|
return suffix
|
|
|
|
|
|
def _nft_marker(plan: VpcDataplanePlan) -> str:
|
|
"""Return a bounded ownership marker safe for nft comment userdata.
|
|
|
|
libnftnl limits rule comments to 128 bytes. VPC/host identifiers are
|
|
intentionally allowed to be long, so embedding them and the full digest
|
|
in every rule would make otherwise valid plans fail at ``nft -c``. The
|
|
state file keeps the full identifiers; comments only need a collision
|
|
resistant, deterministic token for ownership/readback.
|
|
"""
|
|
seed = (f"{plan.vpc.vpc_id}:{plan.host_id}:{plan.desired_revision}:"
|
|
f"{plan.desired_digest}").encode()
|
|
token = hashlib.sha256(seed).hexdigest()[:16]
|
|
return f"{DATAPLANE_OWNER_MARKER}:{plan.resources.token}:{token}"
|
|
|
|
|
|
def render_vpc_nft_rules(plan: VpcDataplanePlan) -> str:
|
|
"""Render scoped VPC ACL/anti-spoof rules for routed and L2 paths.
|
|
|
|
The base chains use ``policy accept`` deliberately. A VPC table must not
|
|
become a host-wide firewall that drops public bridges or unrelated Fabric
|
|
traffic; only packets carrying this VPC's member addresses are subject to
|
|
its default deny. A separate bridge-family table covers same-host VM to
|
|
VM forwarding and ARP, which do not reliably traverse an inet forward
|
|
hook.
|
|
"""
|
|
r = plan.resources
|
|
marker = _nft_marker(plan)
|
|
dataplane_routes = _routes(plan)
|
|
member_cidrs = {f"{m.private_ip}/32" for m in plan.members}
|
|
local_member_cidrs = {f"{m.private_ip}/32" for m in plan.local_members}
|
|
# Custom VPC routes are exact /32s too. Keep source sets restricted to
|
|
# actual member addresses, while destination sets include routed service
|
|
# addresses; otherwise the isolation rules would silently drop a valid
|
|
# route before the ACL/default-deny decision.
|
|
destination_cidrs = member_cidrs | {
|
|
str(item["destination_cidr"]) for item in dataplane_routes
|
|
}
|
|
# A packet entering a local tap may be routed either to another local
|
|
# tap or out through wg-fabric0 to a remote member/service /32. The
|
|
# bridge-family unknown-destination guard therefore needs the complete
|
|
# VPC destination set; restricting it to local routes would drop every
|
|
# same-VPC cross-host flow before the inet routing policy can run.
|
|
local_destination_cidrs = set(destination_cidrs)
|
|
# Use explicit proxy-neighbour entries for the guest's routed /32 model.
|
|
# The generic kernel proxy responder stays disabled, so a nftables reload
|
|
# or a short ordering window cannot make the host answer for arbitrary
|
|
# addresses. proxy_arp_pvlan is enabled only for the same-bridge
|
|
# next-hop case and is still bounded by the exact neighbour set and ARP
|
|
# family output chain below.
|
|
proxy_ips = sorted(
|
|
str(ipaddress.ip_network(item["destination_cidr"], strict=True).network_address)
|
|
for item in dataplane_routes)
|
|
members = ", ".join(sorted(member_cidrs))
|
|
local_members = ", ".join(sorted(local_member_cidrs))
|
|
destinations = ", ".join(sorted(destination_cidrs))
|
|
local_destinations = ", ".join(sorted(local_destination_cidrs))
|
|
proxy_destinations = ", ".join(proxy_ips)
|
|
local_sources = ", ".join(
|
|
f"{m.private_mac} . {m.private_ip}" for m in plan.local_members)
|
|
|
|
def nft_set(name: str, type_name: str, elements: str, *, interval: bool = False) -> str:
|
|
flags = " flags interval;" if interval else ""
|
|
body = f" set {name} {{ type {type_name};{flags}"
|
|
if elements:
|
|
body += f" elements = {{ {elements} }};"
|
|
return body + " }"
|
|
|
|
lines = [
|
|
f"# {DATAPLANE_OWNER_MARKER}", f"# vpc-token={r.token}",
|
|
f"# vpc={plan.vpc.vpc_id} revision={plan.desired_revision} digest={plan.desired_digest}",
|
|
f"table inet {r.nft_table} {{",
|
|
nft_set("member_ips", "ipv4_addr", members, interval=True),
|
|
nft_set("local_member_ips", "ipv4_addr", local_members, interval=True),
|
|
nft_set("vpc_dest_ips", "ipv4_addr", destinations, interval=True),
|
|
nft_set("local_member_sources", "ether_addr . ipv4_addr", local_sources),
|
|
" chain forward { type filter hook forward priority -190; policy accept;",
|
|
f' iifname "{r.bridge}" ip saddr != @local_member_ips drop comment "{marker}:anti-spoof-local-ip"',
|
|
f' iifname "{r.bridge}" ip6 saddr ::/0 drop comment "{marker}:ipv6-drop-bridge"',
|
|
# A source address is not an ingress identity. A packet arriving on
|
|
# the public/management side must never be able to spoof a member /32
|
|
# and use the VPC policy route to reach a private destination. Close
|
|
# both egress legs by interface before the source-address checks.
|
|
# The whole VPC CIDR is guarded, including currently unallocated
|
|
# addresses. A missing /32 route must therefore fail closed instead
|
|
# of falling through to the host's public/default route.
|
|
f' oifname "{r.bridge}" ip daddr {plan.vpc.cidr} iifname != "{r.bridge}" iifname != "{FABRIC_INTERFACE}" drop comment "{marker}:anti-spoof-public-to-vpc-bridge"',
|
|
f' oifname "{FABRIC_INTERFACE}" ip daddr {plan.vpc.cidr} iifname != "{r.bridge}" iifname != "{FABRIC_INTERFACE}" drop comment "{marker}:anti-spoof-public-to-vpc-fabric"',
|
|
f' iifname "{r.bridge}" ip daddr {plan.vpc.cidr} oifname != "{r.bridge}" oifname != "{FABRIC_INTERFACE}" drop comment "{marker}:anti-spoof-bridge-vpc-to-foreign"',
|
|
f' iifname "{FABRIC_INTERFACE}" ip daddr {plan.vpc.cidr} oifname != "{r.bridge}" oifname != "{FABRIC_INTERFACE}" drop comment "{marker}:anti-spoof-fabric-vpc-to-foreign"',
|
|
f' oifname "{r.bridge}" ip daddr @vpc_dest_ips iifname != "{r.bridge}" iifname != "{FABRIC_INTERFACE}" drop comment "{marker}:anti-spoof-public-to-bridge"',
|
|
f' oifname "{FABRIC_INTERFACE}" ip daddr @vpc_dest_ips iifname != "{r.bridge}" iifname != "{FABRIC_INTERFACE}" drop comment "{marker}:anti-spoof-public-to-fabric"',
|
|
# A VPC destination must also leave through one of the two owned legs.
|
|
# If a policy route disappears or a foreign/default route wins, the
|
|
# packet must be dropped rather than sent to a public or management
|
|
# interface. This is deliberately independent of source-IP checks.
|
|
f' iifname "{r.bridge}" ip daddr @vpc_dest_ips oifname != "{r.bridge}" oifname != "{FABRIC_INTERFACE}" drop comment "{marker}:anti-spoof-bridge-to-foreign"',
|
|
f' iifname "{FABRIC_INTERFACE}" ip daddr @vpc_dest_ips oifname != "{r.bridge}" oifname != "{FABRIC_INTERFACE}" drop comment "{marker}:anti-spoof-fabric-to-foreign"',
|
|
f' oifname "{r.bridge}" ip daddr @vpc_dest_ips ip saddr != @member_ips drop comment "{marker}:anti-spoof-egress"',
|
|
f' iifname "{FABRIC_INTERFACE}" ip daddr @vpc_dest_ips ip saddr != @member_ips drop comment "{marker}:anti-spoof-wg-ingress"',
|
|
f' oifname "{FABRIC_INTERFACE}" ip saddr @member_ips ip daddr != @vpc_dest_ips drop comment "{marker}:anti-spoof-wg-egress"',
|
|
f' iifname "{FABRIC_INTERFACE}" ip daddr @vpc_dest_ips ip saddr != @member_ips drop comment "{marker}:anti-spoof-wg-source"',
|
|
f' iifname "{r.bridge}" ip saddr @local_member_ips ip daddr != @vpc_dest_ips drop comment "{marker}:member-egress-isolation"',
|
|
f' iifname "{FABRIC_INTERFACE}" ip saddr @member_ips ip daddr != @vpc_dest_ips drop comment "{marker}:member-fabric-isolation"',
|
|
]
|
|
# ACL priority is first-match order. Direction is relative to the VPC
|
|
# boundary: ingress is wg -> private bridge; egress is bridge -> wg.
|
|
# Same-host members have no wg leg, so their source/destination tuple is
|
|
# enforced on the bridge path as well.
|
|
for acl in plan.vpc.acls:
|
|
action = "accept" if acl.action == "allow" else "drop"
|
|
clause = (f" ip saddr {acl.source_cidr} ip daddr {acl.destination_cidr}"
|
|
f"{_nft_proto(acl)} {action} comment \"{marker}:acl:{acl.acl_id}\"")
|
|
if acl.direction == "ingress":
|
|
lines.append(f' iifname "{FABRIC_INTERFACE}" oifname "{r.bridge}"{clause}')
|
|
else:
|
|
lines.append(f' iifname "{r.bridge}" oifname "{FABRIC_INTERFACE}"{clause}')
|
|
lines.append(f' iifname "{r.bridge}" oifname "{r.bridge}"{clause}')
|
|
# Scope default-deny to member-to-member VPC traffic. The following
|
|
# interface-wide accepts preserve unrelated host/public forwarding.
|
|
lines += [
|
|
f' iifname "{FABRIC_INTERFACE}" oifname "{r.bridge}" ip saddr @member_ips ip daddr @vpc_dest_ips counter drop comment "{marker}:default-deny"',
|
|
f' iifname "{r.bridge}" oifname "{FABRIC_INTERFACE}" ip saddr @member_ips ip daddr @vpc_dest_ips counter drop comment "{marker}:default-deny"',
|
|
f' iifname "{r.bridge}" oifname "{r.bridge}" ip saddr @member_ips ip daddr @vpc_dest_ips counter drop comment "{marker}:default-deny"',
|
|
f' iifname "{FABRIC_INTERFACE}" oifname "{FABRIC_INTERFACE}" ip saddr @member_ips ip daddr @vpc_dest_ips counter drop comment "{marker}:wg-transit-deny"',
|
|
f' iifname "{FABRIC_INTERFACE}" oifname "{r.bridge}" ip6 saddr ::/0 drop comment "{marker}:ipv6-drop-fabric"',
|
|
f' iifname "{r.bridge}" accept comment "{marker}:scope-bridge-in"',
|
|
f' oifname "{r.bridge}" accept comment "{marker}:scope-bridge-out"',
|
|
f' iifname "{FABRIC_INTERFACE}" accept comment "{marker}:scope-fabric-in"',
|
|
f' oifname "{FABRIC_INTERFACE}" accept comment "{marker}:scope-fabric-out"',
|
|
" }",
|
|
" chain input { type filter hook input priority -190; policy accept;",
|
|
f' iifname "{r.bridge}" ip saddr @member_ips drop comment "{marker}:host-isolation"',
|
|
f' iifname "{FABRIC_INTERFACE}" ip daddr @vpc_dest_ips drop comment "{marker}:host-isolation-wg"',
|
|
f' iifname "{FABRIC_INTERFACE}" ip saddr @member_ips ip daddr != @vpc_dest_ips drop comment "{marker}:host-isolation-wg-source"',
|
|
f' iifname "{r.bridge}" ip6 saddr ::/0 drop comment "{marker}:ipv6-input-drop-bridge"',
|
|
f' iifname "{FABRIC_INTERFACE}" ip6 saddr ::/0 drop comment "{marker}:ipv6-input-drop-fabric"',
|
|
" }",
|
|
" chain output { type filter hook output priority -190; policy accept;",
|
|
f' oifname "{r.bridge}" ip daddr @vpc_dest_ips drop comment "{marker}:host-isolation-out"',
|
|
# Host-originated packets must not be able to bind a tenant /32 and
|
|
# bypass the private egress boundary. Keep this leg unconditional;
|
|
# the source address is not an ingress identity.
|
|
f' oifname "{FABRIC_INTERFACE}" ip daddr @vpc_dest_ips drop comment "{marker}:host-isolation-out-wg"',
|
|
f' oifname "{r.bridge}" ip daddr {plan.vpc.cidr} drop comment "{marker}:host-isolation-vpc-out"',
|
|
f' oifname "{FABRIC_INTERFACE}" ip daddr {plan.vpc.cidr} drop comment "{marker}:host-isolation-vpc-out-wg"',
|
|
f' oifname "{r.bridge}" ip6 daddr ::/0 drop comment "{marker}:ipv6-output-drop-bridge"',
|
|
" }", "}", "",
|
|
]
|
|
|
|
# Bridge-family rules are emitted only when this host has a local VM. An
|
|
# empty VPC still gets the routed table above, while no empty nft set is
|
|
# needed for the L2/ARP path.
|
|
if plan.local_members:
|
|
lines += [
|
|
f"table bridge {r.nft_table} {{",
|
|
nft_set("local_member_ips", "ipv4_addr", local_members, interval=True),
|
|
nft_set("local_dest_ips", "ipv4_addr", local_destinations, interval=True),
|
|
nft_set("local_member_sources", "ether_addr . ipv4_addr", local_sources),
|
|
" chain forward { type filter hook forward priority -190; policy accept;",
|
|
f' meta ibrname "{r.bridge}" ether type ip ether saddr . ip saddr != @local_member_sources drop comment "{marker}:bridge-anti-spoof-ingress"',
|
|
f' meta ibrname "{r.bridge}" ether type arp ether saddr != arp saddr ether drop comment "{marker}:bridge-anti-spoof-arp-mac"',
|
|
f' meta ibrname "{r.bridge}" ether type arp arp saddr ether . arp saddr ip != @local_member_sources drop comment "{marker}:bridge-anti-spoof-arp"',
|
|
f' meta ibrname "{r.bridge}" ether type ip ip saddr != @local_member_ips drop comment "{marker}:bridge-anti-spoof-ip"',
|
|
f' meta ibrname "{r.bridge}" ether type arp arp saddr ip != @local_member_ips drop comment "{marker}:bridge-anti-spoof-arp-ip"',
|
|
# This is an L3 /32 fabric. Do not let broadcast or multicast
|
|
# discovery turn the private bridge back into a shared L2 LAN.
|
|
f' meta ibrname "{r.bridge}" ip daddr 224.0.0.0/4 drop comment "{marker}:bridge-multicast-drop"',
|
|
f' meta ibrname "{r.bridge}" ip daddr 255.255.255.255 drop comment "{marker}:bridge-limited-broadcast-drop"',
|
|
f' meta ibrname "{r.bridge}" ip daddr {ipaddress.ip_network(plan.vpc.cidr).broadcast_address} drop comment "{marker}:bridge-vpc-broadcast-drop"',
|
|
f' meta ibrname "{r.bridge}" ip saddr @local_member_ips ip daddr != @local_dest_ips drop comment "{marker}:bridge-unknown-destination-drop"',
|
|
]
|
|
for acl in plan.vpc.acls:
|
|
action = "accept" if acl.action == "allow" else "drop"
|
|
clause = (f" ip saddr {acl.source_cidr} ip daddr {acl.destination_cidr}"
|
|
f"{_nft_proto(acl)} {action} comment \"{marker}:acl:{acl.acl_id}\"")
|
|
lines.append(f' meta ibrname "{r.bridge}" meta obrname "{r.bridge}"{clause}')
|
|
lines += [
|
|
f' meta ibrname "{r.bridge}" meta obrname "{r.bridge}" ip saddr @local_member_ips ip daddr @local_dest_ips counter drop comment "{marker}:bridge-default-deny"',
|
|
f' meta ibrname "{r.bridge}" ether type arp accept comment "{marker}:bridge-arp-accept"',
|
|
f' meta ibrname "{r.bridge}" ether type ip accept comment "{marker}:bridge-scope-accept"',
|
|
f' meta ibrname "{r.bridge}" counter drop comment "{marker}:bridge-non-ip-drop"',
|
|
" }",
|
|
" chain input { type filter hook input priority -190; policy accept;",
|
|
f' meta ibrname "{r.bridge}" ether type arp ether saddr != arp saddr ether drop comment "{marker}:bridge-anti-spoof-arp-input-mac"',
|
|
f' meta ibrname "{r.bridge}" ether type ip ether saddr . ip saddr != @local_member_sources drop comment "{marker}:bridge-anti-spoof-input-ip"',
|
|
f' meta ibrname "{r.bridge}" ether type arp arp saddr ether . arp saddr ip != @local_member_sources drop comment "{marker}:bridge-anti-spoof-input"',
|
|
f' meta ibrname "{r.bridge}" ether type ip6 drop comment "{marker}:bridge-ipv6-input-drop"',
|
|
f' meta ibrname "{r.bridge}" ether type arp accept comment "{marker}:bridge-arp-input-accept"',
|
|
# Proxy ARP makes remote /32 packets arrive at the bridge MAC and
|
|
# then enter the host routing path. Source/MAC checks above have
|
|
# already authenticated the guest, so IPv4 must be accepted here
|
|
# for the inet forward hook to route it to wg-fabric0.
|
|
f' meta ibrname "{r.bridge}" ether type ip accept comment "{marker}:bridge-ip-input-accept"',
|
|
f' meta ibrname "{r.bridge}" counter drop comment "{marker}:bridge-input-non-ip-drop"',
|
|
" }",
|
|
"}", "",
|
|
]
|
|
lines += [
|
|
f"table arp {r.nft_table} {{",
|
|
nft_set("proxy_ips", "ipv4_addr", proxy_destinations, interval=True),
|
|
" chain output { type filter hook output priority -190; policy accept;",
|
|
f' oifname "{r.bridge}" arp operation reply arp saddr ip != @proxy_ips drop comment "{marker}:arp-proxy-output"',
|
|
" }",
|
|
"}", "",
|
|
]
|
|
return "\n".join(lines)
|
|
|
|
|
|
def _here(path: str, content: str) -> str:
|
|
marker = "__VC_VPC_" + hashlib.sha256(content.encode()).hexdigest()[:16]
|
|
return f"cat > {_q(path)} <<'{marker}'\n{content}{'' if content.endswith(chr(10)) else chr(10)}{marker}\n"
|
|
|
|
|
|
def _state(plan: VpcDataplanePlan, routes: Sequence[Mapping[str, Any]]) -> str:
|
|
r = plan.resources
|
|
lines = [f"owner|{DATAPLANE_OWNER_MARKER}", f"vpc|{plan.vpc.vpc_id}",
|
|
f"cidr|{plan.vpc.cidr}", f"fabric|{plan.vpc.fabric_id}",
|
|
f"host|{plan.host_id}", f"revision|{plan.desired_revision}",
|
|
f"digest|{plan.desired_digest}", f"bridge|{r.bridge}",
|
|
f"table|{r.route_table}", f"priority|{r.rule_priority}"]
|
|
for route in routes:
|
|
lines.append("route|{destination}|{device}|{table}|{via}|{metric}".format(
|
|
destination=route["destination_cidr"], device=route["device"],
|
|
table=route["table"], via=route.get("via") or "-",
|
|
metric=route.get("metric", 100)))
|
|
for peer in plan.peers:
|
|
ips = sorted(set(
|
|
[m.private_ip + "/32" for m in plan.remote_members
|
|
if m.host_id == peer.host_id]
|
|
+ list(peer.tenant_allowed_ips)))
|
|
if ips:
|
|
lines.append(f"peer|{peer.host_id}|{peer.public_key}|{','.join(ips)}")
|
|
return "\n".join(lines) + "\n"
|
|
|
|
|
|
def _allowed(plan: VpcDataplanePlan, peer: VpcFabricPeer) -> str:
|
|
members = {m.member_id: m for m in plan.members}
|
|
ips = {m.private_ip + "/32" for m in plan.remote_members
|
|
if m.host_id == peer.host_id}
|
|
for route in plan.vpc.routes:
|
|
target = (members.get(route.next_hop_member_id)
|
|
if route.next_hop_member_id is not None else None)
|
|
target_host = target.host_id if target is not None else route.next_hop_host_id
|
|
if target_host == peer.host_id:
|
|
ips.add(route.destination_cidr)
|
|
return ",".join([peer.overlay_cidr] + sorted(ips | set(peer.tenant_allowed_ips)))
|
|
|
|
|
|
def render_apply_script(plan: VpcDataplanePlan) -> str:
|
|
"""Generate a shell transaction with panel ownership and rollback markers."""
|
|
routes = _routes(plan)
|
|
if plan.remote_members and not plan.peers:
|
|
raise VpcDataplaneError("跨宿主机 VPC 缺少 Fabric peer", 409, "vpc_fabric_peer_missing")
|
|
r = plan.resources
|
|
nft = render_vpc_nft_rules(plan)
|
|
state = _state(plan, routes)
|
|
route_specs = "\\n".join(
|
|
f"{x['destination_cidr']}|{x['device']}|{x['table']}" for x in routes)
|
|
peer_specs = "\\n".join(
|
|
f"{p.host_id}|{p.public_key}|{_allowed(plan,p)}" for p in plan.peers)
|
|
route_checks = "\\n".join(
|
|
f'''actual=$("$IP" -o -4 route show table {_q(x["table"])} {_q(x["destination_cidr"])} 2>/dev/null || true)
|
|
if printf '%s\\n' "$actual" | grep -Fq "dev {_q(x["device"])}" && printf '%s\\n' "$actual" | grep -Eq 'proto 186([[:space:]]|$)'; then
|
|
printf '%s\\n' "__VC_VPC_ROUTE__={x["destination_cidr"]}|{x["device"]}|{x["table"]}|owned"
|
|
else
|
|
printf '%s\\n' "__VC_VPC_ROUTE__={x["destination_cidr"]}|{x["device"]}|{x["table"]}|missing"
|
|
fi'''
|
|
for x in routes)
|
|
peer_checks = "\\n".join(
|
|
f'''actual=$("$WG" show "$WG_IFACE" allowed-ips 2>/dev/null | awk -v wanted={_q(p.public_key)} '$1==wanted {{print $2; exit}}' || true)
|
|
tenant={_q(','.join(_allowed(plan, p).split(',')[1:]))}
|
|
case ",$actual," in *",$tenant,"*) printf '%s\\n' "__VC_VPC_WG_ALLOWED__={p.host_id}|{p.public_key}|$tenant";; *) printf '%s\\n' "__VC_VPC_WG_ALLOWED__={p.host_id}|{p.public_key}|missing";; esac'''
|
|
for p in plan.peers)
|
|
script = f'''#!/bin/sh
|
|
set -eu
|
|
umask 077
|
|
PATH=/usr/sbin:/usr/bin:/sbin:/bin
|
|
export PATH
|
|
IP=/usr/sbin/ip; [ -x "$IP" ] || IP=/usr/bin/ip
|
|
WG=/usr/bin/wg; [ -x "$WG" ] || WG=/usr/sbin/wg
|
|
NFT=/usr/sbin/nft; [ -x "$NFT" ] || NFT=/usr/bin/nft
|
|
FLOCK=/usr/bin/flock; [ -x "$FLOCK" ] || FLOCK=/usr/bin/flock
|
|
BASE=/etc/virt-cloud/vpc-dataplane
|
|
BRIDGE={_q(r.bridge)}; TABLE={_q(r.nft_table)}; ROUTE_TABLE={r.route_table}; RULE_PRIORITY={r.rule_priority}
|
|
WG_IFACE={_q(FABRIC_INTERFACE)}; OWNER={_q(r.owner_file)}; STATE={_q(r.state_file)}; LOCK={_q(r.lock_file)}; TXN={_q(r.transaction_dir)}
|
|
COMMITTED=no; BRIDGE_CREATED=no; NFT_BEFORE=no; WG_BEFORE=no; ROLLBACK_FAILED=no
|
|
mkdir -p /run/virt-cloud "$BASE"
|
|
[ ! -L /run/virt-cloud ] && [ ! -L "$BASE" ] || exit 71
|
|
: >> "$LOCK"; chmod 600 "$LOCK"; exec 9>>"$LOCK"; "$FLOCK" -x 9
|
|
rollback() {{
|
|
status=$?; [ "$COMMITTED" = yes ] && return "$status"; trap - EXIT HUP INT TERM
|
|
oldIFS=$IFS; IFS=' '
|
|
for spec in { _q(route_specs) }; do destination=${{spec%%|*}}; device=${{spec#*|}}; "$IP" route del "$destination" table "$ROUTE_TABLE" dev "$device" >/dev/null 2>&1 || true; done
|
|
IFS=$oldIFS
|
|
"$IP" rule del pref "$RULE_PRIORITY" iif "$BRIDGE" to {_q(plan.vpc.cidr)} lookup "$ROUTE_TABLE" >/dev/null 2>&1 || true
|
|
if [ "$WG_BEFORE" = yes ] && [ -f "$TXN/wg.conf" ]; then "$WG" setconf "$WG_IFACE" "$TXN/wg.conf" >/dev/null 2>&1 || ROLLBACK_FAILED=yes; fi
|
|
if [ "$NFT_BEFORE" = yes ] && [ -f "$TXN/nft.before" ]; then "$NFT" delete table inet "$TABLE" >/dev/null 2>&1 || true; "$NFT" -f "$TXN/nft.before" >/dev/null 2>&1 || ROLLBACK_FAILED=yes; else "$NFT" delete table inet "$TABLE" >/dev/null 2>&1 || true; fi
|
|
if [ "$BRIDGE_CREATED" = yes ]; then "$IP" link del dev "$BRIDGE" >/dev/null 2>&1 || ROLLBACK_FAILED=yes; fi
|
|
[ "$ROLLBACK_FAILED" = yes ] && printf '%s\\n' '__VC_VPC_ROLLBACK__=failed' >&2 || printf '%s\\n' '__VC_VPC_ROLLBACK__=ok' >&2
|
|
exit "$status"
|
|
}}
|
|
trap rollback EXIT HUP INT TERM
|
|
[ -x "$IP" ] && [ -x "$WG" ] && [ -x "$NFT" ] && [ -x "$FLOCK" ] || exit 70
|
|
[ -f /etc/virt-cloud/wg-fabric0.owner ] && grep -Fqx { _q(FABRIC_OWNER_MARKER) } /etc/virt-cloud/wg-fabric0.owner || exit 71
|
|
"$IP" link show dev "$WG_IFACE" >/dev/null 2>&1 && "$WG" show "$WG_IFACE" >/dev/null 2>&1 || exit 72
|
|
[ ! -e "$TXN" ] && [ ! -L "$TXN" ] || exit 79
|
|
mkdir -p "$TXN"; printf '%s\\n' '{DATAPLANE_OWNER_MARKER}:{plan.vpc.vpc_id}:{plan.host_id}:{plan.desired_revision}:{plan.desired_digest}' > "$TXN/inflight"
|
|
if [ -f "$STATE" ]; then cp -p "$STATE" "$TXN/state.before"; grep -Fqx 'owner|{DATAPLANE_OWNER_MARKER}' "$STATE" || exit 71; fi
|
|
if [ -f "$OWNER" ]; then grep -Fqx { _q(DATAPLANE_OWNER_MARKER) } "$OWNER" || exit 71; else printf '%s\\n' { _q(DATAPLANE_OWNER_MARKER) } > "$OWNER"; chmod 600 "$OWNER"; fi
|
|
if "$IP" link show dev "$BRIDGE" >/dev/null 2>&1; then
|
|
"$IP" -d link show dev "$BRIDGE" | grep -Eq '(^|[[:space:]])bridge[[:space:]]+(forward_delay|stp_state|hello_time|ageing_time)([[:space:]]|$)' || exit 71
|
|
else
|
|
"$IP" link add name "$BRIDGE" type bridge; BRIDGE_CREATED=yes
|
|
fi
|
|
"$IP" link set dev "$BRIDGE" up
|
|
if "$WG" showconf "$WG_IFACE" > "$TXN/wg.conf" 2>/dev/null; then WG_BEFORE=yes; fi
|
|
if "$NFT" list table inet "$TABLE" > "$TXN/nft.before" 2>/dev/null; then NFT_BEFORE=yes; grep -Fq { _q(DATAPLANE_OWNER_MARKER) } "$TXN/nft.before" || exit 71; fi
|
|
'''+_here('$TXN/rules.nft', nft)+f'''"$NFT" delete table inet "$TABLE" >/dev/null 2>&1 || true
|
|
"$NFT" -f "$TXN/rules.nft"
|
|
oldIFS=$IFS; IFS=' '
|
|
for spec in { _q(route_specs) }; do destination=${{spec%%|*}}; device=${{spec#*|}}; "$IP" route replace "$destination" table "$ROUTE_TABLE" dev "$device" metric 100; done
|
|
IFS=$oldIFS
|
|
"$IP" rule replace pref "$RULE_PRIORITY" iif "$BRIDGE" to {_q(plan.vpc.cidr)} lookup "$ROUTE_TABLE"
|
|
oldIFS=$IFS; IFS=' '
|
|
for spec in { _q(peer_specs) }; do rest=${{spec#*|}}; key=${{rest%%|*}}; allowed=${{rest#*|}}; "$WG" show "$WG_IFACE" peers | grep -Fqx "$key" || exit 74; "$WG" set "$WG_IFACE" peer "$key" allowed-ips "$allowed"; done
|
|
IFS=$oldIFS
|
|
'''+_here('$STATE', state)+f'''chmod 600 "$STATE"
|
|
printf '%s\\n' '__VC_VPC_APPLIED__=yes'
|
|
printf '%s\\n' '__VC_VPC_OWNER__={DATAPLANE_OWNER_MARKER}'
|
|
printf '%s\\n' '__VC_VPC_REVISION__={plan.desired_revision}'
|
|
printf '%s\\n' '__VC_VPC_DIGEST__={plan.desired_digest}'
|
|
COMMITTED=yes; trap - EXIT HUP INT TERM; rm -rf "$TXN"; exit 0
|
|
'''
|
|
if len(script) > MAX_OUTPUT:
|
|
raise VpcDataplaneError("VPC 数据面脚本过大", 413, "vpc_script_too_large")
|
|
return script
|
|
|
|
|
|
def _render_apply_script_v2(
|
|
plan: VpcDataplanePlan,
|
|
*,
|
|
deployment_mode: str = "networkmanager_nmcli",
|
|
ifup_path: str = "/usr/sbin/ifup",
|
|
ifdown_path: str = "/usr/sbin/ifdown",
|
|
ifquery_path: str = "/usr/sbin/ifquery",
|
|
) -> str:
|
|
"""Generate the production transaction used by the host adapter.
|
|
|
|
The original control-plane module predates distribution-specific bridge
|
|
persistence. Keep its public builder for compatibility, but route real
|
|
applies through this stricter builder: every list is newline-delimited,
|
|
every path is fixed/quoted, and rollback only removes exact resources
|
|
tagged with the panel route protocol or ownership marker.
|
|
"""
|
|
if deployment_mode not in {"networkmanager_nmcli", "ifupdown"}:
|
|
raise VpcDataplaneError("宿主机网络管理模式不支持", 409,
|
|
"vpc_dataplane_network_manager_conflict")
|
|
allowed_ifup = {"/usr/sbin/ifup", "/usr/bin/ifup", "/sbin/ifup", "/bin/ifup"}
|
|
allowed_ifdown = {"/usr/sbin/ifdown", "/usr/bin/ifdown", "/sbin/ifdown", "/bin/ifdown"}
|
|
allowed_ifquery = {"/usr/sbin/ifquery", "/usr/bin/ifquery", "/sbin/ifquery", "/bin/ifquery"}
|
|
if ifup_path not in allowed_ifup or ifdown_path not in allowed_ifdown or ifquery_path not in allowed_ifquery:
|
|
raise VpcDataplaneError("ifupdown 工具路径不受支持", 409,
|
|
"vpc_dataplane_network_manager_conflict")
|
|
from .fabric import _VPC_STATE_LOADER
|
|
|
|
routes = _routes(plan)
|
|
if plan.remote_members and not plan.peers:
|
|
raise VpcDataplaneError("跨宿主机 VPC 缺少 Fabric peer", 409,
|
|
"vpc_fabric_peer_missing")
|
|
r = plan.resources
|
|
nft = render_vpc_nft_rules(plan)
|
|
include = f"/etc/nftables.d/virt-cloud-vpc-{r.token}.nft"
|
|
include_line = f'include "{include}"'
|
|
fragment = f"/etc/network/interfaces.d/virt-cloud-vpc-{r.token}"
|
|
sysctl_file = f"/etc/sysctl.d/99-virt-cloud-vpc-{r.token}.conf"
|
|
state_dir = f"/etc/virt-cloud/vpc-dataplane"
|
|
proxy_file = f"{state_dir}/{r.token}.proxy"
|
|
route_lines = "\n".join(
|
|
f"{item['destination_cidr']}|{item['device']}|{item['table']}|"
|
|
f"{item.get('via') or '-'}|{item.get('metric', 100)}"
|
|
for item in routes)
|
|
bridge_route_lines = "\n".join(
|
|
f"{item['destination_cidr']}|{item['device']}|{item['table']}|"
|
|
f"{item.get('via') or '-'}|{item.get('metric', 100)}|186"
|
|
for item in routes if item["device"] == r.bridge)
|
|
proxy_lines = "\n".join(
|
|
str(ipaddress.ip_network(item["destination_cidr"], strict=True).network_address)
|
|
for item in routes)
|
|
peer_lines = "\n".join(
|
|
f"{peer.host_id}|{peer.public_key}|{_allowed(plan, peer)}"
|
|
for peer in plan.peers)
|
|
state = _state(plan, routes)
|
|
# Local service /32s use a guest address as the next hop on an otherwise
|
|
# unnumbered private bridge. Linux therefore needs the explicit `onlink`
|
|
# assertion; without it iproute2 rejects the route as an invalid gateway.
|
|
# Use a quoted heredoc for data, while keeping the destination as a shell
|
|
# variable expanded by the surrounding script. This avoids the literal
|
|
# ``$TXN`` path bug in the former builder.
|
|
nft_marker = "__VC_VPC_NFT_" + hashlib.sha256(nft.encode()).hexdigest()[:12]
|
|
state_marker = "__VC_VPC_STATE_" + hashlib.sha256(state.encode()).hexdigest()[:12]
|
|
frag_marker = "__VC_VPC_IFUP_" + r.token
|
|
route_marker = "__VC_VPC_ROUTES_" + r.token
|
|
proxy_marker = "__VC_VPC_PROXY_" + r.token
|
|
peer_marker = "__VC_VPC_PEERS_" + r.token
|
|
state_loader = _VPC_STATE_LOADER.replace(
|
|
"__VC_FABRIC_HOST_ID__", plan.host_id).replace(
|
|
"__VC_FABRIC_ID__", plan.vpc.fabric_id).replace(
|
|
"__VC_FABRIC_NETWORK__", "100.64.0.0/10").replace(
|
|
"__VC_VPC_STATE_OBSERVE__", "no")
|
|
script = f'''#!/bin/sh
|
|
set -eu
|
|
umask 077
|
|
PATH=/usr/sbin:/usr/bin:/sbin:/bin
|
|
export PATH
|
|
IP=$(for p in /usr/sbin/ip /usr/bin/ip /sbin/ip /bin/ip; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
|
|
WG=$(for p in /usr/bin/wg /usr/sbin/wg /bin/wg /sbin/wg; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
|
|
NFT=$(for p in /usr/sbin/nft /usr/bin/nft /sbin/nft /bin/nft; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
|
|
FLOCK=$(for p in /usr/bin/flock /usr/sbin/flock /bin/flock /sbin/flock; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
|
|
STAT=$(for p in /usr/bin/stat /bin/stat /usr/sbin/stat /sbin/stat; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
|
|
CMP=$(for p in /usr/bin/cmp /bin/cmp /usr/sbin/cmp /sbin/cmp; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
|
|
SYSCTL=$(for p in /usr/sbin/sysctl /usr/bin/sysctl /sbin/sysctl /bin/sysctl; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
|
|
BRIDGE_CMD=$(for p in /usr/sbin/bridge /usr/bin/bridge /sbin/bridge /bin/bridge; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
|
|
BRIDGE={_q(r.bridge)}; TABLE={_q(r.nft_table)}; ROUTE_TABLE={r.route_table}; RULE_PRIORITY={r.rule_priority}; WG_RULE_PRIORITY=$((RULE_PRIORITY + 1))
|
|
WG_IFACE={_q(FABRIC_INTERFACE)}; OWNER_MARKER={_q(DATAPLANE_OWNER_MARKER)}; FABRIC_OWNER={_q(FABRIC_OWNER_MARKER)}
|
|
OWNER_FILE={_q(r.owner_file)}; STATE_FILE={_q(r.state_file)}; LOCK={_q(r.lock_file)}; TXN={_q(r.transaction_dir)}; PROXY_FILE={_q(proxy_file)}
|
|
NFT_FILE={_q(include)}; NFT_INCLUDE={_q(include_line)}; IF_FRAGMENT={_q(fragment)}; SYSCTL_FILE={_q(sysctl_file)}; MODE={_q(deployment_mode)}; NFT_MAIN=
|
|
IFUP={_q(ifup_path)}; IFDOWN={_q(ifdown_path)}; IFQUERY={_q(ifquery_path)}; NMCLI=
|
|
NM_PROFILE_MARKER={_q(DATAPLANE_OWNER_MARKER + " vpc=" + plan.vpc.vpc_id)}
|
|
COMMITTED=no; BRIDGE_CREATED=no; BRIDGE_WAS_UP=no; NM_CREATED=no; NM_WAS_ACTIVE=no; NM_ACTIVATION_ATTEMPTED=no; IF_CREATED=no; IF_ACTIVATION_ATTEMPTED=no; IF_FRAGMENT_CHANGED=no; SYSCTL_CHANGED=no; RP_FILTER_CHANGED=no; PROXY_ARP_CHANGED=no; PROXY_ARP_PVLAN_CHANGED=no; NFT_FILE_CHANGED=no; NFT_MAIN_CHANGED=no; NFT_FILE_BEFORE=no; NFT_MAIN_BEFORE=no; BRIDGE_NFT_BEFORE=no; ARP_NFT_BEFORE=no; FABRIC_NFT_BEFORE=no; PROXY_ENTRIES_CHANGED=no; PROXY_FILE_CHANGED=no; ROLLBACK_FAILED=no
|
|
[ -n "$IP" ] && [ -n "$WG" ] && [ -n "$NFT" ] && [ -n "$FLOCK" ] && [ -n "$STAT" ] && [ -n "$CMP" ] && [ -n "$SYSCTL" ] && [ -n "$BRIDGE_CMD" ] || exit 70
|
|
if [ "$MODE" = networkmanager_nmcli ]; then
|
|
NMCLI=$(for p in /usr/bin/nmcli /usr/sbin/nmcli /bin/nmcli /sbin/nmcli; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
|
|
[ -n "$NMCLI" ] || exit 70
|
|
fi
|
|
if [ -L /run/virt-cloud ] || {{ [ -e /run/virt-cloud ] && [ ! -d /run/virt-cloud ]; }}; then exit 71; fi
|
|
if [ -L /etc/virt-cloud ] || {{ [ -e /etc/virt-cloud ] && [ ! -d /etc/virt-cloud ]; }}; then exit 71; fi
|
|
if [ -L {state_dir} ] || {{ [ -e {state_dir} ] && [ ! -d {state_dir} ]; }}; then exit 71; fi
|
|
mkdir -p /run/virt-cloud {state_dir}
|
|
chmod 700 /run/virt-cloud {state_dir}
|
|
[ ! -L {state_dir} ] && [ "$($STAT -c '%u:%a' {state_dir} 2>/dev/null)" = 0:700 ] || exit 71
|
|
[ ! -L "$LOCK" ] && {{ [ ! -e "$LOCK" ] || [ -f "$LOCK" ]; }} || exit 71
|
|
: >> "$LOCK"; chmod 600 "$LOCK"
|
|
exec 8>>{_q(FABRIC_LOCK)}; "$FLOCK" -x 8
|
|
exec 9>>"$LOCK"; "$FLOCK" -x 9
|
|
[ -f /etc/virt-cloud/wg-fabric0.owner ] && [ "$(cat /etc/virt-cloud/wg-fabric0.owner)" = "$FABRIC_OWNER" ] || exit 71
|
|
"$WG" show "$WG_IFACE" >/dev/null 2>&1 || exit 71
|
|
NFT_MAIN_COUNT=0
|
|
for nft_candidate in /etc/nftables.conf /etc/sysconfig/nftables.conf; do
|
|
if [ -f "$nft_candidate" ] && [ ! -L "$nft_candidate" ] && [ "$("$STAT" -c '%u' "$nft_candidate" 2>/dev/null)" = 0 ]; then
|
|
case "$("$STAT" -c '%a' "$nft_candidate" 2>/dev/null)" in
|
|
[0-7][0145][0145]) : ;;
|
|
*) continue ;;
|
|
esac
|
|
NFT_MAIN="$nft_candidate"; NFT_MAIN_COUNT=$((NFT_MAIN_COUNT + 1))
|
|
fi
|
|
done
|
|
[ "$NFT_MAIN_COUNT" -eq 1 ] || exit 71
|
|
if [ -L /etc/nftables.d ] || {{ [ -e /etc/nftables.d ] && [ ! -d /etc/nftables.d ]; }}; then exit 71; fi
|
|
if [ -e /etc/nftables.d ]; then
|
|
[ "$("$STAT" -c '%u:%a' /etc/nftables.d 2>/dev/null)" = 0:755 ] || exit 71
|
|
else
|
|
mkdir /etc/nftables.d; chmod 755 /etc/nftables.d
|
|
fi
|
|
if [ -L /etc/sysctl.d ] || {{ [ -e /etc/sysctl.d ] && [ ! -d /etc/sysctl.d ]; }}; then exit 71; fi
|
|
if [ -e /etc/sysctl.d ]; then
|
|
[ "$("$STAT" -c '%u:%a' /etc/sysctl.d 2>/dev/null)" = 0:755 ] || exit 71
|
|
else
|
|
mkdir /etc/sysctl.d; chmod 755 /etc/sysctl.d
|
|
fi
|
|
if [ -e "$SYSCTL_FILE" ] || [ -L "$SYSCTL_FILE" ]; then
|
|
[ -f "$SYSCTL_FILE" ] && [ ! -L "$SYSCTL_FILE" ] || exit 71
|
|
[ "$("$STAT" -c '%u' "$SYSCTL_FILE" 2>/dev/null)" = 0 ] || exit 71
|
|
grep -Fqx '# virt-cloud-vpc-dataplane-v1' "$SYSCTL_FILE" || exit 71
|
|
fi
|
|
if [ -e "$PROXY_FILE" ] || [ -L "$PROXY_FILE" ]; then
|
|
[ -f "$PROXY_FILE" ] && [ ! -L "$PROXY_FILE" ] || exit 71
|
|
[ "$($STAT -c '%u:%a' "$PROXY_FILE" 2>/dev/null)" = 0:600 ] || exit 71
|
|
sed -n '1p' "$PROXY_FILE" | grep -Fqx '# virt-cloud-vpc-proxy-v1' || exit 71
|
|
awk 'NR > 1 && $0 !~ /^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$/ {{ exit 1 }}' "$PROXY_FILE" || exit 71
|
|
fi
|
|
if [ -e "$NFT_FILE" ] || [ -L "$NFT_FILE" ]; then
|
|
[ -f "$NFT_FILE" ] && [ ! -L "$NFT_FILE" ] || exit 71
|
|
[ "$("$STAT" -c '%u' "$NFT_FILE" 2>/dev/null)" = 0 ] || exit 71
|
|
grep -Fqx '# virt-cloud-vpc-dataplane-v1' "$NFT_FILE" || exit 71
|
|
fi
|
|
if [ -L "$OWNER_FILE" ] || {{ [ -e "$OWNER_FILE" ] && [ ! -f "$OWNER_FILE" ]; }}; then exit 71; fi
|
|
if [ -L "$STATE_FILE" ] || {{ [ -e "$STATE_FILE" ] && [ ! -f "$STATE_FILE" ]; }}; then exit 71; fi
|
|
if [ -f "$STATE_FILE" ]; then
|
|
[ "$($STAT -c '%u:%a' "$STATE_FILE" 2>/dev/null)" = 0:600 ] || exit 71
|
|
grep -Fqx 'owner|{DATAPLANE_OWNER_MARKER}' "$STATE_FILE" || exit 71
|
|
grep -Fqx 'vpc|{plan.vpc.vpc_id}' "$STATE_FILE" || exit 71
|
|
grep -Fqx 'fabric|{plan.vpc.fabric_id}' "$STATE_FILE" || exit 71
|
|
grep -Fqx 'host|{plan.host_id}' "$STATE_FILE" || exit 71
|
|
grep -Fqx 'bridge|{r.bridge}' "$STATE_FILE" || exit 71
|
|
grep -Fqx 'table|{r.route_table}' "$STATE_FILE" || exit 71
|
|
state_revision=$(awk -F'|' '$1=="revision" {{print $2; exit}}' "$STATE_FILE")
|
|
state_digest=$(awk -F'|' '$1=="digest" {{print $2; exit}}' "$STATE_FILE")
|
|
case "$state_revision" in ''|*[!0-9]*) exit 79;; esac
|
|
printf '%s\n' "$state_digest" | grep -Eq '^[0-9a-f]{{64}}$' || exit 79
|
|
[ "$state_revision" -le {plan.desired_revision} ] || exit 79
|
|
if [ "$state_revision" = {plan.desired_revision} ] && [ "$state_digest" != { _q(plan.desired_digest) } ]; then exit 79; fi
|
|
fi
|
|
if [ -e "$TXN" ] || [ -L "$TXN" ]; then
|
|
# A process can die after writing the committed marker but before removing
|
|
# its transaction directory. That state is safe to reap after validating
|
|
# ownership; preparing/ready/rollback_failed states remain blocked so a
|
|
# later reconcile cannot guess whether runtime resources were partially
|
|
# applied.
|
|
[ -d "$TXN" ] && [ ! -L "$TXN" ] || exit 79
|
|
[ "$("$STAT" -c '%u:%a' "$TXN" 2>/dev/null)" = 0:700 ] || exit 79
|
|
[ -f "$TXN/inflight" ] && [ ! -L "$TXN/inflight" ] || exit 79
|
|
grep -Fqx '{DATAPLANE_OWNER_MARKER}:{plan.vpc.vpc_id}:{plan.host_id}:{plan.desired_revision}:{plan.desired_digest}' "$TXN/inflight" || exit 79
|
|
[ -f "$TXN/state" ] && [ "$(cat "$TXN/state" 2>/dev/null)" = committed ] || exit 79
|
|
rm -rf "$TXN" || exit 79
|
|
fi
|
|
mkdir -m 700 "$TXN"
|
|
printf '%s\\n' preparing > "$TXN/state"; chmod 600 "$TXN/state"
|
|
rollback() {{
|
|
status=$?; [ "$COMMITTED" = yes ] && exit "$status"; trap - EXIT HUP INT TERM
|
|
set +e
|
|
while IFS='|' read -r destination device table via metric; do
|
|
[ -n "$destination" ] || continue
|
|
actual=$("$IP" -o -4 route show table "$table" "$destination" 2>/dev/null || true)
|
|
[ -n "$actual" ] || continue
|
|
actual_count=$(printf '%s\n' "$actual" | awk 'NF {{count++}} END {{print count + 0}}')
|
|
actual_device=$(printf '%s\n' "$actual" | awk '{{for (i=1;i<=NF;i++) if ($i=="dev" && i<NF) {{print $(i+1); exit}}}}')
|
|
actual_via=$(printf '%s\n' "$actual" | awk '{{for (i=1;i<=NF;i++) if ($i=="via" && i<NF) {{print $(i+1); exit}}}}')
|
|
actual_metric=$(printf '%s\n' "$actual" | awk '{{for (i=1;i<=NF;i++) if ($i=="metric" && i<NF) {{print $(i+1); exit}}}}')
|
|
actual_proto=$(printf '%s\n' "$actual" | awk '{{for (i=1;i<=NF;i++) if ($i=="proto" && i<NF) {{print $(i+1); exit}}}}')
|
|
[ -n "$actual_metric" ] || actual_metric=100
|
|
if [ "$actual_count" != 1 ] || [ "$actual_device" != "$device" ] \
|
|
|| [ "${{actual_via:--}}" != "${{via:--}}" ] \
|
|
|| [ "$actual_metric" != "$metric" ] || [ "$actual_proto" != 186 ]; then
|
|
# A route with the same destination may have been replaced by an
|
|
# operator or another VPC while this transaction was failing. Never
|
|
# delete that route merely because its destination matches ours.
|
|
ROLLBACK_FAILED=yes
|
|
continue
|
|
fi
|
|
if [ "$via" = - ]; then
|
|
"$IP" route del "$destination" table "$table" dev "$device" proto 186 metric "$metric" 2>/dev/null || ROLLBACK_FAILED=yes
|
|
else
|
|
"$IP" route del "$destination" table "$table" via "$via" dev "$device" proto 186 metric "$metric" 2>/dev/null || ROLLBACK_FAILED=yes
|
|
fi
|
|
done <<'{route_marker}'
|
|
{route_lines}
|
|
{route_marker}
|
|
if [ "$ROLLBACK_FAILED" = no ] && [ -f "$TXN/routes.before" ]; then
|
|
while IFS='|' read -r old_destination old_device old_table old_via old_metric; do
|
|
[ -n "$old_destination" ] || continue
|
|
if [ "$old_via" = - ]; then
|
|
"$IP" route replace "$old_destination" table "$old_table" dev "$old_device" proto 186 metric "$old_metric" 2>/dev/null || ROLLBACK_FAILED=yes
|
|
else
|
|
"$IP" route replace "$old_destination" table "$old_table" via "$old_via" dev "$old_device" proto 186 metric "$old_metric" 2>/dev/null || ROLLBACK_FAILED=yes
|
|
fi
|
|
done < "$TXN/routes.before"
|
|
fi
|
|
if [ "$ROLLBACK_FAILED" = no ]; then
|
|
for rule_spec in "$RULE_PRIORITY|$ROUTE_TABLE|-" "$WG_RULE_PRIORITY|$ROUTE_TABLE|$WG_IFACE"; do
|
|
rule_pref=${{rule_spec%%|*}}; rule_rest=${{rule_spec#*|}}; rule_table=${{rule_rest%%|*}}; rule_iif=${{rule_rest#*|}}
|
|
actual_rule=$("$IP" -o rule show pref "$rule_pref" 2>/dev/null || true)
|
|
[ -n "$actual_rule" ] || continue
|
|
actual_rule_count=$(printf '%s\n' "$actual_rule" | awk 'NF {{count++}} END {{print count + 0}}')
|
|
if [ "$actual_rule_count" != 1 ] || ! printf '%s\n' "$actual_rule" | grep -Fq "to {_q(plan.vpc.cidr)} lookup $rule_table"; then
|
|
ROLLBACK_FAILED=yes
|
|
continue
|
|
fi
|
|
if [ "$rule_iif" = - ]; then
|
|
if printf '%s\n' "$actual_rule" | grep -Eq '(^|[[:space:]])iif[[:space:]]'; then
|
|
ROLLBACK_FAILED=yes
|
|
continue
|
|
fi
|
|
elif ! printf '%s\n' "$actual_rule" | grep -Fq "iif $rule_iif"; then
|
|
ROLLBACK_FAILED=yes
|
|
continue
|
|
fi
|
|
"$IP" rule del pref "$rule_pref" 2>/dev/null || ROLLBACK_FAILED=yes
|
|
remaining_rule=$("$IP" -o rule show pref "$rule_pref" 2>/dev/null || true)
|
|
[ -z "$remaining_rule" ] || ROLLBACK_FAILED=yes
|
|
done
|
|
fi
|
|
if [ "$ROLLBACK_FAILED" = no ] && [ -f "$TXN/rules.before" ]; then
|
|
while IFS='|' read -r old_pref old_iif old_to old_lookup; do
|
|
[ -n "$old_pref" ] || continue
|
|
if [ "$old_iif" = - ]; then
|
|
"$IP" rule add pref "$old_pref" to "$old_to" lookup "$old_lookup" 2>/dev/null || ROLLBACK_FAILED=yes
|
|
else
|
|
"$IP" rule add pref "$old_pref" iif "$old_iif" to "$old_to" lookup "$old_lookup" 2>/dev/null || ROLLBACK_FAILED=yes
|
|
fi
|
|
done < "$TXN/rules.before"
|
|
fi
|
|
if [ -f "$TXN/wg.conf" ]; then "$WG" setconf "$WG_IFACE" "$TXN/wg.conf" >/dev/null 2>&1 || ROLLBACK_FAILED=yes; fi
|
|
if [ "$PROXY_ENTRIES_CHANGED" = yes ]; then
|
|
"$IP" neigh show proxy dev "$BRIDGE" 2>/dev/null | awk '$1 ~ /^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$/ {{print $1}}' | LC_ALL=C sort -u > "$TXN/proxy.rollback.current"
|
|
cat <<'{proxy_marker}' | LC_ALL=C sort -u > "$TXN/proxy.rollback.desired"
|
|
{proxy_lines}
|
|
{proxy_marker}
|
|
if "$CMP" -s "$TXN/proxy.rollback.current" "$TXN/proxy.rollback.desired"; then
|
|
while IFS= read -r rollback_proxy; do
|
|
[ -n "$rollback_proxy" ] || continue
|
|
"$IP" neigh del proxy "$rollback_proxy" dev "$BRIDGE" >/dev/null 2>&1 || ROLLBACK_FAILED=yes
|
|
done < "$TXN/proxy.rollback.desired"
|
|
while IFS= read -r rollback_proxy; do
|
|
[ -n "$rollback_proxy" ] || continue
|
|
"$IP" neigh replace proxy "$rollback_proxy" dev "$BRIDGE" >/dev/null 2>&1 || ROLLBACK_FAILED=yes
|
|
done < "$TXN/proxy.before"
|
|
else
|
|
# Do not delete an entry added concurrently by an operator or another
|
|
# controller. Leave the transaction recoverable and report rollback
|
|
# failure rather than risking an unrelated ARP outage.
|
|
ROLLBACK_FAILED=yes
|
|
fi
|
|
fi
|
|
if [ "$PROXY_FILE_CHANGED" = yes ]; then
|
|
if [ -f "$TXN/proxy_file.before" ]; then
|
|
cp -p "$TXN/proxy_file.before" "$PROXY_FILE" || ROLLBACK_FAILED=yes
|
|
else
|
|
rm -f "$PROXY_FILE" || ROLLBACK_FAILED=yes
|
|
fi
|
|
fi
|
|
# Restore runtime sysctls while the new ARP guard is still installed; this
|
|
# avoids a rollback window in which proxy_arp is enabled without its exact
|
|
# output filter.
|
|
if [ "$PROXY_ARP_CHANGED" = yes ] && [ -e "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp" ]; then
|
|
if [ -f "$TXN/proxy_arp.before" ]; then
|
|
cat "$TXN/proxy_arp.before" > "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp" || ROLLBACK_FAILED=yes
|
|
else
|
|
printf '%s\\n' 0 > "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp" || ROLLBACK_FAILED=yes
|
|
fi
|
|
fi
|
|
if [ "$PROXY_ARP_PVLAN_CHANGED" = yes ] && [ -e "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp_pvlan" ]; then
|
|
if [ -f "$TXN/proxy_arp_pvlan.before" ]; then
|
|
cat "$TXN/proxy_arp_pvlan.before" > "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp_pvlan" || ROLLBACK_FAILED=yes
|
|
else
|
|
printf '%s\n' 0 > "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp_pvlan" || ROLLBACK_FAILED=yes
|
|
fi
|
|
fi
|
|
if [ "$RP_FILTER_CHANGED" = yes ] && [ -e "/proc/sys/net/ipv4/conf/$BRIDGE/rp_filter" ]; then
|
|
if [ -f "$TXN/rp_filter.before" ]; then
|
|
cat "$TXN/rp_filter.before" > "/proc/sys/net/ipv4/conf/$BRIDGE/rp_filter" || ROLLBACK_FAILED=yes
|
|
fi
|
|
fi
|
|
NFT_ROLLBACK="$TXN/rollback.nft"
|
|
: > "$NFT_ROLLBACK"
|
|
if "$NFT" list table inet "$TABLE" >/dev/null 2>&1; then printf 'delete table inet %s\\n' "$TABLE" >> "$NFT_ROLLBACK"; fi
|
|
if "$NFT" list table bridge "$TABLE" >/dev/null 2>&1; then printf 'delete table bridge %s\\n' "$TABLE" >> "$NFT_ROLLBACK"; fi
|
|
if "$NFT" list table arp "$TABLE" >/dev/null 2>&1; then printf 'delete table arp %s\\n' "$TABLE" >> "$NFT_ROLLBACK"; fi
|
|
if [ -f "$TXN/nft.before" ]; then cat "$TXN/nft.before" >> "$NFT_ROLLBACK"; fi
|
|
if [ -f "$TXN/nft.bridge.before" ]; then cat "$TXN/nft.bridge.before" >> "$NFT_ROLLBACK"; fi
|
|
if [ -f "$TXN/nft.arp.before" ]; then cat "$TXN/nft.arp.before" >> "$NFT_ROLLBACK"; fi
|
|
if [ -s "$NFT_ROLLBACK" ]; then
|
|
"$NFT" -c -f "$NFT_ROLLBACK" >/dev/null 2>&1 || ROLLBACK_FAILED=yes
|
|
"$NFT" -f "$NFT_ROLLBACK" >/dev/null 2>&1 || ROLLBACK_FAILED=yes
|
|
fi
|
|
if [ -f "$TXN/fabric.nft.before" ]; then
|
|
"$NFT" delete table inet virt_cloud_wg_fabric >/dev/null 2>&1 || true
|
|
"$NFT" -f "$TXN/fabric.nft.before" >/dev/null 2>&1 || ROLLBACK_FAILED=yes
|
|
fi
|
|
if [ "$NFT_FILE_CHANGED" = yes ]; then
|
|
if [ -f "$TXN/nft_file.before" ]; then cp -p "$TXN/nft_file.before" "$NFT_FILE" || ROLLBACK_FAILED=yes; else rm -f "$NFT_FILE" || ROLLBACK_FAILED=yes; fi
|
|
fi
|
|
if [ "$NFT_MAIN_CHANGED" = yes ] && [ -f "$TXN/nft_main.before" ]; then
|
|
cp -p "$TXN/nft_main.before" "$NFT_MAIN" || ROLLBACK_FAILED=yes
|
|
fi
|
|
if [ "$MODE" = networkmanager_nmcli ] && [ -n "$NMCLI" ]; then
|
|
if [ "$NM_ACTIVATION_ATTEMPTED" = yes ] && [ "$NM_WAS_ACTIVE" = no ]; then
|
|
"$NMCLI" connection down "$BRIDGE" >/dev/null 2>&1 || ROLLBACK_FAILED=yes
|
|
if "$NMCLI" -t --escape no -f NAME connection show --active 2>/dev/null | grep -Fqx "$BRIDGE"; then ROLLBACK_FAILED=yes; fi
|
|
fi
|
|
[ "$NM_CREATED" = yes ] && "$NMCLI" connection delete "$BRIDGE" >/dev/null 2>&1 || true
|
|
elif [ "$MODE" = ifupdown ]; then
|
|
if [ "$IF_ACTIVATION_ATTEMPTED" = yes ] && [ "$BRIDGE_WAS_UP" = no ]; then
|
|
"$IFDOWN" --force "$BRIDGE" >/dev/null 2>&1 || ROLLBACK_FAILED=yes
|
|
fi
|
|
[ "$IF_CREATED" = yes ] && "$IFDOWN" --force "$BRIDGE" >/dev/null 2>&1 || true
|
|
fi
|
|
if [ "$IF_FRAGMENT_CHANGED" = yes ]; then
|
|
if [ -f "$TXN/if_fragment.before" ]; then cp -p "$TXN/if_fragment.before" "$IF_FRAGMENT" || ROLLBACK_FAILED=yes; else rm -f "$IF_FRAGMENT" || ROLLBACK_FAILED=yes; fi
|
|
fi
|
|
if [ "$SYSCTL_CHANGED" = yes ]; then
|
|
if [ -f "$TXN/sysctl.before" ]; then
|
|
cp -p "$TXN/sysctl.before" "$SYSCTL_FILE" || ROLLBACK_FAILED=yes
|
|
else
|
|
rm -f "$SYSCTL_FILE" || ROLLBACK_FAILED=yes
|
|
fi
|
|
fi
|
|
if [ "$BRIDGE_CREATED" = yes ]; then
|
|
# Never delete a bridge that acquired a libvirt tap or an operator port.
|
|
ports=$($BRIDGE_CMD link show master "$BRIDGE" 2>/dev/null)
|
|
if [ "$?" -ne 0 ]; then
|
|
# An inability to inspect ports is a rollback failure. Treating an
|
|
# empty error result as "no ports" could delete a bridge still carrying
|
|
# a libvirt tap and strand the host network.
|
|
ROLLBACK_FAILED=yes
|
|
elif [ -z "$ports" ]; then
|
|
"$IP" link del dev "$BRIDGE" type bridge 2>/dev/null || ROLLBACK_FAILED=yes
|
|
fi
|
|
elif [ "$BRIDGE_WAS_UP" = no ] && "$IP" link show dev "$BRIDGE" >/dev/null 2>&1; then
|
|
# Applying a revision may have activated an existing panel bridge through
|
|
# NetworkManager/ifupdown. Restore its prior DOWN state only when no tap
|
|
# or operator port appeared while the transaction was running.
|
|
ports=$($BRIDGE_CMD link show master "$BRIDGE" 2>/dev/null)
|
|
if [ "$?" -ne 0 ] || [ -n "$ports" ]; then
|
|
ROLLBACK_FAILED=yes
|
|
else
|
|
"$IP" link set dev "$BRIDGE" down 2>/dev/null || ROLLBACK_FAILED=yes
|
|
"$IP" link show dev "$BRIDGE" 2>/dev/null | grep -q 'state UP' && ROLLBACK_FAILED=yes
|
|
fi
|
|
fi
|
|
if [ -f "$TXN/owner.before" ]; then cp -p "$TXN/owner.before" "$OWNER_FILE" || ROLLBACK_FAILED=yes; else rm -f "$OWNER_FILE"; fi
|
|
if [ -f "$TXN/state.before" ]; then cp -p "$TXN/state.before" "$STATE_FILE" || ROLLBACK_FAILED=yes; else rm -f "$STATE_FILE"; fi
|
|
if [ "$ROLLBACK_FAILED" = yes ]; then printf '%s\\n' rollback_failed > "$TXN/state"; chmod 600 "$TXN/state"; exit 79; fi
|
|
rm -rf "$TXN"; exit "$status"
|
|
}}
|
|
trap rollback EXIT HUP INT TERM
|
|
if [ -f "$OWNER_FILE" ]; then
|
|
[ ! -L "$OWNER_FILE" ] && grep -Fqx "$OWNER_MARKER" "$OWNER_FILE" || exit 71
|
|
cp -p "$OWNER_FILE" "$TXN/owner.before"
|
|
fi
|
|
if [ -f "$STATE_FILE" ]; then
|
|
[ ! -L "$STATE_FILE" ] || exit 71
|
|
grep -Fqx 'owner|{DATAPLANE_OWNER_MARKER}' "$STATE_FILE" || exit 71
|
|
cp -p "$STATE_FILE" "$TXN/state.before"
|
|
fi
|
|
"$WG" showconf "$WG_IFACE" > "$TXN/wg.conf"; chmod 600 "$TXN/wg.conf"
|
|
if "$NFT" list table inet virt_cloud_wg_fabric > "$TXN/fabric.nft.before" 2>/dev/null; then
|
|
grep -Fq 'virt-cloud-wg-fabric-v1' "$TXN/fabric.nft.before" || exit 71
|
|
else
|
|
exit 71
|
|
fi
|
|
if "$NFT" list table inet "$TABLE" > "$TXN/nft.before" 2>/dev/null; then
|
|
grep -Fq '{DATAPLANE_OWNER_MARKER}' "$TXN/nft.before" || exit 71
|
|
else
|
|
rm -f "$TXN/nft.before"
|
|
fi
|
|
if "$NFT" list table bridge "$TABLE" > "$TXN/nft.bridge.before" 2>/dev/null; then
|
|
grep -Fq '{DATAPLANE_OWNER_MARKER}' "$TXN/nft.bridge.before" || exit 71
|
|
else
|
|
rm -f "$TXN/nft.bridge.before"
|
|
fi
|
|
if "$NFT" list table arp "$TABLE" > "$TXN/nft.arp.before" 2>/dev/null; then
|
|
grep -Fq '{DATAPLANE_OWNER_MARKER}' "$TXN/nft.arp.before" || exit 71
|
|
else
|
|
rm -f "$TXN/nft.arp.before"
|
|
fi
|
|
cp -p "$NFT_MAIN" "$TXN/nft_main.before"; NFT_MAIN_BEFORE=yes
|
|
if [ -f "$NFT_FILE" ]; then cp -p "$NFT_FILE" "$TXN/nft_file.before"; NFT_FILE_BEFORE=yes; fi
|
|
if [ -f "$SYSCTL_FILE" ]; then cp -p "$SYSCTL_FILE" "$TXN/sysctl.before"; fi
|
|
if [ -f "$PROXY_FILE" ]; then cp -p "$PROXY_FILE" "$TXN/proxy_file.before"; fi
|
|
if "$IP" link show dev "$BRIDGE" >/dev/null 2>&1; then
|
|
"$IP" -d link show dev "$BRIDGE" | grep -Eq '(^|[[:space:]])bridge[[:space:]]+(forward_delay|stp_state|hello_time|ageing_time)([[:space:]]|$)' || exit 71
|
|
[ -f "$OWNER_FILE" ] || exit 71
|
|
else
|
|
"$IP" link add name "$BRIDGE" type bridge
|
|
BRIDGE_CREATED=yes
|
|
printf '%s\\n' "$OWNER_MARKER" > "$OWNER_FILE"; chmod 600 "$OWNER_FILE"
|
|
fi
|
|
if "$IP" link show dev "$BRIDGE" 2>/dev/null | grep -q 'state UP'; then BRIDGE_WAS_UP=yes; fi
|
|
"$IP" link set dev "$BRIDGE" up
|
|
# A private bridge is an unnumbered, panel-owned L3 boundary. Reject an
|
|
# address, a default/connected route, or a physical/foreign bridge port before
|
|
# installing any policy. Libvirt tap names are the only accepted members;
|
|
# attaching br0, an underlay NIC, or an operator bridge would make VPC traffic
|
|
# escape into the public network.
|
|
bridge_addrs=$($IP -o addr show dev "$BRIDGE" 2>/dev/null || true)
|
|
printf '%s\\n' "$bridge_addrs" | grep -Eq '[[:space:]]inet6?[[:space:]]' && exit 71 || true
|
|
bridge_ports=$($BRIDGE_CMD link show master "$BRIDGE" 2>/dev/null) || exit 71
|
|
while IFS= read -r bridge_port_line; do
|
|
[ -n "$bridge_port_line" ] || continue
|
|
bridge_port=$(printf '%s\\n' "$bridge_port_line" | sed -n 's/^[0-9][0-9]*: \\([^:[:space:]]*\\).*/\\1/p')
|
|
[ -n "$bridge_port" ] || exit 71
|
|
case "$bridge_port" in
|
|
vnet[0-9]*|tap[0-9A-Za-z_.-]*|macvtap[0-9]*|fwpr[0-9A-Za-z_.-]*|fwln[0-9A-Za-z_.-]*) : ;;
|
|
*) exit 71 ;;
|
|
esac
|
|
done <<__VC_VPC_BRIDGE_PORTS__
|
|
$bridge_ports
|
|
__VC_VPC_BRIDGE_PORTS__
|
|
cat > "$TXN/bridge.routes.allowed" <<'{route_marker}.bridge'
|
|
{bridge_route_lines}
|
|
{route_marker}.bridge
|
|
if [ -f "$TXN/state.before" ]; then
|
|
awk -F'|' '$1 == "route" && $3 == "{r.bridge}" {{print $2 "|" $3 "|" $4 "|" $5 "|" $6 "|186"}}' "$TXN/state.before" >> "$TXN/bridge.routes.allowed"
|
|
fi
|
|
sort -u "$TXN/bridge.routes.allowed" -o "$TXN/bridge.routes.allowed"
|
|
bridge_routes=$($IP -o -4 route show table all dev "$BRIDGE" 2>/dev/null) || exit 71
|
|
while IFS='|' read -r destination device table via metric proto; do
|
|
[ -n "$destination" ] || continue
|
|
printf '%s\\n' "$destination|$device|$table|$via|$metric|$proto" | grep -Fqx -f "$TXN/bridge.routes.allowed" || exit 71
|
|
done <<EOF
|
|
$(printf '%s\\n' "$bridge_routes" | awk '{{
|
|
destination=$1; device=""; via="-"; metric="100"; table="main"; proto=""
|
|
for (i=1;i<=NF;i++) {{
|
|
if ($i=="dev" && i<NF) device=$(i+1)
|
|
if ($i=="via" && i<NF) via=$(i+1)
|
|
if ($i=="metric" && i<NF) metric=$(i+1)
|
|
if ($i=="table" && i<NF) table=$(i+1)
|
|
if ($i=="proto" && i<NF) proto=$(i+1)
|
|
}}
|
|
if (table=="254") table="main"
|
|
if (destination != "") printf "%s|%s|%s|%s|%s|%s\\n", destination, device, table, via, metric, proto
|
|
}}')
|
|
EOF
|
|
bridge_routes6=$($IP -o -6 route show table all dev "$BRIDGE" 2>/dev/null) || exit 71
|
|
[ -z "$bridge_routes6" ] || exit 71
|
|
: > "$TXN/proxy.before"
|
|
"$IP" neigh show proxy dev "$BRIDGE" 2>/dev/null | awk '$1 ~ /^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$/ {{print $1}}' | LC_ALL=C sort -u > "$TXN/proxy.before"
|
|
if [ -f "$PROXY_FILE" ]; then
|
|
sed '1d' "$PROXY_FILE" | LC_ALL=C sort -u > "$TXN/proxy.file.before"
|
|
else
|
|
: > "$TXN/proxy.file.before"
|
|
fi
|
|
"$CMP" -s "$TXN/proxy.before" "$TXN/proxy.file.before" || exit 71
|
|
if [ -r "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp" ]; then
|
|
cat "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp" > "$TXN/proxy_arp.before"
|
|
else
|
|
exit 72
|
|
fi
|
|
if [ -r "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp_pvlan" ]; then
|
|
cat "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp_pvlan" > "$TXN/proxy_arp_pvlan.before"
|
|
else
|
|
exit 72
|
|
fi
|
|
if [ -r "/proc/sys/net/ipv4/conf/$BRIDGE/rp_filter" ]; then
|
|
cat "/proc/sys/net/ipv4/conf/$BRIDGE/rp_filter" > "$TXN/rp_filter.before"
|
|
else
|
|
exit 72
|
|
fi
|
|
SYSCTL_TMP=$(mktemp "/etc/sysctl.d/.virt-cloud-vpc-{r.token}.XXXXXX") || exit 72
|
|
cat > "$SYSCTL_TMP" <<'__VC_VPC_SYSCTL_{r.token}'
|
|
# virt-cloud-vpc-dataplane-v1
|
|
# vpc={plan.vpc.vpc_id} revision={plan.desired_revision} digest={plan.desired_digest}
|
|
net.ipv4.conf.{r.bridge}.rp_filter = 2
|
|
net.ipv4.conf.{r.bridge}.proxy_arp = 0
|
|
net.ipv4.conf.{r.bridge}.proxy_arp_pvlan = 1
|
|
__VC_VPC_SYSCTL_{r.token}
|
|
chmod 600 "$SYSCTL_TMP"
|
|
mv -f "$SYSCTL_TMP" "$SYSCTL_FILE"; SYSCTL_CHANGED=yes
|
|
if [ "$MODE" = networkmanager_nmcli ]; then
|
|
[ -n "$NMCLI" ] || exit 70
|
|
nm_field() {{ "$NMCLI" -t --escape no -g "$1" connection show "$BRIDGE" 2>/dev/null | sed -n '1p'; }}
|
|
nm_empty() {{ value=$(nm_field "$1"); [ -z "$value" ] || [ "$value" = "--" ]; }}
|
|
NM_PROFILE_COUNT=$("$NMCLI" -t --escape no -f NAME connection show 2>/dev/null | awk -v wanted="$BRIDGE" '$0 == wanted {{ count++ }} END {{ print count + 0 }}')
|
|
if [ "$NM_PROFILE_COUNT" = 0 ]; then
|
|
"$NMCLI" connection add type bridge ifname "$BRIDGE" con-name "$BRIDGE" \
|
|
ipv4.method disabled ipv6.method disabled bridge.stp no \
|
|
connection.autoconnect yes connection.stable-id "$NM_PROFILE_MARKER" >/dev/null
|
|
NM_CREATED=yes
|
|
elif [ "$NM_PROFILE_COUNT" = 1 ]; then
|
|
# A deterministic profile name is not ownership by itself. Reusing an
|
|
# operator profile with an address, route, controller, or non-bridge type
|
|
# could leak VPC traffic into the public network. Require the exact
|
|
# panel marker (stored in NetworkManager's supported stable-id property)
|
|
# and validate every field that can add L3 reachability before activating
|
|
# an existing profile.
|
|
[ "$(nm_field connection.type)" = bridge ] || exit 71
|
|
[ "$(nm_field connection.interface-name)" = "$BRIDGE" ] || exit 71
|
|
[ "$(nm_field ipv4.method)" = disabled ] || exit 71
|
|
[ "$(nm_field ipv6.method)" = disabled ] || exit 71
|
|
nm_empty ipv4.addresses || exit 71
|
|
nm_empty ipv4.routes || exit 71
|
|
nm_empty ipv4.routing-rules || exit 71
|
|
nm_empty ipv4.gateway || exit 71
|
|
nm_empty ipv4.dns || exit 71
|
|
nm_empty ipv4.dns-search || exit 71
|
|
nm_empty ipv6.addresses || exit 71
|
|
nm_empty ipv6.routes || exit 71
|
|
nm_empty ipv6.routing-rules || exit 71
|
|
nm_empty ipv6.gateway || exit 71
|
|
nm_empty ipv6.dns || exit 71
|
|
nm_empty ipv6.dns-search || exit 71
|
|
nm_empty connection.master || exit 71
|
|
nm_empty connection.controller || exit 71
|
|
nm_empty connection.slave-type || exit 71
|
|
nm_empty connection.port-type || exit 71
|
|
nm_empty connection.secondaries || exit 71
|
|
nm_empty connection.zone || exit 71
|
|
case "$(nm_field bridge.stp)" in no|false|off|0|--) : ;; *) exit 71 ;; esac
|
|
NM_PROFILE_STABLE_ID=$(nm_field connection.stable-id)
|
|
[ "$NM_PROFILE_STABLE_ID" = "$NM_PROFILE_MARKER" ] || exit 71
|
|
else
|
|
# More than one same-name profile makes `connection up <name>` ambiguous;
|
|
# fail closed instead of possibly activating a public profile.
|
|
exit 71
|
|
fi
|
|
if "$NMCLI" -t --escape no -f NAME connection show --active 2>/dev/null | grep -Fqx "$BRIDGE"; then NM_WAS_ACTIVE=yes; fi
|
|
NM_ACTIVATION_ATTEMPTED=yes
|
|
"$NMCLI" connection up "$BRIDGE" >/dev/null
|
|
else
|
|
[ -x "$IFUP" ] && [ -x "$IFQUERY" ] && [ -x "$IFDOWN" ] || exit 70
|
|
[ ! -L /etc/network ] && [ ! -L /etc/network/interfaces.d ] || exit 71
|
|
# A stanza in the administrator-owned main file wins over an
|
|
# interfaces.d fragment on some ifupdown versions. Refuse that ambiguous
|
|
# ownership instead of letting a same-name bridge inherit ports, addresses,
|
|
# routes, or hooks from /etc/network/interfaces.
|
|
if [ -f /etc/network/interfaces ] &&
|
|
grep -Eq '^[[:space:]]*(auto|allow-[^[:space:]]+|iface)[[:space:]]+'"$BRIDGE"'([[:space:]]|$)' /etc/network/interfaces; then
|
|
exit 71
|
|
fi
|
|
mkdir -p /etc/network/interfaces.d
|
|
if [ -e "$IF_FRAGMENT" ] || [ -L "$IF_FRAGMENT" ]; then
|
|
[ -f "$IF_FRAGMENT" ] && [ ! -L "$IF_FRAGMENT" ] || exit 71
|
|
grep -Fq '# {DATAPLANE_OWNER_MARKER} vpc=' "$IF_FRAGMENT" || exit 71
|
|
cp -p "$IF_FRAGMENT" "$TXN/if_fragment.before"
|
|
fi
|
|
IF_TMP=$(mktemp "/etc/network/interfaces.d/.virt-cloud-vpc-{r.token}.XXXXXX") || exit 72
|
|
cat > "$IF_TMP" <<'{frag_marker}'
|
|
# {DATAPLANE_OWNER_MARKER} vpc={plan.vpc.vpc_id} revision={plan.desired_revision} digest={plan.desired_digest}
|
|
auto {r.bridge}
|
|
iface {r.bridge} inet manual
|
|
bridge_ports none
|
|
bridge-stp off
|
|
{frag_marker}
|
|
chmod 600 "$IF_TMP"
|
|
mv -f "$IF_TMP" "$IF_FRAGMENT"; IF_FRAGMENT_CHANGED=yes
|
|
# Classic Debian ifquery accepts the interface as its positional operand;
|
|
# `--list <iface>` is an ifquery2/listing form and fails on ifupdown.
|
|
"$IFQUERY" "$BRIDGE" >/dev/null 2>&1 || exit 72
|
|
# An existing bridge is shared with libvirt; never ifdown it during
|
|
# rollback. A newly created bridge is the only interface this transaction
|
|
# may tear down.
|
|
[ "$BRIDGE_CREATED" = yes ] && IF_CREATED=yes
|
|
IF_ACTIVATION_ATTEMPTED=yes
|
|
VC_FABRIC_APPLYING=1 "$IFUP" --force "$BRIDGE" >/dev/null 2>&1 || exit 72
|
|
fi
|
|
cat > "$TXN/rules.nft" <<'{nft_marker}'
|
|
{nft.rstrip()}
|
|
{nft_marker}
|
|
chmod 600 "$TXN/rules.nft"
|
|
"$NFT" -c -f "$TXN/rules.nft" >/dev/null || exit 72
|
|
NFT_TMP=$(mktemp "/etc/nftables.d/.virt-cloud-vpc-{r.token}.XXXXXX") || exit 72
|
|
cp -p "$TXN/rules.nft" "$NFT_TMP" || exit 72
|
|
chmod 600 "$NFT_TMP"
|
|
mv -f "$NFT_TMP" "$NFT_FILE"; NFT_FILE_CHANGED=yes
|
|
if ! grep -Fqx "$NFT_INCLUDE" "$NFT_MAIN" &&
|
|
! grep -Fqx 'include "/etc/nftables.d/*.nft"' "$NFT_MAIN" &&
|
|
! grep -Fqx 'include "/etc/nftables.d/*"' "$NFT_MAIN"; then
|
|
NFT_MAIN_TMP=$(mktemp "$NFT_MAIN.virt-cloud-vpc.XXXXXX") || exit 72
|
|
cat "$NFT_MAIN" > "$NFT_MAIN_TMP"
|
|
printf '%s\\n' "$NFT_INCLUDE" >> "$NFT_MAIN_TMP"
|
|
"$NFT" -c -f "$NFT_MAIN_TMP" >/dev/null || exit 72
|
|
chmod "$($STAT -c '%a' "$NFT_MAIN")" "$NFT_MAIN_TMP"
|
|
mv -f "$NFT_MAIN_TMP" "$NFT_MAIN"; NFT_MAIN_CHANGED=yes
|
|
fi
|
|
"$NFT" -c -f "$NFT_MAIN" >/dev/null || exit 72
|
|
NFT_BATCH="$TXN/apply.nft"
|
|
: > "$NFT_BATCH"
|
|
if [ -f "$TXN/nft.before" ]; then printf 'delete table inet %s\\n' "$TABLE" >> "$NFT_BATCH"; fi
|
|
if [ -f "$TXN/nft.bridge.before" ]; then printf 'delete table bridge %s\\n' "$TABLE" >> "$NFT_BATCH"; fi
|
|
if [ -f "$TXN/nft.arp.before" ]; then printf 'delete table arp %s\\n' "$TABLE" >> "$NFT_BATCH"; fi
|
|
cat "$TXN/rules.nft" >> "$NFT_BATCH"
|
|
chmod 600 "$NFT_BATCH"
|
|
"$NFT" -c -f "$NFT_BATCH" >/dev/null || exit 72
|
|
# A single nft batch is atomic in the kernel: the old owned tables remain in
|
|
# place if any delete or new-table rule fails validation/application.
|
|
"$NFT" -f "$NFT_BATCH" >/dev/null || exit 72
|
|
RP_FILTER_CHANGED=yes
|
|
printf '%s\\n' 2 > "/proc/sys/net/ipv4/conf/$BRIDGE/rp_filter" || exit 72
|
|
[ "$(cat "/proc/sys/net/ipv4/conf/$BRIDGE/rp_filter" 2>/dev/null)" = 2 ] || exit 72
|
|
PROXY_ARP_CHANGED=yes
|
|
printf '%s\\n' 0 > "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp" || exit 72
|
|
[ "$(cat "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp" 2>/dev/null)" = 0 ] || exit 72
|
|
PROXY_ARP_PVLAN_CHANGED=yes
|
|
printf '%s\\n' 1 > "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp_pvlan" || exit 72
|
|
[ "$(cat "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp_pvlan" 2>/dev/null)" = 1 ] || exit 72
|
|
PROXY_ENTRIES_CHANGED=yes
|
|
while IFS= read -r old_proxy; do
|
|
[ -n "$old_proxy" ] || continue
|
|
"$IP" neigh del proxy "$old_proxy" dev "$BRIDGE" >/dev/null 2>&1 || exit 72
|
|
done < "$TXN/proxy.before"
|
|
while IFS= read -r proxy_ip; do
|
|
[ -n "$proxy_ip" ] || continue
|
|
"$IP" neigh replace proxy "$proxy_ip" dev "$BRIDGE" >/dev/null 2>&1 || exit 72
|
|
done <<'{proxy_marker}'
|
|
{proxy_lines}
|
|
{proxy_marker}
|
|
PROXY_TMP=$(mktemp "$PROXY_FILE.XXXXXX") || exit 72
|
|
{{
|
|
printf '%s\\n' '# virt-cloud-vpc-proxy-v1'
|
|
cat <<'{proxy_marker}'
|
|
{proxy_lines}
|
|
{proxy_marker}
|
|
}} > "$PROXY_TMP"
|
|
chmod 600 "$PROXY_TMP"
|
|
mv -f "$PROXY_TMP" "$PROXY_FILE"; PROXY_FILE_CHANGED=yes
|
|
"$IP" neigh show proxy dev "$BRIDGE" 2>/dev/null | awk '$1 ~ /^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$/ {{print $1}}' | LC_ALL=C sort -u > "$TXN/proxy.after"
|
|
cat <<'{proxy_marker}' | LC_ALL=C sort -u > "$TXN/proxy.desired"
|
|
{proxy_lines}
|
|
{proxy_marker}
|
|
"$CMP" -s "$TXN/proxy.after" "$TXN/proxy.desired" || exit 72
|
|
: > "$TXN/routes.before"
|
|
cat > "$TXN/routes.desired" <<'{route_marker}'
|
|
{route_lines}
|
|
{route_marker}
|
|
if [ -f "$STATE_FILE" ]; then
|
|
while IFS='|' read -r old_kind old_destination old_device old_table old_via old_metric; do
|
|
[ "$old_kind" = route ] || continue
|
|
printf '%s\\n' "$old_destination|$old_device|$ROUTE_TABLE|${{old_via:--}}|${{old_metric:-100}}" | grep -Fqx -f "$TXN/routes.desired" && continue
|
|
old_existing=$("$IP" -o route show table "$ROUTE_TABLE" "$old_destination" 2>/dev/null || true)
|
|
[ -n "$old_existing" ] || continue
|
|
printf '%s\\n' "$old_existing" | grep -Eq 'proto 186([[:space:]]|$)' || exit 73
|
|
old_metric_actual=$(printf '%s\\n' "$old_existing" | awk '{{for (i=1; i<=NF; i++) if ($i == "metric" && i < NF) {{print $(i+1); exit}}}}')
|
|
[ -n "$old_metric_actual" ] || old_metric_actual=100
|
|
old_actual_device=$(printf '%s\\n' "$old_existing" | awk '{{for (i=1; i<=NF; i++) if ($i == "dev" && i < NF) {{print $(i+1); exit}}}}')
|
|
[ "$old_actual_device" = "$old_device" ] || exit 73
|
|
old_via_actual=$(printf '%s\\n' "$old_existing" | awk '{{for (i=1; i<=NF; i++) if ($i == "via" && i < NF) {{print $(i+1); exit}}}}')
|
|
[ "${{old_via_actual:--}}" = "${{old_via:--}}" ] || exit 73
|
|
printf '%s|%s|%s|%s|%s\\n' "$old_destination" "$old_device" "$ROUTE_TABLE" "${{old_via:--}}" "$old_metric_actual" >> "$TXN/routes.before"
|
|
if [ "${{old_via:--}}" = - ]; then
|
|
"$IP" route del "$old_destination" table "$ROUTE_TABLE" dev "$old_device" proto 186 || exit 73
|
|
else
|
|
"$IP" route del "$old_destination" table "$ROUTE_TABLE" via "$old_via" dev "$old_device" proto 186 || exit 73
|
|
fi
|
|
done < "$STATE_FILE"
|
|
fi
|
|
while IFS='|' read -r destination device table via metric; do
|
|
[ -n "$destination" ] || continue
|
|
existing=$("$IP" -o route show table "$table" "$destination" 2>/dev/null || true)
|
|
[ -z "$existing" ] || printf '%s\\n' "$existing" | grep -Eq 'proto 186([[:space:]]|$)' || exit 73
|
|
if [ -n "$existing" ]; then
|
|
old_device=$(printf '%s\\n' "$existing" | awk '{{for (i=1; i<=NF; i++) if ($i == "dev" && i < NF) {{print $(i+1); exit}}}}')
|
|
old_metric=$(printf '%s\\n' "$existing" | awk '{{for (i=1; i<=NF; i++) if ($i == "metric" && i < NF) {{print $(i+1); exit}}}}')
|
|
[ -n "$old_device" ] || exit 73
|
|
[ "$old_device" = "$device" ] || exit 73
|
|
old_via=$(printf '%s\\n' "$existing" | awk '{{for (i=1; i<=NF; i++) if ($i == "via" && i < NF) {{print $(i+1); exit}}}}')
|
|
[ "${{old_via:--}}" = "$via" ] || exit 73
|
|
[ -n "$old_metric" ] || old_metric=100
|
|
printf '%s|%s|%s|%s|%s\\n' "$destination" "$old_device" "$table" "${{old_via:--}}" "$old_metric" >> "$TXN/routes.before"
|
|
fi
|
|
[ -n "$metric" ] || metric=100
|
|
if [ "$via" = - ]; then
|
|
"$IP" route replace "$destination" table "$table" dev "$device" proto 186 metric "$metric" || exit 73
|
|
else
|
|
"$IP" route replace "$destination" table "$table" via "$via" dev "$device" onlink proto 186 metric "$metric" || exit 73
|
|
fi
|
|
done <<'{route_marker}'
|
|
{route_lines}
|
|
{route_marker}
|
|
: > "$TXN/rules.before"
|
|
for pref in "$RULE_PRIORITY" "$WG_RULE_PRIORITY"; do
|
|
existing=$("$IP" -o rule show pref "$pref" 2>/dev/null || true)
|
|
if [ -n "$existing" ]; then
|
|
printf '%s\\n' "$existing" | grep -Fq "lookup $ROUTE_TABLE" || exit 73
|
|
old_iif=$(printf '%s\\n' "$existing" | awk '{{for (i=1; i<=NF; i++) if ($i == "iif" && i < NF) {{print $(i+1); exit}}}}')
|
|
old_to=$(printf '%s\\n' "$existing" | awk '{{for (i=1; i<=NF; i++) if ($i == "to" && i < NF) {{print $(i+1); exit}}}}')
|
|
old_lookup=$(printf '%s\\n' "$existing" | awk '{{for (i=1; i<=NF; i++) if ($i == "lookup" && i < NF) {{print $(i+1); exit}}}}')
|
|
[ -n "$old_to" ] && [ -n "$old_lookup" ] || exit 73
|
|
if [ "$pref" = "$RULE_PRIORITY" ]; then
|
|
[ -z "$old_iif" ] && [ "$old_to" = {_q(plan.vpc.cidr)} ] || exit 73
|
|
else
|
|
[ "$old_iif" = "$WG_IFACE" ] && [ "$old_to" = {_q(plan.vpc.cidr)} ] || exit 73
|
|
fi
|
|
printf '%s|%s|%s|%s\\n' "$pref" "${{old_iif:--}}" "$old_to" "$old_lookup" >> "$TXN/rules.before"
|
|
fi
|
|
done
|
|
"$IP" rule replace pref "$RULE_PRIORITY" to {_q(plan.vpc.cidr)} lookup "$ROUTE_TABLE"
|
|
"$IP" rule replace pref "$WG_RULE_PRIORITY" iif "$WG_IFACE" to {_q(plan.vpc.cidr)} lookup "$ROUTE_TABLE"
|
|
cat > "$STATE_FILE" <<'{state_marker}'
|
|
{state.rstrip()}
|
|
{state_marker}
|
|
chmod 600 "$STATE_FILE"
|
|
{state_loader}
|
|
vpc_state_load || exit 74
|
|
vpc_state_refresh_nft || exit 74
|
|
while IFS='|' read -r peer_host key allowed; do
|
|
[ -n "$peer_host" ] || continue
|
|
"$WG" show "$WG_IFACE" peers | grep -Fqx "$key" || exit 74
|
|
case ",$allowed," in *',0.0.0.0/0,'*|*',0.0.0.0/1,'*|*',128.0.0.0/1,'* ) exit 74 ;; esac
|
|
# Fabric has one AllowedIPs list per public key. Merge the complete,
|
|
# validated tenant set from every panel-owned VPC state file while keeping
|
|
# the peer's single overlay /32. Never replace the list with this VPC's
|
|
# partial view: doing so would black-hole another concurrently active VPC.
|
|
merge_actual=$("$WG" show "$WG_IFACE" allowed-ips 2>/dev/null | awk -v wanted="$key" '$1 == wanted {{print $2; exit}}' || true)
|
|
merge_overlay=$(printf '%s\\n' "$merge_actual" | tr ',' '\\n' | awk '/^100\\.64\\.[0-9]+\\.[0-9]+\\/32$/ {{print; count++}} END {{exit count == 1 ? 0 : 1}}') || exit 74
|
|
merge_extra=$(vpc_state_allowed_for_peer "$key") || exit 74
|
|
merge_allowed="$merge_overlay"
|
|
[ -z "$merge_extra" ] || merge_allowed="$merge_allowed,$merge_extra"
|
|
"$WG" set "$WG_IFACE" peer "$key" allowed-ips "$merge_allowed" || exit 74
|
|
done <<'{peer_marker}'
|
|
{peer_lines}
|
|
{peer_marker}
|
|
rm -f "$VPC_PEER_FILE" "$VPC_ROUTE_FILE" "$VPC_TENANT_FILE" "$VPC_TENANT_FILE.nft"
|
|
printf '%s\\n' ready > "$TXN/state"; chmod 600 "$TXN/state"
|
|
printf '%s\\n' '__VC_VPC_APPLIED__=yes'
|
|
printf '%s\\n' '__VC_VPC_OWNER__={DATAPLANE_OWNER_MARKER}'
|
|
printf '%s\\n' '__VC_VPC_REVISION__={plan.desired_revision}'
|
|
printf '%s\\n' '__VC_VPC_DIGEST__={plan.desired_digest}'
|
|
COMMITTED=yes
|
|
printf '%s\\n' committed > "$TXN/state"; chmod 600 "$TXN/state"
|
|
rm -rf "$TXN"
|
|
trap - EXIT HUP INT TERM
|
|
exit 0
|
|
'''
|
|
if len(script) > MAX_OUTPUT:
|
|
raise VpcDataplaneError("VPC 数据面脚本过大", 413, "vpc_script_too_large")
|
|
return script
|
|
|
|
|
|
def render_remove_script(
|
|
plan: VpcDataplanePlan,
|
|
*,
|
|
deployment_mode: str = "networkmanager_nmcli",
|
|
ifup_path: str = "/usr/sbin/ifup",
|
|
ifdown_path: str = "/usr/sbin/ifdown",
|
|
ifquery_path: str = "/usr/sbin/ifquery",
|
|
) -> str:
|
|
"""Render a fail-closed removal transaction for an empty host."""
|
|
if plan.local_members:
|
|
raise VpcDataplaneError(
|
|
"仍有本机私网成员,不能执行宿主机 VPC 销毁",
|
|
409, "vpc_remove_has_local_members")
|
|
if deployment_mode not in {"networkmanager_nmcli", "ifupdown"}:
|
|
raise VpcDataplaneError("宿主机网络管理模式不支持", 409,
|
|
"vpc_dataplane_network_manager_conflict")
|
|
allowed_ifup = {"/usr/sbin/ifup", "/usr/bin/ifup", "/sbin/ifup", "/bin/ifup"}
|
|
allowed_ifdown = {"/usr/sbin/ifdown", "/usr/bin/ifdown", "/sbin/ifdown", "/bin/ifdown"}
|
|
allowed_ifquery = {"/usr/sbin/ifquery", "/usr/bin/ifquery", "/sbin/ifquery", "/bin/ifquery"}
|
|
if (ifup_path not in allowed_ifup or ifdown_path not in allowed_ifdown
|
|
or ifquery_path not in allowed_ifquery):
|
|
raise VpcDataplaneError("ifupdown 工具路径不受支持", 409,
|
|
"vpc_dataplane_network_manager_conflict")
|
|
from .fabric import _VPC_STATE_LOADER
|
|
|
|
r = plan.resources
|
|
include = f"/etc/nftables.d/virt-cloud-vpc-{r.token}.nft"
|
|
fragment = f"/etc/network/interfaces.d/virt-cloud-vpc-{r.token}"
|
|
sysctl_file = f"/etc/sysctl.d/99-virt-cloud-vpc-{r.token}.conf"
|
|
proxy_file = f"/etc/virt-cloud/vpc-dataplane/{r.token}.proxy"
|
|
state_loader = _VPC_STATE_LOADER.replace(
|
|
"__VC_FABRIC_HOST_ID__", plan.host_id).replace(
|
|
"__VC_FABRIC_ID__", plan.vpc.fabric_id).replace(
|
|
"__VC_FABRIC_NETWORK__", "100.64.0.0/10").replace(
|
|
"__VC_VPC_STATE_OBSERVE__", "no")
|
|
nm_profile_marker = DATAPLANE_OWNER_MARKER + " vpc=" + plan.vpc.vpc_id
|
|
ifup_marker = f"# {DATAPLANE_OWNER_MARKER} vpc={plan.vpc.vpc_id}"
|
|
inflight_marker = (f"{DATAPLANE_OWNER_MARKER}:{plan.vpc.vpc_id}:"
|
|
f"{plan.host_id}:{plan.desired_revision}:{plan.desired_digest}")
|
|
script = f'''#!/bin/sh
|
|
set -eu
|
|
umask 077
|
|
PATH=/usr/sbin:/usr/bin:/sbin:/bin
|
|
export PATH
|
|
IP=$(for p in /usr/sbin/ip /usr/bin/ip /sbin/ip /bin/ip; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
|
|
WG=$(for p in /usr/bin/wg /usr/sbin/wg /bin/wg /sbin/wg; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
|
|
NFT=$(for p in /usr/sbin/nft /usr/bin/nft /sbin/nft /bin/nft; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
|
|
FLOCK=$(for p in /usr/bin/flock /usr/sbin/flock /bin/flock /sbin/flock; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
|
|
STAT=$(for p in /usr/bin/stat /bin/stat /usr/sbin/stat /sbin/stat; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
|
|
MKTEMP=$(for p in /usr/bin/mktemp /bin/mktemp /usr/sbin/mktemp /sbin/mktemp; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
|
|
BRIDGE_CMD=$(for p in /usr/sbin/bridge /usr/bin/bridge /sbin/bridge /bin/bridge; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
|
|
BRIDGE={_q(r.bridge)}; TABLE={_q(r.nft_table)}; ROUTE_TABLE={r.route_table}; WG_IFACE={_q(FABRIC_INTERFACE)}
|
|
OWNER_FILE={_q(r.owner_file)}; STATE_FILE={_q(r.state_file)}; LOCK={_q(r.lock_file)}; TXN={_q(r.transaction_dir)}
|
|
NFT_FILE={_q(include)}; IF_FRAGMENT={_q(fragment)}; SYSCTL_FILE={_q(sysctl_file)}; PROXY_FILE={_q(proxy_file)}; STATE_DIR=/etc/virt-cloud/vpc-dataplane
|
|
MODE={_q(deployment_mode)}; IFUP={_q(ifup_path)}; IFDOWN={_q(ifdown_path)}; IFQUERY={_q(ifquery_path)}; NMCLI=; NFT_MAIN=; NFT_INCLUDE={_q(f'include "{include}"')}
|
|
[ -n "$IP" ] && [ -n "$WG" ] && [ -n "$NFT" ] && [ -n "$FLOCK" ] && [ -n "$STAT" ] && [ -n "$MKTEMP" ] && [ -n "$BRIDGE_CMD" ] || exit 70
|
|
if [ "$MODE" = networkmanager_nmcli ]; then
|
|
NMCLI=$(for p in /usr/bin/nmcli /usr/sbin/nmcli /bin/nmcli /sbin/nmcli; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
|
|
[ -n "$NMCLI" ] || exit 70
|
|
fi
|
|
[ ! -L /run/virt-cloud ] && [ ! -L /etc/virt-cloud ] || exit 71
|
|
mkdir -p /run/virt-cloud; chmod 700 /run/virt-cloud
|
|
if [ -e "$STATE_DIR" ] || [ -L "$STATE_DIR" ]; then
|
|
[ -d "$STATE_DIR" ] && [ ! -L "$STATE_DIR" ] || exit 71
|
|
[ "$($STAT -c '%u:%a' "$STATE_DIR" 2>/dev/null)" = 0:700 ] || exit 71
|
|
fi
|
|
if [ -e /etc/nftables.d ] || [ -L /etc/nftables.d ]; then
|
|
[ -d /etc/nftables.d ] && [ ! -L /etc/nftables.d ] || exit 71
|
|
[ "$($STAT -c '%u:%a' /etc/nftables.d 2>/dev/null)" = 0:755 ] || exit 71
|
|
fi
|
|
if [ -e /etc/sysctl.d ] || [ -L /etc/sysctl.d ]; then
|
|
[ -d /etc/sysctl.d ] && [ ! -L /etc/sysctl.d ] || exit 71
|
|
[ "$($STAT -c '%u:%a' /etc/sysctl.d 2>/dev/null)" = 0:755 ] || exit 71
|
|
fi
|
|
[ ! -L "$LOCK" ] && {{ [ ! -e "$LOCK" ] || [ -f "$LOCK" ]; }} || exit 71
|
|
: >> "$LOCK"; chmod 600 "$LOCK"; exec 9>>"$LOCK"; "$FLOCK" -x 9
|
|
exec 8>>{_q(FABRIC_LOCK)}; "$FLOCK" -x 8
|
|
[ -f /etc/virt-cloud/wg-fabric0.owner ] && grep -Fqx 'virt-cloud-wg-fabric-v1' /etc/virt-cloud/wg-fabric0.owner || exit 71
|
|
"$WG" show "$WG_IFACE" >/dev/null 2>&1 || exit 71
|
|
BRIDGE_PRESENT=no
|
|
if "$IP" link show dev "$BRIDGE" >/dev/null 2>&1; then BRIDGE_PRESENT=yes; fi
|
|
PHASE=preparing; RESUME=no
|
|
if [ -e "$TXN" ]; then
|
|
[ -d "$TXN" ] && [ ! -L "$TXN" ] || exit 79
|
|
[ "$($STAT -c '%u:%a' "$TXN" 2>/dev/null)" = 0:700 ] || exit 79
|
|
[ -f "$TXN/inflight" ] && [ ! -L "$TXN/inflight" ] || exit 79
|
|
[ "$($STAT -c '%u:%a' "$TXN/inflight" 2>/dev/null)" = 0:600 ] || exit 79
|
|
grep -Fqx '{inflight_marker}' "$TXN/inflight" || exit 79
|
|
[ -f "$TXN/state" ] && [ ! -L "$TXN/state" ] || exit 79
|
|
[ "$($STAT -c '%u:%a' "$TXN/state" 2>/dev/null)" = 0:600 ] || exit 79
|
|
PHASE=$(cat "$TXN/state" 2>/dev/null) || exit 79
|
|
case "$PHASE" in
|
|
preparing|routes_removed|manager_removed|bridge_removed|nft_removed|files_removed|wg_merged|committed) : ;;
|
|
*) exit 79 ;;
|
|
esac
|
|
RESUME=yes
|
|
for snapshot in "$TXN/state.before" "$TXN/routes.before" "$TXN/rules.before" "$TXN/proxy.before"; do
|
|
[ -e "$snapshot" ] || continue
|
|
[ -f "$snapshot" ] && [ ! -L "$snapshot" ] || exit 79
|
|
[ "$($STAT -c '%u:%a' "$snapshot" 2>/dev/null)" = 0:600 ] || exit 79
|
|
done
|
|
fi
|
|
set_phase() {{
|
|
phase_tmp="$TXN/.state.$$"
|
|
printf '%s\\n' "$1" > "$phase_tmp"
|
|
chmod 600 "$phase_tmp"
|
|
mv -f "$phase_tmp" "$TXN/state"
|
|
PHASE="$1"
|
|
}}
|
|
verify_routes_removed() {{
|
|
verify_routes=$($IP -o -4 route show table "$ROUTE_TABLE" 2>/dev/null || true)
|
|
[ -z "$verify_routes" ] || return 1
|
|
verify_rule=$($IP -o rule show pref "{r.rule_priority}" 2>/dev/null || true)
|
|
[ -z "$verify_rule" ] || return 1
|
|
verify_rule=$($IP -o rule show pref "$(( {r.rule_priority} + 1 ))" 2>/dev/null || true)
|
|
[ -z "$verify_rule" ] || return 1
|
|
if $IP link show dev "$BRIDGE" >/dev/null 2>&1; then
|
|
verify_proxy=$($IP neigh show proxy dev "$BRIDGE" 2>/dev/null | awk '$1 ~ /^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$/') || return 1
|
|
[ -z "$verify_proxy" ] || return 1
|
|
fi
|
|
return 0
|
|
}}
|
|
case "$PHASE" in
|
|
preparing|routes_removed|manager_removed|bridge_removed|nft_removed) PRE_FILES=yes ;;
|
|
files_removed|wg_merged|committed) PRE_FILES=no ;;
|
|
*) exit 79 ;;
|
|
esac
|
|
if [ "$PRE_FILES" = yes ]; then
|
|
if [ -f "$OWNER_FILE" ]; then
|
|
[ ! -L "$OWNER_FILE" ] || exit 71
|
|
[ "$($STAT -c '%u:%a' "$OWNER_FILE" 2>/dev/null)" = 0:600 ] || exit 71
|
|
grep -Fqx 'virt-cloud-vpc-dataplane-v1' "$OWNER_FILE" || exit 71
|
|
elif [ -e "$STATE_FILE" ] || [ "$BRIDGE_PRESENT" = yes ] || [ -e "$NFT_FILE" ] ||
|
|
[ -e "$SYSCTL_FILE" ] || [ -e "$PROXY_FILE" ] || [ -e "$IF_FRAGMENT" ]; then
|
|
# A deterministic bridge/file name is not ownership. Refuse to remove
|
|
# anything when the panel's owner marker disappeared or was replaced.
|
|
exit 71
|
|
fi
|
|
if [ -f "$STATE_FILE" ]; then
|
|
[ ! -L "$STATE_FILE" ] || exit 71
|
|
[ "$($STAT -c '%u:%a' "$STATE_FILE" 2>/dev/null)" = 0:600 ] || exit 71
|
|
grep -Fqx 'owner|{DATAPLANE_OWNER_MARKER}' "$STATE_FILE" || exit 71
|
|
grep -Fqx 'vpc|{plan.vpc.vpc_id}' "$STATE_FILE" || exit 71
|
|
grep -Fqx 'cidr|{plan.vpc.cidr}' "$STATE_FILE" || exit 71
|
|
grep -Fqx 'fabric|{plan.vpc.fabric_id}' "$STATE_FILE" || exit 71
|
|
grep -Fqx 'host|{plan.host_id}' "$STATE_FILE" || exit 71
|
|
grep -Fqx 'bridge|{r.bridge}' "$STATE_FILE" || exit 71
|
|
grep -Fqx 'table|{r.route_table}' "$STATE_FILE" || exit 71
|
|
state_revision=$(awk -F'|' '$1=="revision" {{print $2; exit}}' "$STATE_FILE")
|
|
state_digest=$(awk -F'|' '$1=="digest" {{print $2; exit}}' "$STATE_FILE")
|
|
case "$state_revision" in ''|*[!0-9]*) exit 79;; esac
|
|
printf '%s\n' "$state_digest" | grep -Eq '^[0-9a-f]{{64}}$' || exit 79
|
|
[ "$state_revision" -le {plan.desired_revision} ] || exit 79
|
|
if [ "$state_revision" = {plan.desired_revision} ] && [ "$state_digest" != { _q(plan.desired_digest) } ]; then exit 79; fi
|
|
elif [ -f "$OWNER_FILE" ] && [ "$BRIDGE_PRESENT" = yes ]; then
|
|
# A live panel bridge without its state cannot be reconciled safely.
|
|
exit 71
|
|
else
|
|
# Without the state file there is no safe owner proof for deterministic
|
|
# policy routes/rules. Refuse to claim a clean removal while any of those
|
|
# resources remain; an operator can restore the state and retry.
|
|
orphan_routes=$("$IP" -o -4 route show table "$ROUTE_TABLE" 2>/dev/null || true)
|
|
[ -z "$orphan_routes" ] || exit 71
|
|
orphan_rule=$("$IP" -o rule show pref "{r.rule_priority}" 2>/dev/null || true)
|
|
[ -z "$orphan_rule" ] || exit 71
|
|
orphan_rule=$("$IP" -o rule show pref "$(( {r.rule_priority} + 1 ))" 2>/dev/null || true)
|
|
[ -z "$orphan_rule" ] || exit 71
|
|
fi
|
|
for artifact in "$NFT_FILE" "$SYSCTL_FILE" "$PROXY_FILE" "$IF_FRAGMENT"; do
|
|
[ -e "$artifact" ] || continue
|
|
[ -f "$artifact" ] && [ ! -L "$artifact" ] || exit 71
|
|
done
|
|
if [ -f "$NFT_FILE" ]; then
|
|
[ "$($STAT -c '%u:%a' "$NFT_FILE" 2>/dev/null)" = 0:600 ] || exit 71
|
|
grep -Fqx '# virt-cloud-vpc-dataplane-v1' "$NFT_FILE" || exit 71
|
|
fi
|
|
if [ -f "$SYSCTL_FILE" ]; then
|
|
[ "$($STAT -c '%u:%a' "$SYSCTL_FILE" 2>/dev/null)" = 0:600 ] || exit 71
|
|
sed -n '1p' "$SYSCTL_FILE" | grep -Fqx '# virt-cloud-vpc-dataplane-v1' || exit 71
|
|
fi
|
|
if [ -f "$PROXY_FILE" ]; then
|
|
[ "$($STAT -c '%u:%a' "$PROXY_FILE" 2>/dev/null)" = 0:600 ] || exit 71
|
|
sed -n '1p' "$PROXY_FILE" | grep -Fqx '# virt-cloud-vpc-proxy-v1' || exit 71
|
|
awk 'NR > 1 && $0 !~ /^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$/ {{ exit 1 }}' "$PROXY_FILE" || exit 71
|
|
fi
|
|
if [ -f "$IF_FRAGMENT" ]; then
|
|
[ "$($STAT -c '%u:%a' "$IF_FRAGMENT" 2>/dev/null)" = 0:600 ] || exit 71
|
|
sed -n '1p' "$IF_FRAGMENT" | grep -Fqx '{ifup_marker}' || exit 71
|
|
grep -Fqx 'auto {r.bridge}' "$IF_FRAGMENT" || exit 71
|
|
grep -Fqx 'iface {r.bridge} inet manual' "$IF_FRAGMENT" || exit 71
|
|
if grep -Eq '^[[:space:]]*bridge_ports([[:space:]]|$)' "$IF_FRAGMENT"; then
|
|
grep -Eq '^[[:space:]]*bridge_ports[[:space:]]+none[[:space:]]*$' "$IF_FRAGMENT" || exit 71
|
|
fi
|
|
grep -Eq '^[[:space:]]*(address|gateway|up|down|pre-up|post-up|pre-down|post-down)([[:space:]]|$)' "$IF_FRAGMENT" && exit 71 || true
|
|
fi
|
|
else
|
|
for artifact in "$NFT_FILE" "$SYSCTL_FILE" "$PROXY_FILE" "$IF_FRAGMENT" "$OWNER_FILE" "$STATE_FILE"; do
|
|
[ ! -e "$artifact" ] && [ ! -L "$artifact" ] || exit 79
|
|
done
|
|
fi
|
|
if [ "$MODE" = networkmanager_nmcli ]; then
|
|
if [ "$PHASE" = preparing ] || [ "$PHASE" = routes_removed ]; then
|
|
nm_field() {{ "$NMCLI" -t --escape no -g "$1" connection show "$BRIDGE" 2>/dev/null | sed -n '1p'; }}
|
|
nm_empty() {{ value=$(nm_field "$1"); [ -z "$value" ] || [ "$value" = "--" ]; }}
|
|
NM_PROFILE_COUNT=$("$NMCLI" -t --escape no -f NAME connection show 2>/dev/null | awk -v wanted="$BRIDGE" '$0 == wanted {{ count++ }} END {{ print count + 0 }}')
|
|
case "$NM_PROFILE_COUNT" in ''|*[!0-9]*) exit 71;; esac
|
|
if [ "$NM_PROFILE_COUNT" = 1 ]; then
|
|
# The deterministic profile name is not sufficient ownership evidence.
|
|
# Match the apply-side contract before taking a profile down or deleting
|
|
# it, so a public/operator bridge can never be removed by this cleanup.
|
|
[ "$(nm_field connection.type)" = bridge ] || exit 71
|
|
[ "$(nm_field connection.interface-name)" = "$BRIDGE" ] || exit 71
|
|
[ "$(nm_field connection.stable-id)" = "{nm_profile_marker}" ] || exit 71
|
|
[ "$(nm_field ipv4.method)" = disabled ] || exit 71
|
|
[ "$(nm_field ipv6.method)" = disabled ] || exit 71
|
|
nm_empty ipv4.addresses || exit 71
|
|
nm_empty ipv4.routes || exit 71
|
|
nm_empty ipv4.routing-rules || exit 71
|
|
nm_empty ipv4.gateway || exit 71
|
|
nm_empty ipv4.dns || exit 71
|
|
nm_empty ipv4.dns-search || exit 71
|
|
nm_empty ipv6.addresses || exit 71
|
|
nm_empty ipv6.routes || exit 71
|
|
nm_empty ipv6.routing-rules || exit 71
|
|
nm_empty ipv6.gateway || exit 71
|
|
nm_empty ipv6.dns || exit 71
|
|
nm_empty ipv6.dns-search || exit 71
|
|
nm_empty connection.master || exit 71
|
|
nm_empty connection.controller || exit 71
|
|
nm_empty connection.slave-type || exit 71
|
|
nm_empty connection.port-type || exit 71
|
|
nm_empty connection.secondaries || exit 71
|
|
nm_empty connection.zone || exit 71
|
|
case "$(nm_field bridge.stp)" in no|false|off|0|--) : ;; *) exit 71 ;; esac
|
|
elif [ "$NM_PROFILE_COUNT" -gt 1 ]; then
|
|
exit 71
|
|
elif [ "$BRIDGE_PRESENT" = yes ]; then
|
|
# An unmanaged live bridge is foreign/ambiguous; do not delete it.
|
|
exit 71
|
|
fi
|
|
else
|
|
if "$NMCLI" connection show "$BRIDGE" >/dev/null 2>&1; then exit 79; fi
|
|
fi
|
|
else
|
|
if [ "$PHASE" = preparing ] || [ "$PHASE" = routes_removed ]; then
|
|
if [ "$BRIDGE_PRESENT" = yes ] || [ -e "$IF_FRAGMENT" ]; then
|
|
[ -f "$IF_FRAGMENT" ] || exit 71
|
|
[ ! -L /etc/network ] && [ ! -L /etc/network/interfaces ] && [ ! -L /etc/network/interfaces.d ] || exit 71
|
|
if [ -f /etc/network/interfaces ] &&
|
|
grep -Eq '^[[:space:]]*(auto|allow-[^[:space:]]+|iface)[[:space:]]+'"$BRIDGE"'([[:space:]]|$)' /etc/network/interfaces; then
|
|
exit 71
|
|
fi
|
|
"$IFQUERY" "$BRIDGE" >/dev/null 2>&1 || exit 71
|
|
fi
|
|
fi
|
|
fi
|
|
if [ "$PHASE" = preparing ] || [ "$PHASE" = routes_removed ] || [ "$PHASE" = manager_removed ]; then
|
|
if "$IP" link show dev "$BRIDGE" >/dev/null 2>&1; then
|
|
[ -f "$OWNER_FILE" ] || exit 71
|
|
bridge_info=$("$IP" -d link show dev "$BRIDGE" 2>/dev/null) || exit 71
|
|
printf '%s\\n' "$bridge_info" | grep -Eq '(^|[[:space:]])bridge[[:space:]]+(forward_delay|stp_state|hello_time|ageing_time)([[:space:]]|$)' || exit 71
|
|
bridge_ports=$("$BRIDGE_CMD" link show master "$BRIDGE" 2>/dev/null) || exit 71
|
|
[ -z "$bridge_ports" ] || exit 79
|
|
bridge_addrs=$("$IP" -o addr show dev "$BRIDGE" 2>/dev/null) || exit 71
|
|
[ -z "$bridge_addrs" ] || exit 79
|
|
fi
|
|
else
|
|
if "$IP" link show dev "$BRIDGE" >/dev/null 2>&1; then exit 79; fi
|
|
fi
|
|
NFT_MAIN_COUNT=0
|
|
for nft_candidate in /etc/nftables.conf /etc/sysconfig/nftables.conf; do
|
|
if [ -f "$nft_candidate" ] && [ ! -L "$nft_candidate" ] && [ "$($STAT -c '%u' "$nft_candidate" 2>/dev/null)" = 0 ]; then
|
|
case "$($STAT -c '%a' "$nft_candidate" 2>/dev/null)" in
|
|
[0-7][0145][0145]) : ;;
|
|
*) continue ;;
|
|
esac
|
|
NFT_MAIN="$nft_candidate"; NFT_MAIN_COUNT=$((NFT_MAIN_COUNT + 1))
|
|
fi
|
|
done
|
|
[ "$NFT_MAIN_COUNT" -eq 1 ] || exit 71
|
|
if [ "$PHASE" = preparing ] || [ "$PHASE" = routes_removed ] || [ "$PHASE" = manager_removed ] || [ "$PHASE" = bridge_removed ]; then
|
|
if [ -f "$NFT_FILE" ]; then
|
|
nft_tables=$(awk '$1=="table" {{print $2 "|" $3}}' "$NFT_FILE")
|
|
nft_table_count=0
|
|
while IFS='|' read -r family name; do
|
|
[ -n "$family" ] || continue
|
|
case "$family:$name" in inet:*|bridge:*|arp:*) : ;; *) exit 71 ;; esac
|
|
[ "$name" = "$TABLE" ] || exit 71
|
|
table_dump=$("$NFT" list table "$family" "$name" 2>/dev/null) || exit 71
|
|
printf '%s\\n' "$table_dump" | grep -Fq 'virt-cloud-vpc-dataplane-v1' || exit 71
|
|
nft_table_count=$((nft_table_count + 1))
|
|
done <<EOF
|
|
$nft_tables
|
|
EOF
|
|
[ "$nft_table_count" -gt 0 ] || exit 71
|
|
fi
|
|
else
|
|
if [ -f "$NFT_FILE" ]; then
|
|
nft_tables=$(awk '$1=="table" {{print $2 "|" $3}}' "$NFT_FILE")
|
|
while IFS='|' read -r family name; do
|
|
[ -n "$family" ] || continue
|
|
case "$family:$name" in inet:*|bridge:*|arp:*) : ;; *) exit 79 ;; esac
|
|
[ "$name" = "$TABLE" ] || exit 79
|
|
if "$NFT" list table "$family" "$name" >/dev/null 2>&1; then exit 79; fi
|
|
done <<EOF
|
|
$nft_tables
|
|
EOF
|
|
fi
|
|
grep -Fqx "$NFT_INCLUDE" "$NFT_MAIN" && exit 79 || true
|
|
fi
|
|
# A partially completed removal is deliberately fail-closed. The journal is
|
|
# retained for an operator/reconciler to inspect; rerunning it must never repeat
|
|
# route, bridge, nft, or WireGuard deletions against an unknown intermediate
|
|
# state. A fully committed journal only needs the already-verified cleanup.
|
|
if [ "$RESUME" = yes ]; then
|
|
case "$PHASE" in
|
|
committed)
|
|
rm -rf "$TXN" || exit 79
|
|
printf '%s\\n' '__VC_VPC_REMOVED__=yes'
|
|
printf '%s\\n' '__VC_VPC_OWNER__={DATAPLANE_OWNER_MARKER}'
|
|
printf '%s\\n' '__VC_VPC_REVISION__={plan.desired_revision}'
|
|
printf '%s\\n' '__VC_VPC_DIGEST__={plan.desired_digest}'
|
|
printf '%s\\n' '__VC_VPC_ROLLBACK__=none'
|
|
exit 0
|
|
;;
|
|
files_removed|wg_merged)
|
|
# All destructive host resources are already absent. Re-running only
|
|
# the Fabric state load and peer merge below is idempotent and closes
|
|
# the crash window between file removal and AllowedIPs convergence.
|
|
:
|
|
;;
|
|
*) exit 79 ;;
|
|
esac
|
|
fi
|
|
if [ "$RESUME" = no ]; then
|
|
mkdir -m 700 "$TXN"
|
|
printf '%s\\n' '{inflight_marker}' > "$TXN/inflight"; chmod 600 "$TXN/inflight"
|
|
set_phase preparing
|
|
if [ -f "$STATE_FILE" ]; then cp -p "$STATE_FILE" "$TXN/state.before"; fi
|
|
: > "$TXN/routes.before"; : > "$TXN/rules.before"; : > "$TXN/proxy.before"
|
|
chmod 600 "$TXN/routes.before" "$TXN/rules.before" "$TXN/proxy.before"
|
|
else
|
|
# A resumed transaction must use the original snapshots. Creating an empty
|
|
# snapshot after a partial failure would turn an unknown host state into an
|
|
# apparently clean state and could make the next destructive step unsafe.
|
|
for snapshot in "$TXN/routes.before" "$TXN/rules.before" "$TXN/proxy.before"; do
|
|
[ -f "$snapshot" ] && [ ! -L "$snapshot" ] || exit 79
|
|
[ "$($STAT -c '%u:%a' "$snapshot" 2>/dev/null)" = 0:600 ] || exit 79
|
|
done
|
|
fi
|
|
if [ -f "$TXN/state.before" ]; then
|
|
while IFS='|' read -r kind destination device table via metric; do
|
|
[ "$kind" = route ] || continue
|
|
[ "$table" = "$ROUTE_TABLE" ] || exit 71
|
|
[ "$device" = "$BRIDGE" ] || [ "$device" = "$WG_IFACE" ] || exit 71
|
|
printf '%s\\n' "$destination" | grep -Eq '^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+/32$' || exit 71
|
|
case "$via" in
|
|
-|'') : ;;
|
|
*) printf '%s\\n' "$via" | grep -Eq '^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$' || exit 71 ;;
|
|
esac
|
|
case "$metric" in ''|*[!0-9]*) exit 71;; esac
|
|
existing=$("$IP" -o -4 route show table "$table" "$destination" 2>/dev/null || true)
|
|
[ -n "$existing" ] || continue
|
|
existing_count=$(printf '%s\\n' "$existing" | awk 'NF {{ count++ }} END {{ print count + 0 }}')
|
|
[ "$existing_count" = 1 ] || exit 71
|
|
actual_device=$(printf '%s\\n' "$existing" | awk '{{for (i=1;i<=NF;i++) if ($i=="dev" && i<NF) {{print $(i+1); exit}}}}')
|
|
[ "$actual_device" = "$device" ] || exit 71
|
|
printf '%s\\n' "$existing" | grep -Eq 'proto 186([[:space:]]|$)' || exit 71
|
|
old_via=$(printf '%s\\n' "$existing" | awk '{{for (i=1;i<=NF;i++) if ($i=="via" && i<NF) {{print $(i+1); exit}}}}')
|
|
old_metric=$(printf '%s\\n' "$existing" | awk '{{for (i=1;i<=NF;i++) if ($i=="metric" && i<NF) {{print $(i+1); exit}}}}')
|
|
[ -n "$old_metric" ] || old_metric=100
|
|
[ "$old_metric" = "$metric" ] || exit 71
|
|
[ "${{old_via:--}}" = "${{via:--}}" ] || exit 71
|
|
printf '%s|%s|%s|%s|%s\\n' "$destination" "$device" "$table" "$old_via" "$old_metric" >> "$TXN/routes.before"
|
|
if [ "${{old_via:--}}" != - ]; then
|
|
"$IP" route del "$destination" table "$table" via "$old_via" dev "$device" proto 186 metric "$old_metric" || exit 72
|
|
else
|
|
"$IP" route del "$destination" table "$table" dev "$device" proto 186 metric "$old_metric" || exit 72
|
|
fi
|
|
done < "$TXN/state.before"
|
|
old_priority=$(awk -F'|' '$1=="priority" {{print $2; exit}}' "$TXN/state.before")
|
|
old_cidr=$(awk -F'|' '$1=="cidr" {{print $2; exit}}' "$TXN/state.before")
|
|
[ -n "$old_priority" ] && [ -n "$old_cidr" ] || exit 71
|
|
case "$old_priority" in ''|*[!0-9]*) exit 71;; esac
|
|
[ "$old_priority" = "{r.rule_priority}" ] || exit 71
|
|
for pref in "$old_priority" "$((old_priority + 1))"; do
|
|
existing=$("$IP" -o rule show pref "$pref" 2>/dev/null || true)
|
|
[ -n "$existing" ] || continue
|
|
existing_count=$(printf '%s\\n' "$existing" | awk 'NF {{ count++ }} END {{ print count + 0 }}')
|
|
[ "$existing_count" = 1 ] || exit 71
|
|
printf '%s\\n' "$existing" | grep -Fq "to $old_cidr lookup $ROUTE_TABLE" || exit 71
|
|
old_iif=$(printf '%s\\n' "$existing" | awk '{{for (i=1;i<=NF;i++) if ($i=="iif" && i<NF) {{print $(i+1); exit}}}}')
|
|
if [ "$pref" = "$old_priority" ]; then
|
|
[ -z "$old_iif" ] || exit 71
|
|
else
|
|
[ "$old_iif" = "$WG_IFACE" ] || exit 71
|
|
fi
|
|
printf '%s|%s|%s|%s\\n' "$pref" "$old_iif" "$old_cidr" "$ROUTE_TABLE" >> "$TXN/rules.before"
|
|
"$IP" rule del pref "$pref" || exit 72
|
|
done
|
|
fi
|
|
if [ -f "$PROXY_FILE" ]; then
|
|
while IFS= read -r proxy_ip; do
|
|
[ -n "$proxy_ip" ] || continue
|
|
if "$IP" neigh show proxy dev "$BRIDGE" 2>/dev/null | grep -Fqx "$proxy_ip"; then
|
|
printf '%s\\n' "$proxy_ip" >> "$TXN/proxy.before"
|
|
"$IP" neigh del proxy "$proxy_ip" dev "$BRIDGE" || exit 72
|
|
fi
|
|
done <<EOF
|
|
$(sed '1d' "$PROXY_FILE")
|
|
EOF
|
|
fi
|
|
verify_routes_removed || exit 72
|
|
set_phase routes_removed
|
|
if [ "$MODE" = networkmanager_nmcli ]; then
|
|
if [ -n "$NMCLI" ] && "$NMCLI" connection show "$BRIDGE" >/dev/null 2>&1; then
|
|
"$NMCLI" connection down "$BRIDGE" >/dev/null 2>&1 || true
|
|
"$NMCLI" connection delete "$BRIDGE" >/dev/null 2>&1 || exit 72
|
|
if "$NMCLI" connection show "$BRIDGE" >/dev/null 2>&1; then exit 72; fi
|
|
fi
|
|
else
|
|
if [ -f "$IF_FRAGMENT" ]; then
|
|
"$IFQUERY" "$BRIDGE" >/dev/null 2>&1 || exit 71
|
|
"$IFDOWN" --force "$BRIDGE" >/dev/null 2>&1 || exit 72
|
|
fi
|
|
fi
|
|
set_phase manager_removed
|
|
if "$IP" link show dev "$BRIDGE" >/dev/null 2>&1; then
|
|
bridge_info=$("$IP" -d link show dev "$BRIDGE" 2>/dev/null) || exit 71
|
|
printf '%s\\n' "$bridge_info" | grep -Eq '(^|[[:space:]])bridge[[:space:]]+(forward_delay|stp_state|hello_time|ageing_time)([[:space:]]|$)' || exit 71
|
|
ports=$("$BRIDGE_CMD" link show master "$BRIDGE" 2>/dev/null) || exit 71
|
|
[ -z "$ports" ] || exit 79
|
|
bridge_addrs=$("$IP" -o addr show dev "$BRIDGE" 2>/dev/null) || exit 71
|
|
[ -z "$bridge_addrs" ] || exit 79
|
|
bridge_routes=$("$IP" -o -4 route show table all dev "$BRIDGE" 2>/dev/null) || exit 71
|
|
[ -z "$bridge_routes" ] || exit 79
|
|
bridge_routes6=$("$IP" -o -6 route show table all dev "$BRIDGE" 2>/dev/null) || exit 71
|
|
[ -z "$bridge_routes6" ] || exit 79
|
|
proxy_left=$("$IP" neigh show proxy dev "$BRIDGE" 2>/dev/null | awk '$1 ~ /^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$/') || exit 71
|
|
[ -z "$proxy_left" ] || exit 79
|
|
"$IP" link del dev "$BRIDGE" type bridge || exit 72
|
|
fi
|
|
if "$IP" link show dev "$BRIDGE" >/dev/null 2>&1; then exit 72; fi
|
|
set_phase bridge_removed
|
|
if [ -f "$NFT_FILE" ]; then
|
|
nft_tables=$(awk '$1=="table" {{print $2 "|" $3}}' "$NFT_FILE")
|
|
nft_table_count=0
|
|
while IFS='|' read -r family name; do
|
|
[ -n "$family" ] || continue
|
|
case "$family:$name" in inet:*|bridge:*|arp:*) : ;; *) exit 71 ;; esac
|
|
[ "$name" = "$TABLE" ] || exit 71
|
|
table_dump=$("$NFT" list table "$family" "$name" 2>/dev/null) || exit 71
|
|
printf '%s\\n' "$table_dump" | grep -Fq 'virt-cloud-vpc-dataplane-v1' || exit 71
|
|
"$NFT" delete table "$family" "$name" || exit 72
|
|
if "$NFT" list table "$family" "$name" >/dev/null 2>&1; then exit 72; fi
|
|
nft_table_count=$((nft_table_count + 1))
|
|
done <<EOF
|
|
$nft_tables
|
|
EOF
|
|
[ "$nft_table_count" -gt 0 ] || exit 71
|
|
fi
|
|
if grep -Fqx "$NFT_INCLUDE" "$NFT_MAIN"; then
|
|
NFT_MAIN_TMP=$("$MKTEMP" "$NFT_MAIN.virt-cloud-vpc-remove.XXXXXX") || exit 72
|
|
awk -v needle="$NFT_INCLUDE" '$0 != needle' "$NFT_MAIN" > "$NFT_MAIN_TMP" || {{ rm -f "$NFT_MAIN_TMP"; exit 72; }}
|
|
"$NFT" -c -f "$NFT_MAIN_TMP" >/dev/null || {{ rm -f "$NFT_MAIN_TMP"; exit 72; }}
|
|
chmod "$($STAT -c '%a' "$NFT_MAIN")" "$NFT_MAIN_TMP" || {{ rm -f "$NFT_MAIN_TMP"; exit 72; }}
|
|
mv -f "$NFT_MAIN_TMP" "$NFT_MAIN" || exit 72
|
|
grep -Fqx "$NFT_INCLUDE" "$NFT_MAIN" && exit 72 || true
|
|
fi
|
|
set_phase nft_removed
|
|
if [ -e "$NFT_FILE" ] || [ -L "$NFT_FILE" ]; then
|
|
[ -f "$NFT_FILE" ] && [ ! -L "$NFT_FILE" ] || exit 71
|
|
grep -Fqx '# virt-cloud-vpc-dataplane-v1' "$NFT_FILE" || exit 71
|
|
grep -Fqx '# vpc={plan.vpc.vpc_id} revision={plan.desired_revision} digest={plan.desired_digest}' "$NFT_FILE" || exit 71
|
|
fi
|
|
if [ -e "$SYSCTL_FILE" ] || [ -L "$SYSCTL_FILE" ]; then
|
|
[ -f "$SYSCTL_FILE" ] && [ ! -L "$SYSCTL_FILE" ] || exit 71
|
|
sed -n '1p' "$SYSCTL_FILE" | grep -Fqx '# virt-cloud-vpc-dataplane-v1' || exit 71
|
|
fi
|
|
if [ -e "$PROXY_FILE" ] || [ -L "$PROXY_FILE" ]; then
|
|
[ -f "$PROXY_FILE" ] && [ ! -L "$PROXY_FILE" ] || exit 71
|
|
sed -n '1p' "$PROXY_FILE" | grep -Fqx '# virt-cloud-vpc-proxy-v1' || exit 71
|
|
fi
|
|
if [ -e "$IF_FRAGMENT" ] || [ -L "$IF_FRAGMENT" ]; then
|
|
[ -f "$IF_FRAGMENT" ] && [ ! -L "$IF_FRAGMENT" ] || exit 71
|
|
sed -n '1p' "$IF_FRAGMENT" | grep -Fqx '{ifup_marker}' || exit 71
|
|
fi
|
|
if [ -f "$STATE_FILE" ]; then
|
|
[ ! -L "$STATE_FILE" ] || exit 71
|
|
grep -Fqx 'owner|{DATAPLANE_OWNER_MARKER}' "$STATE_FILE" || exit 71
|
|
grep -Fqx 'vpc|{plan.vpc.vpc_id}' "$STATE_FILE" || exit 71
|
|
grep -Fqx 'fabric|{plan.vpc.fabric_id}' "$STATE_FILE" || exit 71
|
|
grep -Fqx 'host|{plan.host_id}' "$STATE_FILE" || exit 71
|
|
grep -Fqx 'bridge|{r.bridge}' "$STATE_FILE" || exit 71
|
|
grep -Fqx 'table|{r.route_table}' "$STATE_FILE" || exit 71
|
|
state_revision=$(awk -F'|' '$1=="revision" {{print $2; exit}}' "$STATE_FILE")
|
|
state_digest=$(awk -F'|' '$1=="digest" {{print $2; exit}}' "$STATE_FILE")
|
|
case "$state_revision" in ''|*[!0-9]*) exit 79;; esac
|
|
[ "$state_revision" -le {plan.desired_revision} ] || exit 79
|
|
if [ "$state_revision" = {plan.desired_revision} ] && [ "$state_digest" != { _q(plan.desired_digest) } ]; then exit 79; fi
|
|
fi
|
|
rm -f "$NFT_FILE" "$SYSCTL_FILE" "$PROXY_FILE" "$IF_FRAGMENT" "$OWNER_FILE" "$STATE_FILE"
|
|
[ ! -e "$NFT_FILE" ] && [ ! -e "$SYSCTL_FILE" ] && [ ! -e "$PROXY_FILE" ] &&
|
|
[ ! -e "$IF_FRAGMENT" ] && [ ! -e "$OWNER_FILE" ] && [ ! -e "$STATE_FILE" ] || exit 72
|
|
set_phase files_removed
|
|
{state_loader}
|
|
vpc_state_load || exit 74
|
|
vpc_state_refresh_nft || exit 74
|
|
for key in $("$WG" show "$WG_IFACE" peers 2>/dev/null); do
|
|
actual=$("$WG" show "$WG_IFACE" allowed-ips 2>/dev/null | awk -v wanted="$key" '$1==wanted {{print $2; exit}}' || true)
|
|
overlay=$(printf '%s\\n' "$actual" | tr ',' '\\n' | awk '/^100\\.64\\.[0-9]+\\.[0-9]+\\/32$/ {{print; count++}} END {{exit count == 1 ? 0 : 1}}') || exit 74
|
|
extra=$(vpc_state_allowed_for_peer "$key") || exit 74
|
|
merged="$overlay"; [ -z "$extra" ] || merged="$merged,$extra"
|
|
"$WG" set "$WG_IFACE" peer "$key" allowed-ips "$merged" || exit 74
|
|
done
|
|
set_phase wg_merged
|
|
verify_routes_removed || exit 79
|
|
for family in inet bridge arp; do
|
|
"$NFT" list table "$family" "$TABLE" >/dev/null 2>&1 && exit 79 || true
|
|
done
|
|
grep -Fqx "$NFT_INCLUDE" "$NFT_MAIN" && exit 79 || true
|
|
if [ "$MODE" = networkmanager_nmcli ] && [ -n "$NMCLI" ]; then
|
|
"$NMCLI" connection show "$BRIDGE" >/dev/null 2>&1 && exit 79 || true
|
|
fi
|
|
[ ! -e "$STATE_FILE" ] && [ ! -e "$OWNER_FILE" ] || exit 79
|
|
if "$IP" link show dev "$BRIDGE" >/dev/null 2>&1; then exit 79; fi
|
|
printf '%s\\n' '__VC_VPC_REMOVED__=yes'
|
|
printf '%s\\n' '__VC_VPC_OWNER__={DATAPLANE_OWNER_MARKER}'
|
|
printf '%s\\n' '__VC_VPC_REVISION__={plan.desired_revision}'
|
|
printf '%s\\n' '__VC_VPC_DIGEST__={plan.desired_digest}'
|
|
set_phase committed
|
|
rm -rf "$TXN"
|
|
printf '%s\\n' '__VC_VPC_ROLLBACK__=none'
|
|
exit 0
|
|
'''
|
|
if len(script) > MAX_OUTPUT:
|
|
raise VpcDataplaneError("VPC 销毁脚本过大", 413, "vpc_script_too_large")
|
|
return script
|
|
|
|
|
|
# Real host calls use the strict, manager-aware builder. The old positional
|
|
# function remains above for compatibility with callers that only render a
|
|
# fixture.
|
|
render_apply_script = _render_apply_script_v2
|
|
|
|
|
|
def render_observe_script(plan: VpcDataplanePlan) -> str:
|
|
routes = _routes(plan)
|
|
r = plan.resources
|
|
proxy_file = f"/etc/virt-cloud/vpc-dataplane/{r.token}.proxy"
|
|
nft_file = f"/etc/nftables.d/virt-cloud-vpc-{r.token}.nft"
|
|
nft_include = f'include "{nft_file}"'
|
|
sysctl_file = f"/etc/sysctl.d/99-virt-cloud-vpc-{r.token}.conf"
|
|
ifup_fragment = f"/etc/network/interfaces.d/virt-cloud-vpc-{r.token}"
|
|
nm_profile_marker = DATAPLANE_OWNER_MARKER + " vpc=" + plan.vpc.vpc_id
|
|
route_lines = "\n".join(
|
|
f"{x['destination_cidr']}|{x['device']}|{x.get('via') or '-'}|{x.get('metric', 100)}"
|
|
for x in routes)
|
|
peer_lines = "\n".join(
|
|
f"{p.host_id}|{p.public_key}|{_allowed(plan, p)}"
|
|
for p in plan.peers)
|
|
fabric_peer_key_words = " ".join(sorted(plan.fabric_peer_keys or ()))
|
|
fabric_peer_key_arg = _q(fabric_peer_key_words)
|
|
fabric_peer_keys_provided = "yes" if plan.fabric_peer_keys is not None else "no"
|
|
route_marker = "__VC_VPC_OBS_ROUTES_" + hashlib.sha256(
|
|
route_lines.encode()).hexdigest()[:12]
|
|
peer_marker = "__VC_VPC_OBS_PEERS_" + hashlib.sha256(
|
|
peer_lines.encode()).hexdigest()[:12]
|
|
nft_marker = _nft_marker(plan)
|
|
# These comments are emitted by render_vpc_nft_rules. Requiring the full
|
|
# owned shape prevents a stale or foreign table being reported as active
|
|
# merely because the deterministic table name exists.
|
|
nft_suffixes = (
|
|
"anti-spoof-local-ip", "anti-spoof-egress",
|
|
"anti-spoof-public-to-bridge", "anti-spoof-public-to-fabric",
|
|
"anti-spoof-bridge-to-foreign", "anti-spoof-fabric-to-foreign",
|
|
"anti-spoof-public-to-vpc-bridge", "anti-spoof-public-to-vpc-fabric",
|
|
"anti-spoof-bridge-vpc-to-foreign", "anti-spoof-fabric-vpc-to-foreign",
|
|
"anti-spoof-wg-ingress", "anti-spoof-wg-egress",
|
|
"anti-spoof-wg-source", "member-egress-isolation",
|
|
"member-fabric-isolation",
|
|
"default-deny",
|
|
"wg-transit-deny",
|
|
"ipv6-drop-bridge", "ipv6-drop-fabric",
|
|
"ipv6-input-drop-bridge", "ipv6-input-drop-fabric",
|
|
"ipv6-output-drop-bridge",
|
|
"host-isolation", "host-isolation-wg", "host-isolation-out",
|
|
"host-isolation-wg-source", "host-isolation-out-wg",
|
|
"host-isolation-vpc-out", "host-isolation-vpc-out-wg",
|
|
"scope-bridge-in", "scope-bridge-out", "scope-fabric-in",
|
|
"scope-fabric-out",
|
|
*(f"acl:{acl.acl_id}" for acl in plan.vpc.acls),
|
|
)
|
|
nft_checks = "\n".join(
|
|
f''' printf '%s\\n' "$nft_text" | grep -Fq "comment \\\"$NFT_MARKER:{suffix}\\\"" || nft_verified=no'''
|
|
for suffix in nft_suffixes)
|
|
bridge_suffixes = (
|
|
"bridge-anti-spoof-ingress", "bridge-anti-spoof-arp",
|
|
"bridge-anti-spoof-arp-mac", "bridge-anti-spoof-ip",
|
|
"bridge-anti-spoof-arp-ip",
|
|
"bridge-multicast-drop", "bridge-limited-broadcast-drop",
|
|
"bridge-vpc-broadcast-drop",
|
|
"bridge-unknown-destination-drop",
|
|
"bridge-default-deny", "bridge-scope-accept",
|
|
"bridge-arp-accept", "bridge-non-ip-drop",
|
|
"bridge-anti-spoof-arp-input-mac", "bridge-anti-spoof-input",
|
|
"bridge-anti-spoof-input-ip", "bridge-ipv6-input-drop",
|
|
"bridge-arp-input-accept", "bridge-ip-input-accept",
|
|
"bridge-input-non-ip-drop",
|
|
*(f"acl:{acl.acl_id}" for acl in plan.vpc.acls),
|
|
)
|
|
bridge_checks = "\n".join(
|
|
f''' printf '%s\\n' "$bridge_text" | grep -Fq "comment \\\"$NFT_MARKER:{suffix}\\\"" || bridge_verified=no'''
|
|
for suffix in bridge_suffixes)
|
|
if plan.local_members:
|
|
bridge_observe = f'''bridge_text=$($NFT list table bridge "$TABLE" 2>/dev/null || true)
|
|
bridge_verified=yes
|
|
printf '%s\\n' "$bridge_text" | grep -Fq "table bridge $TABLE" || bridge_verified=no
|
|
printf '%s\\n' "$bridge_text" | grep -Fq "chain forward" || bridge_verified=no
|
|
printf '%s\\n' "$bridge_text" | grep -Fq "hook forward priority -190" || bridge_verified=no
|
|
printf '%s\\n' "$bridge_text" | grep -Fq "policy accept" || bridge_verified=no
|
|
printf '%s\\n' "$bridge_text" | grep -Fq "set local_member_ips" || bridge_verified=no
|
|
printf '%s\\n' "$bridge_text" | grep -Fq "set local_dest_ips" || bridge_verified=no
|
|
{bridge_checks}
|
|
if [ "$bridge_verified" = yes ]; then printf '%s\\n' '__VC_VPC_BRIDGE_NFT__=yes'; else printf '%s\\n' '__VC_VPC_BRIDGE_NFT__=no'; fi'''
|
|
else:
|
|
# An empty VPC must not silently retain a stale bridge-family table
|
|
# from a previous revision or host membership.
|
|
bridge_observe = '''
|
|
if $NFT list table bridge "$TABLE" >/dev/null 2>&1; then
|
|
printf '%s\\n' '__VC_VPC_BRIDGE_NFT__=no'
|
|
else
|
|
printf '%s\\n' '__VC_VPC_BRIDGE_NFT__=none'
|
|
fi'''.strip()
|
|
if plan.local_members:
|
|
arp_observe = f'''arp_text=$($NFT list table arp "$TABLE" 2>/dev/null || true)
|
|
arp_verified=yes
|
|
printf '%s\\n' "$arp_text" | grep -Fq "table arp $TABLE" || arp_verified=no
|
|
printf '%s\\n' "$arp_text" | grep -Fq "set proxy_ips" || arp_verified=no
|
|
printf '%s\\n' "$arp_text" | grep -Fq "chain output" || arp_verified=no
|
|
printf '%s\\n' "$arp_text" | grep -Fq "hook output priority -190" || arp_verified=no
|
|
printf '%s\\n' "$arp_text" | grep -Fq "policy accept" || arp_verified=no
|
|
printf '%s\\n' "$arp_text" | grep -Fq "comment \\\"$NFT_MARKER:arp-proxy-output\\\"" || arp_verified=no
|
|
if [ "$arp_verified" = yes ]; then printf '%s\\n' '__VC_VPC_ARP_NFT__=yes'; else printf '%s\\n' '__VC_VPC_ARP_NFT__=no'; fi'''
|
|
else:
|
|
arp_observe = '''
|
|
if $NFT list table arp "$TABLE" >/dev/null 2>&1; then
|
|
printf '%s\\n' '__VC_VPC_ARP_NFT__=no'
|
|
else
|
|
printf '%s\\n' '__VC_VPC_ARP_NFT__=none'
|
|
fi'''.strip()
|
|
route_extra_awk = "\n".join(
|
|
f' $0 == "{x["destination_cidr"]}|{x["device"]}|{x.get("via") or "-"}|{x.get("metric", 100)}|186" {{ next }}'
|
|
for x in routes)
|
|
route_checks = "\n".join(
|
|
f'''state=missing
|
|
if printf '%s\\n' "$route_records" | awk -F'|' -v wanted={_q(x['destination_cidr'])} -v expected_device={_q(x['device'])} -v expected_via={_q(x.get('via') or '-')} -v expected_metric={_q(str(x.get('metric', 100)))} '$1 == wanted && $2 == expected_device && $3 == expected_via && $4 == expected_metric && $5 == "186" {{found=1}} END {{exit found ? 0 : 1}}'; then
|
|
state=owned
|
|
fi
|
|
printf '%s\\n' '__VC_VPC_ROUTE__={x['destination_cidr']}|{x['device']}|'"$ROUTE_TABLE"'|{x.get('via') or '-'}|{x.get('metric', 100)}|'"$state"
|
|
'''.rstrip()
|
|
for x in routes)
|
|
if not route_checks:
|
|
route_checks = ": # no expected VPC routes"
|
|
rule_specs = (
|
|
(str(r.rule_priority), "-", plan.vpc.cidr, str(r.route_table)),
|
|
(str(r.rule_priority + 1), FABRIC_INTERFACE, plan.vpc.cidr,
|
|
str(r.route_table)),
|
|
)
|
|
rule_checks = "\n".join(
|
|
f'''rule_state=missing
|
|
if printf '%s\\n' "$rule_records" | awk -F'|' -v pref={_q(pref)} -v expected_iif={_q(iif)} -v expected_to={_q(destination)} -v expected_table={_q(table)} '$1 == pref && $2 == expected_iif && $3 == expected_to && $4 == expected_table {{found=1}} END {{exit found ? 0 : 1}}'; then
|
|
rule_state=owned
|
|
fi
|
|
printf '%s\\n' '__VC_VPC_RULE__={pref}|{iif}|{destination}|{table}|'"$rule_state"
|
|
'''.rstrip()
|
|
for pref, iif, destination, table in rule_specs)
|
|
return f'''#!/bin/sh
|
|
set -eu
|
|
PATH=/usr/sbin:/usr/bin:/sbin:/bin
|
|
IP=/usr/sbin/ip; [ -x "$IP" ] || IP=/usr/bin/ip
|
|
WG=/usr/bin/wg; [ -x "$WG" ] || WG=/usr/sbin/wg
|
|
NFT=/usr/sbin/nft; [ -x "$NFT" ] || NFT=/usr/bin/nft
|
|
FLOCK=/usr/bin/flock; [ -x "$FLOCK" ] || FLOCK=/usr/sbin/flock
|
|
LOCK=/run/virt-cloud/wg-fabric0.lock
|
|
if [ -L /run/virt-cloud ] || {{ [ -e /run/virt-cloud ] && [ ! -d /run/virt-cloud ]; }}; then exit 71; fi
|
|
mkdir -p /run/virt-cloud; chmod 700 /run/virt-cloud
|
|
[ ! -L "$LOCK" ] && {{ [ ! -e "$LOCK" ] || [ -f "$LOCK" ]; }} || exit 71
|
|
: >> "$LOCK"; chmod 600 "$LOCK"; exec 8>>"$LOCK"; "$FLOCK" -s 8
|
|
BRIDGE_CMD=
|
|
for p in /usr/sbin/bridge /usr/bin/bridge /sbin/bridge /bin/bridge; do
|
|
if [ -x "$p" ]; then BRIDGE_CMD=$p; break; fi
|
|
done
|
|
BRIDGE={_q(r.bridge)}; TABLE={_q(r.nft_table)}; ROUTE_TABLE={r.route_table}; WG_IFACE={_q(FABRIC_INTERFACE)}; OWNER={_q(r.owner_file)}; STATE={_q(r.state_file)}; PROXY_FILE={_q(proxy_file)}
|
|
NFT_FILE={_q(nft_file)}; NFT_INCLUDE={_q(nft_include)}; SYSCTL_FILE={_q(sysctl_file)}; IF_FRAGMENT={_q(ifup_fragment)}; NM_PROFILE_MARKER={_q(nm_profile_marker)}; NFT_MAIN=; NFT_MAIN_COUNT=0
|
|
NFT_MARKER={_q(nft_marker)}
|
|
owner=missing; [ -f "$OWNER" ] && owner=$(cat "$OWNER" 2>/dev/null || printf missing)
|
|
printf '%s\\n' "__VC_VPC_BRIDGE_OWNER__=$owner"
|
|
for nft_candidate in /etc/nftables.conf /etc/sysconfig/nftables.conf; do
|
|
if [ -f "$nft_candidate" ] && [ ! -L "$nft_candidate" ] && [ "$(stat -c '%u' "$nft_candidate" 2>/dev/null)" = 0 ]; then
|
|
case "$(stat -c '%a' "$nft_candidate" 2>/dev/null)" in
|
|
[0-7][0145][0145]) NFT_MAIN="$nft_candidate"; NFT_MAIN_COUNT=$((NFT_MAIN_COUNT + 1)) ;;
|
|
*) : ;;
|
|
esac
|
|
fi
|
|
done
|
|
persistence=no
|
|
nft_file_ok=no
|
|
if [ -f "$NFT_FILE" ] && [ ! -L "$NFT_FILE" ] && [ "$(stat -c '%u:%a' "$NFT_FILE" 2>/dev/null)" = 0:600 ] &&
|
|
grep -Fqx '# virt-cloud-vpc-dataplane-v1' "$NFT_FILE" &&
|
|
grep -Fqx '# vpc={plan.vpc.vpc_id} revision={plan.desired_revision} digest={plan.desired_digest}' "$NFT_FILE"; then
|
|
nft_file_ok=yes
|
|
fi
|
|
sysctl_file_ok=no
|
|
if [ -f "$SYSCTL_FILE" ] && [ ! -L "$SYSCTL_FILE" ] && [ "$(stat -c '%u:%a' "$SYSCTL_FILE" 2>/dev/null)" = 0:600 ] &&
|
|
grep -Fqx '# virt-cloud-vpc-dataplane-v1' "$SYSCTL_FILE" &&
|
|
grep -Fqx 'net.ipv4.conf.{r.bridge}.rp_filter = 2' "$SYSCTL_FILE" &&
|
|
grep -Fqx 'net.ipv4.conf.{r.bridge}.proxy_arp = 0' "$SYSCTL_FILE" &&
|
|
grep -Fqx 'net.ipv4.conf.{r.bridge}.proxy_arp_pvlan = 1' "$SYSCTL_FILE"; then
|
|
sysctl_file_ok=yes
|
|
fi
|
|
nm_owned=no
|
|
NMCLI=$(for p in /usr/bin/nmcli /usr/sbin/nmcli /bin/nmcli /sbin/nmcli; do [ -x "$p" ] && {{ printf '%s' "$p"; break; }}; done)
|
|
if [ -n "$NMCLI" ] && [ "$($NMCLI -t --escape no -f NAME connection show 2>/dev/null | awk -v wanted="$BRIDGE" '$0 == wanted {{count++}} END {{print count + 0}}')" = 1 ]; then
|
|
nm_stable=$($NMCLI -t --escape no -g connection.stable-id connection show "$BRIDGE" 2>/dev/null | sed -n '1p')
|
|
nm_type=$($NMCLI -t --escape no -g connection.type connection show "$BRIDGE" 2>/dev/null | sed -n '1p')
|
|
nm_iface=$($NMCLI -t --escape no -g connection.interface-name connection show "$BRIDGE" 2>/dev/null | sed -n '1p')
|
|
nm_method=$($NMCLI -t --escape no -g ipv4.method connection show "$BRIDGE" 2>/dev/null | sed -n '1p')
|
|
nm6_method=$($NMCLI -t --escape no -g ipv6.method connection show "$BRIDGE" 2>/dev/null | sed -n '1p')
|
|
if [ "$nm_stable" = "$NM_PROFILE_MARKER" ] && [ "$nm_type" = bridge ] && [ "$nm_iface" = "$BRIDGE" ] && [ "$nm_method" = disabled ] && [ "$nm6_method" = disabled ]; then nm_owned=yes; fi
|
|
fi
|
|
ifup_owned=no
|
|
if [ -f "$IF_FRAGMENT" ] && [ ! -L "$IF_FRAGMENT" ] && [ "$(stat -c '%u:%a' "$IF_FRAGMENT" 2>/dev/null)" = 0:600 ] &&
|
|
grep -Fqx '# {DATAPLANE_OWNER_MARKER} vpc={plan.vpc.vpc_id} revision={plan.desired_revision} digest={plan.desired_digest}' "$IF_FRAGMENT" &&
|
|
grep -Fqx 'auto {r.bridge}' "$IF_FRAGMENT" && grep -Fqx 'iface {r.bridge} inet manual' "$IF_FRAGMENT"; then
|
|
ifup_owned=yes
|
|
fi
|
|
if [ "$NFT_MAIN_COUNT" = 1 ] && [ "$nft_file_ok" = yes ] && [ "$sysctl_file_ok" = yes ] &&
|
|
grep -Fqx "$NFT_INCLUDE" "$NFT_MAIN" && "$NFT" -c -f "$NFT_MAIN" >/dev/null 2>&1 &&
|
|
( [ "$nm_owned" = yes ] && [ "$ifup_owned" = no ] ||
|
|
[ "$nm_owned" = no ] && [ "$ifup_owned" = yes ] ); then
|
|
persistence=yes
|
|
fi
|
|
printf '%s\\n' "__VC_VPC_PERSISTENCE__=$persistence"
|
|
if "$IP" -d link show dev "$BRIDGE" >/dev/null 2>&1; then printf '%s\\n' '__VC_VPC_BRIDGE__='"$BRIDGE"; else printf '%s\\n' '__VC_VPC_BRIDGE__=missing'; fi
|
|
bridge_boundary=no
|
|
bridge_shape=no; bridge_addr_ok=no; bridge_ports_ok=no; bridge_routes_ok=no; bridge_routes6_ok=no
|
|
if [ -n "$BRIDGE_CMD" ]; then
|
|
if bridge_shape_text=$("$IP" -d link show dev "$BRIDGE" 2>/dev/null) &&
|
|
printf '%s\n' "$bridge_shape_text" | grep -Eq '(^|[[:space:]])bridge[[:space:]]+(forward_delay|stp_state|hello_time|ageing_time)([[:space:]]|$)'; then
|
|
bridge_shape=yes
|
|
fi
|
|
if bridge_addr4=$("$IP" -o -4 addr show dev "$BRIDGE" 2>/dev/null) &&
|
|
bridge_addr6=$("$IP" -o -6 addr show dev "$BRIDGE" 2>/dev/null) &&
|
|
[ -z "$bridge_addr4$bridge_addr6" ]; then
|
|
bridge_addr_ok=yes
|
|
fi
|
|
if bridge_ports=$("$BRIDGE_CMD" link show master "$BRIDGE" 2>/dev/null); then
|
|
bridge_ports_ok=yes
|
|
fi
|
|
if bridge_routes=$("$IP" -o -4 route show table all dev "$BRIDGE" 2>/dev/null) &&
|
|
printf '%s\n' "$bridge_routes" | awk -v wanted_table="$ROUTE_TABLE" '
|
|
NF {{table="main"; proto=""; prefix=$1
|
|
for (i=1;i<=NF;i++) {{if ($i=="table") table=$(i+1); if ($i=="proto") proto=$(i+1)}}
|
|
if (table=="254") table="main"
|
|
if (prefix=="default" || table!=wanted_table || proto!="186") bad=1}}
|
|
END {{exit bad ? 1 : 0}}'; then
|
|
bridge_routes_ok=yes
|
|
fi
|
|
if bridge_routes6=$($IP -o -6 route show table all dev "$BRIDGE" 2>/dev/null) && [ -z "$bridge_routes6" ]; then
|
|
bridge_routes6_ok=yes
|
|
fi
|
|
fi
|
|
if [ "$bridge_shape" = yes ] && [ "$bridge_addr_ok" = yes ] &&
|
|
[ "$bridge_ports_ok" = yes ] && [ "$bridge_routes_ok" = yes ] &&
|
|
[ "$bridge_routes6_ok" = yes ]; then
|
|
bridge_bad=no
|
|
while IFS= read -r bridge_port_line; do
|
|
[ -n "$bridge_port_line" ] || continue
|
|
bridge_port=$(printf '%s\\n' "$bridge_port_line" | sed -n 's/^[0-9][0-9]*: \\([^:[:space:]]*\\).*/\\1/p')
|
|
case "$bridge_port" in
|
|
vnet[0-9]*|tap[0-9A-Za-z_.-]*|macvtap[0-9]*|fwpr[0-9A-Za-z_.-]*|fwln[0-9A-Za-z_.-]*) : ;;
|
|
*) bridge_bad=yes ;;
|
|
esac
|
|
done <<__VC_VPC_OBS_PORTS__
|
|
$bridge_ports
|
|
__VC_VPC_OBS_PORTS__
|
|
if [ "$bridge_bad" = no ]; then
|
|
bridge_boundary=yes
|
|
fi
|
|
fi
|
|
printf '%s\\n' "__VC_VPC_BRIDGE_BOUNDARY__=$bridge_boundary"
|
|
if [ -r "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp" ]; then printf '%s\\n' '__VC_VPC_PROXY_ARP__='"$(cat "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp")"; else printf '%s\\n' '__VC_VPC_PROXY_ARP__=missing'; fi
|
|
if [ -r "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp_pvlan" ]; then printf '%s\\n' '__VC_VPC_PROXY_ARP_PVLAN__='"$(cat "/proc/sys/net/ipv4/conf/$BRIDGE/proxy_arp_pvlan")"; else printf '%s\\n' '__VC_VPC_PROXY_ARP_PVLAN__=missing'; fi
|
|
if [ -r "/proc/sys/net/ipv4/conf/$BRIDGE/rp_filter" ]; then printf '%s\\n' '__VC_VPC_RP_FILTER__='"$(cat "/proc/sys/net/ipv4/conf/$BRIDGE/rp_filter")"; else printf '%s\\n' '__VC_VPC_RP_FILTER__=missing'; fi
|
|
proxy_records=$($IP neigh show proxy dev "$BRIDGE" 2>/dev/null | awk '$1 ~ /^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$/ {{print $1}}' | LC_ALL=C sort -u || true)
|
|
printf '%s\\n' "$proxy_records" | while IFS= read -r proxy_ip; do
|
|
[ -n "$proxy_ip" ] && printf '%s\\n' "__VC_VPC_PROXY_IP__=$proxy_ip"
|
|
done
|
|
proxy_state=missing
|
|
if [ -f "$PROXY_FILE" ] && sed -n '1p' "$PROXY_FILE" | grep -Fqx '# virt-cloud-vpc-proxy-v1'; then
|
|
expected_proxy=$(sed '1d' "$PROXY_FILE" | LC_ALL=C sort -u)
|
|
if [ "$proxy_records" = "$expected_proxy" ]; then proxy_state=owned; fi
|
|
fi
|
|
printf '%s\\n' '__VC_VPC_PROXY_STATE__='"$proxy_state"
|
|
if "$NFT" list table inet "$TABLE" >/dev/null 2>&1; then printf '%s\\n' '__VC_VPC_NFT_TABLE__='"$TABLE"; else printf '%s\\n' '__VC_VPC_NFT_TABLE__=missing'; fi
|
|
if [ -f "$STATE" ] && grep -Fqx 'owner|{DATAPLANE_OWNER_MARKER}' "$STATE"; then
|
|
printf '%s\\n' '__VC_VPC_STATE_OWNER__=owned'
|
|
printf '%s\\n' "__VC_VPC_STATE_REVISION__=$(awk -F'|' '$1==\"revision\"{{print $2; exit}}' \"$STATE\")"
|
|
printf '%s\\n' "__VC_VPC_STATE_DIGEST__=$(awk -F'|' '$1==\"digest\"{{print $2; exit}}' \"$STATE\")"
|
|
else
|
|
printf '%s\\n' '__VC_VPC_STATE_OWNER__=missing'
|
|
fi
|
|
nft_text=$($NFT list table inet "$TABLE" 2>/dev/null || true)
|
|
nft_verified=yes
|
|
printf '%s\\n' "$nft_text" | grep -Fq "table inet $TABLE" || nft_verified=no
|
|
printf '%s\\n' "$nft_text" | grep -Fq "chain forward" || nft_verified=no
|
|
printf '%s\\n' "$nft_text" | grep -Fq "hook forward priority -190" || nft_verified=no
|
|
printf '%s\\n' "$nft_text" | grep -Fq "chain input" || nft_verified=no
|
|
printf '%s\\n' "$nft_text" | grep -Fq "chain output" || nft_verified=no
|
|
printf '%s\\n' "$nft_text" | grep -Fq "set member_ips" || nft_verified=no
|
|
printf '%s\\n' "$nft_text" | grep -Fq "set vpc_dest_ips" || nft_verified=no
|
|
{nft_checks}
|
|
if [ "$nft_verified" = yes ]; then
|
|
printf '%s\\n' '__VC_VPC_ACL_REVISION__={plan.desired_revision}'
|
|
printf '%s\\n' '__VC_VPC_ACL_DIGEST__={plan.desired_digest}'
|
|
printf '%s\\n' '__VC_VPC_ANTI_SPOOF__=yes'
|
|
else
|
|
printf '%s\\n' '__VC_VPC_ACL_REVISION__=missing'
|
|
printf '%s\\n' '__VC_VPC_ACL_DIGEST__=missing'
|
|
printf '%s\\n' '__VC_VPC_ANTI_SPOOF__=no'
|
|
fi
|
|
{bridge_observe}
|
|
{arp_observe}
|
|
route_records=$($IP -o -4 route show table "$ROUTE_TABLE" 2>/dev/null | awk '{{
|
|
destination=$1; device=""; via="-"; metric="100"; proto=""
|
|
for (i=1; i<=NF; i++) {{
|
|
if ($i == "dev" && i < NF) device=$(i+1)
|
|
if ($i == "via" && i < NF) via=$(i+1)
|
|
if ($i == "metric" && i < NF) metric=$(i+1)
|
|
if ($i == "proto" && i < NF) proto=$(i+1)
|
|
}}
|
|
if (destination != "") printf "%s|%s|%s|%s|%s\\n", destination, device, via, metric, proto
|
|
}}' || true)
|
|
rule_records=$($IP -o rule show 2>/dev/null | awk '{{
|
|
pref=$1; sub(/:$/, "", pref); iif="-"; destination=""; table=""
|
|
for (i=1;i<=NF;i++) {{
|
|
if ($i == "iif" && i < NF) iif=$(i+1)
|
|
if ($i == "to" && i < NF) destination=$(i+1)
|
|
if ($i == "lookup" && i < NF) table=$(i+1)
|
|
}}
|
|
if (pref != "" && iif != "" && destination != "" && table != "") printf "%s|%s|%s|%s\\n", pref, iif, destination, table
|
|
}}' || true)
|
|
{rule_checks}
|
|
route_extra=$(printf '%s\\n' "$route_records" | awk '{{
|
|
if (length($0) == 0) next
|
|
count++
|
|
{route_extra_awk}
|
|
extra=1
|
|
}} END {{ if (count != {len(routes)}) extra=1; print extra ? "yes" : "no" }}')
|
|
printf '%s\\n' "__VC_VPC_ROUTE_EXTRA__=$route_extra"
|
|
{route_checks}
|
|
while IFS='|' read -r peer_host key expected_allowed; do
|
|
[ -n "$peer_host" ] || continue
|
|
actual=$(printf '%s\\n' "$($WG show "$WG_IFACE" allowed-ips 2>/dev/null || true)" | awk -v wanted="$key" '$1 == wanted {{print $2; exit}}')
|
|
# ``wg show`` does not promise AllowedIPs order. Canonicalize both
|
|
# comma-separated lists before comparing, and retain the complete set in
|
|
# the marker. The Python adapter projects this to this VPC's tenant /32s
|
|
# after proving that no extra or missing entry exists.
|
|
expected_set=$(printf '%s\\n' "$expected_allowed" | tr ',' '\\n' | awk 'NF' | LC_ALL=C sort -u)
|
|
actual_set=$(printf '%s\\n' "$actual" | tr ',' '\\n' | awk 'NF' | LC_ALL=C sort -u)
|
|
allowed=$(printf '%s\\n' "$actual_set" | awk 'BEGIN {{ ORS=""; sep="" }} NF {{ printf "%s%s", sep, $0; sep="," }}')
|
|
state=missing
|
|
if [ -n "$actual_set" ] && [ "$actual_set" = "$expected_set" ]; then
|
|
state=owned
|
|
else
|
|
allowed=missing
|
|
fi
|
|
printf '%s\\n' "__VC_VPC_WG_ALLOWED__=$peer_host|$key|$allowed"
|
|
printf '%s\\n' "__VC_VPC_WG_STATE__=$peer_host|$state"
|
|
done <<'{peer_marker}'
|
|
{peer_lines}
|
|
{peer_marker}
|
|
expected_fabric_peer_keys={fabric_peer_key_arg}
|
|
wg_peer_set=unverified
|
|
if [ "{fabric_peer_keys_provided}" = yes ]; then
|
|
live_fabric_peer_keys=$("$WG" show "$WG_IFACE" peers 2>/dev/null || true)
|
|
wg_peer_set=exact
|
|
for live_peer_key in $live_fabric_peer_keys; do
|
|
case " $expected_fabric_peer_keys " in
|
|
*" $live_peer_key "*) : ;;
|
|
*) wg_peer_set=extra ;;
|
|
esac
|
|
done
|
|
for expected_peer_key in $expected_fabric_peer_keys; do
|
|
printf '%s\n' "$live_fabric_peer_keys" | grep -Fqx "$expected_peer_key" || wg_peer_set=missing
|
|
done
|
|
fi
|
|
printf '%s\n' "__VC_VPC_WG_PEER_SET__=$wg_peer_set"
|
|
exit 0
|
|
'''
|
|
|
|
|
|
def _markers(output: str) -> dict[str, Any]:
|
|
if not isinstance(output, str) or len(output) > MAX_OUTPUT:
|
|
raise VpcDataplaneError("VPC 回读输出过大", 502, "vpc_readback_invalid")
|
|
result: dict[str, Any] = {"routes": [], "wg_allowed_ips": [],
|
|
"wg_states": {}, "rules": [], "proxy_ips": []}
|
|
for line in output.splitlines():
|
|
if not line.startswith("__VC_VPC_") or "=" not in line:
|
|
continue
|
|
key, value = line.split("=", 1)
|
|
# A peer can legitimately carry many exact /32s from multiple VPCs;
|
|
# keep a bounded upper limit without truncating the authenticated set.
|
|
if len(value) > 65536:
|
|
raise VpcDataplaneError("VPC 回读字段过长", 502, "vpc_readback_invalid")
|
|
if key == "__VC_VPC_ROUTE__":
|
|
p = value.split("|")
|
|
if len(p) == 6:
|
|
result["routes"].append({
|
|
"destination_cidr": p[0], "device": p[1], "table": p[2],
|
|
"via": None if p[3] == "-" else p[3],
|
|
"metric": p[4], "owner": DATAPLANE_OWNER_MARKER,
|
|
"state": p[5],
|
|
})
|
|
elif key == "__VC_VPC_PROXY_IP__":
|
|
if re.fullmatch(r"[0-9]{1,3}(?:\.[0-9]{1,3}){3}", value):
|
|
try:
|
|
address = ipaddress.IPv4Address(value)
|
|
except ipaddress.AddressValueError:
|
|
continue
|
|
result["proxy_ips"].append(str(address))
|
|
elif key == "__VC_VPC_RULE__":
|
|
p = value.split("|")
|
|
if len(p) == 5:
|
|
result["rules"].append({"priority": p[0], "iif": p[1],
|
|
"destination_cidr": p[2],
|
|
"table": p[3], "state": p[4]})
|
|
elif key == "__VC_VPC_WG_ALLOWED__":
|
|
p = value.split("|")
|
|
if len(p) == 3:
|
|
result["wg_allowed_ips"].append({"peer_host_id": p[0], "public_key": p[1], "tenant_allowed_ips": [] if p[2] == "missing" else [x for x in p[2].split(",") if x]})
|
|
elif key == "__VC_VPC_WG_STATE__":
|
|
p = value.split("|")
|
|
if len(p) == 2 and p[1] in {"owned", "missing"}:
|
|
result["wg_states"][p[0]] = p[1]
|
|
else:
|
|
result[key.removeprefix("__VC_VPC_").removesuffix("__").lower()] = value
|
|
return result
|
|
|
|
|
|
def patch_private_interface_xml(xml: str | bytes, *, member: VpcMember,
|
|
resources: VpcDataplaneResources,
|
|
link_state: str = "up",
|
|
vpc_id: str | None = None,
|
|
desired_revision: int | None = None,
|
|
desired_digest: str | None = None) -> str:
|
|
"""Add/validate a private0 NIC on an explicit panel-owned bridge."""
|
|
if link_state not in {"up", "down"}:
|
|
raise VpcDataplaneError("私网网卡 link 状态无效")
|
|
private_mac = _mac(member.private_mac)
|
|
try:
|
|
root = ET.fromstring(xml)
|
|
except ET.ParseError as exc:
|
|
raise VpcDataplaneError("虚拟机 XML 无效", 409, "vpc_private_xml_invalid") from exc
|
|
devices = root.find("./devices")
|
|
if devices is None:
|
|
raise VpcDataplaneError("虚拟机 XML 缺少 devices", 409, "vpc_private_xml_invalid")
|
|
interfaces = list(devices.findall("interface"))
|
|
try:
|
|
public = select_public_interface(interfaces, allow_legacy_single=True)
|
|
# Legacy domains may have a single public NIC without an alias. Give
|
|
# it the stable role name before adding private0 so the next
|
|
# reconciliation does not mistake the public NIC for an unknown role.
|
|
alias = public.find("alias")
|
|
if alias is None:
|
|
alias = ET.SubElement(public, "alias")
|
|
alias.set("name", PUBLIC_INTERFACE_ALIAS)
|
|
private = select_private_interface(interfaces)
|
|
except ValueError as exc:
|
|
raise VpcDataplaneError(str(exc), 409, "vpc_private_xml_role_invalid") from exc
|
|
if private is None:
|
|
private = ET.SubElement(devices, "interface", {"type": "bridge"})
|
|
else:
|
|
if private.get("type") != "bridge":
|
|
raise VpcDataplaneError("private0 必须使用 bridge", 409, "vpc_private_xml_source_invalid")
|
|
existing_filters = private.findall("filterref")
|
|
if existing_filters and any(
|
|
item.get("filter") != "clean-traffic"
|
|
for item in existing_filters):
|
|
raise VpcDataplaneError("private0 nwfilter 不受面板管理", 409,
|
|
"vpc_private_xml_filter_invalid")
|
|
source = private.find("source")
|
|
if source is None or source.get("network") is not None or source.get("bridge") != resources.bridge:
|
|
raise VpcDataplaneError("private0 bridge 不匹配", 409, "vpc_private_xml_source_mismatch")
|
|
source = private.find("source")
|
|
if source is None:
|
|
source = ET.Element("source")
|
|
private.insert(0, source)
|
|
source.attrib.clear(); source.set("bridge", resources.bridge)
|
|
mac = private.find("mac")
|
|
if mac is None:
|
|
mac = ET.Element("mac"); private.insert(0, mac)
|
|
mac.set("address", private_mac)
|
|
alias = private.find("alias")
|
|
if alias is None:
|
|
alias = ET.SubElement(private, "alias")
|
|
alias.set("name", PRIVATE_INTERFACE)
|
|
model = private.find("model")
|
|
if model is None:
|
|
model = ET.SubElement(private, "model")
|
|
model.set("type", "virtio")
|
|
link = private.find("link")
|
|
if link is None:
|
|
link = ET.SubElement(private, "link")
|
|
link.set("state", link_state)
|
|
for item in list(private.findall("filterref")):
|
|
private.remove(item)
|
|
# The private L3 data plane owns anti-spoofing and ACL enforcement in its
|
|
# dedicated bridge/inet/arp nft tables. Do not attach the public
|
|
# libvirt ``clean-traffic`` nwfilter to private0; the role observer
|
|
# deliberately treats any private filter as an unsafe mixed policy.
|
|
metadata = root.find("metadata")
|
|
if metadata is None:
|
|
metadata = ET.Element("metadata"); root.insert(0, metadata)
|
|
managed = metadata.find(f"{{{PANEL_NS}}}managed")
|
|
if managed is None:
|
|
managed = ET.SubElement(metadata, f"{{{PANEL_NS}}}managed")
|
|
managed.set("private-member-id", member.member_id)
|
|
managed.set("private-ip-cidr", f"{member.private_ip}/32")
|
|
managed.set("private-network-bridge", resources.bridge)
|
|
managed.set("private-network-verified", "true")
|
|
managed.set("private-dataplane-owner", DATAPLANE_OWNER_MARKER)
|
|
if vpc_id is not None:
|
|
managed.set("private-vpc-id", vpc_id)
|
|
if desired_revision is not None:
|
|
managed.set("private-network-revision", str(desired_revision))
|
|
if desired_digest is not None:
|
|
managed.set("private-network-digest", desired_digest)
|
|
ET.register_namespace("panel", PANEL_NS)
|
|
return ET.tostring(root, encoding="unicode")
|
|
|
|
|
|
def _member_nics(
|
|
plan: VpcDataplanePlan,
|
|
xml_by_instance: Mapping[str, Any] | None,
|
|
) -> tuple[dict[str, Any], ...]:
|
|
def xml_details(xml: str, member: VpcMember, *, metadata: bool) -> tuple[bool, str | None]:
|
|
try:
|
|
root = ET.fromstring(xml)
|
|
private = select_private_interface(
|
|
list(root.findall("./devices/interface")))
|
|
source = private.find("source") if private is not None else None
|
|
mac = private.find("mac") if private is not None else None
|
|
ok = bool(
|
|
private and private.get("type") == "bridge"
|
|
and source is not None
|
|
and source.get("bridge") == plan.resources.bridge
|
|
and mac is not None
|
|
and mac.get("address", "").lower() == member.private_mac
|
|
# VPC isolation is provided by the dedicated bridge/inet/arp
|
|
# nft context. The public libvirt nwfilter is intentionally
|
|
# absent from private0.
|
|
and not private.findall("filterref")
|
|
)
|
|
link = private.find("link") if private is not None else None
|
|
link_state = link.get("state", "up").lower() if link is not None else "up"
|
|
if link_state not in {"up", "down"}:
|
|
return False, None
|
|
if not metadata:
|
|
return ok, link_state
|
|
managed = root.find(f"./metadata/{{{PANEL_NS}}}managed")
|
|
return bool(
|
|
ok and managed is not None
|
|
and managed.get("private-vpc-id") == plan.vpc.vpc_id
|
|
and managed.get("private-member-id") == member.member_id
|
|
and managed.get("private-ip-cidr") == f"{member.private_ip}/32"
|
|
and managed.get("private-network-revision")
|
|
== str(plan.desired_revision)
|
|
and managed.get("private-network-digest")
|
|
== plan.desired_digest
|
|
), link_state
|
|
except (ET.ParseError, ValueError):
|
|
return False, None
|
|
|
|
result = []
|
|
for member in plan.local_members:
|
|
verified = False
|
|
link_state: str | None = None
|
|
if xml_by_instance and member.instance_uuid in xml_by_instance:
|
|
value = xml_by_instance[member.instance_uuid]
|
|
if isinstance(value, Mapping):
|
|
inactive = value.get("inactive")
|
|
live = value.get("live")
|
|
inactive_ok = False
|
|
live_ok = True
|
|
inactive_link: str | None = None
|
|
live_link: str | None = None
|
|
if isinstance(inactive, str):
|
|
inactive_ok, inactive_link = xml_details(
|
|
inactive, member, metadata=True)
|
|
if live is not None:
|
|
if isinstance(live, str):
|
|
live_ok, live_link = xml_details(
|
|
live, member, metadata=False)
|
|
else:
|
|
live_ok = False
|
|
# A running domain is only healthy when the persistent and
|
|
# live role agree. This makes a stale config-only link state
|
|
# visible to the lifecycle gate instead of silently choosing
|
|
# whichever XML happened to be read first.
|
|
verified = inactive_ok and live_ok and (
|
|
live is None or inactive_link == live_link)
|
|
link_state = live_link if live is not None else inactive_link
|
|
elif isinstance(value, str):
|
|
verified, link_state = xml_details(value, member, metadata=True)
|
|
result.append({"member_id": member.member_id, "instance_uuid": member.instance_uuid,
|
|
"interface": PRIVATE_INTERFACE, "private_ip_cidr": f"{member.private_ip}/32",
|
|
"private_mac": member.private_mac, "xml_verified": verified,
|
|
"link_state": link_state})
|
|
return tuple(result)
|
|
|
|
|
|
class VpcPrivateHostAdapter:
|
|
"""Execute generated scripts through a pinned remote host manager."""
|
|
|
|
def __init__(self, manager: Any, profile: Mapping[str, Any] | None = None):
|
|
self.manager = manager
|
|
self.profile = dict(profile) if isinstance(profile, Mapping) else None
|
|
|
|
def _run(self, script: str) -> str:
|
|
current = self.profile if self.profile is not None else self.manager.current()
|
|
if not isinstance(current, Mapping) or not current:
|
|
raise VpcDataplaneError("宿主机尚未配置", 409, "vpc_host_not_configured")
|
|
if current.get("key_locked"):
|
|
raise VpcDataplaneError("宿主机 SSH 私钥仍锁定", 409, "vpc_host_key_locked")
|
|
argv = [getattr(self.manager, "ssh_binary", "ssh"), "-F", current["ssh_config_file"], "-T", "-o", "ConnectTimeout=10", current["ssh_target"], "/bin/sh -s"]
|
|
try:
|
|
result = self.manager._exec(argv, timeout=35, input_text=script)
|
|
except Exception as exc:
|
|
raise VpcDataplaneError("宿主机 VPC SSH 请求失败", 502, "vpc_host_connection_failed") from exc
|
|
if result.returncode != 0:
|
|
raise VpcDataplaneError("宿主机 VPC 命令失败", 502, "vpc_remote_command_failed")
|
|
return result.stdout
|
|
|
|
def _deployment_contract(self) -> tuple[str, str, str, str]:
|
|
"""Validate the same immutable host capability gate as Fabric."""
|
|
from .fabric import (
|
|
FabricError, _profile_ifdown_path, _profile_ifquery_path,
|
|
_profile_ifup_path, _profile_network_mode,
|
|
)
|
|
|
|
current = self.profile if self.profile is not None else self.manager.current()
|
|
try:
|
|
mode = _profile_network_mode(current)
|
|
except FabricError as exc:
|
|
raise VpcDataplaneError(
|
|
"宿主机网络管理器与 VPC 部署模式不匹配", 409,
|
|
"vpc_dataplane_network_manager_conflict") from exc
|
|
capabilities = current.get("capabilities") if isinstance(current, Mapping) else None
|
|
network = capabilities.get("network") if isinstance(capabilities, Mapping) else None
|
|
wireguard = capabilities.get("wireguard") if isinstance(capabilities, Mapping) else None
|
|
nftables = capabilities.get("nftables") if isinstance(capabilities, Mapping) else None
|
|
if (not isinstance(network, Mapping)
|
|
or not isinstance(wireguard, Mapping)
|
|
or not isinstance(nftables, Mapping)
|
|
or capabilities.get("vpc_ready") is not True
|
|
or network.get("firewall_manager") != "nftables"
|
|
or network.get("ip_forward") != 1
|
|
or nftables.get("ready") is not True
|
|
or wireguard.get("interface") != FABRIC_INTERFACE
|
|
or wireguard.get("interface_present") is not True
|
|
or wireguard.get("interface_is_wireguard") is not True
|
|
or wireguard.get("owned_by_panel") is not True):
|
|
raise VpcDataplaneError(
|
|
"宿主机 VPC 网络或防火墙能力未通过探测", 409,
|
|
"vpc_host_capability_missing")
|
|
return (
|
|
mode,
|
|
_profile_ifup_path(current),
|
|
_profile_ifdown_path(current),
|
|
_profile_ifquery_path(current),
|
|
)
|
|
|
|
def apply(self, plan: VpcDataplanePlan) -> dict[str, Any]:
|
|
mode, ifup_path, ifdown_path, ifquery_path = self._deployment_contract()
|
|
script = render_apply_script(
|
|
plan, deployment_mode=mode,
|
|
ifup_path=ifup_path, ifdown_path=ifdown_path,
|
|
ifquery_path=ifquery_path,
|
|
)
|
|
markers = _markers(self._run(script))
|
|
if markers.get("applied") != "yes":
|
|
raise VpcDataplaneError("宿主机未确认 VPC 应用", 502, "vpc_apply_unconfirmed")
|
|
return {"state": "applying", "readback": "pending", "rollback": "none", "owner": DATAPLANE_OWNER_MARKER}
|
|
|
|
def remove(self, plan: VpcDataplanePlan) -> dict[str, Any]:
|
|
"""Remove a panel-owned empty-host VPC and prove the absence.
|
|
|
|
The removal script is deliberately separate from ``apply``: it may
|
|
only run when the parsed plan has no local members and it validates
|
|
every persisted artifact/bridge owner before deleting anything.
|
|
A positive marker is required before the caller can advance a host
|
|
tombstone.
|
|
"""
|
|
mode, ifup_path, ifdown_path, ifquery_path = self._deployment_contract()
|
|
markers = _markers(self._run(render_remove_script(
|
|
plan, deployment_mode=mode, ifup_path=ifup_path,
|
|
ifdown_path=ifdown_path, ifquery_path=ifquery_path)))
|
|
if markers.get("removed") != "yes":
|
|
raise VpcDataplaneError(
|
|
"宿主机未确认 VPC 清理", 502, "vpc_remove_unconfirmed")
|
|
return {
|
|
"vpc_id": plan.vpc.vpc_id,
|
|
"host_id": plan.host_id,
|
|
"desired_revision": plan.desired_revision,
|
|
"observed_revision": plan.desired_revision,
|
|
"desired_digest": plan.desired_digest,
|
|
"observed_digest": plan.desired_digest,
|
|
"state": "active",
|
|
"removed": True,
|
|
"bridge": None,
|
|
"bridge_owner": None,
|
|
"member_nics": [],
|
|
"routes": [],
|
|
"wg_allowed_ips": [],
|
|
"nft_table": None,
|
|
"acl_revision": plan.desired_revision,
|
|
"anti_spoof": False,
|
|
"rp_filter": None,
|
|
"proxy_arp": None,
|
|
"proxy_arp_pvlan": None,
|
|
"proxy_ips": [],
|
|
"readback": "confirmed",
|
|
"rollback": markers.get("rollback", "none"),
|
|
"reason_code": None,
|
|
}
|
|
|
|
def observe(
|
|
self, plan: VpcDataplanePlan, *,
|
|
xml_by_instance: Mapping[str, Any] | None = None,
|
|
) -> dict[str, Any]:
|
|
self._deployment_contract()
|
|
parsed = _markers(self._run(render_observe_script(plan)))
|
|
expected_routes = {
|
|
(x["destination_cidr"], x["device"], str(x["table"]),
|
|
x.get("via"), str(x.get("metric", 100)))
|
|
for x in _routes(plan)
|
|
}
|
|
expected_proxy_ips = {
|
|
str(ipaddress.ip_network(x["destination_cidr"], strict=True).network_address)
|
|
for x in _routes(plan)
|
|
}
|
|
actual_proxy_ips = set(parsed.get("proxy_ips", []))
|
|
actual_routes = {
|
|
(x.get("destination_cidr"), x.get("device"), str(x.get("table")),
|
|
x.get("via"), str(x.get("metric", 100)))
|
|
for x in parsed.get("routes", [])
|
|
}
|
|
expected_wg_full = {
|
|
# The live `wg show` value includes the peer's overlay /32 as the
|
|
# first AllowedIP. Compare the complete cryptokey-routing set;
|
|
# project tenant entries only after that exact check.
|
|
p.host_id: (p.public_key, set(_allowed(plan, p).split(",")))
|
|
for p in plan.peers
|
|
}
|
|
actual_wg_full = {
|
|
x.get("peer_host_id"): (x.get("public_key"),
|
|
set(x.get("tenant_allowed_ips", [])))
|
|
for x in parsed.get("wg_allowed_ips", [])
|
|
}
|
|
# The live peer may carry /32s belonging to other VPCs sharing this
|
|
# Fabric. Verify the complete aggregate here, then expose only this
|
|
# VPC's tenant set to the PHP contract.
|
|
expected_tenant: dict[str, set[str]] = {
|
|
p.host_id: {m.private_ip + "/32" for m in plan.remote_members
|
|
if m.host_id == p.host_id}
|
|
for p in plan.peers
|
|
}
|
|
members_by_id = {m.member_id: m for m in plan.members}
|
|
for route in plan.vpc.routes:
|
|
target = (members_by_id.get(route.next_hop_member_id)
|
|
if route.next_hop_member_id is not None else None)
|
|
target_host = target.host_id if target is not None else route.next_hop_host_id
|
|
if target_host in expected_tenant:
|
|
expected_tenant[target_host].add(route.destination_cidr)
|
|
expected_wg = {
|
|
p.host_id: (p.public_key, expected_tenant.get(p.host_id, set()))
|
|
for p in plan.peers
|
|
}
|
|
actual_wg = {
|
|
host: (key, ips & expected_wg.get(host, (None, set()))[1])
|
|
for host, (key, ips) in actual_wg_full.items()
|
|
}
|
|
members = _member_nics(plan, xml_by_instance)
|
|
actual_route_state = {
|
|
(x.get("destination_cidr"), x.get("device"), str(x.get("table")),
|
|
x.get("via"), str(x.get("metric", 100)))
|
|
for x in parsed.get("routes", []) if x.get("state") == "owned"
|
|
}
|
|
expected_rules = {
|
|
(str(plan.resources.rule_priority), "-",
|
|
plan.vpc.cidr, str(plan.resources.route_table)),
|
|
(str(plan.resources.rule_priority + 1), FABRIC_INTERFACE,
|
|
plan.vpc.cidr, str(plan.resources.route_table)),
|
|
}
|
|
actual_rules = {
|
|
(str(x.get("priority")), x.get("iif"),
|
|
x.get("destination_cidr"), str(x.get("table")))
|
|
for x in parsed.get("rules", []) if x.get("state") == "owned"
|
|
}
|
|
expected_peer_states = {p.host_id: "owned" for p in plan.peers}
|
|
expected_bridge_nft = "yes" if plan.local_members else "none"
|
|
active = (parsed.get("bridge") == plan.resources.bridge
|
|
and parsed.get("bridge_owner") == DATAPLANE_OWNER_MARKER
|
|
and parsed.get("persistence") == "yes"
|
|
and parsed.get("bridge_boundary") == "yes"
|
|
and parsed.get("proxy_arp") == "0"
|
|
and parsed.get("proxy_arp_pvlan") == "1"
|
|
and parsed.get("proxy_state") == "owned"
|
|
and actual_proxy_ips == expected_proxy_ips
|
|
and parsed.get("rp_filter") == "2"
|
|
and parsed.get("nft_table") == plan.resources.nft_table
|
|
and parsed.get("state_owner") == "owned"
|
|
and str(parsed.get("state_revision")) == str(plan.desired_revision)
|
|
and parsed.get("state_digest") == plan.desired_digest
|
|
and str(parsed.get("acl_revision")) == str(plan.desired_revision)
|
|
and parsed.get("acl_digest") == plan.desired_digest
|
|
and parsed.get("anti_spoof") == "yes"
|
|
and parsed.get("bridge_nft") == expected_bridge_nft
|
|
and parsed.get("route_extra") == "no"
|
|
and expected_routes == actual_routes
|
|
and actual_routes == actual_route_state
|
|
and expected_rules == actual_rules
|
|
and expected_wg_full == actual_wg_full
|
|
and (plan.fabric_peer_keys is None
|
|
or parsed.get("wg_peer_set") == "exact")
|
|
and expected_peer_states == parsed.get("wg_states", {})
|
|
and parsed.get("arp_nft") == ("yes" if plan.local_members else "none")
|
|
and all(x["xml_verified"] for x in members))
|
|
projected_wg = tuple(
|
|
dict(item, tenant_allowed_ips=sorted(
|
|
expected_wg[item.get("peer_host_id")][1]
|
|
& set(item.get("tenant_allowed_ips", []))))
|
|
for item in parsed.get("wg_allowed_ips", [])
|
|
if item.get("peer_host_id") in expected_wg
|
|
and expected_wg[item.get("peer_host_id")][1]
|
|
)
|
|
rp_filter = int(parsed["rp_filter"]) if str(parsed.get("rp_filter", "")).isdigit() else None
|
|
proxy_arp = int(parsed["proxy_arp"]) if str(parsed.get("proxy_arp", "")).isdigit() else None
|
|
proxy_arp_pvlan = (int(parsed["proxy_arp_pvlan"])
|
|
if str(parsed.get("proxy_arp_pvlan", "")).isdigit()
|
|
else None)
|
|
return VpcDataplaneObservation(
|
|
plan.vpc.vpc_id, plan.host_id, plan.desired_revision,
|
|
plan.desired_revision if active else 0, plan.desired_digest,
|
|
plan.desired_digest if active else None,
|
|
"active" if active else "degraded", plan.resources.bridge,
|
|
parsed.get("bridge_owner"), members, tuple(parsed.get("routes", [])),
|
|
projected_wg, parsed.get("nft_table"),
|
|
plan.desired_revision if active else None,
|
|
parsed.get("anti_spoof") == "yes", rp_filter, proxy_arp,
|
|
proxy_arp_pvlan,
|
|
tuple(sorted(actual_proxy_ips)), "confirmed" if active else "mismatch",
|
|
"none", None if active else "vpc_private_dataplane_readback_mismatch",
|
|
).as_dict()
|
|
|
|
|
|
__all__ = ["DATAPLANE_OWNER_MARKER", "FABRIC_INTERFACE", "PRIVATE_INTERFACE", "VpcDataplaneError", "VpcFabricPeer", "VpcDataplaneResources", "VpcDataplanePlan", "VpcDataplaneObservation", "derive_private_mac", "resource_names", "parse_vpc_dataplane_plan", "render_vpc_nft_rules", "render_apply_script", "render_remove_script", "render_observe_script", "patch_private_interface_xml", "VpcPrivateHostAdapter"]
|