You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/agent/vpcrouting/driver_linux.go

548 lines
19 KiB
Go

//go:build linux
package vpcrouting
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/netip"
"os"
"os/exec"
"path/filepath"
"sort"
"strconv"
"strings"
"syscall"
"time"
)
const (
managedRouteProtocol = "186"
maximumCommandOutput = 2 << 20
nftFamily = "inet"
nftTable = "virt_cloud_vpc"
// A readiness probe performs bounded permanent and runtime firewalld
// snapshots. Keep this below the panel's 20-second capability request
// deadline while allowing ordinary D-Bus serialization on loaded hosts.
availabilityTimeout = 15 * time.Second
)
type linuxDriver struct {
ipPath string
nftPath string
firewallCmdPath string
run commandRunner
}
type commandRunner interface {
Run(context.Context, string, []string, []byte) ([]byte, error)
}
type execRunner struct{}
type commandError struct {
path string
exitCode int
cause error
output string
}
func (e *commandError) Error() string {
return fmt.Sprintf("%s exited with code %d", filepath.Base(e.path), e.exitCode)
}
func (e *commandError) Unwrap() error { return e.cause }
type boundedBuffer struct {
buffer bytes.Buffer
limit int
}
func (b *boundedBuffer) Write(value []byte) (int, error) {
if b.buffer.Len()+len(value) > b.limit {
remaining := b.limit - b.buffer.Len()
if remaining > 0 {
_, _ = b.buffer.Write(value[:remaining])
}
return len(value), errors.New("command output exceeded limit")
}
return b.buffer.Write(value)
}
func (execRunner) Run(ctx context.Context, path string, arguments []string, input []byte) ([]byte, error) {
command := exec.CommandContext(ctx, path, arguments...)
command.Env = []string{"LC_ALL=C"}
if input != nil {
command.Stdin = bytes.NewReader(input)
}
output := &boundedBuffer{limit: maximumCommandOutput}
command.Stdout = output
command.Stderr = output
err := command.Run()
if err == nil {
return append([]byte(nil), output.buffer.Bytes()...), nil
}
exitCode := -1
var exit *exec.ExitError
if errors.As(err, &exit) {
exitCode = exit.ExitCode()
}
return nil, &commandError{
path: path, exitCode: exitCode, cause: err,
output: boundedText(output.buffer.String(), 1024),
}
}
func newPlatformDriver() hostDriver {
return &linuxDriver{
ipPath: trustedBinary([]string{"/usr/sbin/ip", "/usr/bin/ip", "/sbin/ip", "/bin/ip"}),
nftPath: trustedBinary([]string{"/usr/sbin/nft", "/usr/bin/nft", "/sbin/nft", "/bin/nft"}),
firewallCmdPath: trustedBinary([]string{"/usr/bin/firewall-cmd", "/usr/sbin/firewall-cmd"}),
run: execRunner{},
}
}
func (d *linuxDriver) Available() error {
return d.availablePrerequisites()
}
// AvailableFor is intentionally read-only. The first mutation may create the
// reserved firewalld objects, but capability reporting must already fail if a
// running firewalld daemon cannot be inspected or if its existing reserved
// objects conflict with the exact projection committed by the Agent.
func (d *linuxDriver) AvailableFor(current map[string]Network) error {
if err := d.availablePrerequisites(); err != nil {
return err
}
ctx, cancel := context.WithTimeout(context.Background(), availabilityTimeout)
defer cancel()
return d.firewalldReadiness(ctx, current)
}
func (d *linuxDriver) availablePrerequisites() error {
if d == nil || d.run == nil {
return unavailable("vpc_l3_apply_unavailable", "VPC L3 routing backend is unavailable", nil)
}
if d.ipPath == "" || d.nftPath == "" {
return unavailable("vpc_l3_tools_unavailable", "trusted ip and nft executables are required for VPC L3 routing", nil)
}
forwarding, err := os.ReadFile("/proc/sys/net/ipv4/ip_forward")
if err != nil || strings.TrimSpace(string(forwarding)) != "1" {
return unavailable("ipv4_forwarding_disabled", "IPv4 forwarding must be enabled for VPC L3 routing", err)
}
return nil
}
func trustedBinary(candidates []string) string {
for _, path := range candidates {
info, err := os.Lstat(path)
if err != nil || !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 || info.Mode()&0o022 != 0 || info.Mode()&0o111 == 0 {
continue
}
stat, ok := info.Sys().(*syscall.Stat_t)
if !ok || stat.Uid != 0 {
continue
}
return path
}
return ""
}
func (d *linuxDriver) Reconcile(ctx context.Context, before, after map[string]Network) error {
if err := d.Available(); err != nil {
return err
}
if err := ctx.Err(); err != nil {
return err
}
firewalld, err := d.prepareFirewalld(ctx, before, after)
if err != nil {
return err
}
// Close every removed firewalld path before changing the authoritative
// routes or nftables projection. Additions are deliberately deferred until
// after the exact nftables membership rules have been installed.
if err := d.restrictFirewalld(ctx, firewalld, before, after); err != nil {
return err
}
orderedAfter := orderedNetworks(after)
for _, network := range orderedAfter {
if err := d.verifyTransport(ctx, network.Transport); err != nil {
return err
}
if err := d.ensureBridge(ctx, network); err != nil {
return err
}
}
if err := d.reconcileRoutes(ctx, before, after); err != nil {
return err
}
if err := d.reconcileFirewall(ctx, after); err != nil {
return err
}
// The dedicated nftables table is the authoritative tenant boundary.
// firewalld is reconciled only after that table is proven installed so a
// zone/policy ACCEPT can never temporarily expose an unvalidated path.
if err := d.applyFirewalld(ctx, firewalld, before, after); err != nil {
return err
}
for _, network := range orderedNetworks(before) {
if _, retained := after[network.VPCKey]; retained {
continue
}
if err := d.removeBridge(ctx, network.VPCKey); err != nil {
return err
}
}
return nil
}
type ipAddressDocument struct {
IfName string `json:"ifname"`
OperState string `json:"operstate"`
MTU int `json:"mtu"`
Addresses []struct {
Family string `json:"family"`
Local string `json:"local"`
PrefixLen int `json:"prefixlen"`
} `json:"addr_info"`
}
func (d *linuxDriver) verifyTransport(ctx context.Context, transport Transport) error {
raw, err := d.run.Run(ctx, d.ipPath, []string{"-j", "-4", "address", "show", "dev", transport.Interface}, nil)
if err != nil {
return unavailable("vpc_transport_unavailable", "configured VPC transport interface is unavailable", err)
}
var documents []ipAddressDocument
if err := json.Unmarshal(raw, &documents); err != nil || len(documents) != 1 || documents[0].IfName != transport.Interface || strings.EqualFold(documents[0].OperState, "DOWN") || documents[0].MTU != transport.MTU {
return unavailable("vpc_transport_mismatch", "configured VPC transport no longer matches the host", err)
}
prefix, _ := netip.ParsePrefix(transport.AddressCIDR)
for _, address := range documents[0].Addresses {
if address.Family == "inet" && address.Local == prefix.Addr().String() && address.PrefixLen == prefix.Bits() {
return nil
}
}
return unavailable("vpc_transport_mismatch", "configured VPC transport address is no longer present", nil)
}
type ipLinkDocument struct {
IfName string `json:"ifname"`
Address string `json:"address"`
MTU int `json:"mtu"`
OperState string `json:"operstate"`
LinkInfo *struct {
InfoKind string `json:"info_kind"`
} `json:"linkinfo"`
}
func (d *linuxDriver) ensureBridge(ctx context.Context, network Network) error {
bridge, _ := BridgeName(network.VPCKey)
ownerMAC, _ := bridgeMAC(network.VPCKey)
raw, err := d.run.Run(ctx, d.ipPath, []string{"-details", "-j", "link", "show", "dev", bridge}, nil)
if err != nil {
if !commandExitCode(err, 1) {
return unavailable("vpc_bridge_inspection_failed", "VPC bridge could not be inspected", err)
}
if _, err := d.run.Run(ctx, d.ipPath, []string{"link", "add", "name", bridge, "address", ownerMAC, "type", "bridge"}, nil); err != nil {
return unavailable("vpc_bridge_create_failed", "VPC bridge could not be created", err)
}
} else {
var links []ipLinkDocument
if json.Unmarshal(raw, &links) != nil || len(links) != 1 || links[0].IfName != bridge ||
!strings.EqualFold(links[0].Address, ownerMAC) || links[0].LinkInfo == nil || links[0].LinkInfo.InfoKind != "bridge" {
return unavailable("vpc_bridge_conflict", "VPC bridge name is already used by another host interface", nil)
}
}
if _, err := d.run.Run(ctx, d.ipPath, []string{"link", "set", "dev", bridge, "mtu", strconv.Itoa(network.Transport.MTU), "up"}, nil); err != nil {
return unavailable("vpc_bridge_configure_failed", "VPC bridge could not be activated", err)
}
prefix, _ := netip.ParsePrefix(network.CIDR)
gatewayCIDR := network.Gateway + "/" + strconv.Itoa(prefix.Bits())
if _, err := d.run.Run(ctx, d.ipPath, []string{"-4", "address", "replace", gatewayCIDR, "dev", bridge}, nil); err != nil {
return unavailable("vpc_bridge_address_failed", "VPC bridge gateway could not be configured", err)
}
proxyARP := filepath.Join("/proc/sys/net/ipv4/conf", bridge, "proxy_arp")
if err := os.WriteFile(proxyARP, []byte("1\n"), 0o644); err != nil {
return unavailable("vpc_proxy_arp_failed", "VPC bridge proxy ARP could not be enabled", err)
}
return nil
}
type routeDocument struct {
Destination string `json:"dst"`
Gateway string `json:"gateway"`
Device string `json:"dev"`
Protocol any `json:"protocol"`
}
type routeKey struct {
destination string
gateway string
device string
}
func (d *linuxDriver) reconcileRoutes(ctx context.Context, before, after map[string]Network) error {
raw, err := d.run.Run(ctx, d.ipPath, []string{"-j", "-4", "route", "show"}, nil)
if err != nil {
return unavailable("vpc_route_inspection_failed", "managed VPC routes could not be inspected", err)
}
var routes []routeDocument
if err := json.Unmarshal(raw, &routes); err != nil {
return unavailable("vpc_route_inspection_failed", "managed VPC route output is invalid", err)
}
desired := make(map[routeKey]struct{})
for _, network := range after {
for _, route := range network.PeerRoutes {
desired[routeKey{destination: route.IP + "/32", gateway: route.Via, device: network.Transport.Interface}] = struct{}{}
}
}
for _, route := range routes {
if _, target := desired[routeKey{destination: route.Destination, gateway: route.Gateway, device: route.Device}]; target && managedProtocol(route.Protocol) {
continue
}
if _, destinationWanted := desiredRouteDestination(desired, route.Destination); destinationWanted && !managedProtocol(route.Protocol) {
return unavailable("vpc_route_conflict", "a VPC destination is already owned by an unmanaged host route", nil)
}
}
relevant := make([]netip.Prefix, 0, len(before)+len(after))
for _, collection := range []map[string]Network{before, after} {
for _, network := range collection {
prefix, _ := netip.ParsePrefix(network.CIDR)
relevant = append(relevant, prefix)
}
}
for _, route := range routes {
key := routeKey{destination: route.Destination, gateway: route.Gateway, device: route.Device}
if _, keep := desired[key]; keep || !managedProtocol(route.Protocol) || !routeWithinPrefixes(route.Destination, relevant) {
continue
}
arguments := []string{"-4", "route", "del", route.Destination}
if route.Gateway != "" {
arguments = append(arguments, "via", route.Gateway)
}
if route.Device != "" {
arguments = append(arguments, "dev", route.Device)
}
arguments = append(arguments, "protocol", managedRouteProtocol)
if _, err := d.run.Run(ctx, d.ipPath, arguments, nil); err != nil {
return unavailable("vpc_route_remove_failed", "a stale VPC route could not be removed", err)
}
}
keys := make([]routeKey, 0, len(desired))
for key := range desired {
keys = append(keys, key)
}
sort.Slice(keys, func(left, right int) bool {
if keys[left].destination != keys[right].destination {
return keys[left].destination < keys[right].destination
}
if keys[left].gateway != keys[right].gateway {
return keys[left].gateway < keys[right].gateway
}
return keys[left].device < keys[right].device
})
for _, key := range keys {
if _, err := d.run.Run(ctx, d.ipPath, []string{
"-4", "route", "replace", key.destination, "via", key.gateway,
"dev", key.device, "protocol", managedRouteProtocol,
}, nil); err != nil {
return unavailable("vpc_route_apply_failed", "a VPC peer route could not be applied", err)
}
}
return nil
}
func (d *linuxDriver) reconcileFirewall(ctx context.Context, networks map[string]Network) error {
exists := true
raw, err := d.run.Run(ctx, d.nftPath, []string{"-j", "list", "table", nftFamily, nftTable}, nil)
if err != nil {
if commandExitCode(err, 1) {
exists = false
} else {
return unavailable("vpc_firewall_inspection_failed", "VPC isolation firewall could not be inspected", err)
}
} else if !nftTableOwned(raw) {
return unavailable("vpc_firewall_conflict", "the reserved VPC nftables table is owned by another configuration", nil)
}
if len(networks) == 0 {
if !exists {
return nil
}
if _, err := d.run.Run(ctx, d.nftPath, []string{"delete", "table", nftFamily, nftTable}, nil); err != nil {
return unavailable("vpc_firewall_remove_failed", "VPC isolation firewall could not be removed", err)
}
return nil
}
script := nftScript(networks, exists)
if _, err := d.run.Run(ctx, d.nftPath, []string{"-f", "-"}, []byte(script)); err != nil {
return unavailable("vpc_firewall_apply_failed", "VPC isolation firewall could not be applied", err)
}
return nil
}
func nftScript(networks map[string]Network, exists bool) string {
var builder strings.Builder
if exists {
builder.WriteString("flush table inet " + nftTable + "\n")
} else {
builder.WriteString("add table inet " + nftTable + "\n")
}
ordered := orderedNetworks(networks)
for _, network := range ordered {
short, _ := nftObjectSuffix(network.VPCKey)
writeAddressSet(&builder, "local_"+short, localAddresses(network.LocalPorts))
writeAddressSet(&builder, "remote_"+short, remoteAddresses(network.PeerRoutes))
}
builder.WriteString("add chain inet " + nftTable + " owner_v1\n")
builder.WriteString("add chain inet " + nftTable + " input { type filter hook input priority -10; policy accept; }\n")
builder.WriteString("add chain inet " + nftTable + " forward { type filter hook forward priority -10; policy accept; }\n")
for _, network := range ordered {
short, _ := nftObjectSuffix(network.VPCKey)
bridge, _ := BridgeName(network.VPCKey)
local := "@local_" + short
remote := "@remote_" + short
builder.WriteString(fmt.Sprintf("add rule inet %s input iifname \"%s\" drop\n", nftTable, bridge))
builder.WriteString(fmt.Sprintf("add rule inet %s forward iifname \"%s\" oifname \"%s\" ip saddr %s ip daddr %s accept\n", nftTable, bridge, bridge, local, local))
builder.WriteString(fmt.Sprintf("add rule inet %s forward iifname \"%s\" oifname \"%s\" ip saddr %s ip daddr %s accept\n", nftTable, bridge, network.Transport.Interface, local, remote))
builder.WriteString(fmt.Sprintf("add rule inet %s forward iifname \"%s\" oifname \"%s\" ip saddr %s ip daddr %s accept\n", nftTable, network.Transport.Interface, bridge, remote, local))
builder.WriteString(fmt.Sprintf("add rule inet %s forward iifname \"%s\" drop\n", nftTable, bridge))
builder.WriteString(fmt.Sprintf("add rule inet %s forward oifname \"%s\" drop\n", nftTable, bridge))
}
return builder.String()
}
func writeAddressSet(builder *strings.Builder, name string, addresses []string) {
builder.WriteString("add set inet " + nftTable + " " + name + " { type ipv4_addr")
if len(addresses) != 0 {
builder.WriteString("; elements = { ")
builder.WriteString(strings.Join(addresses, ", "))
builder.WriteString(" }")
}
builder.WriteString("; }\n")
}
func (d *linuxDriver) removeBridge(ctx context.Context, vpcKey string) error {
bridge, _ := BridgeName(vpcKey)
ownerMAC, _ := bridgeMAC(vpcKey)
raw, err := d.run.Run(ctx, d.ipPath, []string{"-details", "-j", "link", "show", "dev", bridge}, nil)
if err != nil {
if commandExitCode(err, 1) {
return nil
}
return unavailable("vpc_bridge_inspection_failed", "VPC bridge could not be inspected", err)
}
var links []ipLinkDocument
if json.Unmarshal(raw, &links) != nil || len(links) != 1 || links[0].IfName != bridge {
return unavailable("vpc_bridge_inspection_failed", "VPC bridge ownership could not be inspected", nil)
}
if links[0].LinkInfo == nil || links[0].LinkInfo.InfoKind != "bridge" || !strings.EqualFold(links[0].Address, ownerMAC) {
// The derived name is reserved, but absence of the exact deterministic
// ownership marker means it belongs to the host administrator. Never
// delete it during forward reconciliation or rollback.
return nil
}
if _, err := d.run.Run(ctx, d.ipPath, []string{"link", "delete", "dev", bridge, "type", "bridge"}, nil); err != nil {
return unavailable("vpc_bridge_remove_failed", "VPC bridge could not be removed", err)
}
return nil
}
func orderedNetworks(networks map[string]Network) []Network {
result := make([]Network, 0, len(networks))
for _, network := range networks {
result = append(result, network)
}
sort.Slice(result, func(left, right int) bool { return result[left].VPCKey < result[right].VPCKey })
return result
}
func localAddresses(ports []LocalPort) []string {
result := make([]string, 0, len(ports))
for _, port := range ports {
result = append(result, port.IP)
}
return result
}
func remoteAddresses(routes []PeerRoute) []string {
result := make([]string, 0, len(routes))
for _, route := range routes {
result = append(result, route.IP)
}
return result
}
func routeWithinPrefixes(destination string, prefixes []netip.Prefix) bool {
prefix, err := netip.ParsePrefix(destination)
if err != nil || !prefix.Addr().Is4() {
return false
}
for _, candidate := range prefixes {
if candidate.Contains(prefix.Addr()) {
return true
}
}
return false
}
func desiredRouteDestination(routes map[routeKey]struct{}, destination string) (routeKey, bool) {
for candidate := range routes {
if candidate.destination == destination {
return candidate, true
}
}
return routeKey{}, false
}
func managedProtocol(value any) bool {
switch protocol := value.(type) {
case float64:
return protocol == 186
case string:
return protocol == managedRouteProtocol
default:
return false
}
}
func nftTableOwned(raw []byte) bool {
var document struct {
Objects []map[string]json.RawMessage `json:"nftables"`
}
if json.Unmarshal(raw, &document) != nil {
return false
}
for _, object := range document.Objects {
var chain struct {
Family string `json:"family"`
Table string `json:"table"`
Name string `json:"name"`
}
if encoded, exists := object["chain"]; exists && json.Unmarshal(encoded, &chain) == nil &&
chain.Family == nftFamily && chain.Table == nftTable && chain.Name == "owner_v1" {
return true
}
}
return false
}
func commandExitCode(err error, expected int) bool {
var command *commandError
return errors.As(err, &command) && command.exitCode == expected
}
func boundedText(value string, maximum int) string {
value = strings.TrimSpace(value)
if len(value) <= maximum {
return value
}
return value[:maximum]
}
var _ io.Writer = (*boundedBuffer)(nil)