//go:build linux package vpcrouting import ( "bytes" "context" "encoding/json" "errors" "fmt" "io" "net/netip" "os" "os/exec" "path/filepath" "sort" "strconv" "strings" "syscall" "time" ) const ( managedRouteProtocol = "186" maximumCommandOutput = 2 << 20 nftFamily = "inet" nftTable = "virt_cloud_vpc" // A readiness probe performs bounded permanent and runtime firewalld // snapshots. Keep this below the panel's 20-second capability request // deadline while allowing ordinary D-Bus serialization on loaded hosts. availabilityTimeout = 15 * time.Second ) type linuxDriver struct { ipPath string nftPath string firewallCmdPath string run commandRunner } type commandRunner interface { Run(context.Context, string, []string, []byte) ([]byte, error) } type execRunner struct{} type commandError struct { path string exitCode int cause error output string } func (e *commandError) Error() string { return fmt.Sprintf("%s exited with code %d", filepath.Base(e.path), e.exitCode) } func (e *commandError) Unwrap() error { return e.cause } type boundedBuffer struct { buffer bytes.Buffer limit int } func (b *boundedBuffer) Write(value []byte) (int, error) { if b.buffer.Len()+len(value) > b.limit { remaining := b.limit - b.buffer.Len() if remaining > 0 { _, _ = b.buffer.Write(value[:remaining]) } return len(value), errors.New("command output exceeded limit") } return b.buffer.Write(value) } func (execRunner) Run(ctx context.Context, path string, arguments []string, input []byte) ([]byte, error) { command := exec.CommandContext(ctx, path, arguments...) command.Env = []string{"LC_ALL=C"} if input != nil { command.Stdin = bytes.NewReader(input) } output := &boundedBuffer{limit: maximumCommandOutput} command.Stdout = output command.Stderr = output err := command.Run() if err == nil { return append([]byte(nil), output.buffer.Bytes()...), nil } exitCode := -1 var exit *exec.ExitError if errors.As(err, &exit) { exitCode = exit.ExitCode() } return nil, &commandError{ path: path, exitCode: exitCode, cause: err, output: boundedText(output.buffer.String(), 1024), } } func newPlatformDriver() hostDriver { return &linuxDriver{ ipPath: trustedBinary([]string{"/usr/sbin/ip", "/usr/bin/ip", "/sbin/ip", "/bin/ip"}), nftPath: trustedBinary([]string{"/usr/sbin/nft", "/usr/bin/nft", "/sbin/nft", "/bin/nft"}), firewallCmdPath: trustedBinary([]string{"/usr/bin/firewall-cmd", "/usr/sbin/firewall-cmd"}), run: execRunner{}, } } func (d *linuxDriver) Available() error { return d.availablePrerequisites() } // AvailableFor is intentionally read-only. The first mutation may create the // reserved firewalld objects, but capability reporting must already fail if a // running firewalld daemon cannot be inspected or if its existing reserved // objects conflict with the exact projection committed by the Agent. func (d *linuxDriver) AvailableFor(current map[string]Network) error { if err := d.availablePrerequisites(); err != nil { return err } ctx, cancel := context.WithTimeout(context.Background(), availabilityTimeout) defer cancel() return d.firewalldReadiness(ctx, current) } func (d *linuxDriver) availablePrerequisites() error { if d == nil || d.run == nil { return unavailable("vpc_l3_apply_unavailable", "VPC L3 routing backend is unavailable", nil) } if d.ipPath == "" || d.nftPath == "" { return unavailable("vpc_l3_tools_unavailable", "trusted ip and nft executables are required for VPC L3 routing", nil) } forwarding, err := os.ReadFile("/proc/sys/net/ipv4/ip_forward") if err != nil || strings.TrimSpace(string(forwarding)) != "1" { return unavailable("ipv4_forwarding_disabled", "IPv4 forwarding must be enabled for VPC L3 routing", err) } return nil } func trustedBinary(candidates []string) string { for _, path := range candidates { info, err := os.Lstat(path) if err != nil || !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 || info.Mode()&0o022 != 0 || info.Mode()&0o111 == 0 { continue } stat, ok := info.Sys().(*syscall.Stat_t) if !ok || stat.Uid != 0 { continue } return path } return "" } func (d *linuxDriver) Reconcile(ctx context.Context, before, after map[string]Network) error { if err := d.Available(); err != nil { return err } if err := ctx.Err(); err != nil { return err } firewalld, err := d.prepareFirewalld(ctx, before, after) if err != nil { return err } // Close every removed firewalld path before changing the authoritative // routes or nftables projection. Additions are deliberately deferred until // after the exact nftables membership rules have been installed. if err := d.restrictFirewalld(ctx, firewalld, before, after); err != nil { return err } orderedAfter := orderedNetworks(after) for _, network := range orderedAfter { if err := d.verifyTransport(ctx, network.Transport); err != nil { return err } if err := d.ensureBridge(ctx, network); err != nil { return err } } if err := d.reconcileRoutes(ctx, before, after); err != nil { return err } if err := d.reconcileFirewall(ctx, after); err != nil { return err } // The dedicated nftables table is the authoritative tenant boundary. // firewalld is reconciled only after that table is proven installed so a // zone/policy ACCEPT can never temporarily expose an unvalidated path. if err := d.applyFirewalld(ctx, firewalld, before, after); err != nil { return err } for _, network := range orderedNetworks(before) { if _, retained := after[network.VPCKey]; retained { continue } if err := d.removeBridge(ctx, network.VPCKey); err != nil { return err } } return nil } type ipAddressDocument struct { IfName string `json:"ifname"` OperState string `json:"operstate"` MTU int `json:"mtu"` Addresses []struct { Family string `json:"family"` Local string `json:"local"` PrefixLen int `json:"prefixlen"` } `json:"addr_info"` } func (d *linuxDriver) verifyTransport(ctx context.Context, transport Transport) error { raw, err := d.run.Run(ctx, d.ipPath, []string{"-j", "-4", "address", "show", "dev", transport.Interface}, nil) if err != nil { return unavailable("vpc_transport_unavailable", "configured VPC transport interface is unavailable", err) } var documents []ipAddressDocument if err := json.Unmarshal(raw, &documents); err != nil || len(documents) != 1 || documents[0].IfName != transport.Interface || strings.EqualFold(documents[0].OperState, "DOWN") || documents[0].MTU != transport.MTU { return unavailable("vpc_transport_mismatch", "configured VPC transport no longer matches the host", err) } prefix, _ := netip.ParsePrefix(transport.AddressCIDR) for _, address := range documents[0].Addresses { if address.Family == "inet" && address.Local == prefix.Addr().String() && address.PrefixLen == prefix.Bits() { return nil } } return unavailable("vpc_transport_mismatch", "configured VPC transport address is no longer present", nil) } type ipLinkDocument struct { IfName string `json:"ifname"` Address string `json:"address"` MTU int `json:"mtu"` OperState string `json:"operstate"` LinkInfo *struct { InfoKind string `json:"info_kind"` } `json:"linkinfo"` } func (d *linuxDriver) ensureBridge(ctx context.Context, network Network) error { bridge, _ := BridgeName(network.VPCKey) ownerMAC, _ := bridgeMAC(network.VPCKey) raw, err := d.run.Run(ctx, d.ipPath, []string{"-details", "-j", "link", "show", "dev", bridge}, nil) if err != nil { if !commandExitCode(err, 1) { return unavailable("vpc_bridge_inspection_failed", "VPC bridge could not be inspected", err) } if _, err := d.run.Run(ctx, d.ipPath, []string{"link", "add", "name", bridge, "address", ownerMAC, "type", "bridge"}, nil); err != nil { return unavailable("vpc_bridge_create_failed", "VPC bridge could not be created", err) } } else { var links []ipLinkDocument if json.Unmarshal(raw, &links) != nil || len(links) != 1 || links[0].IfName != bridge || !strings.EqualFold(links[0].Address, ownerMAC) || links[0].LinkInfo == nil || links[0].LinkInfo.InfoKind != "bridge" { return unavailable("vpc_bridge_conflict", "VPC bridge name is already used by another host interface", nil) } } if _, err := d.run.Run(ctx, d.ipPath, []string{"link", "set", "dev", bridge, "mtu", strconv.Itoa(network.Transport.MTU), "up"}, nil); err != nil { return unavailable("vpc_bridge_configure_failed", "VPC bridge could not be activated", err) } prefix, _ := netip.ParsePrefix(network.CIDR) gatewayCIDR := network.Gateway + "/" + strconv.Itoa(prefix.Bits()) if _, err := d.run.Run(ctx, d.ipPath, []string{"-4", "address", "replace", gatewayCIDR, "dev", bridge}, nil); err != nil { return unavailable("vpc_bridge_address_failed", "VPC bridge gateway could not be configured", err) } proxyARP := filepath.Join("/proc/sys/net/ipv4/conf", bridge, "proxy_arp") if err := os.WriteFile(proxyARP, []byte("1\n"), 0o644); err != nil { return unavailable("vpc_proxy_arp_failed", "VPC bridge proxy ARP could not be enabled", err) } return nil } type routeDocument struct { Destination string `json:"dst"` Gateway string `json:"gateway"` Device string `json:"dev"` Protocol any `json:"protocol"` } type routeKey struct { destination string gateway string device string } func (d *linuxDriver) reconcileRoutes(ctx context.Context, before, after map[string]Network) error { raw, err := d.run.Run(ctx, d.ipPath, []string{"-j", "-4", "route", "show"}, nil) if err != nil { return unavailable("vpc_route_inspection_failed", "managed VPC routes could not be inspected", err) } var routes []routeDocument if err := json.Unmarshal(raw, &routes); err != nil { return unavailable("vpc_route_inspection_failed", "managed VPC route output is invalid", err) } desired := make(map[routeKey]struct{}) for _, network := range after { for _, route := range network.PeerRoutes { desired[routeKey{destination: route.IP + "/32", gateway: route.Via, device: network.Transport.Interface}] = struct{}{} } } for _, route := range routes { if _, target := desired[routeKey{destination: route.Destination, gateway: route.Gateway, device: route.Device}]; target && managedProtocol(route.Protocol) { continue } if _, destinationWanted := desiredRouteDestination(desired, route.Destination); destinationWanted && !managedProtocol(route.Protocol) { return unavailable("vpc_route_conflict", "a VPC destination is already owned by an unmanaged host route", nil) } } relevant := make([]netip.Prefix, 0, len(before)+len(after)) for _, collection := range []map[string]Network{before, after} { for _, network := range collection { prefix, _ := netip.ParsePrefix(network.CIDR) relevant = append(relevant, prefix) } } for _, route := range routes { key := routeKey{destination: route.Destination, gateway: route.Gateway, device: route.Device} if _, keep := desired[key]; keep || !managedProtocol(route.Protocol) || !routeWithinPrefixes(route.Destination, relevant) { continue } arguments := []string{"-4", "route", "del", route.Destination} if route.Gateway != "" { arguments = append(arguments, "via", route.Gateway) } if route.Device != "" { arguments = append(arguments, "dev", route.Device) } arguments = append(arguments, "protocol", managedRouteProtocol) if _, err := d.run.Run(ctx, d.ipPath, arguments, nil); err != nil { return unavailable("vpc_route_remove_failed", "a stale VPC route could not be removed", err) } } keys := make([]routeKey, 0, len(desired)) for key := range desired { keys = append(keys, key) } sort.Slice(keys, func(left, right int) bool { if keys[left].destination != keys[right].destination { return keys[left].destination < keys[right].destination } if keys[left].gateway != keys[right].gateway { return keys[left].gateway < keys[right].gateway } return keys[left].device < keys[right].device }) for _, key := range keys { if _, err := d.run.Run(ctx, d.ipPath, []string{ "-4", "route", "replace", key.destination, "via", key.gateway, "dev", key.device, "protocol", managedRouteProtocol, }, nil); err != nil { return unavailable("vpc_route_apply_failed", "a VPC peer route could not be applied", err) } } return nil } func (d *linuxDriver) reconcileFirewall(ctx context.Context, networks map[string]Network) error { exists := true raw, err := d.run.Run(ctx, d.nftPath, []string{"-j", "list", "table", nftFamily, nftTable}, nil) if err != nil { if commandExitCode(err, 1) { exists = false } else { return unavailable("vpc_firewall_inspection_failed", "VPC isolation firewall could not be inspected", err) } } else if !nftTableOwned(raw) { return unavailable("vpc_firewall_conflict", "the reserved VPC nftables table is owned by another configuration", nil) } if len(networks) == 0 { if !exists { return nil } if _, err := d.run.Run(ctx, d.nftPath, []string{"delete", "table", nftFamily, nftTable}, nil); err != nil { return unavailable("vpc_firewall_remove_failed", "VPC isolation firewall could not be removed", err) } return nil } script := nftScript(networks, exists) if _, err := d.run.Run(ctx, d.nftPath, []string{"-f", "-"}, []byte(script)); err != nil { return unavailable("vpc_firewall_apply_failed", "VPC isolation firewall could not be applied", err) } return nil } func nftScript(networks map[string]Network, exists bool) string { var builder strings.Builder if exists { builder.WriteString("flush table inet " + nftTable + "\n") } else { builder.WriteString("add table inet " + nftTable + "\n") } ordered := orderedNetworks(networks) for _, network := range ordered { short, _ := nftObjectSuffix(network.VPCKey) writeAddressSet(&builder, "local_"+short, localAddresses(network.LocalPorts)) writeAddressSet(&builder, "remote_"+short, remoteAddresses(network.PeerRoutes)) } builder.WriteString("add chain inet " + nftTable + " owner_v1\n") builder.WriteString("add chain inet " + nftTable + " input { type filter hook input priority -10; policy accept; }\n") builder.WriteString("add chain inet " + nftTable + " forward { type filter hook forward priority -10; policy accept; }\n") for _, network := range ordered { short, _ := nftObjectSuffix(network.VPCKey) bridge, _ := BridgeName(network.VPCKey) local := "@local_" + short remote := "@remote_" + short builder.WriteString(fmt.Sprintf("add rule inet %s input iifname \"%s\" drop\n", nftTable, bridge)) builder.WriteString(fmt.Sprintf("add rule inet %s forward iifname \"%s\" oifname \"%s\" ip saddr %s ip daddr %s accept\n", nftTable, bridge, bridge, local, local)) builder.WriteString(fmt.Sprintf("add rule inet %s forward iifname \"%s\" oifname \"%s\" ip saddr %s ip daddr %s accept\n", nftTable, bridge, network.Transport.Interface, local, remote)) builder.WriteString(fmt.Sprintf("add rule inet %s forward iifname \"%s\" oifname \"%s\" ip saddr %s ip daddr %s accept\n", nftTable, network.Transport.Interface, bridge, remote, local)) builder.WriteString(fmt.Sprintf("add rule inet %s forward iifname \"%s\" drop\n", nftTable, bridge)) builder.WriteString(fmt.Sprintf("add rule inet %s forward oifname \"%s\" drop\n", nftTable, bridge)) } return builder.String() } func writeAddressSet(builder *strings.Builder, name string, addresses []string) { builder.WriteString("add set inet " + nftTable + " " + name + " { type ipv4_addr") if len(addresses) != 0 { builder.WriteString("; elements = { ") builder.WriteString(strings.Join(addresses, ", ")) builder.WriteString(" }") } builder.WriteString("; }\n") } func (d *linuxDriver) removeBridge(ctx context.Context, vpcKey string) error { bridge, _ := BridgeName(vpcKey) ownerMAC, _ := bridgeMAC(vpcKey) raw, err := d.run.Run(ctx, d.ipPath, []string{"-details", "-j", "link", "show", "dev", bridge}, nil) if err != nil { if commandExitCode(err, 1) { return nil } return unavailable("vpc_bridge_inspection_failed", "VPC bridge could not be inspected", err) } var links []ipLinkDocument if json.Unmarshal(raw, &links) != nil || len(links) != 1 || links[0].IfName != bridge { return unavailable("vpc_bridge_inspection_failed", "VPC bridge ownership could not be inspected", nil) } if links[0].LinkInfo == nil || links[0].LinkInfo.InfoKind != "bridge" || !strings.EqualFold(links[0].Address, ownerMAC) { // The derived name is reserved, but absence of the exact deterministic // ownership marker means it belongs to the host administrator. Never // delete it during forward reconciliation or rollback. return nil } if _, err := d.run.Run(ctx, d.ipPath, []string{"link", "delete", "dev", bridge, "type", "bridge"}, nil); err != nil { return unavailable("vpc_bridge_remove_failed", "VPC bridge could not be removed", err) } return nil } func orderedNetworks(networks map[string]Network) []Network { result := make([]Network, 0, len(networks)) for _, network := range networks { result = append(result, network) } sort.Slice(result, func(left, right int) bool { return result[left].VPCKey < result[right].VPCKey }) return result } func localAddresses(ports []LocalPort) []string { result := make([]string, 0, len(ports)) for _, port := range ports { result = append(result, port.IP) } return result } func remoteAddresses(routes []PeerRoute) []string { result := make([]string, 0, len(routes)) for _, route := range routes { result = append(result, route.IP) } return result } func routeWithinPrefixes(destination string, prefixes []netip.Prefix) bool { prefix, err := netip.ParsePrefix(destination) if err != nil || !prefix.Addr().Is4() { return false } for _, candidate := range prefixes { if candidate.Contains(prefix.Addr()) { return true } } return false } func desiredRouteDestination(routes map[routeKey]struct{}, destination string) (routeKey, bool) { for candidate := range routes { if candidate.destination == destination { return candidate, true } } return routeKey{}, false } func managedProtocol(value any) bool { switch protocol := value.(type) { case float64: return protocol == 186 case string: return protocol == managedRouteProtocol default: return false } } func nftTableOwned(raw []byte) bool { var document struct { Objects []map[string]json.RawMessage `json:"nftables"` } if json.Unmarshal(raw, &document) != nil { return false } for _, object := range document.Objects { var chain struct { Family string `json:"family"` Table string `json:"table"` Name string `json:"name"` } if encoded, exists := object["chain"]; exists && json.Unmarshal(encoded, &chain) == nil && chain.Family == nftFamily && chain.Table == nftTable && chain.Name == "owner_v1" { return true } } return false } func commandExitCode(err error, expected int) bool { var command *commandError return errors.As(err, &command) && command.exitCode == expected } func boundedText(value string, maximum int) string { value = strings.TrimSpace(value) if len(value) <= maximum { return value } return value[:maximum] } var _ io.Writer = (*boundedBuffer)(nil)