You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/agent/vpcrouting/manager.go

710 lines
24 KiB
Go

package vpcrouting
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"time"
"git.luoli.live/root/virt/agent/operation"
"golang.org/x/sys/unix"
)
const (
maximumStateBytes = 8 << 20
rollbackTimeout = 30 * time.Second
)
type storedNetwork struct {
Network Network `json:"network"`
Digest string `json:"digest"`
OperationID string `json:"operation_id"`
}
type removalMarker struct {
VPCKey string `json:"vpc_key"`
PreviousRevision uint64 `json:"previous_revision"`
PreviousDigest string `json:"previous_digest"`
OperationID string `json:"operation_id"`
}
type stateDocument struct {
Version int `json:"version"`
Networks []storedNetwork `json:"networks"`
Removed []removalMarker `json:"removed"`
}
type hostDriver interface {
Available() error
Reconcile(context.Context, map[string]Network, map[string]Network) error
}
// projectionAvailability lets a platform backend validate read-only host
// dependencies against the exact projection already committed by this
// manager. In particular, a Linux backend can distinguish its own firewalld
// interface bindings from a foreign use of the reserved VPC objects without
// mutating the firewall during a capability probe.
type projectionAvailability interface {
AvailableFor(map[string]Network) error
}
type Manager struct {
statePath string
driver hostDriver
gate sync.RWMutex
}
// PortLease pins the exact committed projection while a VM seed and domain
// definition are changed. Multiple unrelated VM ports may hold read leases at
// once; Apply/Remove require the writer side and therefore cannot invalidate
// their authorization midway through a cold restart.
type PortLease struct {
manager *Manager
file *os.File
once sync.Once
}
func (lease *PortLease) Release() {
if lease == nil {
return
}
lease.once.Do(func() {
if lease.file != nil {
_ = unix.Flock(int(lease.file.Fd()), unix.LOCK_UN)
_ = lease.file.Close()
}
if lease.manager != nil {
lease.manager.gate.RUnlock()
}
})
}
func Open(statePath string) (*Manager, error) {
if err := validateStatePath(statePath); err != nil {
return nil, err
}
return &Manager{statePath: statePath, driver: newPlatformDriver()}, nil
}
func (m *Manager) Available() error {
if m == nil || m.driver == nil {
return unavailable("vpc_router_unavailable", "VPC host router is unavailable", nil)
}
if driver, ok := m.driver.(projectionAvailability); ok {
document, err := m.load()
if err != nil {
return err
}
return driver.AvailableFor(document.networkMap())
}
return m.driver.Available()
}
// mutationAvailable checks only immutable execution prerequisites. A
// reconcile must still be allowed to repair missing runtime firewalld objects
// after a daemon or host restart; the stricter projection-aware check belongs
// to read-only capability reporting, not the recovery entrance itself.
func (m *Manager) mutationAvailable() error {
if m == nil || m.driver == nil {
return unavailable("vpc_router_unavailable", "VPC host router is unavailable", nil)
}
return m.driver.Available()
}
// Restore reasserts the last durably committed full projection after a host
// reboot. It never invents desired state and performs no work when the store
// is empty, preserving ordinary non-VPC Agent startup.
func (m *Manager) Restore(ctx context.Context) error {
return m.withLock(func() error {
document, err := m.load()
if err != nil {
return err
}
desired := document.networkMap()
if len(desired) == 0 {
return nil
}
if err := m.mutationAvailable(); err != nil {
return err
}
if err := m.driver.Reconcile(ctx, desired, desired); err != nil {
return operation.MarkUncertain(&publicError{
code: "vpc_restore_unconfirmed",
message: "durable VPC host routing could not be restored",
cause: err,
})
}
return nil
})
}
func (m *Manager) Apply(ctx context.Context, request ApplyRequest) (Result, error) {
normalized, err := ValidateApplyRequest(request)
if err != nil {
return Result{}, err
}
if err := m.mutationAvailable(); err != nil {
return Result{}, err
}
var result Result
err = m.withLock(func() error {
document, loadErr := m.load()
if loadErr != nil {
return loadErr
}
before := document.networkMap()
if normalized.VPCRepair {
return m.repairApplyLocked(
ctx, document, before, normalized, &result,
)
}
if existing, exists := document.network(normalized.Network.VPCKey); exists {
switch {
case existing.Network.DesiredRevision == normalized.Network.DesiredRevision && existing.Digest == normalized.DesiredDigest:
if existing.OperationID != normalized.OperationID {
document.putNetwork(storedNetwork{
Network: existing.Network, Digest: existing.Digest,
OperationID: normalized.OperationID,
})
if err := m.save(document); err != nil {
if operation.IsUncertain(err) {
return err
}
return &notAppliedError{error: &publicError{
code: "vpc_marker_not_persisted",
message: "VPC routing already matches but the operation marker could not be persisted",
cause: err,
}}
}
}
bridge, _ := BridgeName(existing.Network.VPCKey)
result = Result{VPCKey: existing.Network.VPCKey, DesiredRevision: existing.Network.DesiredRevision, DesiredDigest: existing.Digest, Bridge: bridge, Applied: true}
return nil
case normalized.Network.DesiredRevision <= existing.Network.DesiredRevision:
return invalid("stale_vpc_revision", "VPC desired revision is stale or conflicts with the applied state")
}
}
after := cloneNetworkMap(before)
after[normalized.Network.VPCKey] = normalized.Network
if err := validateUniqueBridgeNames(after); err != nil {
return unavailable("vpc_bridge_collision", "VPC bridge identity could not be allocated", err)
}
if err := m.driver.Reconcile(ctx, before, after); err != nil {
return m.rollback(before, after, err)
}
document.putNetwork(storedNetwork{
Network: normalized.Network, Digest: normalized.DesiredDigest,
OperationID: normalized.OperationID,
})
if err := m.save(document); err != nil {
if operation.IsUncertain(err) {
return err
}
return m.rollback(before, after, err)
}
bridge, _ := BridgeName(normalized.Network.VPCKey)
result = Result{
VPCKey: normalized.Network.VPCKey, DesiredRevision: normalized.Network.DesiredRevision,
DesiredDigest: normalized.DesiredDigest, Bridge: bridge, Applied: true,
}
return nil
})
return result, err
}
func (m *Manager) Remove(ctx context.Context, request RemoveRequest) (Result, error) {
if err := ValidateRemoveRequest(request); err != nil {
return Result{}, err
}
if err := m.mutationAvailable(); err != nil {
return Result{}, err
}
var result Result
err := m.withLock(func() error {
document, loadErr := m.load()
if loadErr != nil {
return loadErr
}
existing, exists := document.network(request.VPCKey)
if request.VPCRepair {
return m.repairRemoveLocked(
ctx, document, existing, exists, request, &result,
)
}
if !exists || existing.Network.DesiredRevision != request.ExpectedRevision || existing.Digest != request.ExpectedDigest {
return invalid("vpc_remove_precondition_failed", "VPC removal no longer matches the applied host state")
}
before := document.networkMap()
after := cloneNetworkMap(before)
delete(after, request.VPCKey)
if err := m.driver.Reconcile(ctx, before, after); err != nil {
return m.rollback(before, after, err)
}
document.removeNetwork(removalMarker{
VPCKey: request.VPCKey, PreviousRevision: request.ExpectedRevision,
PreviousDigest: request.ExpectedDigest, OperationID: request.OperationID,
})
if err := m.save(document); err != nil {
if operation.IsUncertain(err) {
return err
}
return m.rollback(before, after, err)
}
result = Result{
VPCKey: request.VPCKey, DesiredRevision: request.ExpectedRevision,
DesiredDigest: request.ExpectedDigest, Removed: true,
}
return nil
})
return result, err
}
// repairApplyLocked deliberately re-runs the complete host projection even
// when the durable document already matches. It is reachable only through an
// Agent-authorized successor bound to one exact fenced predecessor; ordinary
// retries never enter this path.
func (m *Manager) repairApplyLocked(ctx context.Context, document stateDocument,
before map[string]Network, request ApplyRequest, result *Result) error {
if existing, exists := document.network(request.Network.VPCKey); exists &&
(existing.Network.DesiredRevision != request.Network.DesiredRevision ||
existing.Digest != request.DesiredDigest) {
return invalid("vpc_repair_projection_changed", "VPC repair no longer matches the durable host projection")
}
after := cloneNetworkMap(before)
after[request.Network.VPCKey] = request.Network
if err := validateUniqueBridgeNames(after); err != nil {
return unavailable("vpc_bridge_collision", "VPC bridge identity could not be allocated", err)
}
if err := m.driver.Reconcile(ctx, before, after); err != nil {
return m.rollback(before, after, err)
}
document.putNetwork(storedNetwork{
Network: request.Network, Digest: request.DesiredDigest,
OperationID: request.OperationID,
})
if err := m.save(document); err != nil {
if operation.IsUncertain(err) {
return err
}
return m.rollback(before, after, err)
}
bridge, _ := BridgeName(request.Network.VPCKey)
*result = Result{
VPCKey: request.Network.VPCKey,
DesiredRevision: request.Network.DesiredRevision,
DesiredDigest: request.DesiredDigest,
Bridge: bridge, Applied: true,
}
return nil
}
// repairRemoveLocked converges the host to an absent projection even when a
// prior partial removal already deleted the durable entry. Conflicting newer
// projections remain fail-closed.
func (m *Manager) repairRemoveLocked(ctx context.Context, document stateDocument,
existing storedNetwork, exists bool, request RemoveRequest,
result *Result) error {
if exists && (existing.Network.DesiredRevision != request.ExpectedRevision ||
existing.Digest != request.ExpectedDigest) {
return invalid("vpc_repair_projection_changed", "VPC repair no longer matches the durable host projection")
}
before := document.networkMap()
after := cloneNetworkMap(before)
delete(after, request.VPCKey)
if err := m.driver.Reconcile(ctx, before, after); err != nil {
return m.rollback(before, after, err)
}
document.removeNetwork(removalMarker{
VPCKey: request.VPCKey,
PreviousRevision: request.ExpectedRevision,
PreviousDigest: request.ExpectedDigest,
OperationID: request.OperationID,
})
if err := m.save(document); err != nil {
if operation.IsUncertain(err) {
return err
}
return m.rollback(before, after, err)
}
*result = Result{
VPCKey: request.VPCKey,
DesiredRevision: request.ExpectedRevision,
DesiredDigest: request.ExpectedDigest,
Removed: true,
}
return nil
}
func (m *Manager) Inspect(operationID, vpcKey string, revision uint64, digest string, present bool) (Inspection, error) {
if m == nil || operationID == "" || !vpcKeyPattern.MatchString(vpcKey) || revision == 0 || !lowerHexDigest.MatchString(digest) {
return Inspection{}, nil
}
var inspection Inspection
err := m.withLock(func() error {
document, loadErr := m.load()
if loadErr != nil {
return loadErr
}
if present {
entry, exists := document.network(vpcKey)
if !exists || entry.OperationID != operationID || entry.Network.DesiredRevision != revision || entry.Digest != digest {
return nil
}
bridge, _ := BridgeName(vpcKey)
inspection = Inspection{Applied: true, Result: Result{
VPCKey: vpcKey, DesiredRevision: revision, DesiredDigest: digest,
Bridge: bridge, Applied: true,
}}
return nil
}
marker, exists := document.removal(vpcKey)
if !exists || marker.OperationID != operationID || marker.PreviousRevision != revision || marker.PreviousDigest != digest {
return nil
}
inspection = Inspection{Removed: true, Result: Result{
VPCKey: vpcKey, DesiredRevision: revision, DesiredDigest: digest, Removed: true,
}}
return nil
})
return inspection, err
}
// AuthorizeLocalPort verifies that a VM port is already part of the exact
// durably committed host projection. Domain attachment is intentionally a
// separate operation: the panel must converge host routing first, then attach
// the NIC, and reverse that order during detach.
func (m *Manager) AuthorizeLocalPort(vpcKey string, revision uint64, digest,
cidr, gateway, ip, mac string) (Network, error) {
authorized, lease, err := m.AcquireLocalPortLease(
vpcKey, revision, digest, cidr, gateway, ip, mac,
)
if lease != nil {
lease.Release()
}
return authorized, err
}
// AcquireLocalPortLease authorizes one exact port and keeps the shared state
// lease until the caller has durably completed or rolled back its VM mutation.
func (m *Manager) AcquireLocalPortLease(vpcKey string, revision uint64, digest,
cidr, gateway, ip, mac string) (Network, *PortLease, error) {
if !vpcKeyPattern.MatchString(vpcKey) || revision == 0 ||
!lowerHexDigest.MatchString(digest) || !canonicalMACPattern.MatchString(mac) {
return Network{}, nil, invalid("invalid_vpc_port_authorization", "VPC port authorization identity is invalid")
}
if m == nil {
return Network{}, nil, unavailable("vpc_router_unavailable", "VPC host router is unavailable", nil)
}
m.gate.RLock()
file, err := m.lockFile(unix.LOCK_SH)
if err != nil {
m.gate.RUnlock()
return Network{}, nil, err
}
lease := &PortLease{manager: m, file: file}
var authorized Network
document, err := m.load()
if err == nil {
entry, exists := document.network(vpcKey)
if !exists || entry.Network.DesiredRevision != revision ||
entry.Digest != digest || entry.Network.CIDR != cidr ||
entry.Network.Gateway != gateway {
err = unavailable("vpc_host_projection_not_ready", "VPC host routing projection is not ready for this port", nil)
} else {
for _, port := range entry.Network.LocalPorts {
if port.IP == ip && port.MAC == mac {
authorized = entry.Network
break
}
}
if authorized.VPCKey == "" {
err = unavailable("vpc_host_projection_not_ready", "VPC port is absent from the committed host routing projection", nil)
}
}
}
if err != nil {
lease.Release()
return Network{}, nil, err
}
return authorized, lease, nil
}
func (m *Manager) rollback(before, after map[string]Network, cause error) error {
rollbackContext, cancel := context.WithTimeout(context.Background(), rollbackTimeout)
defer cancel()
if err := m.driver.Reconcile(rollbackContext, after, before); err != nil {
public := &publicError{
code: "vpc_reconcile_unconfirmed",
message: "VPC host routing failed and the previous state could not be confirmed",
cause: errors.Join(cause, err),
}
return operation.MarkUncertain(public)
}
return rolledBack(cause)
}
func (m *Manager) withLock(call func() error) error {
if m == nil {
return unavailable("vpc_router_unavailable", "VPC host router is unavailable", nil)
}
m.gate.Lock()
defer m.gate.Unlock()
file, err := m.lockFile(unix.LOCK_EX)
if err != nil {
return err
}
defer file.Close()
defer unix.Flock(int(file.Fd()), unix.LOCK_UN) //nolint:errcheck
return call()
}
func (m *Manager) lockFile(mode int) (*os.File, error) {
lockPath := m.statePath + ".lock"
descriptor, err := unix.Open(lockPath, unix.O_CREAT|unix.O_RDWR|unix.O_CLOEXEC|unix.O_NOFOLLOW, 0o600)
if err != nil {
return nil, unavailable("vpc_state_unavailable", "VPC host state lock could not be opened", err)
}
file := os.NewFile(uintptr(descriptor), lockPath)
if file == nil {
_ = unix.Close(descriptor)
return nil, unavailable("vpc_state_unavailable", "VPC host state lock could not be opened", nil)
}
if err := unix.Flock(descriptor, mode); err != nil {
_ = file.Close()
return nil, unavailable("vpc_state_unavailable", "VPC host state lock could not be acquired", err)
}
return file, nil
}
func (m *Manager) load() (stateDocument, error) {
descriptor, err := unix.Open(m.statePath, unix.O_RDONLY|unix.O_CLOEXEC|unix.O_NOFOLLOW, 0)
if errors.Is(err, unix.ENOENT) {
return stateDocument{Version: stateVersion, Networks: []storedNetwork{}, Removed: []removalMarker{}}, nil
}
if err != nil {
return stateDocument{}, unavailable("vpc_state_unavailable", "VPC host state could not be read", err)
}
file := os.NewFile(uintptr(descriptor), m.statePath)
if file == nil {
_ = unix.Close(descriptor)
return stateDocument{}, unavailable("vpc_state_unavailable", "VPC host state could not be opened", nil)
}
defer file.Close()
info, err := file.Stat()
if err != nil || !info.Mode().IsRegular() {
return stateDocument{}, unavailable("vpc_state_invalid", "VPC host state is not a regular file", err)
}
raw, err := io.ReadAll(io.LimitReader(file, maximumStateBytes+1))
if err != nil || len(raw) > maximumStateBytes {
return stateDocument{}, unavailable("vpc_state_invalid", "VPC host state is unreadable or oversized", err)
}
decoder := json.NewDecoder(strings.NewReader(string(raw)))
decoder.DisallowUnknownFields()
var document stateDocument
if err := decoder.Decode(&document); err != nil {
return stateDocument{}, unavailable("vpc_state_invalid", "VPC host state is invalid", err)
}
var extra any
if err := decoder.Decode(&extra); !errors.Is(err, io.EOF) {
return stateDocument{}, unavailable("vpc_state_invalid", "VPC host state contains trailing data", err)
}
if err := document.validate(); err != nil {
return stateDocument{}, unavailable("vpc_state_invalid", "VPC host state is inconsistent", err)
}
return document, nil
}
func (m *Manager) save(document stateDocument) error {
if err := document.validate(); err != nil {
return unavailable("vpc_state_invalid", "VPC host state could not be validated", err)
}
document.sort()
encoded, err := json.Marshal(document)
if err != nil || len(encoded) > maximumStateBytes {
return unavailable("vpc_state_invalid", "VPC host state could not be encoded", err)
}
temporary, err := os.CreateTemp(filepath.Dir(m.statePath), ".virt-vpc-state-*")
if err != nil {
return unavailable("vpc_state_unavailable", "VPC host state could not be persisted", err)
}
temporaryPath := temporary.Name()
keep := false
defer func() {
_ = temporary.Close()
if !keep {
_ = os.Remove(temporaryPath)
}
}()
if err := temporary.Chmod(0o600); err != nil {
return unavailable("vpc_state_unavailable", "VPC host state could not be secured", err)
}
if _, err := temporary.Write(encoded); err != nil {
return unavailable("vpc_state_unavailable", "VPC host state could not be persisted", err)
}
if err := temporary.Sync(); err != nil {
return unavailable("vpc_state_unavailable", "VPC host state could not be synchronized", err)
}
if err := temporary.Close(); err != nil {
return unavailable("vpc_state_unavailable", "VPC host state could not be finalized", err)
}
if err := os.Rename(temporaryPath, m.statePath); err != nil {
return unavailable("vpc_state_unavailable", "VPC host state could not be installed", err)
}
keep = true
directory, err := os.Open(filepath.Dir(m.statePath))
if err != nil {
return operation.MarkUncertain(unavailable("vpc_state_commit_unconfirmed", "VPC host state was installed but directory durability could not be confirmed", err))
}
defer directory.Close()
if err := directory.Sync(); err != nil {
return operation.MarkUncertain(unavailable("vpc_state_commit_unconfirmed", "VPC host state was installed but directory durability could not be confirmed", err))
}
return nil
}
func validateStatePath(path string) error {
if path == "" || !filepath.IsAbs(path) || filepath.Clean(path) != path || path == string(filepath.Separator) || strings.ContainsRune(path, 0) {
return unavailable("vpc_state_path_invalid", "VPC host state path is invalid", nil)
}
directory := filepath.Dir(path)
info, err := os.Lstat(directory)
if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 {
return unavailable("vpc_state_path_invalid", "VPC host state directory is unavailable", err)
}
if info, err := os.Lstat(path); err == nil && (info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular()) {
return unavailable("vpc_state_path_invalid", "VPC host state file is not a regular file", nil)
} else if err != nil && !errors.Is(err, os.ErrNotExist) {
return unavailable("vpc_state_path_invalid", "VPC host state file could not be inspected", err)
}
return nil
}
func (document stateDocument) validate() error {
if document.Version != stateVersion || document.Networks == nil || document.Removed == nil {
return errors.New("invalid state document header")
}
networkKeys := make(map[string]struct{}, len(document.Networks))
for _, entry := range document.Networks {
normalized, err := Normalize(entry.Network)
if err != nil || !lowerHexDigest.MatchString(entry.Digest) || !operationIDPattern.MatchString(entry.OperationID) {
return errors.New("invalid stored network")
}
digest, err := Digest(normalized)
if err != nil || digest != entry.Digest || normalized.VPCKey != entry.Network.VPCKey {
return errors.New("stored network digest mismatch")
}
if _, exists := networkKeys[entry.Network.VPCKey]; exists {
return errors.New("duplicate stored network")
}
networkKeys[entry.Network.VPCKey] = struct{}{}
}
removedKeys := make(map[string]struct{}, len(document.Removed))
for _, marker := range document.Removed {
if !vpcKeyPattern.MatchString(marker.VPCKey) || marker.PreviousRevision == 0 || !lowerHexDigest.MatchString(marker.PreviousDigest) || !operationIDPattern.MatchString(marker.OperationID) {
return errors.New("invalid removal marker")
}
if _, active := networkKeys[marker.VPCKey]; active {
return errors.New("active network has a removal marker")
}
if _, exists := removedKeys[marker.VPCKey]; exists {
return errors.New("duplicate removal marker")
}
removedKeys[marker.VPCKey] = struct{}{}
}
return validateUniqueBridgeNames(document.networkMap())
}
func (document *stateDocument) sort() {
sort.Slice(document.Networks, func(left, right int) bool {
return document.Networks[left].Network.VPCKey < document.Networks[right].Network.VPCKey
})
sort.Slice(document.Removed, func(left, right int) bool {
return document.Removed[left].VPCKey < document.Removed[right].VPCKey
})
}
func (document stateDocument) network(vpcKey string) (storedNetwork, bool) {
for _, entry := range document.Networks {
if entry.Network.VPCKey == vpcKey {
return entry, true
}
}
return storedNetwork{}, false
}
func (document stateDocument) removal(vpcKey string) (removalMarker, bool) {
for _, marker := range document.Removed {
if marker.VPCKey == vpcKey {
return marker, true
}
}
return removalMarker{}, false
}
func (document stateDocument) networkMap() map[string]Network {
result := make(map[string]Network, len(document.Networks))
for _, entry := range document.Networks {
result[entry.Network.VPCKey] = entry.Network
}
return result
}
func (document *stateDocument) putNetwork(entry storedNetwork) {
filtered := document.Networks[:0]
for _, candidate := range document.Networks {
if candidate.Network.VPCKey != entry.Network.VPCKey {
filtered = append(filtered, candidate)
}
}
document.Networks = append(filtered, entry)
removed := document.Removed[:0]
for _, marker := range document.Removed {
if marker.VPCKey != entry.Network.VPCKey {
removed = append(removed, marker)
}
}
document.Removed = removed
}
func (document *stateDocument) removeNetwork(marker removalMarker) {
filtered := document.Networks[:0]
for _, candidate := range document.Networks {
if candidate.Network.VPCKey != marker.VPCKey {
filtered = append(filtered, candidate)
}
}
document.Networks = filtered
removed := document.Removed[:0]
for _, candidate := range document.Removed {
if candidate.VPCKey != marker.VPCKey {
removed = append(removed, candidate)
}
}
document.Removed = append(removed, marker)
}
func cloneNetworkMap(source map[string]Network) map[string]Network {
result := make(map[string]Network, len(source))
for key, network := range source {
network.LocalPorts = append([]LocalPort(nil), network.LocalPorts...)
network.PeerRoutes = append([]PeerRoute(nil), network.PeerRoutes...)
result[key] = network
}
return result
}
func (m *Manager) String() string {
if m == nil {
return "vpcrouting.Manager(nil)"
}
return fmt.Sprintf("vpcrouting.Manager(%s)", filepath.Base(m.statePath))
}