diff --git a/agent/go.mod b/agent/go.mod index 3c40efbc..10d4374f 100644 --- a/agent/go.mod +++ b/agent/go.mod @@ -6,8 +6,10 @@ require ( github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5 github.com/bgrewell/iso-kit v0.0.0-20260806144545-daea9c2909b4 github.com/coder/websocket v1.8.15 + github.com/mdlayher/netlink v1.7.2 + github.com/ti-mo/conntrack v0.6.0 go.etcd.io/bbolt v1.4.0 - golang.org/x/sys v0.30.0 + golang.org/x/sys v0.33.0 libvirt.org/go/libvirt v1.12007.0 libvirt.org/go/libvirtxml v1.12008.0 ) @@ -15,6 +17,13 @@ require ( require ( github.com/fatih/color v1.18.0 // indirect github.com/go-logr/logr v1.4.2 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/josharian/native v1.1.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mdlayher/socket v0.5.1 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/ti-mo/netfilter v0.5.3 // indirect + golang.org/x/net v0.39.0 // indirect + golang.org/x/sync v0.14.0 // indirect ) diff --git a/agent/go.sum b/agent/go.sum index fcf45948..04c2af99 100644 --- a/agent/go.sum +++ b/agent/go.sum @@ -10,23 +10,41 @@ github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA= +github.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mdlayher/netlink v1.7.2 h1:/UtM3ofJap7Vl4QWCPDGXY8d3GIY2UGSDbK+QWmY8/g= +github.com/mdlayher/netlink v1.7.2/go.mod h1:xraEF7uJbxLhc5fpHL4cPe221LI2bdttWlU+ZGLfQSw= +github.com/mdlayher/socket v0.5.1 h1:VZaqt6RkGkt2OE9l3GcC6nZkqD3xKeQLyfleW/uBcos= +github.com/mdlayher/socket v0.5.1/go.mod h1:TjPLHI1UgwEv5J1B5q0zTZq12A/6H7nKmtTanQE37IQ= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/ti-mo/conntrack v0.6.0 h1:laiW2+dzKyS2u0aVr6FeRQs+v7cj4t7q+twolL/ZkjQ= +github.com/ti-mo/conntrack v0.6.0/go.mod h1:4HZrFQQLOSuBzgQNid3H/wYyyp1kfGXUYxueXjIGibo= +github.com/ti-mo/netfilter v0.5.3 h1:ikzduvnaUMwre5bhbNwWOd6bjqLMVb33vv0XXbK0xGQ= +github.com/ti-mo/netfilter v0.5.3/go.mod h1:08SyBCg6hu1qyQk4s3DjjJKNrm3RTb32nm6AzyT972E= +github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8= +github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk= go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk= -golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= -golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= +golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= +golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= +golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= -golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= +golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= libvirt.org/go/libvirt v1.12007.0 h1:y8QrAZQXac5lfOXj3NRwT18uT81bILtSvIZL92EQ/hs= diff --git a/agent/networkobserve/collector.go b/agent/networkobserve/collector.go new file mode 100644 index 00000000..a870d564 --- /dev/null +++ b/agent/networkobserve/collector.go @@ -0,0 +1,524 @@ +package networkobserve + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "net/netip" + "slices" + "sort" + "strings" + "sync" + "sync/atomic" + "time" +) + +const ( + HardMaxObservedVMs = 4_096 + // ti-mo/conntrack and mdlayher/netlink each retain a full dump before this + // package converts it. Keep the kernel-table ceiling materially below the + // contract's per-window aggregation ceiling so a query has a bounded peak. + HardMaxKernelFlows = 262_144 + DefaultRingCapacity = 16_384 +) + +// VMTarget is the complete, public-only inventory required to attribute one +// conntrack tuple to a managed VM. Private/VPC addresses are deliberately not +// accepted by this contract. +type VMTarget struct { + UUID string + IPv4 netip.Addr + IPv6 netip.Addr + Interface InterfaceCounters +} + +type InventoryProvider interface { + NetworkObservationInventory(context.Context) ([]VMTarget, error) +} + +// KernelFlow is the small, library-independent subset of one conntrack entry +// consumed by the collector. It contains tuple identity only in Agent memory; +// no address or port is copied into a Window or persisted by this package. +type KernelFlow struct { + SourceAddress netip.Addr + DestinationAddress netip.Addr + SourcePort uint16 + DestinationPort uint16 + Protocol uint8 + Zone uint16 + TCPState uint8 + SeenReply bool +} + +type KernelSnapshot struct { + Flows []KernelFlow + PacketAccounting bool + Overflow bool +} + +type ConntrackSource interface { + ProbeBackend + Snapshot(context.Context) (KernelSnapshot, error) + Close() error +} + +type CollectorConfig struct { + Limits Limits + MaxVMs int + MaxKernelFlows int + RingCapacity int +} + +func DefaultCollectorConfig() CollectorConfig { + return CollectorConfig{ + Limits: DefaultLimits(), MaxVMs: HardMaxObservedVMs, + MaxKernelFlows: HardMaxKernelFlows, RingCapacity: DefaultRingCapacity, + } +} + +func (config CollectorConfig) validate() error { + if err := config.Limits.Validate(); err != nil { + return err + } + if config.MaxVMs <= 0 || config.MaxVMs > HardMaxObservedVMs || + config.MaxKernelFlows <= 0 || config.MaxKernelFlows > HardMaxKernelFlows || + config.RingCapacity < config.MaxVMs || config.RingCapacity > HardMaxRingCapacity { + return errors.New("network observation collector limits are outside hard bounds") + } + return nil +} + +type flowKey struct { + source, destination netip.Addr + sourcePort uint16 + destinationPort uint16 + zone uint16 + protocol uint8 +} + +type targetIndex struct { + targets []VMTarget + byAddress map[netip.Addr]int + fingerprint string +} + +type Collector struct { + mu sync.Mutex + inventory InventoryProvider + source ConntrackSource + config CollectorConfig + ring *Ring + capability Capability + previous map[flowKey]struct{} + flowBaselineValid bool + previousInterface map[string]InterfaceCounters + mapping string + lastBucket time.Time + samplingGapPending bool + closed atomic.Bool + closeOnce sync.Once + closeErr error + done chan struct{} +} + +func NewCollector(inventory InventoryProvider, source ConntrackSource, generation string, + config CollectorConfig) (*Collector, error) { + if inventory == nil || source == nil { + return nil, errors.New("network observation inventory and conntrack source are required") + } + if err := config.validate(); err != nil { + return nil, err + } + ring, err := NewRing(generation, config.RingCapacity) + if err != nil { + return nil, err + } + capability := ProbeCapability(context.Background(), source) + return &Collector{ + inventory: inventory, source: source, config: config, ring: ring, + capability: capability, previous: make(map[flowKey]struct{}), + previousInterface: make(map[string]InterfaceCounters), + done: make(chan struct{}), + }, nil +} + +func (collector *Collector) Generation() string { return collector.ring.Generation() } + +func (collector *Collector) Capability() Capability { + collector.mu.Lock() + defer collector.mu.Unlock() + return collector.capability +} + +func (collector *Collector) Batch(after uint64, limit int) (Batch, error) { + collector.mu.Lock() + capability := collector.capability + collector.mu.Unlock() + return collector.ring.Batch(capability, after, limit) +} + +// Collect samples one minute without mutating conntrack, firewall policy, +// sysctls, libvirt domains, instance lifecycle, or billing state. +func (collector *Collector) Collect(ctx context.Context, observedAt time.Time) error { + if err := ctx.Err(); err != nil { + return err + } + bucket := observedAt.UTC().Truncate(time.Duration(BucketSeconds) * time.Second) + if bucket.IsZero() { + return errors.New("network observation bucket is invalid") + } + + if collector.closed.Load() { + return errors.New("network observation collector is closed") + } + collector.mu.Lock() + defer collector.mu.Unlock() + if collector.closed.Load() { + return errors.New("network observation collector is closed") + } + if bucket.Equal(collector.lastBucket) { + return nil + } + // Never let wall-clock regression append an older bucket: doing so would + // advance the cursor and overwrite Ring.LatestForVM with stale evidence. + // Remember the discontinuity so the first future bucket cannot present + // multi-minute flow deltas as one complete minute. + if !collector.lastBucket.IsZero() && bucket.Before(collector.lastBucket) { + collector.samplingGapPending = true + return nil + } + + index, err := collector.readInventory(ctx) + if err != nil { + if ctx.Err() == nil { + collector.samplingGapPending = true + } + return err + } + snapshot, snapshotErr := collector.source.Snapshot(ctx) + if err := ctx.Err(); err != nil { + return err + } + if collector.closed.Load() { + return errors.New("network observation collector is closed") + } + + if snapshotErr == nil { + collector.capability = capabilityFromSnapshot(snapshot) + } else if !collector.capability.Available { + // An unavailable collector has no trustworthy address-family contract, + // so it must not manufacture empty complete-looking windows. + return nil + } + + flags := make(map[QualityFlag]struct{}) + firstObservation := collector.lastBucket.IsZero() + samplingGap := collector.samplingGapPending || + (!collector.lastBucket.IsZero() && bucket.After(collector.lastBucket.Add(time.Duration(BucketSeconds)*time.Second))) + mappingChanged := collector.mapping != "" && collector.mapping != index.fingerprint + flowBaselineUsable := collector.flowBaselineValid && !mappingChanged && !samplingGap + if firstObservation || !collector.flowBaselineValid { + flags[QualityColdStart] = struct{}{} + flags[QualityIncomplete] = struct{}{} + } + if samplingGap { + flags[QualitySamplingGap] = struct{}{} + flags[QualityIncomplete] = struct{}{} + } + if mappingChanged { + flags[QualityMappingChanged] = struct{}{} + flags[QualityIncomplete] = struct{}{} + } + if snapshotErr != nil { + flags[QualityConntrackUnavailable] = struct{}{} + flags[QualityIncomplete] = struct{}{} + snapshot.Flows = nil + collector.samplingGapPending = true + } + flowInputComplete := snapshotErr == nil && !snapshot.Overflow + if snapshot.Overflow { + flags[QualityOverflow] = struct{}{} + flags[QualityIncomplete] = struct{}{} + } + if !snapshot.PacketAccounting { + flags[QualityPacketAccountingUnavailable] = struct{}{} + } + if len(snapshot.Flows) > collector.config.MaxKernelFlows { + flags[QualityOverflow] = struct{}{} + flags[QualityIncomplete] = struct{}{} + snapshot.Flows = snapshot.Flows[:collector.config.MaxKernelFlows] + flowInputComplete = false + } + + aggregators := make([]*Aggregator, len(index.targets)) + for i := range aggregators { + aggregators[i], err = NewAggregator(collector.config.Limits) + if err != nil { + return err + } + } + current := make(map[flowKey]struct{}, min(len(snapshot.Flows), collector.config.MaxKernelFlows)) + for _, observed := range snapshot.Flows { + key, ok := canonicalFlowKey(observed) + if !ok { + flags[QualityIncomplete] = struct{}{} + flowInputComplete = false + continue + } + sourceIndex, sourceManaged := index.byAddress[key.source] + destinationIndex, destinationManaged := index.byAddress[key.destination] + if !sourceManaged && !destinationManaged { + continue + } + if len(current) < collector.config.MaxKernelFlows { + current[key] = struct{}{} + } else { + flags[QualityFlowLimitReached] = struct{}{} + flags[QualityIncomplete] = struct{}{} + } + _, existed := collector.previous[key] + isNew := flowBaselineUsable && !existed + protocol := flowProtocol(observed.Protocol) + syn := protocol == ProtocolTCP && (observed.TCPState == 1 || observed.TCPState == 2) + if sourceManaged { + aggregators[sourceIndex].Add(Flow{ + IPv6: key.source.Is6(), Direction: DirectionOutbound, Protocol: protocol, + RemoteIP: key.destination, LocalPort: key.sourcePort, RemotePort: key.destinationPort, + New: isNew, SYN: syn, Unreplied: !observed.SeenReply, + }) + } + if destinationManaged { + aggregators[destinationIndex].Add(Flow{ + IPv6: key.destination.Is6(), Direction: DirectionInbound, Protocol: protocol, + RemoteIP: key.source, LocalPort: key.destinationPort, RemotePort: key.sourcePort, + New: isNew, SYN: syn, Unreplied: !observed.SeenReply, + }) + } + } + + windows := make([]Window, 0, len(index.targets)) + for i, target := range index.targets { + ipv4, ipv6, aggregateFlags := aggregators[i].Result() + windowFlags := cloneFlags(flags) + for _, flag := range aggregateFlags { + windowFlags[flag] = struct{}{} + } + interfaceCounters := cloneInterface(target.Interface) + if interfaceCounters.hasMissingCounter() { + windowFlags[QualityInterfaceUnavailable] = struct{}{} + windowFlags[QualityIncomplete] = struct{}{} + } + if previous, found := collector.previousInterface[target.UUID]; found && + counterRegressed(&previous, &interfaceCounters) { + windowFlags[QualityCounterReset] = struct{}{} + windowFlags[QualityIncomplete] = struct{}{} + } + if collector.capability.IPv4 && ipv4 == nil { + ipv4 = emptyFamilyObservation() + } + if collector.capability.IPv6 && ipv6 == nil { + ipv6 = emptyFamilyObservation() + } + quality := flagSlice(windowFlags) + complete := !slices.Contains(quality, QualityIncomplete) + windows = append(windows, Window{ + VMUUID: target.UUID, BucketStart: bucket, BucketSeconds: BucketSeconds, + Complete: complete, QualityFlags: quality, Interface: &interfaceCounters, + IPv4: ipv4, IPv6: ipv6, + }) + } + if err := collector.ring.Append(windows); err != nil { + return err + } + if flowInputComplete { + collector.previous = current + collector.flowBaselineValid = true + collector.samplingGapPending = false + } else { + collector.flowBaselineValid = false + } + collector.mapping = index.fingerprint + collector.lastBucket = bucket + nextInterfaces := make(map[string]InterfaceCounters, len(index.targets)) + for _, target := range index.targets { + nextInterfaces[target.UUID] = cloneInterface(target.Interface) + } + collector.previousInterface = nextInterfaces + return nil +} + +func (collector *Collector) readInventory(ctx context.Context) (targetIndex, error) { + targets, err := collector.inventory.NetworkObservationInventory(ctx) + if err != nil { + return targetIndex{}, err + } + if len(targets) > collector.config.MaxVMs { + return targetIndex{}, errors.New("network observation inventory exceeds VM limit") + } + sort.Slice(targets, func(i, j int) bool { return targets[i].UUID < targets[j].UUID }) + index := targetIndex{targets: targets, byAddress: make(map[netip.Addr]int, len(targets)*2)} + var identity strings.Builder + for position := range targets { + target := &index.targets[position] + if !uuidPattern.MatchString(target.UUID) { + return targetIndex{}, errors.New("network observation inventory contains invalid VM UUID") + } + if position > 0 && index.targets[position-1].UUID == target.UUID { + return targetIndex{}, errors.New("network observation inventory contains duplicate VM UUID") + } + target.IPv4 = target.IPv4.Unmap() + target.IPv6 = target.IPv6.Unmap() + if !target.IPv4.Is4() || !target.IPv6.Is6() || target.IPv6.Is4In6() || + !isPublicUnicast(target.IPv4) || !isPublicUnicast(target.IPv6) { + return targetIndex{}, errors.New("network observation inventory requires canonical dual-stack public addresses") + } + for _, address := range []netip.Addr{target.IPv4, target.IPv6} { + if _, duplicate := index.byAddress[address]; duplicate { + return targetIndex{}, errors.New("network observation inventory contains duplicate public address") + } + index.byAddress[address] = position + } + identity.WriteString(target.UUID) + identity.WriteByte('\n') + identity.WriteString(target.IPv4.String()) + identity.WriteByte('\n') + identity.WriteString(target.IPv6.String()) + identity.WriteByte('\n') + } + digest := sha256.Sum256([]byte(identity.String())) + index.fingerprint = hex.EncodeToString(digest[:]) + return index, nil +} + +func isPublicUnicast(address netip.Addr) bool { + address = address.Unmap() + return address.IsValid() && address.IsGlobalUnicast() && + !address.IsPrivate() && !address.IsUnspecified() && + !address.IsLoopback() && !address.IsLinkLocalUnicast() && + !address.IsLinkLocalMulticast() && !address.IsMulticast() +} + +func canonicalFlowKey(flow KernelFlow) (flowKey, bool) { + source := flow.SourceAddress.Unmap() + destination := flow.DestinationAddress.Unmap() + if !source.IsValid() || !destination.IsValid() || source.Is6() != destination.Is6() { + return flowKey{}, false + } + return flowKey{source: source, destination: destination, + sourcePort: flow.SourcePort, destinationPort: flow.DestinationPort, + zone: flow.Zone, protocol: flow.Protocol}, true +} + +func flowProtocol(value uint8) Protocol { + switch value { + case 6: + return ProtocolTCP + case 17: + return ProtocolUDP + default: + return ProtocolOther + } +} + +func capabilityFromSnapshot(snapshot KernelSnapshot) Capability { + return Capability{ + Name: CapabilityName, ContractVersion: ContractVersion, Available: true, + NoPayload: true, IPv4: true, IPv6: true, Conntrack: true, + PacketAccounting: snapshot.PacketAccounting, PolicyDecisions: false, + BucketSeconds: BucketSeconds, + } +} + +func emptyFamilyObservation() *FamilyObservation { + return &FamilyObservation{Inbound: &FlowSummary{}, Outbound: &FlowSummary{}} +} + +func cloneFlags(source map[QualityFlag]struct{}) map[QualityFlag]struct{} { + result := make(map[QualityFlag]struct{}, len(source)) + for flag := range source { + result[flag] = struct{}{} + } + return result +} + +func flagSlice(source map[QualityFlag]struct{}) []QualityFlag { + result := make([]QualityFlag, 0, len(source)) + for flag := range source { + result = append(result, flag) + } + result, _ = CanonicalQualityFlags(result) + return result +} + +func cloneInterface(value InterfaceCounters) InterfaceCounters { + return InterfaceCounters{ + RXBytes: cloneCounter(value.RXBytes), TXBytes: cloneCounter(value.TXBytes), + RXPackets: cloneCounter(value.RXPackets), TXPackets: cloneCounter(value.TXPackets), + RXErrors: cloneCounter(value.RXErrors), TXErrors: cloneCounter(value.TXErrors), + RXDrops: cloneCounter(value.RXDrops), TXDrops: cloneCounter(value.TXDrops), + } +} + +func cloneCounter(value *uint64) *uint64 { + if value == nil { + return nil + } + copy := *value + return © +} + +func counterRegressed(previous, current *InterfaceCounters) bool { + if previous == nil || current == nil { + return false + } + left := []*uint64{previous.RXBytes, previous.TXBytes, previous.RXPackets, previous.TXPackets, + previous.RXErrors, previous.TXErrors, previous.RXDrops, previous.TXDrops} + right := []*uint64{current.RXBytes, current.TXBytes, current.RXPackets, current.TXPackets, + current.RXErrors, current.TXErrors, current.RXDrops, current.TXDrops} + for i := range left { + if left[i] != nil && right[i] != nil && *right[i] < *left[i] { + return true + } + } + return false +} + +func (collector *Collector) Run(ctx context.Context) error { + for { + now := time.Now().UTC() + next := now.Truncate(time.Duration(BucketSeconds) * time.Second). + Add(time.Duration(BucketSeconds) * time.Second) + timer := time.NewTimer(time.Until(next)) + select { + case <-ctx.Done(): + timer.Stop() + return ctx.Err() + case <-collector.done: + timer.Stop() + return nil + case <-timer.C: + if err := collector.Collect(ctx, next); err != nil { + if errors.Is(err, context.Canceled) { + return err + } + collector.mu.Lock() + collector.mu.Unlock() + if collector.closed.Load() { + return nil + } + } + } + } +} + +func (collector *Collector) Close() error { + collector.closeOnce.Do(func() { + collector.closed.Store(true) + close(collector.done) + // Do not take collector.mu here. Snapshot may be blocked while Collect + // owns it; the source Close path is what interrupts that blocking read. + collector.closeErr = collector.source.Close() + }) + return collector.closeErr +} diff --git a/agent/networkobserve/collector_test.go b/agent/networkobserve/collector_test.go new file mode 100644 index 00000000..a64df41e --- /dev/null +++ b/agent/networkobserve/collector_test.go @@ -0,0 +1,544 @@ +package networkobserve + +import ( + "context" + "errors" + "net/netip" + "slices" + "sync" + "testing" + "time" +) + +type fakeInventory struct { + mu sync.Mutex + targets []VMTarget + err error +} + +func (inventory *fakeInventory) NetworkObservationInventory(context.Context) ([]VMTarget, error) { + inventory.mu.Lock() + defer inventory.mu.Unlock() + result := make([]VMTarget, len(inventory.targets)) + copy(result, inventory.targets) + return result, inventory.err +} + +func (inventory *fakeInventory) set(targets []VMTarget) { + inventory.mu.Lock() + inventory.targets = targets + inventory.mu.Unlock() +} + +type snapshotResult struct { + snapshot KernelSnapshot + err error +} + +type fakeSource struct { + mu sync.Mutex + evidence ProbeEvidence + probeErr error + snapshots []snapshotResult + closed int +} + +type blockingSource struct { + started chan struct{} + closed chan struct{} + startOnce sync.Once + closeOnce sync.Once +} + +func newBlockingSource() *blockingSource { + return &blockingSource{started: make(chan struct{}), closed: make(chan struct{})} +} + +func (source *blockingSource) Probe(context.Context) (ProbeEvidence, error) { + return ProbeEvidence{Conntrack: true, IPv4: true, IPv6: true}, nil +} + +func (source *blockingSource) Snapshot(ctx context.Context) (KernelSnapshot, error) { + source.startOnce.Do(func() { close(source.started) }) + select { + case <-ctx.Done(): + return KernelSnapshot{}, ctx.Err() + case <-source.closed: + return KernelSnapshot{}, ErrConntrackUnavailable + } +} + +func (source *blockingSource) Close() error { + source.closeOnce.Do(func() { close(source.closed) }) + return nil +} + +func (source *fakeSource) Probe(context.Context) (ProbeEvidence, error) { + return source.evidence, source.probeErr +} + +func (source *fakeSource) Snapshot(context.Context) (KernelSnapshot, error) { + source.mu.Lock() + defer source.mu.Unlock() + if len(source.snapshots) == 0 { + return KernelSnapshot{}, errors.New("no fake snapshot") + } + result := source.snapshots[0] + source.snapshots = source.snapshots[1:] + return result.snapshot, result.err +} + +func (source *fakeSource) Close() error { + source.mu.Lock() + source.closed++ + source.mu.Unlock() + return nil +} + +func observedTarget(rx uint64) VMTarget { + counter := func(value uint64) *uint64 { return &value } + return VMTarget{ + UUID: testVMUUID, IPv4: netip.MustParseAddr("192.0.2.10"), IPv6: netip.MustParseAddr("2001:db8::10"), + Interface: InterfaceCounters{ + RXBytes: counter(rx), TXBytes: counter(200), RXPackets: counter(10), TXPackets: counter(20), + RXErrors: counter(0), TXErrors: counter(0), RXDrops: counter(0), TXDrops: counter(0), + }, + } +} + +func availableFakeSource(results ...snapshotResult) *fakeSource { + return &fakeSource{ + evidence: ProbeEvidence{Conntrack: true, IPv4: true, IPv6: true}, + snapshots: results, + } +} + +func newTestCollector(t *testing.T, inventory *fakeInventory, source *fakeSource, config CollectorConfig) *Collector { + t.Helper() + collector, err := NewCollector(inventory, source, testGeneration, config) + if err != nil { + t.Fatal(err) + } + return collector +} + +func TestCollectorMapsPublicDualStackDirectionsAndNewFlows(t *testing.T) { + outbound := KernelFlow{SourceAddress: netip.MustParseAddr("192.0.2.10"), + DestinationAddress: netip.MustParseAddr("198.51.100.20"), SourcePort: 50000, + DestinationPort: 443, Protocol: 6, TCPState: 1} + inbound := KernelFlow{SourceAddress: netip.MustParseAddr("198.51.100.30"), + DestinationAddress: netip.MustParseAddr("192.0.2.10"), SourcePort: 51000, + DestinationPort: 22, Protocol: 6, TCPState: 3, SeenReply: true} + v6 := KernelFlow{SourceAddress: netip.MustParseAddr("2001:db8::10"), + DestinationAddress: netip.MustParseAddr("2001:db8:1::20"), SourcePort: 53000, + DestinationPort: 53, Protocol: 17} + added := KernelFlow{SourceAddress: netip.MustParseAddr("192.0.2.10"), + DestinationAddress: netip.MustParseAddr("198.51.100.40"), SourcePort: 50001, + DestinationPort: 25, Protocol: 6, TCPState: 1} + source := availableFakeSource( + snapshotResult{snapshot: KernelSnapshot{Flows: []KernelFlow{outbound, inbound, v6}}}, + snapshotResult{snapshot: KernelSnapshot{Flows: []KernelFlow{outbound, inbound, v6, added}}}, + ) + inventory := &fakeInventory{targets: []VMTarget{observedTarget(100)}} + collector := newTestCollector(t, inventory, source, DefaultCollectorConfig()) + start := time.Date(2026, 11, 8, 1, 1, 0, 0, time.UTC) + if err := collector.Collect(context.Background(), start); err != nil { + t.Fatal(err) + } + inventory.set([]VMTarget{observedTarget(150)}) + if err := collector.Collect(context.Background(), start.Add(time.Minute)); err != nil { + t.Fatal(err) + } + batch, err := collector.Batch(1, MaxBatchWindows) + if err != nil { + t.Fatal(err) + } + if len(batch.Windows) != 1 || !batch.Windows[0].Complete { + t.Fatalf("second window = %#v", batch.Windows) + } + window := batch.Windows[0] + if window.IPv4.Outbound.TCPActiveFlows != 2 || window.IPv4.Outbound.TCPNewFlows != 1 || + window.IPv4.Outbound.TCPSYNFlows != 2 || window.IPv4.Outbound.UniqueRemotePorts != 2 || + window.IPv4.Inbound.TCPActiveFlows != 1 || window.IPv4.Inbound.UniqueLocalPorts != 1 || + window.IPv4.Inbound.MaxRemoteIPsPerLocalPort != 1 || + window.IPv6.Outbound.UDPActiveFlows != 1 || window.IPv6.Outbound.UDPNewFlows != 0 { + t.Fatalf("flow summaries = ipv4 %#v ipv6 %#v", window.IPv4, window.IPv6) + } + if *window.Interface.RXBytes != 150 { + t.Fatalf("interface counters = %#v", window.Interface) + } +} + +func TestCollectorIgnoresUnknownAndCountsSameHostVMFlowTwice(t *testing.T) { + second := observedTarget(300) + second.UUID = "123e4567-e89b-42d3-a456-426614174001" + second.IPv4 = netip.MustParseAddr("192.0.2.11") + second.IPv6 = netip.MustParseAddr("2001:db8::11") + managed := KernelFlow{SourceAddress: netip.MustParseAddr("192.0.2.10"), + DestinationAddress: netip.MustParseAddr("192.0.2.11"), SourcePort: 50000, + DestinationPort: 8080, Protocol: 6, SeenReply: true} + unknown := KernelFlow{SourceAddress: netip.MustParseAddr("203.0.113.10"), + DestinationAddress: netip.MustParseAddr("203.0.113.11"), Protocol: 17} + source := availableFakeSource(snapshotResult{snapshot: KernelSnapshot{Flows: []KernelFlow{managed, unknown}}}) + collector := newTestCollector(t, &fakeInventory{targets: []VMTarget{observedTarget(100), second}}, + source, DefaultCollectorConfig()) + if err := collector.Collect(context.Background(), time.Date(2026, 11, 8, 2, 0, 0, 0, time.UTC)); err != nil { + t.Fatal(err) + } + batch, err := collector.Batch(0, MaxBatchWindows) + if err != nil { + t.Fatal(err) + } + if len(batch.Windows) != 2 || batch.Windows[0].IPv4.Outbound.TCPActiveFlows != 1 || + batch.Windows[1].IPv4.Inbound.TCPActiveFlows != 1 { + t.Fatalf("same-host flow projection = %#v", batch.Windows) + } +} + +func TestCollectorMarksColdStartMappingChangeCounterResetAndOverflow(t *testing.T) { + flow := KernelFlow{SourceAddress: netip.MustParseAddr("192.0.2.10"), + DestinationAddress: netip.MustParseAddr("198.51.100.20"), Protocol: 17} + source := availableFakeSource( + snapshotResult{snapshot: KernelSnapshot{Flows: []KernelFlow{flow}}}, + snapshotResult{snapshot: KernelSnapshot{Flows: []KernelFlow{flow, flow}}}, + ) + inventory := &fakeInventory{targets: []VMTarget{observedTarget(100)}} + config := DefaultCollectorConfig() + config.MaxKernelFlows = 1 + collector := newTestCollector(t, inventory, source, config) + start := time.Date(2026, 11, 8, 2, 0, 0, 0, time.UTC) + if err := collector.Collect(context.Background(), start); err != nil { + t.Fatal(err) + } + changed := observedTarget(50) + changed.IPv6 = netip.MustParseAddr("2001:db8::20") + inventory.set([]VMTarget{changed}) + if err := collector.Collect(context.Background(), start.Add(time.Minute)); err != nil { + t.Fatal(err) + } + batch, err := collector.Batch(0, MaxBatchWindows) + if err != nil { + t.Fatal(err) + } + if !slices.Contains(batch.Windows[0].QualityFlags, QualityColdStart) { + t.Fatalf("cold-start flags = %v", batch.Windows[0].QualityFlags) + } + flags := batch.Windows[1].QualityFlags + for _, want := range []QualityFlag{QualityMappingChanged, QualityCounterReset, QualityOverflow, QualityIncomplete} { + if !slices.Contains(flags, want) { + t.Fatalf("flags %v missing %s", flags, want) + } + } +} + +func TestCollectorFailureDoesNotTurnExistingFlowsIntoNew(t *testing.T) { + flow := KernelFlow{SourceAddress: netip.MustParseAddr("192.0.2.10"), + DestinationAddress: netip.MustParseAddr("198.51.100.20"), Protocol: 6} + source := availableFakeSource( + snapshotResult{snapshot: KernelSnapshot{Flows: []KernelFlow{flow}}}, + snapshotResult{err: ErrConntrackUnavailable}, + snapshotResult{snapshot: KernelSnapshot{Flows: []KernelFlow{flow}}}, + snapshotResult{snapshot: KernelSnapshot{Flows: []KernelFlow{flow}}}, + ) + collector := newTestCollector(t, &fakeInventory{targets: []VMTarget{observedTarget(100)}}, + source, DefaultCollectorConfig()) + start := time.Date(2026, 11, 8, 4, 0, 0, 0, time.UTC) + for offset := 0; offset < 4; offset++ { + if err := collector.Collect(context.Background(), start.Add(time.Duration(offset)*time.Minute)); err != nil { + t.Fatal(err) + } + } + batch, err := collector.Batch(0, MaxBatchWindows) + if err != nil { + t.Fatal(err) + } + if len(batch.Windows) != 4 || !slices.Contains(batch.Windows[2].QualityFlags, QualityColdStart) || + batch.Windows[2].IPv4.Outbound.TCPNewFlows != 0 || batch.Windows[3].IPv4.Outbound.TCPNewFlows != 0 { + t.Fatalf("recovery windows = %#v", batch.Windows) + } +} + +func TestCollectorOverflowForcesOneCleanBaselineRebuild(t *testing.T) { + flow := KernelFlow{SourceAddress: netip.MustParseAddr("192.0.2.10"), + DestinationAddress: netip.MustParseAddr("198.51.100.20"), Protocol: 6} + source := availableFakeSource( + snapshotResult{snapshot: KernelSnapshot{Flows: []KernelFlow{flow}}}, + snapshotResult{snapshot: KernelSnapshot{Flows: []KernelFlow{flow}, Overflow: true}}, + snapshotResult{snapshot: KernelSnapshot{Flows: []KernelFlow{flow}}}, + snapshotResult{snapshot: KernelSnapshot{Flows: []KernelFlow{flow}}}, + ) + collector := newTestCollector(t, &fakeInventory{targets: []VMTarget{observedTarget(100)}}, + source, DefaultCollectorConfig()) + start := time.Date(2026, 11, 8, 4, 10, 0, 0, time.UTC) + for offset := 0; offset < 4; offset++ { + if err := collector.Collect(context.Background(), start.Add(time.Duration(offset)*time.Minute)); err != nil { + t.Fatal(err) + } + } + batch, err := collector.Batch(0, MaxBatchWindows) + if err != nil { + t.Fatal(err) + } + if !slices.Contains(batch.Windows[1].QualityFlags, QualityOverflow) || + !slices.Contains(batch.Windows[2].QualityFlags, QualityColdStart) || + batch.Windows[2].IPv4.Outbound.TCPNewFlows != 0 || batch.Windows[3].IPv4.Outbound.TCPNewFlows != 0 { + t.Fatalf("overflow recovery windows = %#v", batch.Windows) + } +} + +func TestCollectorRejectsAmbiguousOrUnboundedInventory(t *testing.T) { + duplicate := observedTarget(100) + duplicate.UUID = "123e4567-e89b-42d3-a456-426614174001" + inventory := &fakeInventory{targets: []VMTarget{observedTarget(100), duplicate}} + collector := newTestCollector(t, inventory, availableFakeSource(), DefaultCollectorConfig()) + if err := collector.Collect(context.Background(), time.Date(2026, 11, 8, 4, 20, 0, 0, time.UTC)); err == nil { + t.Fatal("duplicate public address was accepted") + } + config := DefaultCollectorConfig() + config.MaxVMs = 1 + collector = newTestCollector(t, inventory, availableFakeSource(), config) + if err := collector.Collect(context.Background(), time.Date(2026, 11, 8, 4, 20, 0, 0, time.UTC)); err == nil { + t.Fatal("unbounded VM inventory was accepted") + } +} + +func TestCollectorRejectsPrivateOrNonPublicInventory(t *testing.T) { + tests := []struct { + name string + address string + ipv6 bool + }{ + {name: "private IPv4", address: "10.0.0.10"}, + {name: "loopback IPv4", address: "127.0.0.1"}, + {name: "link-local IPv4", address: "169.254.1.10"}, + {name: "unspecified IPv4", address: "0.0.0.0"}, + {name: "ULA IPv6", address: "fd00::10", ipv6: true}, + {name: "loopback IPv6", address: "::1", ipv6: true}, + {name: "link-local IPv6", address: "fe80::10", ipv6: true}, + {name: "unspecified IPv6", address: "::", ipv6: true}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + target := observedTarget(100) + if test.ipv6 { + target.IPv6 = netip.MustParseAddr(test.address) + } else { + target.IPv4 = netip.MustParseAddr(test.address) + } + collector := newTestCollector(t, &fakeInventory{targets: []VMTarget{target}}, + availableFakeSource(), DefaultCollectorConfig()) + if err := collector.Collect(context.Background(), time.Date(2026, 11, 8, 4, 30, 0, 0, time.UTC)); err == nil { + t.Fatalf("non-public address %s was accepted", test.address) + } + }) + } +} + +func TestCollectorConfigRequiresOneFullMinuteOfRingCapacity(t *testing.T) { + config := DefaultCollectorConfig() + config.RingCapacity = config.MaxVMs - 1 + if _, err := NewCollector(&fakeInventory{}, availableFakeSource(), testGeneration, config); err == nil { + t.Fatal("ring smaller than one maximum inventory minute was accepted") + } +} + +func TestCollectorClockRegressionDoesNotRewindBaselines(t *testing.T) { + flow := KernelFlow{SourceAddress: netip.MustParseAddr("192.0.2.10"), + DestinationAddress: netip.MustParseAddr("198.51.100.20"), Protocol: 6} + added := KernelFlow{SourceAddress: netip.MustParseAddr("192.0.2.10"), + DestinationAddress: netip.MustParseAddr("198.51.100.21"), Protocol: 6} + source := availableFakeSource( + snapshotResult{snapshot: KernelSnapshot{Flows: []KernelFlow{flow}}}, + snapshotResult{snapshot: KernelSnapshot{Flows: []KernelFlow{flow, added}}}, + snapshotResult{snapshot: KernelSnapshot{Flows: []KernelFlow{flow, added}}}, + ) + collector := newTestCollector(t, &fakeInventory{targets: []VMTarget{observedTarget(100)}}, source, DefaultCollectorConfig()) + start := time.Date(2026, 11, 8, 5, 1, 0, 0, time.UTC) + if err := collector.Collect(context.Background(), start); err != nil { + t.Fatal(err) + } + if err := collector.Collect(context.Background(), start.Add(-time.Minute)); err != nil { + t.Fatal(err) + } + if !collector.lastBucket.Equal(start) { + t.Fatalf("clock regression rewound last bucket to %s", collector.lastBucket) + } + before, err := collector.Batch(0, MaxBatchWindows) + if err != nil { + t.Fatal(err) + } + if len(before.Windows) != 1 || before.NextCursor != 1 { + t.Fatalf("clock regression appended stale evidence: %#v", before) + } + if err := collector.Collect(context.Background(), start.Add(time.Minute)); err != nil { + t.Fatal(err) + } + if err := collector.Collect(context.Background(), start.Add(2*time.Minute)); err != nil { + t.Fatal(err) + } + batch, err := collector.Batch(0, MaxBatchWindows) + if err != nil { + t.Fatal(err) + } + if len(batch.Windows) != 3 || batch.Windows[1].Cursor != 2 || + batch.Windows[1].BucketStart.Before(batch.Windows[0].BucketStart) || + batch.Windows[1].Complete || + !slices.Contains(batch.Windows[1].QualityFlags, QualitySamplingGap) || + batch.Windows[1].IPv4.Outbound.TCPNewFlows != 0 || + !batch.Windows[2].Complete || batch.Windows[2].IPv4.Outbound.TCPNewFlows != 0 { + t.Fatalf("clock-regression windows = %#v", batch.Windows) + } +} + +func TestCollectorMarksForwardSamplingGapAndRebuildsBaseline(t *testing.T) { + flow := KernelFlow{SourceAddress: netip.MustParseAddr("192.0.2.10"), + DestinationAddress: netip.MustParseAddr("198.51.100.20"), Protocol: 6} + source := availableFakeSource( + snapshotResult{snapshot: KernelSnapshot{Flows: []KernelFlow{flow}}}, + snapshotResult{snapshot: KernelSnapshot{Flows: []KernelFlow{flow}}}, + snapshotResult{snapshot: KernelSnapshot{Flows: []KernelFlow{flow}}}, + ) + collector := newTestCollector(t, &fakeInventory{targets: []VMTarget{observedTarget(100)}}, source, + DefaultCollectorConfig()) + start := time.Date(2026, 11, 8, 5, 10, 0, 0, time.UTC) + for _, observedAt := range []time.Time{start, start.Add(2 * time.Minute), start.Add(3 * time.Minute)} { + if err := collector.Collect(context.Background(), observedAt); err != nil { + t.Fatal(err) + } + } + batch, err := collector.Batch(0, MaxBatchWindows) + if err != nil { + t.Fatal(err) + } + if len(batch.Windows) != 3 || batch.Windows[1].Complete || + !slices.Contains(batch.Windows[1].QualityFlags, QualitySamplingGap) || + batch.Windows[1].IPv4.Outbound.TCPNewFlows != 0 || + !batch.Windows[2].Complete || batch.Windows[2].IPv4.Outbound.TCPNewFlows != 0 { + t.Fatalf("sampling-gap windows = %#v", batch.Windows) + } +} + +func TestCollectorInventoryFailureMarksNextSuccessAsSamplingGap(t *testing.T) { + flow := KernelFlow{SourceAddress: netip.MustParseAddr("192.0.2.10"), + DestinationAddress: netip.MustParseAddr("198.51.100.20"), Protocol: 6} + inventory := &fakeInventory{targets: []VMTarget{observedTarget(100)}} + source := availableFakeSource( + snapshotResult{snapshot: KernelSnapshot{Flows: []KernelFlow{flow}}}, + snapshotResult{snapshot: KernelSnapshot{Flows: []KernelFlow{flow}}}, + ) + collector := newTestCollector(t, inventory, source, DefaultCollectorConfig()) + start := time.Date(2026, 11, 8, 5, 20, 0, 0, time.UTC) + if err := collector.Collect(context.Background(), start); err != nil { + t.Fatal(err) + } + inventory.mu.Lock() + inventory.err = errors.New("inventory unavailable") + inventory.mu.Unlock() + if err := collector.Collect(context.Background(), start.Add(time.Minute)); err == nil { + t.Fatal("inventory failure was not returned") + } + inventory.mu.Lock() + inventory.err = nil + inventory.mu.Unlock() + if err := collector.Collect(context.Background(), start.Add(2*time.Minute)); err != nil { + t.Fatal(err) + } + batch, err := collector.Batch(0, MaxBatchWindows) + if err != nil { + t.Fatal(err) + } + if len(batch.Windows) != 2 || batch.Windows[1].Complete || + !slices.Contains(batch.Windows[1].QualityFlags, QualitySamplingGap) || + batch.Windows[1].IPv4.Outbound.TCPNewFlows != 0 { + t.Fatalf("inventory-gap windows = %#v", batch.Windows) + } +} + +func TestCollectorRecoversUnavailableCapabilityWithoutInventingWindows(t *testing.T) { + source := availableFakeSource(snapshotResult{snapshot: KernelSnapshot{}}) + source.probeErr = ErrPermissionDenied + source.evidence = ProbeEvidence{} + collector := newTestCollector(t, &fakeInventory{targets: []VMTarget{observedTarget(100)}}, source, DefaultCollectorConfig()) + if collector.Capability().Available { + t.Fatal("failed probe reported available") + } + if err := collector.Collect(context.Background(), time.Date(2026, 11, 8, 3, 0, 0, 0, time.UTC)); err != nil { + t.Fatal(err) + } + batch, err := collector.Batch(0, MaxBatchWindows) + if err != nil { + t.Fatal(err) + } + if !batch.Capability.Available || len(batch.Windows) != 1 { + t.Fatalf("recovered batch = %#v", batch) + } +} + +func TestCollectorCancellationAndCloseAreBounded(t *testing.T) { + source := availableFakeSource() + collector := newTestCollector(t, &fakeInventory{}, source, DefaultCollectorConfig()) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + started := time.Now() + if err := collector.Run(ctx); !errors.Is(err, context.Canceled) { + t.Fatalf("Run error = %v", err) + } + if time.Since(started) > time.Second { + t.Fatal("canceled collector did not stop promptly") + } + done := make(chan error, 1) + go func() { done <- collector.Run(context.Background()) }() + if err := collector.Close(); err != nil { + t.Fatal(err) + } + select { + case err := <-done: + if err != nil { + t.Fatalf("closed Run returned %v", err) + } + case <-time.After(time.Second): + t.Fatal("Close did not stop Run") + } + if err := collector.Close(); err != nil { + t.Fatal(err) + } + if source.closed != 1 { + t.Fatalf("source closed %d times", source.closed) + } +} + +func TestCollectorCloseInterruptsSnapshotWhileCollectOwnsStateLock(t *testing.T) { + source := newBlockingSource() + collector, err := NewCollector(&fakeInventory{targets: []VMTarget{observedTarget(100)}}, source, + testGeneration, DefaultCollectorConfig()) + if err != nil { + t.Fatal(err) + } + collected := make(chan error, 1) + go func() { + collected <- collector.Collect(context.Background(), time.Date(2026, 11, 8, 6, 0, 0, 0, time.UTC)) + }() + select { + case <-source.started: + case <-time.After(time.Second): + t.Fatal("Snapshot did not start") + } + closed := make(chan error, 1) + go func() { closed <- collector.Close() }() + select { + case err := <-closed: + if err != nil { + t.Fatalf("Close error = %v", err) + } + case <-time.After(time.Second): + t.Fatal("Collector.Close waited behind the collection state lock") + } + select { + case err := <-collected: + if err == nil { + t.Fatal("interrupted Collect returned nil") + } + case <-time.After(time.Second): + t.Fatal("Collect remained blocked after Close") + } +} diff --git a/agent/networkobserve/netlink_linux.go b/agent/networkobserve/netlink_linux.go new file mode 100644 index 00000000..b4bf834f --- /dev/null +++ b/agent/networkobserve/netlink_linux.go @@ -0,0 +1,251 @@ +//go:build linux + +package networkobserve + +import ( + "context" + "errors" + "os" + "strconv" + "strings" + "sync" + "time" + + "github.com/mdlayher/netlink" + conntrack "github.com/ti-mo/conntrack" + "golang.org/x/sys/unix" +) + +const conntrackAccountingPath = "/proc/sys/net/netfilter/nf_conntrack_acct" +const conntrackMaxPath = "/proc/sys/net/netfilter/nf_conntrack_max" +const defaultConntrackDumpTimeout = 10 * time.Second + +type conntrackDumper interface { + Dump(*conntrack.DumpOptions) ([]conntrack.Flow, error) + Close() error +} + +type activeConntrackDump struct { + conn conntrackDumper + closeOnce sync.Once + closeErr error +} + +func (dump *activeConntrackDump) Close() error { + dump.closeOnce.Do(func() { dump.closeErr = dump.conn.Close() }) + return dump.closeErr +} + +// NetlinkSource is intentionally read-only. The interface exposed here has no +// Flush, Delete, Update, Create or socket-option mutation method. +type NetlinkSource struct { + mu sync.Mutex + dial func() (conntrackDumper, error) + tableLimit func() (uint64, error) + dumpTimeout time.Duration + closed bool + nextDumpID uint64 + active map[uint64]*activeConntrackDump + inflight sync.WaitGroup +} + +func OpenNetlinkSource() (*NetlinkSource, error) { + return newNetlinkSource(func() (conntrackDumper, error) { + connection, err := conntrack.Dial(&netlink.Config{Strict: true}) + if err != nil { + return nil, classifyNetlinkError(err) + } + return connection, nil + }, readConntrackTableLimit, defaultConntrackDumpTimeout), nil +} + +func newNetlinkSource(dial func() (conntrackDumper, error), tableLimit func() (uint64, error), + dumpTimeout time.Duration) *NetlinkSource { + return &NetlinkSource{ + dial: dial, tableLimit: tableLimit, dumpTimeout: dumpTimeout, + active: make(map[uint64]*activeConntrackDump), + } +} + +func (source *NetlinkSource) Probe(ctx context.Context) (ProbeEvidence, error) { + snapshot, err := source.Snapshot(ctx) + if err != nil { + return ProbeEvidence{}, err + } + return ProbeEvidence{ + Conntrack: true, IPv4: true, IPv6: true, + PacketAccounting: snapshot.PacketAccounting, + }, nil +} + +func (source *NetlinkSource) Snapshot(ctx context.Context) (KernelSnapshot, error) { + if err := ctx.Err(); err != nil { + return KernelSnapshot{}, err + } + if source.tableLimit == nil { + return KernelSnapshot{}, ErrConntrackUnavailable + } + tableLimit, err := source.tableLimit() + if err != nil || tableLimit == 0 || tableLimit > HardMaxKernelFlows { + return KernelSnapshot{}, ErrConntrackUnavailable + } + if source.dial == nil { + return KernelSnapshot{}, ErrConntrackUnavailable + } + connection, err := source.dial() + if err != nil { + return KernelSnapshot{}, classifyNetlinkError(err) + } + active, dumpID, err := source.register(connection) + if err != nil { + _ = connection.Close() + return KernelSnapshot{}, err + } + defer source.release(dumpID, active) + + timeout := source.dumpTimeout + if timeout <= 0 { + timeout = defaultConntrackDumpTimeout + } + dumpContext, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + dumpFinished := make(chan struct{}) + go func() { + select { + case <-dumpContext.Done(): + // mdlayher/netlink supports a concurrent Close specifically to + // interrupt a blocked receive. Each Snapshot owns its socket, so + // cancellation cannot poison a later collection. + _ = active.Close() + case <-dumpFinished: + } + }() + flows, err := connection.Dump(nil) + close(dumpFinished) + if contextErr := dumpContext.Err(); contextErr != nil { + return KernelSnapshot{}, contextErr + } + if err != nil { + return KernelSnapshot{}, classifyNetlinkError(err) + } + if err := ctx.Err(); err != nil { + return KernelSnapshot{}, err + } + result := KernelSnapshot{ + Flows: make([]KernelFlow, 0, min(len(flows), HardMaxKernelFlows)), + PacketAccounting: packetAccountingEnabled(flows), + Overflow: len(flows) > HardMaxKernelFlows, + } + for index := range flows { + if len(result.Flows) >= HardMaxKernelFlows { + break + } + flow := &flows[index] + if flow.Status.Template() || flow.Status.Dying() { + continue + } + observed := KernelFlow{ + SourceAddress: flow.TupleOrig.IP.SourceAddress, + DestinationAddress: flow.TupleOrig.IP.DestinationAddress, + SourcePort: flow.TupleOrig.Proto.SourcePort, + DestinationPort: flow.TupleOrig.Proto.DestinationPort, + Protocol: flow.TupleOrig.Proto.Protocol, + Zone: flow.Zone, + SeenReply: flow.Status.SeenReply(), + } + if flow.ProtoInfo.TCP != nil { + observed.TCPState = flow.ProtoInfo.TCP.State + } + result.Flows = append(result.Flows, observed) + } + return result, nil +} + +func (source *NetlinkSource) register(connection conntrackDumper) (*activeConntrackDump, uint64, error) { + source.mu.Lock() + defer source.mu.Unlock() + if source.closed { + return nil, 0, ErrConntrackUnavailable + } + source.nextDumpID++ + if source.nextDumpID == 0 { + return nil, 0, ErrConntrackUnavailable + } + active := &activeConntrackDump{conn: connection} + source.active[source.nextDumpID] = active + source.inflight.Add(1) + return active, source.nextDumpID, nil +} + +func (source *NetlinkSource) release(dumpID uint64, active *activeConntrackDump) { + _ = active.Close() + source.mu.Lock() + delete(source.active, dumpID) + source.mu.Unlock() + source.inflight.Done() +} + +func readConntrackTableLimit() (uint64, error) { + value, err := os.ReadFile(conntrackMaxPath) + if err != nil { + return 0, err + } + return strconv.ParseUint(strings.TrimSpace(string(value)), 10, 64) +} + +func (source *NetlinkSource) Close() error { + source.mu.Lock() + if source.closed { + source.mu.Unlock() + return nil + } + source.closed = true + active := make([]*activeConntrackDump, 0, len(source.active)) + for _, dump := range source.active { + active = append(active, dump) + } + source.mu.Unlock() + + var result error + for _, dump := range active { + result = errors.Join(result, dump.Close()) + } + source.inflight.Wait() + return result +} + +func packetAccountingEnabled(flows []conntrack.Flow) bool { + if value, err := os.ReadFile(conntrackAccountingPath); err == nil { + switch strings.TrimSpace(string(value)) { + case "1": + return true + case "0": + return false + } + } + for index := range flows { + flow := &flows[index] + if flow.CountersOrig.Packets != 0 || flow.CountersOrig.Bytes != 0 || + flow.CountersReply.Packets != 0 || flow.CountersReply.Bytes != 0 { + return true + } + } + return false +} + +func classifyNetlinkError(err error) error { + if err == nil { + return nil + } + switch { + case errors.Is(err, os.ErrPermission), errors.Is(err, unix.EPERM), errors.Is(err, unix.EACCES): + return errors.Join(ErrPermissionDenied, err) + case errors.Is(err, unix.EAFNOSUPPORT), errors.Is(err, unix.EPROTONOSUPPORT), + errors.Is(err, unix.ENOPROTOOPT): + return errors.Join(ErrNetlinkUnavailable, err) + case errors.Is(err, unix.ENOENT), errors.Is(err, unix.EOPNOTSUPP), errors.Is(err, unix.ENODEV): + return errors.Join(ErrConntrackUnavailable, err) + default: + return errors.Join(ErrConntrackUnavailable, err) + } +} diff --git a/agent/networkobserve/netlink_linux_test.go b/agent/networkobserve/netlink_linux_test.go new file mode 100644 index 00000000..0be08ade --- /dev/null +++ b/agent/networkobserve/netlink_linux_test.go @@ -0,0 +1,216 @@ +//go:build linux + +package networkobserve + +import ( + "context" + "errors" + "net/netip" + "sync" + "testing" + "time" + + conntrack "github.com/ti-mo/conntrack" + "golang.org/x/sys/unix" +) + +type fakeConntrackDumper struct { + mu sync.Mutex + flows []conntrack.Flow + err error + dumpCalls int + closeCalls int +} + +func (dumper *fakeConntrackDumper) Dump(*conntrack.DumpOptions) ([]conntrack.Flow, error) { + dumper.mu.Lock() + defer dumper.mu.Unlock() + dumper.dumpCalls++ + return dumper.flows, dumper.err +} + +func (dumper *fakeConntrackDumper) Close() error { + dumper.mu.Lock() + defer dumper.mu.Unlock() + dumper.closeCalls++ + return nil +} + +func (dumper *fakeConntrackDumper) calls() (int, int) { + dumper.mu.Lock() + defer dumper.mu.Unlock() + return dumper.dumpCalls, dumper.closeCalls +} + +func sourceForDumper(dumper conntrackDumper, tableLimit uint64) *NetlinkSource { + return newNetlinkSource(func() (conntrackDumper, error) { return dumper, nil }, + func() (uint64, error) { return tableLimit, nil }, time.Second) +} + +func TestNetlinkSourceConvertsOriginalTupleReadOnly(t *testing.T) { + dumper := &fakeConntrackDumper{flows: []conntrack.Flow{{ + TupleOrig: conntrack.Tuple{ + IP: conntrack.IPTuple{SourceAddress: netip.MustParseAddr("192.0.2.10"), + DestinationAddress: netip.MustParseAddr("198.51.100.20")}, + Proto: conntrack.ProtoTuple{Protocol: 6, SourcePort: 50000, DestinationPort: 443}, + }, + Zone: 7, + Status: conntrack.StatusSeenReply, + ProtoInfo: conntrack.ProtoInfo{TCP: &conntrack.ProtoInfoTCP{State: 3}}, + CountersOrig: conntrack.Counter{Packets: 1, Bytes: 64}, + }}} + source := sourceForDumper(dumper, 1024) + snapshot, err := source.Snapshot(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(snapshot.Flows) != 1 || snapshot.Flows[0].SourcePort != 50000 || + snapshot.Flows[0].DestinationPort != 443 || snapshot.Flows[0].Zone != 7 || + !snapshot.Flows[0].SeenReply || snapshot.Flows[0].TCPState != 3 { + t.Fatalf("snapshot = %#v", snapshot) + } + dumpCalls, _ := dumper.calls() + if dumpCalls != 1 { + t.Fatalf("Dump calls = %d", dumpCalls) + } +} + +func TestNetlinkSourceRefusesUnboundedKernelTableBeforeDump(t *testing.T) { + dumper := &fakeConntrackDumper{} + source := sourceForDumper(dumper, HardMaxKernelFlows+1) + if _, err := source.Snapshot(context.Background()); !errors.Is(err, ErrConntrackUnavailable) { + t.Fatalf("Snapshot error = %v", err) + } + dumpCalls, _ := dumper.calls() + if dumpCalls != 0 { + t.Fatalf("unbounded table was dumped %d times", dumpCalls) + } +} + +func TestNetlinkErrorsMapToStableCapabilityReasons(t *testing.T) { + tests := []struct { + err error + want error + }{ + {err: unix.EPERM, want: ErrPermissionDenied}, + {err: unix.EPROTONOSUPPORT, want: ErrNetlinkUnavailable}, + {err: unix.ENOENT, want: ErrConntrackUnavailable}, + {err: errors.New("private kernel detail"), want: ErrConntrackUnavailable}, + } + for _, test := range tests { + if got := classifyNetlinkError(test.err); !errors.Is(got, test.want) { + t.Fatalf("classify(%v) = %v, want %v", test.err, got, test.want) + } + } +} + +func TestNetlinkSourceCloseIsIdempotent(t *testing.T) { + dumper := &fakeConntrackDumper{} + source := sourceForDumper(dumper, 1) + // No connection is opened until Snapshot, so closing an idle source must + // not create or close a socket merely to satisfy shutdown. + if err := source.Close(); err != nil { + t.Fatal(err) + } + if err := source.Close(); err != nil { + t.Fatal(err) + } + _, closeCalls := dumper.calls() + if closeCalls != 0 { + t.Fatalf("idle Close calls = %d", closeCalls) + } +} + +type blockingConntrackDumper struct { + started chan struct{} + closed chan struct{} + startOnce sync.Once + closeOnce sync.Once +} + +func newBlockingConntrackDumper() *blockingConntrackDumper { + return &blockingConntrackDumper{started: make(chan struct{}), closed: make(chan struct{})} +} + +func (dumper *blockingConntrackDumper) Dump(*conntrack.DumpOptions) ([]conntrack.Flow, error) { + dumper.startOnce.Do(func() { close(dumper.started) }) + <-dumper.closed + return nil, errors.New("netlink socket closed") +} + +func (dumper *blockingConntrackDumper) Close() error { + dumper.closeOnce.Do(func() { close(dumper.closed) }) + return nil +} + +func TestNetlinkSourceCancellationInterruptsBlockedDump(t *testing.T) { + dumper := newBlockingConntrackDumper() + source := sourceForDumper(dumper, 1024) + ctx, cancel := context.WithCancel(context.Background()) + result := make(chan error, 1) + go func() { + _, err := source.Snapshot(ctx) + result <- err + }() + select { + case <-dumper.started: + case <-time.After(time.Second): + t.Fatal("Dump did not start") + } + cancel() + select { + case err := <-result: + if !errors.Is(err, context.Canceled) { + t.Fatalf("Snapshot error = %v", err) + } + case <-time.After(time.Second): + t.Fatal("context cancellation did not interrupt Dump") + } +} + +func TestNetlinkSourceHardTimeoutInterruptsBlockedDump(t *testing.T) { + dumper := newBlockingConntrackDumper() + source := newNetlinkSource(func() (conntrackDumper, error) { return dumper, nil }, + func() (uint64, error) { return 1024, nil }, 20*time.Millisecond) + started := time.Now() + _, err := source.Snapshot(context.Background()) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("Snapshot error = %v", err) + } + if time.Since(started) > time.Second { + t.Fatal("hard timeout did not interrupt Dump") + } +} + +func TestNetlinkSourceCloseInterruptsBlockedDump(t *testing.T) { + dumper := newBlockingConntrackDumper() + source := sourceForDumper(dumper, 1024) + result := make(chan error, 1) + go func() { + _, err := source.Snapshot(context.Background()) + result <- err + }() + select { + case <-dumper.started: + case <-time.After(time.Second): + t.Fatal("Dump did not start") + } + closed := make(chan error, 1) + go func() { closed <- source.Close() }() + select { + case err := <-closed: + if err != nil { + t.Fatalf("Close error = %v", err) + } + case <-time.After(time.Second): + t.Fatal("Close did not interrupt Dump") + } + select { + case err := <-result: + if !errors.Is(err, ErrConntrackUnavailable) { + t.Fatalf("Snapshot error = %v", err) + } + case <-time.After(time.Second): + t.Fatal("Snapshot remained blocked after Close") + } +} diff --git a/agent/networkobserve/netlink_other.go b/agent/networkobserve/netlink_other.go new file mode 100644 index 00000000..511212d4 --- /dev/null +++ b/agent/networkobserve/netlink_other.go @@ -0,0 +1,19 @@ +//go:build !linux + +package networkobserve + +import "context" + +type NetlinkSource struct{} + +func OpenNetlinkSource() (*NetlinkSource, error) { return &NetlinkSource{}, nil } + +func (*NetlinkSource) Probe(context.Context) (ProbeEvidence, error) { + return ProbeEvidence{}, ErrNetlinkUnavailable +} + +func (*NetlinkSource) Snapshot(context.Context) (KernelSnapshot, error) { + return KernelSnapshot{}, ErrNetlinkUnavailable +} + +func (*NetlinkSource) Close() error { return nil } diff --git a/agent/networkobserve/ring.go b/agent/networkobserve/ring.go new file mode 100644 index 00000000..f8ef102e --- /dev/null +++ b/agent/networkobserve/ring.go @@ -0,0 +1,183 @@ +package networkobserve + +import ( + "crypto/rand" + "encoding/hex" + "errors" + "slices" + "sync" +) + +const HardMaxRingCapacity = 262_144 + +func NewGeneration() (string, error) { + var raw [16]byte + if _, err := rand.Read(raw[:]); err != nil { + return "", err + } + return hex.EncodeToString(raw[:]), nil +} + +type Ring struct { + mu sync.RWMutex + generation string + capacity int + nextCursor uint64 + windows []Window + latestByVM map[string]Window + countsByVM map[string]int +} + +func NewRing(generation string, capacity int) (*Ring, error) { + if !generationPattern.MatchString(generation) || capacity <= 0 || capacity > HardMaxRingCapacity { + return nil, errors.New("invalid network observation ring configuration") + } + return &Ring{ + generation: generation, capacity: capacity, windows: make([]Window, 0, capacity), + latestByVM: make(map[string]Window), countsByVM: make(map[string]int), + }, nil +} + +func (ring *Ring) Generation() string { return ring.generation } + +func (ring *Ring) Append(windows []Window) error { + ring.mu.Lock() + defer ring.mu.Unlock() + if len(windows) > ring.capacity { + return errors.New("network observation append exceeds ring capacity") + } + prepared := make([]Window, len(windows)) + nextCursor := ring.nextCursor + for index := range windows { + window := cloneWindow(windows[index]) + nextCursor++ + if nextCursor == 0 { + return errors.New("network observation cursor exhausted") + } + window.Cursor = nextCursor + windowID, err := StableWindowID(ring.generation, window.Cursor, window.VMUUID, window.BucketStart) + if err != nil { + return err + } + window.WindowID = windowID + prepared[index] = window + } + ring.nextCursor = nextCursor + ring.windows = append(ring.windows, prepared...) + for _, window := range prepared { + ring.countsByVM[window.VMUUID]++ + ring.latestByVM[window.VMUUID] = cloneWindow(window) + } + if overflow := len(ring.windows) - ring.capacity; overflow > 0 { + removedLatest := make(map[string]struct{}) + for index := 0; index < overflow; index++ { + removed := ring.windows[index] + ring.countsByVM[removed.VMUUID]-- + if ring.countsByVM[removed.VMUUID] == 0 { + delete(ring.countsByVM, removed.VMUUID) + delete(ring.latestByVM, removed.VMUUID) + } else if latest, ok := ring.latestByVM[removed.VMUUID]; ok && latest.Cursor == removed.Cursor { + removedLatest[removed.VMUUID] = struct{}{} + } + } + copy(ring.windows, ring.windows[overflow:]) + for index := len(ring.windows) - overflow; index < len(ring.windows); index++ { + ring.windows[index] = Window{} + } + ring.windows = ring.windows[:len(ring.windows)-overflow] + for uuid := range removedLatest { + for index := len(ring.windows) - 1; index >= 0; index-- { + if ring.windows[index].VMUUID == uuid { + ring.latestByVM[uuid] = cloneWindow(ring.windows[index]) + break + } + } + } + } + return nil +} + +func (ring *Ring) Batch(capability Capability, after uint64, limit int) (Batch, error) { + if err := capability.Validate(); err != nil { + return Batch{}, err + } + if limit <= 0 || limit > MaxBatchWindows { + return Batch{}, errors.New("network observation batch limit is invalid") + } + ring.mu.RLock() + defer ring.mu.RUnlock() + result := Batch{ContractVersion: ContractVersion, CollectorGeneration: ring.generation, Capability: capability} + if len(ring.windows) == 0 || !capability.Available { + return result, result.Validate() + } + result.OldestCursor = ring.windows[0].Cursor + start := 0 + if after > 0 { + if after < result.OldestCursor-1 { + result.GapBefore = true + } else { + start = sortSearchWindow(ring.windows, after) + } + } + if start >= len(ring.windows) { + return result, result.Validate() + } + end := min(start+limit, len(ring.windows)) + result.Windows = make([]Window, end-start) + for i := start; i < end; i++ { + result.Windows[i-start] = cloneWindow(ring.windows[i]) + } + result.NextCursor = result.Windows[len(result.Windows)-1].Cursor + result.HasMore = end < len(ring.windows) + if err := result.Validate(); err != nil { + return Batch{}, err + } + return result, nil +} + +func sortSearchWindow(windows []Window, after uint64) int { + left, right := 0, len(windows) + for left < right { + middle := left + (right-left)/2 + if windows[middle].Cursor <= after { + left = middle + 1 + } else { + right = middle + } + } + return left +} + +func (ring *Ring) LatestForVM(uuid string) (Window, bool) { + ring.mu.RLock() + defer ring.mu.RUnlock() + window, ok := ring.latestByVM[uuid] + return cloneWindow(window), ok +} + +func cloneWindow(value Window) Window { + value.QualityFlags = slices.Clone(value.QualityFlags) + if value.Interface != nil { + copy := cloneInterface(*value.Interface) + value.Interface = © + } + value.IPv4 = cloneFamily(value.IPv4) + value.IPv6 = cloneFamily(value.IPv6) + return value +} + +func cloneFamily(value *FamilyObservation) *FamilyObservation { + if value == nil { + return nil + } + result := &FamilyObservation{} + if value.Inbound != nil { + copy := *value.Inbound + result.Inbound = © + } + if value.Outbound != nil { + copy := *value.Outbound + result.Outbound = © + } + return result +} diff --git a/agent/networkobserve/ring_test.go b/agent/networkobserve/ring_test.go new file mode 100644 index 00000000..9c192653 --- /dev/null +++ b/agent/networkobserve/ring_test.go @@ -0,0 +1,126 @@ +package networkobserve + +import ( + "testing" + "time" +) + +func ringTestCapability() Capability { + return Capability{Name: CapabilityName, ContractVersion: ContractVersion, + Available: true, NoPayload: true, IPv4: true, IPv6: true, Conntrack: true, + BucketSeconds: BucketSeconds} +} + +func ringTestWindow(uuid string, minute int) Window { + return Window{ + VMUUID: uuid, BucketStart: time.Date(2026, 11, 8, 1, minute, 0, 0, time.UTC), + BucketSeconds: BucketSeconds, Complete: true, QualityFlags: []QualityFlag{}, + Interface: completeTestInterface(), IPv4: emptyFamilyObservation(), IPv6: emptyFamilyObservation(), + } +} + +func TestRingPaginatesAndReportsRetentionGap(t *testing.T) { + ring, err := NewRing(testGeneration, 3) + if err != nil { + t.Fatal(err) + } + for minute := 1; minute <= 5; minute++ { + if err := ring.Append([]Window{ringTestWindow(testVMUUID, minute)}); err != nil { + t.Fatal(err) + } + } + first, err := ring.Batch(ringTestCapability(), 1, 2) + if err != nil { + t.Fatal(err) + } + if !first.GapBefore || first.OldestCursor != 3 || len(first.Windows) != 2 || + first.Windows[0].Cursor != 3 || first.NextCursor != 4 || !first.HasMore { + t.Fatalf("gap batch = %#v", first) + } + second, err := ring.Batch(ringTestCapability(), first.NextCursor, 2) + if err != nil { + t.Fatal(err) + } + if second.GapBefore || len(second.Windows) != 1 || second.NextCursor != 5 || second.HasMore { + t.Fatalf("second batch = %#v", second) + } + empty, err := ring.Batch(ringTestCapability(), second.NextCursor, 2) + if err != nil { + t.Fatal(err) + } + if len(empty.Windows) != 0 || empty.NextCursor != 0 || empty.OldestCursor != 3 { + t.Fatalf("empty batch = %#v", empty) + } +} + +func TestRingCopiesCallerAndResponseData(t *testing.T) { + ring, err := NewRing(testGeneration, 2) + if err != nil { + t.Fatal(err) + } + window := ringTestWindow(testVMUUID, 1) + if err := ring.Append([]Window{window}); err != nil { + t.Fatal(err) + } + *window.Interface.RXBytes = 999 + batch, err := ring.Batch(ringTestCapability(), 0, 2) + if err != nil { + t.Fatal(err) + } + if got := *batch.Windows[0].Interface.RXBytes; got != 1 { + t.Fatalf("ring retained caller mutation: %d", got) + } + *batch.Windows[0].Interface.RXBytes = 888 + again, err := ring.Batch(ringTestCapability(), 0, 2) + if err != nil { + t.Fatal(err) + } + if got := *again.Windows[0].Interface.RXBytes; got != 1 { + t.Fatalf("ring retained response mutation: %d", got) + } + latest, ok := ring.LatestForVM(testVMUUID) + if !ok || *latest.Interface.RXBytes != 1 { + t.Fatalf("latest index = %#v, %v", latest, ok) + } +} + +func TestRingAppendFailureDoesNotConsumeCursor(t *testing.T) { + ring, err := NewRing(testGeneration, 2) + if err != nil { + t.Fatal(err) + } + bad := ringTestWindow("invalid", 1) + if err := ring.Append([]Window{bad}); err == nil { + t.Fatal("invalid append succeeded") + } + if err := ring.Append([]Window{ringTestWindow(testVMUUID, 1)}); err != nil { + t.Fatal(err) + } + batch, err := ring.Batch(ringTestCapability(), 0, 2) + if err != nil { + t.Fatal(err) + } + if batch.Windows[0].Cursor != 1 { + t.Fatalf("failed append consumed cursor: %d", batch.Windows[0].Cursor) + } +} + +func TestRingRejectsUnboundedCapacity(t *testing.T) { + if _, err := NewRing(testGeneration, HardMaxRingCapacity+1); err == nil { + t.Fatal("unbounded ring capacity accepted") + } +} + +func TestNewGenerationIsCanonicalAndFresh(t *testing.T) { + first, err := NewGeneration() + if err != nil { + t.Fatal(err) + } + second, err := NewGeneration() + if err != nil { + t.Fatal(err) + } + if !generationPattern.MatchString(first) || !generationPattern.MatchString(second) || first == second { + t.Fatalf("generations = %q, %q", first, second) + } +} diff --git a/agent/networkobserve/types.go b/agent/networkobserve/types.go index a6628927..8c2cef4c 100644 --- a/agent/networkobserve/types.go +++ b/agent/networkobserve/types.go @@ -40,6 +40,7 @@ const ( QualityLocalPortCardinality QualityFlag = "local_port_cardinality_saturated" QualityRelationCardinality QualityFlag = "relation_cardinality_saturated" QualityMappingChanged QualityFlag = "mapping_changed" + QualitySamplingGap QualityFlag = "sampling_gap" QualityClockRegression QualityFlag = "clock_regression" QualityCounterReset QualityFlag = "counter_reset" QualityInterfaceUnavailable QualityFlag = "interface_unavailable" @@ -51,7 +52,7 @@ var allowedQualityFlags = map[QualityFlag]struct{}{ QualityColdStart: {}, QualityIncomplete: {}, QualityOverflow: {}, QualityFlowLimitReached: {}, QualityRemoteIPCardinality: {}, QualityRemotePortCardinality: {}, QualityLocalPortCardinality: {}, QualityRelationCardinality: {}, - QualityMappingChanged: {}, QualityClockRegression: {}, QualityCounterReset: {}, + QualityMappingChanged: {}, QualitySamplingGap: {}, QualityClockRegression: {}, QualityCounterReset: {}, QualityInterfaceUnavailable: {}, QualityConntrackUnavailable: {}, QualityPacketAccountingUnavailable: {}, } @@ -60,7 +61,7 @@ var riskUnsafeQualityFlags = map[QualityFlag]struct{}{ QualityColdStart: {}, QualityIncomplete: {}, QualityOverflow: {}, QualityFlowLimitReached: {}, QualityRemoteIPCardinality: {}, QualityRemotePortCardinality: {}, QualityLocalPortCardinality: {}, QualityRelationCardinality: {}, - QualityMappingChanged: {}, QualityClockRegression: {}, QualityCounterReset: {}, + QualityMappingChanged: {}, QualitySamplingGap: {}, QualityClockRegression: {}, QualityCounterReset: {}, QualityInterfaceUnavailable: {}, QualityConntrackUnavailable: {}, } diff --git a/agent/networkobserve/types_test.go b/agent/networkobserve/types_test.go index 58a97be5..4018b506 100644 --- a/agent/networkobserve/types_test.go +++ b/agent/networkobserve/types_test.go @@ -269,6 +269,30 @@ func TestWindowRejectsRiskUnsafeFlagOnCompleteObservation(t *testing.T) { } } +func TestSamplingGapIsCanonicalRiskUnsafeEvidence(t *testing.T) { + start := time.Date(2026, 11, 8, 1, 3, 0, 0, time.UTC) + const generation = "0123456789abcdef0123456789abcdef" + const uuid = "123e4567-e89b-42d3-a456-426614174000" + id, err := StableWindowID(generation, 1, uuid, start) + if err != nil { + t.Fatal(err) + } + window := Window{Cursor: 1, WindowID: id, VMUUID: uuid, BucketStart: start, + BucketSeconds: BucketSeconds, Complete: true, + QualityFlags: []QualityFlag{QualitySamplingGap}} + if err := window.validate(generation); err == nil { + t.Fatal("complete window accepted sampling-gap evidence") + } + window.Complete = false + window.QualityFlags, err = CanonicalQualityFlags([]QualityFlag{QualitySamplingGap, QualityIncomplete}) + if err != nil { + t.Fatal(err) + } + if err := window.validate(generation); err != nil { + t.Fatalf("incomplete sampling-gap evidence was rejected: %v", err) + } +} + func TestPacketAccountingUnavailableDoesNotInvalidateFlowWindow(t *testing.T) { start := time.Date(2026, 11, 8, 1, 2, 0, 0, time.UTC) const generation = "0123456789abcdef0123456789abcdef"