feat(agent): collect bounded network abuse observations
parent
a8edc2780e
commit
219bfd32a9
@ -0,0 +1,524 @@
|
||||
package networkobserve
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"net/netip"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
HardMaxObservedVMs = 4_096
|
||||
// ti-mo/conntrack and mdlayher/netlink each retain a full dump before this
|
||||
// package converts it. Keep the kernel-table ceiling materially below the
|
||||
// contract's per-window aggregation ceiling so a query has a bounded peak.
|
||||
HardMaxKernelFlows = 262_144
|
||||
DefaultRingCapacity = 16_384
|
||||
)
|
||||
|
||||
// VMTarget is the complete, public-only inventory required to attribute one
|
||||
// conntrack tuple to a managed VM. Private/VPC addresses are deliberately not
|
||||
// accepted by this contract.
|
||||
type VMTarget struct {
|
||||
UUID string
|
||||
IPv4 netip.Addr
|
||||
IPv6 netip.Addr
|
||||
Interface InterfaceCounters
|
||||
}
|
||||
|
||||
type InventoryProvider interface {
|
||||
NetworkObservationInventory(context.Context) ([]VMTarget, error)
|
||||
}
|
||||
|
||||
// KernelFlow is the small, library-independent subset of one conntrack entry
|
||||
// consumed by the collector. It contains tuple identity only in Agent memory;
|
||||
// no address or port is copied into a Window or persisted by this package.
|
||||
type KernelFlow struct {
|
||||
SourceAddress netip.Addr
|
||||
DestinationAddress netip.Addr
|
||||
SourcePort uint16
|
||||
DestinationPort uint16
|
||||
Protocol uint8
|
||||
Zone uint16
|
||||
TCPState uint8
|
||||
SeenReply bool
|
||||
}
|
||||
|
||||
type KernelSnapshot struct {
|
||||
Flows []KernelFlow
|
||||
PacketAccounting bool
|
||||
Overflow bool
|
||||
}
|
||||
|
||||
type ConntrackSource interface {
|
||||
ProbeBackend
|
||||
Snapshot(context.Context) (KernelSnapshot, error)
|
||||
Close() error
|
||||
}
|
||||
|
||||
type CollectorConfig struct {
|
||||
Limits Limits
|
||||
MaxVMs int
|
||||
MaxKernelFlows int
|
||||
RingCapacity int
|
||||
}
|
||||
|
||||
func DefaultCollectorConfig() CollectorConfig {
|
||||
return CollectorConfig{
|
||||
Limits: DefaultLimits(), MaxVMs: HardMaxObservedVMs,
|
||||
MaxKernelFlows: HardMaxKernelFlows, RingCapacity: DefaultRingCapacity,
|
||||
}
|
||||
}
|
||||
|
||||
func (config CollectorConfig) validate() error {
|
||||
if err := config.Limits.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
if config.MaxVMs <= 0 || config.MaxVMs > HardMaxObservedVMs ||
|
||||
config.MaxKernelFlows <= 0 || config.MaxKernelFlows > HardMaxKernelFlows ||
|
||||
config.RingCapacity < config.MaxVMs || config.RingCapacity > HardMaxRingCapacity {
|
||||
return errors.New("network observation collector limits are outside hard bounds")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type flowKey struct {
|
||||
source, destination netip.Addr
|
||||
sourcePort uint16
|
||||
destinationPort uint16
|
||||
zone uint16
|
||||
protocol uint8
|
||||
}
|
||||
|
||||
type targetIndex struct {
|
||||
targets []VMTarget
|
||||
byAddress map[netip.Addr]int
|
||||
fingerprint string
|
||||
}
|
||||
|
||||
type Collector struct {
|
||||
mu sync.Mutex
|
||||
inventory InventoryProvider
|
||||
source ConntrackSource
|
||||
config CollectorConfig
|
||||
ring *Ring
|
||||
capability Capability
|
||||
previous map[flowKey]struct{}
|
||||
flowBaselineValid bool
|
||||
previousInterface map[string]InterfaceCounters
|
||||
mapping string
|
||||
lastBucket time.Time
|
||||
samplingGapPending bool
|
||||
closed atomic.Bool
|
||||
closeOnce sync.Once
|
||||
closeErr error
|
||||
done chan struct{}
|
||||
}
|
||||
|
||||
func NewCollector(inventory InventoryProvider, source ConntrackSource, generation string,
|
||||
config CollectorConfig) (*Collector, error) {
|
||||
if inventory == nil || source == nil {
|
||||
return nil, errors.New("network observation inventory and conntrack source are required")
|
||||
}
|
||||
if err := config.validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ring, err := NewRing(generation, config.RingCapacity)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
capability := ProbeCapability(context.Background(), source)
|
||||
return &Collector{
|
||||
inventory: inventory, source: source, config: config, ring: ring,
|
||||
capability: capability, previous: make(map[flowKey]struct{}),
|
||||
previousInterface: make(map[string]InterfaceCounters),
|
||||
done: make(chan struct{}),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (collector *Collector) Generation() string { return collector.ring.Generation() }
|
||||
|
||||
func (collector *Collector) Capability() Capability {
|
||||
collector.mu.Lock()
|
||||
defer collector.mu.Unlock()
|
||||
return collector.capability
|
||||
}
|
||||
|
||||
func (collector *Collector) Batch(after uint64, limit int) (Batch, error) {
|
||||
collector.mu.Lock()
|
||||
capability := collector.capability
|
||||
collector.mu.Unlock()
|
||||
return collector.ring.Batch(capability, after, limit)
|
||||
}
|
||||
|
||||
// Collect samples one minute without mutating conntrack, firewall policy,
|
||||
// sysctls, libvirt domains, instance lifecycle, or billing state.
|
||||
func (collector *Collector) Collect(ctx context.Context, observedAt time.Time) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
bucket := observedAt.UTC().Truncate(time.Duration(BucketSeconds) * time.Second)
|
||||
if bucket.IsZero() {
|
||||
return errors.New("network observation bucket is invalid")
|
||||
}
|
||||
|
||||
if collector.closed.Load() {
|
||||
return errors.New("network observation collector is closed")
|
||||
}
|
||||
collector.mu.Lock()
|
||||
defer collector.mu.Unlock()
|
||||
if collector.closed.Load() {
|
||||
return errors.New("network observation collector is closed")
|
||||
}
|
||||
if bucket.Equal(collector.lastBucket) {
|
||||
return nil
|
||||
}
|
||||
// Never let wall-clock regression append an older bucket: doing so would
|
||||
// advance the cursor and overwrite Ring.LatestForVM with stale evidence.
|
||||
// Remember the discontinuity so the first future bucket cannot present
|
||||
// multi-minute flow deltas as one complete minute.
|
||||
if !collector.lastBucket.IsZero() && bucket.Before(collector.lastBucket) {
|
||||
collector.samplingGapPending = true
|
||||
return nil
|
||||
}
|
||||
|
||||
index, err := collector.readInventory(ctx)
|
||||
if err != nil {
|
||||
if ctx.Err() == nil {
|
||||
collector.samplingGapPending = true
|
||||
}
|
||||
return err
|
||||
}
|
||||
snapshot, snapshotErr := collector.source.Snapshot(ctx)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if collector.closed.Load() {
|
||||
return errors.New("network observation collector is closed")
|
||||
}
|
||||
|
||||
if snapshotErr == nil {
|
||||
collector.capability = capabilityFromSnapshot(snapshot)
|
||||
} else if !collector.capability.Available {
|
||||
// An unavailable collector has no trustworthy address-family contract,
|
||||
// so it must not manufacture empty complete-looking windows.
|
||||
return nil
|
||||
}
|
||||
|
||||
flags := make(map[QualityFlag]struct{})
|
||||
firstObservation := collector.lastBucket.IsZero()
|
||||
samplingGap := collector.samplingGapPending ||
|
||||
(!collector.lastBucket.IsZero() && bucket.After(collector.lastBucket.Add(time.Duration(BucketSeconds)*time.Second)))
|
||||
mappingChanged := collector.mapping != "" && collector.mapping != index.fingerprint
|
||||
flowBaselineUsable := collector.flowBaselineValid && !mappingChanged && !samplingGap
|
||||
if firstObservation || !collector.flowBaselineValid {
|
||||
flags[QualityColdStart] = struct{}{}
|
||||
flags[QualityIncomplete] = struct{}{}
|
||||
}
|
||||
if samplingGap {
|
||||
flags[QualitySamplingGap] = struct{}{}
|
||||
flags[QualityIncomplete] = struct{}{}
|
||||
}
|
||||
if mappingChanged {
|
||||
flags[QualityMappingChanged] = struct{}{}
|
||||
flags[QualityIncomplete] = struct{}{}
|
||||
}
|
||||
if snapshotErr != nil {
|
||||
flags[QualityConntrackUnavailable] = struct{}{}
|
||||
flags[QualityIncomplete] = struct{}{}
|
||||
snapshot.Flows = nil
|
||||
collector.samplingGapPending = true
|
||||
}
|
||||
flowInputComplete := snapshotErr == nil && !snapshot.Overflow
|
||||
if snapshot.Overflow {
|
||||
flags[QualityOverflow] = struct{}{}
|
||||
flags[QualityIncomplete] = struct{}{}
|
||||
}
|
||||
if !snapshot.PacketAccounting {
|
||||
flags[QualityPacketAccountingUnavailable] = struct{}{}
|
||||
}
|
||||
if len(snapshot.Flows) > collector.config.MaxKernelFlows {
|
||||
flags[QualityOverflow] = struct{}{}
|
||||
flags[QualityIncomplete] = struct{}{}
|
||||
snapshot.Flows = snapshot.Flows[:collector.config.MaxKernelFlows]
|
||||
flowInputComplete = false
|
||||
}
|
||||
|
||||
aggregators := make([]*Aggregator, len(index.targets))
|
||||
for i := range aggregators {
|
||||
aggregators[i], err = NewAggregator(collector.config.Limits)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
current := make(map[flowKey]struct{}, min(len(snapshot.Flows), collector.config.MaxKernelFlows))
|
||||
for _, observed := range snapshot.Flows {
|
||||
key, ok := canonicalFlowKey(observed)
|
||||
if !ok {
|
||||
flags[QualityIncomplete] = struct{}{}
|
||||
flowInputComplete = false
|
||||
continue
|
||||
}
|
||||
sourceIndex, sourceManaged := index.byAddress[key.source]
|
||||
destinationIndex, destinationManaged := index.byAddress[key.destination]
|
||||
if !sourceManaged && !destinationManaged {
|
||||
continue
|
||||
}
|
||||
if len(current) < collector.config.MaxKernelFlows {
|
||||
current[key] = struct{}{}
|
||||
} else {
|
||||
flags[QualityFlowLimitReached] = struct{}{}
|
||||
flags[QualityIncomplete] = struct{}{}
|
||||
}
|
||||
_, existed := collector.previous[key]
|
||||
isNew := flowBaselineUsable && !existed
|
||||
protocol := flowProtocol(observed.Protocol)
|
||||
syn := protocol == ProtocolTCP && (observed.TCPState == 1 || observed.TCPState == 2)
|
||||
if sourceManaged {
|
||||
aggregators[sourceIndex].Add(Flow{
|
||||
IPv6: key.source.Is6(), Direction: DirectionOutbound, Protocol: protocol,
|
||||
RemoteIP: key.destination, LocalPort: key.sourcePort, RemotePort: key.destinationPort,
|
||||
New: isNew, SYN: syn, Unreplied: !observed.SeenReply,
|
||||
})
|
||||
}
|
||||
if destinationManaged {
|
||||
aggregators[destinationIndex].Add(Flow{
|
||||
IPv6: key.destination.Is6(), Direction: DirectionInbound, Protocol: protocol,
|
||||
RemoteIP: key.source, LocalPort: key.destinationPort, RemotePort: key.sourcePort,
|
||||
New: isNew, SYN: syn, Unreplied: !observed.SeenReply,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
windows := make([]Window, 0, len(index.targets))
|
||||
for i, target := range index.targets {
|
||||
ipv4, ipv6, aggregateFlags := aggregators[i].Result()
|
||||
windowFlags := cloneFlags(flags)
|
||||
for _, flag := range aggregateFlags {
|
||||
windowFlags[flag] = struct{}{}
|
||||
}
|
||||
interfaceCounters := cloneInterface(target.Interface)
|
||||
if interfaceCounters.hasMissingCounter() {
|
||||
windowFlags[QualityInterfaceUnavailable] = struct{}{}
|
||||
windowFlags[QualityIncomplete] = struct{}{}
|
||||
}
|
||||
if previous, found := collector.previousInterface[target.UUID]; found &&
|
||||
counterRegressed(&previous, &interfaceCounters) {
|
||||
windowFlags[QualityCounterReset] = struct{}{}
|
||||
windowFlags[QualityIncomplete] = struct{}{}
|
||||
}
|
||||
if collector.capability.IPv4 && ipv4 == nil {
|
||||
ipv4 = emptyFamilyObservation()
|
||||
}
|
||||
if collector.capability.IPv6 && ipv6 == nil {
|
||||
ipv6 = emptyFamilyObservation()
|
||||
}
|
||||
quality := flagSlice(windowFlags)
|
||||
complete := !slices.Contains(quality, QualityIncomplete)
|
||||
windows = append(windows, Window{
|
||||
VMUUID: target.UUID, BucketStart: bucket, BucketSeconds: BucketSeconds,
|
||||
Complete: complete, QualityFlags: quality, Interface: &interfaceCounters,
|
||||
IPv4: ipv4, IPv6: ipv6,
|
||||
})
|
||||
}
|
||||
if err := collector.ring.Append(windows); err != nil {
|
||||
return err
|
||||
}
|
||||
if flowInputComplete {
|
||||
collector.previous = current
|
||||
collector.flowBaselineValid = true
|
||||
collector.samplingGapPending = false
|
||||
} else {
|
||||
collector.flowBaselineValid = false
|
||||
}
|
||||
collector.mapping = index.fingerprint
|
||||
collector.lastBucket = bucket
|
||||
nextInterfaces := make(map[string]InterfaceCounters, len(index.targets))
|
||||
for _, target := range index.targets {
|
||||
nextInterfaces[target.UUID] = cloneInterface(target.Interface)
|
||||
}
|
||||
collector.previousInterface = nextInterfaces
|
||||
return nil
|
||||
}
|
||||
|
||||
func (collector *Collector) readInventory(ctx context.Context) (targetIndex, error) {
|
||||
targets, err := collector.inventory.NetworkObservationInventory(ctx)
|
||||
if err != nil {
|
||||
return targetIndex{}, err
|
||||
}
|
||||
if len(targets) > collector.config.MaxVMs {
|
||||
return targetIndex{}, errors.New("network observation inventory exceeds VM limit")
|
||||
}
|
||||
sort.Slice(targets, func(i, j int) bool { return targets[i].UUID < targets[j].UUID })
|
||||
index := targetIndex{targets: targets, byAddress: make(map[netip.Addr]int, len(targets)*2)}
|
||||
var identity strings.Builder
|
||||
for position := range targets {
|
||||
target := &index.targets[position]
|
||||
if !uuidPattern.MatchString(target.UUID) {
|
||||
return targetIndex{}, errors.New("network observation inventory contains invalid VM UUID")
|
||||
}
|
||||
if position > 0 && index.targets[position-1].UUID == target.UUID {
|
||||
return targetIndex{}, errors.New("network observation inventory contains duplicate VM UUID")
|
||||
}
|
||||
target.IPv4 = target.IPv4.Unmap()
|
||||
target.IPv6 = target.IPv6.Unmap()
|
||||
if !target.IPv4.Is4() || !target.IPv6.Is6() || target.IPv6.Is4In6() ||
|
||||
!isPublicUnicast(target.IPv4) || !isPublicUnicast(target.IPv6) {
|
||||
return targetIndex{}, errors.New("network observation inventory requires canonical dual-stack public addresses")
|
||||
}
|
||||
for _, address := range []netip.Addr{target.IPv4, target.IPv6} {
|
||||
if _, duplicate := index.byAddress[address]; duplicate {
|
||||
return targetIndex{}, errors.New("network observation inventory contains duplicate public address")
|
||||
}
|
||||
index.byAddress[address] = position
|
||||
}
|
||||
identity.WriteString(target.UUID)
|
||||
identity.WriteByte('\n')
|
||||
identity.WriteString(target.IPv4.String())
|
||||
identity.WriteByte('\n')
|
||||
identity.WriteString(target.IPv6.String())
|
||||
identity.WriteByte('\n')
|
||||
}
|
||||
digest := sha256.Sum256([]byte(identity.String()))
|
||||
index.fingerprint = hex.EncodeToString(digest[:])
|
||||
return index, nil
|
||||
}
|
||||
|
||||
func isPublicUnicast(address netip.Addr) bool {
|
||||
address = address.Unmap()
|
||||
return address.IsValid() && address.IsGlobalUnicast() &&
|
||||
!address.IsPrivate() && !address.IsUnspecified() &&
|
||||
!address.IsLoopback() && !address.IsLinkLocalUnicast() &&
|
||||
!address.IsLinkLocalMulticast() && !address.IsMulticast()
|
||||
}
|
||||
|
||||
func canonicalFlowKey(flow KernelFlow) (flowKey, bool) {
|
||||
source := flow.SourceAddress.Unmap()
|
||||
destination := flow.DestinationAddress.Unmap()
|
||||
if !source.IsValid() || !destination.IsValid() || source.Is6() != destination.Is6() {
|
||||
return flowKey{}, false
|
||||
}
|
||||
return flowKey{source: source, destination: destination,
|
||||
sourcePort: flow.SourcePort, destinationPort: flow.DestinationPort,
|
||||
zone: flow.Zone, protocol: flow.Protocol}, true
|
||||
}
|
||||
|
||||
func flowProtocol(value uint8) Protocol {
|
||||
switch value {
|
||||
case 6:
|
||||
return ProtocolTCP
|
||||
case 17:
|
||||
return ProtocolUDP
|
||||
default:
|
||||
return ProtocolOther
|
||||
}
|
||||
}
|
||||
|
||||
func capabilityFromSnapshot(snapshot KernelSnapshot) Capability {
|
||||
return Capability{
|
||||
Name: CapabilityName, ContractVersion: ContractVersion, Available: true,
|
||||
NoPayload: true, IPv4: true, IPv6: true, Conntrack: true,
|
||||
PacketAccounting: snapshot.PacketAccounting, PolicyDecisions: false,
|
||||
BucketSeconds: BucketSeconds,
|
||||
}
|
||||
}
|
||||
|
||||
func emptyFamilyObservation() *FamilyObservation {
|
||||
return &FamilyObservation{Inbound: &FlowSummary{}, Outbound: &FlowSummary{}}
|
||||
}
|
||||
|
||||
func cloneFlags(source map[QualityFlag]struct{}) map[QualityFlag]struct{} {
|
||||
result := make(map[QualityFlag]struct{}, len(source))
|
||||
for flag := range source {
|
||||
result[flag] = struct{}{}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func flagSlice(source map[QualityFlag]struct{}) []QualityFlag {
|
||||
result := make([]QualityFlag, 0, len(source))
|
||||
for flag := range source {
|
||||
result = append(result, flag)
|
||||
}
|
||||
result, _ = CanonicalQualityFlags(result)
|
||||
return result
|
||||
}
|
||||
|
||||
func cloneInterface(value InterfaceCounters) InterfaceCounters {
|
||||
return InterfaceCounters{
|
||||
RXBytes: cloneCounter(value.RXBytes), TXBytes: cloneCounter(value.TXBytes),
|
||||
RXPackets: cloneCounter(value.RXPackets), TXPackets: cloneCounter(value.TXPackets),
|
||||
RXErrors: cloneCounter(value.RXErrors), TXErrors: cloneCounter(value.TXErrors),
|
||||
RXDrops: cloneCounter(value.RXDrops), TXDrops: cloneCounter(value.TXDrops),
|
||||
}
|
||||
}
|
||||
|
||||
func cloneCounter(value *uint64) *uint64 {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
copy := *value
|
||||
return ©
|
||||
}
|
||||
|
||||
func counterRegressed(previous, current *InterfaceCounters) bool {
|
||||
if previous == nil || current == nil {
|
||||
return false
|
||||
}
|
||||
left := []*uint64{previous.RXBytes, previous.TXBytes, previous.RXPackets, previous.TXPackets,
|
||||
previous.RXErrors, previous.TXErrors, previous.RXDrops, previous.TXDrops}
|
||||
right := []*uint64{current.RXBytes, current.TXBytes, current.RXPackets, current.TXPackets,
|
||||
current.RXErrors, current.TXErrors, current.RXDrops, current.TXDrops}
|
||||
for i := range left {
|
||||
if left[i] != nil && right[i] != nil && *right[i] < *left[i] {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (collector *Collector) Run(ctx context.Context) error {
|
||||
for {
|
||||
now := time.Now().UTC()
|
||||
next := now.Truncate(time.Duration(BucketSeconds) * time.Second).
|
||||
Add(time.Duration(BucketSeconds) * time.Second)
|
||||
timer := time.NewTimer(time.Until(next))
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
return ctx.Err()
|
||||
case <-collector.done:
|
||||
timer.Stop()
|
||||
return nil
|
||||
case <-timer.C:
|
||||
if err := collector.Collect(ctx, next); err != nil {
|
||||
if errors.Is(err, context.Canceled) {
|
||||
return err
|
||||
}
|
||||
collector.mu.Lock()
|
||||
collector.mu.Unlock()
|
||||
if collector.closed.Load() {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (collector *Collector) Close() error {
|
||||
collector.closeOnce.Do(func() {
|
||||
collector.closed.Store(true)
|
||||
close(collector.done)
|
||||
// Do not take collector.mu here. Snapshot may be blocked while Collect
|
||||
// owns it; the source Close path is what interrupts that blocking read.
|
||||
collector.closeErr = collector.source.Close()
|
||||
})
|
||||
return collector.closeErr
|
||||
}
|
||||
@ -0,0 +1,544 @@
|
||||
package networkobserve
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/netip"
|
||||
"slices"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type fakeInventory struct {
|
||||
mu sync.Mutex
|
||||
targets []VMTarget
|
||||
err error
|
||||
}
|
||||
|
||||
func (inventory *fakeInventory) NetworkObservationInventory(context.Context) ([]VMTarget, error) {
|
||||
inventory.mu.Lock()
|
||||
defer inventory.mu.Unlock()
|
||||
result := make([]VMTarget, len(inventory.targets))
|
||||
copy(result, inventory.targets)
|
||||
return result, inventory.err
|
||||
}
|
||||
|
||||
func (inventory *fakeInventory) set(targets []VMTarget) {
|
||||
inventory.mu.Lock()
|
||||
inventory.targets = targets
|
||||
inventory.mu.Unlock()
|
||||
}
|
||||
|
||||
type snapshotResult struct {
|
||||
snapshot KernelSnapshot
|
||||
err error
|
||||
}
|
||||
|
||||
type fakeSource struct {
|
||||
mu sync.Mutex
|
||||
evidence ProbeEvidence
|
||||
probeErr error
|
||||
snapshots []snapshotResult
|
||||
closed int
|
||||
}
|
||||
|
||||
type blockingSource struct {
|
||||
started chan struct{}
|
||||
closed chan struct{}
|
||||
startOnce sync.Once
|
||||
closeOnce sync.Once
|
||||
}
|
||||
|
||||
func newBlockingSource() *blockingSource {
|
||||
return &blockingSource{started: make(chan struct{}), closed: make(chan struct{})}
|
||||
}
|
||||
|
||||
func (source *blockingSource) Probe(context.Context) (ProbeEvidence, error) {
|
||||
return ProbeEvidence{Conntrack: true, IPv4: true, IPv6: true}, nil
|
||||
}
|
||||
|
||||
func (source *blockingSource) Snapshot(ctx context.Context) (KernelSnapshot, error) {
|
||||
source.startOnce.Do(func() { close(source.started) })
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return KernelSnapshot{}, ctx.Err()
|
||||
case <-source.closed:
|
||||
return KernelSnapshot{}, ErrConntrackUnavailable
|
||||
}
|
||||
}
|
||||
|
||||
func (source *blockingSource) Close() error {
|
||||
source.closeOnce.Do(func() { close(source.closed) })
|
||||
return nil
|
||||
}
|
||||
|
||||
func (source *fakeSource) Probe(context.Context) (ProbeEvidence, error) {
|
||||
return source.evidence, source.probeErr
|
||||
}
|
||||
|
||||
func (source *fakeSource) Snapshot(context.Context) (KernelSnapshot, error) {
|
||||
source.mu.Lock()
|
||||
defer source.mu.Unlock()
|
||||
if len(source.snapshots) == 0 {
|
||||
return KernelSnapshot{}, errors.New("no fake snapshot")
|
||||
}
|
||||
result := source.snapshots[0]
|
||||
source.snapshots = source.snapshots[1:]
|
||||
return result.snapshot, result.err
|
||||
}
|
||||
|
||||
func (source *fakeSource) Close() error {
|
||||
source.mu.Lock()
|
||||
source.closed++
|
||||
source.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func observedTarget(rx uint64) VMTarget {
|
||||
counter := func(value uint64) *uint64 { return &value }
|
||||
return VMTarget{
|
||||
UUID: testVMUUID, IPv4: netip.MustParseAddr("192.0.2.10"), IPv6: netip.MustParseAddr("2001:db8::10"),
|
||||
Interface: InterfaceCounters{
|
||||
RXBytes: counter(rx), TXBytes: counter(200), RXPackets: counter(10), TXPackets: counter(20),
|
||||
RXErrors: counter(0), TXErrors: counter(0), RXDrops: counter(0), TXDrops: counter(0),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func availableFakeSource(results ...snapshotResult) *fakeSource {
|
||||
return &fakeSource{
|
||||
evidence: ProbeEvidence{Conntrack: true, IPv4: true, IPv6: true},
|
||||
snapshots: results,
|
||||
}
|
||||
}
|
||||
|
||||
func newTestCollector(t *testing.T, inventory *fakeInventory, source *fakeSource, config CollectorConfig) *Collector {
|
||||
t.Helper()
|
||||
collector, err := NewCollector(inventory, source, testGeneration, config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return collector
|
||||
}
|
||||
|
||||
func TestCollectorMapsPublicDualStackDirectionsAndNewFlows(t *testing.T) {
|
||||
outbound := KernelFlow{SourceAddress: netip.MustParseAddr("192.0.2.10"),
|
||||
DestinationAddress: netip.MustParseAddr("198.51.100.20"), SourcePort: 50000,
|
||||
DestinationPort: 443, Protocol: 6, TCPState: 1}
|
||||
inbound := KernelFlow{SourceAddress: netip.MustParseAddr("198.51.100.30"),
|
||||
DestinationAddress: netip.MustParseAddr("192.0.2.10"), SourcePort: 51000,
|
||||
DestinationPort: 22, Protocol: 6, TCPState: 3, SeenReply: true}
|
||||
v6 := KernelFlow{SourceAddress: netip.MustParseAddr("2001:db8::10"),
|
||||
DestinationAddress: netip.MustParseAddr("2001:db8:1::20"), SourcePort: 53000,
|
||||
DestinationPort: 53, Protocol: 17}
|
||||
added := KernelFlow{SourceAddress: netip.MustParseAddr("192.0.2.10"),
|
||||
DestinationAddress: netip.MustParseAddr("198.51.100.40"), SourcePort: 50001,
|
||||
DestinationPort: 25, Protocol: 6, TCPState: 1}
|
||||
source := availableFakeSource(
|
||||
snapshotResult{snapshot: KernelSnapshot{Flows: []KernelFlow{outbound, inbound, v6}}},
|
||||
snapshotResult{snapshot: KernelSnapshot{Flows: []KernelFlow{outbound, inbound, v6, added}}},
|
||||
)
|
||||
inventory := &fakeInventory{targets: []VMTarget{observedTarget(100)}}
|
||||
collector := newTestCollector(t, inventory, source, DefaultCollectorConfig())
|
||||
start := time.Date(2026, 11, 8, 1, 1, 0, 0, time.UTC)
|
||||
if err := collector.Collect(context.Background(), start); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inventory.set([]VMTarget{observedTarget(150)})
|
||||
if err := collector.Collect(context.Background(), start.Add(time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
batch, err := collector.Batch(1, MaxBatchWindows)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(batch.Windows) != 1 || !batch.Windows[0].Complete {
|
||||
t.Fatalf("second window = %#v", batch.Windows)
|
||||
}
|
||||
window := batch.Windows[0]
|
||||
if window.IPv4.Outbound.TCPActiveFlows != 2 || window.IPv4.Outbound.TCPNewFlows != 1 ||
|
||||
window.IPv4.Outbound.TCPSYNFlows != 2 || window.IPv4.Outbound.UniqueRemotePorts != 2 ||
|
||||
window.IPv4.Inbound.TCPActiveFlows != 1 || window.IPv4.Inbound.UniqueLocalPorts != 1 ||
|
||||
window.IPv4.Inbound.MaxRemoteIPsPerLocalPort != 1 ||
|
||||
window.IPv6.Outbound.UDPActiveFlows != 1 || window.IPv6.Outbound.UDPNewFlows != 0 {
|
||||
t.Fatalf("flow summaries = ipv4 %#v ipv6 %#v", window.IPv4, window.IPv6)
|
||||
}
|
||||
if *window.Interface.RXBytes != 150 {
|
||||
t.Fatalf("interface counters = %#v", window.Interface)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollectorIgnoresUnknownAndCountsSameHostVMFlowTwice(t *testing.T) {
|
||||
second := observedTarget(300)
|
||||
second.UUID = "123e4567-e89b-42d3-a456-426614174001"
|
||||
second.IPv4 = netip.MustParseAddr("192.0.2.11")
|
||||
second.IPv6 = netip.MustParseAddr("2001:db8::11")
|
||||
managed := KernelFlow{SourceAddress: netip.MustParseAddr("192.0.2.10"),
|
||||
DestinationAddress: netip.MustParseAddr("192.0.2.11"), SourcePort: 50000,
|
||||
DestinationPort: 8080, Protocol: 6, SeenReply: true}
|
||||
unknown := KernelFlow{SourceAddress: netip.MustParseAddr("203.0.113.10"),
|
||||
DestinationAddress: netip.MustParseAddr("203.0.113.11"), Protocol: 17}
|
||||
source := availableFakeSource(snapshotResult{snapshot: KernelSnapshot{Flows: []KernelFlow{managed, unknown}}})
|
||||
collector := newTestCollector(t, &fakeInventory{targets: []VMTarget{observedTarget(100), second}},
|
||||
source, DefaultCollectorConfig())
|
||||
if err := collector.Collect(context.Background(), time.Date(2026, 11, 8, 2, 0, 0, 0, time.UTC)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
batch, err := collector.Batch(0, MaxBatchWindows)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(batch.Windows) != 2 || batch.Windows[0].IPv4.Outbound.TCPActiveFlows != 1 ||
|
||||
batch.Windows[1].IPv4.Inbound.TCPActiveFlows != 1 {
|
||||
t.Fatalf("same-host flow projection = %#v", batch.Windows)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollectorMarksColdStartMappingChangeCounterResetAndOverflow(t *testing.T) {
|
||||
flow := KernelFlow{SourceAddress: netip.MustParseAddr("192.0.2.10"),
|
||||
DestinationAddress: netip.MustParseAddr("198.51.100.20"), Protocol: 17}
|
||||
source := availableFakeSource(
|
||||
snapshotResult{snapshot: KernelSnapshot{Flows: []KernelFlow{flow}}},
|
||||
snapshotResult{snapshot: KernelSnapshot{Flows: []KernelFlow{flow, flow}}},
|
||||
)
|
||||
inventory := &fakeInventory{targets: []VMTarget{observedTarget(100)}}
|
||||
config := DefaultCollectorConfig()
|
||||
config.MaxKernelFlows = 1
|
||||
collector := newTestCollector(t, inventory, source, config)
|
||||
start := time.Date(2026, 11, 8, 2, 0, 0, 0, time.UTC)
|
||||
if err := collector.Collect(context.Background(), start); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
changed := observedTarget(50)
|
||||
changed.IPv6 = netip.MustParseAddr("2001:db8::20")
|
||||
inventory.set([]VMTarget{changed})
|
||||
if err := collector.Collect(context.Background(), start.Add(time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
batch, err := collector.Batch(0, MaxBatchWindows)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !slices.Contains(batch.Windows[0].QualityFlags, QualityColdStart) {
|
||||
t.Fatalf("cold-start flags = %v", batch.Windows[0].QualityFlags)
|
||||
}
|
||||
flags := batch.Windows[1].QualityFlags
|
||||
for _, want := range []QualityFlag{QualityMappingChanged, QualityCounterReset, QualityOverflow, QualityIncomplete} {
|
||||
if !slices.Contains(flags, want) {
|
||||
t.Fatalf("flags %v missing %s", flags, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollectorFailureDoesNotTurnExistingFlowsIntoNew(t *testing.T) {
|
||||
flow := KernelFlow{SourceAddress: netip.MustParseAddr("192.0.2.10"),
|
||||
DestinationAddress: netip.MustParseAddr("198.51.100.20"), Protocol: 6}
|
||||
source := availableFakeSource(
|
||||
snapshotResult{snapshot: KernelSnapshot{Flows: []KernelFlow{flow}}},
|
||||
snapshotResult{err: ErrConntrackUnavailable},
|
||||
snapshotResult{snapshot: KernelSnapshot{Flows: []KernelFlow{flow}}},
|
||||
snapshotResult{snapshot: KernelSnapshot{Flows: []KernelFlow{flow}}},
|
||||
)
|
||||
collector := newTestCollector(t, &fakeInventory{targets: []VMTarget{observedTarget(100)}},
|
||||
source, DefaultCollectorConfig())
|
||||
start := time.Date(2026, 11, 8, 4, 0, 0, 0, time.UTC)
|
||||
for offset := 0; offset < 4; offset++ {
|
||||
if err := collector.Collect(context.Background(), start.Add(time.Duration(offset)*time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
batch, err := collector.Batch(0, MaxBatchWindows)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(batch.Windows) != 4 || !slices.Contains(batch.Windows[2].QualityFlags, QualityColdStart) ||
|
||||
batch.Windows[2].IPv4.Outbound.TCPNewFlows != 0 || batch.Windows[3].IPv4.Outbound.TCPNewFlows != 0 {
|
||||
t.Fatalf("recovery windows = %#v", batch.Windows)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollectorOverflowForcesOneCleanBaselineRebuild(t *testing.T) {
|
||||
flow := KernelFlow{SourceAddress: netip.MustParseAddr("192.0.2.10"),
|
||||
DestinationAddress: netip.MustParseAddr("198.51.100.20"), Protocol: 6}
|
||||
source := availableFakeSource(
|
||||
snapshotResult{snapshot: KernelSnapshot{Flows: []KernelFlow{flow}}},
|
||||
snapshotResult{snapshot: KernelSnapshot{Flows: []KernelFlow{flow}, Overflow: true}},
|
||||
snapshotResult{snapshot: KernelSnapshot{Flows: []KernelFlow{flow}}},
|
||||
snapshotResult{snapshot: KernelSnapshot{Flows: []KernelFlow{flow}}},
|
||||
)
|
||||
collector := newTestCollector(t, &fakeInventory{targets: []VMTarget{observedTarget(100)}},
|
||||
source, DefaultCollectorConfig())
|
||||
start := time.Date(2026, 11, 8, 4, 10, 0, 0, time.UTC)
|
||||
for offset := 0; offset < 4; offset++ {
|
||||
if err := collector.Collect(context.Background(), start.Add(time.Duration(offset)*time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
batch, err := collector.Batch(0, MaxBatchWindows)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !slices.Contains(batch.Windows[1].QualityFlags, QualityOverflow) ||
|
||||
!slices.Contains(batch.Windows[2].QualityFlags, QualityColdStart) ||
|
||||
batch.Windows[2].IPv4.Outbound.TCPNewFlows != 0 || batch.Windows[3].IPv4.Outbound.TCPNewFlows != 0 {
|
||||
t.Fatalf("overflow recovery windows = %#v", batch.Windows)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollectorRejectsAmbiguousOrUnboundedInventory(t *testing.T) {
|
||||
duplicate := observedTarget(100)
|
||||
duplicate.UUID = "123e4567-e89b-42d3-a456-426614174001"
|
||||
inventory := &fakeInventory{targets: []VMTarget{observedTarget(100), duplicate}}
|
||||
collector := newTestCollector(t, inventory, availableFakeSource(), DefaultCollectorConfig())
|
||||
if err := collector.Collect(context.Background(), time.Date(2026, 11, 8, 4, 20, 0, 0, time.UTC)); err == nil {
|
||||
t.Fatal("duplicate public address was accepted")
|
||||
}
|
||||
config := DefaultCollectorConfig()
|
||||
config.MaxVMs = 1
|
||||
collector = newTestCollector(t, inventory, availableFakeSource(), config)
|
||||
if err := collector.Collect(context.Background(), time.Date(2026, 11, 8, 4, 20, 0, 0, time.UTC)); err == nil {
|
||||
t.Fatal("unbounded VM inventory was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollectorRejectsPrivateOrNonPublicInventory(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
address string
|
||||
ipv6 bool
|
||||
}{
|
||||
{name: "private IPv4", address: "10.0.0.10"},
|
||||
{name: "loopback IPv4", address: "127.0.0.1"},
|
||||
{name: "link-local IPv4", address: "169.254.1.10"},
|
||||
{name: "unspecified IPv4", address: "0.0.0.0"},
|
||||
{name: "ULA IPv6", address: "fd00::10", ipv6: true},
|
||||
{name: "loopback IPv6", address: "::1", ipv6: true},
|
||||
{name: "link-local IPv6", address: "fe80::10", ipv6: true},
|
||||
{name: "unspecified IPv6", address: "::", ipv6: true},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
target := observedTarget(100)
|
||||
if test.ipv6 {
|
||||
target.IPv6 = netip.MustParseAddr(test.address)
|
||||
} else {
|
||||
target.IPv4 = netip.MustParseAddr(test.address)
|
||||
}
|
||||
collector := newTestCollector(t, &fakeInventory{targets: []VMTarget{target}},
|
||||
availableFakeSource(), DefaultCollectorConfig())
|
||||
if err := collector.Collect(context.Background(), time.Date(2026, 11, 8, 4, 30, 0, 0, time.UTC)); err == nil {
|
||||
t.Fatalf("non-public address %s was accepted", test.address)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollectorConfigRequiresOneFullMinuteOfRingCapacity(t *testing.T) {
|
||||
config := DefaultCollectorConfig()
|
||||
config.RingCapacity = config.MaxVMs - 1
|
||||
if _, err := NewCollector(&fakeInventory{}, availableFakeSource(), testGeneration, config); err == nil {
|
||||
t.Fatal("ring smaller than one maximum inventory minute was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollectorClockRegressionDoesNotRewindBaselines(t *testing.T) {
|
||||
flow := KernelFlow{SourceAddress: netip.MustParseAddr("192.0.2.10"),
|
||||
DestinationAddress: netip.MustParseAddr("198.51.100.20"), Protocol: 6}
|
||||
added := KernelFlow{SourceAddress: netip.MustParseAddr("192.0.2.10"),
|
||||
DestinationAddress: netip.MustParseAddr("198.51.100.21"), Protocol: 6}
|
||||
source := availableFakeSource(
|
||||
snapshotResult{snapshot: KernelSnapshot{Flows: []KernelFlow{flow}}},
|
||||
snapshotResult{snapshot: KernelSnapshot{Flows: []KernelFlow{flow, added}}},
|
||||
snapshotResult{snapshot: KernelSnapshot{Flows: []KernelFlow{flow, added}}},
|
||||
)
|
||||
collector := newTestCollector(t, &fakeInventory{targets: []VMTarget{observedTarget(100)}}, source, DefaultCollectorConfig())
|
||||
start := time.Date(2026, 11, 8, 5, 1, 0, 0, time.UTC)
|
||||
if err := collector.Collect(context.Background(), start); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := collector.Collect(context.Background(), start.Add(-time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !collector.lastBucket.Equal(start) {
|
||||
t.Fatalf("clock regression rewound last bucket to %s", collector.lastBucket)
|
||||
}
|
||||
before, err := collector.Batch(0, MaxBatchWindows)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(before.Windows) != 1 || before.NextCursor != 1 {
|
||||
t.Fatalf("clock regression appended stale evidence: %#v", before)
|
||||
}
|
||||
if err := collector.Collect(context.Background(), start.Add(time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := collector.Collect(context.Background(), start.Add(2*time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
batch, err := collector.Batch(0, MaxBatchWindows)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(batch.Windows) != 3 || batch.Windows[1].Cursor != 2 ||
|
||||
batch.Windows[1].BucketStart.Before(batch.Windows[0].BucketStart) ||
|
||||
batch.Windows[1].Complete ||
|
||||
!slices.Contains(batch.Windows[1].QualityFlags, QualitySamplingGap) ||
|
||||
batch.Windows[1].IPv4.Outbound.TCPNewFlows != 0 ||
|
||||
!batch.Windows[2].Complete || batch.Windows[2].IPv4.Outbound.TCPNewFlows != 0 {
|
||||
t.Fatalf("clock-regression windows = %#v", batch.Windows)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollectorMarksForwardSamplingGapAndRebuildsBaseline(t *testing.T) {
|
||||
flow := KernelFlow{SourceAddress: netip.MustParseAddr("192.0.2.10"),
|
||||
DestinationAddress: netip.MustParseAddr("198.51.100.20"), Protocol: 6}
|
||||
source := availableFakeSource(
|
||||
snapshotResult{snapshot: KernelSnapshot{Flows: []KernelFlow{flow}}},
|
||||
snapshotResult{snapshot: KernelSnapshot{Flows: []KernelFlow{flow}}},
|
||||
snapshotResult{snapshot: KernelSnapshot{Flows: []KernelFlow{flow}}},
|
||||
)
|
||||
collector := newTestCollector(t, &fakeInventory{targets: []VMTarget{observedTarget(100)}}, source,
|
||||
DefaultCollectorConfig())
|
||||
start := time.Date(2026, 11, 8, 5, 10, 0, 0, time.UTC)
|
||||
for _, observedAt := range []time.Time{start, start.Add(2 * time.Minute), start.Add(3 * time.Minute)} {
|
||||
if err := collector.Collect(context.Background(), observedAt); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
batch, err := collector.Batch(0, MaxBatchWindows)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(batch.Windows) != 3 || batch.Windows[1].Complete ||
|
||||
!slices.Contains(batch.Windows[1].QualityFlags, QualitySamplingGap) ||
|
||||
batch.Windows[1].IPv4.Outbound.TCPNewFlows != 0 ||
|
||||
!batch.Windows[2].Complete || batch.Windows[2].IPv4.Outbound.TCPNewFlows != 0 {
|
||||
t.Fatalf("sampling-gap windows = %#v", batch.Windows)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollectorInventoryFailureMarksNextSuccessAsSamplingGap(t *testing.T) {
|
||||
flow := KernelFlow{SourceAddress: netip.MustParseAddr("192.0.2.10"),
|
||||
DestinationAddress: netip.MustParseAddr("198.51.100.20"), Protocol: 6}
|
||||
inventory := &fakeInventory{targets: []VMTarget{observedTarget(100)}}
|
||||
source := availableFakeSource(
|
||||
snapshotResult{snapshot: KernelSnapshot{Flows: []KernelFlow{flow}}},
|
||||
snapshotResult{snapshot: KernelSnapshot{Flows: []KernelFlow{flow}}},
|
||||
)
|
||||
collector := newTestCollector(t, inventory, source, DefaultCollectorConfig())
|
||||
start := time.Date(2026, 11, 8, 5, 20, 0, 0, time.UTC)
|
||||
if err := collector.Collect(context.Background(), start); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inventory.mu.Lock()
|
||||
inventory.err = errors.New("inventory unavailable")
|
||||
inventory.mu.Unlock()
|
||||
if err := collector.Collect(context.Background(), start.Add(time.Minute)); err == nil {
|
||||
t.Fatal("inventory failure was not returned")
|
||||
}
|
||||
inventory.mu.Lock()
|
||||
inventory.err = nil
|
||||
inventory.mu.Unlock()
|
||||
if err := collector.Collect(context.Background(), start.Add(2*time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
batch, err := collector.Batch(0, MaxBatchWindows)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(batch.Windows) != 2 || batch.Windows[1].Complete ||
|
||||
!slices.Contains(batch.Windows[1].QualityFlags, QualitySamplingGap) ||
|
||||
batch.Windows[1].IPv4.Outbound.TCPNewFlows != 0 {
|
||||
t.Fatalf("inventory-gap windows = %#v", batch.Windows)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollectorRecoversUnavailableCapabilityWithoutInventingWindows(t *testing.T) {
|
||||
source := availableFakeSource(snapshotResult{snapshot: KernelSnapshot{}})
|
||||
source.probeErr = ErrPermissionDenied
|
||||
source.evidence = ProbeEvidence{}
|
||||
collector := newTestCollector(t, &fakeInventory{targets: []VMTarget{observedTarget(100)}}, source, DefaultCollectorConfig())
|
||||
if collector.Capability().Available {
|
||||
t.Fatal("failed probe reported available")
|
||||
}
|
||||
if err := collector.Collect(context.Background(), time.Date(2026, 11, 8, 3, 0, 0, 0, time.UTC)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
batch, err := collector.Batch(0, MaxBatchWindows)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !batch.Capability.Available || len(batch.Windows) != 1 {
|
||||
t.Fatalf("recovered batch = %#v", batch)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollectorCancellationAndCloseAreBounded(t *testing.T) {
|
||||
source := availableFakeSource()
|
||||
collector := newTestCollector(t, &fakeInventory{}, source, DefaultCollectorConfig())
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
started := time.Now()
|
||||
if err := collector.Run(ctx); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("Run error = %v", err)
|
||||
}
|
||||
if time.Since(started) > time.Second {
|
||||
t.Fatal("canceled collector did not stop promptly")
|
||||
}
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- collector.Run(context.Background()) }()
|
||||
if err := collector.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
t.Fatalf("closed Run returned %v", err)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("Close did not stop Run")
|
||||
}
|
||||
if err := collector.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if source.closed != 1 {
|
||||
t.Fatalf("source closed %d times", source.closed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollectorCloseInterruptsSnapshotWhileCollectOwnsStateLock(t *testing.T) {
|
||||
source := newBlockingSource()
|
||||
collector, err := NewCollector(&fakeInventory{targets: []VMTarget{observedTarget(100)}}, source,
|
||||
testGeneration, DefaultCollectorConfig())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
collected := make(chan error, 1)
|
||||
go func() {
|
||||
collected <- collector.Collect(context.Background(), time.Date(2026, 11, 8, 6, 0, 0, 0, time.UTC))
|
||||
}()
|
||||
select {
|
||||
case <-source.started:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("Snapshot did not start")
|
||||
}
|
||||
closed := make(chan error, 1)
|
||||
go func() { closed <- collector.Close() }()
|
||||
select {
|
||||
case err := <-closed:
|
||||
if err != nil {
|
||||
t.Fatalf("Close error = %v", err)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("Collector.Close waited behind the collection state lock")
|
||||
}
|
||||
select {
|
||||
case err := <-collected:
|
||||
if err == nil {
|
||||
t.Fatal("interrupted Collect returned nil")
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("Collect remained blocked after Close")
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,251 @@
|
||||
//go:build linux
|
||||
|
||||
package networkobserve
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/mdlayher/netlink"
|
||||
conntrack "github.com/ti-mo/conntrack"
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
const conntrackAccountingPath = "/proc/sys/net/netfilter/nf_conntrack_acct"
|
||||
const conntrackMaxPath = "/proc/sys/net/netfilter/nf_conntrack_max"
|
||||
const defaultConntrackDumpTimeout = 10 * time.Second
|
||||
|
||||
type conntrackDumper interface {
|
||||
Dump(*conntrack.DumpOptions) ([]conntrack.Flow, error)
|
||||
Close() error
|
||||
}
|
||||
|
||||
type activeConntrackDump struct {
|
||||
conn conntrackDumper
|
||||
closeOnce sync.Once
|
||||
closeErr error
|
||||
}
|
||||
|
||||
func (dump *activeConntrackDump) Close() error {
|
||||
dump.closeOnce.Do(func() { dump.closeErr = dump.conn.Close() })
|
||||
return dump.closeErr
|
||||
}
|
||||
|
||||
// NetlinkSource is intentionally read-only. The interface exposed here has no
|
||||
// Flush, Delete, Update, Create or socket-option mutation method.
|
||||
type NetlinkSource struct {
|
||||
mu sync.Mutex
|
||||
dial func() (conntrackDumper, error)
|
||||
tableLimit func() (uint64, error)
|
||||
dumpTimeout time.Duration
|
||||
closed bool
|
||||
nextDumpID uint64
|
||||
active map[uint64]*activeConntrackDump
|
||||
inflight sync.WaitGroup
|
||||
}
|
||||
|
||||
func OpenNetlinkSource() (*NetlinkSource, error) {
|
||||
return newNetlinkSource(func() (conntrackDumper, error) {
|
||||
connection, err := conntrack.Dial(&netlink.Config{Strict: true})
|
||||
if err != nil {
|
||||
return nil, classifyNetlinkError(err)
|
||||
}
|
||||
return connection, nil
|
||||
}, readConntrackTableLimit, defaultConntrackDumpTimeout), nil
|
||||
}
|
||||
|
||||
func newNetlinkSource(dial func() (conntrackDumper, error), tableLimit func() (uint64, error),
|
||||
dumpTimeout time.Duration) *NetlinkSource {
|
||||
return &NetlinkSource{
|
||||
dial: dial, tableLimit: tableLimit, dumpTimeout: dumpTimeout,
|
||||
active: make(map[uint64]*activeConntrackDump),
|
||||
}
|
||||
}
|
||||
|
||||
func (source *NetlinkSource) Probe(ctx context.Context) (ProbeEvidence, error) {
|
||||
snapshot, err := source.Snapshot(ctx)
|
||||
if err != nil {
|
||||
return ProbeEvidence{}, err
|
||||
}
|
||||
return ProbeEvidence{
|
||||
Conntrack: true, IPv4: true, IPv6: true,
|
||||
PacketAccounting: snapshot.PacketAccounting,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (source *NetlinkSource) Snapshot(ctx context.Context) (KernelSnapshot, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return KernelSnapshot{}, err
|
||||
}
|
||||
if source.tableLimit == nil {
|
||||
return KernelSnapshot{}, ErrConntrackUnavailable
|
||||
}
|
||||
tableLimit, err := source.tableLimit()
|
||||
if err != nil || tableLimit == 0 || tableLimit > HardMaxKernelFlows {
|
||||
return KernelSnapshot{}, ErrConntrackUnavailable
|
||||
}
|
||||
if source.dial == nil {
|
||||
return KernelSnapshot{}, ErrConntrackUnavailable
|
||||
}
|
||||
connection, err := source.dial()
|
||||
if err != nil {
|
||||
return KernelSnapshot{}, classifyNetlinkError(err)
|
||||
}
|
||||
active, dumpID, err := source.register(connection)
|
||||
if err != nil {
|
||||
_ = connection.Close()
|
||||
return KernelSnapshot{}, err
|
||||
}
|
||||
defer source.release(dumpID, active)
|
||||
|
||||
timeout := source.dumpTimeout
|
||||
if timeout <= 0 {
|
||||
timeout = defaultConntrackDumpTimeout
|
||||
}
|
||||
dumpContext, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
dumpFinished := make(chan struct{})
|
||||
go func() {
|
||||
select {
|
||||
case <-dumpContext.Done():
|
||||
// mdlayher/netlink supports a concurrent Close specifically to
|
||||
// interrupt a blocked receive. Each Snapshot owns its socket, so
|
||||
// cancellation cannot poison a later collection.
|
||||
_ = active.Close()
|
||||
case <-dumpFinished:
|
||||
}
|
||||
}()
|
||||
flows, err := connection.Dump(nil)
|
||||
close(dumpFinished)
|
||||
if contextErr := dumpContext.Err(); contextErr != nil {
|
||||
return KernelSnapshot{}, contextErr
|
||||
}
|
||||
if err != nil {
|
||||
return KernelSnapshot{}, classifyNetlinkError(err)
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return KernelSnapshot{}, err
|
||||
}
|
||||
result := KernelSnapshot{
|
||||
Flows: make([]KernelFlow, 0, min(len(flows), HardMaxKernelFlows)),
|
||||
PacketAccounting: packetAccountingEnabled(flows),
|
||||
Overflow: len(flows) > HardMaxKernelFlows,
|
||||
}
|
||||
for index := range flows {
|
||||
if len(result.Flows) >= HardMaxKernelFlows {
|
||||
break
|
||||
}
|
||||
flow := &flows[index]
|
||||
if flow.Status.Template() || flow.Status.Dying() {
|
||||
continue
|
||||
}
|
||||
observed := KernelFlow{
|
||||
SourceAddress: flow.TupleOrig.IP.SourceAddress,
|
||||
DestinationAddress: flow.TupleOrig.IP.DestinationAddress,
|
||||
SourcePort: flow.TupleOrig.Proto.SourcePort,
|
||||
DestinationPort: flow.TupleOrig.Proto.DestinationPort,
|
||||
Protocol: flow.TupleOrig.Proto.Protocol,
|
||||
Zone: flow.Zone,
|
||||
SeenReply: flow.Status.SeenReply(),
|
||||
}
|
||||
if flow.ProtoInfo.TCP != nil {
|
||||
observed.TCPState = flow.ProtoInfo.TCP.State
|
||||
}
|
||||
result.Flows = append(result.Flows, observed)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (source *NetlinkSource) register(connection conntrackDumper) (*activeConntrackDump, uint64, error) {
|
||||
source.mu.Lock()
|
||||
defer source.mu.Unlock()
|
||||
if source.closed {
|
||||
return nil, 0, ErrConntrackUnavailable
|
||||
}
|
||||
source.nextDumpID++
|
||||
if source.nextDumpID == 0 {
|
||||
return nil, 0, ErrConntrackUnavailable
|
||||
}
|
||||
active := &activeConntrackDump{conn: connection}
|
||||
source.active[source.nextDumpID] = active
|
||||
source.inflight.Add(1)
|
||||
return active, source.nextDumpID, nil
|
||||
}
|
||||
|
||||
func (source *NetlinkSource) release(dumpID uint64, active *activeConntrackDump) {
|
||||
_ = active.Close()
|
||||
source.mu.Lock()
|
||||
delete(source.active, dumpID)
|
||||
source.mu.Unlock()
|
||||
source.inflight.Done()
|
||||
}
|
||||
|
||||
func readConntrackTableLimit() (uint64, error) {
|
||||
value, err := os.ReadFile(conntrackMaxPath)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return strconv.ParseUint(strings.TrimSpace(string(value)), 10, 64)
|
||||
}
|
||||
|
||||
func (source *NetlinkSource) Close() error {
|
||||
source.mu.Lock()
|
||||
if source.closed {
|
||||
source.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
source.closed = true
|
||||
active := make([]*activeConntrackDump, 0, len(source.active))
|
||||
for _, dump := range source.active {
|
||||
active = append(active, dump)
|
||||
}
|
||||
source.mu.Unlock()
|
||||
|
||||
var result error
|
||||
for _, dump := range active {
|
||||
result = errors.Join(result, dump.Close())
|
||||
}
|
||||
source.inflight.Wait()
|
||||
return result
|
||||
}
|
||||
|
||||
func packetAccountingEnabled(flows []conntrack.Flow) bool {
|
||||
if value, err := os.ReadFile(conntrackAccountingPath); err == nil {
|
||||
switch strings.TrimSpace(string(value)) {
|
||||
case "1":
|
||||
return true
|
||||
case "0":
|
||||
return false
|
||||
}
|
||||
}
|
||||
for index := range flows {
|
||||
flow := &flows[index]
|
||||
if flow.CountersOrig.Packets != 0 || flow.CountersOrig.Bytes != 0 ||
|
||||
flow.CountersReply.Packets != 0 || flow.CountersReply.Bytes != 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func classifyNetlinkError(err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
switch {
|
||||
case errors.Is(err, os.ErrPermission), errors.Is(err, unix.EPERM), errors.Is(err, unix.EACCES):
|
||||
return errors.Join(ErrPermissionDenied, err)
|
||||
case errors.Is(err, unix.EAFNOSUPPORT), errors.Is(err, unix.EPROTONOSUPPORT),
|
||||
errors.Is(err, unix.ENOPROTOOPT):
|
||||
return errors.Join(ErrNetlinkUnavailable, err)
|
||||
case errors.Is(err, unix.ENOENT), errors.Is(err, unix.EOPNOTSUPP), errors.Is(err, unix.ENODEV):
|
||||
return errors.Join(ErrConntrackUnavailable, err)
|
||||
default:
|
||||
return errors.Join(ErrConntrackUnavailable, err)
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,216 @@
|
||||
//go:build linux
|
||||
|
||||
package networkobserve
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/netip"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
conntrack "github.com/ti-mo/conntrack"
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
type fakeConntrackDumper struct {
|
||||
mu sync.Mutex
|
||||
flows []conntrack.Flow
|
||||
err error
|
||||
dumpCalls int
|
||||
closeCalls int
|
||||
}
|
||||
|
||||
func (dumper *fakeConntrackDumper) Dump(*conntrack.DumpOptions) ([]conntrack.Flow, error) {
|
||||
dumper.mu.Lock()
|
||||
defer dumper.mu.Unlock()
|
||||
dumper.dumpCalls++
|
||||
return dumper.flows, dumper.err
|
||||
}
|
||||
|
||||
func (dumper *fakeConntrackDumper) Close() error {
|
||||
dumper.mu.Lock()
|
||||
defer dumper.mu.Unlock()
|
||||
dumper.closeCalls++
|
||||
return nil
|
||||
}
|
||||
|
||||
func (dumper *fakeConntrackDumper) calls() (int, int) {
|
||||
dumper.mu.Lock()
|
||||
defer dumper.mu.Unlock()
|
||||
return dumper.dumpCalls, dumper.closeCalls
|
||||
}
|
||||
|
||||
func sourceForDumper(dumper conntrackDumper, tableLimit uint64) *NetlinkSource {
|
||||
return newNetlinkSource(func() (conntrackDumper, error) { return dumper, nil },
|
||||
func() (uint64, error) { return tableLimit, nil }, time.Second)
|
||||
}
|
||||
|
||||
func TestNetlinkSourceConvertsOriginalTupleReadOnly(t *testing.T) {
|
||||
dumper := &fakeConntrackDumper{flows: []conntrack.Flow{{
|
||||
TupleOrig: conntrack.Tuple{
|
||||
IP: conntrack.IPTuple{SourceAddress: netip.MustParseAddr("192.0.2.10"),
|
||||
DestinationAddress: netip.MustParseAddr("198.51.100.20")},
|
||||
Proto: conntrack.ProtoTuple{Protocol: 6, SourcePort: 50000, DestinationPort: 443},
|
||||
},
|
||||
Zone: 7,
|
||||
Status: conntrack.StatusSeenReply,
|
||||
ProtoInfo: conntrack.ProtoInfo{TCP: &conntrack.ProtoInfoTCP{State: 3}},
|
||||
CountersOrig: conntrack.Counter{Packets: 1, Bytes: 64},
|
||||
}}}
|
||||
source := sourceForDumper(dumper, 1024)
|
||||
snapshot, err := source.Snapshot(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(snapshot.Flows) != 1 || snapshot.Flows[0].SourcePort != 50000 ||
|
||||
snapshot.Flows[0].DestinationPort != 443 || snapshot.Flows[0].Zone != 7 ||
|
||||
!snapshot.Flows[0].SeenReply || snapshot.Flows[0].TCPState != 3 {
|
||||
t.Fatalf("snapshot = %#v", snapshot)
|
||||
}
|
||||
dumpCalls, _ := dumper.calls()
|
||||
if dumpCalls != 1 {
|
||||
t.Fatalf("Dump calls = %d", dumpCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNetlinkSourceRefusesUnboundedKernelTableBeforeDump(t *testing.T) {
|
||||
dumper := &fakeConntrackDumper{}
|
||||
source := sourceForDumper(dumper, HardMaxKernelFlows+1)
|
||||
if _, err := source.Snapshot(context.Background()); !errors.Is(err, ErrConntrackUnavailable) {
|
||||
t.Fatalf("Snapshot error = %v", err)
|
||||
}
|
||||
dumpCalls, _ := dumper.calls()
|
||||
if dumpCalls != 0 {
|
||||
t.Fatalf("unbounded table was dumped %d times", dumpCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNetlinkErrorsMapToStableCapabilityReasons(t *testing.T) {
|
||||
tests := []struct {
|
||||
err error
|
||||
want error
|
||||
}{
|
||||
{err: unix.EPERM, want: ErrPermissionDenied},
|
||||
{err: unix.EPROTONOSUPPORT, want: ErrNetlinkUnavailable},
|
||||
{err: unix.ENOENT, want: ErrConntrackUnavailable},
|
||||
{err: errors.New("private kernel detail"), want: ErrConntrackUnavailable},
|
||||
}
|
||||
for _, test := range tests {
|
||||
if got := classifyNetlinkError(test.err); !errors.Is(got, test.want) {
|
||||
t.Fatalf("classify(%v) = %v, want %v", test.err, got, test.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNetlinkSourceCloseIsIdempotent(t *testing.T) {
|
||||
dumper := &fakeConntrackDumper{}
|
||||
source := sourceForDumper(dumper, 1)
|
||||
// No connection is opened until Snapshot, so closing an idle source must
|
||||
// not create or close a socket merely to satisfy shutdown.
|
||||
if err := source.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := source.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, closeCalls := dumper.calls()
|
||||
if closeCalls != 0 {
|
||||
t.Fatalf("idle Close calls = %d", closeCalls)
|
||||
}
|
||||
}
|
||||
|
||||
type blockingConntrackDumper struct {
|
||||
started chan struct{}
|
||||
closed chan struct{}
|
||||
startOnce sync.Once
|
||||
closeOnce sync.Once
|
||||
}
|
||||
|
||||
func newBlockingConntrackDumper() *blockingConntrackDumper {
|
||||
return &blockingConntrackDumper{started: make(chan struct{}), closed: make(chan struct{})}
|
||||
}
|
||||
|
||||
func (dumper *blockingConntrackDumper) Dump(*conntrack.DumpOptions) ([]conntrack.Flow, error) {
|
||||
dumper.startOnce.Do(func() { close(dumper.started) })
|
||||
<-dumper.closed
|
||||
return nil, errors.New("netlink socket closed")
|
||||
}
|
||||
|
||||
func (dumper *blockingConntrackDumper) Close() error {
|
||||
dumper.closeOnce.Do(func() { close(dumper.closed) })
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestNetlinkSourceCancellationInterruptsBlockedDump(t *testing.T) {
|
||||
dumper := newBlockingConntrackDumper()
|
||||
source := sourceForDumper(dumper, 1024)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
result := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := source.Snapshot(ctx)
|
||||
result <- err
|
||||
}()
|
||||
select {
|
||||
case <-dumper.started:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("Dump did not start")
|
||||
}
|
||||
cancel()
|
||||
select {
|
||||
case err := <-result:
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("Snapshot error = %v", err)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("context cancellation did not interrupt Dump")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNetlinkSourceHardTimeoutInterruptsBlockedDump(t *testing.T) {
|
||||
dumper := newBlockingConntrackDumper()
|
||||
source := newNetlinkSource(func() (conntrackDumper, error) { return dumper, nil },
|
||||
func() (uint64, error) { return 1024, nil }, 20*time.Millisecond)
|
||||
started := time.Now()
|
||||
_, err := source.Snapshot(context.Background())
|
||||
if !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("Snapshot error = %v", err)
|
||||
}
|
||||
if time.Since(started) > time.Second {
|
||||
t.Fatal("hard timeout did not interrupt Dump")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNetlinkSourceCloseInterruptsBlockedDump(t *testing.T) {
|
||||
dumper := newBlockingConntrackDumper()
|
||||
source := sourceForDumper(dumper, 1024)
|
||||
result := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := source.Snapshot(context.Background())
|
||||
result <- err
|
||||
}()
|
||||
select {
|
||||
case <-dumper.started:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("Dump did not start")
|
||||
}
|
||||
closed := make(chan error, 1)
|
||||
go func() { closed <- source.Close() }()
|
||||
select {
|
||||
case err := <-closed:
|
||||
if err != nil {
|
||||
t.Fatalf("Close error = %v", err)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("Close did not interrupt Dump")
|
||||
}
|
||||
select {
|
||||
case err := <-result:
|
||||
if !errors.Is(err, ErrConntrackUnavailable) {
|
||||
t.Fatalf("Snapshot error = %v", err)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("Snapshot remained blocked after Close")
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,19 @@
|
||||
//go:build !linux
|
||||
|
||||
package networkobserve
|
||||
|
||||
import "context"
|
||||
|
||||
type NetlinkSource struct{}
|
||||
|
||||
func OpenNetlinkSource() (*NetlinkSource, error) { return &NetlinkSource{}, nil }
|
||||
|
||||
func (*NetlinkSource) Probe(context.Context) (ProbeEvidence, error) {
|
||||
return ProbeEvidence{}, ErrNetlinkUnavailable
|
||||
}
|
||||
|
||||
func (*NetlinkSource) Snapshot(context.Context) (KernelSnapshot, error) {
|
||||
return KernelSnapshot{}, ErrNetlinkUnavailable
|
||||
}
|
||||
|
||||
func (*NetlinkSource) Close() error { return nil }
|
||||
@ -0,0 +1,183 @@
|
||||
package networkobserve
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"slices"
|
||||
"sync"
|
||||
)
|
||||
|
||||
const HardMaxRingCapacity = 262_144
|
||||
|
||||
func NewGeneration() (string, error) {
|
||||
var raw [16]byte
|
||||
if _, err := rand.Read(raw[:]); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return hex.EncodeToString(raw[:]), nil
|
||||
}
|
||||
|
||||
type Ring struct {
|
||||
mu sync.RWMutex
|
||||
generation string
|
||||
capacity int
|
||||
nextCursor uint64
|
||||
windows []Window
|
||||
latestByVM map[string]Window
|
||||
countsByVM map[string]int
|
||||
}
|
||||
|
||||
func NewRing(generation string, capacity int) (*Ring, error) {
|
||||
if !generationPattern.MatchString(generation) || capacity <= 0 || capacity > HardMaxRingCapacity {
|
||||
return nil, errors.New("invalid network observation ring configuration")
|
||||
}
|
||||
return &Ring{
|
||||
generation: generation, capacity: capacity, windows: make([]Window, 0, capacity),
|
||||
latestByVM: make(map[string]Window), countsByVM: make(map[string]int),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (ring *Ring) Generation() string { return ring.generation }
|
||||
|
||||
func (ring *Ring) Append(windows []Window) error {
|
||||
ring.mu.Lock()
|
||||
defer ring.mu.Unlock()
|
||||
if len(windows) > ring.capacity {
|
||||
return errors.New("network observation append exceeds ring capacity")
|
||||
}
|
||||
prepared := make([]Window, len(windows))
|
||||
nextCursor := ring.nextCursor
|
||||
for index := range windows {
|
||||
window := cloneWindow(windows[index])
|
||||
nextCursor++
|
||||
if nextCursor == 0 {
|
||||
return errors.New("network observation cursor exhausted")
|
||||
}
|
||||
window.Cursor = nextCursor
|
||||
windowID, err := StableWindowID(ring.generation, window.Cursor, window.VMUUID, window.BucketStart)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
window.WindowID = windowID
|
||||
prepared[index] = window
|
||||
}
|
||||
ring.nextCursor = nextCursor
|
||||
ring.windows = append(ring.windows, prepared...)
|
||||
for _, window := range prepared {
|
||||
ring.countsByVM[window.VMUUID]++
|
||||
ring.latestByVM[window.VMUUID] = cloneWindow(window)
|
||||
}
|
||||
if overflow := len(ring.windows) - ring.capacity; overflow > 0 {
|
||||
removedLatest := make(map[string]struct{})
|
||||
for index := 0; index < overflow; index++ {
|
||||
removed := ring.windows[index]
|
||||
ring.countsByVM[removed.VMUUID]--
|
||||
if ring.countsByVM[removed.VMUUID] == 0 {
|
||||
delete(ring.countsByVM, removed.VMUUID)
|
||||
delete(ring.latestByVM, removed.VMUUID)
|
||||
} else if latest, ok := ring.latestByVM[removed.VMUUID]; ok && latest.Cursor == removed.Cursor {
|
||||
removedLatest[removed.VMUUID] = struct{}{}
|
||||
}
|
||||
}
|
||||
copy(ring.windows, ring.windows[overflow:])
|
||||
for index := len(ring.windows) - overflow; index < len(ring.windows); index++ {
|
||||
ring.windows[index] = Window{}
|
||||
}
|
||||
ring.windows = ring.windows[:len(ring.windows)-overflow]
|
||||
for uuid := range removedLatest {
|
||||
for index := len(ring.windows) - 1; index >= 0; index-- {
|
||||
if ring.windows[index].VMUUID == uuid {
|
||||
ring.latestByVM[uuid] = cloneWindow(ring.windows[index])
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ring *Ring) Batch(capability Capability, after uint64, limit int) (Batch, error) {
|
||||
if err := capability.Validate(); err != nil {
|
||||
return Batch{}, err
|
||||
}
|
||||
if limit <= 0 || limit > MaxBatchWindows {
|
||||
return Batch{}, errors.New("network observation batch limit is invalid")
|
||||
}
|
||||
ring.mu.RLock()
|
||||
defer ring.mu.RUnlock()
|
||||
result := Batch{ContractVersion: ContractVersion, CollectorGeneration: ring.generation, Capability: capability}
|
||||
if len(ring.windows) == 0 || !capability.Available {
|
||||
return result, result.Validate()
|
||||
}
|
||||
result.OldestCursor = ring.windows[0].Cursor
|
||||
start := 0
|
||||
if after > 0 {
|
||||
if after < result.OldestCursor-1 {
|
||||
result.GapBefore = true
|
||||
} else {
|
||||
start = sortSearchWindow(ring.windows, after)
|
||||
}
|
||||
}
|
||||
if start >= len(ring.windows) {
|
||||
return result, result.Validate()
|
||||
}
|
||||
end := min(start+limit, len(ring.windows))
|
||||
result.Windows = make([]Window, end-start)
|
||||
for i := start; i < end; i++ {
|
||||
result.Windows[i-start] = cloneWindow(ring.windows[i])
|
||||
}
|
||||
result.NextCursor = result.Windows[len(result.Windows)-1].Cursor
|
||||
result.HasMore = end < len(ring.windows)
|
||||
if err := result.Validate(); err != nil {
|
||||
return Batch{}, err
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func sortSearchWindow(windows []Window, after uint64) int {
|
||||
left, right := 0, len(windows)
|
||||
for left < right {
|
||||
middle := left + (right-left)/2
|
||||
if windows[middle].Cursor <= after {
|
||||
left = middle + 1
|
||||
} else {
|
||||
right = middle
|
||||
}
|
||||
}
|
||||
return left
|
||||
}
|
||||
|
||||
func (ring *Ring) LatestForVM(uuid string) (Window, bool) {
|
||||
ring.mu.RLock()
|
||||
defer ring.mu.RUnlock()
|
||||
window, ok := ring.latestByVM[uuid]
|
||||
return cloneWindow(window), ok
|
||||
}
|
||||
|
||||
func cloneWindow(value Window) Window {
|
||||
value.QualityFlags = slices.Clone(value.QualityFlags)
|
||||
if value.Interface != nil {
|
||||
copy := cloneInterface(*value.Interface)
|
||||
value.Interface = ©
|
||||
}
|
||||
value.IPv4 = cloneFamily(value.IPv4)
|
||||
value.IPv6 = cloneFamily(value.IPv6)
|
||||
return value
|
||||
}
|
||||
|
||||
func cloneFamily(value *FamilyObservation) *FamilyObservation {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
result := &FamilyObservation{}
|
||||
if value.Inbound != nil {
|
||||
copy := *value.Inbound
|
||||
result.Inbound = ©
|
||||
}
|
||||
if value.Outbound != nil {
|
||||
copy := *value.Outbound
|
||||
result.Outbound = ©
|
||||
}
|
||||
return result
|
||||
}
|
||||
@ -0,0 +1,126 @@
|
||||
package networkobserve
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func ringTestCapability() Capability {
|
||||
return Capability{Name: CapabilityName, ContractVersion: ContractVersion,
|
||||
Available: true, NoPayload: true, IPv4: true, IPv6: true, Conntrack: true,
|
||||
BucketSeconds: BucketSeconds}
|
||||
}
|
||||
|
||||
func ringTestWindow(uuid string, minute int) Window {
|
||||
return Window{
|
||||
VMUUID: uuid, BucketStart: time.Date(2026, 11, 8, 1, minute, 0, 0, time.UTC),
|
||||
BucketSeconds: BucketSeconds, Complete: true, QualityFlags: []QualityFlag{},
|
||||
Interface: completeTestInterface(), IPv4: emptyFamilyObservation(), IPv6: emptyFamilyObservation(),
|
||||
}
|
||||
}
|
||||
|
||||
func TestRingPaginatesAndReportsRetentionGap(t *testing.T) {
|
||||
ring, err := NewRing(testGeneration, 3)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for minute := 1; minute <= 5; minute++ {
|
||||
if err := ring.Append([]Window{ringTestWindow(testVMUUID, minute)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
first, err := ring.Batch(ringTestCapability(), 1, 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !first.GapBefore || first.OldestCursor != 3 || len(first.Windows) != 2 ||
|
||||
first.Windows[0].Cursor != 3 || first.NextCursor != 4 || !first.HasMore {
|
||||
t.Fatalf("gap batch = %#v", first)
|
||||
}
|
||||
second, err := ring.Batch(ringTestCapability(), first.NextCursor, 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if second.GapBefore || len(second.Windows) != 1 || second.NextCursor != 5 || second.HasMore {
|
||||
t.Fatalf("second batch = %#v", second)
|
||||
}
|
||||
empty, err := ring.Batch(ringTestCapability(), second.NextCursor, 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(empty.Windows) != 0 || empty.NextCursor != 0 || empty.OldestCursor != 3 {
|
||||
t.Fatalf("empty batch = %#v", empty)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRingCopiesCallerAndResponseData(t *testing.T) {
|
||||
ring, err := NewRing(testGeneration, 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
window := ringTestWindow(testVMUUID, 1)
|
||||
if err := ring.Append([]Window{window}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
*window.Interface.RXBytes = 999
|
||||
batch, err := ring.Batch(ringTestCapability(), 0, 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := *batch.Windows[0].Interface.RXBytes; got != 1 {
|
||||
t.Fatalf("ring retained caller mutation: %d", got)
|
||||
}
|
||||
*batch.Windows[0].Interface.RXBytes = 888
|
||||
again, err := ring.Batch(ringTestCapability(), 0, 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := *again.Windows[0].Interface.RXBytes; got != 1 {
|
||||
t.Fatalf("ring retained response mutation: %d", got)
|
||||
}
|
||||
latest, ok := ring.LatestForVM(testVMUUID)
|
||||
if !ok || *latest.Interface.RXBytes != 1 {
|
||||
t.Fatalf("latest index = %#v, %v", latest, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRingAppendFailureDoesNotConsumeCursor(t *testing.T) {
|
||||
ring, err := NewRing(testGeneration, 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bad := ringTestWindow("invalid", 1)
|
||||
if err := ring.Append([]Window{bad}); err == nil {
|
||||
t.Fatal("invalid append succeeded")
|
||||
}
|
||||
if err := ring.Append([]Window{ringTestWindow(testVMUUID, 1)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
batch, err := ring.Batch(ringTestCapability(), 0, 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if batch.Windows[0].Cursor != 1 {
|
||||
t.Fatalf("failed append consumed cursor: %d", batch.Windows[0].Cursor)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRingRejectsUnboundedCapacity(t *testing.T) {
|
||||
if _, err := NewRing(testGeneration, HardMaxRingCapacity+1); err == nil {
|
||||
t.Fatal("unbounded ring capacity accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewGenerationIsCanonicalAndFresh(t *testing.T) {
|
||||
first, err := NewGeneration()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second, err := NewGeneration()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !generationPattern.MatchString(first) || !generationPattern.MatchString(second) || first == second {
|
||||
t.Fatalf("generations = %q, %q", first, second)
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue