You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
433 lines
12 KiB
Go
433 lines
12 KiB
Go
package hypervisor
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"syscall"
|
|
"time"
|
|
)
|
|
|
|
const (
|
|
maxTrafficStoreBytes = 16 << 20
|
|
legacyTrafficStateVersion = 1
|
|
trafficStateVersion = 2
|
|
)
|
|
|
|
type trafficObservation struct {
|
|
RXBytes uint64
|
|
TXBytes uint64
|
|
RXAvailable bool
|
|
TXAvailable bool
|
|
}
|
|
|
|
func availableTrafficObservation(rx, tx uint64) trafficObservation {
|
|
return trafficObservation{
|
|
RXBytes: rx, TXBytes: tx, RXAvailable: true, TXAvailable: true,
|
|
}
|
|
}
|
|
|
|
type interfaceTrafficSample struct {
|
|
TargetAvailable bool
|
|
RXSet bool
|
|
RXBytes int64
|
|
TXSet bool
|
|
TXBytes int64
|
|
}
|
|
|
|
func aggregateInterfaceTraffic(samples []interfaceTrafficSample) trafficObservation {
|
|
result := trafficObservation{
|
|
RXAvailable: len(samples) > 0,
|
|
TXAvailable: len(samples) > 0,
|
|
}
|
|
for _, sample := range samples {
|
|
if !sample.TargetAvailable {
|
|
result.RXAvailable = false
|
|
result.TXAvailable = false
|
|
continue
|
|
}
|
|
if !sample.RXSet || sample.RXBytes < 0 {
|
|
result.RXAvailable = false
|
|
} else {
|
|
result.RXBytes += uint64(sample.RXBytes)
|
|
}
|
|
if !sample.TXSet || sample.TXBytes < 0 {
|
|
result.TXAvailable = false
|
|
} else {
|
|
result.TXBytes += uint64(sample.TXBytes)
|
|
}
|
|
}
|
|
return result
|
|
}
|
|
|
|
type trafficEntry struct {
|
|
Initialized bool `json:"initialized"`
|
|
RXInitialized bool `json:"rx_initialized,omitempty"`
|
|
TXInitialized bool `json:"tx_initialized,omitempty"`
|
|
LastObservedUnixNano int64 `json:"last_observed_unix_nano,omitempty"`
|
|
LastRX uint64 `json:"last_rx"`
|
|
LastTX uint64 `json:"last_tx"`
|
|
TotalRX uint64 `json:"total_rx"`
|
|
TotalTX uint64 `json:"total_tx"`
|
|
}
|
|
|
|
type trafficFile struct {
|
|
Version int `json:"version"`
|
|
VMs map[string]trafficEntry `json:"vms"`
|
|
}
|
|
|
|
// legacyTrafficEntryV1 is decode-only. Version 1 duplicated PHP-owned daily
|
|
// buckets on every host; version 2 keeps only the durable monotonic counters
|
|
// required to survive libvirt counter resets and Agent restarts.
|
|
type legacyTrafficEntryV1 struct {
|
|
Initialized bool `json:"initialized"`
|
|
RXInitialized bool `json:"rx_initialized,omitempty"`
|
|
TXInitialized bool `json:"tx_initialized,omitempty"`
|
|
LastObservedUnixNano int64 `json:"last_observed_unix_nano,omitempty"`
|
|
LastRX uint64 `json:"last_rx"`
|
|
LastTX uint64 `json:"last_tx"`
|
|
TotalRX uint64 `json:"total_rx"`
|
|
TotalTX uint64 `json:"total_tx"`
|
|
LegacyDays map[string]json.RawMessage `json:"days"`
|
|
}
|
|
|
|
type legacyTrafficFileV1 struct {
|
|
Version int `json:"version"`
|
|
VMs map[string]legacyTrafficEntryV1 `json:"vms"`
|
|
}
|
|
|
|
type trafficStore struct {
|
|
mu sync.Mutex
|
|
path string
|
|
entries map[string]trafficEntry
|
|
sampleLocks sync.Map
|
|
}
|
|
|
|
func openTrafficStore(path string) (*trafficStore, error) {
|
|
if strings.TrimSpace(path) == "" || !filepath.IsAbs(path) || filepath.Clean(path) != path {
|
|
return nil, errors.New("traffic state path must be a clean absolute path")
|
|
}
|
|
store := &trafficStore{path: path, entries: make(map[string]trafficEntry)}
|
|
if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil {
|
|
return nil, fmt.Errorf("create traffic state directory: %w", err)
|
|
}
|
|
if err := store.withFileLock(store.reloadLocked); err != nil {
|
|
return nil, err
|
|
}
|
|
return store, nil
|
|
}
|
|
|
|
func (s *trafficStore) reloadLocked() error {
|
|
file, err := os.Open(s.path)
|
|
if errors.Is(err, os.ErrNotExist) {
|
|
s.entries = make(map[string]trafficEntry)
|
|
return nil
|
|
}
|
|
if err != nil {
|
|
return fmt.Errorf("open traffic state: %w", err)
|
|
}
|
|
defer file.Close()
|
|
contents, err := io.ReadAll(io.LimitReader(file, maxTrafficStoreBytes+1))
|
|
if err != nil {
|
|
return fmt.Errorf("read traffic state: %w", err)
|
|
}
|
|
if len(contents) > maxTrafficStoreBytes {
|
|
return errors.New("traffic state exceeds size limit")
|
|
}
|
|
var header struct {
|
|
Version int `json:"version"`
|
|
VMs json.RawMessage `json:"vms"`
|
|
}
|
|
if err := decodeTrafficDocument(contents, &header); err != nil {
|
|
return fmt.Errorf("decode traffic state: %w", err)
|
|
}
|
|
var entries map[string]trafficEntry
|
|
migrate := false
|
|
switch header.Version {
|
|
case legacyTrafficStateVersion:
|
|
var legacy legacyTrafficFileV1
|
|
if err := decodeTrafficDocument(contents, &legacy); err != nil {
|
|
return fmt.Errorf("decode traffic state: %w", err)
|
|
}
|
|
if legacy.VMs == nil {
|
|
return errors.New("decode traffic state: incomplete document")
|
|
}
|
|
entries = make(map[string]trafficEntry, len(legacy.VMs))
|
|
for uuid, entry := range legacy.VMs {
|
|
entries[uuid] = trafficEntry{
|
|
Initialized: entry.Initialized,
|
|
RXInitialized: entry.RXInitialized,
|
|
TXInitialized: entry.TXInitialized,
|
|
LastObservedUnixNano: entry.LastObservedUnixNano,
|
|
LastRX: entry.LastRX,
|
|
LastTX: entry.LastTX,
|
|
TotalRX: entry.TotalRX,
|
|
TotalTX: entry.TotalTX,
|
|
}
|
|
}
|
|
migrate = true
|
|
case trafficStateVersion:
|
|
var document trafficFile
|
|
if err := decodeTrafficDocument(contents, &document); err != nil {
|
|
return fmt.Errorf("decode traffic state: %w", err)
|
|
}
|
|
if document.VMs == nil {
|
|
return errors.New("decode traffic state: incomplete document")
|
|
}
|
|
entries = document.VMs
|
|
default:
|
|
return errors.New("decode traffic state: unsupported document version")
|
|
}
|
|
for uuid, entry := range entries {
|
|
entries[uuid] = normalizeTrafficEntry(entry)
|
|
}
|
|
s.entries = entries
|
|
if migrate {
|
|
if err := s.persistLocked(); err != nil {
|
|
return fmt.Errorf("migrate traffic state: %w", err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func decodeTrafficDocument(contents []byte, target any) error {
|
|
decoder := json.NewDecoder(strings.NewReader(string(contents)))
|
|
decoder.DisallowUnknownFields()
|
|
if err := decoder.Decode(target); err != nil {
|
|
return err
|
|
}
|
|
var trailer any
|
|
if err := decoder.Decode(&trailer); err != io.EOF {
|
|
return errors.New("trailing JSON values are not allowed")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func normalizeTrafficEntry(entry trafficEntry) trafficEntry {
|
|
// Version 1 originally had a single initialization bit. Promote old
|
|
// entries to directional baselines while retaining on-disk compatibility.
|
|
if entry.Initialized && !entry.RXInitialized && !entry.TXInitialized {
|
|
entry.RXInitialized = true
|
|
entry.TXInitialized = true
|
|
}
|
|
entry.Initialized = entry.RXInitialized && entry.TXInitialized
|
|
return entry
|
|
}
|
|
|
|
func (s *trafficStore) withFileLock(call func() error) error {
|
|
lock, err := os.OpenFile(s.path+".lock", os.O_CREATE|os.O_RDWR, 0o600)
|
|
if err != nil {
|
|
return fmt.Errorf("open traffic state lock: %w", err)
|
|
}
|
|
defer lock.Close()
|
|
if err := syscall.Flock(int(lock.Fd()), syscall.LOCK_EX); err != nil {
|
|
return fmt.Errorf("lock traffic state: %w", err)
|
|
}
|
|
defer func() { _ = syscall.Flock(int(lock.Fd()), syscall.LOCK_UN) }()
|
|
return call()
|
|
}
|
|
|
|
func (s *trafficStore) lockSample(uuid string) func() {
|
|
value, _ := s.sampleLocks.LoadOrStore(uuid, &sync.Mutex{})
|
|
lock := value.(*sync.Mutex)
|
|
lock.Lock()
|
|
return lock.Unlock
|
|
}
|
|
|
|
func (s *trafficStore) initialize(uuid string, at time.Time) (bool, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
var initialized bool
|
|
err := s.withFileLock(func() error {
|
|
if err := s.reloadLocked(); err != nil {
|
|
return err
|
|
}
|
|
var err error
|
|
initialized, err = s.initializeLocked(uuid, at)
|
|
return err
|
|
})
|
|
return initialized, err
|
|
}
|
|
|
|
func (s *trafficStore) initializeLocked(uuid string, at time.Time) (bool, error) {
|
|
previous, exists := s.entries[uuid]
|
|
entry := normalizeTrafficEntry(previous)
|
|
if entry.RXInitialized || entry.TXInitialized {
|
|
return false, nil
|
|
}
|
|
entry.Initialized = true
|
|
entry.RXInitialized = true
|
|
entry.TXInitialized = true
|
|
entry.LastRX = 0
|
|
entry.LastTX = 0
|
|
entry.LastObservedUnixNano = at.UTC().UnixNano()
|
|
s.entries[uuid] = entry
|
|
if err := s.persistLocked(); err != nil {
|
|
if exists {
|
|
s.entries[uuid] = previous
|
|
} else {
|
|
delete(s.entries, uuid)
|
|
}
|
|
return false, err
|
|
}
|
|
return !exists, nil
|
|
}
|
|
|
|
func (s *trafficStore) observe(uuid string, at time.Time, sample trafficObservation) (uint64, uint64, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
var totalRX, totalTX uint64
|
|
err := s.withFileLock(func() error {
|
|
if err := s.reloadLocked(); err != nil {
|
|
return err
|
|
}
|
|
var err error
|
|
totalRX, totalTX, err = s.observeLocked(uuid, at, sample)
|
|
return err
|
|
})
|
|
return totalRX, totalTX, err
|
|
}
|
|
|
|
func (s *trafficStore) observeLocked(uuid string, at time.Time, sample trafficObservation) (uint64, uint64, error) {
|
|
entry := normalizeTrafficEntry(s.entries[uuid])
|
|
if !sample.RXAvailable && !sample.TXAvailable {
|
|
return entry.TotalRX, entry.TotalTX, nil
|
|
}
|
|
observedAt := at.UTC().UnixNano()
|
|
duplicateRX := !sample.RXAvailable || (entry.RXInitialized && sample.RXBytes == entry.LastRX)
|
|
duplicateTX := !sample.TXAvailable || (entry.TXInitialized && sample.TXBytes == entry.LastTX)
|
|
if entry.LastObservedUnixNano != 0 && observedAt == entry.LastObservedUnixNano && duplicateRX && duplicateTX {
|
|
return entry.TotalRX, entry.TotalTX, nil
|
|
}
|
|
var deltaRX, deltaTX uint64
|
|
if sample.RXAvailable {
|
|
if entry.RXInitialized {
|
|
deltaRX = sample.RXBytes
|
|
if sample.RXBytes >= entry.LastRX {
|
|
deltaRX = sample.RXBytes - entry.LastRX
|
|
}
|
|
} else {
|
|
entry.RXInitialized = true
|
|
}
|
|
entry.LastRX = sample.RXBytes
|
|
}
|
|
if sample.TXAvailable {
|
|
if entry.TXInitialized {
|
|
deltaTX = sample.TXBytes
|
|
if sample.TXBytes >= entry.LastTX {
|
|
deltaTX = sample.TXBytes - entry.LastTX
|
|
}
|
|
} else {
|
|
entry.TXInitialized = true
|
|
}
|
|
entry.LastTX = sample.TXBytes
|
|
}
|
|
entry.Initialized = entry.RXInitialized && entry.TXInitialized
|
|
entry.LastObservedUnixNano = observedAt
|
|
entry.TotalRX += deltaRX
|
|
entry.TotalTX += deltaTX
|
|
previous, exists := s.entries[uuid]
|
|
s.entries[uuid] = entry
|
|
if err := s.persistLocked(); err != nil {
|
|
if exists {
|
|
s.entries[uuid] = previous
|
|
} else {
|
|
delete(s.entries, uuid)
|
|
}
|
|
return 0, 0, err
|
|
}
|
|
return entry.TotalRX, entry.TotalTX, nil
|
|
}
|
|
|
|
func (s *trafficStore) contains(uuid string) bool {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
if err := s.withFileLock(s.reloadLocked); err != nil {
|
|
return true
|
|
}
|
|
_, exists := s.entries[uuid]
|
|
return exists
|
|
}
|
|
|
|
func (s *trafficStore) remove(uuid string) error {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
return s.withFileLock(func() error {
|
|
if err := s.reloadLocked(); err != nil {
|
|
return err
|
|
}
|
|
return s.removeLocked(uuid)
|
|
})
|
|
}
|
|
|
|
func (s *trafficStore) removeLocked(uuid string) error {
|
|
previous, exists := s.entries[uuid]
|
|
if exists {
|
|
delete(s.entries, uuid)
|
|
}
|
|
// Persist even when the in-memory entry is absent. A previous atomic rename
|
|
// may have reached disk before its directory fsync reported an error, while
|
|
// the caller conservatively restored only the in-memory map. Rewriting the
|
|
// authoritative map here is required before reconciliation can attest that
|
|
// no traffic artifact remains.
|
|
if err := s.persistLocked(); err != nil {
|
|
if exists {
|
|
s.entries[uuid] = previous
|
|
}
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *trafficStore) persistLocked() error {
|
|
document := trafficFile{Version: trafficStateVersion, VMs: s.entries}
|
|
encoded, err := json.Marshal(document)
|
|
if err != nil {
|
|
return fmt.Errorf("encode traffic state: %w", err)
|
|
}
|
|
if len(encoded) > maxTrafficStoreBytes {
|
|
return errors.New("traffic state exceeds size limit")
|
|
}
|
|
directory := filepath.Dir(s.path)
|
|
if err := os.MkdirAll(directory, 0o750); err != nil {
|
|
return fmt.Errorf("create traffic state directory: %w", err)
|
|
}
|
|
temporary, err := os.CreateTemp(directory, ".traffic-*.tmp")
|
|
if err != nil {
|
|
return fmt.Errorf("create traffic state temp file: %w", err)
|
|
}
|
|
temporaryName := temporary.Name()
|
|
remove := true
|
|
defer func() {
|
|
if remove {
|
|
_ = os.Remove(temporaryName)
|
|
}
|
|
}()
|
|
if _, err := temporary.Write(encoded); err != nil {
|
|
temporary.Close()
|
|
return fmt.Errorf("write traffic state: %w", err)
|
|
}
|
|
if err := temporary.Sync(); err != nil {
|
|
temporary.Close()
|
|
return fmt.Errorf("sync traffic state: %w", err)
|
|
}
|
|
if err := temporary.Close(); err != nil {
|
|
return fmt.Errorf("close traffic state: %w", err)
|
|
}
|
|
if err := os.Rename(temporaryName, s.path); err != nil {
|
|
return fmt.Errorf("replace traffic state: %w", err)
|
|
}
|
|
remove = false
|
|
dir, err := os.Open(directory)
|
|
if err == nil {
|
|
err = dir.Sync()
|
|
_ = dir.Close()
|
|
}
|
|
return err
|
|
}
|