You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/agent/hypervisor/traffic_store.go

433 lines
12 KiB
Go

package hypervisor
import (
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"sync"
"syscall"
"time"
)
const (
maxTrafficStoreBytes = 16 << 20
legacyTrafficStateVersion = 1
trafficStateVersion = 2
)
type trafficObservation struct {
RXBytes uint64
TXBytes uint64
RXAvailable bool
TXAvailable bool
}
func availableTrafficObservation(rx, tx uint64) trafficObservation {
return trafficObservation{
RXBytes: rx, TXBytes: tx, RXAvailable: true, TXAvailable: true,
}
}
type interfaceTrafficSample struct {
TargetAvailable bool
RXSet bool
RXBytes int64
TXSet bool
TXBytes int64
}
func aggregateInterfaceTraffic(samples []interfaceTrafficSample) trafficObservation {
result := trafficObservation{
RXAvailable: len(samples) > 0,
TXAvailable: len(samples) > 0,
}
for _, sample := range samples {
if !sample.TargetAvailable {
result.RXAvailable = false
result.TXAvailable = false
continue
}
if !sample.RXSet || sample.RXBytes < 0 {
result.RXAvailable = false
} else {
result.RXBytes += uint64(sample.RXBytes)
}
if !sample.TXSet || sample.TXBytes < 0 {
result.TXAvailable = false
} else {
result.TXBytes += uint64(sample.TXBytes)
}
}
return result
}
type trafficEntry struct {
Initialized bool `json:"initialized"`
RXInitialized bool `json:"rx_initialized,omitempty"`
TXInitialized bool `json:"tx_initialized,omitempty"`
LastObservedUnixNano int64 `json:"last_observed_unix_nano,omitempty"`
LastRX uint64 `json:"last_rx"`
LastTX uint64 `json:"last_tx"`
TotalRX uint64 `json:"total_rx"`
TotalTX uint64 `json:"total_tx"`
}
type trafficFile struct {
Version int `json:"version"`
VMs map[string]trafficEntry `json:"vms"`
}
// legacyTrafficEntryV1 is decode-only. Version 1 duplicated PHP-owned daily
// buckets on every host; version 2 keeps only the durable monotonic counters
// required to survive libvirt counter resets and Agent restarts.
type legacyTrafficEntryV1 struct {
Initialized bool `json:"initialized"`
RXInitialized bool `json:"rx_initialized,omitempty"`
TXInitialized bool `json:"tx_initialized,omitempty"`
LastObservedUnixNano int64 `json:"last_observed_unix_nano,omitempty"`
LastRX uint64 `json:"last_rx"`
LastTX uint64 `json:"last_tx"`
TotalRX uint64 `json:"total_rx"`
TotalTX uint64 `json:"total_tx"`
LegacyDays map[string]json.RawMessage `json:"days"`
}
type legacyTrafficFileV1 struct {
Version int `json:"version"`
VMs map[string]legacyTrafficEntryV1 `json:"vms"`
}
type trafficStore struct {
mu sync.Mutex
path string
entries map[string]trafficEntry
sampleLocks sync.Map
}
func openTrafficStore(path string) (*trafficStore, error) {
if strings.TrimSpace(path) == "" || !filepath.IsAbs(path) || filepath.Clean(path) != path {
return nil, errors.New("traffic state path must be a clean absolute path")
}
store := &trafficStore{path: path, entries: make(map[string]trafficEntry)}
if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil {
return nil, fmt.Errorf("create traffic state directory: %w", err)
}
if err := store.withFileLock(store.reloadLocked); err != nil {
return nil, err
}
return store, nil
}
func (s *trafficStore) reloadLocked() error {
file, err := os.Open(s.path)
if errors.Is(err, os.ErrNotExist) {
s.entries = make(map[string]trafficEntry)
return nil
}
if err != nil {
return fmt.Errorf("open traffic state: %w", err)
}
defer file.Close()
contents, err := io.ReadAll(io.LimitReader(file, maxTrafficStoreBytes+1))
if err != nil {
return fmt.Errorf("read traffic state: %w", err)
}
if len(contents) > maxTrafficStoreBytes {
return errors.New("traffic state exceeds size limit")
}
var header struct {
Version int `json:"version"`
VMs json.RawMessage `json:"vms"`
}
if err := decodeTrafficDocument(contents, &header); err != nil {
return fmt.Errorf("decode traffic state: %w", err)
}
var entries map[string]trafficEntry
migrate := false
switch header.Version {
case legacyTrafficStateVersion:
var legacy legacyTrafficFileV1
if err := decodeTrafficDocument(contents, &legacy); err != nil {
return fmt.Errorf("decode traffic state: %w", err)
}
if legacy.VMs == nil {
return errors.New("decode traffic state: incomplete document")
}
entries = make(map[string]trafficEntry, len(legacy.VMs))
for uuid, entry := range legacy.VMs {
entries[uuid] = trafficEntry{
Initialized: entry.Initialized,
RXInitialized: entry.RXInitialized,
TXInitialized: entry.TXInitialized,
LastObservedUnixNano: entry.LastObservedUnixNano,
LastRX: entry.LastRX,
LastTX: entry.LastTX,
TotalRX: entry.TotalRX,
TotalTX: entry.TotalTX,
}
}
migrate = true
case trafficStateVersion:
var document trafficFile
if err := decodeTrafficDocument(contents, &document); err != nil {
return fmt.Errorf("decode traffic state: %w", err)
}
if document.VMs == nil {
return errors.New("decode traffic state: incomplete document")
}
entries = document.VMs
default:
return errors.New("decode traffic state: unsupported document version")
}
for uuid, entry := range entries {
entries[uuid] = normalizeTrafficEntry(entry)
}
s.entries = entries
if migrate {
if err := s.persistLocked(); err != nil {
return fmt.Errorf("migrate traffic state: %w", err)
}
}
return nil
}
func decodeTrafficDocument(contents []byte, target any) error {
decoder := json.NewDecoder(strings.NewReader(string(contents)))
decoder.DisallowUnknownFields()
if err := decoder.Decode(target); err != nil {
return err
}
var trailer any
if err := decoder.Decode(&trailer); err != io.EOF {
return errors.New("trailing JSON values are not allowed")
}
return nil
}
func normalizeTrafficEntry(entry trafficEntry) trafficEntry {
// Version 1 originally had a single initialization bit. Promote old
// entries to directional baselines while retaining on-disk compatibility.
if entry.Initialized && !entry.RXInitialized && !entry.TXInitialized {
entry.RXInitialized = true
entry.TXInitialized = true
}
entry.Initialized = entry.RXInitialized && entry.TXInitialized
return entry
}
func (s *trafficStore) withFileLock(call func() error) error {
lock, err := os.OpenFile(s.path+".lock", os.O_CREATE|os.O_RDWR, 0o600)
if err != nil {
return fmt.Errorf("open traffic state lock: %w", err)
}
defer lock.Close()
if err := syscall.Flock(int(lock.Fd()), syscall.LOCK_EX); err != nil {
return fmt.Errorf("lock traffic state: %w", err)
}
defer func() { _ = syscall.Flock(int(lock.Fd()), syscall.LOCK_UN) }()
return call()
}
func (s *trafficStore) lockSample(uuid string) func() {
value, _ := s.sampleLocks.LoadOrStore(uuid, &sync.Mutex{})
lock := value.(*sync.Mutex)
lock.Lock()
return lock.Unlock
}
func (s *trafficStore) initialize(uuid string, at time.Time) (bool, error) {
s.mu.Lock()
defer s.mu.Unlock()
var initialized bool
err := s.withFileLock(func() error {
if err := s.reloadLocked(); err != nil {
return err
}
var err error
initialized, err = s.initializeLocked(uuid, at)
return err
})
return initialized, err
}
func (s *trafficStore) initializeLocked(uuid string, at time.Time) (bool, error) {
previous, exists := s.entries[uuid]
entry := normalizeTrafficEntry(previous)
if entry.RXInitialized || entry.TXInitialized {
return false, nil
}
entry.Initialized = true
entry.RXInitialized = true
entry.TXInitialized = true
entry.LastRX = 0
entry.LastTX = 0
entry.LastObservedUnixNano = at.UTC().UnixNano()
s.entries[uuid] = entry
if err := s.persistLocked(); err != nil {
if exists {
s.entries[uuid] = previous
} else {
delete(s.entries, uuid)
}
return false, err
}
return !exists, nil
}
func (s *trafficStore) observe(uuid string, at time.Time, sample trafficObservation) (uint64, uint64, error) {
s.mu.Lock()
defer s.mu.Unlock()
var totalRX, totalTX uint64
err := s.withFileLock(func() error {
if err := s.reloadLocked(); err != nil {
return err
}
var err error
totalRX, totalTX, err = s.observeLocked(uuid, at, sample)
return err
})
return totalRX, totalTX, err
}
func (s *trafficStore) observeLocked(uuid string, at time.Time, sample trafficObservation) (uint64, uint64, error) {
entry := normalizeTrafficEntry(s.entries[uuid])
if !sample.RXAvailable && !sample.TXAvailable {
return entry.TotalRX, entry.TotalTX, nil
}
observedAt := at.UTC().UnixNano()
duplicateRX := !sample.RXAvailable || (entry.RXInitialized && sample.RXBytes == entry.LastRX)
duplicateTX := !sample.TXAvailable || (entry.TXInitialized && sample.TXBytes == entry.LastTX)
if entry.LastObservedUnixNano != 0 && observedAt == entry.LastObservedUnixNano && duplicateRX && duplicateTX {
return entry.TotalRX, entry.TotalTX, nil
}
var deltaRX, deltaTX uint64
if sample.RXAvailable {
if entry.RXInitialized {
deltaRX = sample.RXBytes
if sample.RXBytes >= entry.LastRX {
deltaRX = sample.RXBytes - entry.LastRX
}
} else {
entry.RXInitialized = true
}
entry.LastRX = sample.RXBytes
}
if sample.TXAvailable {
if entry.TXInitialized {
deltaTX = sample.TXBytes
if sample.TXBytes >= entry.LastTX {
deltaTX = sample.TXBytes - entry.LastTX
}
} else {
entry.TXInitialized = true
}
entry.LastTX = sample.TXBytes
}
entry.Initialized = entry.RXInitialized && entry.TXInitialized
entry.LastObservedUnixNano = observedAt
entry.TotalRX += deltaRX
entry.TotalTX += deltaTX
previous, exists := s.entries[uuid]
s.entries[uuid] = entry
if err := s.persistLocked(); err != nil {
if exists {
s.entries[uuid] = previous
} else {
delete(s.entries, uuid)
}
return 0, 0, err
}
return entry.TotalRX, entry.TotalTX, nil
}
func (s *trafficStore) contains(uuid string) bool {
s.mu.Lock()
defer s.mu.Unlock()
if err := s.withFileLock(s.reloadLocked); err != nil {
return true
}
_, exists := s.entries[uuid]
return exists
}
func (s *trafficStore) remove(uuid string) error {
s.mu.Lock()
defer s.mu.Unlock()
return s.withFileLock(func() error {
if err := s.reloadLocked(); err != nil {
return err
}
return s.removeLocked(uuid)
})
}
func (s *trafficStore) removeLocked(uuid string) error {
previous, exists := s.entries[uuid]
if exists {
delete(s.entries, uuid)
}
// Persist even when the in-memory entry is absent. A previous atomic rename
// may have reached disk before its directory fsync reported an error, while
// the caller conservatively restored only the in-memory map. Rewriting the
// authoritative map here is required before reconciliation can attest that
// no traffic artifact remains.
if err := s.persistLocked(); err != nil {
if exists {
s.entries[uuid] = previous
}
return err
}
return nil
}
func (s *trafficStore) persistLocked() error {
document := trafficFile{Version: trafficStateVersion, VMs: s.entries}
encoded, err := json.Marshal(document)
if err != nil {
return fmt.Errorf("encode traffic state: %w", err)
}
if len(encoded) > maxTrafficStoreBytes {
return errors.New("traffic state exceeds size limit")
}
directory := filepath.Dir(s.path)
if err := os.MkdirAll(directory, 0o750); err != nil {
return fmt.Errorf("create traffic state directory: %w", err)
}
temporary, err := os.CreateTemp(directory, ".traffic-*.tmp")
if err != nil {
return fmt.Errorf("create traffic state temp file: %w", err)
}
temporaryName := temporary.Name()
remove := true
defer func() {
if remove {
_ = os.Remove(temporaryName)
}
}()
if _, err := temporary.Write(encoded); err != nil {
temporary.Close()
return fmt.Errorf("write traffic state: %w", err)
}
if err := temporary.Sync(); err != nil {
temporary.Close()
return fmt.Errorf("sync traffic state: %w", err)
}
if err := temporary.Close(); err != nil {
return fmt.Errorf("close traffic state: %w", err)
}
if err := os.Rename(temporaryName, s.path); err != nil {
return fmt.Errorf("replace traffic state: %w", err)
}
remove = false
dir, err := os.Open(directory)
if err == nil {
err = dir.Sync()
_ = dir.Close()
}
return err
}