"""Periodic libvirt VM measurements and safe counter-to-rate conversion. The collector runs independently of browser visits. Graph-rate baselines are process-local, while billing byte-counter baselines are committed by the log store so traffic accounting survives an engine restart. """ from __future__ import annotations import hashlib import json import logging import math import re import threading import time import uuid from dataclasses import dataclass from typing import Any, Callable, Mapping LOG = logging.getLogger(__name__) SAMPLE_INTERVAL_SECONDS = 60 RETENTION_SECONDS = 30 * 24 * 60 * 60 MAX_QUERY_POINTS = 600 RANGES = {"1h": 3600, "24h": 24 * 3600, "7d": 7 * 24 * 3600, "30d": RETENTION_SECONDS} _COUNTER = re.compile(r"^(block|net)\.(\d+)\.(rd|wr|rx|tx)\.bytes$") _MEMORY_MAX_AGE_SECONDS = 180 _MAX_UNSIGNED_BIGINT = (1 << 64) - 1 def host_scope(profile: Mapping[str, Any]) -> str: """Keep VM history when pool settings change, but isolate a new SSH host key.""" if profile.get("host_id"): return hashlib.sha256(("virt-cloud/host/" + profile["host_id"]).encode("ascii")).hexdigest() identity = [profile[key] for key in ("host", "port", "fingerprint")] if (not isinstance(identity[0], str) or not isinstance(identity[1], int) or not isinstance(identity[2], str)): raise ValueError("Host identity is invalid") raw = json.dumps(identity, ensure_ascii=False, separators=(",", ":")) return hashlib.sha256(raw.encode("utf-8")).hexdigest() def query_interval_seconds(window_seconds: int) -> int: """Epoch-aligned buckets cannot exceed MAX_QUERY_POINTS in any range.""" if window_seconds <= 0: raise ValueError("Metric range must be positive") return max(SAMPLE_INTERVAL_SECONDS, math.ceil(window_seconds / (MAX_QUERY_POINTS - 1) / SAMPLE_INTERVAL_SECONDS) * SAMPLE_INTERVAL_SECONDS) def _nonnegative_int(value: Any) -> int | None: if isinstance(value, bool) or not isinstance(value, int) or value < 0: return None return value def _device_counters(stats: Mapping[str, Any], group: str, directions: tuple[str, str]) -> tuple[tuple[tuple[str, str], ...], dict[str, dict[str, int] | None]]: count = _nonnegative_int(stats.get(f"{group}.count")) devices: dict[str, dict[str, int]] = { str(number): {} for number in range(min(count, 1024)) } if count is not None else {} for key, value in stats.items(): match = _COUNTER.fullmatch(key) if not match or match.group(1) != group or match.group(3) not in directions: continue devices.setdefault(match.group(2), {}) valid = _nonnegative_int(value) if valid is not None: devices[match.group(2)][match.group(3)] = valid signature = tuple(sorted( (number, str(stats.get(f"{group}.{number}.name", number))) for number in devices )) counters_by_direction: dict[str, dict[str, int] | None] = {} for direction in directions: counters = {number: fields[direction] for number, fields in devices.items() if direction in fields} # CD-ROM and unsupported devices may never expose byte counters. Use # only reporting devices, but _rate() resets if that coverage changes. counters_by_direction[direction] = counters or None return signature, counters_by_direction def _network_counter_devices(stats: Mapping[str, Any]) -> dict[str, list[int]] | None: """Return validated cumulative counters keyed by stable guest NIC identity. libvirt reports rx/tx from the guest's point of view: rx is downloaded by the guest and tx is uploaded by it. metric_counters enriches each row with the NIC MAC; the host tap name is only a compatibility fallback. Device indexes are deliberately not identities because their order can change when another NIC is attached. """ count = _nonnegative_int(stats.get("net.count")) if count is None or count > 1024: return None candidates: dict[str, list[int] | None] = {} for number in range(count): name = stats.get(f"net.{number}.name") identity = stats.get(f"net.{number}.identity", name) rx = _nonnegative_int(stats.get(f"net.{number}.rx.bytes")) tx = _nonnegative_int(stats.get(f"net.{number}.tx.bytes")) if (not isinstance(identity, str) or not 1 <= len(identity) <= 255 or any(ord(char) < 32 or ord(char) == 127 for char in identity) or rx is None or tx is None or rx > _MAX_UNSIGNED_BIGINT or tx > _MAX_UNSIGNED_BIGINT): return None # Duplicate names cannot be safely matched to a previous observation. if identity in candidates: return None candidates[identity] = [rx, tx] return {identity: counters for identity, counters in candidates.items() if counters is not None} def _memory_mib(stats: Mapping[str, Any], sampled_at: int) -> tuple[float | None, float | None]: available = _nonnegative_int(stats.get("balloon.available")) usable = _nonnegative_int(stats.get("balloon.usable")) last_update = _nonnegative_int(stats.get("balloon.last-update")) # The QEMU RSS and balloon.current fields describe host process/allocation, # not guest-used memory. Only guest balloon statistics can support used MiB. if (available is None or usable is None or last_update is None or last_update > sampled_at + SAMPLE_INTERVAL_SECONDS or sampled_at - last_update > _MEMORY_MAX_AGE_SECONDS or usable > available): return None, None return (available - usable) / 1024, available / 1024 @dataclass(frozen=True) class RawVMStats: vm_id: str generation: int vcpus: int values: Mapping[str, Any] @dataclass(frozen=True) class _Baseline: monotonic: float generation: int vcpus: int cpu_time: int | None disk_signature: tuple[tuple[str, str], ...] disk: Mapping[str, Mapping[str, int] | None] network_signature: tuple[tuple[str, str], ...] network: Mapping[str, Mapping[str, int] | None] class MetricSampler: def __init__(self) -> None: self._previous: dict[tuple[str, str], _Baseline] = {} @staticmethod def _rate(current: Mapping[str, int] | None, previous: Mapping[str, int] | None, seconds: float, same_devices: bool) -> float | None: if (not same_devices or current is None or previous is None or seconds <= 0 or current.keys() != previous.keys() or any(value < previous[name] for name, value in current.items())): return None return sum(value - previous[name] for name, value in current.items()) / seconds def observe(self, scope: str, raw: RawVMStats, *, timestamp: int, monotonic: float) -> dict[str, Any]: vm_id = str(uuid.UUID(raw.vm_id)) values = raw.values cpu_time = _nonnegative_int(values.get("cpu.time")) disk_signature, disk = _device_counters(values, "block", ("rd", "wr")) network_signature, network = _device_counters(values, "net", ("rx", "tx")) memory_used, memory_total = _memory_mib(values, timestamp) key = (scope, vm_id) previous = self._previous.get(key) seconds = monotonic - previous.monotonic if previous else 0 # A long pause means a graph gap rather than a misleading average. same_run = bool(previous and previous.generation == raw.generation and previous.vcpus == raw.vcpus and 0 < seconds <= SAMPLE_INTERVAL_SECONDS * 3) cpu_percent = None if (same_run and cpu_time is not None and previous.cpu_time is not None and raw.vcpus > 0 and cpu_time >= previous.cpu_time): fraction = (cpu_time - previous.cpu_time) / (seconds * 1e9 * raw.vcpus) # libvirt's total CPU time can include QEMU auxiliary threads, so # a healthy sample can exceed the allotted vCPU time. The chart is # defined as 0–100% of VM vCPU capacity. cpu_percent = min(100.0, fraction * 100) disk_same = same_run and bool(disk_signature) and disk_signature == previous.disk_signature network_same = (same_run and bool(network_signature) and network_signature == previous.network_signature) point = { "vm_id": vm_id, "timestamp": timestamp, "cpu_percent": cpu_percent, "memory_used_mib": memory_used, "memory_total_mib": memory_total, "disk_read_bps": self._rate(disk["rd"], previous.disk["rd"] if previous else None, seconds, disk_same), "disk_write_bps": self._rate(disk["wr"], previous.disk["wr"] if previous else None, seconds, disk_same), "network_rx_bps": self._rate(network["rx"], previous.network["rx"] if previous else None, seconds, network_same), "network_tx_bps": self._rate(network["tx"], previous.network["tx"] if previous else None, seconds, network_same), # Raw per-NIC cumulative counters are never summed into billing # totals here. The database store locks and advances its durable # baseline in the same transaction as this sample. "network_devices": _network_counter_devices(values), } self._previous[key] = _Baseline(monotonic, raw.generation, raw.vcpus, cpu_time, disk_signature, disk, network_signature, network) return point def forget_unseen(self, scope: str, seen: set[str]) -> None: for key in list(self._previous): if key[0] == scope and key[1] not in seen: del self._previous[key] class MetricsCollector: """One daemon sampler owned by the single Uvicorn worker's lifespan.""" def __init__(self, store_getter: Callable[[], Any], profile_getter: Callable[[], Mapping[str, Any] | None], service_factory: Callable[[Mapping[str, Any]], Any], *, owner_getter: Callable[[str, str], int | None] | None = None, interval_seconds: int = SAMPLE_INTERVAL_SECONDS) -> None: self._store_getter = store_getter self._profile_getter = profile_getter self._service_factory = service_factory self._owner_getter = owner_getter self._interval_seconds = interval_seconds self._sampler = MetricSampler() self._stop = threading.Event() self._thread: threading.Thread | None = None self._prune_signal = threading.Event() self._prune_thread: threading.Thread | None = None def sample_once(self, *, timestamp: int | None = None, monotonic: float | None = None) -> None: store = self._store_getter() if store is None or not hasattr(store, "prune_metric_samples"): return timestamp = int(time.time()) if timestamp is None else timestamp try: profile = self._profile_getter() if profile is None or profile.get("key_locked"): return scope = host_scope(profile) rows = self._service_factory(profile).metric_counters() monotonic = time.monotonic() if monotonic is None else monotonic points = [] host_id = profile.get("host_id") for row in rows: point = self._sampler.observe(scope, row, timestamp=timestamp, monotonic=monotonic) point["host_id"] = host_id if isinstance(host_id, str) else None point["user_id"] = (self._owner_getter(scope, point["vm_id"]) if self._owner_getter is not None else None) points.append(point) self._sampler.forget_unseen(scope, {str(uuid.UUID(row.vm_id)) for row in rows}) if points: store.save_metric_samples(scope, points) finally: # Signal after sample/save (or a failed host access). The separate # cleanup thread never delays the next libvirt measurement. self._prune_signal.set() def _prune_loop(self) -> None: while not self._stop.is_set(): self._prune_signal.wait(timeout=self._interval_seconds) self._prune_signal.clear() if self._stop.is_set(): break store = self._store_getter() if store is None or not hasattr(store, "prune_metric_samples"): continue while not self._stop.is_set(): try: removed = store.prune_metric_samples( int(time.time()) - RETENTION_SECONDS) except Exception as exc: # Never echo database driver errors or connection details. LOG.warning("VM metric retention cleanup failed: %s", type(exc).__name__) break if not isinstance(removed, int) or removed < 5000: break # Sustain catch-up without occupying MariaDB continuously. if self._stop.wait(0.1): break def _run(self) -> None: while not self._stop.is_set(): started = time.monotonic() try: self.sample_once() except Exception as exc: # Never print libvirt/DB exception text: it can contain a URI, # account name, or other connection details. LOG.warning("VM metric collection failed: %s", type(exc).__name__) self._stop.wait(max(0, self._interval_seconds - (time.monotonic() - started))) def start(self) -> None: if self._thread is not None: return self._prune_thread = threading.Thread(target=self._prune_loop, daemon=True, name="vm-metrics-retention") self._prune_thread.start() self._thread = threading.Thread(target=self._run, daemon=True, name="vm-metrics-collector") self._thread.start() def close(self) -> None: self._stop.set() self._prune_signal.set() deadline = time.monotonic() + 15 for worker in (self._thread, self._prune_thread): if worker is not None: worker.join(timeout=max(0, deadline - time.monotonic()))