|
|
"""Periodic libvirt VM measurements and safe counter-to-rate conversion.
|
|
|
|
|
|
The collector runs independently of browser visits. A process restart deliberately
|
|
|
loses counter baselines: the first subsequent rate sample is unknown, not zero.
|
|
|
"""
|
|
|
|
|
|
from __future__ import annotations
|
|
|
|
|
|
import hashlib
|
|
|
import json
|
|
|
import logging
|
|
|
import math
|
|
|
import re
|
|
|
import threading
|
|
|
import time
|
|
|
import uuid
|
|
|
from dataclasses import dataclass
|
|
|
from typing import Any, Callable, Mapping
|
|
|
|
|
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
SAMPLE_INTERVAL_SECONDS = 60
|
|
|
RETENTION_SECONDS = 30 * 24 * 60 * 60
|
|
|
MAX_QUERY_POINTS = 600
|
|
|
RANGES = {"1h": 3600, "24h": 24 * 3600, "7d": 7 * 24 * 3600,
|
|
|
"30d": RETENTION_SECONDS}
|
|
|
_COUNTER = re.compile(r"^(block|net)\.(\d+)\.(rd|wr|rx|tx)\.bytes$")
|
|
|
_MEMORY_MAX_AGE_SECONDS = 180
|
|
|
|
|
|
|
|
|
def host_scope(profile: Mapping[str, Any]) -> str:
|
|
|
"""Keep VM history when pool settings change, but isolate a new SSH host key."""
|
|
|
identity = [profile[key] for key in ("host", "port", "fingerprint")]
|
|
|
if (not isinstance(identity[0], str) or not isinstance(identity[1], int)
|
|
|
or not isinstance(identity[2], str)):
|
|
|
raise ValueError("Host identity is invalid")
|
|
|
raw = json.dumps(identity, ensure_ascii=False, separators=(",", ":"))
|
|
|
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
|
|
|
|
|
|
|
|
|
def query_interval_seconds(window_seconds: int) -> int:
|
|
|
"""Epoch-aligned buckets cannot exceed MAX_QUERY_POINTS in any range."""
|
|
|
if window_seconds <= 0:
|
|
|
raise ValueError("Metric range must be positive")
|
|
|
return max(SAMPLE_INTERVAL_SECONDS,
|
|
|
math.ceil(window_seconds / (MAX_QUERY_POINTS - 1)
|
|
|
/ SAMPLE_INTERVAL_SECONDS) * SAMPLE_INTERVAL_SECONDS)
|
|
|
|
|
|
|
|
|
def _nonnegative_int(value: Any) -> int | None:
|
|
|
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
|
|
|
return None
|
|
|
return value
|
|
|
|
|
|
|
|
|
def _device_counters(stats: Mapping[str, Any], group: str,
|
|
|
directions: tuple[str, str]) -> tuple[tuple[tuple[str, str], ...],
|
|
|
dict[str, dict[str, int] | None]]:
|
|
|
count = _nonnegative_int(stats.get(f"{group}.count"))
|
|
|
devices: dict[str, dict[str, int]] = {
|
|
|
str(number): {} for number in range(min(count, 1024))
|
|
|
} if count is not None else {}
|
|
|
for key, value in stats.items():
|
|
|
match = _COUNTER.fullmatch(key)
|
|
|
if not match or match.group(1) != group or match.group(3) not in directions:
|
|
|
continue
|
|
|
devices.setdefault(match.group(2), {})
|
|
|
valid = _nonnegative_int(value)
|
|
|
if valid is not None:
|
|
|
devices[match.group(2)][match.group(3)] = valid
|
|
|
signature = tuple(sorted(
|
|
|
(number, str(stats.get(f"{group}.{number}.name", number)))
|
|
|
for number in devices
|
|
|
))
|
|
|
counters_by_direction: dict[str, dict[str, int] | None] = {}
|
|
|
for direction in directions:
|
|
|
counters = {number: fields[direction] for number, fields in devices.items()
|
|
|
if direction in fields}
|
|
|
# CD-ROM and unsupported devices may never expose byte counters. Use
|
|
|
# only reporting devices, but _rate() resets if that coverage changes.
|
|
|
counters_by_direction[direction] = counters or None
|
|
|
return signature, counters_by_direction
|
|
|
|
|
|
|
|
|
def _memory_mib(stats: Mapping[str, Any], sampled_at: int) -> tuple[float | None,
|
|
|
float | None]:
|
|
|
available = _nonnegative_int(stats.get("balloon.available"))
|
|
|
usable = _nonnegative_int(stats.get("balloon.usable"))
|
|
|
last_update = _nonnegative_int(stats.get("balloon.last-update"))
|
|
|
# The QEMU RSS and balloon.current fields describe host process/allocation,
|
|
|
# not guest-used memory. Only guest balloon statistics can support used MiB.
|
|
|
if (available is None or usable is None or last_update is None
|
|
|
or last_update > sampled_at + SAMPLE_INTERVAL_SECONDS
|
|
|
or sampled_at - last_update > _MEMORY_MAX_AGE_SECONDS
|
|
|
or usable > available):
|
|
|
return None, None
|
|
|
return (available - usable) / 1024, available / 1024
|
|
|
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
|
class RawVMStats:
|
|
|
vm_id: str
|
|
|
generation: int
|
|
|
vcpus: int
|
|
|
values: Mapping[str, Any]
|
|
|
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
|
class _Baseline:
|
|
|
monotonic: float
|
|
|
generation: int
|
|
|
vcpus: int
|
|
|
cpu_time: int | None
|
|
|
disk_signature: tuple[tuple[str, str], ...]
|
|
|
disk: Mapping[str, Mapping[str, int] | None]
|
|
|
network_signature: tuple[tuple[str, str], ...]
|
|
|
network: Mapping[str, Mapping[str, int] | None]
|
|
|
|
|
|
|
|
|
class MetricSampler:
|
|
|
def __init__(self) -> None:
|
|
|
self._previous: dict[tuple[str, str], _Baseline] = {}
|
|
|
|
|
|
@staticmethod
|
|
|
def _rate(current: Mapping[str, int] | None,
|
|
|
previous: Mapping[str, int] | None,
|
|
|
seconds: float, same_devices: bool) -> float | None:
|
|
|
if (not same_devices or current is None or previous is None
|
|
|
or seconds <= 0 or current.keys() != previous.keys()
|
|
|
or any(value < previous[name] for name, value in current.items())):
|
|
|
return None
|
|
|
return sum(value - previous[name] for name, value in current.items()) / seconds
|
|
|
|
|
|
def observe(self, scope: str, raw: RawVMStats, *, timestamp: int,
|
|
|
monotonic: float) -> dict[str, Any]:
|
|
|
vm_id = str(uuid.UUID(raw.vm_id))
|
|
|
values = raw.values
|
|
|
cpu_time = _nonnegative_int(values.get("cpu.time"))
|
|
|
disk_signature, disk = _device_counters(values, "block", ("rd", "wr"))
|
|
|
network_signature, network = _device_counters(values, "net", ("rx", "tx"))
|
|
|
memory_used, memory_total = _memory_mib(values, timestamp)
|
|
|
key = (scope, vm_id)
|
|
|
previous = self._previous.get(key)
|
|
|
seconds = monotonic - previous.monotonic if previous else 0
|
|
|
# A long pause means a graph gap rather than a misleading average.
|
|
|
same_run = bool(previous and previous.generation == raw.generation
|
|
|
and previous.vcpus == raw.vcpus
|
|
|
and 0 < seconds <= SAMPLE_INTERVAL_SECONDS * 3)
|
|
|
cpu_percent = None
|
|
|
if (same_run and cpu_time is not None and previous.cpu_time is not None
|
|
|
and raw.vcpus > 0 and cpu_time >= previous.cpu_time):
|
|
|
fraction = (cpu_time - previous.cpu_time) / (seconds * 1e9 * raw.vcpus)
|
|
|
# libvirt's total CPU time can include QEMU auxiliary threads, so
|
|
|
# a healthy sample can exceed the allotted vCPU time. The chart is
|
|
|
# defined as 0–100% of VM vCPU capacity.
|
|
|
cpu_percent = min(100.0, fraction * 100)
|
|
|
disk_same = same_run and bool(disk_signature) and disk_signature == previous.disk_signature
|
|
|
network_same = (same_run and bool(network_signature)
|
|
|
and network_signature == previous.network_signature)
|
|
|
point = {
|
|
|
"vm_id": vm_id,
|
|
|
"timestamp": timestamp,
|
|
|
"cpu_percent": cpu_percent,
|
|
|
"memory_used_mib": memory_used,
|
|
|
"memory_total_mib": memory_total,
|
|
|
"disk_read_bps": self._rate(disk["rd"], previous.disk["rd"] if previous else None,
|
|
|
seconds, disk_same),
|
|
|
"disk_write_bps": self._rate(disk["wr"], previous.disk["wr"] if previous else None,
|
|
|
seconds, disk_same),
|
|
|
"network_rx_bps": self._rate(network["rx"], previous.network["rx"] if previous else None,
|
|
|
seconds, network_same),
|
|
|
"network_tx_bps": self._rate(network["tx"], previous.network["tx"] if previous else None,
|
|
|
seconds, network_same),
|
|
|
}
|
|
|
self._previous[key] = _Baseline(monotonic, raw.generation, raw.vcpus, cpu_time,
|
|
|
disk_signature, disk, network_signature, network)
|
|
|
return point
|
|
|
|
|
|
def forget_unseen(self, scope: str, seen: set[str]) -> None:
|
|
|
for key in list(self._previous):
|
|
|
if key[0] == scope and key[1] not in seen:
|
|
|
del self._previous[key]
|
|
|
|
|
|
|
|
|
class MetricsCollector:
|
|
|
"""One daemon sampler owned by the single Uvicorn worker's lifespan."""
|
|
|
|
|
|
def __init__(self, store_getter: Callable[[], Any],
|
|
|
profile_getter: Callable[[], Mapping[str, Any] | None],
|
|
|
service_factory: Callable[[Mapping[str, Any]], Any],
|
|
|
*, interval_seconds: int = SAMPLE_INTERVAL_SECONDS) -> None:
|
|
|
self._store_getter = store_getter
|
|
|
self._profile_getter = profile_getter
|
|
|
self._service_factory = service_factory
|
|
|
self._interval_seconds = interval_seconds
|
|
|
self._sampler = MetricSampler()
|
|
|
self._stop = threading.Event()
|
|
|
self._thread: threading.Thread | None = None
|
|
|
self._prune_signal = threading.Event()
|
|
|
self._prune_thread: threading.Thread | None = None
|
|
|
|
|
|
def sample_once(self, *, timestamp: int | None = None,
|
|
|
monotonic: float | None = None) -> None:
|
|
|
store = self._store_getter()
|
|
|
if store is None or not hasattr(store, "prune_metric_samples"):
|
|
|
return
|
|
|
timestamp = int(time.time()) if timestamp is None else timestamp
|
|
|
try:
|
|
|
profile = self._profile_getter()
|
|
|
if profile is None or profile.get("key_locked"):
|
|
|
return
|
|
|
scope = host_scope(profile)
|
|
|
rows = self._service_factory(profile).metric_counters()
|
|
|
monotonic = time.monotonic() if monotonic is None else monotonic
|
|
|
points = [self._sampler.observe(scope, row, timestamp=timestamp,
|
|
|
monotonic=monotonic) for row in rows]
|
|
|
self._sampler.forget_unseen(scope, {str(uuid.UUID(row.vm_id)) for row in rows})
|
|
|
if points:
|
|
|
store.save_metric_samples(scope, points)
|
|
|
finally:
|
|
|
# Signal after sample/save (or a failed host access). The separate
|
|
|
# cleanup thread never delays the next libvirt measurement.
|
|
|
self._prune_signal.set()
|
|
|
|
|
|
def _prune_loop(self) -> None:
|
|
|
while not self._stop.is_set():
|
|
|
self._prune_signal.wait(timeout=self._interval_seconds)
|
|
|
self._prune_signal.clear()
|
|
|
if self._stop.is_set():
|
|
|
break
|
|
|
store = self._store_getter()
|
|
|
if store is None or not hasattr(store, "prune_metric_samples"):
|
|
|
continue
|
|
|
while not self._stop.is_set():
|
|
|
try:
|
|
|
removed = store.prune_metric_samples(
|
|
|
int(time.time()) - RETENTION_SECONDS)
|
|
|
except Exception as exc:
|
|
|
# Never echo database driver errors or connection details.
|
|
|
LOG.warning("VM metric retention cleanup failed: %s",
|
|
|
type(exc).__name__)
|
|
|
break
|
|
|
if not isinstance(removed, int) or removed < 5000:
|
|
|
break
|
|
|
# Sustain catch-up without occupying MariaDB continuously.
|
|
|
if self._stop.wait(0.1):
|
|
|
break
|
|
|
|
|
|
def _run(self) -> None:
|
|
|
while not self._stop.is_set():
|
|
|
started = time.monotonic()
|
|
|
try:
|
|
|
self.sample_once()
|
|
|
except Exception as exc:
|
|
|
# Never print libvirt/DB exception text: it can contain a URI,
|
|
|
# account name, or other connection details.
|
|
|
LOG.warning("VM metric collection failed: %s", type(exc).__name__)
|
|
|
self._stop.wait(max(0, self._interval_seconds - (time.monotonic() - started)))
|
|
|
|
|
|
def start(self) -> None:
|
|
|
if self._thread is not None:
|
|
|
return
|
|
|
self._prune_thread = threading.Thread(target=self._prune_loop, daemon=True,
|
|
|
name="vm-metrics-retention")
|
|
|
self._prune_thread.start()
|
|
|
self._thread = threading.Thread(target=self._run, daemon=True,
|
|
|
name="vm-metrics-collector")
|
|
|
self._thread.start()
|
|
|
|
|
|
def close(self) -> None:
|
|
|
self._stop.set()
|
|
|
self._prune_signal.set()
|
|
|
deadline = time.monotonic() + 15
|
|
|
for worker in (self._thread, self._prune_thread):
|
|
|
if worker is not None:
|
|
|
worker.join(timeout=max(0, deadline - time.monotonic()))
|