You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/virsh_panel/metrics.py

277 lines
12 KiB
Python

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

"""Periodic libvirt VM measurements and safe counter-to-rate conversion.
The collector runs independently of browser visits. A process restart deliberately
loses counter baselines: the first subsequent rate sample is unknown, not zero.
"""
from __future__ import annotations
import hashlib
import json
import logging
import math
import re
import threading
import time
import uuid
from dataclasses import dataclass
from typing import Any, Callable, Mapping
LOG = logging.getLogger(__name__)
SAMPLE_INTERVAL_SECONDS = 60
RETENTION_SECONDS = 30 * 24 * 60 * 60
MAX_QUERY_POINTS = 600
RANGES = {"1h": 3600, "24h": 24 * 3600, "7d": 7 * 24 * 3600,
"30d": RETENTION_SECONDS}
_COUNTER = re.compile(r"^(block|net)\.(\d+)\.(rd|wr|rx|tx)\.bytes$")
_MEMORY_MAX_AGE_SECONDS = 180
def host_scope(profile: Mapping[str, Any]) -> str:
"""Keep VM history when pool settings change, but isolate a new SSH host key."""
identity = [profile[key] for key in ("host", "port", "fingerprint")]
if (not isinstance(identity[0], str) or not isinstance(identity[1], int)
or not isinstance(identity[2], str)):
raise ValueError("Host identity is invalid")
raw = json.dumps(identity, ensure_ascii=False, separators=(",", ":"))
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
def query_interval_seconds(window_seconds: int) -> int:
"""Epoch-aligned buckets cannot exceed MAX_QUERY_POINTS in any range."""
if window_seconds <= 0:
raise ValueError("Metric range must be positive")
return max(SAMPLE_INTERVAL_SECONDS,
math.ceil(window_seconds / (MAX_QUERY_POINTS - 1)
/ SAMPLE_INTERVAL_SECONDS) * SAMPLE_INTERVAL_SECONDS)
def _nonnegative_int(value: Any) -> int | None:
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
return None
return value
def _device_counters(stats: Mapping[str, Any], group: str,
directions: tuple[str, str]) -> tuple[tuple[tuple[str, str], ...],
dict[str, dict[str, int] | None]]:
count = _nonnegative_int(stats.get(f"{group}.count"))
devices: dict[str, dict[str, int]] = {
str(number): {} for number in range(min(count, 1024))
} if count is not None else {}
for key, value in stats.items():
match = _COUNTER.fullmatch(key)
if not match or match.group(1) != group or match.group(3) not in directions:
continue
devices.setdefault(match.group(2), {})
valid = _nonnegative_int(value)
if valid is not None:
devices[match.group(2)][match.group(3)] = valid
signature = tuple(sorted(
(number, str(stats.get(f"{group}.{number}.name", number)))
for number in devices
))
counters_by_direction: dict[str, dict[str, int] | None] = {}
for direction in directions:
counters = {number: fields[direction] for number, fields in devices.items()
if direction in fields}
# CD-ROM and unsupported devices may never expose byte counters. Use
# only reporting devices, but _rate() resets if that coverage changes.
counters_by_direction[direction] = counters or None
return signature, counters_by_direction
def _memory_mib(stats: Mapping[str, Any], sampled_at: int) -> tuple[float | None,
float | None]:
available = _nonnegative_int(stats.get("balloon.available"))
usable = _nonnegative_int(stats.get("balloon.usable"))
last_update = _nonnegative_int(stats.get("balloon.last-update"))
# The QEMU RSS and balloon.current fields describe host process/allocation,
# not guest-used memory. Only guest balloon statistics can support used MiB.
if (available is None or usable is None or last_update is None
or last_update > sampled_at + SAMPLE_INTERVAL_SECONDS
or sampled_at - last_update > _MEMORY_MAX_AGE_SECONDS
or usable > available):
return None, None
return (available - usable) / 1024, available / 1024
@dataclass(frozen=True)
class RawVMStats:
vm_id: str
generation: int
vcpus: int
values: Mapping[str, Any]
@dataclass(frozen=True)
class _Baseline:
monotonic: float
generation: int
vcpus: int
cpu_time: int | None
disk_signature: tuple[tuple[str, str], ...]
disk: Mapping[str, Mapping[str, int] | None]
network_signature: tuple[tuple[str, str], ...]
network: Mapping[str, Mapping[str, int] | None]
class MetricSampler:
def __init__(self) -> None:
self._previous: dict[tuple[str, str], _Baseline] = {}
@staticmethod
def _rate(current: Mapping[str, int] | None,
previous: Mapping[str, int] | None,
seconds: float, same_devices: bool) -> float | None:
if (not same_devices or current is None or previous is None
or seconds <= 0 or current.keys() != previous.keys()
or any(value < previous[name] for name, value in current.items())):
return None
return sum(value - previous[name] for name, value in current.items()) / seconds
def observe(self, scope: str, raw: RawVMStats, *, timestamp: int,
monotonic: float) -> dict[str, Any]:
vm_id = str(uuid.UUID(raw.vm_id))
values = raw.values
cpu_time = _nonnegative_int(values.get("cpu.time"))
disk_signature, disk = _device_counters(values, "block", ("rd", "wr"))
network_signature, network = _device_counters(values, "net", ("rx", "tx"))
memory_used, memory_total = _memory_mib(values, timestamp)
key = (scope, vm_id)
previous = self._previous.get(key)
seconds = monotonic - previous.monotonic if previous else 0
# A long pause means a graph gap rather than a misleading average.
same_run = bool(previous and previous.generation == raw.generation
and previous.vcpus == raw.vcpus
and 0 < seconds <= SAMPLE_INTERVAL_SECONDS * 3)
cpu_percent = None
if (same_run and cpu_time is not None and previous.cpu_time is not None
and raw.vcpus > 0 and cpu_time >= previous.cpu_time):
fraction = (cpu_time - previous.cpu_time) / (seconds * 1e9 * raw.vcpus)
# libvirt's total CPU time can include QEMU auxiliary threads, so
# a healthy sample can exceed the allotted vCPU time. The chart is
# defined as 0–100% of VM vCPU capacity.
cpu_percent = min(100.0, fraction * 100)
disk_same = same_run and bool(disk_signature) and disk_signature == previous.disk_signature
network_same = (same_run and bool(network_signature)
and network_signature == previous.network_signature)
point = {
"vm_id": vm_id,
"timestamp": timestamp,
"cpu_percent": cpu_percent,
"memory_used_mib": memory_used,
"memory_total_mib": memory_total,
"disk_read_bps": self._rate(disk["rd"], previous.disk["rd"] if previous else None,
seconds, disk_same),
"disk_write_bps": self._rate(disk["wr"], previous.disk["wr"] if previous else None,
seconds, disk_same),
"network_rx_bps": self._rate(network["rx"], previous.network["rx"] if previous else None,
seconds, network_same),
"network_tx_bps": self._rate(network["tx"], previous.network["tx"] if previous else None,
seconds, network_same),
}
self._previous[key] = _Baseline(monotonic, raw.generation, raw.vcpus, cpu_time,
disk_signature, disk, network_signature, network)
return point
def forget_unseen(self, scope: str, seen: set[str]) -> None:
for key in list(self._previous):
if key[0] == scope and key[1] not in seen:
del self._previous[key]
class MetricsCollector:
"""One daemon sampler owned by the single Uvicorn worker's lifespan."""
def __init__(self, store_getter: Callable[[], Any],
profile_getter: Callable[[], Mapping[str, Any] | None],
service_factory: Callable[[Mapping[str, Any]], Any],
*, interval_seconds: int = SAMPLE_INTERVAL_SECONDS) -> None:
self._store_getter = store_getter
self._profile_getter = profile_getter
self._service_factory = service_factory
self._interval_seconds = interval_seconds
self._sampler = MetricSampler()
self._stop = threading.Event()
self._thread: threading.Thread | None = None
self._prune_signal = threading.Event()
self._prune_thread: threading.Thread | None = None
def sample_once(self, *, timestamp: int | None = None,
monotonic: float | None = None) -> None:
store = self._store_getter()
if store is None or not hasattr(store, "prune_metric_samples"):
return
timestamp = int(time.time()) if timestamp is None else timestamp
try:
profile = self._profile_getter()
if profile is None or profile.get("key_locked"):
return
scope = host_scope(profile)
rows = self._service_factory(profile).metric_counters()
monotonic = time.monotonic() if monotonic is None else monotonic
points = [self._sampler.observe(scope, row, timestamp=timestamp,
monotonic=monotonic) for row in rows]
self._sampler.forget_unseen(scope, {str(uuid.UUID(row.vm_id)) for row in rows})
if points:
store.save_metric_samples(scope, points)
finally:
# Signal after sample/save (or a failed host access). The separate
# cleanup thread never delays the next libvirt measurement.
self._prune_signal.set()
def _prune_loop(self) -> None:
while not self._stop.is_set():
self._prune_signal.wait(timeout=self._interval_seconds)
self._prune_signal.clear()
if self._stop.is_set():
break
store = self._store_getter()
if store is None or not hasattr(store, "prune_metric_samples"):
continue
while not self._stop.is_set():
try:
removed = store.prune_metric_samples(
int(time.time()) - RETENTION_SECONDS)
except Exception as exc:
# Never echo database driver errors or connection details.
LOG.warning("VM metric retention cleanup failed: %s",
type(exc).__name__)
break
if not isinstance(removed, int) or removed < 5000:
break
# Sustain catch-up without occupying MariaDB continuously.
if self._stop.wait(0.1):
break
def _run(self) -> None:
while not self._stop.is_set():
started = time.monotonic()
try:
self.sample_once()
except Exception as exc:
# Never print libvirt/DB exception text: it can contain a URI,
# account name, or other connection details.
LOG.warning("VM metric collection failed: %s", type(exc).__name__)
self._stop.wait(max(0, self._interval_seconds - (time.monotonic() - started)))
def start(self) -> None:
if self._thread is not None:
return
self._prune_thread = threading.Thread(target=self._prune_loop, daemon=True,
name="vm-metrics-retention")
self._prune_thread.start()
self._thread = threading.Thread(target=self._run, daemon=True,
name="vm-metrics-collector")
self._thread.start()
def close(self) -> None:
self._stop.set()
self._prune_signal.set()
deadline = time.monotonic() + 15
for worker in (self._thread, self._prune_thread):
if worker is not None:
worker.join(timeout=max(0, deadline - time.monotonic()))