You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/tests/test_metrics.py

471 lines
17 KiB
Python

"""VM metric math, DB persistence, and the process-local collection lifecycle."""
from __future__ import annotations
import threading
import pytest
from virsh_panel.database import DatabaseStore
from virsh_panel.metrics import (
MAX_QUERY_POINTS, RANGES, MetricSampler, MetricsCollector, RawVMStats,
host_scope, query_interval_seconds,
)
VM_ID = "00000000-0000-4000-8000-000000000001"
HOST = {"host": "hypervisor.example", "port": 2222,
"fingerprint": "SHA256:one", "storage_pool": "/home/kvm"}
def raw(*, generation=12, cpu=1_000_000_000, block_read=100,
block_write=200, network_rx=1000, network_tx=2000,
memory_time=1000, extra=None):
values = {
"cpu.time": cpu,
"balloon.available": 2 * 1024 * 1024,
"balloon.usable": 512 * 1024,
"balloon.last-update": memory_time,
"block.count": 1,
"block.0.name": "vda",
"block.0.rd.bytes": block_read,
"block.0.wr.bytes": block_write,
"net.count": 1,
"net.0.name": "vnet1",
"net.0.rx.bytes": network_rx,
"net.0.tx.bytes": network_tx,
}
values.update(extra or {})
return RawVMStats(VM_ID, generation, 2, values)
def test_host_scope_ignores_pool_but_changes_with_ssh_host_identity():
first = host_scope(HOST)
assert first == host_scope({**HOST, "storage_pool": "/different"})
assert first != host_scope({**HOST, "fingerprint": "SHA256:two"})
assert first != host_scope({**HOST, "port": 22})
def test_interval_caps_all_ranges_at_600_epoch_aligned_buckets():
for seconds in RANGES.values():
interval = query_interval_seconds(seconds)
assert interval % 60 == 0
assert seconds // interval + 2 <= MAX_QUERY_POINTS
def test_initial_sample_and_fresh_guest_memory_then_rates():
sampler = MetricSampler()
scope = host_scope(HOST)
first = sampler.observe(scope, raw(), timestamp=1000, monotonic=10)
assert first["cpu_percent"] is None
assert first["disk_read_bps"] is None
assert first["network_rx_bps"] is None
assert first["memory_used_mib"] == 1536
assert first["memory_total_mib"] == 2048
second = sampler.observe(
scope,
raw(cpu=61_000_000_000, block_read=700, block_write=1400,
network_rx=2200, network_tx=2600, memory_time=1050),
timestamp=1060, monotonic=70,
)
assert second["cpu_percent"] == 50
assert second["disk_read_bps"] == 10
assert second["disk_write_bps"] == 20
assert second["network_rx_bps"] == 20
assert second["network_tx_bps"] == 10
def test_cpu_time_over_vcpu_capacity_is_displayed_as_100_percent():
sampler = MetricSampler()
scope = host_scope(HOST)
sampler.observe(scope, raw(), timestamp=1000, monotonic=10)
# 130 CPU-seconds across two vCPUs in a 60-second interval; libvirt can
# include QEMU auxiliary threads in cpu.time.
point = sampler.observe(scope, raw(cpu=131_000_000_000),
timestamp=1060, monotonic=70)
assert point["cpu_percent"] == 100
def test_memory_missing_or_stale_balloon_stats_never_uses_rss_or_current():
sampler = MetricSampler()
scope = host_scope(HOST)
values = {"balloon.current": 2 * 1024 * 1024, "balloon.rss": 512 * 1024}
for extra in (
{"balloon.last-update": 1},
{"balloon.usable": None},
{"balloon.available": None},
):
point = sampler.observe(scope, raw(extra={**values, **extra}),
timestamp=1000, monotonic=10)
assert point["memory_used_mib"] is None
assert point["memory_total_mib"] is None
def test_counter_reset_reboot_device_change_and_partial_data_are_unknown():
sampler = MetricSampler()
scope = host_scope(HOST)
sampler.observe(scope, raw(), timestamp=1000, monotonic=10)
reset = sampler.observe(scope, raw(cpu=100, block_read=1, network_rx=1),
timestamp=1060, monotonic=70)
assert reset["cpu_percent"] is None
assert reset["disk_read_bps"] is None
assert reset["network_rx_bps"] is None
# Even if a second disk masks the first disk's decreasing counter in the
# aggregate, device change must not produce a transient spike.
changed = sampler.observe(scope, raw(extra={
"block.count": 2, "block.1.name": "vdb",
"block.1.rd.bytes": 10_000, "block.1.wr.bytes": 20_000,
}), timestamp=1120, monotonic=130)
assert changed["disk_read_bps"] is None
assert changed["disk_write_bps"] is None
partial = sampler.observe(scope, raw(extra={
"block.count": 2, "block.1.name": "vdb",
"block.1.rd.bytes": 11_000,
}), timestamp=1180, monotonic=190)
assert partial["disk_write_bps"] is None # second device missing wr.bytes
assert partial["disk_read_bps"] is not None
reboot = sampler.observe(scope, raw(generation=13, cpu=200_000_000_000),
timestamp=1240, monotonic=250)
assert reboot["cpu_percent"] is None
assert reboot["network_tx_bps"] is None
def test_cdrom_without_byte_counters_does_not_hide_disk_rates():
sampler = MetricSampler()
scope = host_scope(HOST)
cdrom = {"block.count": 2, "block.1.name": "sda"}
sampler.observe(scope, raw(extra=cdrom), timestamp=1000, monotonic=10)
second = sampler.observe(
scope, raw(block_read=700, block_write=1400, extra={
**cdrom, "balloon.last-update": 1060,
}), timestamp=1060, monotonic=70,
)
assert second["disk_read_bps"] == 10
assert second["disk_write_bps"] == 20
def test_single_device_counter_rollback_is_not_masked_by_another_device():
sampler = MetricSampler()
scope = host_scope(HOST)
second_disk = {"block.count": 2, "block.1.name": "vdb",
"block.1.rd.bytes": 1_000, "block.1.wr.bytes": 1_000}
sampler.observe(scope, raw(extra=second_disk), timestamp=1000, monotonic=10)
point = sampler.observe(scope, raw(block_read=1, block_write=1, extra={
**second_disk, "block.1.rd.bytes": 10_000,
"block.1.wr.bytes": 10_000,
}), timestamp=1060, monotonic=70)
assert point["disk_read_bps"] is None
assert point["disk_write_bps"] is None
def test_unobserved_long_gap_is_not_averaged_as_current_rate():
sampler = MetricSampler()
scope = host_scope(HOST)
sampler.observe(scope, raw(), timestamp=1000, monotonic=10)
delayed = sampler.observe(scope, raw(cpu=50_000_000_000, block_read=2000),
timestamp=1600, monotonic=610)
assert delayed["cpu_percent"] is None
assert delayed["disk_read_bps"] is None
class FakeCursor:
def __init__(self, rows=(), delete_counts=()):
self.rows = rows
self.commands = []
self.delete_counts = list(delete_counts)
self.rowcount = 0
def __enter__(self):
return self
def __exit__(self, *_args):
return None
def execute(self, sql, args=None):
self.commands.append((sql, args))
self.rowcount = (self.delete_counts.pop(0)
if sql.startswith("DELETE FROM vm_metric_samples")
and self.delete_counts else 0)
def executemany(self, sql, args):
self.commands.append((sql, args))
def fetchall(self):
return self.rows
class FakeConnection:
def __init__(self, rows=(), delete_counts=()):
self.fake_cursor = FakeCursor(rows, delete_counts)
self.commits = 0
self.closed = False
def cursor(self):
return self.fake_cursor
def commit(self):
self.commits += 1
def rollback(self):
raise AssertionError("unexpected rollback")
def close(self):
self.closed = True
def test_metric_and_user_tables_created_with_record_table():
connection = FakeConnection()
store = DatabaseStore.__new__(DatabaseStore)
store._connect = lambda: connection
store._initialize_schema()
assert len(connection.fake_cursor.commands) == 6
assert "vm_metric_samples" in connection.fake_cursor.commands[1][0]
assert "vm_metric_retention" in connection.fake_cursor.commands[1][0]
assert "CREATE TABLE IF NOT EXISTS users" in connection.fake_cursor.commands[2][0]
assert "CREATE TABLE IF NOT EXISTS registration_codes" in connection.fake_cursor.commands[3][0]
assert "CREATE TABLE IF NOT EXISTS vm_ownership" in connection.fake_cursor.commands[4][0]
assert "CREATE TABLE IF NOT EXISTS user_ssh_keys" in connection.fake_cursor.commands[5][0]
assert connection.commits == 1 and connection.closed
def test_metric_store_prunes_physically_and_queries_only_scoped_recent_data():
scope = host_scope(HOST)
connection = FakeConnection([{
"timestamp": 1000,
"cpu_percent": 12.5,
"memory_used_mib": 1024,
"memory_total_mib": 2048,
"disk_read_bps": None,
"disk_write_bps": None,
"network_rx_bps": 256,
"network_tx_bps": 0,
}])
store = DatabaseStore.__new__(DatabaseStore)
store._connect = lambda: connection
point = {"vm_id": VM_ID, "timestamp": 1000, "cpu_percent": 12.5,
"memory_used_mib": 1024, "memory_total_mib": 2048,
"disk_read_bps": None, "disk_write_bps": None,
"network_rx_bps": 256, "network_tx_bps": 0}
store.save_metric_samples(scope, [point])
store.prune_metric_samples(500)
result = store.query_metric_samples(scope, VM_ID, since=500, until=1100,
interval_seconds=60, limit=600)
assert result[0]["cpu_percent"] == 12.5
assert result[0]["disk_read_bps"] is None
insert, delete, query = connection.fake_cursor.commands
assert "INSERT INTO vm_metric_samples" in insert[0]
assert "DELETE FROM vm_metric_samples WHERE sampled_at < %s" in delete[0]
assert delete[1] == (500, 5000)
assert "host_scope=%s AND vm_uuid=%s AND sampled_at >= %s" in query[0]
assert "LIMIT %s" in query[0]
assert query[1] == (scope, VM_ID, 500, 1100, 60, 600)
assert connection.commits == 2 and connection.closed
def test_metric_prune_is_one_index_ordered_batch_per_transaction():
connection = FakeConnection(delete_counts=[5000] * 10)
store = DatabaseStore.__new__(DatabaseStore)
store._connect = lambda: connection
assert store.prune_metric_samples(1234) == 5000
assert len(connection.fake_cursor.commands) == 1
assert connection.commits == 1 and connection.closed
assert all("ORDER BY sampled_at LIMIT %s" in command
and args == (1234, 5000)
for command, args in connection.fake_cursor.commands)
def test_metric_prune_reports_short_final_batch():
connection = FakeConnection(delete_counts=[275])
store = DatabaseStore.__new__(DatabaseStore)
store._connect = lambda: connection
assert store.prune_metric_samples(1234) == 275
assert len(connection.fake_cursor.commands) == 1
assert connection.commits == 1
def test_metric_store_rejects_invalid_host_and_uuid_before_db_query():
store = DatabaseStore.__new__(DatabaseStore)
store._connect = lambda: pytest.fail("invalid scope must not query database")
with pytest.raises(ValueError):
store.query_metric_samples("bad-scope", VM_ID, since=0, until=1,
interval_seconds=60)
with pytest.raises(ValueError):
store.query_metric_samples(host_scope(HOST), "not-uuid", since=0, until=1,
interval_seconds=60)
def test_collector_prunes_when_ssh_locked_and_starts_after_db_configuration():
class Store:
def __init__(self):
self.pruned = []
self.saved = []
self.pruned_event = threading.Event()
self.saved_event = threading.Event()
def prune_metric_samples(self, cutoff):
self.pruned.append(cutoff)
self.pruned_event.set()
return 0
def save_metric_samples(self, scope, points):
self.saved.append((scope, points))
self.saved_event.set()
current = {"store": None, "profile": None}
store = Store()
class Service:
def metric_counters(self):
return [raw()]
collector = MetricsCollector(lambda: current["store"],
lambda: current["profile"],
lambda _profile: Service(), interval_seconds=0.01)
collector.start()
current["profile"] = {**HOST, "key_locked": True}
current["store"] = store
try:
assert store.pruned_event.wait(timeout=2)
assert store.pruned and not store.saved
current["profile"]["key_locked"] = False
assert store.saved_event.wait(timeout=2)
assert store.saved
finally:
collector.close()
def test_background_collector_keeps_sampling_without_an_http_session():
"""The 30-day history must not depend on a logged-in browser polling it."""
saved_twice = threading.Event()
class Store:
def __init__(self):
self.saved = []
def save_metric_samples(self, scope, points):
self.saved.append((scope, points))
if len(self.saved) >= 2:
saved_twice.set()
def prune_metric_samples(self, _cutoff):
return 0
class Service:
calls = 0
def metric_counters(self):
self.calls += 1
return [raw(cpu=self.calls * 1_000_000_000,
block_read=self.calls * 1000,
network_rx=self.calls * 1000)]
store = Store()
service = Service()
profile = {**HOST, "key_locked": False}
collector = MetricsCollector(lambda: store, lambda: profile,
lambda _profile: service,
interval_seconds=0.01)
collector.start()
try:
assert saved_twice.wait(timeout=2)
finally:
collector.close()
assert service.calls >= 2
assert all(scope == host_scope(HOST) for scope, _ in store.saved)
assert all(points[0]["vm_id"] == VM_ID for _, points in store.saved)
def test_metric_cleanup_failure_does_not_discard_current_sample(caplog):
events = []
attempted = threading.Event()
class Store:
def save_metric_samples(self, _scope, points):
events.append(("saved", points))
def prune_metric_samples(self, _cutoff):
events.append(("prune", None))
attempted.set()
raise RuntimeError("secret database details")
class Service:
def metric_counters(self):
return [raw()]
collector = MetricsCollector(lambda: Store(), lambda: HOST,
lambda _profile: Service(), interval_seconds=1)
collector.start()
assert attempted.wait(timeout=2)
collector.close()
assert [event[0] for event in events] == ["saved", "prune"]
assert "RuntimeError" in caplog.text
assert "secret database details" not in caplog.text
def test_backlog_cleanup_continues_in_other_thread_without_stalling_samples():
second_saved = threading.Event()
prune_entered = threading.Event()
release_prune = threading.Event()
backlog_drained = threading.Event()
class Store:
def __init__(self):
self.saved = 0
self.pruned = 0
def save_metric_samples(self, _scope, _points):
self.saved += 1
if self.saved >= 2:
second_saved.set()
def prune_metric_samples(self, _cutoff):
self.pruned += 1
if self.pruned == 1:
prune_entered.set()
assert release_prune.wait(timeout=2)
if self.pruned < 3:
return 5000
backlog_drained.set()
return 0
class Service:
def metric_counters(self):
return [raw()]
store = Store()
collector = MetricsCollector(lambda: store, lambda: HOST,
lambda _profile: Service(), interval_seconds=0.01)
collector.start()
try:
assert prune_entered.wait(timeout=2)
assert second_saved.wait(timeout=2)
release_prune.set()
assert backlog_drained.wait(timeout=2)
assert store.pruned >= 3
finally:
release_prune.set()
collector.close()
def test_collector_thread_starts_before_db_configuration_and_stops():
ready = threading.Event()
current = {"store": None}
class Store:
def prune_metric_samples(self, _cutoff):
ready.set()
return 0
collector = MetricsCollector(lambda: current["store"], lambda: None,
lambda _profile: pytest.fail("no host configured"),
interval_seconds=0.01)
collector.start()
current["store"] = Store()
assert ready.wait(timeout=2)
collector.close()
assert collector._thread is not None and not collector._thread.is_alive()
assert collector._prune_thread is not None and not collector._prune_thread.is_alive()