Add 30-day VM performance history and detail charts
parent
ff29462f2a
commit
0dc36cefc2
@ -0,0 +1,424 @@
|
||||
"""VM metric math, DB persistence, and the process-local collection lifecycle."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
|
||||
import pytest
|
||||
|
||||
from virsh_panel.database import DatabaseStore
|
||||
from virsh_panel.metrics import (
|
||||
MAX_QUERY_POINTS, RANGES, MetricSampler, MetricsCollector, RawVMStats,
|
||||
host_scope, query_interval_seconds,
|
||||
)
|
||||
|
||||
|
||||
VM_ID = "00000000-0000-4000-8000-000000000001"
|
||||
HOST = {"host": "hypervisor.example", "port": 2222,
|
||||
"fingerprint": "SHA256:one", "storage_pool": "/home/kvm"}
|
||||
|
||||
|
||||
def raw(*, generation=12, cpu=1_000_000_000, block_read=100,
|
||||
block_write=200, network_rx=1000, network_tx=2000,
|
||||
memory_time=1000, extra=None):
|
||||
values = {
|
||||
"cpu.time": cpu,
|
||||
"balloon.available": 2 * 1024 * 1024,
|
||||
"balloon.usable": 512 * 1024,
|
||||
"balloon.last-update": memory_time,
|
||||
"block.count": 1,
|
||||
"block.0.name": "vda",
|
||||
"block.0.rd.bytes": block_read,
|
||||
"block.0.wr.bytes": block_write,
|
||||
"net.count": 1,
|
||||
"net.0.name": "vnet1",
|
||||
"net.0.rx.bytes": network_rx,
|
||||
"net.0.tx.bytes": network_tx,
|
||||
}
|
||||
values.update(extra or {})
|
||||
return RawVMStats(VM_ID, generation, 2, values)
|
||||
|
||||
|
||||
def test_host_scope_ignores_pool_but_changes_with_ssh_host_identity():
|
||||
first = host_scope(HOST)
|
||||
assert first == host_scope({**HOST, "storage_pool": "/different"})
|
||||
assert first != host_scope({**HOST, "fingerprint": "SHA256:two"})
|
||||
assert first != host_scope({**HOST, "port": 22})
|
||||
|
||||
|
||||
def test_interval_caps_all_ranges_at_600_epoch_aligned_buckets():
|
||||
for seconds in RANGES.values():
|
||||
interval = query_interval_seconds(seconds)
|
||||
assert interval % 60 == 0
|
||||
assert seconds // interval + 2 <= MAX_QUERY_POINTS
|
||||
|
||||
|
||||
def test_initial_sample_and_fresh_guest_memory_then_rates():
|
||||
sampler = MetricSampler()
|
||||
scope = host_scope(HOST)
|
||||
first = sampler.observe(scope, raw(), timestamp=1000, monotonic=10)
|
||||
assert first["cpu_percent"] is None
|
||||
assert first["disk_read_bps"] is None
|
||||
assert first["network_rx_bps"] is None
|
||||
assert first["memory_used_mib"] == 1536
|
||||
assert first["memory_total_mib"] == 2048
|
||||
|
||||
second = sampler.observe(
|
||||
scope,
|
||||
raw(cpu=61_000_000_000, block_read=700, block_write=1400,
|
||||
network_rx=2200, network_tx=2600, memory_time=1050),
|
||||
timestamp=1060, monotonic=70,
|
||||
)
|
||||
assert second["cpu_percent"] == 50
|
||||
assert second["disk_read_bps"] == 10
|
||||
assert second["disk_write_bps"] == 20
|
||||
assert second["network_rx_bps"] == 20
|
||||
assert second["network_tx_bps"] == 10
|
||||
|
||||
|
||||
def test_cpu_time_over_vcpu_capacity_is_displayed_as_100_percent():
|
||||
sampler = MetricSampler()
|
||||
scope = host_scope(HOST)
|
||||
sampler.observe(scope, raw(), timestamp=1000, monotonic=10)
|
||||
# 130 CPU-seconds across two vCPUs in a 60-second interval; libvirt can
|
||||
# include QEMU auxiliary threads in cpu.time.
|
||||
point = sampler.observe(scope, raw(cpu=131_000_000_000),
|
||||
timestamp=1060, monotonic=70)
|
||||
assert point["cpu_percent"] == 100
|
||||
|
||||
|
||||
def test_memory_missing_or_stale_balloon_stats_never_uses_rss_or_current():
|
||||
sampler = MetricSampler()
|
||||
scope = host_scope(HOST)
|
||||
values = {"balloon.current": 2 * 1024 * 1024, "balloon.rss": 512 * 1024}
|
||||
for extra in (
|
||||
{"balloon.last-update": 1},
|
||||
{"balloon.usable": None},
|
||||
{"balloon.available": None},
|
||||
):
|
||||
point = sampler.observe(scope, raw(extra={**values, **extra}),
|
||||
timestamp=1000, monotonic=10)
|
||||
assert point["memory_used_mib"] is None
|
||||
assert point["memory_total_mib"] is None
|
||||
|
||||
|
||||
def test_counter_reset_reboot_device_change_and_partial_data_are_unknown():
|
||||
sampler = MetricSampler()
|
||||
scope = host_scope(HOST)
|
||||
sampler.observe(scope, raw(), timestamp=1000, monotonic=10)
|
||||
reset = sampler.observe(scope, raw(cpu=100, block_read=1, network_rx=1),
|
||||
timestamp=1060, monotonic=70)
|
||||
assert reset["cpu_percent"] is None
|
||||
assert reset["disk_read_bps"] is None
|
||||
assert reset["network_rx_bps"] is None
|
||||
|
||||
# Even if a second disk masks the first disk's decreasing counter in the
|
||||
# aggregate, device change must not produce a transient spike.
|
||||
changed = sampler.observe(scope, raw(extra={
|
||||
"block.count": 2, "block.1.name": "vdb",
|
||||
"block.1.rd.bytes": 10_000, "block.1.wr.bytes": 20_000,
|
||||
}), timestamp=1120, monotonic=130)
|
||||
assert changed["disk_read_bps"] is None
|
||||
assert changed["disk_write_bps"] is None
|
||||
|
||||
partial = sampler.observe(scope, raw(extra={
|
||||
"block.count": 2, "block.1.name": "vdb",
|
||||
"block.1.rd.bytes": 11_000,
|
||||
}), timestamp=1180, monotonic=190)
|
||||
assert partial["disk_write_bps"] is None # second device missing wr.bytes
|
||||
assert partial["disk_read_bps"] is not None
|
||||
|
||||
reboot = sampler.observe(scope, raw(generation=13, cpu=200_000_000_000),
|
||||
timestamp=1240, monotonic=250)
|
||||
assert reboot["cpu_percent"] is None
|
||||
assert reboot["network_tx_bps"] is None
|
||||
|
||||
|
||||
def test_cdrom_without_byte_counters_does_not_hide_disk_rates():
|
||||
sampler = MetricSampler()
|
||||
scope = host_scope(HOST)
|
||||
cdrom = {"block.count": 2, "block.1.name": "sda"}
|
||||
sampler.observe(scope, raw(extra=cdrom), timestamp=1000, monotonic=10)
|
||||
second = sampler.observe(
|
||||
scope, raw(block_read=700, block_write=1400, extra={
|
||||
**cdrom, "balloon.last-update": 1060,
|
||||
}), timestamp=1060, monotonic=70,
|
||||
)
|
||||
assert second["disk_read_bps"] == 10
|
||||
assert second["disk_write_bps"] == 20
|
||||
|
||||
|
||||
def test_single_device_counter_rollback_is_not_masked_by_another_device():
|
||||
sampler = MetricSampler()
|
||||
scope = host_scope(HOST)
|
||||
second_disk = {"block.count": 2, "block.1.name": "vdb",
|
||||
"block.1.rd.bytes": 1_000, "block.1.wr.bytes": 1_000}
|
||||
sampler.observe(scope, raw(extra=second_disk), timestamp=1000, monotonic=10)
|
||||
point = sampler.observe(scope, raw(block_read=1, block_write=1, extra={
|
||||
**second_disk, "block.1.rd.bytes": 10_000,
|
||||
"block.1.wr.bytes": 10_000,
|
||||
}), timestamp=1060, monotonic=70)
|
||||
assert point["disk_read_bps"] is None
|
||||
assert point["disk_write_bps"] is None
|
||||
|
||||
|
||||
def test_unobserved_long_gap_is_not_averaged_as_current_rate():
|
||||
sampler = MetricSampler()
|
||||
scope = host_scope(HOST)
|
||||
sampler.observe(scope, raw(), timestamp=1000, monotonic=10)
|
||||
delayed = sampler.observe(scope, raw(cpu=50_000_000_000, block_read=2000),
|
||||
timestamp=1600, monotonic=610)
|
||||
assert delayed["cpu_percent"] is None
|
||||
assert delayed["disk_read_bps"] is None
|
||||
|
||||
|
||||
class FakeCursor:
|
||||
def __init__(self, rows=(), delete_counts=()):
|
||||
self.rows = rows
|
||||
self.commands = []
|
||||
self.delete_counts = list(delete_counts)
|
||||
self.rowcount = 0
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return None
|
||||
|
||||
def execute(self, sql, args=None):
|
||||
self.commands.append((sql, args))
|
||||
self.rowcount = (self.delete_counts.pop(0)
|
||||
if sql.startswith("DELETE FROM vm_metric_samples")
|
||||
and self.delete_counts else 0)
|
||||
|
||||
def executemany(self, sql, args):
|
||||
self.commands.append((sql, args))
|
||||
|
||||
def fetchall(self):
|
||||
return self.rows
|
||||
|
||||
|
||||
class FakeConnection:
|
||||
def __init__(self, rows=(), delete_counts=()):
|
||||
self.fake_cursor = FakeCursor(rows, delete_counts)
|
||||
self.commits = 0
|
||||
self.closed = False
|
||||
|
||||
def cursor(self):
|
||||
return self.fake_cursor
|
||||
|
||||
def commit(self):
|
||||
self.commits += 1
|
||||
|
||||
def rollback(self):
|
||||
raise AssertionError("unexpected rollback")
|
||||
|
||||
def close(self):
|
||||
self.closed = True
|
||||
|
||||
|
||||
def test_metric_table_created_with_record_table():
|
||||
connection = FakeConnection()
|
||||
store = DatabaseStore.__new__(DatabaseStore)
|
||||
store._connect = lambda: connection
|
||||
store._initialize_schema()
|
||||
assert len(connection.fake_cursor.commands) == 2
|
||||
assert "vm_metric_samples" in connection.fake_cursor.commands[1][0]
|
||||
assert "vm_metric_retention" in connection.fake_cursor.commands[1][0]
|
||||
assert connection.commits == 1 and connection.closed
|
||||
|
||||
|
||||
def test_metric_store_prunes_physically_and_queries_only_scoped_recent_data():
|
||||
scope = host_scope(HOST)
|
||||
connection = FakeConnection([{
|
||||
"timestamp": 1000,
|
||||
"cpu_percent": 12.5,
|
||||
"memory_used_mib": 1024,
|
||||
"memory_total_mib": 2048,
|
||||
"disk_read_bps": None,
|
||||
"disk_write_bps": None,
|
||||
"network_rx_bps": 256,
|
||||
"network_tx_bps": 0,
|
||||
}])
|
||||
store = DatabaseStore.__new__(DatabaseStore)
|
||||
store._connect = lambda: connection
|
||||
point = {"vm_id": VM_ID, "timestamp": 1000, "cpu_percent": 12.5,
|
||||
"memory_used_mib": 1024, "memory_total_mib": 2048,
|
||||
"disk_read_bps": None, "disk_write_bps": None,
|
||||
"network_rx_bps": 256, "network_tx_bps": 0}
|
||||
store.save_metric_samples(scope, [point])
|
||||
store.prune_metric_samples(500)
|
||||
result = store.query_metric_samples(scope, VM_ID, since=500, until=1100,
|
||||
interval_seconds=60, limit=600)
|
||||
assert result[0]["cpu_percent"] == 12.5
|
||||
assert result[0]["disk_read_bps"] is None
|
||||
insert, delete, query = connection.fake_cursor.commands
|
||||
assert "INSERT INTO vm_metric_samples" in insert[0]
|
||||
assert "DELETE FROM vm_metric_samples WHERE sampled_at < %s" in delete[0]
|
||||
assert delete[1] == (500, 5000)
|
||||
assert "host_scope=%s AND vm_uuid=%s AND sampled_at >= %s" in query[0]
|
||||
assert "LIMIT %s" in query[0]
|
||||
assert query[1] == (scope, VM_ID, 500, 1100, 60, 600)
|
||||
assert connection.commits == 2 and connection.closed
|
||||
|
||||
|
||||
def test_metric_prune_is_one_index_ordered_batch_per_transaction():
|
||||
connection = FakeConnection(delete_counts=[5000] * 10)
|
||||
store = DatabaseStore.__new__(DatabaseStore)
|
||||
store._connect = lambda: connection
|
||||
assert store.prune_metric_samples(1234) == 5000
|
||||
assert len(connection.fake_cursor.commands) == 1
|
||||
assert connection.commits == 1 and connection.closed
|
||||
assert all("ORDER BY sampled_at LIMIT %s" in command
|
||||
and args == (1234, 5000)
|
||||
for command, args in connection.fake_cursor.commands)
|
||||
|
||||
|
||||
def test_metric_prune_reports_short_final_batch():
|
||||
connection = FakeConnection(delete_counts=[275])
|
||||
store = DatabaseStore.__new__(DatabaseStore)
|
||||
store._connect = lambda: connection
|
||||
assert store.prune_metric_samples(1234) == 275
|
||||
assert len(connection.fake_cursor.commands) == 1
|
||||
assert connection.commits == 1
|
||||
|
||||
|
||||
def test_metric_store_rejects_invalid_host_and_uuid_before_db_query():
|
||||
store = DatabaseStore.__new__(DatabaseStore)
|
||||
store._connect = lambda: pytest.fail("invalid scope must not query database")
|
||||
with pytest.raises(ValueError):
|
||||
store.query_metric_samples("bad-scope", VM_ID, since=0, until=1,
|
||||
interval_seconds=60)
|
||||
with pytest.raises(ValueError):
|
||||
store.query_metric_samples(host_scope(HOST), "not-uuid", since=0, until=1,
|
||||
interval_seconds=60)
|
||||
|
||||
|
||||
def test_collector_prunes_when_ssh_locked_and_starts_after_db_configuration():
|
||||
class Store:
|
||||
def __init__(self):
|
||||
self.pruned = []
|
||||
self.saved = []
|
||||
self.pruned_event = threading.Event()
|
||||
self.saved_event = threading.Event()
|
||||
|
||||
def prune_metric_samples(self, cutoff):
|
||||
self.pruned.append(cutoff)
|
||||
self.pruned_event.set()
|
||||
return 0
|
||||
|
||||
def save_metric_samples(self, scope, points):
|
||||
self.saved.append((scope, points))
|
||||
self.saved_event.set()
|
||||
|
||||
current = {"store": None, "profile": None}
|
||||
store = Store()
|
||||
class Service:
|
||||
def metric_counters(self):
|
||||
return [raw()]
|
||||
collector = MetricsCollector(lambda: current["store"],
|
||||
lambda: current["profile"],
|
||||
lambda _profile: Service(), interval_seconds=0.01)
|
||||
collector.start()
|
||||
current["profile"] = {**HOST, "key_locked": True}
|
||||
current["store"] = store
|
||||
try:
|
||||
assert store.pruned_event.wait(timeout=2)
|
||||
assert store.pruned and not store.saved
|
||||
current["profile"]["key_locked"] = False
|
||||
assert store.saved_event.wait(timeout=2)
|
||||
assert store.saved
|
||||
finally:
|
||||
collector.close()
|
||||
|
||||
|
||||
def test_metric_cleanup_failure_does_not_discard_current_sample(caplog):
|
||||
events = []
|
||||
attempted = threading.Event()
|
||||
|
||||
class Store:
|
||||
def save_metric_samples(self, _scope, points):
|
||||
events.append(("saved", points))
|
||||
|
||||
def prune_metric_samples(self, _cutoff):
|
||||
events.append(("prune", None))
|
||||
attempted.set()
|
||||
raise RuntimeError("secret database details")
|
||||
|
||||
class Service:
|
||||
def metric_counters(self):
|
||||
return [raw()]
|
||||
|
||||
collector = MetricsCollector(lambda: Store(), lambda: HOST,
|
||||
lambda _profile: Service(), interval_seconds=1)
|
||||
collector.start()
|
||||
assert attempted.wait(timeout=2)
|
||||
collector.close()
|
||||
assert [event[0] for event in events] == ["saved", "prune"]
|
||||
assert "RuntimeError" in caplog.text
|
||||
assert "secret database details" not in caplog.text
|
||||
|
||||
|
||||
def test_backlog_cleanup_continues_in_other_thread_without_stalling_samples():
|
||||
second_saved = threading.Event()
|
||||
prune_entered = threading.Event()
|
||||
release_prune = threading.Event()
|
||||
backlog_drained = threading.Event()
|
||||
|
||||
class Store:
|
||||
def __init__(self):
|
||||
self.saved = 0
|
||||
self.pruned = 0
|
||||
|
||||
def save_metric_samples(self, _scope, _points):
|
||||
self.saved += 1
|
||||
if self.saved >= 2:
|
||||
second_saved.set()
|
||||
|
||||
def prune_metric_samples(self, _cutoff):
|
||||
self.pruned += 1
|
||||
if self.pruned == 1:
|
||||
prune_entered.set()
|
||||
assert release_prune.wait(timeout=2)
|
||||
if self.pruned < 3:
|
||||
return 5000
|
||||
backlog_drained.set()
|
||||
return 0
|
||||
|
||||
class Service:
|
||||
def metric_counters(self):
|
||||
return [raw()]
|
||||
|
||||
store = Store()
|
||||
collector = MetricsCollector(lambda: store, lambda: HOST,
|
||||
lambda _profile: Service(), interval_seconds=0.01)
|
||||
collector.start()
|
||||
try:
|
||||
assert prune_entered.wait(timeout=2)
|
||||
assert second_saved.wait(timeout=2)
|
||||
release_prune.set()
|
||||
assert backlog_drained.wait(timeout=2)
|
||||
assert store.pruned >= 3
|
||||
finally:
|
||||
release_prune.set()
|
||||
collector.close()
|
||||
|
||||
|
||||
def test_collector_thread_starts_before_db_configuration_and_stops():
|
||||
ready = threading.Event()
|
||||
current = {"store": None}
|
||||
|
||||
class Store:
|
||||
def prune_metric_samples(self, _cutoff):
|
||||
ready.set()
|
||||
return 0
|
||||
|
||||
collector = MetricsCollector(lambda: current["store"], lambda: None,
|
||||
lambda _profile: pytest.fail("no host configured"),
|
||||
interval_seconds=0.01)
|
||||
collector.start()
|
||||
current["store"] = Store()
|
||||
assert ready.wait(timeout=2)
|
||||
collector.close()
|
||||
assert collector._thread is not None and not collector._thread.is_alive()
|
||||
assert collector._prune_thread is not None and not collector._prune_thread.is_alive()
|
||||
Loading…
Reference in New Issue