You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
471 lines
17 KiB
Python
471 lines
17 KiB
Python
"""VM metric math, DB persistence, and the process-local collection lifecycle."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import threading
|
|
|
|
import pytest
|
|
|
|
from virsh_panel.database import DatabaseStore
|
|
from virsh_panel.metrics import (
|
|
MAX_QUERY_POINTS, RANGES, MetricSampler, MetricsCollector, RawVMStats,
|
|
host_scope, query_interval_seconds,
|
|
)
|
|
|
|
|
|
VM_ID = "00000000-0000-4000-8000-000000000001"
|
|
HOST = {"host": "hypervisor.example", "port": 2222,
|
|
"fingerprint": "SHA256:one", "storage_pool": "/home/kvm"}
|
|
|
|
|
|
def raw(*, generation=12, cpu=1_000_000_000, block_read=100,
|
|
block_write=200, network_rx=1000, network_tx=2000,
|
|
memory_time=1000, extra=None):
|
|
values = {
|
|
"cpu.time": cpu,
|
|
"balloon.available": 2 * 1024 * 1024,
|
|
"balloon.usable": 512 * 1024,
|
|
"balloon.last-update": memory_time,
|
|
"block.count": 1,
|
|
"block.0.name": "vda",
|
|
"block.0.rd.bytes": block_read,
|
|
"block.0.wr.bytes": block_write,
|
|
"net.count": 1,
|
|
"net.0.name": "vnet1",
|
|
"net.0.rx.bytes": network_rx,
|
|
"net.0.tx.bytes": network_tx,
|
|
}
|
|
values.update(extra or {})
|
|
return RawVMStats(VM_ID, generation, 2, values)
|
|
|
|
|
|
def test_host_scope_ignores_pool_but_changes_with_ssh_host_identity():
|
|
first = host_scope(HOST)
|
|
assert first == host_scope({**HOST, "storage_pool": "/different"})
|
|
assert first != host_scope({**HOST, "fingerprint": "SHA256:two"})
|
|
assert first != host_scope({**HOST, "port": 22})
|
|
|
|
|
|
def test_interval_caps_all_ranges_at_600_epoch_aligned_buckets():
|
|
for seconds in RANGES.values():
|
|
interval = query_interval_seconds(seconds)
|
|
assert interval % 60 == 0
|
|
assert seconds // interval + 2 <= MAX_QUERY_POINTS
|
|
|
|
|
|
def test_initial_sample_and_fresh_guest_memory_then_rates():
|
|
sampler = MetricSampler()
|
|
scope = host_scope(HOST)
|
|
first = sampler.observe(scope, raw(), timestamp=1000, monotonic=10)
|
|
assert first["cpu_percent"] is None
|
|
assert first["disk_read_bps"] is None
|
|
assert first["network_rx_bps"] is None
|
|
assert first["memory_used_mib"] == 1536
|
|
assert first["memory_total_mib"] == 2048
|
|
|
|
second = sampler.observe(
|
|
scope,
|
|
raw(cpu=61_000_000_000, block_read=700, block_write=1400,
|
|
network_rx=2200, network_tx=2600, memory_time=1050),
|
|
timestamp=1060, monotonic=70,
|
|
)
|
|
assert second["cpu_percent"] == 50
|
|
assert second["disk_read_bps"] == 10
|
|
assert second["disk_write_bps"] == 20
|
|
assert second["network_rx_bps"] == 20
|
|
assert second["network_tx_bps"] == 10
|
|
|
|
|
|
def test_cpu_time_over_vcpu_capacity_is_displayed_as_100_percent():
|
|
sampler = MetricSampler()
|
|
scope = host_scope(HOST)
|
|
sampler.observe(scope, raw(), timestamp=1000, monotonic=10)
|
|
# 130 CPU-seconds across two vCPUs in a 60-second interval; libvirt can
|
|
# include QEMU auxiliary threads in cpu.time.
|
|
point = sampler.observe(scope, raw(cpu=131_000_000_000),
|
|
timestamp=1060, monotonic=70)
|
|
assert point["cpu_percent"] == 100
|
|
|
|
|
|
def test_memory_missing_or_stale_balloon_stats_never_uses_rss_or_current():
|
|
sampler = MetricSampler()
|
|
scope = host_scope(HOST)
|
|
values = {"balloon.current": 2 * 1024 * 1024, "balloon.rss": 512 * 1024}
|
|
for extra in (
|
|
{"balloon.last-update": 1},
|
|
{"balloon.usable": None},
|
|
{"balloon.available": None},
|
|
):
|
|
point = sampler.observe(scope, raw(extra={**values, **extra}),
|
|
timestamp=1000, monotonic=10)
|
|
assert point["memory_used_mib"] is None
|
|
assert point["memory_total_mib"] is None
|
|
|
|
|
|
def test_counter_reset_reboot_device_change_and_partial_data_are_unknown():
|
|
sampler = MetricSampler()
|
|
scope = host_scope(HOST)
|
|
sampler.observe(scope, raw(), timestamp=1000, monotonic=10)
|
|
reset = sampler.observe(scope, raw(cpu=100, block_read=1, network_rx=1),
|
|
timestamp=1060, monotonic=70)
|
|
assert reset["cpu_percent"] is None
|
|
assert reset["disk_read_bps"] is None
|
|
assert reset["network_rx_bps"] is None
|
|
|
|
# Even if a second disk masks the first disk's decreasing counter in the
|
|
# aggregate, device change must not produce a transient spike.
|
|
changed = sampler.observe(scope, raw(extra={
|
|
"block.count": 2, "block.1.name": "vdb",
|
|
"block.1.rd.bytes": 10_000, "block.1.wr.bytes": 20_000,
|
|
}), timestamp=1120, monotonic=130)
|
|
assert changed["disk_read_bps"] is None
|
|
assert changed["disk_write_bps"] is None
|
|
|
|
partial = sampler.observe(scope, raw(extra={
|
|
"block.count": 2, "block.1.name": "vdb",
|
|
"block.1.rd.bytes": 11_000,
|
|
}), timestamp=1180, monotonic=190)
|
|
assert partial["disk_write_bps"] is None # second device missing wr.bytes
|
|
assert partial["disk_read_bps"] is not None
|
|
|
|
reboot = sampler.observe(scope, raw(generation=13, cpu=200_000_000_000),
|
|
timestamp=1240, monotonic=250)
|
|
assert reboot["cpu_percent"] is None
|
|
assert reboot["network_tx_bps"] is None
|
|
|
|
|
|
def test_cdrom_without_byte_counters_does_not_hide_disk_rates():
|
|
sampler = MetricSampler()
|
|
scope = host_scope(HOST)
|
|
cdrom = {"block.count": 2, "block.1.name": "sda"}
|
|
sampler.observe(scope, raw(extra=cdrom), timestamp=1000, monotonic=10)
|
|
second = sampler.observe(
|
|
scope, raw(block_read=700, block_write=1400, extra={
|
|
**cdrom, "balloon.last-update": 1060,
|
|
}), timestamp=1060, monotonic=70,
|
|
)
|
|
assert second["disk_read_bps"] == 10
|
|
assert second["disk_write_bps"] == 20
|
|
|
|
|
|
def test_single_device_counter_rollback_is_not_masked_by_another_device():
|
|
sampler = MetricSampler()
|
|
scope = host_scope(HOST)
|
|
second_disk = {"block.count": 2, "block.1.name": "vdb",
|
|
"block.1.rd.bytes": 1_000, "block.1.wr.bytes": 1_000}
|
|
sampler.observe(scope, raw(extra=second_disk), timestamp=1000, monotonic=10)
|
|
point = sampler.observe(scope, raw(block_read=1, block_write=1, extra={
|
|
**second_disk, "block.1.rd.bytes": 10_000,
|
|
"block.1.wr.bytes": 10_000,
|
|
}), timestamp=1060, monotonic=70)
|
|
assert point["disk_read_bps"] is None
|
|
assert point["disk_write_bps"] is None
|
|
|
|
|
|
def test_unobserved_long_gap_is_not_averaged_as_current_rate():
|
|
sampler = MetricSampler()
|
|
scope = host_scope(HOST)
|
|
sampler.observe(scope, raw(), timestamp=1000, monotonic=10)
|
|
delayed = sampler.observe(scope, raw(cpu=50_000_000_000, block_read=2000),
|
|
timestamp=1600, monotonic=610)
|
|
assert delayed["cpu_percent"] is None
|
|
assert delayed["disk_read_bps"] is None
|
|
|
|
|
|
class FakeCursor:
|
|
def __init__(self, rows=(), delete_counts=()):
|
|
self.rows = rows
|
|
self.commands = []
|
|
self.delete_counts = list(delete_counts)
|
|
self.rowcount = 0
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *_args):
|
|
return None
|
|
|
|
def execute(self, sql, args=None):
|
|
self.commands.append((sql, args))
|
|
self.rowcount = (self.delete_counts.pop(0)
|
|
if sql.startswith("DELETE FROM vm_metric_samples")
|
|
and self.delete_counts else 0)
|
|
|
|
def executemany(self, sql, args):
|
|
self.commands.append((sql, args))
|
|
|
|
def fetchall(self):
|
|
return self.rows
|
|
|
|
|
|
class FakeConnection:
|
|
def __init__(self, rows=(), delete_counts=()):
|
|
self.fake_cursor = FakeCursor(rows, delete_counts)
|
|
self.commits = 0
|
|
self.closed = False
|
|
|
|
def cursor(self):
|
|
return self.fake_cursor
|
|
|
|
def commit(self):
|
|
self.commits += 1
|
|
|
|
def rollback(self):
|
|
raise AssertionError("unexpected rollback")
|
|
|
|
def close(self):
|
|
self.closed = True
|
|
|
|
|
|
def test_metric_and_user_tables_created_with_record_table():
|
|
connection = FakeConnection()
|
|
store = DatabaseStore.__new__(DatabaseStore)
|
|
store._connect = lambda: connection
|
|
store._initialize_schema()
|
|
assert len(connection.fake_cursor.commands) == 6
|
|
assert "vm_metric_samples" in connection.fake_cursor.commands[1][0]
|
|
assert "vm_metric_retention" in connection.fake_cursor.commands[1][0]
|
|
assert "CREATE TABLE IF NOT EXISTS users" in connection.fake_cursor.commands[2][0]
|
|
assert "CREATE TABLE IF NOT EXISTS registration_codes" in connection.fake_cursor.commands[3][0]
|
|
assert "CREATE TABLE IF NOT EXISTS vm_ownership" in connection.fake_cursor.commands[4][0]
|
|
assert "CREATE TABLE IF NOT EXISTS user_ssh_keys" in connection.fake_cursor.commands[5][0]
|
|
assert connection.commits == 1 and connection.closed
|
|
|
|
|
|
def test_metric_store_prunes_physically_and_queries_only_scoped_recent_data():
|
|
scope = host_scope(HOST)
|
|
connection = FakeConnection([{
|
|
"timestamp": 1000,
|
|
"cpu_percent": 12.5,
|
|
"memory_used_mib": 1024,
|
|
"memory_total_mib": 2048,
|
|
"disk_read_bps": None,
|
|
"disk_write_bps": None,
|
|
"network_rx_bps": 256,
|
|
"network_tx_bps": 0,
|
|
}])
|
|
store = DatabaseStore.__new__(DatabaseStore)
|
|
store._connect = lambda: connection
|
|
point = {"vm_id": VM_ID, "timestamp": 1000, "cpu_percent": 12.5,
|
|
"memory_used_mib": 1024, "memory_total_mib": 2048,
|
|
"disk_read_bps": None, "disk_write_bps": None,
|
|
"network_rx_bps": 256, "network_tx_bps": 0}
|
|
store.save_metric_samples(scope, [point])
|
|
store.prune_metric_samples(500)
|
|
result = store.query_metric_samples(scope, VM_ID, since=500, until=1100,
|
|
interval_seconds=60, limit=600)
|
|
assert result[0]["cpu_percent"] == 12.5
|
|
assert result[0]["disk_read_bps"] is None
|
|
insert, delete, query = connection.fake_cursor.commands
|
|
assert "INSERT INTO vm_metric_samples" in insert[0]
|
|
assert "DELETE FROM vm_metric_samples WHERE sampled_at < %s" in delete[0]
|
|
assert delete[1] == (500, 5000)
|
|
assert "host_scope=%s AND vm_uuid=%s AND sampled_at >= %s" in query[0]
|
|
assert "LIMIT %s" in query[0]
|
|
assert query[1] == (scope, VM_ID, 500, 1100, 60, 600)
|
|
assert connection.commits == 2 and connection.closed
|
|
|
|
|
|
def test_metric_prune_is_one_index_ordered_batch_per_transaction():
|
|
connection = FakeConnection(delete_counts=[5000] * 10)
|
|
store = DatabaseStore.__new__(DatabaseStore)
|
|
store._connect = lambda: connection
|
|
assert store.prune_metric_samples(1234) == 5000
|
|
assert len(connection.fake_cursor.commands) == 1
|
|
assert connection.commits == 1 and connection.closed
|
|
assert all("ORDER BY sampled_at LIMIT %s" in command
|
|
and args == (1234, 5000)
|
|
for command, args in connection.fake_cursor.commands)
|
|
|
|
|
|
def test_metric_prune_reports_short_final_batch():
|
|
connection = FakeConnection(delete_counts=[275])
|
|
store = DatabaseStore.__new__(DatabaseStore)
|
|
store._connect = lambda: connection
|
|
assert store.prune_metric_samples(1234) == 275
|
|
assert len(connection.fake_cursor.commands) == 1
|
|
assert connection.commits == 1
|
|
|
|
|
|
def test_metric_store_rejects_invalid_host_and_uuid_before_db_query():
|
|
store = DatabaseStore.__new__(DatabaseStore)
|
|
store._connect = lambda: pytest.fail("invalid scope must not query database")
|
|
with pytest.raises(ValueError):
|
|
store.query_metric_samples("bad-scope", VM_ID, since=0, until=1,
|
|
interval_seconds=60)
|
|
with pytest.raises(ValueError):
|
|
store.query_metric_samples(host_scope(HOST), "not-uuid", since=0, until=1,
|
|
interval_seconds=60)
|
|
|
|
|
|
def test_collector_prunes_when_ssh_locked_and_starts_after_db_configuration():
|
|
class Store:
|
|
def __init__(self):
|
|
self.pruned = []
|
|
self.saved = []
|
|
self.pruned_event = threading.Event()
|
|
self.saved_event = threading.Event()
|
|
|
|
def prune_metric_samples(self, cutoff):
|
|
self.pruned.append(cutoff)
|
|
self.pruned_event.set()
|
|
return 0
|
|
|
|
def save_metric_samples(self, scope, points):
|
|
self.saved.append((scope, points))
|
|
self.saved_event.set()
|
|
|
|
current = {"store": None, "profile": None}
|
|
store = Store()
|
|
class Service:
|
|
def metric_counters(self):
|
|
return [raw()]
|
|
collector = MetricsCollector(lambda: current["store"],
|
|
lambda: current["profile"],
|
|
lambda _profile: Service(), interval_seconds=0.01)
|
|
collector.start()
|
|
current["profile"] = {**HOST, "key_locked": True}
|
|
current["store"] = store
|
|
try:
|
|
assert store.pruned_event.wait(timeout=2)
|
|
assert store.pruned and not store.saved
|
|
current["profile"]["key_locked"] = False
|
|
assert store.saved_event.wait(timeout=2)
|
|
assert store.saved
|
|
finally:
|
|
collector.close()
|
|
|
|
|
|
def test_background_collector_keeps_sampling_without_an_http_session():
|
|
"""The 30-day history must not depend on a logged-in browser polling it."""
|
|
saved_twice = threading.Event()
|
|
|
|
class Store:
|
|
def __init__(self):
|
|
self.saved = []
|
|
|
|
def save_metric_samples(self, scope, points):
|
|
self.saved.append((scope, points))
|
|
if len(self.saved) >= 2:
|
|
saved_twice.set()
|
|
|
|
def prune_metric_samples(self, _cutoff):
|
|
return 0
|
|
|
|
class Service:
|
|
calls = 0
|
|
|
|
def metric_counters(self):
|
|
self.calls += 1
|
|
return [raw(cpu=self.calls * 1_000_000_000,
|
|
block_read=self.calls * 1000,
|
|
network_rx=self.calls * 1000)]
|
|
|
|
store = Store()
|
|
service = Service()
|
|
profile = {**HOST, "key_locked": False}
|
|
collector = MetricsCollector(lambda: store, lambda: profile,
|
|
lambda _profile: service,
|
|
interval_seconds=0.01)
|
|
collector.start()
|
|
try:
|
|
assert saved_twice.wait(timeout=2)
|
|
finally:
|
|
collector.close()
|
|
|
|
assert service.calls >= 2
|
|
assert all(scope == host_scope(HOST) for scope, _ in store.saved)
|
|
assert all(points[0]["vm_id"] == VM_ID for _, points in store.saved)
|
|
|
|
|
|
def test_metric_cleanup_failure_does_not_discard_current_sample(caplog):
|
|
events = []
|
|
attempted = threading.Event()
|
|
|
|
class Store:
|
|
def save_metric_samples(self, _scope, points):
|
|
events.append(("saved", points))
|
|
|
|
def prune_metric_samples(self, _cutoff):
|
|
events.append(("prune", None))
|
|
attempted.set()
|
|
raise RuntimeError("secret database details")
|
|
|
|
class Service:
|
|
def metric_counters(self):
|
|
return [raw()]
|
|
|
|
collector = MetricsCollector(lambda: Store(), lambda: HOST,
|
|
lambda _profile: Service(), interval_seconds=1)
|
|
collector.start()
|
|
assert attempted.wait(timeout=2)
|
|
collector.close()
|
|
assert [event[0] for event in events] == ["saved", "prune"]
|
|
assert "RuntimeError" in caplog.text
|
|
assert "secret database details" not in caplog.text
|
|
|
|
|
|
def test_backlog_cleanup_continues_in_other_thread_without_stalling_samples():
|
|
second_saved = threading.Event()
|
|
prune_entered = threading.Event()
|
|
release_prune = threading.Event()
|
|
backlog_drained = threading.Event()
|
|
|
|
class Store:
|
|
def __init__(self):
|
|
self.saved = 0
|
|
self.pruned = 0
|
|
|
|
def save_metric_samples(self, _scope, _points):
|
|
self.saved += 1
|
|
if self.saved >= 2:
|
|
second_saved.set()
|
|
|
|
def prune_metric_samples(self, _cutoff):
|
|
self.pruned += 1
|
|
if self.pruned == 1:
|
|
prune_entered.set()
|
|
assert release_prune.wait(timeout=2)
|
|
if self.pruned < 3:
|
|
return 5000
|
|
backlog_drained.set()
|
|
return 0
|
|
|
|
class Service:
|
|
def metric_counters(self):
|
|
return [raw()]
|
|
|
|
store = Store()
|
|
collector = MetricsCollector(lambda: store, lambda: HOST,
|
|
lambda _profile: Service(), interval_seconds=0.01)
|
|
collector.start()
|
|
try:
|
|
assert prune_entered.wait(timeout=2)
|
|
assert second_saved.wait(timeout=2)
|
|
release_prune.set()
|
|
assert backlog_drained.wait(timeout=2)
|
|
assert store.pruned >= 3
|
|
finally:
|
|
release_prune.set()
|
|
collector.close()
|
|
|
|
|
|
def test_collector_thread_starts_before_db_configuration_and_stops():
|
|
ready = threading.Event()
|
|
current = {"store": None}
|
|
|
|
class Store:
|
|
def prune_metric_samples(self, _cutoff):
|
|
ready.set()
|
|
return 0
|
|
|
|
collector = MetricsCollector(lambda: current["store"], lambda: None,
|
|
lambda _profile: pytest.fail("no host configured"),
|
|
interval_seconds=0.01)
|
|
collector.start()
|
|
current["store"] = Store()
|
|
assert ready.wait(timeout=2)
|
|
collector.close()
|
|
assert collector._thread is not None and not collector._thread.is_alive()
|
|
assert collector._prune_thread is not None and not collector._prune_thread.is_alive()
|