You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
750 lines
28 KiB
Python
750 lines
28 KiB
Python
"""VM metric math, DB persistence, and the process-local collection lifecycle."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import threading
|
|
from decimal import Decimal
|
|
|
|
import pytest
|
|
|
|
from virsh_panel.database import DatabaseStore
|
|
from virsh_panel.metrics import (
|
|
MAX_QUERY_POINTS, RANGES, MetricSampler, MetricsCollector, RawVMStats,
|
|
host_scope, query_interval_seconds,
|
|
)
|
|
|
|
|
|
VM_ID = "00000000-0000-4000-8000-000000000001"
|
|
HOST = {"host": "hypervisor.example", "port": 2222,
|
|
"fingerprint": "SHA256:one", "storage_pool": "/home/kvm"}
|
|
|
|
|
|
def raw(*, generation=12, cpu=1_000_000_000, block_read=100,
|
|
block_write=200, network_rx=1000, network_tx=2000,
|
|
memory_time=1000, extra=None):
|
|
values = {
|
|
"cpu.time": cpu,
|
|
"balloon.available": 2 * 1024 * 1024,
|
|
"balloon.usable": 512 * 1024,
|
|
"balloon.last-update": memory_time,
|
|
"block.count": 1,
|
|
"block.0.name": "vda",
|
|
"block.0.rd.bytes": block_read,
|
|
"block.0.wr.bytes": block_write,
|
|
"net.count": 1,
|
|
"net.0.name": "vnet1",
|
|
"net.0.rx.bytes": network_rx,
|
|
"net.0.tx.bytes": network_tx,
|
|
}
|
|
values.update(extra or {})
|
|
return RawVMStats(VM_ID, generation, 2, values)
|
|
|
|
|
|
def test_host_scope_ignores_pool_but_changes_with_ssh_host_identity():
|
|
first = host_scope(HOST)
|
|
assert first == host_scope({**HOST, "storage_pool": "/different"})
|
|
assert first != host_scope({**HOST, "fingerprint": "SHA256:two"})
|
|
assert first != host_scope({**HOST, "port": 22})
|
|
|
|
|
|
def test_interval_caps_all_ranges_at_600_epoch_aligned_buckets():
|
|
for seconds in RANGES.values():
|
|
interval = query_interval_seconds(seconds)
|
|
assert interval % 60 == 0
|
|
assert seconds // interval + 2 <= MAX_QUERY_POINTS
|
|
|
|
|
|
def test_initial_sample_and_fresh_guest_memory_then_rates():
|
|
sampler = MetricSampler()
|
|
scope = host_scope(HOST)
|
|
first = sampler.observe(scope, raw(), timestamp=1000, monotonic=10)
|
|
assert first["cpu_percent"] is None
|
|
assert first["disk_read_bps"] is None
|
|
assert first["network_rx_bps"] is None
|
|
assert first["memory_used_mib"] == 1536
|
|
assert first["memory_total_mib"] == 2048
|
|
|
|
second = sampler.observe(
|
|
scope,
|
|
raw(cpu=61_000_000_000, block_read=700, block_write=1400,
|
|
network_rx=2200, network_tx=2600, memory_time=1050),
|
|
timestamp=1060, monotonic=70,
|
|
)
|
|
assert second["cpu_percent"] == 50
|
|
assert second["disk_read_bps"] == 10
|
|
assert second["disk_write_bps"] == 20
|
|
assert second["network_rx_bps"] == 20
|
|
assert second["network_tx_bps"] == 10
|
|
|
|
|
|
def test_cpu_time_over_vcpu_capacity_is_displayed_as_100_percent():
|
|
sampler = MetricSampler()
|
|
scope = host_scope(HOST)
|
|
sampler.observe(scope, raw(), timestamp=1000, monotonic=10)
|
|
# 130 CPU-seconds across two vCPUs in a 60-second interval; libvirt can
|
|
# include QEMU auxiliary threads in cpu.time.
|
|
point = sampler.observe(scope, raw(cpu=131_000_000_000),
|
|
timestamp=1060, monotonic=70)
|
|
assert point["cpu_percent"] == 100
|
|
|
|
|
|
def test_memory_missing_or_stale_balloon_stats_never_uses_rss_or_current():
|
|
sampler = MetricSampler()
|
|
scope = host_scope(HOST)
|
|
values = {"balloon.current": 2 * 1024 * 1024, "balloon.rss": 512 * 1024}
|
|
for extra in (
|
|
{"balloon.last-update": 1},
|
|
{"balloon.usable": None},
|
|
{"balloon.available": None},
|
|
):
|
|
point = sampler.observe(scope, raw(extra={**values, **extra}),
|
|
timestamp=1000, monotonic=10)
|
|
assert point["memory_used_mib"] is None
|
|
assert point["memory_total_mib"] is None
|
|
|
|
|
|
def test_counter_reset_reboot_device_change_and_partial_data_are_unknown():
|
|
sampler = MetricSampler()
|
|
scope = host_scope(HOST)
|
|
sampler.observe(scope, raw(), timestamp=1000, monotonic=10)
|
|
reset = sampler.observe(scope, raw(cpu=100, block_read=1, network_rx=1),
|
|
timestamp=1060, monotonic=70)
|
|
assert reset["cpu_percent"] is None
|
|
assert reset["disk_read_bps"] is None
|
|
assert reset["network_rx_bps"] is None
|
|
|
|
# Even if a second disk masks the first disk's decreasing counter in the
|
|
# aggregate, device change must not produce a transient spike.
|
|
changed = sampler.observe(scope, raw(extra={
|
|
"block.count": 2, "block.1.name": "vdb",
|
|
"block.1.rd.bytes": 10_000, "block.1.wr.bytes": 20_000,
|
|
}), timestamp=1120, monotonic=130)
|
|
assert changed["disk_read_bps"] is None
|
|
assert changed["disk_write_bps"] is None
|
|
|
|
partial = sampler.observe(scope, raw(extra={
|
|
"block.count": 2, "block.1.name": "vdb",
|
|
"block.1.rd.bytes": 11_000,
|
|
}), timestamp=1180, monotonic=190)
|
|
assert partial["disk_write_bps"] is None # second device missing wr.bytes
|
|
assert partial["disk_read_bps"] is not None
|
|
|
|
reboot = sampler.observe(scope, raw(generation=13, cpu=200_000_000_000),
|
|
timestamp=1240, monotonic=250)
|
|
assert reboot["cpu_percent"] is None
|
|
assert reboot["network_tx_bps"] is None
|
|
|
|
|
|
def test_cdrom_without_byte_counters_does_not_hide_disk_rates():
|
|
sampler = MetricSampler()
|
|
scope = host_scope(HOST)
|
|
cdrom = {"block.count": 2, "block.1.name": "sda"}
|
|
sampler.observe(scope, raw(extra=cdrom), timestamp=1000, monotonic=10)
|
|
second = sampler.observe(
|
|
scope, raw(block_read=700, block_write=1400, extra={
|
|
**cdrom, "balloon.last-update": 1060,
|
|
}), timestamp=1060, monotonic=70,
|
|
)
|
|
assert second["disk_read_bps"] == 10
|
|
assert second["disk_write_bps"] == 20
|
|
|
|
|
|
def test_single_device_counter_rollback_is_not_masked_by_another_device():
|
|
sampler = MetricSampler()
|
|
scope = host_scope(HOST)
|
|
second_disk = {"block.count": 2, "block.1.name": "vdb",
|
|
"block.1.rd.bytes": 1_000, "block.1.wr.bytes": 1_000}
|
|
sampler.observe(scope, raw(extra=second_disk), timestamp=1000, monotonic=10)
|
|
point = sampler.observe(scope, raw(block_read=1, block_write=1, extra={
|
|
**second_disk, "block.1.rd.bytes": 10_000,
|
|
"block.1.wr.bytes": 10_000,
|
|
}), timestamp=1060, monotonic=70)
|
|
assert point["disk_read_bps"] is None
|
|
assert point["disk_write_bps"] is None
|
|
|
|
|
|
def test_unobserved_long_gap_is_not_averaged_as_current_rate():
|
|
sampler = MetricSampler()
|
|
scope = host_scope(HOST)
|
|
sampler.observe(scope, raw(), timestamp=1000, monotonic=10)
|
|
delayed = sampler.observe(scope, raw(cpu=50_000_000_000, block_read=2000),
|
|
timestamp=1600, monotonic=610)
|
|
assert delayed["cpu_percent"] is None
|
|
assert delayed["disk_read_bps"] is None
|
|
|
|
|
|
class FakeCursor:
|
|
def __init__(self, rows=(), delete_counts=()):
|
|
self.rows = rows
|
|
self.commands = []
|
|
self.delete_counts = list(delete_counts)
|
|
self.rowcount = 0
|
|
self.one = {
|
|
"last_observed_at": None,
|
|
"network_baseline_json": "{}",
|
|
"download_rx_bytes": 0,
|
|
"upload_tx_bytes": 0,
|
|
}
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *_args):
|
|
return None
|
|
|
|
def execute(self, sql, args=None):
|
|
self.commands.append((sql, args))
|
|
self.rowcount = (self.delete_counts.pop(0)
|
|
if sql.startswith("DELETE FROM vm_metric_samples")
|
|
and self.delete_counts else 0)
|
|
|
|
def executemany(self, sql, args):
|
|
self.commands.append((sql, args))
|
|
|
|
def fetchall(self):
|
|
return self.rows
|
|
|
|
def fetchone(self):
|
|
return self.one
|
|
|
|
|
|
class FakeConnection:
|
|
def __init__(self, rows=(), delete_counts=()):
|
|
self.fake_cursor = FakeCursor(rows, delete_counts)
|
|
self.commits = 0
|
|
self.closed = False
|
|
|
|
def cursor(self):
|
|
return self.fake_cursor
|
|
|
|
def commit(self):
|
|
self.commits += 1
|
|
|
|
def rollback(self):
|
|
raise AssertionError("unexpected rollback")
|
|
|
|
def close(self):
|
|
self.closed = True
|
|
|
|
|
|
def test_metric_and_user_tables_created_with_record_table():
|
|
connection = FakeConnection()
|
|
store = DatabaseStore.__new__(DatabaseStore)
|
|
store._connect = lambda: connection
|
|
store._initialize_schema()
|
|
assert len(connection.fake_cursor.commands) == 13
|
|
assert "vm_metric_samples" in connection.fake_cursor.commands[1][0]
|
|
assert "vm_metric_retention" in connection.fake_cursor.commands[1][0]
|
|
assert "CREATE TABLE IF NOT EXISTS vm_traffic_totals" in connection.fake_cursor.commands[2][0]
|
|
assert "CREATE TABLE IF NOT EXISTS vm_traffic_checkpoints" in connection.fake_cursor.commands[3][0]
|
|
assert all("ALTER TABLE vm_metric_samples" in connection.fake_cursor.commands[index][0]
|
|
for index in range(4, 9))
|
|
assert "CREATE TABLE IF NOT EXISTS users" in connection.fake_cursor.commands[9][0]
|
|
assert "CREATE TABLE IF NOT EXISTS registration_codes" in connection.fake_cursor.commands[10][0]
|
|
assert "CREATE TABLE IF NOT EXISTS vm_ownership" in connection.fake_cursor.commands[11][0]
|
|
assert "CREATE TABLE IF NOT EXISTS user_ssh_keys" in connection.fake_cursor.commands[12][0]
|
|
assert connection.commits == 1 and connection.closed
|
|
|
|
|
|
def test_metric_store_prunes_physically_and_queries_only_scoped_recent_data():
|
|
scope = host_scope(HOST)
|
|
connection = FakeConnection([{
|
|
"timestamp": 1000,
|
|
"cpu_percent": 12.5,
|
|
"memory_used_mib": 1024,
|
|
"memory_total_mib": 2048,
|
|
"disk_read_bps": None,
|
|
"disk_write_bps": None,
|
|
"network_rx_bps": 256,
|
|
"network_tx_bps": 0,
|
|
}])
|
|
store = DatabaseStore.__new__(DatabaseStore)
|
|
store._connect = lambda: connection
|
|
point = {"vm_id": VM_ID, "timestamp": 1000, "cpu_percent": 12.5,
|
|
"memory_used_mib": 1024, "memory_total_mib": 2048,
|
|
"disk_read_bps": None, "disk_write_bps": None,
|
|
"network_rx_bps": 256, "network_tx_bps": 0,
|
|
"network_devices": {}}
|
|
store.save_metric_samples(scope, [point])
|
|
store.prune_metric_samples(500)
|
|
result = store.query_metric_samples(scope, VM_ID, since=500, until=1100,
|
|
interval_seconds=60, limit=600)
|
|
assert result[0]["cpu_percent"] == 12.5
|
|
assert result[0]["disk_read_bps"] is None
|
|
traffic_insert, locked, traffic_update, checkpoint, insert, delete, query = connection.fake_cursor.commands
|
|
assert "INSERT IGNORE INTO vm_traffic_totals" in traffic_insert[0]
|
|
assert "FOR UPDATE" in locked[0]
|
|
assert "UPDATE vm_traffic_totals" in traffic_update[0]
|
|
assert "INSERT IGNORE INTO vm_traffic_checkpoints" in checkpoint[0]
|
|
assert checkpoint[1] == (scope, VM_ID, 0, None, None, 1000, 0, 0)
|
|
assert "INSERT INTO vm_metric_samples" in insert[0]
|
|
assert "DELETE FROM vm_metric_samples WHERE sampled_at < %s" in delete[0]
|
|
assert delete[1] == (500, 5000)
|
|
assert "host_scope=%s AND vm_uuid=%s AND sampled_at >= %s" in query[0]
|
|
assert "LIMIT %s" in query[0]
|
|
assert query[1] == (scope, VM_ID, 500, 1100, 60, 600)
|
|
assert connection.commits == 2 and connection.closed
|
|
|
|
|
|
def test_daily_traffic_query_sums_scoped_owner_bytes_by_utc_day():
|
|
scope = host_scope({**HOST, "host_id": "hk"})
|
|
connection = FakeConnection([
|
|
{"day_start": Decimal(0), "download_bytes": Decimal(1500),
|
|
"upload_bytes": Decimal(250)},
|
|
{"day_start": Decimal(86400), "download_bytes": None,
|
|
"upload_bytes": None},
|
|
])
|
|
store = DatabaseStore.__new__(DatabaseStore)
|
|
store._connect = lambda: connection
|
|
|
|
result = store.query_daily_traffic(
|
|
scope, VM_ID, host_id="hk", user_id=41,
|
|
since=0, until=172799, limit=2,
|
|
)
|
|
|
|
assert result == [
|
|
{"day_start": 0, "download_bytes": 1500,
|
|
"upload_bytes": 250, "total_bytes": 1750},
|
|
{"day_start": 86400, "download_bytes": None,
|
|
"upload_bytes": None, "total_bytes": None},
|
|
]
|
|
query, args = connection.fake_cursor.commands[0]
|
|
assert "SUM(network_rx_bytes) AS download_bytes" in query
|
|
assert "SUM(network_tx_bytes) AS upload_bytes" in query
|
|
assert "host_scope=%s AND vm_uuid=%s" in query
|
|
assert "host_id <=> %s AND user_id <=> %s" in query
|
|
assert "GROUP BY FLOOR(sampled_at / 86400)" in query
|
|
assert args == (scope, VM_ID, "hk", 41, 0, 172799, 2)
|
|
assert connection.closed
|
|
|
|
|
|
class TrafficBackend:
|
|
"""Tiny transaction fake for durable accounting behavior tests."""
|
|
def __init__(self):
|
|
self.states = {}
|
|
self.samples = {}
|
|
self.checkpoints = {}
|
|
self.commands = []
|
|
self.commits = 0
|
|
|
|
def connection(self):
|
|
backend = self
|
|
|
|
class Cursor:
|
|
selected = None
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *_args):
|
|
return None
|
|
|
|
def execute(self, sql, args=None):
|
|
backend.commands.append((sql, args))
|
|
if sql.startswith("INSERT IGNORE INTO vm_traffic_totals"):
|
|
key = args[0], args[1]
|
|
backend.states.setdefault(key, {
|
|
"host_id": args[2], "user_id": args[3],
|
|
"last_observed_at": None,
|
|
"network_baseline_json": "{}",
|
|
"download_rx_bytes": 0, "upload_tx_bytes": 0,
|
|
})
|
|
elif sql.startswith("SELECT last_observed_at"):
|
|
self.selected = backend.states[(args[0], args[1])]
|
|
elif sql.startswith("UPDATE vm_traffic_totals"):
|
|
key = args[-2], args[-1]
|
|
backend.states[key].update({
|
|
"host_id": args[0], "user_id": args[1],
|
|
"last_observed_at": args[2],
|
|
"network_baseline_json": args[4],
|
|
"download_rx_bytes": args[5], "upload_tx_bytes": args[6],
|
|
})
|
|
elif sql.startswith("INSERT IGNORE INTO vm_traffic_checkpoints"):
|
|
key = args[0], args[1], args[2]
|
|
backend.checkpoints.setdefault(key, {
|
|
"host_id": args[3], "user_id": args[4],
|
|
"first_observed_at": args[5],
|
|
"download_rx_bytes": args[6], "upload_tx_bytes": args[7],
|
|
})
|
|
elif sql.startswith("INSERT INTO vm_metric_samples"):
|
|
key = args[0], args[1], args[4]
|
|
# The production ON DUPLICATE clause deliberately keeps
|
|
# the first interval byte deltas immutable.
|
|
backend.samples.setdefault(key, {
|
|
"network_rx_bytes": args[-2], "network_tx_bytes": args[-1],
|
|
"host_id": args[2], "user_id": args[3],
|
|
})
|
|
|
|
def fetchone(self):
|
|
return dict(self.selected)
|
|
|
|
class Connection:
|
|
def cursor(self):
|
|
return Cursor()
|
|
|
|
def commit(self):
|
|
backend.commits += 1
|
|
|
|
def rollback(self):
|
|
raise AssertionError("unexpected rollback")
|
|
|
|
def close(self):
|
|
return None
|
|
|
|
return Connection()
|
|
|
|
|
|
def traffic_point(timestamp, devices, *, host_id="hk", user_id=41):
|
|
return {
|
|
"vm_id": VM_ID, "timestamp": timestamp,
|
|
"cpu_percent": None, "memory_used_mib": None, "memory_total_mib": None,
|
|
"disk_read_bps": None, "disk_write_bps": None,
|
|
"network_rx_bps": None, "network_tx_bps": None,
|
|
"network_devices": devices, "host_id": host_id, "user_id": user_id,
|
|
}
|
|
|
|
|
|
def test_durable_traffic_baseline_survives_store_restart_and_is_idempotent():
|
|
scope = host_scope({**HOST, "host_id": "hk"})
|
|
backend = TrafficBackend()
|
|
first = DatabaseStore.__new__(DatabaseStore)
|
|
first._connect = backend.connection
|
|
first.save_metric_samples(scope, [traffic_point(1000, {"vnet0": [1000, 2000]})])
|
|
|
|
# A fresh object represents an engine restart; the baseline is in MariaDB.
|
|
restarted = DatabaseStore.__new__(DatabaseStore)
|
|
restarted._connect = backend.connection
|
|
restarted.save_metric_samples(scope, [traffic_point(1060, {"vnet0": [1250, 2400]})])
|
|
state = backend.states[(scope, VM_ID)]
|
|
assert state["download_rx_bytes"] == 250 # guest download
|
|
assert state["upload_tx_bytes"] == 400 # guest upload
|
|
assert json.loads(state["network_baseline_json"]) == {"vnet0": [1250, 2400]}
|
|
assert backend.samples[(scope, VM_ID, 1060)] == {
|
|
"network_rx_bytes": 250, "network_tx_bytes": 400,
|
|
"host_id": "hk", "user_id": 41,
|
|
}
|
|
assert backend.checkpoints[(scope, VM_ID, 0)] == {
|
|
"host_id": "hk", "user_id": 41, "first_observed_at": 1000,
|
|
"download_rx_bytes": 0, "upload_tx_bytes": 0,
|
|
}
|
|
|
|
# Duplicate and older observations neither advance totals nor replace the
|
|
# durable baseline used by the next valid interval.
|
|
restarted.save_metric_samples(scope, [traffic_point(1060, {"vnet0": [9999, 9999]})])
|
|
restarted.save_metric_samples(scope, [traffic_point(1030, {"vnet0": [8000, 8000]})])
|
|
restarted.save_metric_samples(scope, [traffic_point(1120, {"vnet0": [1300, 2500]})])
|
|
state = backend.states[(scope, VM_ID)]
|
|
assert state["download_rx_bytes"] == 300
|
|
assert state["upload_tx_bytes"] == 500
|
|
assert backend.samples[(scope, VM_ID, 1060)]["network_rx_bytes"] == 250
|
|
|
|
|
|
def test_incomplete_network_frame_does_not_replace_durable_baseline():
|
|
scope = host_scope({**HOST, "host_id": "hk"})
|
|
backend = TrafficBackend()
|
|
store = DatabaseStore.__new__(DatabaseStore)
|
|
store._connect = backend.connection
|
|
identity = "mac:52:54:00:12:34:56"
|
|
store.save_metric_samples(scope, [traffic_point(1000, {identity: [1000, 2000]})])
|
|
store.save_metric_samples(scope, [traffic_point(1060, None)])
|
|
|
|
state = backend.states[(scope, VM_ID)]
|
|
assert state["last_observed_at"] == 1000
|
|
assert json.loads(state["network_baseline_json"]) == {identity: [1000, 2000]}
|
|
assert backend.samples[(scope, VM_ID, 1060)]["network_rx_bytes"] is None
|
|
|
|
store.save_metric_samples(scope, [traffic_point(1120, {identity: [1400, 2600]})])
|
|
state = backend.states[(scope, VM_ID)]
|
|
assert state["download_rx_bytes"] == 400
|
|
assert state["upload_tx_bytes"] == 600
|
|
|
|
|
|
def test_same_mac_counter_reset_and_host_tap_rename_still_accumulates():
|
|
previous = {"mac:52:54:00:12:34:56": [9000, 12000]}
|
|
current = {"mac:52:54:00:12:34:56": [700, 1100]}
|
|
assert DatabaseStore._traffic_increment(previous, current) == (700, 1100)
|
|
|
|
|
|
def test_traffic_rollbacks_new_and_removed_nics_do_not_hide_stable_nics():
|
|
previous = {"vnet0": [1000, 2000], "vnet1": [500, 800], "removed": [9, 9]}
|
|
current = {"vnet0": [10, 20], "vnet1": [650, 1100], "new": [9999, 9999]}
|
|
# vnet0 is a known stable identity whose counters restarted at zero, so
|
|
# its post-reset bytes remain billable. New/removed NICs establish/no
|
|
# longer have a baseline, while the stable vnet1 increment is exact.
|
|
assert DatabaseStore._traffic_increment(previous, current) == (160, 320)
|
|
|
|
|
|
def test_sampler_rejects_frame_with_duplicate_network_identity():
|
|
sampler = MetricSampler()
|
|
point = sampler.observe(host_scope(HOST), raw(extra={
|
|
"net.count": 4,
|
|
"net.1.name": "vnet2", "net.1.rx.bytes": 3000, "net.1.tx.bytes": 4000,
|
|
"net.2.name": "duplicate", "net.2.rx.bytes": 1, "net.2.tx.bytes": 2,
|
|
"net.3.name": "duplicate", "net.3.rx.bytes": 3, "net.3.tx.bytes": 4,
|
|
}), timestamp=1000, monotonic=10)
|
|
assert point["network_devices"] is None
|
|
|
|
|
|
def test_sampler_prefers_stable_network_identity_over_renamed_host_tap():
|
|
sampler = MetricSampler()
|
|
point = sampler.observe(host_scope(HOST), raw(extra={
|
|
"net.0.identity": "mac:52:54:00:12:34:56",
|
|
}), timestamp=1000, monotonic=10)
|
|
assert point["network_devices"] == {
|
|
"mac:52:54:00:12:34:56": [1000, 2000],
|
|
}
|
|
|
|
|
|
def test_collector_associates_host_and_current_owner_without_hiding_unowned_vms():
|
|
saved = []
|
|
|
|
class Store:
|
|
def save_metric_samples(self, scope, points):
|
|
saved.append((scope, points))
|
|
|
|
def prune_metric_samples(self, _cutoff):
|
|
return 0
|
|
|
|
class Service:
|
|
def metric_counters(self):
|
|
second = raw()
|
|
return [second, RawVMStats(
|
|
"00000000-0000-4000-8000-000000000002", second.generation,
|
|
second.vcpus, second.values)]
|
|
|
|
owners = {VM_ID: 41}
|
|
profile = {**HOST, "host_id": "hk", "key_locked": False}
|
|
collector = MetricsCollector(
|
|
lambda: Store(), lambda: profile, lambda _profile: Service(),
|
|
owner_getter=lambda _scope, vm_id: owners.get(vm_id),
|
|
)
|
|
collector.sample_once(timestamp=1000, monotonic=10)
|
|
_, points = saved[0]
|
|
assert [(point["host_id"], point["user_id"]) for point in points] == [
|
|
("hk", 41), ("hk", None),
|
|
]
|
|
assert all(point["network_devices"] == {"vnet1": [1000, 2000]}
|
|
for point in points)
|
|
|
|
|
|
def test_metric_prune_is_one_index_ordered_batch_per_transaction():
|
|
connection = FakeConnection(delete_counts=[5000] * 10)
|
|
store = DatabaseStore.__new__(DatabaseStore)
|
|
store._connect = lambda: connection
|
|
assert store.prune_metric_samples(1234) == 5000
|
|
assert len(connection.fake_cursor.commands) == 1
|
|
assert connection.commits == 1 and connection.closed
|
|
assert all("ORDER BY sampled_at LIMIT %s" in command
|
|
and args == (1234, 5000)
|
|
for command, args in connection.fake_cursor.commands)
|
|
|
|
|
|
def test_metric_prune_reports_short_final_batch():
|
|
connection = FakeConnection(delete_counts=[275])
|
|
store = DatabaseStore.__new__(DatabaseStore)
|
|
store._connect = lambda: connection
|
|
assert store.prune_metric_samples(1234) == 275
|
|
assert len(connection.fake_cursor.commands) == 1
|
|
assert connection.commits == 1
|
|
|
|
|
|
def test_metric_store_rejects_invalid_host_and_uuid_before_db_query():
|
|
store = DatabaseStore.__new__(DatabaseStore)
|
|
store._connect = lambda: pytest.fail("invalid scope must not query database")
|
|
with pytest.raises(ValueError):
|
|
store.query_metric_samples("bad-scope", VM_ID, since=0, until=1,
|
|
interval_seconds=60)
|
|
with pytest.raises(ValueError):
|
|
store.query_metric_samples(host_scope(HOST), "not-uuid", since=0, until=1,
|
|
interval_seconds=60)
|
|
|
|
|
|
def test_daily_traffic_query_rejects_invalid_identity_owner_and_range_before_db():
|
|
store = DatabaseStore.__new__(DatabaseStore)
|
|
store._connect = lambda: pytest.fail("invalid daily query must not reach database")
|
|
scope = host_scope(HOST)
|
|
common = {"host_id": "hk", "user_id": 41, "since": 0, "until": 1}
|
|
for vm_scope, vm_id, changes in (
|
|
("bad-scope", VM_ID, {}),
|
|
(scope, "not-a-uuid", {}),
|
|
(scope, VM_ID, {"host_id": "bad host"}),
|
|
(scope, VM_ID, {"user_id": 0}),
|
|
(scope, VM_ID, {"until": 30 * 86400 + 1}),
|
|
(scope, VM_ID, {"limit": 31}),
|
|
):
|
|
with pytest.raises(ValueError):
|
|
store.query_daily_traffic(vm_scope, vm_id, **{**common, **changes})
|
|
|
|
|
|
def test_collector_prunes_when_ssh_locked_and_starts_after_db_configuration():
|
|
class Store:
|
|
def __init__(self):
|
|
self.pruned = []
|
|
self.saved = []
|
|
self.pruned_event = threading.Event()
|
|
self.saved_event = threading.Event()
|
|
|
|
def prune_metric_samples(self, cutoff):
|
|
self.pruned.append(cutoff)
|
|
self.pruned_event.set()
|
|
return 0
|
|
|
|
def save_metric_samples(self, scope, points):
|
|
self.saved.append((scope, points))
|
|
self.saved_event.set()
|
|
|
|
current = {"store": None, "profile": None}
|
|
store = Store()
|
|
class Service:
|
|
def metric_counters(self):
|
|
return [raw()]
|
|
collector = MetricsCollector(lambda: current["store"],
|
|
lambda: current["profile"],
|
|
lambda _profile: Service(), interval_seconds=0.01)
|
|
collector.start()
|
|
current["profile"] = {**HOST, "key_locked": True}
|
|
current["store"] = store
|
|
try:
|
|
assert store.pruned_event.wait(timeout=2)
|
|
assert store.pruned and not store.saved
|
|
current["profile"]["key_locked"] = False
|
|
assert store.saved_event.wait(timeout=2)
|
|
assert store.saved
|
|
finally:
|
|
collector.close()
|
|
|
|
|
|
def test_background_collector_keeps_sampling_without_an_http_session():
|
|
"""The 30-day history must not depend on a logged-in browser polling it."""
|
|
saved_twice = threading.Event()
|
|
|
|
class Store:
|
|
def __init__(self):
|
|
self.saved = []
|
|
|
|
def save_metric_samples(self, scope, points):
|
|
self.saved.append((scope, points))
|
|
if len(self.saved) >= 2:
|
|
saved_twice.set()
|
|
|
|
def prune_metric_samples(self, _cutoff):
|
|
return 0
|
|
|
|
class Service:
|
|
calls = 0
|
|
|
|
def metric_counters(self):
|
|
self.calls += 1
|
|
return [raw(cpu=self.calls * 1_000_000_000,
|
|
block_read=self.calls * 1000,
|
|
network_rx=self.calls * 1000)]
|
|
|
|
store = Store()
|
|
service = Service()
|
|
profile = {**HOST, "key_locked": False}
|
|
collector = MetricsCollector(lambda: store, lambda: profile,
|
|
lambda _profile: service,
|
|
interval_seconds=0.01)
|
|
collector.start()
|
|
try:
|
|
assert saved_twice.wait(timeout=2)
|
|
finally:
|
|
collector.close()
|
|
|
|
assert service.calls >= 2
|
|
assert all(scope == host_scope(HOST) for scope, _ in store.saved)
|
|
assert all(points[0]["vm_id"] == VM_ID for _, points in store.saved)
|
|
|
|
|
|
def test_metric_cleanup_failure_does_not_discard_current_sample(caplog):
|
|
events = []
|
|
attempted = threading.Event()
|
|
|
|
class Store:
|
|
def save_metric_samples(self, _scope, points):
|
|
events.append(("saved", points))
|
|
|
|
def prune_metric_samples(self, _cutoff):
|
|
events.append(("prune", None))
|
|
attempted.set()
|
|
raise RuntimeError("secret database details")
|
|
|
|
class Service:
|
|
def metric_counters(self):
|
|
return [raw()]
|
|
|
|
collector = MetricsCollector(lambda: Store(), lambda: HOST,
|
|
lambda _profile: Service(), interval_seconds=1)
|
|
collector.start()
|
|
assert attempted.wait(timeout=2)
|
|
collector.close()
|
|
assert [event[0] for event in events] == ["saved", "prune"]
|
|
assert "RuntimeError" in caplog.text
|
|
assert "secret database details" not in caplog.text
|
|
|
|
|
|
def test_backlog_cleanup_continues_in_other_thread_without_stalling_samples():
|
|
second_saved = threading.Event()
|
|
prune_entered = threading.Event()
|
|
release_prune = threading.Event()
|
|
backlog_drained = threading.Event()
|
|
|
|
class Store:
|
|
def __init__(self):
|
|
self.saved = 0
|
|
self.pruned = 0
|
|
|
|
def save_metric_samples(self, _scope, _points):
|
|
self.saved += 1
|
|
if self.saved >= 2:
|
|
second_saved.set()
|
|
|
|
def prune_metric_samples(self, _cutoff):
|
|
self.pruned += 1
|
|
if self.pruned == 1:
|
|
prune_entered.set()
|
|
assert release_prune.wait(timeout=2)
|
|
if self.pruned < 3:
|
|
return 5000
|
|
backlog_drained.set()
|
|
return 0
|
|
|
|
class Service:
|
|
def metric_counters(self):
|
|
return [raw()]
|
|
|
|
store = Store()
|
|
collector = MetricsCollector(lambda: store, lambda: HOST,
|
|
lambda _profile: Service(), interval_seconds=0.01)
|
|
collector.start()
|
|
try:
|
|
assert prune_entered.wait(timeout=2)
|
|
assert second_saved.wait(timeout=2)
|
|
release_prune.set()
|
|
assert backlog_drained.wait(timeout=2)
|
|
assert store.pruned >= 3
|
|
finally:
|
|
release_prune.set()
|
|
collector.close()
|
|
|
|
|
|
def test_collector_thread_starts_before_db_configuration_and_stops():
|
|
ready = threading.Event()
|
|
current = {"store": None}
|
|
|
|
class Store:
|
|
def prune_metric_samples(self, _cutoff):
|
|
ready.set()
|
|
return 0
|
|
|
|
collector = MetricsCollector(lambda: current["store"], lambda: None,
|
|
lambda _profile: pytest.fail("no host configured"),
|
|
interval_seconds=0.01)
|
|
collector.start()
|
|
current["store"] = Store()
|
|
assert ready.wait(timeout=2)
|
|
collector.close()
|
|
assert collector._thread is not None and not collector._thread.is_alive()
|
|
assert collector._prune_thread is not None and not collector._prune_thread.is_alive()
|