You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/engine/tests/test_metrics.py

778 lines
29 KiB
Python

"""VM metric math, DB persistence, and the process-local collection lifecycle."""
from __future__ import annotations
import json
import threading
from decimal import Decimal
import pytest
from virsh_panel.database import DatabaseStore
from virsh_panel.metrics import (
MAX_QUERY_POINTS, RANGES, MetricSampler, MetricsCollector, RawVMStats,
host_scope, query_interval_seconds,
)
VM_ID = "00000000-0000-4000-8000-000000000001"
HOST = {"host": "hypervisor.example", "port": 2222,
"fingerprint": "SHA256:one", "storage_pool": "/home/kvm"}
def raw(*, generation=12, cpu=1_000_000_000, block_read=100,
block_write=200, network_rx=1000, network_tx=2000,
memory_time=1000, extra=None):
values = {
"cpu.time": cpu,
"balloon.available": 2 * 1024 * 1024,
"balloon.usable": 512 * 1024,
"balloon.last-update": memory_time,
"block.count": 1,
"block.0.name": "vda",
"block.0.rd.bytes": block_read,
"block.0.wr.bytes": block_write,
"net.count": 1,
"net.0.name": "vnet1",
"net.0.rx.bytes": network_rx,
"net.0.tx.bytes": network_tx,
}
values.update(extra or {})
return RawVMStats(VM_ID, generation, 2, values)
def test_host_scope_ignores_pool_but_changes_with_ssh_host_identity():
first = host_scope(HOST)
assert first == host_scope({**HOST, "storage_pool": "/different"})
assert first != host_scope({**HOST, "fingerprint": "SHA256:two"})
assert first != host_scope({**HOST, "port": 22})
def test_interval_caps_all_ranges_at_600_epoch_aligned_buckets():
for seconds in RANGES.values():
interval = query_interval_seconds(seconds)
assert interval % 60 == 0
assert seconds // interval + 2 <= MAX_QUERY_POINTS
def test_initial_sample_and_fresh_guest_memory_then_rates():
sampler = MetricSampler()
scope = host_scope(HOST)
first = sampler.observe(scope, raw(), timestamp=1000, monotonic=10)
assert first["cpu_percent"] is None
assert first["disk_read_bps"] is None
assert first["network_rx_bps"] is None
assert first["memory_used_mib"] == 1536
assert first["memory_total_mib"] == 2048
second = sampler.observe(
scope,
raw(cpu=61_000_000_000, block_read=700, block_write=1400,
network_rx=2200, network_tx=2600, memory_time=1050),
timestamp=1060, monotonic=70,
)
assert second["cpu_percent"] == 50
assert second["disk_read_bps"] == 10
assert second["disk_write_bps"] == 20
assert second["network_rx_bps"] == 20
assert second["network_tx_bps"] == 10
def test_cpu_time_over_vcpu_capacity_is_displayed_as_100_percent():
sampler = MetricSampler()
scope = host_scope(HOST)
sampler.observe(scope, raw(), timestamp=1000, monotonic=10)
# 130 CPU-seconds across two vCPUs in a 60-second interval; libvirt can
# include QEMU auxiliary threads in cpu.time.
point = sampler.observe(scope, raw(cpu=131_000_000_000),
timestamp=1060, monotonic=70)
assert point["cpu_percent"] == 100
def test_memory_missing_or_stale_balloon_stats_never_uses_rss_or_current():
sampler = MetricSampler()
scope = host_scope(HOST)
values = {"balloon.current": 2 * 1024 * 1024, "balloon.rss": 512 * 1024}
for extra in (
{"balloon.last-update": 1},
{"balloon.usable": None},
{"balloon.available": None},
):
point = sampler.observe(scope, raw(extra={**values, **extra}),
timestamp=1000, monotonic=10)
assert point["memory_used_mib"] is None
assert point["memory_total_mib"] is None
def test_counter_reset_reboot_device_change_and_partial_data_are_unknown():
sampler = MetricSampler()
scope = host_scope(HOST)
sampler.observe(scope, raw(), timestamp=1000, monotonic=10)
reset = sampler.observe(scope, raw(cpu=100, block_read=1, network_rx=1),
timestamp=1060, monotonic=70)
assert reset["cpu_percent"] is None
assert reset["disk_read_bps"] is None
assert reset["network_rx_bps"] is None
# Even if a second disk masks the first disk's decreasing counter in the
# aggregate, device change must not produce a transient spike.
changed = sampler.observe(scope, raw(extra={
"block.count": 2, "block.1.name": "vdb",
"block.1.rd.bytes": 10_000, "block.1.wr.bytes": 20_000,
}), timestamp=1120, monotonic=130)
assert changed["disk_read_bps"] is None
assert changed["disk_write_bps"] is None
partial = sampler.observe(scope, raw(extra={
"block.count": 2, "block.1.name": "vdb",
"block.1.rd.bytes": 11_000,
}), timestamp=1180, monotonic=190)
assert partial["disk_write_bps"] is None # second device missing wr.bytes
assert partial["disk_read_bps"] is not None
reboot = sampler.observe(scope, raw(generation=13, cpu=200_000_000_000),
timestamp=1240, monotonic=250)
assert reboot["cpu_percent"] is None
assert reboot["network_tx_bps"] is None
def test_cdrom_without_byte_counters_does_not_hide_disk_rates():
sampler = MetricSampler()
scope = host_scope(HOST)
cdrom = {"block.count": 2, "block.1.name": "sda"}
sampler.observe(scope, raw(extra=cdrom), timestamp=1000, monotonic=10)
second = sampler.observe(
scope, raw(block_read=700, block_write=1400, extra={
**cdrom, "balloon.last-update": 1060,
}), timestamp=1060, monotonic=70,
)
assert second["disk_read_bps"] == 10
assert second["disk_write_bps"] == 20
def test_single_device_counter_rollback_is_not_masked_by_another_device():
sampler = MetricSampler()
scope = host_scope(HOST)
second_disk = {"block.count": 2, "block.1.name": "vdb",
"block.1.rd.bytes": 1_000, "block.1.wr.bytes": 1_000}
sampler.observe(scope, raw(extra=second_disk), timestamp=1000, monotonic=10)
point = sampler.observe(scope, raw(block_read=1, block_write=1, extra={
**second_disk, "block.1.rd.bytes": 10_000,
"block.1.wr.bytes": 10_000,
}), timestamp=1060, monotonic=70)
assert point["disk_read_bps"] is None
assert point["disk_write_bps"] is None
def test_unobserved_long_gap_is_not_averaged_as_current_rate():
sampler = MetricSampler()
scope = host_scope(HOST)
sampler.observe(scope, raw(), timestamp=1000, monotonic=10)
delayed = sampler.observe(scope, raw(cpu=50_000_000_000, block_read=2000),
timestamp=1600, monotonic=610)
assert delayed["cpu_percent"] is None
assert delayed["disk_read_bps"] is None
class FakeCursor:
def __init__(self, rows=(), delete_counts=()):
self.rows = rows
self.commands = []
self.delete_counts = list(delete_counts)
self.rowcount = 0
self.one = {
"last_observed_at": None,
"network_baseline_json": "{}",
"download_rx_bytes": 0,
"upload_tx_bytes": 0,
}
def __enter__(self):
return self
def __exit__(self, *_args):
return None
def execute(self, sql, args=None):
self.commands.append((sql, args))
self.rowcount = (self.delete_counts.pop(0)
if sql.startswith("DELETE FROM vm_metric_samples")
and self.delete_counts else 0)
def executemany(self, sql, args):
self.commands.append((sql, args))
def fetchall(self):
return self.rows
def fetchone(self):
return self.one
class FakeConnection:
def __init__(self, rows=(), delete_counts=()):
self.fake_cursor = FakeCursor(rows, delete_counts)
self.commits = 0
self.closed = False
def cursor(self):
return self.fake_cursor
def commit(self):
self.commits += 1
def rollback(self):
raise AssertionError("unexpected rollback")
def close(self):
self.closed = True
def test_metric_and_user_tables_created_with_record_table():
connection = FakeConnection()
store = DatabaseStore.__new__(DatabaseStore)
store._connect = lambda: connection
store._initialize_schema()
assert len(connection.fake_cursor.commands) == 13
assert "vm_metric_samples" in connection.fake_cursor.commands[1][0]
assert "vm_metric_retention" in connection.fake_cursor.commands[1][0]
assert "CREATE TABLE IF NOT EXISTS vm_traffic_totals" in connection.fake_cursor.commands[2][0]
assert "CREATE TABLE IF NOT EXISTS vm_traffic_checkpoints" in connection.fake_cursor.commands[3][0]
assert all("ALTER TABLE vm_metric_samples" in connection.fake_cursor.commands[index][0]
for index in range(4, 9))
assert "CREATE TABLE IF NOT EXISTS users" in connection.fake_cursor.commands[9][0]
assert "CREATE TABLE IF NOT EXISTS registration_codes" in connection.fake_cursor.commands[10][0]
assert "CREATE TABLE IF NOT EXISTS vm_ownership" in connection.fake_cursor.commands[11][0]
assert "CREATE TABLE IF NOT EXISTS user_ssh_keys" in connection.fake_cursor.commands[12][0]
assert connection.commits == 1 and connection.closed
def test_metric_store_prunes_physically_and_queries_only_scoped_recent_data():
scope = host_scope(HOST)
connection = FakeConnection([{
"timestamp": 1000,
"cpu_percent": 12.5,
"memory_used_mib": 1024,
"memory_total_mib": 2048,
"disk_read_bps": None,
"disk_write_bps": None,
"network_rx_bps": 256,
"network_tx_bps": 0,
}])
store = DatabaseStore.__new__(DatabaseStore)
store._connect = lambda: connection
point = {"vm_id": VM_ID, "timestamp": 1000, "cpu_percent": 12.5,
"memory_used_mib": 1024, "memory_total_mib": 2048,
"disk_read_bps": None, "disk_write_bps": None,
"network_rx_bps": 256, "network_tx_bps": 0,
"network_devices": {}}
store.save_metric_samples(scope, [point])
store.prune_metric_samples(500)
result = store.query_metric_samples(scope, VM_ID, since=500, until=1100,
interval_seconds=60, limit=600)
assert result[0]["cpu_percent"] == 12.5
assert result[0]["disk_read_bps"] is None
traffic_insert, locked, traffic_update, checkpoint, insert, delete, query = connection.fake_cursor.commands
assert "INSERT IGNORE INTO vm_traffic_totals" in traffic_insert[0]
assert "FOR UPDATE" in locked[0]
assert "UPDATE vm_traffic_totals" in traffic_update[0]
assert "INSERT IGNORE INTO vm_traffic_checkpoints" in checkpoint[0]
assert checkpoint[1] == (scope, VM_ID, 0, None, None, 1000, 0, 0)
assert "INSERT INTO vm_metric_samples" in insert[0]
assert "DELETE FROM vm_metric_samples WHERE sampled_at < %s" in delete[0]
assert delete[1] == (500, 5000)
assert "host_scope=%s AND vm_uuid=%s AND sampled_at >= %s" in query[0]
assert "LIMIT %s" in query[0]
assert query[1] == (scope, VM_ID, 500, 1100, 60, 600)
assert connection.commits == 2 and connection.closed
def test_daily_traffic_query_sums_scoped_owner_bytes_by_utc_day():
scope = host_scope({**HOST, "host_id": "hk"})
connection = FakeConnection([
{"day_start": Decimal(0), "download_bytes": Decimal(1500),
"upload_bytes": Decimal(250)},
{"day_start": Decimal(86400), "download_bytes": None,
"upload_bytes": None},
])
store = DatabaseStore.__new__(DatabaseStore)
store._connect = lambda: connection
result = store.query_daily_traffic(
scope, VM_ID, host_id="hk", user_id=41,
since=0, until=172799, limit=2,
)
assert result == [
{"day_start": 0, "download_bytes": 1500,
"upload_bytes": 250, "total_bytes": 1750},
{"day_start": 86400, "download_bytes": None,
"upload_bytes": None, "total_bytes": None},
]
query, args = connection.fake_cursor.commands[0]
assert "SUM(network_rx_bytes) AS download_bytes" in query
assert "SUM(network_tx_bytes) AS upload_bytes" in query
assert "host_scope=%s AND vm_uuid=%s" in query
assert "host_id <=> %s AND user_id <=> %s" in query
assert "GROUP BY FLOOR(sampled_at / 86400)" in query
assert args == (scope, VM_ID, "hk", 41, 0, 172799, 2)
assert connection.closed
class TrafficBackend:
"""Tiny transaction fake for durable accounting behavior tests."""
def __init__(self):
self.states = {}
self.samples = {}
self.checkpoints = {}
self.commands = []
self.commits = 0
def connection(self):
backend = self
class Cursor:
selected = None
def __enter__(self):
return self
def __exit__(self, *_args):
return None
def execute(self, sql, args=None):
backend.commands.append((sql, args))
if sql.startswith("INSERT IGNORE INTO vm_traffic_totals"):
key = args[0], args[1]
backend.states.setdefault(key, {
"host_id": args[2], "user_id": args[3],
"last_observed_at": None,
"network_baseline_json": "{}",
"download_rx_bytes": 0, "upload_tx_bytes": 0,
})
elif sql.startswith("SELECT last_observed_at"):
self.selected = backend.states[(args[0], args[1])]
elif sql.startswith("UPDATE vm_traffic_totals"):
key = args[-2], args[-1]
backend.states[key].update({
"host_id": args[0], "user_id": args[1],
"last_observed_at": args[2],
"network_baseline_json": args[4],
"download_rx_bytes": args[5], "upload_tx_bytes": args[6],
})
elif sql.startswith("INSERT IGNORE INTO vm_traffic_checkpoints"):
key = args[0], args[1], args[2]
backend.checkpoints.setdefault(key, {
"host_id": args[3], "user_id": args[4],
"first_observed_at": args[5],
"download_rx_bytes": args[6], "upload_tx_bytes": args[7],
})
elif sql.startswith("INSERT INTO vm_metric_samples"):
key = args[0], args[1], args[4]
# The production ON DUPLICATE clause deliberately keeps
# the first interval byte deltas immutable.
backend.samples.setdefault(key, {
"network_rx_bytes": args[-2], "network_tx_bytes": args[-1],
"host_id": args[2], "user_id": args[3],
})
def fetchone(self):
return dict(self.selected)
class Connection:
def cursor(self):
return Cursor()
def commit(self):
backend.commits += 1
def rollback(self):
raise AssertionError("unexpected rollback")
def close(self):
return None
return Connection()
def traffic_point(timestamp, devices, *, host_id="hk", user_id=41):
return {
"vm_id": VM_ID, "timestamp": timestamp,
"cpu_percent": None, "memory_used_mib": None, "memory_total_mib": None,
"disk_read_bps": None, "disk_write_bps": None,
"network_rx_bps": None, "network_tx_bps": None,
"network_devices": devices, "host_id": host_id, "user_id": user_id,
}
def test_durable_traffic_baseline_survives_store_restart_and_is_idempotent():
scope = host_scope({**HOST, "host_id": "hk"})
backend = TrafficBackend()
first = DatabaseStore.__new__(DatabaseStore)
first._connect = backend.connection
first.save_metric_samples(scope, [traffic_point(1000, {"vnet0": [1000, 2000]})])
# A fresh object represents an engine restart; the baseline is in MariaDB.
restarted = DatabaseStore.__new__(DatabaseStore)
restarted._connect = backend.connection
restarted.save_metric_samples(scope, [traffic_point(1060, {"vnet0": [1250, 2400]})])
state = backend.states[(scope, VM_ID)]
assert state["download_rx_bytes"] == 250 # guest download
assert state["upload_tx_bytes"] == 400 # guest upload
assert json.loads(state["network_baseline_json"]) == {"vnet0": [1250, 2400]}
assert backend.samples[(scope, VM_ID, 1060)] == {
"network_rx_bytes": 250, "network_tx_bytes": 400,
"host_id": "hk", "user_id": 41,
}
assert backend.checkpoints[(scope, VM_ID, 0)] == {
"host_id": "hk", "user_id": 41, "first_observed_at": 1000,
"download_rx_bytes": 0, "upload_tx_bytes": 0,
}
# Duplicate and older observations neither advance totals nor replace the
# durable baseline used by the next valid interval.
restarted.save_metric_samples(scope, [traffic_point(1060, {"vnet0": [9999, 9999]})])
restarted.save_metric_samples(scope, [traffic_point(1030, {"vnet0": [8000, 8000]})])
restarted.save_metric_samples(scope, [traffic_point(1120, {"vnet0": [1300, 2500]})])
state = backend.states[(scope, VM_ID)]
assert state["download_rx_bytes"] == 300
assert state["upload_tx_bytes"] == 500
assert backend.samples[(scope, VM_ID, 1060)]["network_rx_bytes"] == 250
def test_incomplete_network_frame_does_not_replace_durable_baseline():
scope = host_scope({**HOST, "host_id": "hk"})
backend = TrafficBackend()
store = DatabaseStore.__new__(DatabaseStore)
store._connect = backend.connection
identity = "mac:52:54:00:12:34:56"
store.save_metric_samples(scope, [traffic_point(1000, {identity: [1000, 2000]})])
store.save_metric_samples(scope, [traffic_point(1060, None)])
state = backend.states[(scope, VM_ID)]
assert state["last_observed_at"] == 1000
assert json.loads(state["network_baseline_json"]) == {identity: [1000, 2000]}
assert backend.samples[(scope, VM_ID, 1060)]["network_rx_bytes"] is None
store.save_metric_samples(scope, [traffic_point(1120, {identity: [1400, 2600]})])
state = backend.states[(scope, VM_ID)]
assert state["download_rx_bytes"] == 400
assert state["upload_tx_bytes"] == 600
def test_same_mac_counter_reset_and_host_tap_rename_still_accumulates():
previous = {"mac:52:54:00:12:34:56": [9000, 12000]}
current = {"mac:52:54:00:12:34:56": [700, 1100]}
assert DatabaseStore._traffic_increment(previous, current) == (700, 1100)
def test_traffic_rollbacks_new_and_removed_nics_do_not_hide_stable_nics():
previous = {"vnet0": [1000, 2000], "vnet1": [500, 800], "removed": [9, 9]}
current = {"vnet0": [10, 20], "vnet1": [650, 1100], "new": [9999, 9999]}
# vnet0 is a known stable identity whose counters restarted at zero, so
# its post-reset bytes remain billable. New/removed NICs establish/no
# longer have a baseline, while the stable vnet1 increment is exact.
assert DatabaseStore._traffic_increment(previous, current) == (160, 320)
def test_sampler_rejects_frame_with_duplicate_network_identity():
sampler = MetricSampler()
point = sampler.observe(host_scope(HOST), raw(extra={
"net.count": 4,
"net.1.name": "vnet2", "net.1.rx.bytes": 3000, "net.1.tx.bytes": 4000,
"net.2.name": "duplicate", "net.2.rx.bytes": 1, "net.2.tx.bytes": 2,
"net.3.name": "duplicate", "net.3.rx.bytes": 3, "net.3.tx.bytes": 4,
}), timestamp=1000, monotonic=10)
assert point["network_devices"] is None
def test_sampler_prefers_stable_network_identity_over_renamed_host_tap():
sampler = MetricSampler()
point = sampler.observe(host_scope(HOST), raw(extra={
"net.0.identity": "mac:52:54:00:12:34:56",
}), timestamp=1000, monotonic=10)
assert point["network_devices"] == {
"mac:52:54:00:12:34:56": [1000, 2000],
}
def test_sampler_billing_frame_only_contains_explicit_public_role():
sampler = MetricSampler()
point = sampler.observe(host_scope(HOST), raw(extra={
"net.count": 2,
"net.0.identity": "mac:52:54:00:12:34:56",
"net.0.role": "public",
"net.1.name": "vnet2",
"net.1.identity": "mac:52:54:00:aa:bb:cc",
"net.1.role": "private",
"net.1.rx.bytes": 3000,
"net.1.tx.bytes": 4000,
}), timestamp=1000, monotonic=10)
assert point["network_billing_devices"] == {
"mac:52:54:00:12:34:56": [1000, 2000],
}
def test_sampler_billing_rejects_ambiguous_multinic_frame():
sampler = MetricSampler()
point = sampler.observe(host_scope(HOST), raw(extra={
"net.count": 2,
"net.1.name": "vnet2",
"net.1.rx.bytes": 3000,
"net.1.tx.bytes": 4000,
}), timestamp=1000, monotonic=10)
assert point["network_billing_devices"] is None
def test_collector_associates_host_and_current_owner_without_hiding_unowned_vms():
saved = []
class Store:
def save_metric_samples(self, scope, points):
saved.append((scope, points))
def prune_metric_samples(self, _cutoff):
return 0
class Service:
def metric_counters(self):
second = raw()
return [second, RawVMStats(
"00000000-0000-4000-8000-000000000002", second.generation,
second.vcpus, second.values)]
owners = {VM_ID: 41}
profile = {**HOST, "host_id": "hk", "key_locked": False}
collector = MetricsCollector(
lambda: Store(), lambda: profile, lambda _profile: Service(),
owner_getter=lambda _scope, vm_id: owners.get(vm_id),
)
collector.sample_once(timestamp=1000, monotonic=10)
_, points = saved[0]
assert [(point["host_id"], point["user_id"]) for point in points] == [
("hk", 41), ("hk", None),
]
assert all(point["network_devices"] == {"vnet1": [1000, 2000]}
for point in points)
def test_metric_prune_is_one_index_ordered_batch_per_transaction():
connection = FakeConnection(delete_counts=[5000] * 10)
store = DatabaseStore.__new__(DatabaseStore)
store._connect = lambda: connection
assert store.prune_metric_samples(1234) == 5000
assert len(connection.fake_cursor.commands) == 1
assert connection.commits == 1 and connection.closed
assert all("ORDER BY sampled_at LIMIT %s" in command
and args == (1234, 5000)
for command, args in connection.fake_cursor.commands)
def test_metric_prune_reports_short_final_batch():
connection = FakeConnection(delete_counts=[275])
store = DatabaseStore.__new__(DatabaseStore)
store._connect = lambda: connection
assert store.prune_metric_samples(1234) == 275
assert len(connection.fake_cursor.commands) == 1
assert connection.commits == 1
def test_metric_store_rejects_invalid_host_and_uuid_before_db_query():
store = DatabaseStore.__new__(DatabaseStore)
store._connect = lambda: pytest.fail("invalid scope must not query database")
with pytest.raises(ValueError):
store.query_metric_samples("bad-scope", VM_ID, since=0, until=1,
interval_seconds=60)
with pytest.raises(ValueError):
store.query_metric_samples(host_scope(HOST), "not-uuid", since=0, until=1,
interval_seconds=60)
def test_daily_traffic_query_rejects_invalid_identity_owner_and_range_before_db():
store = DatabaseStore.__new__(DatabaseStore)
store._connect = lambda: pytest.fail("invalid daily query must not reach database")
scope = host_scope(HOST)
common = {"host_id": "hk", "user_id": 41, "since": 0, "until": 1}
for vm_scope, vm_id, changes in (
("bad-scope", VM_ID, {}),
(scope, "not-a-uuid", {}),
(scope, VM_ID, {"host_id": "bad host"}),
(scope, VM_ID, {"user_id": 0}),
(scope, VM_ID, {"until": 30 * 86400 + 1}),
(scope, VM_ID, {"limit": 31}),
):
with pytest.raises(ValueError):
store.query_daily_traffic(vm_scope, vm_id, **{**common, **changes})
def test_collector_prunes_when_ssh_locked_and_starts_after_db_configuration():
class Store:
def __init__(self):
self.pruned = []
self.saved = []
self.pruned_event = threading.Event()
self.saved_event = threading.Event()
def prune_metric_samples(self, cutoff):
self.pruned.append(cutoff)
self.pruned_event.set()
return 0
def save_metric_samples(self, scope, points):
self.saved.append((scope, points))
self.saved_event.set()
current = {"store": None, "profile": None}
store = Store()
class Service:
def metric_counters(self):
return [raw()]
collector = MetricsCollector(lambda: current["store"],
lambda: current["profile"],
lambda _profile: Service(), interval_seconds=0.01)
collector.start()
current["profile"] = {**HOST, "key_locked": True}
current["store"] = store
try:
assert store.pruned_event.wait(timeout=2)
assert store.pruned and not store.saved
current["profile"]["key_locked"] = False
assert store.saved_event.wait(timeout=2)
assert store.saved
finally:
collector.close()
def test_background_collector_keeps_sampling_without_an_http_session():
"""The 30-day history must not depend on a logged-in browser polling it."""
saved_twice = threading.Event()
class Store:
def __init__(self):
self.saved = []
def save_metric_samples(self, scope, points):
self.saved.append((scope, points))
if len(self.saved) >= 2:
saved_twice.set()
def prune_metric_samples(self, _cutoff):
return 0
class Service:
calls = 0
def metric_counters(self):
self.calls += 1
return [raw(cpu=self.calls * 1_000_000_000,
block_read=self.calls * 1000,
network_rx=self.calls * 1000)]
store = Store()
service = Service()
profile = {**HOST, "key_locked": False}
collector = MetricsCollector(lambda: store, lambda: profile,
lambda _profile: service,
interval_seconds=0.01)
collector.start()
try:
assert saved_twice.wait(timeout=2)
finally:
collector.close()
assert service.calls >= 2
assert all(scope == host_scope(HOST) for scope, _ in store.saved)
assert all(points[0]["vm_id"] == VM_ID for _, points in store.saved)
def test_metric_cleanup_failure_does_not_discard_current_sample(caplog):
events = []
attempted = threading.Event()
class Store:
def save_metric_samples(self, _scope, points):
events.append(("saved", points))
def prune_metric_samples(self, _cutoff):
events.append(("prune", None))
attempted.set()
raise RuntimeError("secret database details")
class Service:
def metric_counters(self):
return [raw()]
collector = MetricsCollector(lambda: Store(), lambda: HOST,
lambda _profile: Service(), interval_seconds=1)
collector.start()
assert attempted.wait(timeout=2)
collector.close()
assert [event[0] for event in events] == ["saved", "prune"]
assert "RuntimeError" in caplog.text
assert "secret database details" not in caplog.text
def test_backlog_cleanup_continues_in_other_thread_without_stalling_samples():
second_saved = threading.Event()
prune_entered = threading.Event()
release_prune = threading.Event()
backlog_drained = threading.Event()
class Store:
def __init__(self):
self.saved = 0
self.pruned = 0
def save_metric_samples(self, _scope, _points):
self.saved += 1
if self.saved >= 2:
second_saved.set()
def prune_metric_samples(self, _cutoff):
self.pruned += 1
if self.pruned == 1:
prune_entered.set()
assert release_prune.wait(timeout=2)
if self.pruned < 3:
return 5000
backlog_drained.set()
return 0
class Service:
def metric_counters(self):
return [raw()]
store = Store()
collector = MetricsCollector(lambda: store, lambda: HOST,
lambda _profile: Service(), interval_seconds=0.01)
collector.start()
try:
assert prune_entered.wait(timeout=2)
assert second_saved.wait(timeout=2)
release_prune.set()
assert backlog_drained.wait(timeout=2)
assert store.pruned >= 3
finally:
release_prune.set()
collector.close()
def test_collector_thread_starts_before_db_configuration_and_stops():
ready = threading.Event()
current = {"store": None}
class Store:
def prune_metric_samples(self, _cutoff):
ready.set()
return 0
collector = MetricsCollector(lambda: current["store"], lambda: None,
lambda _profile: pytest.fail("no host configured"),
interval_seconds=0.01)
collector.start()
current["store"] = Store()
assert ready.wait(timeout=2)
collector.close()
assert collector._thread is not None and not collector._thread.is_alive()
assert collector._prune_thread is not None and not collector._prune_thread.is_alive()