"""VM metric math, DB persistence, and the process-local collection lifecycle.""" from __future__ import annotations import json import threading from decimal import Decimal import pytest from virsh_panel.database import DatabaseStore from virsh_panel.metrics import ( MAX_QUERY_POINTS, RANGES, MetricSampler, MetricsCollector, RawVMStats, host_scope, query_interval_seconds, ) VM_ID = "00000000-0000-4000-8000-000000000001" HOST = {"host": "hypervisor.example", "port": 2222, "fingerprint": "SHA256:one", "storage_pool": "/home/kvm"} def raw(*, generation=12, cpu=1_000_000_000, block_read=100, block_write=200, network_rx=1000, network_tx=2000, memory_time=1000, extra=None): values = { "cpu.time": cpu, "balloon.available": 2 * 1024 * 1024, "balloon.usable": 512 * 1024, "balloon.last-update": memory_time, "block.count": 1, "block.0.name": "vda", "block.0.rd.bytes": block_read, "block.0.wr.bytes": block_write, "net.count": 1, "net.0.name": "vnet1", "net.0.rx.bytes": network_rx, "net.0.tx.bytes": network_tx, } values.update(extra or {}) return RawVMStats(VM_ID, generation, 2, values) def test_host_scope_ignores_pool_but_changes_with_ssh_host_identity(): first = host_scope(HOST) assert first == host_scope({**HOST, "storage_pool": "/different"}) assert first != host_scope({**HOST, "fingerprint": "SHA256:two"}) assert first != host_scope({**HOST, "port": 22}) def test_interval_caps_all_ranges_at_600_epoch_aligned_buckets(): for seconds in RANGES.values(): interval = query_interval_seconds(seconds) assert interval % 60 == 0 assert seconds // interval + 2 <= MAX_QUERY_POINTS def test_initial_sample_and_fresh_guest_memory_then_rates(): sampler = MetricSampler() scope = host_scope(HOST) first = sampler.observe(scope, raw(), timestamp=1000, monotonic=10) assert first["cpu_percent"] is None assert first["disk_read_bps"] is None assert first["network_rx_bps"] is None assert first["memory_used_mib"] == 1536 assert first["memory_total_mib"] == 2048 second = sampler.observe( scope, raw(cpu=61_000_000_000, block_read=700, block_write=1400, network_rx=2200, network_tx=2600, memory_time=1050), timestamp=1060, monotonic=70, ) assert second["cpu_percent"] == 50 assert second["disk_read_bps"] == 10 assert second["disk_write_bps"] == 20 assert second["network_rx_bps"] == 20 assert second["network_tx_bps"] == 10 def test_cpu_time_over_vcpu_capacity_is_displayed_as_100_percent(): sampler = MetricSampler() scope = host_scope(HOST) sampler.observe(scope, raw(), timestamp=1000, monotonic=10) # 130 CPU-seconds across two vCPUs in a 60-second interval; libvirt can # include QEMU auxiliary threads in cpu.time. point = sampler.observe(scope, raw(cpu=131_000_000_000), timestamp=1060, monotonic=70) assert point["cpu_percent"] == 100 def test_memory_missing_or_stale_balloon_stats_never_uses_rss_or_current(): sampler = MetricSampler() scope = host_scope(HOST) values = {"balloon.current": 2 * 1024 * 1024, "balloon.rss": 512 * 1024} for extra in ( {"balloon.last-update": 1}, {"balloon.usable": None}, {"balloon.available": None}, ): point = sampler.observe(scope, raw(extra={**values, **extra}), timestamp=1000, monotonic=10) assert point["memory_used_mib"] is None assert point["memory_total_mib"] is None def test_counter_reset_reboot_device_change_and_partial_data_are_unknown(): sampler = MetricSampler() scope = host_scope(HOST) sampler.observe(scope, raw(), timestamp=1000, monotonic=10) reset = sampler.observe(scope, raw(cpu=100, block_read=1, network_rx=1), timestamp=1060, monotonic=70) assert reset["cpu_percent"] is None assert reset["disk_read_bps"] is None assert reset["network_rx_bps"] is None # Even if a second disk masks the first disk's decreasing counter in the # aggregate, device change must not produce a transient spike. changed = sampler.observe(scope, raw(extra={ "block.count": 2, "block.1.name": "vdb", "block.1.rd.bytes": 10_000, "block.1.wr.bytes": 20_000, }), timestamp=1120, monotonic=130) assert changed["disk_read_bps"] is None assert changed["disk_write_bps"] is None partial = sampler.observe(scope, raw(extra={ "block.count": 2, "block.1.name": "vdb", "block.1.rd.bytes": 11_000, }), timestamp=1180, monotonic=190) assert partial["disk_write_bps"] is None # second device missing wr.bytes assert partial["disk_read_bps"] is not None reboot = sampler.observe(scope, raw(generation=13, cpu=200_000_000_000), timestamp=1240, monotonic=250) assert reboot["cpu_percent"] is None assert reboot["network_tx_bps"] is None def test_cdrom_without_byte_counters_does_not_hide_disk_rates(): sampler = MetricSampler() scope = host_scope(HOST) cdrom = {"block.count": 2, "block.1.name": "sda"} sampler.observe(scope, raw(extra=cdrom), timestamp=1000, monotonic=10) second = sampler.observe( scope, raw(block_read=700, block_write=1400, extra={ **cdrom, "balloon.last-update": 1060, }), timestamp=1060, monotonic=70, ) assert second["disk_read_bps"] == 10 assert second["disk_write_bps"] == 20 def test_single_device_counter_rollback_is_not_masked_by_another_device(): sampler = MetricSampler() scope = host_scope(HOST) second_disk = {"block.count": 2, "block.1.name": "vdb", "block.1.rd.bytes": 1_000, "block.1.wr.bytes": 1_000} sampler.observe(scope, raw(extra=second_disk), timestamp=1000, monotonic=10) point = sampler.observe(scope, raw(block_read=1, block_write=1, extra={ **second_disk, "block.1.rd.bytes": 10_000, "block.1.wr.bytes": 10_000, }), timestamp=1060, monotonic=70) assert point["disk_read_bps"] is None assert point["disk_write_bps"] is None def test_unobserved_long_gap_is_not_averaged_as_current_rate(): sampler = MetricSampler() scope = host_scope(HOST) sampler.observe(scope, raw(), timestamp=1000, monotonic=10) delayed = sampler.observe(scope, raw(cpu=50_000_000_000, block_read=2000), timestamp=1600, monotonic=610) assert delayed["cpu_percent"] is None assert delayed["disk_read_bps"] is None class FakeCursor: def __init__(self, rows=(), delete_counts=()): self.rows = rows self.commands = [] self.delete_counts = list(delete_counts) self.rowcount = 0 self.one = { "last_observed_at": None, "network_baseline_json": "{}", "download_rx_bytes": 0, "upload_tx_bytes": 0, } def __enter__(self): return self def __exit__(self, *_args): return None def execute(self, sql, args=None): self.commands.append((sql, args)) self.rowcount = (self.delete_counts.pop(0) if sql.startswith("DELETE FROM vm_metric_samples") and self.delete_counts else 0) def executemany(self, sql, args): self.commands.append((sql, args)) def fetchall(self): return self.rows def fetchone(self): return self.one class FakeConnection: def __init__(self, rows=(), delete_counts=()): self.fake_cursor = FakeCursor(rows, delete_counts) self.commits = 0 self.closed = False def cursor(self): return self.fake_cursor def commit(self): self.commits += 1 def rollback(self): raise AssertionError("unexpected rollback") def close(self): self.closed = True def test_metric_and_user_tables_created_with_record_table(): connection = FakeConnection() store = DatabaseStore.__new__(DatabaseStore) store._connect = lambda: connection store._initialize_schema() assert len(connection.fake_cursor.commands) == 13 assert "vm_metric_samples" in connection.fake_cursor.commands[1][0] assert "vm_metric_retention" in connection.fake_cursor.commands[1][0] assert "CREATE TABLE IF NOT EXISTS vm_traffic_totals" in connection.fake_cursor.commands[2][0] assert "CREATE TABLE IF NOT EXISTS vm_traffic_checkpoints" in connection.fake_cursor.commands[3][0] assert all("ALTER TABLE vm_metric_samples" in connection.fake_cursor.commands[index][0] for index in range(4, 9)) assert "CREATE TABLE IF NOT EXISTS users" in connection.fake_cursor.commands[9][0] assert "CREATE TABLE IF NOT EXISTS registration_codes" in connection.fake_cursor.commands[10][0] assert "CREATE TABLE IF NOT EXISTS vm_ownership" in connection.fake_cursor.commands[11][0] assert "CREATE TABLE IF NOT EXISTS user_ssh_keys" in connection.fake_cursor.commands[12][0] assert connection.commits == 1 and connection.closed def test_metric_store_prunes_physically_and_queries_only_scoped_recent_data(): scope = host_scope(HOST) connection = FakeConnection([{ "timestamp": 1000, "cpu_percent": 12.5, "memory_used_mib": 1024, "memory_total_mib": 2048, "disk_read_bps": None, "disk_write_bps": None, "network_rx_bps": 256, "network_tx_bps": 0, }]) store = DatabaseStore.__new__(DatabaseStore) store._connect = lambda: connection point = {"vm_id": VM_ID, "timestamp": 1000, "cpu_percent": 12.5, "memory_used_mib": 1024, "memory_total_mib": 2048, "disk_read_bps": None, "disk_write_bps": None, "network_rx_bps": 256, "network_tx_bps": 0, "network_devices": {}} store.save_metric_samples(scope, [point]) store.prune_metric_samples(500) result = store.query_metric_samples(scope, VM_ID, since=500, until=1100, interval_seconds=60, limit=600) assert result[0]["cpu_percent"] == 12.5 assert result[0]["disk_read_bps"] is None traffic_insert, locked, traffic_update, checkpoint, insert, delete, query = connection.fake_cursor.commands assert "INSERT IGNORE INTO vm_traffic_totals" in traffic_insert[0] assert "FOR UPDATE" in locked[0] assert "UPDATE vm_traffic_totals" in traffic_update[0] assert "INSERT IGNORE INTO vm_traffic_checkpoints" in checkpoint[0] assert checkpoint[1] == (scope, VM_ID, 0, None, None, 1000, 0, 0) assert "INSERT INTO vm_metric_samples" in insert[0] assert "DELETE FROM vm_metric_samples WHERE sampled_at < %s" in delete[0] assert delete[1] == (500, 5000) assert "host_scope=%s AND vm_uuid=%s AND sampled_at >= %s" in query[0] assert "LIMIT %s" in query[0] assert query[1] == (scope, VM_ID, 500, 1100, 60, 600) assert connection.commits == 2 and connection.closed def test_daily_traffic_query_sums_scoped_owner_bytes_by_utc_day(): scope = host_scope({**HOST, "host_id": "hk"}) connection = FakeConnection([ {"day_start": Decimal(0), "download_bytes": Decimal(1500), "upload_bytes": Decimal(250)}, {"day_start": Decimal(86400), "download_bytes": None, "upload_bytes": None}, ]) store = DatabaseStore.__new__(DatabaseStore) store._connect = lambda: connection result = store.query_daily_traffic( scope, VM_ID, host_id="hk", user_id=41, since=0, until=172799, limit=2, ) assert result == [ {"day_start": 0, "download_bytes": 1500, "upload_bytes": 250, "total_bytes": 1750}, {"day_start": 86400, "download_bytes": None, "upload_bytes": None, "total_bytes": None}, ] query, args = connection.fake_cursor.commands[0] assert "SUM(network_rx_bytes) AS download_bytes" in query assert "SUM(network_tx_bytes) AS upload_bytes" in query assert "host_scope=%s AND vm_uuid=%s" in query assert "host_id <=> %s AND user_id <=> %s" in query assert "GROUP BY FLOOR(sampled_at / 86400)" in query assert args == (scope, VM_ID, "hk", 41, 0, 172799, 2) assert connection.closed class TrafficBackend: """Tiny transaction fake for durable accounting behavior tests.""" def __init__(self): self.states = {} self.samples = {} self.checkpoints = {} self.commands = [] self.commits = 0 def connection(self): backend = self class Cursor: selected = None def __enter__(self): return self def __exit__(self, *_args): return None def execute(self, sql, args=None): backend.commands.append((sql, args)) if sql.startswith("INSERT IGNORE INTO vm_traffic_totals"): key = args[0], args[1] backend.states.setdefault(key, { "host_id": args[2], "user_id": args[3], "last_observed_at": None, "network_baseline_json": "{}", "download_rx_bytes": 0, "upload_tx_bytes": 0, }) elif sql.startswith("SELECT last_observed_at"): self.selected = backend.states[(args[0], args[1])] elif sql.startswith("UPDATE vm_traffic_totals"): key = args[-2], args[-1] backend.states[key].update({ "host_id": args[0], "user_id": args[1], "last_observed_at": args[2], "network_baseline_json": args[4], "download_rx_bytes": args[5], "upload_tx_bytes": args[6], }) elif sql.startswith("INSERT IGNORE INTO vm_traffic_checkpoints"): key = args[0], args[1], args[2] backend.checkpoints.setdefault(key, { "host_id": args[3], "user_id": args[4], "first_observed_at": args[5], "download_rx_bytes": args[6], "upload_tx_bytes": args[7], }) elif sql.startswith("INSERT INTO vm_metric_samples"): key = args[0], args[1], args[4] # The production ON DUPLICATE clause deliberately keeps # the first interval byte deltas immutable. backend.samples.setdefault(key, { "network_rx_bytes": args[-2], "network_tx_bytes": args[-1], "host_id": args[2], "user_id": args[3], }) def fetchone(self): return dict(self.selected) class Connection: def cursor(self): return Cursor() def commit(self): backend.commits += 1 def rollback(self): raise AssertionError("unexpected rollback") def close(self): return None return Connection() def traffic_point(timestamp, devices, *, host_id="hk", user_id=41): return { "vm_id": VM_ID, "timestamp": timestamp, "cpu_percent": None, "memory_used_mib": None, "memory_total_mib": None, "disk_read_bps": None, "disk_write_bps": None, "network_rx_bps": None, "network_tx_bps": None, "network_devices": devices, "host_id": host_id, "user_id": user_id, } def test_durable_traffic_baseline_survives_store_restart_and_is_idempotent(): scope = host_scope({**HOST, "host_id": "hk"}) backend = TrafficBackend() first = DatabaseStore.__new__(DatabaseStore) first._connect = backend.connection first.save_metric_samples(scope, [traffic_point(1000, {"vnet0": [1000, 2000]})]) # A fresh object represents an engine restart; the baseline is in MariaDB. restarted = DatabaseStore.__new__(DatabaseStore) restarted._connect = backend.connection restarted.save_metric_samples(scope, [traffic_point(1060, {"vnet0": [1250, 2400]})]) state = backend.states[(scope, VM_ID)] assert state["download_rx_bytes"] == 250 # guest download assert state["upload_tx_bytes"] == 400 # guest upload assert json.loads(state["network_baseline_json"]) == {"vnet0": [1250, 2400]} assert backend.samples[(scope, VM_ID, 1060)] == { "network_rx_bytes": 250, "network_tx_bytes": 400, "host_id": "hk", "user_id": 41, } assert backend.checkpoints[(scope, VM_ID, 0)] == { "host_id": "hk", "user_id": 41, "first_observed_at": 1000, "download_rx_bytes": 0, "upload_tx_bytes": 0, } # Duplicate and older observations neither advance totals nor replace the # durable baseline used by the next valid interval. restarted.save_metric_samples(scope, [traffic_point(1060, {"vnet0": [9999, 9999]})]) restarted.save_metric_samples(scope, [traffic_point(1030, {"vnet0": [8000, 8000]})]) restarted.save_metric_samples(scope, [traffic_point(1120, {"vnet0": [1300, 2500]})]) state = backend.states[(scope, VM_ID)] assert state["download_rx_bytes"] == 300 assert state["upload_tx_bytes"] == 500 assert backend.samples[(scope, VM_ID, 1060)]["network_rx_bytes"] == 250 def test_incomplete_network_frame_does_not_replace_durable_baseline(): scope = host_scope({**HOST, "host_id": "hk"}) backend = TrafficBackend() store = DatabaseStore.__new__(DatabaseStore) store._connect = backend.connection identity = "mac:52:54:00:12:34:56" store.save_metric_samples(scope, [traffic_point(1000, {identity: [1000, 2000]})]) store.save_metric_samples(scope, [traffic_point(1060, None)]) state = backend.states[(scope, VM_ID)] assert state["last_observed_at"] == 1000 assert json.loads(state["network_baseline_json"]) == {identity: [1000, 2000]} assert backend.samples[(scope, VM_ID, 1060)]["network_rx_bytes"] is None store.save_metric_samples(scope, [traffic_point(1120, {identity: [1400, 2600]})]) state = backend.states[(scope, VM_ID)] assert state["download_rx_bytes"] == 400 assert state["upload_tx_bytes"] == 600 def test_same_mac_counter_reset_and_host_tap_rename_still_accumulates(): previous = {"mac:52:54:00:12:34:56": [9000, 12000]} current = {"mac:52:54:00:12:34:56": [700, 1100]} assert DatabaseStore._traffic_increment(previous, current) == (700, 1100) def test_traffic_rollbacks_new_and_removed_nics_do_not_hide_stable_nics(): previous = {"vnet0": [1000, 2000], "vnet1": [500, 800], "removed": [9, 9]} current = {"vnet0": [10, 20], "vnet1": [650, 1100], "new": [9999, 9999]} # vnet0 is a known stable identity whose counters restarted at zero, so # its post-reset bytes remain billable. New/removed NICs establish/no # longer have a baseline, while the stable vnet1 increment is exact. assert DatabaseStore._traffic_increment(previous, current) == (160, 320) def test_sampler_rejects_frame_with_duplicate_network_identity(): sampler = MetricSampler() point = sampler.observe(host_scope(HOST), raw(extra={ "net.count": 4, "net.1.name": "vnet2", "net.1.rx.bytes": 3000, "net.1.tx.bytes": 4000, "net.2.name": "duplicate", "net.2.rx.bytes": 1, "net.2.tx.bytes": 2, "net.3.name": "duplicate", "net.3.rx.bytes": 3, "net.3.tx.bytes": 4, }), timestamp=1000, monotonic=10) assert point["network_devices"] is None def test_sampler_prefers_stable_network_identity_over_renamed_host_tap(): sampler = MetricSampler() point = sampler.observe(host_scope(HOST), raw(extra={ "net.0.identity": "mac:52:54:00:12:34:56", }), timestamp=1000, monotonic=10) assert point["network_devices"] == { "mac:52:54:00:12:34:56": [1000, 2000], } def test_collector_associates_host_and_current_owner_without_hiding_unowned_vms(): saved = [] class Store: def save_metric_samples(self, scope, points): saved.append((scope, points)) def prune_metric_samples(self, _cutoff): return 0 class Service: def metric_counters(self): second = raw() return [second, RawVMStats( "00000000-0000-4000-8000-000000000002", second.generation, second.vcpus, second.values)] owners = {VM_ID: 41} profile = {**HOST, "host_id": "hk", "key_locked": False} collector = MetricsCollector( lambda: Store(), lambda: profile, lambda _profile: Service(), owner_getter=lambda _scope, vm_id: owners.get(vm_id), ) collector.sample_once(timestamp=1000, monotonic=10) _, points = saved[0] assert [(point["host_id"], point["user_id"]) for point in points] == [ ("hk", 41), ("hk", None), ] assert all(point["network_devices"] == {"vnet1": [1000, 2000]} for point in points) def test_metric_prune_is_one_index_ordered_batch_per_transaction(): connection = FakeConnection(delete_counts=[5000] * 10) store = DatabaseStore.__new__(DatabaseStore) store._connect = lambda: connection assert store.prune_metric_samples(1234) == 5000 assert len(connection.fake_cursor.commands) == 1 assert connection.commits == 1 and connection.closed assert all("ORDER BY sampled_at LIMIT %s" in command and args == (1234, 5000) for command, args in connection.fake_cursor.commands) def test_metric_prune_reports_short_final_batch(): connection = FakeConnection(delete_counts=[275]) store = DatabaseStore.__new__(DatabaseStore) store._connect = lambda: connection assert store.prune_metric_samples(1234) == 275 assert len(connection.fake_cursor.commands) == 1 assert connection.commits == 1 def test_metric_store_rejects_invalid_host_and_uuid_before_db_query(): store = DatabaseStore.__new__(DatabaseStore) store._connect = lambda: pytest.fail("invalid scope must not query database") with pytest.raises(ValueError): store.query_metric_samples("bad-scope", VM_ID, since=0, until=1, interval_seconds=60) with pytest.raises(ValueError): store.query_metric_samples(host_scope(HOST), "not-uuid", since=0, until=1, interval_seconds=60) def test_daily_traffic_query_rejects_invalid_identity_owner_and_range_before_db(): store = DatabaseStore.__new__(DatabaseStore) store._connect = lambda: pytest.fail("invalid daily query must not reach database") scope = host_scope(HOST) common = {"host_id": "hk", "user_id": 41, "since": 0, "until": 1} for vm_scope, vm_id, changes in ( ("bad-scope", VM_ID, {}), (scope, "not-a-uuid", {}), (scope, VM_ID, {"host_id": "bad host"}), (scope, VM_ID, {"user_id": 0}), (scope, VM_ID, {"until": 30 * 86400 + 1}), (scope, VM_ID, {"limit": 31}), ): with pytest.raises(ValueError): store.query_daily_traffic(vm_scope, vm_id, **{**common, **changes}) def test_collector_prunes_when_ssh_locked_and_starts_after_db_configuration(): class Store: def __init__(self): self.pruned = [] self.saved = [] self.pruned_event = threading.Event() self.saved_event = threading.Event() def prune_metric_samples(self, cutoff): self.pruned.append(cutoff) self.pruned_event.set() return 0 def save_metric_samples(self, scope, points): self.saved.append((scope, points)) self.saved_event.set() current = {"store": None, "profile": None} store = Store() class Service: def metric_counters(self): return [raw()] collector = MetricsCollector(lambda: current["store"], lambda: current["profile"], lambda _profile: Service(), interval_seconds=0.01) collector.start() current["profile"] = {**HOST, "key_locked": True} current["store"] = store try: assert store.pruned_event.wait(timeout=2) assert store.pruned and not store.saved current["profile"]["key_locked"] = False assert store.saved_event.wait(timeout=2) assert store.saved finally: collector.close() def test_background_collector_keeps_sampling_without_an_http_session(): """The 30-day history must not depend on a logged-in browser polling it.""" saved_twice = threading.Event() class Store: def __init__(self): self.saved = [] def save_metric_samples(self, scope, points): self.saved.append((scope, points)) if len(self.saved) >= 2: saved_twice.set() def prune_metric_samples(self, _cutoff): return 0 class Service: calls = 0 def metric_counters(self): self.calls += 1 return [raw(cpu=self.calls * 1_000_000_000, block_read=self.calls * 1000, network_rx=self.calls * 1000)] store = Store() service = Service() profile = {**HOST, "key_locked": False} collector = MetricsCollector(lambda: store, lambda: profile, lambda _profile: service, interval_seconds=0.01) collector.start() try: assert saved_twice.wait(timeout=2) finally: collector.close() assert service.calls >= 2 assert all(scope == host_scope(HOST) for scope, _ in store.saved) assert all(points[0]["vm_id"] == VM_ID for _, points in store.saved) def test_metric_cleanup_failure_does_not_discard_current_sample(caplog): events = [] attempted = threading.Event() class Store: def save_metric_samples(self, _scope, points): events.append(("saved", points)) def prune_metric_samples(self, _cutoff): events.append(("prune", None)) attempted.set() raise RuntimeError("secret database details") class Service: def metric_counters(self): return [raw()] collector = MetricsCollector(lambda: Store(), lambda: HOST, lambda _profile: Service(), interval_seconds=1) collector.start() assert attempted.wait(timeout=2) collector.close() assert [event[0] for event in events] == ["saved", "prune"] assert "RuntimeError" in caplog.text assert "secret database details" not in caplog.text def test_backlog_cleanup_continues_in_other_thread_without_stalling_samples(): second_saved = threading.Event() prune_entered = threading.Event() release_prune = threading.Event() backlog_drained = threading.Event() class Store: def __init__(self): self.saved = 0 self.pruned = 0 def save_metric_samples(self, _scope, _points): self.saved += 1 if self.saved >= 2: second_saved.set() def prune_metric_samples(self, _cutoff): self.pruned += 1 if self.pruned == 1: prune_entered.set() assert release_prune.wait(timeout=2) if self.pruned < 3: return 5000 backlog_drained.set() return 0 class Service: def metric_counters(self): return [raw()] store = Store() collector = MetricsCollector(lambda: store, lambda: HOST, lambda _profile: Service(), interval_seconds=0.01) collector.start() try: assert prune_entered.wait(timeout=2) assert second_saved.wait(timeout=2) release_prune.set() assert backlog_drained.wait(timeout=2) assert store.pruned >= 3 finally: release_prune.set() collector.close() def test_collector_thread_starts_before_db_configuration_and_stops(): ready = threading.Event() current = {"store": None} class Store: def prune_metric_samples(self, _cutoff): ready.set() return 0 collector = MetricsCollector(lambda: current["store"], lambda: None, lambda _profile: pytest.fail("no host configured"), interval_seconds=0.01) collector.start() current["store"] = Store() assert ready.wait(timeout=2) collector.close() assert collector._thread is not None and not collector._thread.is_alive() assert collector._prune_thread is not None and not collector._prune_thread.is_alive()