You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudTrafficCollectionTest.php

573 lines
26 KiB
PHTML

<?php
namespace App\Services\Cloud {
final class CloudDaemonActor
{
public static function resolve(string $sessionId): ?array
{
if ($sessionId !== 'cloud-traffic-sync') {
return null;
}
return [
'user_id' => 99,
'is_admin' => true,
'is_super_admin' => true,
'can_manage' => true,
'session_id' => $sessionId,
];
}
}
}
namespace {
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudInstance;
use App\Services\Cloud\CloudAgentClient;
use App\Services\Cloud\CloudError;
use App\Services\Cloud\CloudLogIdentity;
use App\Services\Cloud\CloudTrafficService;
use GuzzleHttp\Client;
use GuzzleHttp\Handler\MockHandler;
use GuzzleHttp\HandlerStack;
use GuzzleHttp\Middleware;
use GuzzleHttp\Psr7\Response;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$expect = static function (callable $callable, int $status,
string $message) use ($assert): void {
try {
$callable();
} catch (CloudError $error) {
$assert($error->httpStatus === $status,
$message . ' Unexpected status: ' . $error->httpStatus);
return;
}
throw new RuntimeException($message);
};
$capsule = new Capsule();
$capsule->addConnection([
'driver' => 'sqlite', 'database' => ':memory:',
], 'default');
$capsule->addConnection([
'driver' => 'sqlite', 'database' => ':memory:',
], 'log');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$capsule->schema('default')->create('cloud_instances',
static function (Blueprint $table): void {
$table->increments('id');
$table->integer('host_id');
$table->integer('user_id');
$table->string('uuid');
$table->string('status');
$table->string('service_status');
$table->text('spec_json');
$table->integer('lifetime_download_bytes')->default(0);
$table->integer('lifetime_upload_bytes')->default(0);
$table->integer('traffic_synced_at')->default(0);
$table->integer('created_at');
$table->integer('updated_at');
$table->unique(['host_id', 'uuid']);
});
$logSchema = $capsule->schema('log');
$logSchema->create('cloud_vm_metric_samples',
static function (Blueprint $table): void {
$table->integer('instance_id');
$table->integer('user_id');
$table->integer('sampled_at');
foreach (['cpu_percent', 'memory_used_mib', 'memory_total_mib',
'disk_read_bps', 'disk_write_bps', 'network_rx_bps',
'network_tx_bps'] as $field) {
$table->double($field)->nullable();
}
$table->integer('network_rx_bytes')->nullable();
$table->integer('network_tx_bytes')->nullable();
$table->primary(['instance_id', 'sampled_at']);
});
$logSchema->create('cloud_vm_metric_rollups',
static function (Blueprint $table): void {
$table->integer('instance_id');
$table->integer('user_id');
$table->integer('interval_seconds');
$table->integer('bucket_start');
$table->integer('sample_count');
$table->integer('last_sampled_at');
foreach (['cpu_percent', 'memory_used_mib', 'memory_total_mib',
'disk_read_bps', 'disk_write_bps', 'network_rx_bps',
'network_tx_bps'] as $field) {
$table->double($field . '_sum')->nullable();
$table->integer($field . '_count');
}
$table->primary([
'instance_id', 'interval_seconds', 'bucket_start',
]);
});
$logSchema->create('cloud_vm_traffic_totals',
static function (Blueprint $table): void {
$table->integer('host_id');
$table->binary('vm_uuid');
$table->integer('user_id');
$table->integer('last_observed_at')->nullable();
$table->binary('network_signature');
$table->text('network_baseline_json');
$table->integer('download_rx_bytes')->default(0);
$table->integer('upload_tx_bytes')->default(0);
$table->integer('updated_at');
$table->primary(['host_id', 'vm_uuid']);
});
$logSchema->create('cloud_vm_traffic_checkpoints',
static function (Blueprint $table): void {
$table->integer('instance_id');
$table->integer('host_id');
$table->binary('vm_uuid');
$table->integer('checkpoint_hour');
$table->integer('user_id');
$table->integer('first_observed_at');
$table->integer('download_rx_bytes');
$table->integer('upload_tx_bytes');
$table->primary(['host_id', 'vm_uuid', 'checkpoint_hour']);
});
$logSchema->create('cloud_vm_traffic_daily_checkpoints',
static function (Blueprint $table): void {
$table->integer('instance_id');
$table->integer('user_id');
$table->integer('checkpoint_day');
$table->integer('first_observed_at');
$table->integer('first_download_rx_bytes');
$table->integer('first_upload_tx_bytes');
$table->integer('last_observed_at');
$table->integer('last_download_rx_bytes');
$table->integer('last_upload_tx_bytes');
$table->integer('sample_count');
$table->integer('created_at');
$table->integer('updated_at');
$table->primary(['instance_id', 'checkpoint_day']);
});
$logSchema->create('cloud_vm_daily_traffic_cursors',
static function (Blueprint $table): void {
$table->integer('instance_id')->primary();
$table->integer('user_id');
$table->integer('last_observed_at');
$table->integer('download_total_bytes');
$table->integer('upload_total_bytes');
$table->integer('updated_at');
});
$logSchema->create('cloud_vm_daily_traffic',
static function (Blueprint $table): void {
$table->integer('instance_id');
$table->integer('user_id');
$table->integer('day_start');
$table->integer('download_bytes')->default(0);
$table->integer('upload_bytes')->default(0);
$table->integer('sample_count')->default(0);
$table->integer('first_observed_at');
$table->integer('last_observed_at');
$table->integer('created_at');
$table->integer('updated_at');
$table->primary(['instance_id', 'day_start']);
});
$now = time();
$secondInHour = $now % 3600;
if ($secondInHour < 120) {
$firstAt = $now - 180;
$secondAt = $now - 120;
} else {
$firstAt = $now - 120;
$secondAt = $now - 60;
}
$resetAt = $secondAt + 60;
$uuid = '11111111-1111-4111-8111-111111111111';
$instance = new CloudInstance();
$instance->host_id = 7;
$instance->user_id = 17;
$instance->uuid = $uuid;
$instance->status = 'active';
$instance->service_status = 'active';
$instance->spec_json = json_encode([
'vcpu' => 2, 'memory_mib' => 2048,
], JSON_THROW_ON_ERROR);
$instance->lifetime_download_bytes = 0;
$instance->lifetime_upload_bytes = 0;
$instance->traffic_synced_at = 0;
$instance->created_at = $instance->updated_at = $now;
$instance->save();
$metrics = static function (int $timestamp, int $cpu, ?int $memory,
int $read, int $write, int $rx, int $tx) use ($uuid): array {
return [
'uuid' => $uuid,
'collected_at' => gmdate('Y-m-d\TH:i:s\Z', $timestamp),
'cpu_time_ns' => $cpu,
'memory_used_kib' => $memory,
'disk_read_bytes' => $read,
'disk_write_bytes' => $write,
'rx_bytes' => $rx,
'tx_bytes' => $tx,
];
};
$firstMetrics = $metrics($firstAt, 10_000_000_000, 1_048_576,
100_000, 50_000, 100_000, 50_000);
$secondMetrics = $metrics($secondAt, 11_200_000_000, 1_101_824,
106_000, 53_000, 112_000, 56_000);
$resetMetrics = $metrics($resetAt, 100_000_000, 524_288,
100, 50, 1_000, 500);
$unavailableMemoryAt = $resetAt + 1;
$unavailableMemoryMetrics = $metrics(
$unavailableMemoryAt, 100_000_000, null, 100, 50, 1_000, 500
);
$log = $capsule->connection('log');
$log->table('cloud_vm_metric_samples')->insert([
'instance_id' => (int) $instance->id,
'user_id' => 17,
'sampled_at' => $now - 31 * 86400,
]);
$history = [];
$mock = new MockHandler([
new Response(200, ['Content-Type' => 'application/json'],
json_encode($firstMetrics, JSON_THROW_ON_ERROR)),
]);
$stack = HandlerStack::create($mock);
$stack->push(Middleware::history($history));
$agent = new CloudAgentClient('7',
'metric-test-secret-at-least-32-bytes', new Client([
'handler' => $stack, 'http_errors' => false,
]), 20.0, 'http://127.0.0.1:8001/internal/agent/request',
'test-gateway-secret-at-least-32-bytes');
$first = CloudTrafficService::collect(1, $agent);
$firstSyncCount = CloudTrafficService::sync(1);
$total = $log->table('cloud_vm_traffic_totals')->first();
$firstSample = $log->table('cloud_vm_metric_samples')
->where('sampled_at', $firstAt)->first();
$checkpoint = $log->table('cloud_vm_traffic_checkpoints')->first();
$dailyCheckpoint = $log->table(
'cloud_vm_traffic_daily_checkpoints'
)->first();
$dailyCursor = $log->table('cloud_vm_daily_traffic_cursors')->first();
$daily = $log->table('cloud_vm_daily_traffic')->first();
$baseline = json_decode((string) $total->network_baseline_json, true,
flags: JSON_THROW_ON_ERROR);
$assert($first === [
'observed' => 1, 'unavailable' => 0, 'skipped' => 0,
'errors' => [],
]
&& count($history) === 1
&& $firstSyncCount === 1
&& $history[0]['request']->getMethod() === 'POST'
&& (($firstEnvelope = json_decode(
(string) $history[0]['request']->getBody(), true,
flags: JSON_THROW_ON_ERROR
))['method'] ?? null) === 'GET'
&& ($firstEnvelope['request_target'] ?? null)
=== '/v1/vms/' . $uuid . '/metrics'
&& $history[0]['request']->getHeaderLine('X-Virt-Signature') !== ''
&& (int) $total->download_rx_bytes === 100_000
&& (int) $total->upload_tx_bytes === 50_000
&& ($baseline['last_agent_rx_bytes'] ?? null) === 100_000
&& ($baseline['last_agent_tx_bytes'] ?? null) === 50_000
&& $firstSample->network_rx_bytes === null
&& $firstSample->network_tx_bytes === null
&& $firstSample->network_rx_bps === null
&& $firstSample->network_tx_bps === null
&& (float) $firstSample->memory_used_mib === 1024.0
&& (float) $firstSample->memory_total_mib === 2048.0
&& (int) $checkpoint->download_rx_bytes === 100_000
&& (int) $checkpoint->upload_tx_bytes === 50_000
&& (int) $dailyCheckpoint->instance_id === (int) $instance->id
&& (int) $dailyCheckpoint->checkpoint_day
=== intdiv($firstAt, 86400) * 86400
&& (int) $dailyCheckpoint->first_observed_at === $firstAt
&& (int) $dailyCheckpoint->last_observed_at === $firstAt
&& (int) $dailyCheckpoint->first_download_rx_bytes === 100_000
&& (int) $dailyCheckpoint->last_download_rx_bytes === 100_000
&& (int) $dailyCheckpoint->sample_count === 1
&& (int) $dailyCursor->instance_id === (int) $instance->id
&& (int) $dailyCursor->download_total_bytes === 100_000
&& (int) $dailyCursor->upload_total_bytes === 50_000
&& (int) $daily->download_bytes === 0
&& (int) $daily->upload_bytes === 0
&& (int) $daily->sample_count === 1
&& !$log->table('cloud_vm_metric_samples')
->where('sampled_at', $now - 31 * 86400)->exists(),
'First signed Agent Metrics DTO did not restore its durable Agent total or prune old samples.');
$store = new ReflectionMethod(CloudTrafficService::class, 'storeMetrics');
$store->invoke(null, $log, $instance, $secondMetrics);
$secondSample = $log->table('cloud_vm_metric_samples')
->where('sampled_at', $secondAt)->first();
$total = $log->table('cloud_vm_traffic_totals')->first();
$dailyCheckpoint = $log->table(
'cloud_vm_traffic_daily_checkpoints'
)->first();
$assert((int) $total->download_rx_bytes === 112_000
&& (int) $total->upload_tx_bytes === 56_000
&& (int) $secondSample->network_rx_bytes === 12_000
&& (int) $secondSample->network_tx_bytes === 6_000
&& abs((float) $secondSample->network_rx_bps - 200.0) < 0.0001
&& abs((float) $secondSample->network_tx_bps - 100.0) < 0.0001
&& abs((float) $secondSample->disk_read_bps - 100.0) < 0.0001
&& abs((float) $secondSample->disk_write_bps - 50.0) < 0.0001
&& abs((float) $secondSample->cpu_percent - 1.0) < 0.0001
&& $log->table('cloud_vm_traffic_checkpoints')->count() === 1
&& (int) $log->table('cloud_vm_traffic_checkpoints')
->value('first_observed_at') === $firstAt,
'Second Agent sample lost its delta/rate math or overwrote the first hourly checkpoint.');
$assert($log->table('cloud_vm_traffic_daily_checkpoints')->count() === 1
&& (int) $dailyCheckpoint->first_observed_at === $firstAt
&& (int) $dailyCheckpoint->last_observed_at === $secondAt
&& (int) $dailyCheckpoint->first_download_rx_bytes === 100_000
&& (int) $dailyCheckpoint->last_download_rx_bytes === 112_000
&& (int) $dailyCheckpoint->last_upload_tx_bytes === 56_000
&& (int) $dailyCheckpoint->sample_count === 2,
'Same-day Agent samples did not compact into one cumulative daily checkpoint.');
$store->invoke(null, $log, $instance, $resetMetrics);
$resetSample = $log->table('cloud_vm_metric_samples')
->where('sampled_at', $resetAt)->first();
$total = $log->table('cloud_vm_traffic_totals')->first();
$assert((int) $total->download_rx_bytes === 113_000
&& (int) $total->upload_tx_bytes === 56_500
&& (int) $resetSample->network_rx_bytes === 1_000
&& (int) $resetSample->network_tx_bytes === 500
&& $resetSample->network_rx_bps === null
&& $resetSample->network_tx_bps === null,
'Raw Agent counter reset lowered PHP billing totals or emitted a false rate spike.');
$store->invoke(null, $log, $instance, $unavailableMemoryMetrics);
$unavailableMemorySample = $log->table('cloud_vm_metric_samples')
->where('sampled_at', $unavailableMemoryAt)->first();
$total = $log->table('cloud_vm_traffic_totals')->first();
$assert($unavailableMemorySample !== null
&& $unavailableMemorySample->memory_used_mib === null
&& (int) $total->download_rx_bytes === 113_000
&& (int) $total->upload_tx_bytes === 56_500,
'Unavailable balloon statistics fabricated memory usage or disrupted traffic totals.');
$duplicate = $metrics($resetAt, 999_000_000, 999_999,
999_999, 999_999, 1_000, 500);
$store->invoke(null, $log, $instance, $duplicate);
$total = $log->table('cloud_vm_traffic_totals')->first();
$assert((int) $total->download_rx_bytes === 113_000
&& (int) $total->upload_tx_bytes === 56_500
&& $log->table('cloud_vm_metric_samples')->count() === 4,
'A repeated durable Agent counter was charged twice.');
$futureObservedAt = $now + 120;
$log->table('cloud_vm_traffic_totals')->update([
'last_observed_at' => $futureObservedAt,
]);
$rollbackMetrics = $metrics($now, 1_000_000_000, 786_432,
1_000, 500, 1_500, 700);
$rollbackHistory = [];
$rollbackMock = new MockHandler([
new Response(200, ['Content-Type' => 'application/json'],
json_encode($rollbackMetrics, JSON_THROW_ON_ERROR)),
new Response(200, ['Content-Type' => 'application/json'],
json_encode($rollbackMetrics, JSON_THROW_ON_ERROR)),
]);
$rollbackStack = HandlerStack::create($rollbackMock);
$rollbackStack->push(Middleware::history($rollbackHistory));
$rollbackAgent = new CloudAgentClient('7',
'metric-test-secret-at-least-32-bytes', new Client([
'handler' => $rollbackStack, 'http_errors' => false,
]), 20.0, 'http://127.0.0.1:8001/internal/agent/request',
'test-gateway-secret-at-least-32-bytes');
$rollbackFirst = CloudTrafficService::collect(1, $rollbackAgent);
$rollbackSecond = CloudTrafficService::collect(1, $rollbackAgent);
$total = $log->table('cloud_vm_traffic_totals')->first();
$rollbackBaseline = json_decode(
(string) $total->network_baseline_json, true,
flags: JSON_THROW_ON_ERROR
);
$assert($rollbackFirst['observed'] === 1
&& $rollbackSecond['observed'] === 1
&& count($rollbackHistory) === 2
&& (int) $total->download_rx_bytes === 113_500
&& (int) $total->upload_tx_bytes === 56_700
&& (int) $total->last_observed_at === $futureObservedAt
&& ($rollbackBaseline['last_agent_rx_bytes'] ?? null) === 1_500
&& ($rollbackBaseline['last_agent_tx_bytes'] ?? null) === 700
&& $log->table('cloud_vm_metric_samples')->count() === 4,
'A future observation blocked clock-rollback collection, duplicated billing, or inserted an out-of-order chart point.');
$wrongUuid = $resetMetrics;
$wrongUuid['uuid'] = '22222222-2222-4222-8222-222222222222';
$expect(static fn () => $store->invoke(
null, $log, $instance, $wrongUuid
), 502, 'Metrics for another VM UUID were accepted.');
$log->table('cloud_vm_traffic_totals')->update(['user_id' => 18]);
$newer = $resetMetrics;
$newer['collected_at'] = gmdate('Y-m-d\TH:i:s\Z', $resetAt + 2);
$expect(static fn () => $store->invoke(
null, $log, $instance, $newer
), 409, 'A traffic total owned by another user was overwritten.');
$log->table('cloud_vm_traffic_totals')->update([
'user_id' => 17, 'host_id' => '8',
]);
$expect(static fn () => $store->invoke(
null, $log, $instance, $newer
), 409, 'A traffic total bound to another host was overwritten.');
$log->table('cloud_vm_traffic_totals')->update([
'user_id' => 17, 'host_id' => '7',
]);
$instance->lifetime_download_bytes = 200_000;
$instance->lifetime_upload_bytes = 100_000;
$instance->traffic_synced_at = 0;
$instance->save();
$assert(CloudTrafficService::sync(10) === 0
&& (int) $instance->fresh()->lifetime_download_bytes === 200_000
&& (int) $instance->fresh()->lifetime_upload_bytes === 100_000,
'A smaller reconstructed total rolled paid lifetime usage backwards.');
$instance->lifetime_download_bytes = 0;
$instance->lifetime_upload_bytes = 0;
$instance->traffic_synced_at = 0;
$instance->save();
$assert(CloudTrafficService::sync(10) === 1
&& (int) $instance->fresh()->lifetime_download_bytes === 113_500
&& (int) $instance->fresh()->lifetime_upload_bytes === 56_700,
'Owner/host/UUID-bound PHP totals were not synchronized to billing state.');
$log->table('cloud_vm_metric_samples')->insert([
'instance_id' => (int) $instance->id,
'user_id' => 999,
'sampled_at' => $now - 10,
'cpu_percent' => 99,
]);
$chart = CloudTrafficService::metrics($instance->fresh(), '1h');
$dayChart = CloudTrafficService::metrics($instance->fresh(), '24h');
$monthChart = CloudTrafficService::metrics($instance->fresh(), '30d');
$assert($chart['interval_seconds'] === 60
&& $chart['points'] !== []
&& $dayChart['interval_seconds'] === 300
&& $dayChart['points'] !== []
&& $monthChart['interval_seconds'] === 7200
&& $monthChart['points'] !== []
&& !in_array($now - 10,
array_column($chart['points'], 'timestamp'), true),
'Tiered chart reads lost data or accepted another owner binding.');
$expect(static fn () => CloudTrafficService::metrics(
$instance->fresh(), '90d'
), 422, 'An unbounded chart range was accepted.');
$migratedUuid = '33333333-3333-4333-8333-333333333333';
$instance->refresh();
$instance->host_id = 8;
$instance->uuid = $migratedUuid;
$instance->lifetime_download_bytes = 113_500;
$instance->lifetime_upload_bytes = 56_700;
$instance->traffic_synced_at = 0;
$instance->spec_json = json_encode([
'vcpu' => 2,
'memory_mib' => 2048,
'traffic_counter_generation' => [
'host_id' => 8,
'uuid' => $migratedUuid,
'download_base_bytes' => 113_500,
'upload_base_bytes' => 56_700,
],
], JSON_THROW_ON_ERROR);
$instance->save();
$migratedMetrics = $metrics(
$now,
100_000_000,
524_288,
100,
50,
1_250,
750
);
$migratedMetrics['uuid'] = $migratedUuid;
$store->invoke(null, $log, $instance, $migratedMetrics);
$migratedTotal = $log->table('cloud_vm_traffic_totals')
->where('vm_uuid', CloudLogIdentity::uuid($migratedUuid))->first();
$migrationSyncCount = CloudTrafficService::sync(10);
$migratedInstance = $instance->fresh();
$assert($migratedTotal !== null
&& (string) $migratedTotal->host_id === '8'
&& (int) $migratedTotal->download_rx_bytes === 114_750
&& (int) $migratedTotal->upload_tx_bytes === 57_450
&& $migrationSyncCount === 1
&& (int) $migratedInstance->lifetime_download_bytes === 114_750
&& (int) $migratedInstance->lifetime_upload_bytes === 57_450,
'A maintenance migration reset or double-counted the lifetime traffic generation: '
. json_encode([
'total' => $migratedTotal === null ? null : [
'host_id' => (int) $migratedTotal->host_id,
'vm_uuid' => bin2hex((string) $migratedTotal->vm_uuid),
'download' => (int) $migratedTotal->download_rx_bytes,
'upload' => (int) $migratedTotal->upload_tx_bytes,
],
'sync_count' => $migrationSyncCount,
'instance_download' =>
(int) $migratedInstance->lifetime_download_bytes,
'instance_upload' =>
(int) $migratedInstance->lifetime_upload_bytes,
'instance' => $migratedInstance->toArray(),
], JSON_THROW_ON_ERROR));
// A staggered deployment must keep cumulative billing alive even before
// the optional daily-log tables have been installed.
$logSchema->drop('cloud_vm_daily_traffic');
$logSchema->drop('cloud_vm_daily_traffic_cursors');
$log->table('cloud_vm_traffic_totals')
->where('vm_uuid', CloudLogIdentity::uuid($migratedUuid))
->update(['last_observed_at' => 0]);
$withoutDailyMetrics = $metrics(
$now + 1, 110_000_000, 524_288, 110, 55, 1_350, 800
);
$withoutDailyMetrics['uuid'] = $migratedUuid;
$withoutDailyMock = new MockHandler([
new Response(200, ['Content-Type' => 'application/json'],
json_encode($withoutDailyMetrics, JSON_THROW_ON_ERROR)),
]);
$withoutDailyAgent = new CloudAgentClient('8',
'metric-test-secret-at-least-32-bytes', new Client([
'handler' => HandlerStack::create($withoutDailyMock),
'http_errors' => false,
]), 20.0, 'http://127.0.0.1:8001/internal/agent/request',
'test-gateway-secret-at-least-32-bytes');
$withoutDaily = CloudTrafficService::collect(1, $withoutDailyAgent);
$withoutDailySyncCount = CloudTrafficService::sync(1);
$withoutDailyTotal = $log->table('cloud_vm_traffic_totals')
->where('vm_uuid', CloudLogIdentity::uuid($migratedUuid))->first();
$assert($withoutDaily === [
'observed' => 1, 'unavailable' => 0, 'skipped' => 0,
'errors' => [],
]
&& $withoutDailySyncCount === 1
&& (int) $withoutDailyTotal->download_rx_bytes === 114_850
&& (int) $withoutDailyTotal->upload_tx_bytes === 57_500,
'Missing daily-log tables blocked the core cumulative traffic write.');
$controller = file_get_contents(dirname(__DIR__)
. '/app/Controllers/CloudApiController.php');
$lifecycle = file_get_contents(dirname(__DIR__)
. '/app/Services/Cloud/CloudLifecycleService.php');
$metricsStart = strpos($controller, 'public function metrics(');
$metricsEnd = $metricsStart === false ? false
: strpos($controller, 'public function dailyTraffic(', $metricsStart);
$metricsAction = $metricsStart === false || $metricsEnd === false ? ''
: substr($controller, $metricsStart, $metricsEnd - $metricsStart);
$assert(str_contains($metricsAction, 'CloudTrafficService::metrics(')
&& !str_contains($metricsAction, 'CloudAgentSettingsService')
&& !str_contains($metricsAction, "->request(")
&& str_contains($lifecycle, 'CloudTrafficService::collect($batchSize)')
&& str_contains($lifecycle, "\$collection['errors']")
&& str_contains($lifecycle, '$this->observeFailure((string) $code)'),
'Metrics page bypassed PHP storage or lifecycle dropped collector observation errors.');
echo "Agent metrics collection, monotonic billing and retained chart contracts passed.\n";
}