You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/app/Services/Cloud/CloudTrafficService.php

635 lines
29 KiB
PHP

<?php
namespace App\Services\Cloud;
use App\Models\CloudInstance;
use App\Models\User;
final class CloudTrafficService
{
private const AGENT_TIMEOUT_SECONDS = 3.0;
private const SAMPLE_MINIMUM_SECONDS = 45;
private const SAMPLE_RETENTION_SECONDS = 30 * 86400;
private static array $retryAfter = [];
/**
* Poll current cumulative counters from each instance Agent and turn them
* into PHP-owned billing totals and bounded chart samples. A failed host or
* VM read is observation-only: it never lowers the last durable total.
*/
public static function collect(int $limit = 500,
?CloudAgentClient $agent = null): array
{
$result = [
'observed' => 0, 'unavailable' => 0, 'skipped' => 0,
'errors' => [],
];
$limit = max(1, min(2000, $limit));
$now = time();
foreach (self::$retryAfter as $key => $retryAt) {
if ($retryAt <= $now) {
unset(self::$retryAfter[$key]);
}
}
$actor = CloudDaemonActor::resolve('cloud-traffic-sync');
if ($actor === null) {
$result['errors'][] = 'traffic_collection_unconfigured';
return $result;
}
try {
$log = User::getConnectionResolver()->connection('log');
$schema = $log->getSchemaBuilder();
foreach ([
'cloud_vm_metric_samples', 'cloud_vm_traffic_totals',
'cloud_vm_traffic_checkpoints',
] as $table) {
if (!$schema->hasTable($table)) {
$result['errors'][] = 'traffic_log_schema_missing';
return $result;
}
}
// Fetch all identities so a permanently unavailable low-ID host
// cannot occupy every bounded pass. Only $limit due rows are polled.
$instances = CloudInstance::query()
->where('status', '<>', 'deleted')
->where('service_status', '<>', 'exempt')
->orderBy('traffic_synced_at')->orderBy('id')->get([
'id', 'host_id', 'user_id', 'uuid', 'spec_json',
'traffic_synced_at',
]);
} catch (\Throwable $error) {
$result['errors'][] = 'traffic_collection_failed';
return $result;
}
$groups = [];
foreach ($instances as $instance) {
$hostId = (int) $instance->host_id;
$uuid = strtolower(trim((string) $instance->uuid));
if ($hostId < 1 || (int) $instance->user_id < 1
|| preg_match('/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/D',
$uuid) !== 1) {
$result['skipped']++;
continue;
}
$groups[$hostId][] = ['model' => $instance, 'uuid' => $uuid];
}
$attempted = 0;
foreach ($groups as $hostId => $owned) {
if ($attempted >= $limit) {
break;
}
$scope = self::hostScope((int) $hostId);
try {
$totals = $log->table('cloud_vm_traffic_totals')
->where('host_scope', $scope)->whereIn('vm_uuid',
array_column($owned, 'uuid'))->get()->keyBy('vm_uuid');
} catch (\Throwable $error) {
$result['errors'][] = 'traffic_collection_failed';
continue;
}
$due = [];
foreach ($owned as $item) {
if ($attempted + count($due) >= $limit) {
break;
}
$retryKey = $hostId . ':' . $item['uuid'];
if ((self::$retryAfter[$retryKey] ?? 0) > $now) {
$result['skipped']++;
continue;
}
$previous = $totals->get($item['uuid']);
$lastObservedAt = $previous === null ? 0
: (int) ($previous->last_observed_at ?? 0);
if ($lastObservedAt <= $now
&& $lastObservedAt > $now - self::SAMPLE_MINIMUM_SECONDS) {
$result['skipped']++;
continue;
}
$due[] = $item;
}
if ($due === []) {
continue;
}
try {
$client = $agent ?? CloudAgentSettingsService::clientForHost(
(int) $hostId, self::AGENT_TIMEOUT_SECONDS
);
} catch (\Throwable $error) {
foreach ($due as $item) {
self::$retryAfter[$hostId . ':' . $item['uuid']] = $now + 60;
}
$attempted += count($due);
$result['unavailable'] += count($due);
$result['errors'][] = 'traffic_agent_unavailable';
continue;
}
foreach ($due as $item) {
$attempted++;
$retryKey = $hostId . ':' . $item['uuid'];
try {
$metrics = $client->request('GET', '/v1/vms/'
. rawurlencode($item['uuid']) . '/metrics', $actor);
self::storeMetrics($log, $item['model'], $metrics);
unset(self::$retryAfter[$retryKey]);
$result['observed']++;
} catch (\Throwable $error) {
self::$retryAfter[$retryKey] = $now + 60;
$result['unavailable']++;
$result['errors'][] = 'traffic_metric_unavailable';
}
}
}
try {
$log->table('cloud_vm_metric_samples')->where(
'sampled_at', '<', $now - self::SAMPLE_RETENTION_SECONDS
)->delete();
} catch (\Throwable $error) {
$result['errors'][] = 'traffic_retention_failed';
}
$result['errors'] = array_values(array_unique($result['errors']));
return $result;
}
public static function sync(int $limit = 500): int
{
$batchSize = max(1, min(2000, $limit));
$updated = 0;
CloudInstance::where('status', '<>', 'deleted')
->where('service_status', '<>', 'exempt')
->select(['id', 'host_id', 'user_id', 'uuid'])
->chunkById($batchSize, function ($instances) use (&$updated) {
$updated += self::syncBatch($instances);
});
try {
$log = User::getConnectionResolver()->connection('log');
if ($log->getSchemaBuilder()->hasTable(
'cloud_vm_daily_traffic'
)) {
CloudDailyTrafficLedger::prune();
}
} catch (\Throwable $error) {
error_log('cloud daily traffic retention unavailable: '
. get_class($error));
}
return $updated;
}
/** Return a bounded, downsampled chart from PHP-owned log storage. */
public static function metrics(CloudInstance $instance, string $range): array
{
$ranges = [
'1h' => [3600, 60], '24h' => [86400, 300],
'7d' => [7 * 86400, 1800], '30d' => [30 * 86400, 7200],
];
if (!isset($ranges[$range])) {
throw new CloudError('指标时间范围无效');
}
[$seconds, $interval] = $ranges[$range];
$uuid = strtolower(trim((string) $instance->uuid));
if (preg_match('/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/D',
$uuid) !== 1) {
throw new CloudError('实例标识无效', 409);
}
try {
$log = User::getConnectionResolver()->connection('log');
if (!$log->getSchemaBuilder()->hasTable('cloud_vm_metric_samples')) {
throw new CloudError('性能采样库尚未初始化', 503);
}
$driver = $log->getDriverName();
$bucket = $driver === 'sqlite'
? 'CAST(sampled_at / ' . $interval . ' AS INTEGER)'
: 'FLOOR(sampled_at / ' . $interval . ')';
$fields = [
'cpu_percent', 'memory_used_mib', 'memory_total_mib',
'disk_read_bps', 'disk_write_bps', 'network_rx_bps',
'network_tx_bps',
];
$select = $bucket . ' AS metric_bucket, MAX(sampled_at) AS timestamp';
foreach ($fields as $field) {
$select .= ', AVG(' . $field . ') AS ' . $field;
}
$rows = $log->table('cloud_vm_metric_samples')
->where('host_scope', self::hostScope((int) $instance->host_id))
->where('vm_uuid', $uuid)
->where('host_id', (string) (int) $instance->host_id)
->where('user_id', (int) $instance->user_id)
->where('sampled_at', '>=', time() - $seconds - $interval)
->selectRaw($select)->groupByRaw($bucket)
->orderBy('metric_bucket')->limit(600)->get();
} catch (CloudError $error) {
throw $error;
} catch (\Throwable $error) {
throw new CloudError('性能采样记录暂时无法读取', 503);
}
$points = [];
foreach ($rows as $row) {
$point = ['timestamp' => max(0, (int) $row->timestamp)];
foreach ($fields as $field) {
$value = $row->{$field};
$point[$field] = $value === null ? null : max(0.0, (float) $value);
}
$points[] = $point;
}
return ['interval_seconds' => $interval, 'points' => $points];
}
private static function storeMetrics($log, CloudInstance $instance,
array $metrics): void
{
$uuid = strtolower((string) $instance->uuid);
if (!is_string($metrics['uuid'] ?? null)
|| !hash_equals($uuid, strtolower(trim($metrics['uuid'])))) {
throw new CloudError('Agent 指标响应与实例不匹配', 502);
}
$collectedAt = self::metricTimestamp($metrics['collected_at'] ?? null);
$current = [
'cpu_time_ns' => self::counter($metrics, 'cpu_time_ns'),
'memory_used_kib' => self::nullableCounter(
$metrics, 'memory_used_kib'
),
'disk_read_bytes' => self::counter($metrics, 'disk_read_bytes'),
'disk_write_bytes' => self::counter($metrics, 'disk_write_bytes'),
'rx_bytes' => self::counter($metrics, 'rx_bytes'),
'tx_bytes' => self::counter($metrics, 'tx_bytes'),
];
$scope = self::hostScope((int) $instance->host_id);
$log->transaction(static function () use (
$log, $instance, $uuid, $collectedAt, $current, $scope
): void {
$previous = $log->table('cloud_vm_traffic_totals')
->where('host_scope', $scope)->where('vm_uuid', $uuid)
->lockForUpdate()->first();
if ($previous !== null) {
if (($previous->user_id !== null
&& (int) $previous->user_id !== (int) $instance->user_id)
|| ($previous->host_id !== null
&& (string) $previous->host_id
!== (string) (int) $instance->host_id)) {
throw new CloudError('性能采样归属与实例不一致', 409);
}
}
$baseline = $previous === null ? [] : json_decode(
(string) ($previous->network_baseline_json ?? ''), true
);
$baseline = is_array($baseline) && !array_is_list($baseline)
? $baseline : [];
$previousObservedAt = $previous === null ? 0
: (int) ($previous->last_observed_at ?? 0);
$orderedSample = $previous === null
|| $collectedAt > $previousObservedAt;
// The Agent's RX/TX counters are durable cumulative totals. A host
// clock correction must not freeze quota accounting, but its old
// timestamp is not suitable for chart ordering or rate math.
$elapsed = $orderedSample && $previous !== null
? $collectedAt - $previousObservedAt : 0;
$spec = json_decode((string) $instance->spec_json, true);
$spec = is_array($spec) && !array_is_list($spec) ? $spec : [];
$vcpu = filter_var($spec['vcpu'] ?? ($spec['cpu_cores'] ?? null),
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1, 'max_range' => 64]]);
$memoryTotal = filter_var($spec['memory_mib'] ?? null,
FILTER_VALIDATE_INT,
['options' => ['min_range' => 1, 'max_range' => 262144]]);
$delta = static function (string $field) use (
$baseline, $current, $elapsed
): ?float {
if ($elapsed <= 0 || !isset($baseline[$field])
|| !is_int($baseline[$field])) {
return null;
}
// CPU and disk counters can restart with the guest. Bytes
// observed since zero are still a safe delta for this interval.
$change = $current[$field] >= $baseline[$field]
? $current[$field] - $baseline[$field]
: $current[$field];
return $change / $elapsed;
};
$cpuPerSecond = $delta('cpu_time_ns');
$cpu = $cpuPerSecond === null || $vcpu === false ? null
: min(100.0, max(0.0,
$cpuPerSecond / 1000000000 / (int) $vcpu * 100));
$lastAgentRx = $baseline['last_agent_rx_bytes'] ?? null;
$lastAgentTx = $baseline['last_agent_tx_bytes'] ?? null;
$rawBaselineKnown = $previous !== null && is_int($lastAgentRx)
&& $lastAgentRx >= 0 && is_int($lastAgentTx)
&& $lastAgentTx >= 0;
$rxReset = $rawBaselineKnown
&& $current['rx_bytes'] < $lastAgentRx;
$txReset = $rawBaselineKnown
&& $current['tx_bytes'] < $lastAgentTx;
$networkRx = !$rawBaselineKnown ? null
: ($rxReset ? $current['rx_bytes']
: $current['rx_bytes'] - $lastAgentRx);
$networkTx = !$rawBaselineKnown ? null
: ($txReset ? $current['tx_bytes']
: $current['tx_bytes'] - $lastAgentTx);
// The Agent counters come from its durable traffic store. On the
// first PHP observation they are already the authoritative lifetime
// total (including after a panel log-database restore), while there
// is deliberately no interval from which to derive a rate.
$generation = $spec['traffic_counter_generation'] ?? null;
$generation = is_array($generation) && !array_is_list($generation)
&& (int) ($generation['host_id'] ?? 0)
=== (int) $instance->host_id
&& hash_equals(
strtolower((string) ($generation['uuid'] ?? '')),
$uuid
)
&& is_int($generation['download_base_bytes'] ?? null)
&& $generation['download_base_bytes'] >= 0
&& is_int($generation['upload_base_bytes'] ?? null)
&& $generation['upload_base_bytes'] >= 0
? $generation : null;
$baseRx = $previous === null && $generation !== null
? $generation['download_base_bytes'] : 0;
$baseTx = $previous === null && $generation !== null
? $generation['upload_base_bytes'] : 0;
if ($current['rx_bytes'] > PHP_INT_MAX - $baseRx
|| $current['tx_bytes'] > PHP_INT_MAX - $baseTx) {
throw new CloudError('迁移后的累计流量超出可记录范围', 502);
}
$panelRx = $previous === null
? $baseRx + $current['rx_bytes']
: (int) $previous->download_rx_bytes;
$panelTx = $previous === null
? $baseTx + $current['tx_bytes']
: (int) $previous->upload_tx_bytes;
if ($networkRx !== null) {
if ($networkRx > PHP_INT_MAX - $panelRx) {
throw new CloudError('累计下载流量超出可记录范围', 502);
}
$panelRx += $networkRx;
}
if ($networkTx !== null) {
if ($networkTx > PHP_INT_MAX - $panelTx) {
throw new CloudError('累计上传流量超出可记录范围', 502);
}
$panelTx += $networkTx;
}
$sample = [
'host_scope' => $scope,
'vm_uuid' => $uuid,
'host_id' => (string) (int) $instance->host_id,
'user_id' => (int) $instance->user_id,
'sampled_at' => $collectedAt,
'cpu_percent' => $cpu,
'memory_used_mib' => $current['memory_used_kib'] === null
? null : $current['memory_used_kib'] / 1024,
'memory_total_mib' => $memoryTotal === false
? null : (float) $memoryTotal,
'disk_read_bps' => $delta('disk_read_bytes'),
'disk_write_bps' => $delta('disk_write_bytes'),
'network_rx_bps' => $elapsed > 0 && $networkRx !== null
&& !$rxReset
? $networkRx / $elapsed : null,
'network_tx_bps' => $elapsed > 0 && $networkTx !== null
&& !$txReset
? $networkTx / $elapsed : null,
'network_rx_bytes' => $networkRx,
'network_tx_bytes' => $networkTx,
];
if ($orderedSample) {
$log->table('cloud_vm_metric_samples')->insertOrIgnore($sample);
$log->table('cloud_vm_traffic_checkpoints')->insertOrIgnore([
'host_scope' => $scope,
'vm_uuid' => $uuid,
'checkpoint_hour' => intdiv($collectedAt, 3600) * 3600,
'host_id' => (string) (int) $instance->host_id,
'user_id' => (int) $instance->user_id,
'first_observed_at' => $collectedAt,
'download_rx_bytes' => $panelRx,
'upload_tx_bytes' => $panelTx,
]);
}
$encoded = json_encode([
'version' => 1,
'cpu_time_ns' => $current['cpu_time_ns'],
'memory_used_kib' => $current['memory_used_kib'],
'disk_read_bytes' => $current['disk_read_bytes'],
'disk_write_bytes' => $current['disk_write_bytes'],
'last_agent_rx_bytes' => $current['rx_bytes'],
'last_agent_tx_bytes' => $current['tx_bytes'],
],
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES);
$values = [
'host_id' => (string) (int) $instance->host_id,
'user_id' => (int) $instance->user_id,
'last_observed_at' => max($previousObservedAt, $collectedAt),
'network_signature' => hash('sha256',
'virt-agent-durable-traffic/v1'),
'network_baseline_json' => $encoded,
'download_rx_bytes' => $panelRx,
'upload_tx_bytes' => $panelTx,
'updated_at' => time(),
];
if ($previous === null) {
$log->table('cloud_vm_traffic_totals')->insert($values + [
'host_scope' => $scope, 'vm_uuid' => $uuid,
]);
} else {
$log->table('cloud_vm_traffic_totals')
->where('host_scope', $scope)->where('vm_uuid', $uuid)
->update($values);
}
}, 3);
}
private static function counter(array $metrics, string $field): int
{
$value = $metrics[$field] ?? null;
if (!is_int($value) || $value < 0) {
throw new CloudError('Agent 指标计数器格式无效', 502);
}
return $value;
}
private static function nullableCounter(array $metrics,
string $field): ?int
{
if (!array_key_exists($field, $metrics)) {
throw new CloudError('Agent 指标计数器格式无效', 502);
}
$value = $metrics[$field];
if ($value === null) {
return null;
}
if (!is_int($value) || $value < 0) {
throw new CloudError('Agent 指标计数器格式无效', 502);
}
return $value;
}
private static function metricTimestamp($value): int
{
if (!is_string($value) || strlen($value) < 20 || strlen($value) > 40
|| preg_match('/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,9})?Z$/D',
$value) !== 1) {
throw new CloudError('Agent 指标采集时间格式无效', 502);
}
$timestamp = strtotime($value);
if ($timestamp === false || abs(time() - $timestamp) > 300) {
throw new CloudError('Agent 指标采集时间不可信', 502);
}
return $timestamp;
}
private static function hostScope(int $hostId): string
{
return hash('sha256', 'virt-cloud/host/' . $hostId);
}
private static function syncBatch($instances): int
{
if ($instances->isEmpty()) {
return 0;
}
$groups = [];
foreach ($instances as $instance) {
$groups[(int) $instance->host_id][] = $instance;
}
$updated = 0;
$log = User::getConnectionResolver()->connection('log');
$dailyReady = $log->getSchemaBuilder()->hasTable(
'cloud_vm_daily_traffic_cursors'
) && $log->getSchemaBuilder()->hasTable(
'cloud_vm_daily_traffic'
);
foreach ($groups as $hostId => $owned) {
$scope = self::hostScope($hostId);
$uuids = array_map(static fn ($instance) => (string) $instance->uuid, $owned);
$rows = $log->table('cloud_vm_traffic_totals')
->where('host_scope', $scope)->whereIn('vm_uuid', $uuids)->get();
$byUuid = [];
foreach ($rows as $row) {
$byUuid[(string) $row->vm_uuid] = $row;
}
foreach ($owned as $instance) {
$row = $byUuid[(string) $instance->uuid] ?? null;
// An absent or changed owner is unknown usage, not grounds to
// stop another tenant's VM. The next successful sample repairs it.
if ($row === null || $row->user_id === null
|| (int) $row->user_id !== (int) $instance->user_id
|| ($row->host_id !== null && (string) $row->host_id !== (string) $hostId)) {
continue;
}
$download = max(0, (int) $row->download_rx_bytes);
$upload = max(0, (int) $row->upload_tx_bytes);
$observed = max(0, (int) ($row->last_observed_at ?? $row->updated_at ?? 0));
$changed = CloudInstance::where('id', $instance->id)
->where('host_id', $hostId)->where('user_id', $instance->user_id)
->where('uuid', $instance->uuid)
->where('traffic_synced_at', '<=', $observed)
// Agent totals are durable monotonic counters. A rebuilt,
// restored or corrupt log database must never roll paid
// usage backwards and accidentally restore a blocked NIC.
->where('lifetime_download_bytes', '<=', $download)
->where('lifetime_upload_bytes', '<=', $upload)
->update(['lifetime_download_bytes' => $download,
'lifetime_upload_bytes' => $upload, 'traffic_synced_at' => $observed]);
$updated += $changed;
if ($dailyReady) {
try {
// The ledger locks and rechecks the current host/UUID
// after the monotonic main-database sync. A migration
// cutover therefore either sees this total in its base
// or rejects this source generation as stale.
CloudDailyTrafficLedger::record(
$instance, $observed, $download, $upload
);
} catch (\Throwable $error) {
// Daily charts are non-critical. The unchanged cursor
// lets the next successful sync catch up without
// rolling back billing totals.
error_log('cloud daily traffic write unavailable: '
. get_class($error));
}
}
}
}
return $updated;
}
/**
* Reconstruct a conservative period boundary from durable cumulative
* checkpoints. Chart samples are intentionally retained for only 30
* days, while the Agent keeps the first cumulative total in every hour
* indefinitely. Using the first observation *after* the boundary excludes
* the straddling interval and bounds uncharged traffic to less than one
* checkpoint hour even after a long lifecycle-daemon outage.
*/
public static function periodBaseline(CloudInstance $instance,
int $startsAt): ?array
{
if ((int) $instance->traffic_synced_at < $startsAt) {
return null;
}
$log = User::getConnectionResolver()->connection('log');
$scope = self::hostScope((int) $instance->host_id);
if ($log->getSchemaBuilder()->hasTable('cloud_vm_traffic_checkpoints')) {
$checkpoint = $log->table('cloud_vm_traffic_checkpoints')
->where('host_scope', $scope)->where('vm_uuid', (string) $instance->uuid)
->where('user_id', (int) $instance->user_id)
->where('host_id', (string) (int) $instance->host_id)
->where('first_observed_at', '>', $startsAt)
->where('first_observed_at', '<=', (int) $instance->traffic_synced_at)
->orderBy('first_observed_at')->first([
'download_rx_bytes', 'upload_tx_bytes',
]);
if ($checkpoint !== null) {
$download = max(0, (int) $checkpoint->download_rx_bytes);
$upload = max(0, (int) $checkpoint->upload_tx_bytes);
if ($download <= (int) $instance->lifetime_download_bytes
&& $upload <= (int) $instance->lifetime_upload_bytes) {
return ['download' => $download, 'upload' => $upload];
}
return null;
}
}
// Fall back to the bounded sample table when the first post-boundary
// checkpoint is unavailable. Refuse partial retained history: otherwise a
// long outage would subtract only the tail and silently undercount.
if (!$log->getSchemaBuilder()->hasTable('cloud_vm_metric_samples')) {
return null;
}
$rows = $log->table('cloud_vm_metric_samples')
->where('host_scope', $scope)->where('vm_uuid', (string) $instance->uuid)
->where('user_id', (int) $instance->user_id)
->where('host_id', (string) (int) $instance->host_id)
->where('sampled_at', '>', $startsAt)
->where('sampled_at', '<=', (int) $instance->traffic_synced_at)
->orderBy('sampled_at')->get([
'sampled_at', 'network_rx_bytes', 'network_tx_bytes',
]);
$download = 0;
$upload = 0;
$skippedBoundaryInterval = false;
foreach ($rows as $row) {
if ($row->network_rx_bytes === null || $row->network_tx_bytes === null) {
continue;
}
if (!$skippedBoundaryInterval) {
// The sample fallback is safe only when retention still
// covers the boundary. Normal collection is every 60 seconds;
// five minutes tolerates jitter without accepting a truncated
// 30-day tail as complete history.
if ((int) $row->sampled_at > $startsAt + 300) {
return null;
}
$skippedBoundaryInterval = true;
continue;
}
$download += max(0, (int) $row->network_rx_bytes);
$upload += max(0, (int) $row->network_tx_bytes);
}
$lifetimeDownload = max(0, (int) $instance->lifetime_download_bytes);
$lifetimeUpload = max(0, (int) $instance->lifetime_upload_bytes);
if ($download > $lifetimeDownload || $upload > $lifetimeUpload) {
return null;
}
return ['download' => $lifetimeDownload - $download,
'upload' => $lifetimeUpload - $upload];
}
}