You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/app/Services/Cloud/CloudAbuseObservationIngest...

580 lines
22 KiB
PHP

<?php
namespace App\Services\Cloud;
use App\Models\CloudInstance;
use App\Models\User;
/**
* Polls one Agent at a time and durably records canonical observations.
*
* This worker is read-only toward hosts. Samples, projection intents and the
* host cursor commit in one log-database transaction; collection never enters
* the provider-mutation queue because it has no external effect to reconcile.
*/
final class CloudAbuseObservationIngestService
{
private const PAGE_LIMIT = 64;
private const POLL_SECONDS = 15;
private const LEASE_SECONDS = 45;
private const MAX_BACKOFF_SECONDS = 300;
/** @var \Closure():array<int> */
private readonly \Closure $hostIds;
/** @var \Closure(int):object */
private readonly \Closure $clientFactory;
/** @var \Closure():int */
private readonly \Closure $clock;
/** @var list<string> */
private array $errors = [];
/**
* @param null|callable():array<int> $hostIds
* @param null|callable(int):object $clientFactory
* @param null|callable():int $clock
*/
public function __construct(
?callable $hostIds = null,
?callable $clientFactory = null,
?callable $clock = null
) {
$this->hostIds = \Closure::fromCallable($hostIds
?? static fn (): array =>
CloudAgentSettingsService::enabledHostIds());
$this->clientFactory = \Closure::fromCallable($clientFactory
?? static fn (int $hostId): CloudAgentClient =>
CloudAgentSettingsService::clientForHost($hostId, 12.0));
$this->clock = \Closure::fromCallable($clock
?? static fn (): int => time());
}
/** Poll at most one due host. */
public function tick(): bool
{
$this->errors = [];
$now = ($this->clock)();
if (!is_int($now) || $now < 1) {
throw new \LogicException('Abuse monitor clock is invalid');
}
$hostIds = self::normalizeHostIds(($this->hostIds)());
if ($hostIds === []) {
return false;
}
$claim = $this->claim($hostIds, $now);
if ($claim === null) {
return false;
}
try {
$client = ($this->clientFactory)($claim['host_id']);
if (!is_object($client)
|| !is_callable([$client, 'networkObservations'])) {
throw new \LogicException(
'Abuse monitor client does not implement observations'
);
}
$batch = $client->networkObservations(
self::systemActor(),
$claim['generation'],
$claim['cursor'],
self::PAGE_LIMIT
);
if (!is_array($batch)) {
throw new CloudAgentError(
'宿主机 Agent 返回了无效的网络观测合同',
502,
'agent_invalid_network_observation',
true
);
}
$this->commitBatch($claim, $batch, $now);
} catch (\Throwable $error) {
$this->recordFailure($claim, $error, $now);
$this->errors[] = $error instanceof CloudAgentError
? $error->agentCode
: 'network_observation_collection_failed';
}
return true;
}
/** @return list<string> */
public function observationErrors(): array
{
return $this->errors;
}
/** @param array<int> $hostIds @return null|array<string,mixed> */
private function claim(array $hostIds, int $now): ?array
{
$log = User::getConnectionResolver()->connection('log');
return $log->transaction(static function () use (
$log,
$hostIds,
$now
): ?array {
foreach ($hostIds as $hostId) {
$log->table('cloud_vm_network_monitor_cursors')
->insertOrIgnore([
'host_id' => $hostId,
'collector_generation' => null,
'cursor_value' => 0,
'oldest_cursor' => 0,
'gap_before' => 0,
'poll_revision' => 0,
'next_poll_at' => 0,
'poll_lease_token' => null,
'poll_lease_until' => 0,
'collection_state' => 'unknown',
'consecutive_failures' => 0,
'last_window_start' => null,
'last_success_at' => null,
'last_error_code' => '',
'updated_at' => $now,
]);
}
$row = $log->table('cloud_vm_network_monitor_cursors')
->whereIn('host_id', $hostIds)
->where('next_poll_at', '<=', $now)
->where(static function ($lease) use ($now): void {
$lease->whereNull('poll_lease_token')
->orWhere('poll_lease_until', '<=', $now);
})
->orderBy('next_poll_at')->orderBy('host_id')
->lockForUpdate()->first();
if ($row === null) {
return null;
}
$token = bin2hex(random_bytes(16));
$revision = (int) $row->poll_revision + 1;
$updated = $log->table('cloud_vm_network_monitor_cursors')
->where('host_id', (int) $row->host_id)
->where('poll_revision', (int) $row->poll_revision)
->update([
'poll_revision' => $revision,
'poll_lease_token' => $token,
'poll_lease_until' => $now + self::LEASE_SECONDS,
'updated_at' => $now,
]);
if ($updated !== 1) {
return null;
}
return [
'host_id' => (int) $row->host_id,
'generation' => is_string($row->collector_generation)
&& $row->collector_generation !== ''
? $row->collector_generation : null,
'cursor' => max(0, (int) $row->cursor_value),
'oldest_cursor' => max(0, (int) $row->oldest_cursor),
'gap_before' => (int) $row->gap_before === 1,
'revision' => $revision,
'lease_token' => $token,
'failures' => max(0, (int) $row->consecutive_failures),
'last_window_start' => $row->last_window_start === null
? null : max(0, (int) $row->last_window_start),
];
}, 3);
}
/** @param array<string,mixed> $claim @param array<string,mixed> $batch */
private function commitBatch(array $claim, array $batch, int $now): void
{
$generation = (string) ($batch['collector_generation'] ?? '');
if (preg_match('/^[a-f0-9]{32}$/D', $generation) !== 1
|| !is_array($batch['windows'] ?? null)
|| !is_bool($batch['gap_before'] ?? null)
|| !is_bool($batch['has_more'] ?? null)
|| !is_array($batch['capability'] ?? null)) {
throw new CloudAgentError(
'宿主机 Agent 返回了无效的网络观测合同',
502,
'agent_invalid_network_observation',
true
);
}
$windows = $batch['windows'];
$hostId = $claim['host_id'];
$sameGeneration = is_string($claim['generation'])
&& hash_equals($claim['generation'], $generation);
$uuids = array_values(array_unique(array_map(
static fn (array $window): string =>
strtolower((string) ($window['vm_uuid'] ?? '')),
$windows
)));
$main = User::getConnectionResolver()->connection('default');
$main->transaction(function () use (
$main,
$claim,
$batch,
$generation,
$hostId,
$sameGeneration,
$uuids,
$windows,
$now
): void {
$instances = $uuids === [] ? collect() : $main
->table('cloud_instances')
->where('host_id', $hostId)
->whereIn('uuid', $uuids)
->orderBy('id')->lockForUpdate()->get()->keyBy(
static fn (object $row): string =>
strtolower((string) $row->uuid)
);
$mappingMiss = count($instances) !== count($uuids);
$log = User::getConnectionResolver()->connection('log');
$log->transaction(function () use (
$log,
$claim,
$batch,
$generation,
$hostId,
$sameGeneration,
$instances,
$windows,
$mappingMiss,
$now
): void {
$gapActive = ($batch['gap_before'] ?? false) === true
|| ($sameGeneration
&& ($claim['gap_before'] ?? false) === true);
foreach ($windows as $window) {
$uuid = strtolower((string) $window['vm_uuid']);
$instance = $instances->get($uuid);
if ($instance === null) {
continue;
}
$quality = array_values($window['quality_flags']);
$complete = $window['complete'] === true;
if ($gapActive) {
$complete = false;
$quality[] = 'incomplete';
$quality[] = 'panel_ring_gap';
$quality = array_values(array_unique($quality));
sort($quality, SORT_STRING);
}
$metrics = [
'interface' => $window['interface'],
'ipv4' => $window['ipv4'],
'ipv6' => $window['ipv6'],
];
$bucketStart = self::bucketEpoch(
(string) $window['bucket_start']
);
$sampleKey = hash('sha256', implode("\n", [
$hostId,
$generation,
$window['window_id'],
$uuid,
]));
$placementKey = hash('sha256', implode("\n", [
(int) $instance->id,
(int) $instance->user_id,
$hostId,
$uuid,
]));
$sampleValues = [
'sample_key' => $sampleKey,
'instance_id' => (int) $instance->id,
'user_id' => (int) $instance->user_id,
'host_id' => $hostId,
'vm_uuid' => $uuid,
'placement_key' => $placementKey,
'collector_generation' => $generation,
'window_id' => (string) $window['window_id'],
'contract_version' => (int) $batch['contract_version'],
'bucket_start' => $bucketStart,
'bucket_seconds' => (int) $window['bucket_seconds'],
'complete' => $complete ? 1 : 0,
'metrics_json' => self::json($metrics),
'quality_flags_json' => self::json($quality),
'received_at' => $now,
];
$inserted = $log->table(
'cloud_vm_network_observation_samples'
)->insertOrIgnore($sampleValues);
if ($inserted !== 1) {
self::assertSampleReplay($log, $sampleValues);
}
$outboxInserted = (int) $log->table(
'cloud_vm_network_projection_outbox'
)->insertOrIgnore([
'sample_key' => $sampleKey,
'instance_id' => (int) $instance->id,
'bucket_start' => $bucketStart,
'placement_key' => $placementKey,
'state' => 'pending',
'attempts' => 0,
'available_at' => $now,
'lease_token' => null,
'lease_until' => 0,
'last_error_code' => '',
'projected_at' => null,
'terminal_at' => null,
'discard_reason_code' => '',
'created_at' => $now,
'updated_at' => $now,
]);
if ($outboxInserted !== 1) {
self::assertOutboxReplay(
$log,
$sampleKey,
(int) $instance->id,
$bucketStart,
$placementKey
);
}
}
$cursor = $windows === [] && $sameGeneration
? $claim['cursor'] : (int) $batch['next_cursor'];
$oldest = $windows === [] && $sameGeneration
? $claim['oldest_cursor'] : (int) $batch['oldest_cursor'];
$lastWindow = $claim['last_window_start'];
if ($windows !== []) {
$last = $windows[array_key_last($windows)];
$lastWindow = self::bucketEpoch(
(string) $last['bucket_start']
);
}
$available = ($batch['capability']['available'] ?? false)
=== true;
$state = !$available ? 'unavailable'
: (($gapActive || $mappingMiss)
? 'degraded' : 'healthy');
$errorCode = !$available
? (string) ($batch['capability']['reason']
?? 'collection_unavailable')
: ($mappingMiss ? 'instance_mapping_missing'
: ($gapActive ? 'ring_gap' : ''));
$gapPending = $gapActive
&& ($batch['has_more'] ?? false) === true;
$cursorUpdate = [
'collector_generation' => $generation,
'cursor_value' => $cursor,
'oldest_cursor' => $oldest,
'gap_before' => $gapPending ? 1 : 0,
'next_poll_at' => $batch['has_more']
? $now : $now + self::POLL_SECONDS,
'poll_lease_token' => null,
'poll_lease_until' => 0,
'collection_state' => $state,
'consecutive_failures' => 0,
'last_window_start' => $lastWindow,
'last_error_code' => $errorCode,
'updated_at' => $now,
];
if ($available) {
$cursorUpdate['last_success_at'] = $now;
}
$updated = $log->table(
'cloud_vm_network_monitor_cursors'
)->where('host_id', $hostId)
->where('poll_revision', $claim['revision'])
->where('poll_lease_token', $claim['lease_token'])
->update($cursorUpdate);
if ($updated !== 1) {
throw new \RuntimeException(
'Abuse monitor poll lease was lost before commit'
);
}
}, 3);
}, 3);
}
/** @param array<string,mixed> $claim */
private function recordFailure(
array $claim,
\Throwable $error,
int $now
): void {
$failures = $claim['failures'] + 1;
$code = $error instanceof CloudAgentError
? $error->agentCode : 'network_observation_collection_failed';
if (preg_match('/^[a-z0-9_]{3,64}$/D', $code) !== 1) {
$code = 'network_observation_collection_failed';
}
$backoff = min(
self::MAX_BACKOFF_SECONDS,
15 * (2 ** min(4, max(0, $failures - 1)))
);
User::getConnectionResolver()->connection('log')
->table('cloud_vm_network_monitor_cursors')
->where('host_id', $claim['host_id'])
->where('poll_revision', $claim['revision'])
->where('poll_lease_token', $claim['lease_token'])
->update([
'next_poll_at' => $now + $backoff,
'poll_lease_token' => null,
'poll_lease_until' => 0,
'collection_state' => $failures >= 3
? 'unavailable' : 'degraded',
'consecutive_failures' => $failures,
'last_error_code' => $code,
'updated_at' => $now,
]);
}
/** @param array<string,mixed> $expected */
private static function assertSampleReplay(object $log, array $expected): void
{
$existing = $log->table('cloud_vm_network_observation_samples')
->where('sample_key', (string) $expected['sample_key'])->first();
if ($existing === null
|| (int) $existing->instance_id !== (int) $expected['instance_id']
|| (int) $existing->user_id !== (int) $expected['user_id']
|| (int) $existing->host_id !== (int) $expected['host_id']
|| (string) $existing->vm_uuid !== (string) $expected['vm_uuid']
|| !hash_equals(
(string) $existing->placement_key,
(string) $expected['placement_key']
)
|| (string) $existing->collector_generation
!== (string) $expected['collector_generation']
|| (string) $existing->window_id
!== (string) $expected['window_id']
|| (int) $existing->contract_version
!== (int) $expected['contract_version']
|| (int) $existing->bucket_start
!== (int) $expected['bucket_start']
|| (int) $existing->bucket_seconds
!== (int) $expected['bucket_seconds']
|| (int) $existing->complete !== (int) $expected['complete']
|| !self::jsonEquivalent(
(string) $existing->metrics_json,
(string) $expected['metrics_json']
)
|| !self::jsonEquivalent(
(string) $existing->quality_flags_json,
(string) $expected['quality_flags_json']
)) {
throw new \RuntimeException(
'Network observation replay identity or content conflict'
);
}
}
private static function assertOutboxReplay(
object $log,
string $sampleKey,
int $instanceId,
int $bucketStart,
string $placementKey
): void {
$existing = $log->table('cloud_vm_network_projection_outbox')
->where('sample_key', $sampleKey)->first();
if ($existing === null
|| (int) $existing->instance_id !== $instanceId
|| (int) $existing->bucket_start !== $bucketStart
|| !hash_equals((string) $existing->placement_key, $placementKey)) {
throw new \RuntimeException(
'Network observation outbox replay identity conflict'
);
}
}
private static function bucketEpoch(string $value): int
{
$parsed = \DateTimeImmutable::createFromFormat(
'!Y-m-d\TH:i:s\Z',
$value,
new \DateTimeZone('UTC')
);
if ($parsed === false || $parsed->format('Y-m-d\TH:i:s\Z') !== $value
|| $parsed->getTimestamp() < 1
|| $parsed->getTimestamp() % 60 !== 0) {
throw new \RuntimeException(
'Canonical observation bucket became invalid'
);
}
return $parsed->getTimestamp();
}
private static function json(array $value): string
{
return json_encode(
$value,
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
);
}
private static function jsonEquivalent(string $left, string $right): bool
{
try {
$leftValue = json_decode(
$left,
true,
16,
JSON_THROW_ON_ERROR
);
$rightValue = json_decode(
$right,
true,
16,
JSON_THROW_ON_ERROR
);
} catch (\JsonException) {
return false;
}
return self::canonicalJsonValue($leftValue)
=== self::canonicalJsonValue($rightValue);
}
private static function canonicalJsonValue(mixed $value): mixed
{
if (!is_array($value)) {
return $value;
}
if (array_is_list($value)) {
return array_map(
static fn (mixed $item): mixed => self::canonicalJsonValue(
$item
),
$value
);
}
ksort($value, SORT_STRING);
foreach ($value as $key => $item) {
$value[$key] = self::canonicalJsonValue($item);
}
return $value;
}
/** @param mixed $values @return array<int> */
private static function normalizeHostIds(mixed $values): array
{
if (!is_array($values)) {
throw new \LogicException('Abuse monitor host list is invalid');
}
$result = [];
foreach ($values as $value) {
if (!is_int($value) && !ctype_digit((string) $value)) {
throw new \LogicException(
'Abuse monitor host ID is invalid'
);
}
$id = (int) $value;
if ($id < 1) {
throw new \LogicException(
'Abuse monitor host ID is invalid'
);
}
$result[$id] = $id;
}
ksort($result, SORT_NUMERIC);
return array_values($result);
}
/** Dedicated service actor for the signed, read-only inspection route. */
private static function systemActor(): array
{
return [
'user_id' => 1,
'is_admin' => true,
'is_super_admin' => true,
'can_manage' => true,
];
}
}