You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
580 lines
22 KiB
PHP
580 lines
22 KiB
PHP
<?php
|
|
|
|
namespace App\Services\Cloud;
|
|
|
|
use App\Models\CloudInstance;
|
|
use App\Models\User;
|
|
|
|
/**
|
|
* Polls one Agent at a time and durably records canonical observations.
|
|
*
|
|
* This worker is read-only toward hosts. Samples, projection intents and the
|
|
* host cursor commit in one log-database transaction; collection never enters
|
|
* the provider-mutation queue because it has no external effect to reconcile.
|
|
*/
|
|
final class CloudAbuseObservationIngestService
|
|
{
|
|
private const PAGE_LIMIT = 64;
|
|
private const POLL_SECONDS = 15;
|
|
private const LEASE_SECONDS = 45;
|
|
private const MAX_BACKOFF_SECONDS = 300;
|
|
|
|
/** @var \Closure():array<int> */
|
|
private readonly \Closure $hostIds;
|
|
|
|
/** @var \Closure(int):object */
|
|
private readonly \Closure $clientFactory;
|
|
|
|
/** @var \Closure():int */
|
|
private readonly \Closure $clock;
|
|
|
|
/** @var list<string> */
|
|
private array $errors = [];
|
|
|
|
/**
|
|
* @param null|callable():array<int> $hostIds
|
|
* @param null|callable(int):object $clientFactory
|
|
* @param null|callable():int $clock
|
|
*/
|
|
public function __construct(
|
|
?callable $hostIds = null,
|
|
?callable $clientFactory = null,
|
|
?callable $clock = null
|
|
) {
|
|
$this->hostIds = \Closure::fromCallable($hostIds
|
|
?? static fn (): array =>
|
|
CloudAgentSettingsService::enabledHostIds());
|
|
$this->clientFactory = \Closure::fromCallable($clientFactory
|
|
?? static fn (int $hostId): CloudAgentClient =>
|
|
CloudAgentSettingsService::clientForHost($hostId, 12.0));
|
|
$this->clock = \Closure::fromCallable($clock
|
|
?? static fn (): int => time());
|
|
}
|
|
|
|
/** Poll at most one due host. */
|
|
public function tick(): bool
|
|
{
|
|
$this->errors = [];
|
|
$now = ($this->clock)();
|
|
if (!is_int($now) || $now < 1) {
|
|
throw new \LogicException('Abuse monitor clock is invalid');
|
|
}
|
|
$hostIds = self::normalizeHostIds(($this->hostIds)());
|
|
if ($hostIds === []) {
|
|
return false;
|
|
}
|
|
$claim = $this->claim($hostIds, $now);
|
|
if ($claim === null) {
|
|
return false;
|
|
}
|
|
try {
|
|
$client = ($this->clientFactory)($claim['host_id']);
|
|
if (!is_object($client)
|
|
|| !is_callable([$client, 'networkObservations'])) {
|
|
throw new \LogicException(
|
|
'Abuse monitor client does not implement observations'
|
|
);
|
|
}
|
|
$batch = $client->networkObservations(
|
|
self::systemActor(),
|
|
$claim['generation'],
|
|
$claim['cursor'],
|
|
self::PAGE_LIMIT
|
|
);
|
|
if (!is_array($batch)) {
|
|
throw new CloudAgentError(
|
|
'宿主机 Agent 返回了无效的网络观测合同',
|
|
502,
|
|
'agent_invalid_network_observation',
|
|
true
|
|
);
|
|
}
|
|
$this->commitBatch($claim, $batch, $now);
|
|
} catch (\Throwable $error) {
|
|
$this->recordFailure($claim, $error, $now);
|
|
$this->errors[] = $error instanceof CloudAgentError
|
|
? $error->agentCode
|
|
: 'network_observation_collection_failed';
|
|
}
|
|
return true;
|
|
}
|
|
|
|
/** @return list<string> */
|
|
public function observationErrors(): array
|
|
{
|
|
return $this->errors;
|
|
}
|
|
|
|
/** @param array<int> $hostIds @return null|array<string,mixed> */
|
|
private function claim(array $hostIds, int $now): ?array
|
|
{
|
|
$log = User::getConnectionResolver()->connection('log');
|
|
return $log->transaction(static function () use (
|
|
$log,
|
|
$hostIds,
|
|
$now
|
|
): ?array {
|
|
foreach ($hostIds as $hostId) {
|
|
$log->table('cloud_vm_network_monitor_cursors')
|
|
->insertOrIgnore([
|
|
'host_id' => $hostId,
|
|
'collector_generation' => null,
|
|
'cursor_value' => 0,
|
|
'oldest_cursor' => 0,
|
|
'gap_before' => 0,
|
|
'poll_revision' => 0,
|
|
'next_poll_at' => 0,
|
|
'poll_lease_token' => null,
|
|
'poll_lease_until' => 0,
|
|
'collection_state' => 'unknown',
|
|
'consecutive_failures' => 0,
|
|
'last_window_start' => null,
|
|
'last_success_at' => null,
|
|
'last_error_code' => '',
|
|
'updated_at' => $now,
|
|
]);
|
|
}
|
|
$row = $log->table('cloud_vm_network_monitor_cursors')
|
|
->whereIn('host_id', $hostIds)
|
|
->where('next_poll_at', '<=', $now)
|
|
->where(static function ($lease) use ($now): void {
|
|
$lease->whereNull('poll_lease_token')
|
|
->orWhere('poll_lease_until', '<=', $now);
|
|
})
|
|
->orderBy('next_poll_at')->orderBy('host_id')
|
|
->lockForUpdate()->first();
|
|
if ($row === null) {
|
|
return null;
|
|
}
|
|
$token = bin2hex(random_bytes(16));
|
|
$revision = (int) $row->poll_revision + 1;
|
|
$updated = $log->table('cloud_vm_network_monitor_cursors')
|
|
->where('host_id', (int) $row->host_id)
|
|
->where('poll_revision', (int) $row->poll_revision)
|
|
->update([
|
|
'poll_revision' => $revision,
|
|
'poll_lease_token' => $token,
|
|
'poll_lease_until' => $now + self::LEASE_SECONDS,
|
|
'updated_at' => $now,
|
|
]);
|
|
if ($updated !== 1) {
|
|
return null;
|
|
}
|
|
return [
|
|
'host_id' => (int) $row->host_id,
|
|
'generation' => is_string($row->collector_generation)
|
|
&& $row->collector_generation !== ''
|
|
? $row->collector_generation : null,
|
|
'cursor' => max(0, (int) $row->cursor_value),
|
|
'oldest_cursor' => max(0, (int) $row->oldest_cursor),
|
|
'gap_before' => (int) $row->gap_before === 1,
|
|
'revision' => $revision,
|
|
'lease_token' => $token,
|
|
'failures' => max(0, (int) $row->consecutive_failures),
|
|
'last_window_start' => $row->last_window_start === null
|
|
? null : max(0, (int) $row->last_window_start),
|
|
];
|
|
}, 3);
|
|
}
|
|
|
|
/** @param array<string,mixed> $claim @param array<string,mixed> $batch */
|
|
private function commitBatch(array $claim, array $batch, int $now): void
|
|
{
|
|
$generation = (string) ($batch['collector_generation'] ?? '');
|
|
if (preg_match('/^[a-f0-9]{32}$/D', $generation) !== 1
|
|
|| !is_array($batch['windows'] ?? null)
|
|
|| !is_bool($batch['gap_before'] ?? null)
|
|
|| !is_bool($batch['has_more'] ?? null)
|
|
|| !is_array($batch['capability'] ?? null)) {
|
|
throw new CloudAgentError(
|
|
'宿主机 Agent 返回了无效的网络观测合同',
|
|
502,
|
|
'agent_invalid_network_observation',
|
|
true
|
|
);
|
|
}
|
|
$windows = $batch['windows'];
|
|
$hostId = $claim['host_id'];
|
|
$sameGeneration = is_string($claim['generation'])
|
|
&& hash_equals($claim['generation'], $generation);
|
|
$uuids = array_values(array_unique(array_map(
|
|
static fn (array $window): string =>
|
|
strtolower((string) ($window['vm_uuid'] ?? '')),
|
|
$windows
|
|
)));
|
|
$main = User::getConnectionResolver()->connection('default');
|
|
$main->transaction(function () use (
|
|
$main,
|
|
$claim,
|
|
$batch,
|
|
$generation,
|
|
$hostId,
|
|
$sameGeneration,
|
|
$uuids,
|
|
$windows,
|
|
$now
|
|
): void {
|
|
$instances = $uuids === [] ? collect() : $main
|
|
->table('cloud_instances')
|
|
->where('host_id', $hostId)
|
|
->whereIn('uuid', $uuids)
|
|
->orderBy('id')->lockForUpdate()->get()->keyBy(
|
|
static fn (object $row): string =>
|
|
strtolower((string) $row->uuid)
|
|
);
|
|
$mappingMiss = count($instances) !== count($uuids);
|
|
$log = User::getConnectionResolver()->connection('log');
|
|
$log->transaction(function () use (
|
|
$log,
|
|
$claim,
|
|
$batch,
|
|
$generation,
|
|
$hostId,
|
|
$sameGeneration,
|
|
$instances,
|
|
$windows,
|
|
$mappingMiss,
|
|
$now
|
|
): void {
|
|
$gapActive = ($batch['gap_before'] ?? false) === true
|
|
|| ($sameGeneration
|
|
&& ($claim['gap_before'] ?? false) === true);
|
|
foreach ($windows as $window) {
|
|
$uuid = strtolower((string) $window['vm_uuid']);
|
|
$instance = $instances->get($uuid);
|
|
if ($instance === null) {
|
|
continue;
|
|
}
|
|
$quality = array_values($window['quality_flags']);
|
|
$complete = $window['complete'] === true;
|
|
if ($gapActive) {
|
|
$complete = false;
|
|
$quality[] = 'incomplete';
|
|
$quality[] = 'panel_ring_gap';
|
|
$quality = array_values(array_unique($quality));
|
|
sort($quality, SORT_STRING);
|
|
}
|
|
$metrics = [
|
|
'interface' => $window['interface'],
|
|
'ipv4' => $window['ipv4'],
|
|
'ipv6' => $window['ipv6'],
|
|
];
|
|
$bucketStart = self::bucketEpoch(
|
|
(string) $window['bucket_start']
|
|
);
|
|
$sampleKey = hash('sha256', implode("\n", [
|
|
$hostId,
|
|
$generation,
|
|
$window['window_id'],
|
|
$uuid,
|
|
]));
|
|
$placementKey = hash('sha256', implode("\n", [
|
|
(int) $instance->id,
|
|
(int) $instance->user_id,
|
|
$hostId,
|
|
$uuid,
|
|
]));
|
|
$sampleValues = [
|
|
'sample_key' => $sampleKey,
|
|
'instance_id' => (int) $instance->id,
|
|
'user_id' => (int) $instance->user_id,
|
|
'host_id' => $hostId,
|
|
'vm_uuid' => $uuid,
|
|
'placement_key' => $placementKey,
|
|
'collector_generation' => $generation,
|
|
'window_id' => (string) $window['window_id'],
|
|
'contract_version' => (int) $batch['contract_version'],
|
|
'bucket_start' => $bucketStart,
|
|
'bucket_seconds' => (int) $window['bucket_seconds'],
|
|
'complete' => $complete ? 1 : 0,
|
|
'metrics_json' => self::json($metrics),
|
|
'quality_flags_json' => self::json($quality),
|
|
'received_at' => $now,
|
|
];
|
|
$inserted = $log->table(
|
|
'cloud_vm_network_observation_samples'
|
|
)->insertOrIgnore($sampleValues);
|
|
if ($inserted !== 1) {
|
|
self::assertSampleReplay($log, $sampleValues);
|
|
}
|
|
$outboxInserted = (int) $log->table(
|
|
'cloud_vm_network_projection_outbox'
|
|
)->insertOrIgnore([
|
|
'sample_key' => $sampleKey,
|
|
'instance_id' => (int) $instance->id,
|
|
'bucket_start' => $bucketStart,
|
|
'placement_key' => $placementKey,
|
|
'state' => 'pending',
|
|
'attempts' => 0,
|
|
'available_at' => $now,
|
|
'lease_token' => null,
|
|
'lease_until' => 0,
|
|
'last_error_code' => '',
|
|
'projected_at' => null,
|
|
'terminal_at' => null,
|
|
'discard_reason_code' => '',
|
|
'created_at' => $now,
|
|
'updated_at' => $now,
|
|
]);
|
|
if ($outboxInserted !== 1) {
|
|
self::assertOutboxReplay(
|
|
$log,
|
|
$sampleKey,
|
|
(int) $instance->id,
|
|
$bucketStart,
|
|
$placementKey
|
|
);
|
|
}
|
|
}
|
|
|
|
$cursor = $windows === [] && $sameGeneration
|
|
? $claim['cursor'] : (int) $batch['next_cursor'];
|
|
$oldest = $windows === [] && $sameGeneration
|
|
? $claim['oldest_cursor'] : (int) $batch['oldest_cursor'];
|
|
$lastWindow = $claim['last_window_start'];
|
|
if ($windows !== []) {
|
|
$last = $windows[array_key_last($windows)];
|
|
$lastWindow = self::bucketEpoch(
|
|
(string) $last['bucket_start']
|
|
);
|
|
}
|
|
$available = ($batch['capability']['available'] ?? false)
|
|
=== true;
|
|
$state = !$available ? 'unavailable'
|
|
: (($gapActive || $mappingMiss)
|
|
? 'degraded' : 'healthy');
|
|
$errorCode = !$available
|
|
? (string) ($batch['capability']['reason']
|
|
?? 'collection_unavailable')
|
|
: ($mappingMiss ? 'instance_mapping_missing'
|
|
: ($gapActive ? 'ring_gap' : ''));
|
|
$gapPending = $gapActive
|
|
&& ($batch['has_more'] ?? false) === true;
|
|
$cursorUpdate = [
|
|
'collector_generation' => $generation,
|
|
'cursor_value' => $cursor,
|
|
'oldest_cursor' => $oldest,
|
|
'gap_before' => $gapPending ? 1 : 0,
|
|
'next_poll_at' => $batch['has_more']
|
|
? $now : $now + self::POLL_SECONDS,
|
|
'poll_lease_token' => null,
|
|
'poll_lease_until' => 0,
|
|
'collection_state' => $state,
|
|
'consecutive_failures' => 0,
|
|
'last_window_start' => $lastWindow,
|
|
'last_error_code' => $errorCode,
|
|
'updated_at' => $now,
|
|
];
|
|
if ($available) {
|
|
$cursorUpdate['last_success_at'] = $now;
|
|
}
|
|
$updated = $log->table(
|
|
'cloud_vm_network_monitor_cursors'
|
|
)->where('host_id', $hostId)
|
|
->where('poll_revision', $claim['revision'])
|
|
->where('poll_lease_token', $claim['lease_token'])
|
|
->update($cursorUpdate);
|
|
if ($updated !== 1) {
|
|
throw new \RuntimeException(
|
|
'Abuse monitor poll lease was lost before commit'
|
|
);
|
|
}
|
|
}, 3);
|
|
}, 3);
|
|
}
|
|
|
|
/** @param array<string,mixed> $claim */
|
|
private function recordFailure(
|
|
array $claim,
|
|
\Throwable $error,
|
|
int $now
|
|
): void {
|
|
$failures = $claim['failures'] + 1;
|
|
$code = $error instanceof CloudAgentError
|
|
? $error->agentCode : 'network_observation_collection_failed';
|
|
if (preg_match('/^[a-z0-9_]{3,64}$/D', $code) !== 1) {
|
|
$code = 'network_observation_collection_failed';
|
|
}
|
|
$backoff = min(
|
|
self::MAX_BACKOFF_SECONDS,
|
|
15 * (2 ** min(4, max(0, $failures - 1)))
|
|
);
|
|
User::getConnectionResolver()->connection('log')
|
|
->table('cloud_vm_network_monitor_cursors')
|
|
->where('host_id', $claim['host_id'])
|
|
->where('poll_revision', $claim['revision'])
|
|
->where('poll_lease_token', $claim['lease_token'])
|
|
->update([
|
|
'next_poll_at' => $now + $backoff,
|
|
'poll_lease_token' => null,
|
|
'poll_lease_until' => 0,
|
|
'collection_state' => $failures >= 3
|
|
? 'unavailable' : 'degraded',
|
|
'consecutive_failures' => $failures,
|
|
'last_error_code' => $code,
|
|
'updated_at' => $now,
|
|
]);
|
|
}
|
|
|
|
/** @param array<string,mixed> $expected */
|
|
private static function assertSampleReplay(object $log, array $expected): void
|
|
{
|
|
$existing = $log->table('cloud_vm_network_observation_samples')
|
|
->where('sample_key', (string) $expected['sample_key'])->first();
|
|
if ($existing === null
|
|
|| (int) $existing->instance_id !== (int) $expected['instance_id']
|
|
|| (int) $existing->user_id !== (int) $expected['user_id']
|
|
|| (int) $existing->host_id !== (int) $expected['host_id']
|
|
|| (string) $existing->vm_uuid !== (string) $expected['vm_uuid']
|
|
|| !hash_equals(
|
|
(string) $existing->placement_key,
|
|
(string) $expected['placement_key']
|
|
)
|
|
|| (string) $existing->collector_generation
|
|
!== (string) $expected['collector_generation']
|
|
|| (string) $existing->window_id
|
|
!== (string) $expected['window_id']
|
|
|| (int) $existing->contract_version
|
|
!== (int) $expected['contract_version']
|
|
|| (int) $existing->bucket_start
|
|
!== (int) $expected['bucket_start']
|
|
|| (int) $existing->bucket_seconds
|
|
!== (int) $expected['bucket_seconds']
|
|
|| (int) $existing->complete !== (int) $expected['complete']
|
|
|| !self::jsonEquivalent(
|
|
(string) $existing->metrics_json,
|
|
(string) $expected['metrics_json']
|
|
)
|
|
|| !self::jsonEquivalent(
|
|
(string) $existing->quality_flags_json,
|
|
(string) $expected['quality_flags_json']
|
|
)) {
|
|
throw new \RuntimeException(
|
|
'Network observation replay identity or content conflict'
|
|
);
|
|
}
|
|
}
|
|
|
|
private static function assertOutboxReplay(
|
|
object $log,
|
|
string $sampleKey,
|
|
int $instanceId,
|
|
int $bucketStart,
|
|
string $placementKey
|
|
): void {
|
|
$existing = $log->table('cloud_vm_network_projection_outbox')
|
|
->where('sample_key', $sampleKey)->first();
|
|
if ($existing === null
|
|
|| (int) $existing->instance_id !== $instanceId
|
|
|| (int) $existing->bucket_start !== $bucketStart
|
|
|| !hash_equals((string) $existing->placement_key, $placementKey)) {
|
|
throw new \RuntimeException(
|
|
'Network observation outbox replay identity conflict'
|
|
);
|
|
}
|
|
}
|
|
|
|
private static function bucketEpoch(string $value): int
|
|
{
|
|
$parsed = \DateTimeImmutable::createFromFormat(
|
|
'!Y-m-d\TH:i:s\Z',
|
|
$value,
|
|
new \DateTimeZone('UTC')
|
|
);
|
|
if ($parsed === false || $parsed->format('Y-m-d\TH:i:s\Z') !== $value
|
|
|| $parsed->getTimestamp() < 1
|
|
|| $parsed->getTimestamp() % 60 !== 0) {
|
|
throw new \RuntimeException(
|
|
'Canonical observation bucket became invalid'
|
|
);
|
|
}
|
|
return $parsed->getTimestamp();
|
|
}
|
|
|
|
private static function json(array $value): string
|
|
{
|
|
return json_encode(
|
|
$value,
|
|
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
);
|
|
}
|
|
|
|
private static function jsonEquivalent(string $left, string $right): bool
|
|
{
|
|
try {
|
|
$leftValue = json_decode(
|
|
$left,
|
|
true,
|
|
16,
|
|
JSON_THROW_ON_ERROR
|
|
);
|
|
$rightValue = json_decode(
|
|
$right,
|
|
true,
|
|
16,
|
|
JSON_THROW_ON_ERROR
|
|
);
|
|
} catch (\JsonException) {
|
|
return false;
|
|
}
|
|
return self::canonicalJsonValue($leftValue)
|
|
=== self::canonicalJsonValue($rightValue);
|
|
}
|
|
|
|
private static function canonicalJsonValue(mixed $value): mixed
|
|
{
|
|
if (!is_array($value)) {
|
|
return $value;
|
|
}
|
|
if (array_is_list($value)) {
|
|
return array_map(
|
|
static fn (mixed $item): mixed => self::canonicalJsonValue(
|
|
$item
|
|
),
|
|
$value
|
|
);
|
|
}
|
|
ksort($value, SORT_STRING);
|
|
foreach ($value as $key => $item) {
|
|
$value[$key] = self::canonicalJsonValue($item);
|
|
}
|
|
return $value;
|
|
}
|
|
|
|
/** @param mixed $values @return array<int> */
|
|
private static function normalizeHostIds(mixed $values): array
|
|
{
|
|
if (!is_array($values)) {
|
|
throw new \LogicException('Abuse monitor host list is invalid');
|
|
}
|
|
$result = [];
|
|
foreach ($values as $value) {
|
|
if (!is_int($value) && !ctype_digit((string) $value)) {
|
|
throw new \LogicException(
|
|
'Abuse monitor host ID is invalid'
|
|
);
|
|
}
|
|
$id = (int) $value;
|
|
if ($id < 1) {
|
|
throw new \LogicException(
|
|
'Abuse monitor host ID is invalid'
|
|
);
|
|
}
|
|
$result[$id] = $id;
|
|
}
|
|
ksort($result, SORT_NUMERIC);
|
|
return array_values($result);
|
|
}
|
|
|
|
/** Dedicated service actor for the signed, read-only inspection route. */
|
|
private static function systemActor(): array
|
|
{
|
|
return [
|
|
'user_id' => 1,
|
|
'is_admin' => true,
|
|
'is_super_admin' => true,
|
|
'can_manage' => true,
|
|
];
|
|
}
|
|
}
|