You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
406 lines
15 KiB
PHP
406 lines
15 KiB
PHP
<?php
|
|
|
|
namespace App\Services\Cloud;
|
|
|
|
use App\Models\User;
|
|
|
|
/**
|
|
* Bounded retention for observe-only network-abuse telemetry.
|
|
*
|
|
* The log and main databases deliberately remain separate. Each database is
|
|
* pruned in its own transaction and every pass is idempotent; no lifecycle,
|
|
* billing, firewall or provider-operation state is touched here.
|
|
*/
|
|
final class CloudAbuseRetentionService
|
|
{
|
|
public const TELEMETRY_RETENTION_SECONDS = 90 * 86400;
|
|
public const READ_MODEL_RETENTION_SECONDS = 180 * 86400;
|
|
public const DEFAULT_BATCH_SIZE = 200;
|
|
|
|
/** @var \Closure():int */
|
|
private readonly \Closure $clock;
|
|
|
|
/** Last baseline position inspected by this worker process. */
|
|
private ?int $baselineScanAfter = null;
|
|
|
|
/** @var null|array{projected_at:int,sample_key:string} */
|
|
private ?array $receiptScanAfter = null;
|
|
|
|
public function __construct(?callable $clock = null)
|
|
{
|
|
$this->clock = \Closure::fromCallable(
|
|
$clock ?? static fn (): int => time()
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Run one bounded, restart-safe retention pass.
|
|
*
|
|
* @return array{
|
|
* terminal_outboxes:int,
|
|
* samples:int,
|
|
* baselines:int,
|
|
* receipts:int,
|
|
* findings:int
|
|
* }
|
|
*/
|
|
public function prune(int $limit = self::DEFAULT_BATCH_SIZE): array
|
|
{
|
|
if ($limit < 1 || $limit > self::DEFAULT_BATCH_SIZE) {
|
|
throw new \InvalidArgumentException(
|
|
'Abuse retention batch size must be between 1 and '
|
|
. self::DEFAULT_BATCH_SIZE
|
|
);
|
|
}
|
|
$now = ($this->clock)();
|
|
if (!is_int($now) || $now < self::READ_MODEL_RETENTION_SECONDS) {
|
|
throw new \LogicException('Abuse retention clock is invalid');
|
|
}
|
|
|
|
$telemetryCutoff = $now - self::TELEMETRY_RETENTION_SECONDS;
|
|
$readModelCutoff = $now - self::READ_MODEL_RETENTION_SECONDS;
|
|
$terminal = $this->pruneTerminalTelemetry(
|
|
$telemetryCutoff,
|
|
$limit
|
|
);
|
|
|
|
return [
|
|
'terminal_outboxes' => $terminal['outboxes'],
|
|
'samples' => $terminal['samples']
|
|
+ $this->pruneOrphanSamples($telemetryCutoff, $limit),
|
|
'baselines' => $this->pruneOrphanBaselines($limit, $now),
|
|
'receipts' => $this->pruneReceipts(
|
|
$readModelCutoff,
|
|
$limit,
|
|
$now
|
|
),
|
|
'findings' => $this->pruneFindings($readModelCutoff, $limit),
|
|
];
|
|
}
|
|
|
|
/** @return array{outboxes:int,samples:int} */
|
|
private function pruneTerminalTelemetry(int $cutoff, int $limit): array
|
|
{
|
|
$log = User::getConnectionResolver()->connection('log');
|
|
|
|
return $log->transaction(static function () use (
|
|
$log,
|
|
$cutoff,
|
|
$limit
|
|
): array {
|
|
$rows = $log->table(
|
|
'cloud_vm_network_projection_outbox as outbox'
|
|
)->join(
|
|
'cloud_vm_network_observation_samples as sample',
|
|
'sample.sample_key',
|
|
'=',
|
|
'outbox.sample_key'
|
|
)->whereIn('outbox.state', ['done', 'discarded'])
|
|
->where('outbox.terminal_at', '<=', $cutoff)
|
|
->where('sample.bucket_start', '<=', $cutoff)
|
|
->orderBy('outbox.terminal_at')
|
|
->orderBy('outbox.sample_key')
|
|
->limit($limit)
|
|
->lockForUpdate()
|
|
->get(['outbox.sample_key']);
|
|
$keys = [];
|
|
foreach ($rows as $row) {
|
|
$keys[] = (string) $row->sample_key;
|
|
}
|
|
if ($keys === []) {
|
|
return ['outboxes' => 0, 'samples' => 0];
|
|
}
|
|
|
|
// The foreign key is RESTRICT: terminal intent must be removed
|
|
// before its immutable sample, within the same log transaction.
|
|
$outboxes = $log->table('cloud_vm_network_projection_outbox')
|
|
->whereIn('sample_key', $keys)
|
|
->whereIn('state', ['done', 'discarded'])
|
|
->where('terminal_at', '<=', $cutoff)
|
|
->delete();
|
|
if ($outboxes !== count($keys)) {
|
|
throw new \RuntimeException(
|
|
'Abuse retention terminal outbox changed during pruning'
|
|
);
|
|
}
|
|
$samples = $log->table(
|
|
'cloud_vm_network_observation_samples'
|
|
)->whereIn('sample_key', $keys)
|
|
->where('bucket_start', '<=', $cutoff)
|
|
->delete();
|
|
if ($samples !== count($keys)) {
|
|
throw new \RuntimeException(
|
|
'Abuse retention sample changed during pruning'
|
|
);
|
|
}
|
|
|
|
return ['outboxes' => $outboxes, 'samples' => $samples];
|
|
});
|
|
}
|
|
|
|
private function pruneOrphanSamples(int $cutoff, int $limit): int
|
|
{
|
|
$log = User::getConnectionResolver()->connection('log');
|
|
|
|
return $log->transaction(static function () use (
|
|
$log,
|
|
$cutoff,
|
|
$limit
|
|
): int {
|
|
$rows = $log->table(
|
|
'cloud_vm_network_observation_samples as sample'
|
|
)->leftJoin(
|
|
'cloud_vm_network_projection_outbox as outbox',
|
|
'outbox.sample_key',
|
|
'=',
|
|
'sample.sample_key'
|
|
)->whereNull('outbox.sample_key')
|
|
->where('sample.bucket_start', '<=', $cutoff)
|
|
->orderBy('sample.bucket_start')
|
|
->orderBy('sample.sample_key')
|
|
->limit($limit)
|
|
->lockForUpdate()
|
|
->get(['sample.sample_key']);
|
|
$keys = [];
|
|
foreach ($rows as $row) {
|
|
$keys[] = (string) $row->sample_key;
|
|
}
|
|
if ($keys === []) {
|
|
return 0;
|
|
}
|
|
|
|
return $log->table('cloud_vm_network_observation_samples')
|
|
->whereIn('sample_key', $keys)
|
|
->where('bucket_start', '<=', $cutoff)
|
|
->whereNotExists(static function ($query): void {
|
|
$query->selectRaw('1')->from(
|
|
'cloud_vm_network_projection_outbox as outbox'
|
|
)->whereColumn(
|
|
'outbox.sample_key',
|
|
'cloud_vm_network_observation_samples.sample_key'
|
|
);
|
|
})->delete();
|
|
});
|
|
}
|
|
|
|
private function pruneOrphanBaselines(int $limit, int $now): int
|
|
{
|
|
$log = User::getConnectionResolver()->connection('log');
|
|
$main = User::getConnectionResolver()->connection();
|
|
$candidateIds = $this->baselineCandidates($log, $limit, $now);
|
|
if ($candidateIds === []) {
|
|
return 0;
|
|
}
|
|
$live = $main->table('cloud_instances')
|
|
->whereIn('id', $candidateIds)->pluck('id')->all();
|
|
$liveIds = array_fill_keys(array_map('intval', $live), true);
|
|
$orphans = array_values(array_filter(
|
|
$candidateIds,
|
|
static fn (int $id): bool => !isset($liveIds[$id])
|
|
));
|
|
if ($orphans === []) {
|
|
return 0;
|
|
}
|
|
|
|
return $log->transaction(static function () use (
|
|
$log,
|
|
$orphans
|
|
): int {
|
|
// Pending or leased work owns the baseline even after the main
|
|
// row disappears. Projection must decide/discard it first.
|
|
return $log->table('cloud_vm_network_baselines')
|
|
->whereIn('instance_id', $orphans)
|
|
->whereNotExists(static function ($query): void {
|
|
$query->selectRaw('1')->from(
|
|
'cloud_vm_network_projection_outbox as outbox'
|
|
)->whereColumn(
|
|
'outbox.instance_id',
|
|
'cloud_vm_network_baselines.instance_id'
|
|
)->whereIn('outbox.state', ['pending', 'leased']);
|
|
})->delete();
|
|
});
|
|
}
|
|
|
|
private function pruneReceipts(int $cutoff, int $limit, int $now): int
|
|
{
|
|
$main = User::getConnectionResolver()->connection();
|
|
$log = User::getConnectionResolver()->connection('log');
|
|
$candidateKeys = $this->receiptCandidates(
|
|
$main,
|
|
$cutoff,
|
|
$limit,
|
|
$now
|
|
);
|
|
if ($candidateKeys === []) {
|
|
return 0;
|
|
}
|
|
$retained = $log->table('cloud_vm_network_observation_samples')
|
|
->whereIn('sample_key', $candidateKeys)
|
|
->pluck('sample_key')->all();
|
|
$retainedKeys = array_fill_keys(array_map('strval', $retained), true);
|
|
$orphans = array_values(array_filter(
|
|
$candidateKeys,
|
|
static fn (string $key): bool => !isset($retainedKeys[$key])
|
|
));
|
|
if ($orphans === []) {
|
|
return 0;
|
|
}
|
|
|
|
return $main->transaction(static function () use (
|
|
$main,
|
|
$orphans,
|
|
$cutoff
|
|
): int {
|
|
return $main->table('cloud_abuse_projection_receipts')
|
|
->whereIn('sample_key', $orphans)
|
|
->where('projected_at', '<=', $cutoff)
|
|
->delete();
|
|
});
|
|
}
|
|
|
|
/** @return list<int> */
|
|
private function baselineCandidates(
|
|
object $log,
|
|
int $limit,
|
|
int $now
|
|
): array {
|
|
$maximum = (int) ($log->table('cloud_vm_network_baselines')
|
|
->max('instance_id') ?? 0);
|
|
if ($maximum < 1) {
|
|
$this->baselineScanAfter = null;
|
|
return [];
|
|
}
|
|
if ($this->baselineScanAfter === null
|
|
|| $this->baselineScanAfter > $maximum) {
|
|
// A time-derived first anchor prevents repeated --once workers
|
|
// from always starting at the same live-ID prefix after restart.
|
|
$this->baselineScanAfter = intdiv($now, 60)
|
|
% ($maximum + 1);
|
|
}
|
|
$anchor = $this->baselineScanAfter;
|
|
$ids = $log->table('cloud_vm_network_baselines')
|
|
->where('instance_id', '>', $anchor)
|
|
->orderBy('instance_id')->limit($limit)
|
|
->pluck('instance_id')->map(static fn ($id): int => (int) $id)
|
|
->all();
|
|
$remaining = $limit - count($ids);
|
|
if ($remaining > 0) {
|
|
$wrapped = $log->table('cloud_vm_network_baselines')
|
|
->where('instance_id', '<=', $anchor)
|
|
->orderBy('instance_id')->limit($remaining)
|
|
->pluck('instance_id')
|
|
->map(static fn ($id): int => (int) $id)->all();
|
|
$ids = array_merge($ids, $wrapped);
|
|
}
|
|
if ($ids !== []) {
|
|
$this->baselineScanAfter = $ids[array_key_last($ids)];
|
|
}
|
|
return $ids;
|
|
}
|
|
|
|
/** @return list<string> */
|
|
private function receiptCandidates(
|
|
object $main,
|
|
int $cutoff,
|
|
int $limit,
|
|
int $now
|
|
): array {
|
|
$base = static fn () => $main
|
|
->table('cloud_abuse_projection_receipts')
|
|
->where('projected_at', '<=', $cutoff);
|
|
if ($this->receiptScanAfter === null) {
|
|
$bounds = $base()->selectRaw(
|
|
'MIN(projected_at) AS oldest, MAX(projected_at) AS newest'
|
|
)->first();
|
|
if ($bounds === null || $bounds->oldest === null
|
|
|| $bounds->newest === null) {
|
|
return [];
|
|
}
|
|
$oldest = (int) $bounds->oldest;
|
|
$newest = (int) $bounds->newest;
|
|
$span = max(1, $newest - $oldest + 2);
|
|
$this->receiptScanAfter = [
|
|
'projected_at' => $oldest - 1
|
|
+ (intdiv($now, 60) % $span),
|
|
// sample_key values are SHA-256 identities. Varying this
|
|
// anchor by minute keeps repeated --once workers from
|
|
// selecting the same protected prefix after every restart.
|
|
'sample_key' => hash(
|
|
'sha256',
|
|
'abuse-receipt-retention:' . intdiv($now, 60)
|
|
),
|
|
];
|
|
}
|
|
$anchor = $this->receiptScanAfter;
|
|
$rows = $base()->where(static function ($query) use ($anchor): void {
|
|
$query->where('projected_at', '>', $anchor['projected_at'])
|
|
->orWhere(static function ($same) use ($anchor): void {
|
|
$same->where(
|
|
'projected_at',
|
|
$anchor['projected_at']
|
|
)->where('sample_key', '>', $anchor['sample_key']);
|
|
});
|
|
})->orderBy('projected_at')->orderBy('sample_key')
|
|
->limit($limit)->get(['projected_at', 'sample_key'])->all();
|
|
$remaining = $limit - count($rows);
|
|
if ($remaining > 0) {
|
|
$wrapped = $base()->where(
|
|
static function ($query) use ($anchor): void {
|
|
$query->where(
|
|
'projected_at',
|
|
'<',
|
|
$anchor['projected_at']
|
|
)->orWhere(static function ($same) use ($anchor): void {
|
|
$same->where(
|
|
'projected_at',
|
|
$anchor['projected_at']
|
|
)->where('sample_key', '<=', $anchor['sample_key']);
|
|
});
|
|
}
|
|
)->orderBy('projected_at')->orderBy('sample_key')
|
|
->limit($remaining)
|
|
->get(['projected_at', 'sample_key'])->all();
|
|
$rows = array_merge($rows, $wrapped);
|
|
}
|
|
if ($rows === []) {
|
|
return [];
|
|
}
|
|
$last = $rows[array_key_last($rows)];
|
|
$this->receiptScanAfter = [
|
|
'projected_at' => (int) $last->projected_at,
|
|
'sample_key' => (string) $last->sample_key,
|
|
];
|
|
return array_map(
|
|
static fn (object $row): string => (string) $row->sample_key,
|
|
$rows
|
|
);
|
|
}
|
|
|
|
private function pruneFindings(int $cutoff, int $limit): int
|
|
{
|
|
$main = User::getConnectionResolver()->connection();
|
|
|
|
return $main->transaction(static function () use (
|
|
$main,
|
|
$cutoff,
|
|
$limit
|
|
): int {
|
|
$ids = $main->table('cloud_abuse_findings')
|
|
->where('state', 'resolved')
|
|
->whereNotNull('resolved_at')
|
|
->where('resolved_at', '<=', $cutoff)
|
|
->orderBy('resolved_at')->orderBy('id')
|
|
->limit($limit)->lockForUpdate()->pluck('id')->all();
|
|
if ($ids === []) {
|
|
return 0;
|
|
}
|
|
|
|
return $main->table('cloud_abuse_findings')
|
|
->whereIn('id', array_map('intval', $ids))
|
|
->where('state', 'resolved')
|
|
->where('resolved_at', '<=', $cutoff)
|
|
->delete();
|
|
});
|
|
}
|
|
}
|