diff --git a/web/app/Services/Cloud/CloudAbuseAlertDeliveryService.php b/web/app/Services/Cloud/CloudAbuseAlertDeliveryService.php index f22282af..cef1f707 100644 --- a/web/app/Services/Cloud/CloudAbuseAlertDeliveryService.php +++ b/web/app/Services/Cloud/CloudAbuseAlertDeliveryService.php @@ -57,34 +57,48 @@ final class CloudAbuseAlertDeliveryService private function reserveNextEvent(int $now): string { $main = self::connection(); - $event = $main->table('cloud_abuse_alert_events as event') + $event = $main->table('cloud_network_alert_events as event') + ->join( + 'cloud_network_findings as finding', + 'finding.id', + '=', + 'event.finding_id' + )->join( + 'cloud_instances as instance', + 'instance.id', + '=', + 'finding.instance_id' + ) ->whereNotExists(static function ($query): void { $query->selectRaw('1') - ->from('cloud_abuse_alert_deliveries as delivery') + ->from('cloud_network_alert_deliveries as delivery') ->whereColumn('delivery.event_id', 'event.id'); - })->orderBy('event.id')->first(['event.*']); + })->orderBy('event.id')->first([ + 'event.*', + 'finding.instance_id', + 'finding.policy_revision', + 'finding.rule_code', + 'finding.direction', + 'finding.ip_family', + 'instance.user_id', + 'instance.host_id', + ]); if ($event === null) { return 'none'; } - $items = $main->table('cloud_abuse_alert_event_items') - ->where('event_id', (int) $event->id) - ->orderBy('id')->get()->all(); - $formatFailed = false; try { - $message = $this->messageFormatter()->format($event, $items); + $message = $this->messageFormatter()->format($event); } catch (\Throwable $error) { $formatFailed = true; $message = '网络滥用告警事件格式无效,请在管理后台核查事件 #' . (int) $event->id . '。'; } self::assertFrozenMessage($message); - $destination = null; if (!$formatFailed) { try { - $destination = $this->telegramChannel() - ->currentDestination(); + $destination = $this->telegramChannel()->currentDestination(); } catch (AdministratorTelegramLocalUnavailable $error) { return 'local_unavailable'; } @@ -107,12 +121,11 @@ final class CloudAbuseAlertDeliveryService $effectiveDestination = null; } } - $locked = $main->table('cloud_abuse_alert_events') + $locked = $main->table('cloud_network_alert_events') ->where('id', (int) $event->id) - ->where('event_key', (string) $event->event_key) ->lockForUpdate()->first(); if ($locked === null || $main->table( - 'cloud_abuse_alert_deliveries' + 'cloud_network_alert_deliveries' )->where('event_id', (int) $event->id)->exists()) { return 'none'; } @@ -131,7 +144,7 @@ final class CloudAbuseAlertDeliveryService ? 'channel_unavailable' : 'destination_ineligible'; $terminalAt = $now; } - $main->table('cloud_abuse_alert_deliveries')->insert([ + $main->table('cloud_network_alert_deliveries')->insert([ 'event_id' => (int) $locked->id, 'binding_generation' => $effectiveDestination?->bindingGeneration, @@ -159,7 +172,7 @@ final class CloudAbuseAlertDeliveryService { $main = self::connection(); $claim = $main->transaction(function () use ($main, $now): ?array { - $candidate = $main->table('cloud_abuse_alert_deliveries') + $candidate = $main->table('cloud_network_alert_deliveries') ->where('state', 'pending')->orderBy('created_at') ->orderBy('id')->first([ 'id', 'event_id', 'bound_admin_id', @@ -170,13 +183,13 @@ final class CloudAbuseAlertDeliveryService $administrator = $main->table('user') ->where('id', (int) $candidate->bound_admin_id) ->lockForUpdate()->first(['id', 'is_admin', 'enable']); - $event = $main->table('cloud_abuse_alert_events') + $event = $main->table('cloud_network_alert_events') ->where('id', (int) $candidate->event_id) ->lockForUpdate()->first(['id']); if ($event === null) { return null; } - $delivery = $main->table('cloud_abuse_alert_deliveries') + $delivery = $main->table('cloud_network_alert_deliveries') ->where('id', (int) $candidate->id) ->where('event_id', (int) $event->id) ->where('state', 'pending') @@ -190,7 +203,7 @@ final class CloudAbuseAlertDeliveryService || !$this->telegramChannel()->eligibleAdministrator( (int) $administrator->id )) { - $updated = $main->table('cloud_abuse_alert_deliveries') + $updated = $main->table('cloud_network_alert_deliveries') ->where('id', (int) $delivery->id) ->where('state', 'pending')->update([ 'state' => 'skipped', @@ -211,7 +224,7 @@ final class CloudAbuseAlertDeliveryService $message, (string) $delivery->content_hash )) { - $updated = $main->table('cloud_abuse_alert_deliveries') + $updated = $main->table('cloud_network_alert_deliveries') ->where('id', (int) $delivery->id) ->where('state', 'pending')->update([ 'state' => 'failed', @@ -226,7 +239,7 @@ final class CloudAbuseAlertDeliveryService } return ['terminal' => true]; } - $updated = $main->table('cloud_abuse_alert_deliveries') + $updated = $main->table('cloud_network_alert_deliveries') ->where('id', (int) $delivery->id) ->where('state', 'pending')->update([ 'state' => 'dispatching', @@ -306,7 +319,7 @@ final class CloudAbuseAlertDeliveryService try { ($this->afterRemoteSuccess)($messageId); $updated = self::connection()->table( - 'cloud_abuse_alert_deliveries' + 'cloud_network_alert_deliveries' )->where('id', $id)->where('state', 'dispatching')->update([ 'telegram_message_id' => $messageId, 'state' => 'sent', @@ -332,7 +345,7 @@ final class CloudAbuseAlertDeliveryService { $main = self::connection(); return $main->transaction(function () use ($main, $now): bool { - $row = $main->table('cloud_abuse_alert_deliveries') + $row = $main->table('cloud_network_alert_deliveries') ->where('state', 'dispatching') ->where('updated_at', '<=', $now - self::DISPATCH_STALE_SECONDS) @@ -341,7 +354,7 @@ final class CloudAbuseAlertDeliveryService if ($row === null) { return false; } - $updated = $main->table('cloud_abuse_alert_deliveries') + $updated = $main->table('cloud_network_alert_deliveries') ->where('id', (int) $row->id) ->where('state', 'dispatching')->update([ 'state' => 'unknown', @@ -361,7 +374,7 @@ final class CloudAbuseAlertDeliveryService private function returnToPending(int $id, int $now): void { $updated = self::connection()->table( - 'cloud_abuse_alert_deliveries' + 'cloud_network_alert_deliveries' )->where('id', $id)->where('state', 'dispatching')->update([ 'state' => 'pending', 'dispatch_started_at' => null, @@ -381,7 +394,7 @@ final class CloudAbuseAlertDeliveryService ): void { $updated = self::connection()->table( - 'cloud_abuse_alert_deliveries' + 'cloud_network_alert_deliveries' )->where('id', $id)->where('state', 'dispatching')->update([ 'state' => 'skipped', 'skip_reason' => $reason, @@ -399,7 +412,7 @@ final class CloudAbuseAlertDeliveryService private function failInvalidPayload(int $id, int $now): void { $updated = self::connection()->table( - 'cloud_abuse_alert_deliveries' + 'cloud_network_alert_deliveries' )->where('id', $id)->where('state', 'dispatching')->update([ 'state' => 'failed', 'error_code' => 'payload_invalid', @@ -421,7 +434,7 @@ final class CloudAbuseAlertDeliveryService ): void { try { $updated = self::connection()->table( - 'cloud_abuse_alert_deliveries' + 'cloud_network_alert_deliveries' )->where('id', $id)->where('state', 'dispatching')->update([ 'telegram_message_id' => $messageId, 'state' => 'unknown', diff --git a/web/app/Services/Cloud/CloudAbuseAlertMessageFormatter.php b/web/app/Services/Cloud/CloudAbuseAlertMessageFormatter.php index da8c4621..2d82ce1a 100644 --- a/web/app/Services/Cloud/CloudAbuseAlertMessageFormatter.php +++ b/web/app/Services/Cloud/CloudAbuseAlertMessageFormatter.php @@ -4,13 +4,7 @@ namespace App\Services\Cloud; use App\Services\Config; -/** - * Renders one immutable abuse-alert event into a bounded Telegram message. - * - * The formatter accepts only the event/item ledger snapshot. It never reads - * live instances or findings, so a pending delivery cannot drift after an - * instance is renamed, transferred or deleted. - */ +/** Renders one immutable finding transition into a bounded Telegram message. */ final class CloudAbuseAlertMessageFormatter { private const RULE_LABELS = [ @@ -38,167 +32,91 @@ final class CloudAbuseAlertMessageFormatter 'max_remote_ips_per_local_port' => '单本地端口最大远端 IP 数', ]; - private const MAX_LENGTH = 4096; - private const CONTENT_BUDGET = 3700; - - /** - * @param list $items - */ - public function format(object $event, array $items): string + public function format(object $event): string { $eventId = self::positiveInteger(self::value($event, 'id'), 'event id'); - $eventKey = self::hexIdentity(self::value($event, 'event_key'), 'event key'); $instanceId = self::positiveInteger( self::value($event, 'instance_id'), 'instance id' ); $userId = self::positiveInteger(self::value($event, 'user_id'), 'user id'); $hostId = self::positiveInteger(self::value($event, 'host_id'), 'host id'); - $bucketStart = self::positiveInteger( - self::value($event, 'bucket_start'), - 'bucket start' - ); - $ruleVersion = self::positiveInteger( - self::value($event, 'rule_version'), - 'rule version' - ); $policyRevision = self::positiveInteger( self::value($event, 'policy_revision'), 'policy revision' ); - $transitionCount = self::positiveInteger( - self::value($event, 'transition_count'), - 'transition count' + $occurredAt = self::positiveInteger( + self::value($event, 'occurred_at'), + 'occurred at' ); - $highest = (string) self::value($event, 'highest_severity'); - if ($bucketStart % 60 !== 0 - || !in_array($highest, ['suspicious', 'high'], true) - || $transitionCount !== count($items)) { - throw new \InvalidArgumentException('Invalid abuse alert event snapshot'); - } - $url = self::adminUrl((string) Config::get('baseUrl', ''), $instanceId); - - $normalized = []; - $seen = []; - foreach ($items as $item) { - $row = self::normalizeItem($item, $eventId, $bucketStart); - if (isset($seen[$row['item_key']])) { - throw new \InvalidArgumentException( - 'Duplicate abuse alert event item' - ); - } - $seen[$row['item_key']] = true; - $normalized[] = $row; + $rule = (string) self::value($event, 'rule_code'); + $direction = (string) self::value($event, 'direction'); + $family = (string) self::value($event, 'ip_family'); + $type = (string) self::value($event, 'transition_type'); + $from = (string) self::value($event, 'from_severity'); + $to = (string) self::value($event, 'to_severity'); + $reason = (string) self::value($event, 'resolution_reason'); + if (!isset(self::RULE_LABELS[$rule]) + || !in_array($direction, ['inbound', 'outbound'], true) + || !in_array($family, ['ipv4', 'ipv6', 'mixed'], true) + || !self::validTransition($type, $from, $to, $reason)) { + throw new \InvalidArgumentException( + 'Invalid abuse alert transition snapshot' + ); } - usort($normalized, static fn (array $left, array $right): int => [ - $left['transition_type'], - $left['direction'], - $left['rule_code'], - $left['ip_family'], - $left['item_key'], - ] <=> [ - $right['transition_type'], - $right['direction'], - $right['rule_code'], - $right['ip_family'], - $right['item_key'], - ]); - - $onlyResolved = array_reduce( - $normalized, - static fn (bool $carry, array $item): bool => $carry - && $item['transition_type'] === 'resolved', - true - ); - $title = $onlyResolved + $evidence = self::decodeEvidence(self::value($event, 'evidence_json')); + $resolved = $type === 'resolved'; + $title = $resolved ? '✅ 网络风险已恢复' - : ($highest === 'high' + : ($to === 'high' ? '🔴 网络滥用高风险告警' : '🟠 网络异常告警'); + $transition = [ + 'opened' => '新触发', + 'escalated' => '风险升级', + 'resolved' => $reason === 'policy_replaced' + ? '策略更新后关闭' : '已恢复', + ][$type]; + $directionLabel = $direction === 'inbound' + ? '入站受攻击' : '出站可疑行为'; + $severity = $resolved + ? ($from === 'high' ? '高风险已解除' : '可疑状态已解除') + : ($to === 'high' ? '高风险' : '可疑'); + $evidenceParts = []; + foreach ($evidence as $key => $count) { + $evidenceParts[] = self::EVIDENCE_LABELS[$key] . ' ' + . number_format($count, 0, '.', ','); + } $lines = [ $title, '', '实例:#' . $instanceId, '用户:#' . $userId, '宿主机:#' . $hostId, - '发生时间:' . gmdate('Y-m-d H:i:s', $bucketStart) . ' UTC', + '发生时间:' . gmdate('Y-m-d H:i:s', $occurredAt) . ' UTC', '告警:#' . $eventId, - '策略:r' . $policyRevision . ' / 引擎 v' . $ruleVersion, + '策略:r' . $policyRevision, + '', + ($resolved ? '✅ ' : ($to === 'high' ? '🔴 ' : '🟠 ')) + . $transition . ' · ' . self::RULE_LABELS[$rule], + '方向:' . $directionLabel . ' · ' . strtoupper($family) + . ' · ' . $severity, + '证据:' . ($evidenceParts === [] + ? '未记录聚合计数' : implode(';', $evidenceParts)), + '', + '后台:' . self::adminUrl( + (string) Config::get('baseUrl', ''), + $instanceId + ), + '说明:仅包含聚合元数据,不包含通信正文或远端地址明细。', ]; - - $included = 0; - foreach ($normalized as $item) { - $block = self::itemLines($item); - $candidate = implode("\n", array_merge($lines, [''], $block)); - if (mb_strlen($candidate, 'UTF-8') > self::CONTENT_BUDGET) { - break; - } - $lines[] = ''; - array_push($lines, ...$block); - $included++; - } - if ($included < count($normalized)) { - $lines[] = ''; - $lines[] = '其他规则:' . (count($normalized) - $included) - . ' 条,请在后台查看完整证据。'; - } - $lines[] = ''; - $lines[] = '后台:' . $url; - $lines[] = '说明:仅包含聚合元数据,不包含通信正文或远端地址明细。'; $message = implode("\n", $lines); - if (mb_strlen($message, 'UTF-8') > self::MAX_LENGTH) { + if (mb_strlen($message, 'UTF-8') > 4096) { throw new \LogicException('Abuse alert message exceeded Telegram limit'); } return $message; } - /** @return array */ - private static function normalizeItem( - object $item, - int $eventId, - int $bucketStart - ): array { - $itemKey = self::hexIdentity(self::value($item, 'item_key'), 'item key'); - $findingKey = self::hexIdentity( - self::value($item, 'finding_key'), - 'finding key' - ); - $itemEventId = self::positiveInteger( - self::value($item, 'event_id'), - 'item event id' - ); - $rule = (string) self::value($item, 'rule_code'); - $direction = (string) self::value($item, 'direction'); - $family = (string) self::value($item, 'ip_family'); - $type = (string) self::value($item, 'transition_type'); - $from = (string) self::value($item, 'from_severity'); - $to = (string) self::value($item, 'to_severity'); - $reason = (string) self::value($item, 'resolution_reason'); - $occurredAt = self::positiveInteger( - self::value($item, 'occurred_at'), - 'item occurred at' - ); - if ($itemEventId !== $eventId || $occurredAt !== $bucketStart - || !isset(self::RULE_LABELS[$rule]) - || !in_array($direction, ['inbound', 'outbound'], true) - || !in_array($family, ['ipv4', 'ipv6', 'mixed'], true) - || !self::validTransition($type, $from, $to, $reason)) { - throw new \InvalidArgumentException('Invalid abuse alert event item'); - } - $evidence = self::decodeEvidence(self::value($item, 'evidence_json')); - return [ - 'item_key' => $itemKey, - 'finding_key' => $findingKey, - 'rule_code' => $rule, - 'direction' => $direction, - 'ip_family' => $family, - 'transition_type' => $type, - 'from_severity' => $from, - 'to_severity' => $to, - 'evidence' => $evidence, - ]; - } - private static function validTransition( string $type, string $from, @@ -212,7 +130,10 @@ final class CloudAbuseAlertMessageFormatter && $to === 'high' && $reason === '') || ($type === 'resolved' && in_array($from, ['suspicious', 'high'], true) - && $to === 'none' && $reason === 'evidence_clear'); + && $to === 'none' + && in_array($reason, [ + 'evidence_clear', 'policy_replaced', + ], true)); } /** @return array */ @@ -245,35 +166,6 @@ final class CloudAbuseAlertMessageFormatter return $normalized; } - /** @param array $item @return list */ - private static function itemLines(array $item): array - { - $transition = [ - 'opened' => '新触发', - 'escalated' => '风险升级', - 'resolved' => '已恢复', - ][$item['transition_type']]; - $icon = $item['transition_type'] === 'resolved' ? '✅' - : (($item['to_severity'] === 'high') ? '🔴' : '🟠'); - $direction = $item['direction'] === 'inbound' - ? '入站受攻击' - : '出站可疑行为'; - $severity = $item['transition_type'] === 'resolved' - ? (($item['from_severity'] === 'high') ? '高风险已解除' : '可疑状态已解除') - : (($item['to_severity'] === 'high') ? '高风险' : '可疑'); - $evidence = []; - foreach ($item['evidence'] as $key => $count) { - $evidence[] = self::EVIDENCE_LABELS[$key] . ' ' - . number_format($count, 0, '.', ','); - } - return [ - $icon . ' ' . $transition . ' · ' . self::RULE_LABELS[$item['rule_code']], - '方向:' . $direction . ' · ' . strtoupper($item['ip_family']) - . ' · ' . $severity, - '证据:' . ($evidence === [] ? '未记录聚合计数' : implode(';', $evidence)), - ]; - } - private static function adminUrl(string $baseUrl, int $instanceId): string { $baseUrl = rtrim(trim($baseUrl), '/'); @@ -301,15 +193,6 @@ final class CloudAbuseAlertMessageFormatter return (int) $value; } - private static function hexIdentity(mixed $value, string $label): string - { - $value = is_string($value) ? $value : ''; - if (preg_match('/^[a-f0-9]{64}$/D', $value) !== 1) { - throw new \InvalidArgumentException('Invalid abuse alert ' . $label); - } - return $value; - } - private static function value(object $row, string $key): mixed { return $row->{$key} ?? null; diff --git a/web/app/Services/Cloud/CloudAbuseRetentionService.php b/web/app/Services/Cloud/CloudAbuseRetentionService.php index fccfe2a5..3814a16f 100644 --- a/web/app/Services/Cloud/CloudAbuseRetentionService.php +++ b/web/app/Services/Cloud/CloudAbuseRetentionService.php @@ -4,32 +4,17 @@ namespace App\Services\Cloud; use App\Models\User; -/** - * Bounded retention for observe-only network-abuse telemetry. - * - * The log and main databases deliberately remain separate. Each database is - * pruned in its own transaction and every pass is idempotent; no lifecycle, - * billing, firewall or provider-operation state is touched here. - */ +/** Bounded retention for the compact instance-centric abuse ledger. */ final class CloudAbuseRetentionService { - public const RAW_TELEMETRY_RETENTION_SECONDS = 72 * 3600; - public const TERMINAL_OUTBOX_RETENTION_SECONDS = 24 * 3600; - public const RECEIPT_RETENTION_SECONDS = 7 * 86400; - public const FIVE_MINUTE_ROLLUP_RETENTION_SECONDS = 14 * 86400; - public const HOURLY_ROLLUP_RETENTION_SECONDS = 180 * 86400; - public const READ_MODEL_RETENTION_SECONDS = 180 * 86400; + public const DAILY_SUMMARY_RETENTION_SECONDS = 180 * 86400; + public const ALERT_HISTORY_RETENTION_SECONDS = 180 * 86400; + public const ORPHAN_HOST_RETENTION_SECONDS = 7 * 86400; public const DEFAULT_BATCH_SIZE = 2000; /** @var \Closure():int */ private readonly \Closure $clock; - /** Last baseline position inspected by this worker process. */ - private ?int $baselineScanAfter = null; - - /** @var null|array{projected_at:int,sample_key:string} */ - private ?array $receiptScanAfter = null; - public function __construct(?callable $clock = null) { $this->clock = \Closure::fromCallable( @@ -38,17 +23,13 @@ final class CloudAbuseRetentionService } /** - * Run one bounded, restart-safe retention pass. + * Run one bounded, restart-safe main-database retention pass. * * @return array{ - * terminal_outboxes:int, - * samples:int, - * five_minute_rollups:int, - * hourly_rollups:int, - * baselines:int, - * receipts:int, - * findings:int, - * alert_events:int + * daily_summaries:int, + * alert_events:int, + * resolved_findings:int, + * orphan_hosts:int * } */ public function prune(int $limit = self::DEFAULT_BATCH_SIZE): array @@ -60,430 +41,193 @@ final class CloudAbuseRetentionService ); } $now = ($this->clock)(); - if (!is_int($now) || $now < self::READ_MODEL_RETENTION_SECONDS) { + if (!is_int($now) || $now < self::ALERT_HISTORY_RETENTION_SECONDS) { throw new \LogicException('Abuse retention clock is invalid'); } - - $sampleCutoff = $now - self::RAW_TELEMETRY_RETENTION_SECONDS; - $outboxCutoff = $now - self::TERMINAL_OUTBOX_RETENTION_SECONDS; - $receiptCutoff = $now - self::RECEIPT_RETENTION_SECONDS; - $readModelCutoff = $now - self::READ_MODEL_RETENTION_SECONDS; + $historyCutoff = $now - self::ALERT_HISTORY_RETENTION_SECONDS; return [ - 'terminal_outboxes' => $this->pruneTerminalOutboxes( - $outboxCutoff, + 'daily_summaries' => $this->pruneDailySummaries( + $now - self::DAILY_SUMMARY_RETENTION_SECONDS, $limit ), - 'samples' => $this->pruneOrphanSamples($sampleCutoff, $limit), - 'five_minute_rollups' => $this->pruneRollups( - CloudAbuseObservationRollupStore::FIVE_MINUTES, - $now - self::FIVE_MINUTE_ROLLUP_RETENTION_SECONDS, + 'alert_events' => $this->pruneAlertEvents( + $historyCutoff, $limit ), - 'hourly_rollups' => $this->pruneRollups( - CloudAbuseObservationRollupStore::ONE_HOUR, - $now - self::HOURLY_ROLLUP_RETENTION_SECONDS, + 'resolved_findings' => $this->pruneResolvedFindings( + $historyCutoff, $limit ), - 'baselines' => $this->pruneOrphanBaselines($limit, $now), - 'receipts' => $this->pruneReceipts( - $receiptCutoff, - $limit, - $now - ), - 'findings' => $this->pruneFindings($readModelCutoff, $limit), - 'alert_events' => $this->pruneAlertEvents( - $readModelCutoff, + 'orphan_hosts' => $this->pruneOrphanHosts( + $now - self::ORPHAN_HOST_RETENTION_SECONDS, + $now, $limit ), ]; } - private function pruneTerminalOutboxes(int $cutoff, int $limit): int + private function pruneDailySummaries(int $cutoff, int $limit): int { - $log = User::getConnectionResolver()->connection('log'); - - return $log->transaction(static function () use ( - $log, - $cutoff, - $limit - ): int { - $keys = $log->table('cloud_vm_network_projection_outbox') - ->whereIn('state', ['done', 'discarded']) - ->where('terminal_at', '<=', $cutoff) - ->orderBy('terminal_at') - ->orderBy('sample_key') - ->limit($limit) - ->lockForUpdate() - ->pluck('sample_key')->map( - static fn ($key): string => (string) $key - )->all(); - if ($keys === []) { - return 0; - } - - $outboxes = $log->table('cloud_vm_network_projection_outbox') - ->whereIn('sample_key', $keys) - ->whereIn('state', ['done', 'discarded']) - ->where('terminal_at', '<=', $cutoff) - ->delete(); - if ($outboxes !== count($keys)) { - throw new \RuntimeException( - 'Abuse retention terminal outbox changed during pruning' - ); - } - return $outboxes; - }); - } - - private function pruneRollups( - int $interval, - int $cutoff, - int $limit - ): int { - $log = User::getConnectionResolver()->connection('log'); - return $log->transaction(static function () use ( - $log, - $interval, + $main = self::connection(); + return $main->transaction(static function () use ( + $main, $cutoff, $limit ): int { - $rows = $log->table('cloud_vm_network_observation_rollups') - ->where('interval_seconds', $interval) - ->where('bucket_start', '<=', $cutoff) - ->orderBy('bucket_start')->orderBy('instance_id') + $rows = $main->table('cloud_network_daily_summaries') + ->where('day_start', '<=', $cutoff) + ->orderBy('day_start')->orderBy('instance_id') ->limit($limit)->lockForUpdate() - ->get(['instance_id', 'bucket_start']); - if ($rows->isEmpty()) { - return 0; - } + ->get(['instance_id', 'day_start']); $deleted = 0; foreach ($rows as $row) { - $deleted += $log->table( - 'cloud_vm_network_observation_rollups' - )->where('instance_id', (int) $row->instance_id) - ->where('interval_seconds', $interval) - ->where('bucket_start', (int) $row->bucket_start) - ->where('bucket_start', '<=', $cutoff) + $deleted += (int) $main + ->table('cloud_network_daily_summaries') + ->where('instance_id', (int) $row->instance_id) + ->where('day_start', (int) $row->day_start) + ->where('day_start', '<=', $cutoff) ->delete(); } return $deleted; - }); + }, 3); } - private function pruneOrphanSamples(int $cutoff, int $limit): int + /** + * Pending, dispatching, unknown and not-yet-reserved events are permanent + * safety fences. Only terminal known-effect deliveries release an event. + */ + private function pruneAlertEvents(int $cutoff, int $limit): int { - $log = User::getConnectionResolver()->connection('log'); - - return $log->transaction(static function () use ( - $log, + $main = self::connection(); + return $main->transaction(static function () use ( + $main, $cutoff, $limit ): int { - $rows = $log->table( - 'cloud_vm_network_observation_samples as sample' - )->leftJoin( - 'cloud_vm_network_projection_outbox as outbox', - 'outbox.sample_key', - '=', - 'sample.sample_key' - )->whereNull('outbox.sample_key') - ->where('sample.bucket_start', '<=', $cutoff) - ->orderBy('sample.bucket_start') - ->orderBy('sample.sample_key') - ->limit($limit) - ->lockForUpdate() - ->get(['sample.sample_key']); - $keys = []; - foreach ($rows as $row) { - $keys[] = (string) $row->sample_key; - } - if ($keys === []) { + $events = $main->table('cloud_network_alert_events as event') + ->join( + 'cloud_network_alert_deliveries as delivery', + 'delivery.event_id', + '=', + 'event.id' + )->where('event.created_at', '<=', $cutoff) + ->whereIn('delivery.state', ['sent', 'failed', 'skipped']) + ->whereNotNull('delivery.terminal_at') + ->where('delivery.terminal_at', '<=', $cutoff) + ->orderBy('event.created_at')->orderBy('event.id') + ->limit($limit)->lockForUpdate() + ->get(['event.id']); + $ids = $events->map( + static fn (object $row): int => (int) $row->id + )->all(); + if ($ids === []) { return 0; } - - return $log->table('cloud_vm_network_observation_samples') - ->whereIn('sample_key', $keys) - ->where('bucket_start', '<=', $cutoff) - ->whereNotExists(static function ($query): void { - $query->selectRaw('1')->from( - 'cloud_vm_network_projection_outbox as outbox' - )->whereColumn( - 'outbox.sample_key', - 'cloud_vm_network_observation_samples.sample_key' - ); - })->delete(); - }); - } - - private function pruneOrphanBaselines(int $limit, int $now): int - { - $log = User::getConnectionResolver()->connection('log'); - $main = User::getConnectionResolver()->connection(); - $candidateIds = $this->baselineCandidates($log, $limit, $now); - if ($candidateIds === []) { - return 0; - } - $live = $main->table('cloud_instances') - ->whereIn('id', $candidateIds)->pluck('id')->all(); - $liveIds = array_fill_keys(array_map('intval', $live), true); - $orphans = array_values(array_filter( - $candidateIds, - static fn (int $id): bool => !isset($liveIds[$id]) - )); - if ($orphans === []) { - return 0; - } - - return $log->transaction(static function () use ( - $log, - $orphans - ): int { - // Pending or leased work owns the baseline even after the main - // row disappears. Projection must decide/discard it first. - return $log->table('cloud_vm_network_baselines') - ->whereIn('instance_id', $orphans) - ->whereNotExists(static function ($query): void { - $query->selectRaw('1')->from( - 'cloud_vm_network_projection_outbox as outbox' - )->whereColumn( - 'outbox.instance_id', - 'cloud_vm_network_baselines.instance_id' - )->whereIn('outbox.state', ['pending', 'leased']); + return (int) $main->table('cloud_network_alert_events') + ->whereIn('id', $ids) + ->where('created_at', '<=', $cutoff) + ->whereExists(static function ($query) use ($cutoff): void { + $query->selectRaw('1') + ->from('cloud_network_alert_deliveries as delivery') + ->whereColumn( + 'delivery.event_id', + 'cloud_network_alert_events.id' + )->whereIn('delivery.state', [ + 'sent', 'failed', 'skipped', + ])->whereNotNull('delivery.terminal_at') + ->where('delivery.terminal_at', '<=', $cutoff); })->delete(); - }); + }, 3); } - private function pruneReceipts(int $cutoff, int $limit, int $now): int + private function pruneResolvedFindings(int $cutoff, int $limit): int { - $main = User::getConnectionResolver()->connection(); - $candidateKeys = $this->receiptCandidates( - $main, - $cutoff, - $limit, - $now - ); - if ($candidateKeys === []) { - return 0; - } - return $main->transaction(static function () use ( - $main, - $candidateKeys, - $cutoff - ): int { - return $main->table('cloud_abuse_projection_receipts') - ->whereIn('sample_key', $candidateKeys) - ->where('projected_at', '<=', $cutoff) - ->delete(); - }); - } - - /** @return list */ - private function baselineCandidates( - object $log, - int $limit, - int $now - ): array { - $maximum = (int) ($log->table('cloud_vm_network_baselines') - ->max('instance_id') ?? 0); - if ($maximum < 1) { - $this->baselineScanAfter = null; - return []; - } - if ($this->baselineScanAfter === null - || $this->baselineScanAfter > $maximum) { - // A time-derived first anchor prevents repeated --once workers - // from always starting at the same live-ID prefix after restart. - $this->baselineScanAfter = intdiv($now, 60) - % ($maximum + 1); - } - $anchor = $this->baselineScanAfter; - $ids = $log->table('cloud_vm_network_baselines') - ->where('instance_id', '>', $anchor) - ->orderBy('instance_id')->limit($limit) - ->pluck('instance_id')->map(static fn ($id): int => (int) $id) - ->all(); - $remaining = $limit - count($ids); - if ($remaining > 0) { - $wrapped = $log->table('cloud_vm_network_baselines') - ->where('instance_id', '<=', $anchor) - ->orderBy('instance_id')->limit($remaining) - ->pluck('instance_id') - ->map(static fn ($id): int => (int) $id)->all(); - $ids = array_merge($ids, $wrapped); - } - if ($ids !== []) { - $this->baselineScanAfter = $ids[array_key_last($ids)]; - } - return $ids; - } - - /** @return list */ - private function receiptCandidates( - object $main, - int $cutoff, - int $limit, - int $now - ): array { - $base = static fn () => $main - ->table('cloud_abuse_projection_receipts') - ->where('projected_at', '<=', $cutoff); - if ($this->receiptScanAfter === null) { - $bounds = $base()->selectRaw( - 'MIN(projected_at) AS oldest, MAX(projected_at) AS newest' - )->first(); - if ($bounds === null || $bounds->oldest === null - || $bounds->newest === null) { - return []; - } - $oldest = (int) $bounds->oldest; - $newest = (int) $bounds->newest; - $span = max(1, $newest - $oldest + 2); - $this->receiptScanAfter = [ - 'projected_at' => $oldest - 1 - + (intdiv($now, 60) % $span), - // sample_key values are SHA-256 identities. Varying this - // anchor by minute keeps repeated --once workers from - // selecting the same protected prefix after every restart. - 'sample_key' => hash( - 'sha256', - 'abuse-receipt-retention:' . intdiv($now, 60) - ), - ]; - } - $anchor = $this->receiptScanAfter; - $rows = $base()->where(static function ($query) use ($anchor): void { - $query->where('projected_at', '>', $anchor['projected_at']) - ->orWhere(static function ($same) use ($anchor): void { - $same->where( - 'projected_at', - $anchor['projected_at'] - )->where('sample_key', '>', $anchor['sample_key']); - }); - })->orderBy('projected_at')->orderBy('sample_key') - ->limit($limit)->get(['projected_at', 'sample_key'])->all(); - $remaining = $limit - count($rows); - if ($remaining > 0) { - $wrapped = $base()->where( - static function ($query) use ($anchor): void { - $query->where( - 'projected_at', - '<', - $anchor['projected_at'] - )->orWhere(static function ($same) use ($anchor): void { - $same->where( - 'projected_at', - $anchor['projected_at'] - )->where('sample_key', '<=', $anchor['sample_key']); - }); - } - )->orderBy('projected_at')->orderBy('sample_key') - ->limit($remaining) - ->get(['projected_at', 'sample_key'])->all(); - $rows = array_merge($rows, $wrapped); - } - if ($rows === []) { - return []; - } - $last = $rows[array_key_last($rows)]; - $this->receiptScanAfter = [ - 'projected_at' => (int) $last->projected_at, - 'sample_key' => (string) $last->sample_key, - ]; - return array_map( - static fn (object $row): string => (string) $row->sample_key, - $rows - ); - } - - private function pruneFindings(int $cutoff, int $limit): int - { - $main = User::getConnectionResolver()->connection(); - + $main = self::connection(); return $main->transaction(static function () use ( $main, $cutoff, $limit ): int { - $ids = $main->table('cloud_abuse_findings') + $ids = $main->table('cloud_network_findings') ->where('state', 'resolved') ->whereNotNull('resolved_at') ->where('resolved_at', '<=', $cutoff) - ->orderBy('resolved_at')->orderBy('id') - ->limit($limit)->lockForUpdate()->pluck('id')->all(); + ->whereNotExists(static function ($query): void { + $query->selectRaw('1') + ->from('cloud_network_alert_events as event') + ->whereColumn( + 'event.finding_id', + 'cloud_network_findings.id' + ); + })->orderBy('resolved_at')->orderBy('id') + ->limit($limit)->lockForUpdate()->pluck('id') + ->map(static fn ($id): int => (int) $id)->all(); if ($ids === []) { return 0; } - - return $main->table('cloud_abuse_findings') - ->whereIn('id', array_map('intval', $ids)) + return (int) $main->table('cloud_network_findings') + ->whereIn('id', $ids) ->where('state', 'resolved') ->where('resolved_at', '<=', $cutoff) - ->delete(); - }); + ->whereNotExists(static function ($query): void { + $query->selectRaw('1') + ->from('cloud_network_alert_events as event') + ->whereColumn( + 'event.finding_id', + 'cloud_network_findings.id' + ); + })->delete(); + }, 3); } - /** - * Unknown, pending, dispatching and not-yet-decided events are durable - * safety fences. Only old, terminal, known-effect delivery decisions may - * release their immutable event and item snapshots. - */ - private function pruneAlertEvents(int $cutoff, int $limit): int - { - $main = User::getConnectionResolver()->connection(); - - $candidateIds = $main->table('cloud_abuse_alert_events') - ->where('created_at', '<=', $cutoff) - ->whereExists(static function ($query) use ($cutoff): void { - $query->selectRaw('1') - ->from('cloud_abuse_alert_deliveries as delivery') - ->whereColumn( - 'delivery.event_id', - 'cloud_abuse_alert_events.id' - )->whereIn('delivery.state', [ - 'sent', 'failed', 'skipped', - ])->whereNotNull('delivery.terminal_at') - ->where('delivery.terminal_at', '<=', $cutoff); - })->orderBy('created_at')->orderBy('id')->limit($limit) - ->pluck('id')->map(static fn ($id): int => (int) $id)->all(); - sort($candidateIds, SORT_NUMERIC); - if ($candidateIds === []) { - return 0; - } - + private function pruneOrphanHosts( + int $cutoff, + int $now, + int $limit + ): int { + $main = self::connection(); return $main->transaction(static function () use ( $main, $cutoff, - $candidateIds + $now, + $limit ): int { - $eventIds = $main->table('cloud_abuse_alert_events') - ->whereIn('id', $candidateIds) - ->where('created_at', '<=', $cutoff) - ->orderBy('id')->lockForUpdate()->pluck('id') + $ids = $main->table('cloud_network_monitor_hosts') + ->where('updated_at', '<=', $cutoff) + ->where(static function ($lease) use ($now): void { + $lease->whereNull('poll_lease_token') + ->orWhere('poll_lease_until', '<=', $now); + })->whereNotExists(static function ($query): void { + $query->selectRaw('1')->from('cloud_hosts as host') + ->whereColumn( + 'host.id', + 'cloud_network_monitor_hosts.host_id' + ); + })->orderBy('updated_at')->orderBy('host_id') + ->limit($limit)->lockForUpdate()->pluck('host_id') ->map(static fn ($id): int => (int) $id)->all(); - if ($eventIds === []) { - return 0; - } - $deliveries = $main->table('cloud_abuse_alert_deliveries') - ->whereIn('event_id', $eventIds)->orderBy('id')->lockForUpdate() - ->get(['event_id', 'state', 'terminal_at']); - $eligible = []; - foreach ($deliveries as $delivery) { - if (in_array((string) $delivery->state, [ - 'sent', 'failed', 'skipped', - ], true) && $delivery->terminal_at !== null - && (int) $delivery->terminal_at <= $cutoff) { - $eligible[] = (int) $delivery->event_id; - } - } - $eventIds = array_values(array_unique($eligible)); - sort($eventIds, SORT_NUMERIC); - if ($eventIds === []) { + if ($ids === []) { return 0; } - return $main->table('cloud_abuse_alert_events') - ->whereIn('id', $eventIds) - ->where('created_at', '<=', $cutoff)->delete(); - }); + return (int) $main->table('cloud_network_monitor_hosts') + ->whereIn('host_id', $ids) + ->where('updated_at', '<=', $cutoff) + ->where(static function ($lease) use ($now): void { + $lease->whereNull('poll_lease_token') + ->orWhere('poll_lease_until', '<=', $now); + })->whereNotExists(static function ($query): void { + $query->selectRaw('1')->from('cloud_hosts as host') + ->whereColumn( + 'host.id', + 'cloud_network_monitor_hosts.host_id' + ); + })->delete(); + }, 3); + } + + private static function connection(): object + { + return User::getConnectionResolver()->connection('default'); } } diff --git a/web/sql/cloud.sql b/web/sql/cloud.sql index 81ca2836..1a912106 100644 --- a/web/sql/cloud.sql +++ b/web/sql/cloud.sql @@ -3311,6 +3311,7 @@ CREATE TABLE cloud_network_alert_deliveries ( binding_generation, chat_id, telegram_message_id ), KEY cloud_network_alert_delivery_due (state, updated_at, id), + KEY cloud_network_alert_delivery_admin (bound_admin_id, created_at, id), KEY cloud_network_alert_delivery_terminal (terminal_at, state, id), CONSTRAINT chk_cloud_network_alert_delivery_state CHECK ( state IN ('pending','dispatching','sent','failed','unknown','skipped') @@ -3319,6 +3320,55 @@ CREATE TABLE cloud_network_alert_deliveries ( CHAR_LENGTH(message_text) BETWEEN 1 AND 4096 AND content_hash REGEXP '^[a-f0-9]{64}$' ), + CONSTRAINT chk_cloud_network_alert_delivery_destination CHECK ( + ( + destination_redacted_at IS NULL + AND ( + (binding_generation IS NULL AND bound_admin_id IS NULL + AND chat_id IS NULL AND telegram_message_id IS NULL) + OR (binding_generation IS NOT NULL + AND binding_generation REGEXP '^[a-f0-9]{32}$' + AND bound_admin_id IS NOT NULL AND bound_admin_id > 0 + AND chat_id IS NOT NULL + AND chat_id REGEXP '^[1-9][0-9]{0,18}$' + AND (telegram_message_id IS NULL + OR telegram_message_id > 0)) + ) + ) + OR ( + destination_redacted_at IS NOT NULL + AND state IN ('sent','failed','unknown','skipped') + AND binding_generation IS NULL AND bound_admin_id IS NULL + AND chat_id IS NULL AND telegram_message_id IS NULL + ) + ), + CONSTRAINT chk_cloud_network_alert_delivery_shape CHECK ( + (state = 'pending' + AND binding_generation IS NOT NULL AND telegram_message_id IS NULL + AND dispatch_started_at IS NULL AND sent_at IS NULL + AND terminal_at IS NULL AND error_code = '' AND skip_reason = '') + OR (state = 'dispatching' + AND binding_generation IS NOT NULL AND telegram_message_id IS NULL + AND dispatch_started_at IS NOT NULL AND sent_at IS NULL + AND terminal_at IS NULL AND error_code = '' AND skip_reason = '') + OR (state = 'sent' + AND dispatch_started_at IS NOT NULL AND sent_at IS NOT NULL + AND terminal_at IS NOT NULL AND error_code = '' AND skip_reason = '' + AND (destination_redacted_at IS NOT NULL + OR telegram_message_id IS NOT NULL)) + OR (state = 'failed' + AND telegram_message_id IS NULL + AND dispatch_started_at IS NULL AND sent_at IS NULL + AND terminal_at IS NOT NULL + AND error_code <> '' AND skip_reason = '') + OR (state = 'unknown' + AND dispatch_started_at IS NOT NULL AND sent_at IS NULL + AND terminal_at IS NOT NULL + AND error_code <> '' AND skip_reason = '') + OR (state = 'skipped' + AND dispatch_started_at IS NULL AND sent_at IS NULL + AND terminal_at IS NOT NULL AND error_code = '' AND skip_reason <> '') + ), CONSTRAINT fk_cloud_network_alert_delivery_event FOREIGN KEY (event_id) REFERENCES cloud_network_alert_events (id) ON DELETE CASCADE ON UPDATE RESTRICT diff --git a/web/tests/CloudAbuseAlertDeliveryServiceTest.php b/web/tests/CloudAbuseAlertDeliveryServiceTest.php index 135b0619..1b42cf9c 100644 --- a/web/tests/CloudAbuseAlertDeliveryServiceTest.php +++ b/web/tests/CloudAbuseAlertDeliveryServiceTest.php @@ -26,25 +26,23 @@ $db = $capsule->getConnection('default'); $schema = $db->getSchemaBuilder(); $schema->create('user', static function (Blueprint $t): void { $t->increments('id'); $t->integer('is_admin'); $t->integer('enable'); }); -$schema->create('cloud_abuse_alert_events', static function (Blueprint $t): void { - $t->increments('id'); $t->string('event_key'); $t->string('sample_key'); - $t->integer('instance_id'); $t->integer('user_id'); $t->integer('host_id'); - $t->integer('bucket_start'); $t->integer('rule_version'); - $t->integer('policy_revision'); $t->string('policy_hash'); - $t->string('transition_digest'); $t->integer('transition_count'); - $t->string('highest_severity'); $t->integer('created_at'); +$schema->create('cloud_instances', static function (Blueprint $t): void { + $t->increments('id'); $t->integer('user_id'); $t->integer('host_id'); }); -$schema->create('cloud_abuse_alert_event_items', static function (Blueprint $t): void { - $t->increments('id'); $t->integer('event_id'); $t->string('item_key'); - $t->string('finding_key'); $t->integer('occurrence_generation'); - $t->integer('transition_revision'); $t->string('rule_code'); +$schema->create('cloud_network_findings', static function (Blueprint $t): void { + $t->increments('id'); $t->integer('instance_id'); $t->integer('policy_revision'); + $t->string('rule_code'); $t->string('direction'); $t->string('ip_family'); +}); +$schema->create('cloud_network_alert_events', static function (Blueprint $t): void { + $t->increments('id'); $t->integer('finding_id'); + $t->integer('transition_revision'); $t->string('transition_type'); $t->string('from_severity'); $t->string('to_severity'); $t->text('evidence_json'); $t->string('resolution_reason'); $t->integer('occurred_at'); $t->integer('created_at'); }); -$schema->create('cloud_abuse_alert_deliveries', static function (Blueprint $t): void { +$schema->create('cloud_network_alert_deliveries', static function (Blueprint $t): void { $t->increments('id'); $t->integer('event_id')->unique(); $t->string('binding_generation')->nullable(); $t->integer('bound_admin_id')->nullable(); $t->string('chat_id')->nullable(); @@ -119,27 +117,23 @@ final class AbuseDeliveryFakeChannel implements AdministratorTelegramDeliveryCha $identity = 1; $addEvent = static function () use ($db, &$identity): int { $number = $identity++; $bucket = 1800000000 + ($number * 60); - $id = (int) $db->table('cloud_abuse_alert_events')->insertGetId([ - 'event_key' => hash('sha256', 'event-' . $number), - 'sample_key' => hash('sha256', 'sample-' . $number), - 'instance_id' => 100 + $number, 'user_id' => 10, 'host_id' => 20, - 'bucket_start' => $bucket, 'rule_version' => 1, - 'policy_revision' => 1, 'policy_hash' => str_repeat('b', 64), - 'transition_digest' => str_repeat('c', 64), 'transition_count' => 1, - 'highest_severity' => 'high', 'created_at' => $bucket, + $instanceId = 100 + $number; + $db->table('cloud_instances')->insert([ + 'id' => $instanceId, 'user_id' => 10, 'host_id' => 20, ]); - $db->table('cloud_abuse_alert_event_items')->insert([ - 'event_id' => $id, 'item_key' => hash('sha256', 'item-' . $number), - 'finding_key' => hash('sha256', 'finding-' . $number), - 'occurrence_generation' => 1, 'transition_revision' => 1, + $findingId = (int) $db->table('cloud_network_findings')->insertGetId([ + 'instance_id' => $instanceId, 'policy_revision' => 1, 'rule_code' => 'outbound_tcp_syn_sweep', 'direction' => 'outbound', - 'ip_family' => 'ipv4', 'transition_type' => 'opened', + 'ip_family' => 'ipv4', + ]); + return (int) $db->table('cloud_network_alert_events')->insertGetId([ + 'finding_id' => $findingId, 'transition_revision' => 1, + 'transition_type' => 'opened', 'from_severity' => 'none', 'to_severity' => 'high', 'evidence_json' => '{"tcp_syn_flows":5000,"unique_remote_ips":600}', 'resolution_reason' => '', 'occurred_at' => $bucket, 'created_at' => $bucket, ]); - return $id; }; $now = 1900000000; $channel = new AbuseDeliveryFakeChannel(); @@ -149,7 +143,7 @@ $service = new CloudAbuseAlertDeliveryService($channel, $first = $addEvent(); $assert($service->tick() && $channel->calls === 1, 'Fresh alert was not sent exactly once.'); -$sent = $db->table('cloud_abuse_alert_deliveries')->where('event_id', $first)->first(); +$sent = $db->table('cloud_network_alert_deliveries')->where('event_id', $first)->first(); $assert($sent?->state === 'sent' && (int) $sent->telegram_message_id === 7001 && hash_equals((string) $sent->content_hash, hash('sha256', (string) $sent->message_text)), @@ -161,7 +155,7 @@ $assert($service->tick() && $channel->calls === 2, 'A second alert was not delegated to the shared channel.'); $channel->destination = null; $skipped = $addEvent(); -$assert($service->tick() && $db->table('cloud_abuse_alert_deliveries') +$assert($service->tick() && $db->table('cloud_network_alert_deliveries') ->where('event_id', $skipped)->value('state') === 'skipped', 'Unbound channel did not terminally skip the event.'); $channel->destination = new AdministratorTelegramDestination( @@ -169,22 +163,22 @@ $channel->destination = new AdministratorTelegramDestination( ); $local = $addEvent(); $channel->destinationMode = 'local'; $threw = false; try { $service->tick(); } catch (RuntimeException) { $threw = true; } -$assert($threw && !$db->table('cloud_abuse_alert_deliveries') +$assert($threw && !$db->table('cloud_network_alert_deliveries') ->where('event_id', $local)->exists(), 'Local destination failure was hidden or consumed the event.'); $channel->destinationMode = 'ready'; $channel->sendMode = 'preflight'; $now += 2; -$assert($service->tick() && $db->table('cloud_abuse_alert_deliveries') +$assert($service->tick() && $db->table('cloud_network_alert_deliveries') ->where('event_id', $local)->value('state') === 'skipped', 'Destination change was not safely skipped before network I/O.'); $payload = $addEvent(); $channel->sendMode = 'payload'; $now += 2; -$assert($service->tick() && $db->table('cloud_abuse_alert_deliveries') +$assert($service->tick() && $db->table('cloud_network_alert_deliveries') ->where('event_id', $payload)->value('state') === 'failed' - && $db->table('cloud_abuse_alert_deliveries') + && $db->table('cloud_network_alert_deliveries') ->where('event_id', $payload)->value('dispatch_started_at') === null, 'Invalid payload was not a known pre-network failure.'); $disabled = $addEvent(); $channel->sendMode = 'channel'; $now += 2; -$assert($service->tick() && $db->table('cloud_abuse_alert_deliveries') +$assert($service->tick() && $db->table('cloud_network_alert_deliveries') ->where('event_id', $disabled)->value('skip_reason') === 'channel_disabled_or_unavailable', 'Disabled channel was mislabeled as a destination change.'); @@ -192,7 +186,7 @@ $assert($service->tick() && $db->table('cloud_abuse_alert_deliveries') $backlogged = $addEvent(); $channel->sendMode = 'backlog'; $now += 2; $threw = false; try { $service->tick(); } catch (RuntimeException) { $threw = true; } -$assert($threw && $db->table('cloud_abuse_alert_deliveries') +$assert($threw && $db->table('cloud_network_alert_deliveries') ->where('event_id', $backlogged)->value('state') === 'pending', 'Known-no-remote Telegram backlog was not retained as pending.'); $channel->sendMode = 'success'; $now += 2; @@ -200,12 +194,12 @@ $assert($service->tick(), 'Backlogged alert did not resume safely.'); $retryable = $addEvent(); $channel->sendMode = 'local'; $now += 2; $threw = false; try { $service->tick(); } catch (RuntimeException) { $threw = true; } -$assert($threw && $db->table('cloud_abuse_alert_deliveries') +$assert($threw && $db->table('cloud_network_alert_deliveries') ->where('event_id', $retryable)->value('state') === 'pending', 'Local send failure did not return frozen work to pending.'); $channel->sendMode = 'remote'; $now += 2; $threw = false; try { $service->tick(); } catch (RuntimeException) { $threw = true; } -$assert($threw && $db->table('cloud_abuse_alert_deliveries') +$assert($threw && $db->table('cloud_network_alert_deliveries') ->where('event_id', $retryable)->value('state') === 'unknown', 'Remote-effect failure was not fenced as unknown.'); @@ -215,13 +209,13 @@ $failing = new CloudAbuseAlertDeliveryService($channel, static function () use (&$now): int { return $now; }, static function (int $messageId): void { throw new RuntimeException('fail'); }); $threw = false; try { $failing->tick(); } catch (RuntimeException) { $threw = true; } -$row = $db->table('cloud_abuse_alert_deliveries')->where('event_id', $afterRemote)->first(); +$row = $db->table('cloud_network_alert_deliveries')->where('event_id', $afterRemote)->first(); $assert($threw && $row?->state === 'unknown' && (int) $row->telegram_message_id > 0, 'Post-send finalization failure was not retained as unknown.'); $stale = $addEvent(); -$db->table('cloud_abuse_alert_deliveries')->insert([ +$db->table('cloud_network_alert_deliveries')->insert([ 'event_id' => $stale, 'binding_generation' => str_repeat('a', 32), 'bound_admin_id' => 1, 'chat_id' => '123456', 'telegram_message_id' => null, 'state' => 'dispatching', 'message_text' => 'frozen', @@ -231,7 +225,7 @@ $db->table('cloud_abuse_alert_deliveries')->insert([ 'destination_redacted_at' => null, 'created_at' => $now - 40, 'updated_at' => $now - 40, ]); -$assert($service->tick() && $db->table('cloud_abuse_alert_deliveries') +$assert($service->tick() && $db->table('cloud_network_alert_deliveries') ->where('event_id', $stale)->value('state') === 'unknown', 'Stale dispatch was not fenced as unknown.'); diff --git a/web/tests/CloudAbuseAlertMessageFormatterTest.php b/web/tests/CloudAbuseAlertMessageFormatterTest.php index 51f87fa1..985e1dd3 100644 --- a/web/tests/CloudAbuseAlertMessageFormatterTest.php +++ b/web/tests/CloudAbuseAlertMessageFormatterTest.php @@ -19,24 +19,13 @@ $throws = static function (callable $callback, string $message) use ($assert): v } $assert(false, $message); }; - -$event = (object) [ - 'id' => 41, - 'event_key' => str_repeat('a', 64), - 'instance_id' => 91, - 'user_id' => 17, - 'host_id' => 3, - 'bucket_start' => 1794182400, - 'rule_version' => 1, - 'policy_revision' => 4, - 'transition_count' => 2, - 'highest_severity' => 'high', -]; -$item = static function (array $overrides = []): object { +$event = static function (array $overrides = []): object { return (object) array_replace([ - 'event_id' => 41, - 'item_key' => str_repeat('b', 64), - 'finding_key' => str_repeat('c', 64), + 'id' => 41, + 'instance_id' => 91, + 'user_id' => 17, + 'host_id' => 3, + 'policy_revision' => 4, 'rule_code' => 'outbound_tcp_syn_sweep', 'direction' => 'outbound', 'ip_family' => 'ipv4', @@ -51,89 +40,59 @@ $item = static function (array $overrides = []): object { 'occurred_at' => 1794182400, ], $overrides); }; -$items = [ - $item(), - $item([ - 'item_key' => str_repeat('d', 64), - 'finding_key' => str_repeat('e', 64), - 'rule_code' => 'inbound_udp_flood', - 'direction' => 'inbound', - 'ip_family' => 'ipv6', - 'transition_type' => 'resolved', - 'from_severity' => 'suspicious', - 'to_severity' => 'none', - 'evidence_json' => '{"udp_active_flows":0,"udp_new_flows":0}', - 'resolution_reason' => 'evidence_clear', - ]), -]; $formatter = new CloudAbuseAlertMessageFormatter(); Config::set('baseUrl', 'https://panel.example.test'); -$message = $formatter->format($event, $items); +$message = $formatter->format($event()); $assert(str_contains($message, '网络滥用高风险告警') && str_contains($message, '告警:#41') && str_contains($message, '实例:#91') && str_contains($message, '用户:#17') && str_contains($message, '宿主机:#3') && str_contains($message, '出站可疑行为') - && str_contains($message, '入站受攻击') && str_contains($message, 'TCP SYN Flow 4,321') && str_contains($message, 'https://panel.example.test/admin/abuse-monitoring?query=91'), - 'Formatter omitted immutable identity, direction or aggregate evidence.'); + 'Formatter omitted transition identity, direction or aggregate evidence.'); $assert(!str_contains($message, '203.0.113.9') - && !str_contains($message, '通信正文:'), - 'Formatter exposed address or packet-content fields.'); -$assert(mb_strlen($message, 'UTF-8') <= 4096, - 'Formatter exceeded Telegram message length.'); + && !str_contains($message, '通信正文:') + && mb_strlen($message, 'UTF-8') <= 4096, + 'Formatter exposed sensitive data or exceeded Telegram limits.'); -$resolvedEvent = clone $event; -$resolvedEvent->transition_count = 1; -$resolvedEvent->highest_severity = 'suspicious'; -Config::set('baseUrl', 'http://127.0.0.1:8765'); -$resolved = $formatter->format($resolvedEvent, [$items[1]]); +$resolved = $formatter->format($event([ + 'rule_code' => 'inbound_udp_flood', + 'direction' => 'inbound', + 'ip_family' => 'ipv6', + 'transition_type' => 'resolved', + 'from_severity' => 'suspicious', + 'to_severity' => 'none', + 'evidence_json' => '{"udp_active_flows":0,"udp_new_flows":0}', + 'resolution_reason' => 'policy_replaced', +])); $assert(str_contains($resolved, '网络风险已恢复') - && str_contains($resolved, '可疑状态已解除'), - 'Resolved alert was not rendered as a recovery.'); - -$manyItems = []; -for ($index = 0; $index < 80; $index++) { - $manyItems[] = $item([ - 'item_key' => hash('sha256', 'item-' . $index), - 'finding_key' => hash('sha256', 'finding-' . $index), - ]); -} -$manyEvent = clone $event; -$manyEvent->transition_count = count($manyItems); -Config::set('baseUrl', 'https://panel.example.test/base'); -$bounded = $formatter->format($manyEvent, $manyItems); -$assert(mb_strlen($bounded, 'UTF-8') <= 4096 - && str_contains($bounded, '其他规则:'), - 'Large immutable event was not summarized within Telegram limits.'); + && str_contains($resolved, '策略更新后关闭') + && str_contains($resolved, '入站受攻击'), + 'Resolved policy transition was not rendered correctly.'); -$invalidEvidence = array_map(static fn (object $row): object => clone $row, $items); -$invalidEvidence[0]->evidence_json = - '{"remote_ip":"203.0.113.9","tcp_syn_flows":5}'; $throws( - static fn () => $formatter->format($event, $invalidEvidence), + static fn () => $formatter->format($event([ + 'evidence_json' => '{"remote_ip":"203.0.113.9","tcp_syn_flows":5}', + ])), 'Address-bearing evidence was accepted.' ); -$wrongEvent = clone $event; -$wrongEvent->transition_count = 1; $throws( - static fn () => $formatter->format($wrongEvent, $items), - 'Event/item count mismatch was accepted.' -); -$wrongBucket = array_map(static fn (object $row): object => clone $row, $items); -$wrongBucket[0]->occurred_at++; -$throws( - static fn () => $formatter->format($event, $wrongBucket), - 'Cross-bucket alert item was accepted.' + static fn () => $formatter->format($event([ + 'transition_type' => 'resolved', + 'from_severity' => 'high', + 'to_severity' => 'none', + 'resolution_reason' => 'unknown_reason', + ])), + 'Unknown resolution reason was accepted.' ); $throws( - static function () use ($formatter, $event, $items): void { + static function () use ($formatter, $event): void { Config::set('baseUrl', "https://panel.example.test\n.invalid"); - $formatter->format($event, $items); + $formatter->format($event()); }, 'Unsafe administrator base URL was accepted.' ); diff --git a/web/tests/CloudAbuseRetentionServiceTest.php b/web/tests/CloudAbuseRetentionServiceTest.php index 0df13463..02170321 100644 --- a/web/tests/CloudAbuseRetentionServiceTest.php +++ b/web/tests/CloudAbuseRetentionServiceTest.php @@ -1,510 +1,144 @@ addConnection([ - 'driver' => 'sqlite', 'database' => ':memory:', + 'driver' => 'sqlite', 'database' => ':memory:', 'prefix' => '', ], 'default'); -$capsule->addConnection([ - 'driver' => 'sqlite', 'database' => ':memory:', -], 'log'); $capsule->setAsGlobal(); $capsule->bootEloquent(); +User::setConnectionResolver($capsule->getDatabaseManager()); +$db = $capsule->getConnection('default'); +$schema = $db->getSchemaBuilder(); -$mainSchema = $capsule->schema('default'); -$mainSchema->create('cloud_instances', static function (Blueprint $table): void { +$schema->create('cloud_hosts', static function (Blueprint $table): void { $table->increments('id'); }); -$mainSchema->create( - 'cloud_abuse_projection_receipts', - static function (Blueprint $table): void { - $table->string('sample_key')->primary(); - $table->integer('instance_id'); - $table->integer('rule_version'); - $table->integer('policy_revision')->default(1); - $table->string('policy_hash')->default( - '511828bde950c504f92bb7a0603ca8b14453f575ead276c1889baa764f384dae' - ); - $table->integer('bucket_start'); - $table->integer('projected_at'); - } -); -$mainSchema->create('cloud_abuse_findings', - static function (Blueprint $table): void { - $table->increments('id'); - $table->string('finding_key')->unique(); - $table->integer('instance_id'); - $table->integer('user_id'); - $table->integer('host_id'); - $table->string('rule_code'); - $table->integer('rule_version'); - $table->integer('policy_revision')->default(1); - $table->string('policy_hash')->default( - '511828bde950c504f92bb7a0603ca8b14453f575ead276c1889baa764f384dae' - ); - $table->string('direction'); - $table->string('ip_family'); - $table->string('severity'); - $table->string('state'); - $table->integer('first_seen_at'); - $table->integer('last_seen_at'); - $table->integer('consecutive_windows'); - $table->integer('occurrence_generation')->default(1); - $table->integer('transition_revision')->default(1); - $table->text('evidence_json'); - $table->integer('resolved_at')->nullable(); - $table->string('resolution_reason')->default(''); - $table->integer('created_at'); - $table->integer('updated_at'); - }); -$mainSchema->create('cloud_abuse_alert_events', - static function (Blueprint $table): void { - $table->increments('id'); - $table->string('sample_key')->unique(); - $table->integer('instance_id'); - $table->integer('user_id'); - $table->integer('created_at'); - }); -$mainSchema->create('cloud_abuse_alert_event_items', - static function (Blueprint $table): void { - $table->increments('id'); - $table->integer('event_id'); - $table->string('finding_key'); - $table->foreign('event_id')->references('id') - ->on('cloud_abuse_alert_events')->cascadeOnDelete(); - }); -$mainSchema->create('cloud_abuse_alert_deliveries', - static function (Blueprint $table): void { - $table->increments('id'); - $table->integer('event_id')->unique(); - $table->string('state'); - $table->integer('terminal_at')->nullable(); - $table->foreign('event_id')->references('id') - ->on('cloud_abuse_alert_events')->cascadeOnDelete(); - }); +$schema->create('cloud_instances', static function (Blueprint $table): void { + $table->increments('id'); +}); +$schema->create('cloud_network_monitor_hosts', static function (Blueprint $table): void { + $table->integer('host_id')->primary(); + $table->string('poll_lease_token')->nullable(); + $table->integer('poll_lease_until')->default(0); + $table->integer('updated_at'); +}); +$schema->create('cloud_network_daily_summaries', static function (Blueprint $table): void { + $table->integer('instance_id'); + $table->integer('day_start'); + $table->primary(['instance_id', 'day_start']); +}); +$schema->create('cloud_network_findings', static function (Blueprint $table): void { + $table->increments('id'); + $table->string('state'); + $table->integer('resolved_at')->nullable(); +}); +$schema->create('cloud_network_alert_events', static function (Blueprint $table): void { + $table->increments('id'); + $table->integer('finding_id'); + $table->integer('created_at'); +}); +$schema->create('cloud_network_alert_deliveries', static function (Blueprint $table): void { + $table->increments('id'); + $table->integer('event_id')->unique(); + $table->string('state'); + $table->integer('terminal_at')->nullable(); +}); -$logSchema = $capsule->schema('log'); -$logSchema->create( - 'cloud_vm_network_observation_samples', - static function (Blueprint $table): void { - $table->string('sample_key')->primary(); - $table->integer('instance_id'); - $table->integer('bucket_start'); - $table->integer('rule_version')->default(1); - $table->integer('policy_revision')->default(1); - $table->string('policy_hash')->default( - '511828bde950c504f92bb7a0603ca8b14453f575ead276c1889baa764f384dae' - ); - } -); -$logSchema->create( - 'cloud_vm_network_observation_rollups', - static function (Blueprint $table): void { - $table->integer('instance_id'); - $table->integer('interval_seconds'); - $table->integer('bucket_start'); - $table->primary(['instance_id', 'interval_seconds', 'bucket_start']); - } -); -$logSchema->create( - 'cloud_vm_network_projection_outbox', - static function (Blueprint $table): void { - $table->string('sample_key')->primary(); - $table->integer('instance_id'); - $table->integer('bucket_start'); - $table->string('state'); - $table->integer('terminal_at')->nullable(); - } -); -$logSchema->create( - 'cloud_vm_network_baselines', - static function (Blueprint $table): void { - $table->integer('instance_id')->primary(); - $table->integer('user_id'); - $table->integer('rule_version')->default(1); - $table->integer('policy_revision')->default(1); - $table->string('policy_hash')->default( - '511828bde950c504f92bb7a0603ca8b14453f575ead276c1889baa764f384dae' - ); +$assert = static function (bool $condition, string $message): void { + if (!$condition) { + throw new RuntimeException($message); } -); - -$main = $capsule->connection('default'); -$log = $capsule->connection('log'); -$main->statement('PRAGMA foreign_keys = ON'); -$now = 20_000_000; -$sampleCutoff = $now - - CloudAbuseRetentionService::RAW_TELEMETRY_RETENTION_SECONDS; -$outboxCutoff = $now - - CloudAbuseRetentionService::TERMINAL_OUTBOX_RETENTION_SECONDS; -$receiptCutoff = $now - - CloudAbuseRetentionService::RECEIPT_RETENTION_SECONDS; -$readCutoff = $now - - CloudAbuseRetentionService::READ_MODEL_RETENTION_SECONDS; -$oldBucket = intdiv($sampleCutoff - 120, 60) * 60; -$recentBucket = intdiv($sampleCutoff + 120, 60) * 60; -$defaultPolicyHash = - '511828bde950c504f92bb7a0603ca8b14453f575ead276c1889baa764f384dae'; - -$key = static fn (string $name): string => hash('sha256', $name); -$fixtureFindingKey = static fn (string $name): string => $key( - 'finding-' . $defaultPolicyHash . '-' . $name -); -$sample = static function ( - string $name, - int $instanceId, - int $bucket -) use ($log, $key): string { - $sampleKey = $key($name); - $log->table('cloud_vm_network_observation_samples')->insert([ - 'sample_key' => $sampleKey, - 'instance_id' => $instanceId, - 'bucket_start' => $bucket, - ]); - - return $sampleKey; -}; -$outbox = static function ( - string $sampleKey, - int $instanceId, - int $bucket, - string $state, - ?int $terminalAt = null -) use ($log): void { - $log->table('cloud_vm_network_projection_outbox')->insert([ - 'sample_key' => $sampleKey, - 'instance_id' => $instanceId, - 'bucket_start' => $bucket, - 'state' => $state, - 'terminal_at' => $terminalAt, - ]); -}; -$receipt = static function ( - string $sampleKey, - int $instanceId, - int $projectedAt -) use ($main): void { - $main->table('cloud_abuse_projection_receipts')->insert([ - 'sample_key' => $sampleKey, - 'instance_id' => $instanceId, - 'rule_version' => 1, - 'bucket_start' => 60, - 'projected_at' => $projectedAt, - ]); -}; -$finding = static function ( - string $name, - string $state, - ?int $resolvedAt -) use ($main, $fixtureFindingKey, $now): void { - $main->table('cloud_abuse_findings')->insert([ - 'finding_key' => $fixtureFindingKey($name), - 'instance_id' => 1, - 'user_id' => 1, - 'host_id' => 1, - 'rule_code' => 'test_rule', - 'rule_version' => 1, - 'direction' => 'outbound', - 'ip_family' => 'ipv4', - 'severity' => 'info', - 'state' => $state, - 'first_seen_at' => 60, - 'last_seen_at' => 60, - 'consecutive_windows' => 1, - 'evidence_json' => '{}', - 'resolved_at' => $resolvedAt, - 'created_at' => 60, - 'updated_at' => $now, - ]); }; - -$main->table('cloud_instances')->insert(['id' => 1]); -$log->table('cloud_vm_network_baselines')->insert([ - ['instance_id' => 1, 'user_id' => 1], - ['instance_id' => 2, 'user_id' => 2], - ['instance_id' => 3, 'user_id' => 3], +$now = 2_000_000_000; +$historyCutoff = $now - CloudAbuseRetentionService::ALERT_HISTORY_RETENTION_SECONDS; +$dailyCutoff = $now - CloudAbuseRetentionService::DAILY_SUMMARY_RETENTION_SECONDS; +$hostCutoff = $now - CloudAbuseRetentionService::ORPHAN_HOST_RETENTION_SECONDS; + +$db->table('cloud_hosts')->insert(['id' => 1]); +$db->table('cloud_instances')->insert([['id' => 1], ['id' => 2]]); +$db->table('cloud_network_daily_summaries')->insert([ + ['instance_id' => 1, 'day_start' => $dailyCutoff - 86400], + ['instance_id' => 1, 'day_start' => $dailyCutoff + 86400], ]); - -$doneOld = $sample('done-old', 1, $oldBucket); -$outbox($doneOld, 1, $oldBucket, 'done', $outboxCutoff - 1); -$receipt($doneOld, 1, $receiptCutoff - 1); - -$pendingOld = $sample('pending-old', 3, $oldBucket); -$outbox($pendingOld, 3, $oldBucket, 'pending'); -$leasedOld = $sample('leased-old', 1, $oldBucket); -$outbox($leasedOld, 1, $oldBucket, 'leased'); -$doneRecent = $sample('done-recent', 1, $recentBucket); -$outbox($doneRecent, 1, $recentBucket, 'done', $outboxCutoff + 1); -$orphanOld = $sample('orphan-old', 1, $oldBucket); - -$receiptGone = $key('receipt-gone'); -$receipt($receiptGone, 1, $receiptCutoff - 1); -$receiptRecent = $key('receipt-recent'); -$receipt($receiptRecent, 1, $receiptCutoff + 1); -$receiptWithSample = $sample('receipt-with-sample', 1, $recentBucket); -$receipt($receiptWithSample, 1, $receiptCutoff - 1); - -$fiveMinuteCutoff = $now - - CloudAbuseRetentionService::FIVE_MINUTE_ROLLUP_RETENTION_SECONDS; -$hourlyCutoff = $now - - CloudAbuseRetentionService::HOURLY_ROLLUP_RETENTION_SECONDS; -$log->table('cloud_vm_network_observation_rollups')->insert([ - ['instance_id' => 1, 'interval_seconds' => 300, - 'bucket_start' => intdiv($fiveMinuteCutoff - 300, 300) * 300], - ['instance_id' => 1, 'interval_seconds' => 300, - 'bucket_start' => intdiv($fiveMinuteCutoff + 300, 300) * 300], - ['instance_id' => 1, 'interval_seconds' => 3600, - 'bucket_start' => intdiv($hourlyCutoff - 3600, 3600) * 3600], - ['instance_id' => 1, 'interval_seconds' => 3600, - 'bucket_start' => intdiv($hourlyCutoff + 3600, 3600) * 3600], +$db->table('cloud_network_monitor_hosts')->insert([ + ['host_id' => 1, 'poll_lease_token' => null, 'poll_lease_until' => 0, + 'updated_at' => $hostCutoff - 1], + ['host_id' => 2, 'poll_lease_token' => null, 'poll_lease_until' => 0, + 'updated_at' => $hostCutoff - 1], + ['host_id' => 3, 'poll_lease_token' => str_repeat('a', 32), + 'poll_lease_until' => $now + 60, 'updated_at' => $hostCutoff - 1], + ['host_id' => 4, 'poll_lease_token' => null, 'poll_lease_until' => 0, + 'updated_at' => $hostCutoff + 60], ]); - -$finding('resolved-old', 'resolved', $readCutoff - 1); -$finding('resolved-recent', 'resolved', $readCutoff + 1); -$finding('open-old', 'open', null); -$oldEventFindingKey = $fixtureFindingKey('resolved-old'); -$main->table('cloud_abuse_alert_events')->insert([ - 'id' => 1, - 'sample_key' => $key('old-alert-event'), - 'instance_id' => 1, - 'user_id' => 1, - 'created_at' => $readCutoff - 1, +$db->table('cloud_network_findings')->insert([ + ['id' => 1, 'state' => 'resolved', 'resolved_at' => $historyCutoff - 10], + ['id' => 2, 'state' => 'resolved', 'resolved_at' => $historyCutoff - 10], + ['id' => 3, 'state' => 'resolved', 'resolved_at' => $historyCutoff - 10], + ['id' => 4, 'state' => 'open', 'resolved_at' => null], + ['id' => 5, 'state' => 'resolved', 'resolved_at' => $historyCutoff + 60], ]); -$main->table('cloud_abuse_alert_event_items')->insert([ - 'id' => 1, - 'event_id' => 1, - 'finding_key' => $oldEventFindingKey, +$db->table('cloud_network_alert_events')->insert([ + ['id' => 1, 'finding_id' => 1, 'created_at' => $historyCutoff - 10], + ['id' => 2, 'finding_id' => 2, 'created_at' => $historyCutoff - 10], + ['id' => 3, 'finding_id' => 2, 'created_at' => $historyCutoff - 10], + ['id' => 4, 'finding_id' => 2, 'created_at' => $historyCutoff - 10], + ['id' => 5, 'finding_id' => 2, 'created_at' => $historyCutoff - 10], ]); -foreach ([2, 3, 4] as $eventId) { - $main->table('cloud_abuse_alert_events')->insert([ - 'id' => $eventId, - 'sample_key' => $key('delivery-alert-' . $eventId), - 'instance_id' => 1, - 'user_id' => 1, - 'created_at' => $readCutoff - 1, - ]); - $main->table('cloud_abuse_alert_event_items')->insert([ - 'id' => $eventId, - 'event_id' => $eventId, - 'finding_key' => $key('delivery-finding-' . $eventId), - ]); -} -$main->table('cloud_abuse_alert_deliveries')->insert([ - ['event_id' => 2, 'state' => 'sent', - 'terminal_at' => $readCutoff - 1], - ['event_id' => 3, 'state' => 'unknown', - 'terminal_at' => $readCutoff - 1], +$db->table('cloud_network_alert_deliveries')->insert([ + ['event_id' => 1, 'state' => 'sent', 'terminal_at' => $historyCutoff - 5], + ['event_id' => 2, 'state' => 'unknown', 'terminal_at' => $historyCutoff - 5], ['event_id' => 4, 'state' => 'pending', 'terminal_at' => null], + ['event_id' => 5, 'state' => 'skipped', 'terminal_at' => $historyCutoff + 60], ]); $service = new CloudAbuseRetentionService(static fn (): int => $now); $result = $service->prune(); - $assert($result === [ - 'terminal_outboxes' => 1, - 'samples' => 2, - 'five_minute_rollups' => 1, - 'hourly_rollups' => 1, - 'baselines' => 1, - 'receipts' => 3, - 'findings' => 1, + 'daily_summaries' => 1, 'alert_events' => 1, -], 'Retention pass returned unexpected bounded deletion counts.'); -$assert(!$log->table('cloud_vm_network_projection_outbox') - ->where('sample_key', $doneOld)->exists() - && !$log->table('cloud_vm_network_observation_samples') - ->where('sample_key', $doneOld)->exists(), - 'Expired terminal projection and sample were not removed together.'); -$assert(!$log->table('cloud_vm_network_observation_samples') - ->where('sample_key', $orphanOld)->exists(), - 'Expired orphan sample was not removed.'); -foreach ([$pendingOld, $leasedOld, $doneRecent] as $protectedKey) { - $assert($log->table('cloud_vm_network_projection_outbox') - ->where('sample_key', $protectedKey)->exists() - && $log->table('cloud_vm_network_observation_samples') - ->where('sample_key', $protectedKey)->exists(), - 'Pending, leased or recent telemetry was removed.'); -} -$assert($log->table('cloud_vm_network_baselines') - ->where('instance_id', 1)->exists() - && !$log->table('cloud_vm_network_baselines') - ->where('instance_id', 2)->exists() - && $log->table('cloud_vm_network_baselines') - ->where('instance_id', 3)->exists(), - 'Orphan baseline cleanup ignored live ownership or active projection.'); -$assert(!$main->table('cloud_abuse_projection_receipts') - ->whereIn('sample_key', [ - $doneOld, $receiptGone, $receiptWithSample, - ])->exists() - && $main->table('cloud_abuse_projection_receipts') - ->where('sample_key', $receiptRecent)->exists(), - 'Receipt retention ignored the bounded replay horizon.'); -$assert($log->table('cloud_vm_network_observation_rollups')->count() === 2, - 'Observation rollups did not keep the correct bounded horizons.'); -$assert($main->table('cloud_abuse_findings')->count() === 2 - && !$main->table('cloud_abuse_findings') - ->where( - 'finding_key', - $fixtureFindingKey('resolved-old') - )->exists(), - 'Finding retention removed an open/recent finding or kept an old one.'); -$assert($main->table('cloud_abuse_alert_events')->where('id', 1)->exists() - && $main->table('cloud_abuse_alert_event_items') - ->where('event_id', 1)->exists(), - 'Phase-6 retention removed an alert before delivery finality exists.'); -$assert(!$main->table('cloud_abuse_alert_events')->where('id', 2)->exists() - && !$main->table('cloud_abuse_alert_event_items') - ->where('event_id', 2)->exists() - && $main->table('cloud_abuse_alert_events') - ->whereIn('id', [3, 4])->count() === 2, - 'Alert retention removed pending/unknown work or retained known terminal work.'); - -// Terminal projection state and raw evidence have independent horizons. A raw -// deletion failure must leave only the sample for a later retry; it must not -// resurrect already-finalized projection work. -$rollbackKey = $sample('rollback-terminal', 1, $oldBucket); -$outbox( - $rollbackKey, - 1, - $oldBucket, - 'discarded', - $outboxCutoff - 1 -); -$quotedRollback = $log->getPdo()->quote($rollbackKey); -$log->statement( - 'CREATE TRIGGER fail_abuse_sample_delete ' - . 'BEFORE DELETE ON cloud_vm_network_observation_samples ' - . 'WHEN OLD.sample_key = ' . $quotedRollback . ' ' - . "BEGIN SELECT RAISE(ABORT, 'injected retention failure'); END" -); -$failed = false; -try { - $service->prune(); -} catch (Throwable) { - $failed = true; -} -$assert($failed - && !$log->table('cloud_vm_network_projection_outbox') - ->where('sample_key', $rollbackKey)->exists() - && $log->table('cloud_vm_network_observation_samples') - ->where('sample_key', $rollbackKey)->exists(), - 'Independent terminal intent and sample retention was not restart-safe.'); -$log->statement('DROP TRIGGER fail_abuse_sample_delete'); -$retry = $service->prune(); -$assert($retry['samples'] === 1 - && !$log->table('cloud_vm_network_observation_samples') - ->where('sample_key', $rollbackKey)->exists(), - 'Raw sample cleanup did not resume after a transient storage failure.'); - -// The configured bound applies to each independently indexed retention set. -$finding('batch-one', 'resolved', $readCutoff - 2); -$finding('batch-two', 'resolved', $readCutoff - 2); -$bounded = $service->prune(1); -$assert($bounded['findings'] === 1 - && $main->table('cloud_abuse_findings') - ->whereIn('finding_key', [ - $fixtureFindingKey('batch-one'), - $fixtureFindingKey('batch-two'), - ])->count() === 1, - 'Retention batch size did not bound the indexed finding pass.'); + 'resolved_findings' => 2, + 'orphan_hosts' => 1, +], 'Compact retention returned an unexpected deletion ledger.'); +$assert(!$db->table('cloud_network_daily_summaries') + ->where('day_start', $dailyCutoff - 86400)->exists() + && $db->table('cloud_network_daily_summaries') + ->where('day_start', $dailyCutoff + 86400)->exists(), + 'Daily summaries did not respect the retention cutoff.'); +$assert(!$db->table('cloud_network_alert_events')->where('id', 1)->exists() + && $db->table('cloud_network_alert_events')->whereIn('id', [2, 3, 4, 5]) + ->count() === 4, + 'Unknown, pending, undelivered or recent-terminal events lost their fence.'); +$assert(!$db->table('cloud_network_findings')->whereIn('id', [1, 3])->exists() + && $db->table('cloud_network_findings')->whereIn('id', [2, 4, 5]) + ->count() === 3, + 'Finding retention ignored live events, state or age.'); +$assert(!$db->table('cloud_network_monitor_hosts')->where('host_id', 2)->exists() + && $db->table('cloud_network_monitor_hosts')->whereIn('host_id', [1, 3, 4]) + ->count() === 3, + 'Host retention removed a live, leased or recent cursor.'); $assert($service->prune() === [ - 'terminal_outboxes' => 0, - 'samples' => 0, - 'five_minute_rollups' => 0, - 'hourly_rollups' => 0, - 'baselines' => 0, - 'receipts' => 0, - 'findings' => 1, + 'daily_summaries' => 0, 'alert_events' => 0, -], 'A restart-safe retention pass was not idempotent.'); - -// Protected rows at the front of a bounded scan must not permanently starve -// an eligible row beyond the limit. The process cursor advances and wraps; -// a time-derived first anchor also varies repeated --once worker starts. -$log->table('cloud_vm_network_projection_outbox')->delete(); -$log->table('cloud_vm_network_observation_samples')->delete(); -$log->table('cloud_vm_network_baselines')->delete(); -$main->table('cloud_abuse_projection_receipts')->delete(); -$main->table('cloud_instances')->insert(['id' => 2]); -$log->table('cloud_vm_network_baselines')->insert([ - ['instance_id' => 1, 'user_id' => 1], - ['instance_id' => 2, 'user_id' => 2], - ['instance_id' => 3, 'user_id' => 3], -]); -$scanMinute = intdiv($now, 60); -$scanMinute -= $scanMinute % 4; -$scanNow = $scanMinute * 60; -$scanService = new CloudAbuseRetentionService( - static fn (): int => $scanNow -); -$firstBaselineScan = $scanService->prune(2); -$assert($firstBaselineScan['baselines'] === 0 - && $log->table('cloud_vm_network_baselines') - ->where('instance_id', 3)->exists(), - 'A protected baseline prefix was not exercised.'); -$secondBaselineScan = $scanService->prune(2); -$assert($secondBaselineScan['baselines'] === 1 - && !$log->table('cloud_vm_network_baselines') - ->where('instance_id', 3)->exists(), - 'A high-ID orphan baseline remained starved after scan wraparound.'); - -$log->table('cloud_vm_network_projection_outbox')->delete(); -$log->table('cloud_vm_network_observation_samples')->delete(); -$main->table('cloud_abuse_projection_receipts')->delete(); -$receiptKeys = []; -$scanReadCutoff = $scanNow - - CloudAbuseRetentionService::RECEIPT_RETENTION_SECONDS; -foreach ([1, 2, 3] as $number) { - $receiptKeys[$number] = str_pad( - dechex($number), - 64, - '0', - STR_PAD_LEFT - ); - $receipt($receiptKeys[$number], 1, $scanReadCutoff - 1); - if ($number <= 2) { - $log->table('cloud_vm_network_observation_samples')->insert([ - 'sample_key' => $receiptKeys[$number], - 'instance_id' => 1, - 'bucket_start' => $oldBucket, - ]); - $outbox( - $receiptKeys[$number], - 1, - $oldBucket, - 'pending' - ); - } -} -$receiptScanService = new CloudAbuseRetentionService( - static fn (): int => $scanNow -); -$receiptScanService->prune(2); -$receiptScanService->prune(2); -$assert($main->table('cloud_abuse_projection_receipts')->count() === 0, - 'Receipt tombstones outlived the bounded Agent replay horizon.'); -$assert($log->table('cloud_vm_network_projection_outbox')->count() === 2 - && $log->table('cloud_vm_network_observation_samples')->count() === 2, - 'Receipt scan removed pending or leased projection evidence.'); + 'resolved_findings' => 0, + 'orphan_hosts' => 0, +], 'Compact retention was not idempotent.'); foreach ([0, CloudAbuseRetentionService::DEFAULT_BATCH_SIZE + 1] as $limit) { + $threw = false; try { $service->prune($limit); - throw new RuntimeException('Invalid retention batch size was accepted.'); } catch (InvalidArgumentException) { - // Expected. + $threw = true; } + $assert($threw, 'Invalid retention batch size was accepted.'); } -echo "Cloud abuse retention service tests passed.\n"; +echo "Compact abuse retention tests passed.\n"; diff --git a/web/tests/CloudCompactAbuseSchemaContractTest.php b/web/tests/CloudCompactAbuseSchemaContractTest.php index 2db040d5..a096047b 100644 --- a/web/tests/CloudCompactAbuseSchemaContractTest.php +++ b/web/tests/CloudCompactAbuseSchemaContractTest.php @@ -64,7 +64,11 @@ foreach (array_keys($definitions) as $table) { $assert(str_contains($definitions['cloud_network_alert_deliveries'], "state IN ('pending','dispatching','sent','failed','unknown','skipped')") && str_contains($definitions['cloud_network_alert_deliveries'], - 'content_hash CHAR(64)'), + 'content_hash CHAR(64)') + && str_contains($definitions['cloud_network_alert_deliveries'], + 'chk_cloud_network_alert_delivery_destination') + && str_contains($definitions['cloud_network_alert_deliveries'], + 'chk_cloud_network_alert_delivery_shape'), 'Telegram delivery lost its conservative unknown-effect fence.'); echo "Compact abuse monitoring schema contract passed.\n";