diff --git a/web/app/Services/Cloud/CloudAbuseDetector.php b/web/app/Services/Cloud/CloudAbuseDetector.php index 710867d0..5fa68ac9 100644 --- a/web/app/Services/Cloud/CloudAbuseDetector.php +++ b/web/app/Services/Cloud/CloudAbuseDetector.php @@ -76,6 +76,78 @@ final class CloudAbuseDetector ]; } + /** + * Reset statistical history while preserving the minimum state required + * for already-open findings to earn a verified clear transition later. + * + * @param list> $openFindings + * @return array + */ + public static function resetBaseline(array $openFindings = []): array + { + $baseline = self::initialBaseline(); + foreach ($openFindings as $finding) { + $rule = (string) ($finding['rule_code'] ?? ''); + $family = (string) ($finding['ip_family'] ?? ''); + $severity = (string) ($finding['severity'] ?? ''); + if (!isset(self::RULES[$rule]) + || !in_array($family, ['ipv4', 'ipv6'], true) + || !in_array($severity, ['suspicious', 'high'], true) + || (string) ($finding['direction'] ?? '') + !== self::RULES[$rule]['direction']) { + continue; + } + $key = $family . ':' . $rule; + $existing = $baseline['rules'][$key] ?? null; + if (is_array($existing) + && ($existing['severity'] ?? '') === 'high') { + continue; + } + $baseline['rules'][$key] = [ + 'active' => true, + 'severity' => $severity, + 'suspicious_streak' => 0, + 'high_streak' => 0, + 'clear_streak' => 0, + 'transition' => 'continued', + ]; + } + return $baseline; + } + + /** + * Reset statistical history after a known collection gap while retaining + * only validated active rule state from the prior durable baseline. + * + * @param array $previous + * @return array + */ + public static function resetBaselineFromPrevious(array $previous): array + { + $findings = []; + foreach (($previous['rules'] ?? []) as $key => $state) { + if (!is_string($key) || !is_array($state) + || ($state['active'] ?? false) !== true) { + continue; + } + $parts = explode(':', $key, 2); + if (count($parts) !== 2) { + continue; + } + [$family, $rule] = $parts; + if (!isset(self::RULES[$rule])) { + continue; + } + $findings[] = [ + 'rule_code' => $rule, + 'direction' => self::RULES[$rule]['direction'], + 'ip_family' => $family, + 'severity' => $state['severity'] ?? '', + ]; + } + return self::resetBaseline($findings); + } + /** * @param array $sample Canonical allowlisted observation * @param null|array $baseline Previous compact baseline diff --git a/web/app/Services/Cloud/CloudAbuseProjectionService.php b/web/app/Services/Cloud/CloudAbuseProjectionService.php new file mode 100644 index 00000000..69ec0e84 --- /dev/null +++ b/web/app/Services/Cloud/CloudAbuseProjectionService.php @@ -0,0 +1,1204 @@ + */ + private array $errors = []; + + /** + * The optional hook is a test seam for the only unavoidable cross-database + * crash window. Production callers must leave it null. + */ + public function __construct( + ?callable $clock = null, + ?callable $afterMainCommit = null + ) { + $this->clock = \Closure::fromCallable( + $clock ?? static fn (): int => time() + ); + $this->afterMainCommit = $afterMainCommit === null + ? null : \Closure::fromCallable($afterMainCommit); + } + + /** Project at most one ordered sample. */ + public function tick(): bool + { + $this->errors = []; + $now = ($this->clock)(); + if (!is_int($now) || $now < 1) { + throw new \LogicException('Abuse projection clock is invalid'); + } + $claim = $this->claim($now); + if ($claim === null) { + return false; + } + if (isset($claim['claim_error_code'])) { + $this->errors[] = (string) $claim['claim_error_code']; + return true; + } + try { + $claim = self::hydrateClaim($claim); + $decision = $this->projectMain($claim, $now); + if ($this->afterMainCommit !== null) { + ($this->afterMainCommit)($claim['sample_key']); + } + if ($decision['state'] === 'discarded') { + $reason = self::boundedCode( + (string) $decision['reason'], + 'projection_discarded' + ); + $this->discard( + $claim, + $reason, + $now + ); + $this->errors[] = $reason; + } else { + $this->finalize($claim, $decision['baseline'], $now); + } + } catch (\Throwable $error) { + $this->errors[] = $this->retryOrDiscard( + $claim, + $error, + $now + ); + } + return true; + } + + /** @return list */ + public function observationErrors(): array + { + return $this->errors; + } + + /** @return null|array */ + private function claim(int $now): ?array + { + $log = User::getConnectionResolver()->connection('log'); + return $log->transaction(function () use ($log, $now): ?array { + $candidate = $log->table( + 'cloud_vm_network_projection_outbox as candidate' + )->where(static function ($due) use ($now): void { + $due->where(static function ($pending) use ($now): void { + $pending->where('candidate.state', 'pending') + ->where('candidate.available_at', '<=', $now); + })->orWhere(static function ($expired) use ($now): void { + $expired->where('candidate.state', 'leased') + ->where('candidate.lease_until', '<=', $now); + }); + })->whereNotExists(static function ($earlier): void { + $earlier->selectRaw('1')->from( + 'cloud_vm_network_projection_outbox as prior' + )->whereColumn( + 'prior.instance_id', + 'candidate.instance_id' + )->whereIn('prior.state', ['pending', 'leased']) + ->where(static function ($order): void { + $order->whereColumn( + 'prior.bucket_start', + '<', + 'candidate.bucket_start' + )->orWhere(static function ($same): void { + $same->whereColumn( + 'prior.bucket_start', + 'candidate.bucket_start' + )->whereColumn( + 'prior.sample_key', + '<', + 'candidate.sample_key' + ); + }); + }); + })->orderBy('candidate.bucket_start') + ->orderBy('candidate.sample_key')->first(); + if ($candidate === null) { + return null; + } + $outbox = $log->table('cloud_vm_network_projection_outbox') + ->where('sample_key', (string) $candidate->sample_key) + ->lockForUpdate()->first(); + if ($outbox === null + || !in_array((string) $outbox->state, + ['pending', 'leased'], true) + || ((string) $outbox->state === 'pending' + && (int) $outbox->available_at > $now) + || ((string) $outbox->state === 'leased' + && (int) $outbox->lease_until > $now)) { + return null; + } + $sample = $log->table( + 'cloud_vm_network_observation_samples' + )->where('sample_key', (string) $outbox->sample_key) + ->lockForUpdate()->first(); + if ($sample === null + || (int) $sample->instance_id !== (int) $outbox->instance_id + || (int) $sample->bucket_start !== (int) $outbox->bucket_start + || !hash_equals( + (string) $sample->placement_key, + (string) $outbox->placement_key + )) { + $code = 'projection_sample_ledger_inconsistent'; + $reported = $this->recordClaimFailure( + $log, + $outbox, + $code, + $now + ); + return ['claim_error_code' => $reported]; + } + $baseline = $log->table('cloud_vm_network_baselines') + ->where('instance_id', (int) $sample->instance_id) + ->lockForUpdate()->first(); + if ($baseline === null) { + $log->table('cloud_vm_network_baselines')->insert([ + 'instance_id' => (int) $sample->instance_id, + 'user_id' => (int) $sample->user_id, + 'rule_version' => CloudAbuseDetector::RULE_VERSION, + 'sample_count' => 0, + 'warmup_state' => 'warming', + 'metrics_json' => self::json( + CloudAbuseDetector::initialBaseline() + ), + 'last_projection_key' => null, + 'last_bucket_start' => null, + 'projection_lease_token' => null, + 'projection_lease_until' => 0, + 'revision' => 0, + 'created_at' => $now, + 'updated_at' => $now, + ]); + $baseline = $log->table('cloud_vm_network_baselines') + ->where('instance_id', (int) $sample->instance_id) + ->lockForUpdate()->first(); + } + if ($baseline === null + || (int) $baseline->user_id !== (int) $sample->user_id + || (int) $baseline->rule_version + !== CloudAbuseDetector::RULE_VERSION) { + $code = 'projection_baseline_identity_inconsistent'; + $reported = $this->recordClaimFailure( + $log, + $outbox, + $code, + $now + ); + return ['claim_error_code' => $reported]; + } + if ($baseline->projection_lease_token !== null + && (int) $baseline->projection_lease_until > $now) { + return null; + } + $token = bin2hex(random_bytes(16)); + $attempts = (int) $outbox->attempts + 1; + $outboxUpdated = $log->table( + 'cloud_vm_network_projection_outbox' + )->where('sample_key', (string) $outbox->sample_key) + ->where('state', (string) $outbox->state) + ->update([ + 'state' => 'leased', + 'attempts' => $attempts, + 'lease_token' => $token, + 'lease_until' => $now + self::LEASE_SECONDS, + 'last_error_code' => '', + 'updated_at' => $now, + ]); + $baselineUpdated = $log->table( + 'cloud_vm_network_baselines' + )->where('instance_id', (int) $sample->instance_id) + ->where('revision', (int) $baseline->revision) + ->update([ + 'projection_lease_token' => $token, + 'projection_lease_until' => $now + + self::LEASE_SECONDS, + 'updated_at' => $now, + ]); + if ($outboxUpdated !== 1 || $baselineUpdated !== 1) { + throw new \RuntimeException( + 'Abuse projection lease compare-and-swap failed' + ); + } + $complete = (int) $sample->complete === 1; + return [ + 'sample_key' => (string) $sample->sample_key, + 'instance_id' => (int) $sample->instance_id, + 'user_id' => (int) $sample->user_id, + 'host_id' => (int) $sample->host_id, + 'placement_key' => (string) $sample->placement_key, + 'bucket_start' => (int) $sample->bucket_start, + 'complete' => $complete, + 'metrics_json' => (string) $sample->metrics_json, + 'quality_flags_json' => + (string) $sample->quality_flags_json, + 'baseline_json' => (string) $baseline->metrics_json, + 'baseline_revision' => (int) $baseline->revision, + 'last_projection_key' => $baseline->last_projection_key, + 'last_bucket_start' => $baseline->last_bucket_start === null + ? null : (int) $baseline->last_bucket_start, + 'attempts' => $attempts, + 'lease_token' => $token, + ]; + }, 3); + } + + private function recordClaimFailure( + object $log, + object $outbox, + string $code, + int $now + ): string { + $code = self::boundedCode($code, 'projection_claim_failed'); + $attempts = (int) $outbox->attempts + 1; + $terminal = $attempts >= self::MAX_ATTEMPTS; + if ($terminal) { + $receiptState = $this->receiptState([ + 'sample_key' => (string) $outbox->sample_key, + 'instance_id' => (int) $outbox->instance_id, + 'bucket_start' => (int) $outbox->bucket_start, + ]); + if ($receiptState === 'matching') { + $baseline = $log->table('cloud_vm_network_baselines') + ->where('instance_id', (int) $outbox->instance_id) + ->lockForUpdate()->first(); + if ($baseline !== null + && $baseline->projection_lease_token !== null + && (int) $baseline->projection_lease_until > $now) { + $terminal = false; + $code = 'projection_receipt_finalize_pending'; + } else { + try { + $seed = $this->currentOwnerBaselineSeed( + (int) $outbox->instance_id + ); + } catch (\Throwable) { + $seed = false; + } + if ($seed === false) { + $terminal = false; + $code = 'projection_baseline_seed_pending'; + } else { + // The main read model already committed this sample. + // Use its current open findings to reset statistics + // without losing the evidence required for resolution. + if ($seed === null) { + $deleted = $log->table( + 'cloud_vm_network_baselines' + )->where('instance_id', + (int) $outbox->instance_id)->delete(); + if ($baseline !== null && $deleted !== 1) { + throw new \RuntimeException( + 'Abuse baseline delete CAS failed' + ); + } + } else { + $seedBaseline = $seed['baseline']; + $seedBaseline['last_bucket_start'] = + (int) $outbox->bucket_start; + $values = [ + 'user_id' => $seed['user_id'], + 'rule_version' => + CloudAbuseDetector::RULE_VERSION, + 'sample_count' => 0, + 'warmup_state' => 'warming', + 'metrics_json' => self::json($seedBaseline), + 'last_projection_key' => + (string) $outbox->sample_key, + 'last_bucket_start' => + (int) $outbox->bucket_start, + 'projection_lease_token' => null, + 'projection_lease_until' => 0, + 'updated_at' => $now, + ]; + if ($baseline === null) { + $log->table( + 'cloud_vm_network_baselines' + )->insert([ + 'instance_id' => + (int) $outbox->instance_id, + 'revision' => 0, + 'created_at' => $now, + ] + $values); + } else { + $values['revision'] = + (int) $baseline->revision + 1; + $baselineUpdated = $log->table( + 'cloud_vm_network_baselines' + )->where('instance_id', + (int) $outbox->instance_id) + ->where('revision', + (int) $baseline->revision) + ->update($values); + if ($baselineUpdated !== 1) { + throw new \RuntimeException( + 'Abuse baseline reset CAS failed' + ); + } + } + } + $updated = $log->table( + 'cloud_vm_network_projection_outbox' + )->where('sample_key', (string) $outbox->sample_key) + ->where('state', (string) $outbox->state) + ->where('attempts', (int) $outbox->attempts) + ->update([ + 'state' => 'done', + 'attempts' => min( + $attempts, + self::MAX_ATTEMPTS + ), + 'lease_token' => null, + 'lease_until' => 0, + 'last_error_code' => '', + 'projected_at' => $now, + 'terminal_at' => $now, + 'discard_reason_code' => '', + 'updated_at' => $now, + ]); + if ($updated !== 1) { + throw new \RuntimeException( + 'Abuse receipt claim recovery CAS failed' + ); + } + return 'projection_receipt_baseline_reset'; + } + } + } + if ($receiptState === 'conflict') { + $terminal = false; + $code = 'projection_receipt_conflict'; + } elseif ($receiptState === 'unknown') { + $terminal = false; + $code = 'projection_receipt_check_failed'; + } + } + $values = $terminal ? [ + 'state' => 'discarded', + 'attempts' => $attempts, + 'lease_token' => null, + 'lease_until' => 0, + 'last_error_code' => $code, + 'projected_at' => null, + 'terminal_at' => $now, + 'discard_reason_code' => 'projection_retry_exhausted', + 'updated_at' => $now, + ] : [ + 'state' => 'pending', + 'attempts' => min($attempts, self::MAX_ATTEMPTS), + 'available_at' => $now + self::retryDelay($attempts), + 'lease_token' => null, + 'lease_until' => 0, + 'last_error_code' => $code, + 'projected_at' => null, + 'terminal_at' => null, + 'discard_reason_code' => '', + 'updated_at' => $now, + ]; + $updated = $log->table('cloud_vm_network_projection_outbox') + ->where('sample_key', (string) $outbox->sample_key) + ->where('state', (string) $outbox->state) + ->where('attempts', (int) $outbox->attempts) + ->update($values); + if ($updated !== 1) { + throw new \RuntimeException( + 'Abuse projection claim failure compare-and-swap failed' + ); + } + return $terminal ? 'projection_retry_exhausted' : $code; + } + + /** @param array $claim @return array */ + private static function hydrateClaim(array $claim): array + { + $claim['metrics'] = self::canonicalMetrics(self::decodeObject( + (string) $claim['metrics_json'], + 'sample metrics' + )); + $claim['quality_flags'] = self::canonicalQuality(self::decodeList( + (string) $claim['quality_flags_json'], + 'sample quality flags' + )); + $claim['baseline'] = self::decodeObject( + (string) $claim['baseline_json'], + 'network baseline' + ); + unset( + $claim['metrics_json'], + $claim['quality_flags_json'], + $claim['baseline_json'] + ); + if ($claim['complete'] === in_array( + 'incomplete', + $claim['quality_flags'], + true + )) { + throw new \RuntimeException( + 'Sample completeness and quality evidence disagree' + ); + } + self::assertStoredSampleSemantics( + $claim['metrics'], + $claim['quality_flags'], + $claim['complete'] + ); + return $claim; + } + + /** + * Recheck the semantic invariants that made the stored sample canonical. + * The log database is a durable boundary: corrupt rows must never be + * promoted to healthy evidence merely because their JSON shape is valid. + * + * @param array $metrics + * @param array $quality + */ + private static function assertStoredSampleSemantics( + array $metrics, + array $quality, + bool $complete + ): void { + if ($metrics['ipv4'] === null && $metrics['ipv6'] === null) { + throw new \RuntimeException( + 'Sample has no observable address family' + ); + } + $interfaceUnavailable = in_array( + 'interface_unavailable', + $quality, + true + ); + $hasMissingCounter = in_array( + null, + $metrics['interface'], + true + ); + if ($hasMissingCounter !== $interfaceUnavailable) { + throw new \RuntimeException( + 'Sample interface evidence is inconsistent' + ); + } + if (in_array('panel_ring_gap', $quality, true) + && $complete) { + throw new \RuntimeException( + 'Panel ring gap sample cannot be complete' + ); + } + if ($complete && array_intersect($quality, [ + 'clock_regression', 'cold_start', 'conntrack_unavailable', + 'counter_reset', 'flow_limit_reached', + 'interface_unavailable', + 'local_port_cardinality_saturated', 'mapping_changed', + 'overflow', 'relation_cardinality_saturated', + 'remote_ip_cardinality_saturated', + 'remote_port_cardinality_saturated', 'sampling_gap', + ]) !== []) { + throw new \RuntimeException( + 'Complete sample contains risk-unsafe evidence' + ); + } + } + + /** + * @param array $claim + * @return array{state:string,reason?:string,baseline?:array} + */ + private function projectMain(array $claim, int $now): array + { + if ($claim['last_bucket_start'] !== null + && $claim['bucket_start'] <= $claim['last_bucket_start']) { + if (is_string($claim['last_projection_key']) + && hash_equals( + $claim['last_projection_key'], + $claim['sample_key'] + )) { + return [ + 'state' => 'done', + 'baseline' => $claim['baseline'], + ]; + } + return ['state' => 'discarded', + 'reason' => 'out_of_order_sample']; + } + $reset = in_array( + 'panel_ring_gap', + $claim['quality_flags'], + true + ); + $base = $reset + ? CloudAbuseDetector::resetBaselineFromPrevious( + $claim['baseline'] + ) : $claim['baseline']; + $detected = CloudAbuseDetector::evaluate([ + 'bucket_start' => $claim['bucket_start'], + 'complete' => $claim['complete'], + 'quality_flags' => $claim['quality_flags'], + 'metrics' => $claim['metrics'], + ], $base); + $baseline = $detected['baseline']; + $evaluation = $detected['evaluation']; + if (!$claim['complete']) { + // Do not derive a per-second rate across an unobserved minute. + // Statistical history and finding streaks stay frozen, while the + // next complete sample establishes a fresh interface baseline. + $baseline['last_interface'] = null; + } + + $main = User::getConnectionResolver()->connection('default'); + return $main->transaction(function () use ( + $main, + $claim, + $baseline, + $evaluation, + $now + ): array { + $instance = $main->table('cloud_instances') + ->where('id', $claim['instance_id']) + ->lockForUpdate()->first(); + if ($instance === null) { + return ['state' => 'discarded', + 'reason' => 'instance_deleted']; + } + if ((int) $instance->user_id !== $claim['user_id']) { + $this->retireOwnerReadModel( + $main, + $claim['instance_id'], + $claim['user_id'], + $now + ); + return ['state' => 'discarded', + 'reason' => 'instance_owner_changed']; + } + $receipt = $main->table('cloud_abuse_projection_receipts') + ->where('sample_key', $claim['sample_key']) + ->lockForUpdate()->first(); + if ($receipt !== null) { + if ((int) $receipt->instance_id !== $claim['instance_id'] + || (int) $receipt->bucket_start + !== $claim['bucket_start'] + || (int) $receipt->rule_version + !== CloudAbuseDetector::RULE_VERSION) { + throw new \RuntimeException( + 'Abuse projection receipt identity conflict' + ); + } + return ['state' => 'done', 'baseline' => $baseline]; + } + $state = $main->table('cloud_abuse_instance_states') + ->where('instance_id', $claim['instance_id']) + ->lockForUpdate()->first(); + if ($state !== null && $state->last_bucket_start !== null + && (int) $state->last_bucket_start + >= $claim['bucket_start']) { + return ['state' => 'discarded', + 'reason' => 'projection_order_conflict']; + } + + $healthy = ($evaluation['collection_state'] ?? null) + === 'healthy'; + $summary = [ + 'rates' => $evaluation['rates'] ?? [], + 'sample_count' => $evaluation['sample_count'] ?? null, + 'warmup_remaining' => + $evaluation['warmup_remaining'] ?? null, + 'quality_flags' => $claim['quality_flags'], + 'metrics' => $claim['metrics'], + ]; + $stateValues = [ + 'instance_id' => $claim['instance_id'], + 'user_id' => $claim['user_id'], + 'host_id' => (int) $instance->host_id, + 'risk_state' => $healthy + ? (string) $evaluation['risk_state'] + : (string) ($state->risk_state ?? 'observing'), + 'collection_state' => $healthy ? 'healthy' : 'degraded', + 'latest_direction' => $healthy + ? (string) $evaluation['direction'] + : (string) ($state->latest_direction ?? 'none'), + 'latest_rule_code' => $healthy + ? (string) $evaluation['latest_rule_code'] + : (string) ($state->latest_rule_code ?? ''), + 'latest_severity' => $healthy + ? (string) $evaluation['latest_severity'] + : (string) ($state->latest_severity ?? 'none'), + 'summary_json' => self::json($summary), + 'last_projection_key' => $claim['sample_key'], + 'last_bucket_start' => $claim['bucket_start'], + 'last_sample_at' => $claim['bucket_start'] + 60, + 'last_evaluated_at' => $healthy + ? $now : ($state->last_evaluated_at ?? null), + 'updated_at' => $now, + ]; + if ($state === null) { + $stateValues['created_at'] = $now; + $main->table('cloud_abuse_instance_states') + ->insert($stateValues); + } else { + $main->table('cloud_abuse_instance_states') + ->where('instance_id', $claim['instance_id']) + ->update($stateValues); + } + if ($healthy) { + $this->applyFindings( + $main, + $claim, + (int) $instance->host_id, + (array) ($evaluation['findings'] ?? []), + $now + ); + } + $main->table('cloud_abuse_projection_receipts')->insert([ + 'sample_key' => $claim['sample_key'], + 'instance_id' => $claim['instance_id'], + 'rule_version' => CloudAbuseDetector::RULE_VERSION, + 'bucket_start' => $claim['bucket_start'], + 'projected_at' => $now, + ]); + return ['state' => 'done', 'baseline' => $baseline]; + }, 3); + } + + private function retireOwnerReadModel( + object $main, + int $instanceId, + int $oldUserId, + int $now + ): void { + $main->table('cloud_abuse_instance_states') + ->where('instance_id', $instanceId) + ->where('user_id', $oldUserId) + ->delete(); + $findings = $main->table('cloud_abuse_findings') + ->where('instance_id', $instanceId) + ->where('user_id', $oldUserId) + ->where('state', 'open') + ->orderBy('id')->lockForUpdate()->get(); + foreach ($findings as $finding) { + $resolvedAt = max($now, (int) $finding->last_seen_at); + $updated = $main->table('cloud_abuse_findings') + ->where('id', (int) $finding->id) + ->where('instance_id', $instanceId) + ->where('user_id', $oldUserId) + ->where('state', 'open')->update([ + 'state' => 'resolved', + 'resolved_at' => $resolvedAt, + 'updated_at' => $now, + ]); + if ($updated !== 1) { + throw new \RuntimeException( + 'Abuse projection owner cleanup compare-and-swap failed' + ); + } + } + } + + /** + * @return null|array{user_id:int,baseline:array} + */ + private function currentOwnerBaselineSeed(int $instanceId): ?array + { + $main = User::getConnectionResolver()->connection('default'); + $instance = $main->table('cloud_instances') + ->where('id', $instanceId)->first(['user_id']); + if ($instance === null) { + return null; + } + $userId = (int) $instance->user_id; + $rows = $main->table('cloud_abuse_findings') + ->where('instance_id', $instanceId) + ->where('user_id', $userId) + ->where('rule_version', CloudAbuseDetector::RULE_VERSION) + ->where('state', 'open') + ->orderBy('id')->get([ + 'rule_code', 'direction', 'ip_family', 'severity', + ]); + $findings = []; + foreach ($rows as $row) { + $findings[] = [ + 'rule_code' => (string) $row->rule_code, + 'direction' => (string) $row->direction, + 'ip_family' => (string) $row->ip_family, + 'severity' => (string) $row->severity, + ]; + } + return [ + 'user_id' => $userId, + 'baseline' => CloudAbuseDetector::resetBaseline($findings), + ]; + } + + /** + * @param array $claim + * @param array> $findings + */ + private function applyFindings( + object $main, + array $claim, + int $currentHostId, + array $findings, + int $now + ): void { + foreach ($findings as $finding) { + $key = hash('sha256', implode("\n", [ + $claim['instance_id'], + CloudAbuseDetector::RULE_VERSION, + (string) $finding['rule_code'], + (string) $finding['ip_family'], + (string) $finding['direction'], + ])); + $row = $main->table('cloud_abuse_findings') + ->where('finding_key', $key)->lockForUpdate()->first(); + if (($finding['active'] ?? false) === true) { + $values = [ + 'instance_id' => $claim['instance_id'], + 'user_id' => $claim['user_id'], + 'host_id' => $currentHostId, + 'rule_code' => (string) $finding['rule_code'], + 'rule_version' => CloudAbuseDetector::RULE_VERSION, + 'direction' => (string) $finding['direction'], + 'ip_family' => (string) $finding['ip_family'], + 'severity' => (string) $finding['severity'], + 'state' => 'open', + 'first_seen_at' => $row === null + || (string) $row->state === 'resolved' + ? $claim['bucket_start'] + : (int) $row->first_seen_at, + 'last_seen_at' => $claim['bucket_start'], + 'consecutive_windows' => max( + 1, + (int) $finding['consecutive_windows'] + ), + 'evidence_json' => self::json( + (array) $finding['evidence'] + ), + 'resolved_at' => null, + 'updated_at' => $now, + ]; + if ($row === null) { + $values['finding_key'] = $key; + $values['created_at'] = $now; + $main->table('cloud_abuse_findings')->insert($values); + } else { + $main->table('cloud_abuse_findings') + ->where('finding_key', $key)->update($values); + } + } elseif (($finding['transition'] ?? '') === 'resolved' + && $row !== null && (string) $row->state === 'open') { + $main->table('cloud_abuse_findings') + ->where('finding_key', $key) + ->where('state', 'open')->update([ + 'state' => 'resolved', + 'resolved_at' => $claim['bucket_start'], + 'updated_at' => $now, + ]); + } + } + } + + /** @param array $claim @param array $baseline */ + private function finalize(array $claim, array $baseline, int $now): void + { + $log = User::getConnectionResolver()->connection('log'); + $log->transaction(function () use ( + $log, + $claim, + $baseline, + $now + ): void { + $base = $log->table('cloud_vm_network_baselines') + ->where('instance_id', $claim['instance_id']) + ->lockForUpdate()->first(); + $outbox = $log->table('cloud_vm_network_projection_outbox') + ->where('sample_key', $claim['sample_key']) + ->lockForUpdate()->first(); + if ($base === null || $outbox === null + || (string) $base->projection_lease_token + !== $claim['lease_token'] + || (string) $outbox->lease_token !== $claim['lease_token'] + || (string) $outbox->state !== 'leased') { + throw new \RuntimeException( + 'Abuse projection lease was lost before finalization' + ); + } + $sampleCount = max(0, (int) ($baseline['sample_count'] ?? 0)); + $baselineUpdated = $log->table('cloud_vm_network_baselines') + ->where('instance_id', $claim['instance_id']) + ->where('revision', $claim['baseline_revision']) + ->where('projection_lease_token', $claim['lease_token']) + ->update([ + 'sample_count' => $sampleCount, + 'warmup_state' => $sampleCount + >= CloudAbuseDetector::WARMUP_WINDOWS + ? 'ready' : 'warming', + 'metrics_json' => self::json($baseline), + 'last_projection_key' => $claim['sample_key'], + 'last_bucket_start' => $claim['bucket_start'], + 'projection_lease_token' => null, + 'projection_lease_until' => 0, + 'revision' => $claim['baseline_revision'] + 1, + 'updated_at' => $now, + ]); + $updated = $log->table('cloud_vm_network_projection_outbox') + ->where('sample_key', $claim['sample_key']) + ->where('state', 'leased') + ->where('lease_token', $claim['lease_token']) + ->update([ + 'state' => 'done', + 'lease_token' => null, + 'lease_until' => 0, + 'last_error_code' => '', + 'projected_at' => $now, + 'terminal_at' => $now, + 'discard_reason_code' => '', + 'updated_at' => $now, + ]); + if ($baselineUpdated !== 1 || $updated !== 1) { + throw new \RuntimeException( + 'Abuse projection finalization compare-and-swap failed' + ); + } + }, 3); + } + + /** @param array $claim */ + private function discard(array $claim, string $reason, int $now): void + { + if (preg_match('/^[a-z0-9_]{3,64}$/D', $reason) !== 1) { + $reason = 'projection_discarded'; + } + $log = User::getConnectionResolver()->connection('log'); + $log->transaction(function () use ($log, $claim, $reason, $now): void { + if ($reason === 'instance_owner_changed') { + $released = $log->table('cloud_vm_network_baselines') + ->where('instance_id', $claim['instance_id']) + ->where('user_id', $claim['user_id']) + ->where('revision', $claim['baseline_revision']) + ->where('projection_lease_token', $claim['lease_token']) + ->delete(); + } else { + $released = $this->releaseBaselineLease( + $log, + $claim, + $now + ); + } + $updated = $log->table('cloud_vm_network_projection_outbox') + ->where('sample_key', $claim['sample_key']) + ->where('state', 'leased') + ->where('lease_token', $claim['lease_token']) + ->update([ + 'state' => 'discarded', + 'lease_token' => null, + 'lease_until' => 0, + 'last_error_code' => '', + 'projected_at' => null, + 'terminal_at' => $now, + 'discard_reason_code' => $reason, + 'updated_at' => $now, + ]); + if ($released !== 1 || $updated !== 1) { + throw new \RuntimeException( + 'Abuse projection discard finalization failed' + ); + } + }, 3); + } + + /** @param array $claim */ + private function retryOrDiscard( + array $claim, + \Throwable $error, + int $now + ): string { + $code = self::boundedCode( + $error instanceof CloudAgentError + ? $error->agentCode : 'projection_failed', + 'projection_failed' + ); + $terminal = $claim['attempts'] >= self::MAX_ATTEMPTS; + if ($terminal) { + $receiptState = $this->receiptState($claim); + if ($receiptState === 'matching') { + $terminal = false; + $code = 'projection_receipt_finalize_pending'; + } elseif ($receiptState === 'conflict') { + $terminal = false; + $code = 'projection_receipt_conflict'; + } elseif ($receiptState === 'unknown') { + $terminal = false; + $code = 'projection_receipt_check_failed'; + } + } + $log = User::getConnectionResolver()->connection('log'); + $log->transaction(function () use ( + $log, + $claim, + $now, + $code, + $terminal + ): void { + $this->releaseBaselineLease($log, $claim, $now); + $values = $terminal ? [ + 'state' => 'discarded', + 'lease_token' => null, + 'lease_until' => 0, + 'last_error_code' => $code, + 'projected_at' => null, + 'terminal_at' => $now, + 'discard_reason_code' => 'projection_retry_exhausted', + 'updated_at' => $now, + ] : [ + 'state' => 'pending', + 'available_at' => $now + + self::retryDelay($claim['attempts']), + 'lease_token' => null, + 'lease_until' => 0, + 'last_error_code' => $code, + 'projected_at' => null, + 'terminal_at' => null, + 'discard_reason_code' => '', + 'updated_at' => $now, + ]; + $log->table('cloud_vm_network_projection_outbox') + ->where('sample_key', $claim['sample_key']) + ->where('state', 'leased') + ->where('lease_token', $claim['lease_token']) + ->update($values); + }, 3); + return $code; + } + + /** @param array $claim */ + private function receiptState(array $claim): string + { + try { + $receipt = User::getConnectionResolver() + ->connection('default') + ->table('cloud_abuse_projection_receipts') + ->where('sample_key', $claim['sample_key'])->first(); + } catch (\Throwable) { + return 'unknown'; + } + if ($receipt === null) { + return 'absent'; + } + if ((int) $receipt->instance_id !== $claim['instance_id'] + || (int) $receipt->bucket_start !== $claim['bucket_start'] + || (int) $receipt->rule_version + !== CloudAbuseDetector::RULE_VERSION + || (int) $receipt->projected_at < 1) { + return 'conflict'; + } + return 'matching'; + } + + /** @param array $claim */ + private function releaseBaselineLease( + object $log, + array $claim, + int $now + ): int { + return $log->table('cloud_vm_network_baselines') + ->where('instance_id', $claim['instance_id']) + ->where('revision', $claim['baseline_revision']) + ->where('projection_lease_token', $claim['lease_token']) + ->update([ + 'projection_lease_token' => null, + 'projection_lease_until' => 0, + 'updated_at' => $now, + ]); + } + + /** @return array */ + private static function decodeObject(string $json, string $name): array + { + try { + $value = json_decode($json, true, 16, JSON_THROW_ON_ERROR); + } catch (\JsonException $error) { + throw new \RuntimeException("Invalid {$name}", 0, $error); + } + if (!is_array($value) || $value === [] || array_is_list($value)) { + throw new \RuntimeException("Invalid {$name}"); + } + return $value; + } + + /** @return array */ + private static function decodeList(string $json, string $name): array + { + try { + $value = json_decode($json, true, 8, JSON_THROW_ON_ERROR); + } catch (\JsonException $error) { + throw new \RuntimeException("Invalid {$name}", 0, $error); + } + if (!is_array($value) || !array_is_list($value)) { + throw new \RuntimeException("Invalid {$name}"); + } + return $value; + } + + /** @param array $metrics @return array */ + private static function canonicalMetrics(array $metrics): array + { + self::exactKeys($metrics, ['interface', 'ipv4', 'ipv6'], + 'sample metrics'); + $interfaceFields = [ + 'rx_bytes', 'tx_bytes', 'rx_packets', 'tx_packets', + 'rx_errors', 'tx_errors', 'rx_drops', 'tx_drops', + ]; + if (!is_array($metrics['interface']) + || array_is_list($metrics['interface'])) { + throw new \RuntimeException('Invalid sample interface metrics'); + } + self::exactKeys($metrics['interface'], $interfaceFields, + 'sample interface metrics'); + $interface = []; + foreach ($interfaceFields as $field) { + $value = $metrics['interface'][$field]; + if ($value !== null && (!is_int($value) || $value < 0)) { + throw new \RuntimeException( + 'Invalid sample interface counter' + ); + } + $interface[$field] = $value; + } + return [ + 'interface' => $interface, + 'ipv4' => self::canonicalFamily($metrics['ipv4'], 'ipv4'), + 'ipv6' => self::canonicalFamily($metrics['ipv6'], 'ipv6'), + ]; + } + + private static function canonicalFamily(mixed $family, string $name): ?array + { + if ($family === null) { + return null; + } + if (!is_array($family) || array_is_list($family)) { + throw new \RuntimeException("Invalid {$name} sample metrics"); + } + self::exactKeys($family, ['inbound', 'outbound'], + "{$name} sample metrics"); + return [ + 'inbound' => self::canonicalFlow( + $family['inbound'], + "{$name} inbound" + ), + 'outbound' => self::canonicalFlow( + $family['outbound'], + "{$name} outbound" + ), + ]; + } + + /** @return array */ + private static function canonicalFlow(mixed $flow, string $name): array + { + $fields = [ + 'tcp_active_flows', 'tcp_new_flows', 'tcp_syn_flows', + 'tcp_unreplied_flows', 'udp_active_flows', 'udp_new_flows', + 'other_active_flows', 'unique_remote_ips', + 'unique_remote_ports', 'unique_local_ports', + 'max_remote_ports_per_remote_ip', + 'max_remote_ips_per_remote_port', + 'max_local_ports_per_remote_ip', + 'max_remote_ips_per_local_port', + ]; + if (!is_array($flow) || array_is_list($flow)) { + throw new \RuntimeException("Invalid {$name} flow metrics"); + } + self::exactKeys($flow, $fields, "{$name} flow metrics"); + $result = []; + foreach ($fields as $field) { + if (!is_int($flow[$field]) || $flow[$field] < 0) { + throw new \RuntimeException("Invalid {$name} flow counter"); + } + $result[$field] = $flow[$field]; + } + return $result; + } + + /** @param array $quality @return array */ + private static function canonicalQuality(array $quality): array + { + $allowed = [ + 'clock_regression', 'cold_start', 'conntrack_unavailable', + 'counter_reset', 'flow_limit_reached', 'incomplete', + 'interface_unavailable', 'local_port_cardinality_saturated', + 'mapping_changed', 'overflow', 'packet_accounting_unavailable', + 'panel_ring_gap', 'relation_cardinality_saturated', + 'remote_ip_cardinality_saturated', + 'remote_port_cardinality_saturated', 'sampling_gap', + ]; + if (count($quality) > 17) { + throw new \RuntimeException('Too many sample quality flags'); + } + foreach ($quality as $flag) { + if (!is_string($flag) || !in_array($flag, $allowed, true)) { + throw new \RuntimeException('Invalid sample quality flag'); + } + } + $canonical = array_values(array_unique($quality)); + sort($canonical, SORT_STRING); + if ($canonical !== $quality) { + throw new \RuntimeException( + 'Sample quality flags are not canonical' + ); + } + return $quality; + } + + /** @param array $value @param array $keys */ + private static function exactKeys( + array $value, + array $keys, + string $name + ): void { + $actual = array_keys($value); + sort($actual, SORT_STRING); + $expected = $keys; + sort($expected, SORT_STRING); + if ($actual !== $expected) { + throw new \RuntimeException("Invalid {$name} shape"); + } + } + + private static function boundedCode(string $code, string $fallback): string + { + return preg_match('/^[a-z0-9_]{3,64}$/D', $code) === 1 + ? $code : $fallback; + } + + private static function retryDelay(int $attempts): int + { + return min( + self::MAX_BACKOFF_SECONDS, + 15 * (2 ** min(6, max(0, $attempts - 1))) + ); + } + + private static function json(array $value): string + { + return json_encode( + $value, + JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES + ); + } +} diff --git a/web/tests/CloudAbuseProjectionServiceTest.php b/web/tests/CloudAbuseProjectionServiceTest.php new file mode 100644 index 00000000..de335404 --- /dev/null +++ b/web/tests/CloudAbuseProjectionServiceTest.php @@ -0,0 +1,800 @@ +addConnection([ + 'driver' => 'sqlite', 'database' => ':memory:', +], 'default'); +$capsule->addConnection([ + 'driver' => 'sqlite', 'database' => ':memory:', +], 'log'); +$capsule->setAsGlobal(); +$capsule->bootEloquent(); + +$mainSchema = $capsule->schema('default'); +$mainSchema->create('cloud_instances', static function (Blueprint $table): void { + $table->increments('id'); + $table->integer('host_id'); + $table->integer('user_id'); +}); +$mainSchema->create('cloud_abuse_instance_states', + static function (Blueprint $table): void { + $table->integer('instance_id')->primary(); + $table->integer('user_id'); + $table->integer('host_id'); + $table->string('risk_state'); + $table->string('collection_state'); + $table->string('latest_direction'); + $table->string('latest_rule_code'); + $table->string('latest_severity'); + $table->text('summary_json'); + $table->string('last_projection_key')->nullable(); + $table->integer('last_bucket_start')->nullable(); + $table->integer('last_sample_at')->nullable(); + $table->integer('last_evaluated_at')->nullable(); + $table->integer('created_at'); + $table->integer('updated_at'); + }); +$mainSchema->create('cloud_abuse_findings', + static function (Blueprint $table): void { + $table->increments('id'); + $table->string('finding_key')->unique(); + $table->integer('instance_id'); + $table->integer('user_id'); + $table->integer('host_id'); + $table->string('rule_code'); + $table->integer('rule_version'); + $table->string('direction'); + $table->string('ip_family'); + $table->string('severity'); + $table->string('state'); + $table->integer('first_seen_at'); + $table->integer('last_seen_at'); + $table->integer('consecutive_windows'); + $table->text('evidence_json'); + $table->integer('resolved_at')->nullable(); + $table->integer('created_at'); + $table->integer('updated_at'); + }); +$mainSchema->create('cloud_abuse_projection_receipts', + static function (Blueprint $table): void { + $table->string('sample_key')->primary(); + $table->integer('instance_id'); + $table->integer('rule_version'); + $table->integer('bucket_start'); + $table->integer('projected_at'); + }); + +$logSchema = $capsule->schema('log'); +$logSchema->create('cloud_vm_network_observation_samples', + static function (Blueprint $table): void { + $table->string('sample_key')->primary(); + $table->integer('instance_id'); + $table->integer('user_id'); + $table->integer('host_id'); + $table->string('vm_uuid'); + $table->string('placement_key'); + $table->string('collector_generation'); + $table->string('window_id'); + $table->integer('contract_version'); + $table->integer('bucket_start'); + $table->integer('bucket_seconds'); + $table->integer('complete'); + $table->text('metrics_json'); + $table->text('quality_flags_json'); + $table->integer('received_at'); + }); +$logSchema->create('cloud_vm_network_projection_outbox', + static function (Blueprint $table): void { + $table->string('sample_key')->primary(); + $table->integer('instance_id'); + $table->integer('bucket_start'); + $table->string('placement_key'); + $table->string('state'); + $table->integer('attempts')->default(0); + $table->integer('available_at'); + $table->string('lease_token')->nullable(); + $table->integer('lease_until')->default(0); + $table->string('last_error_code')->default(''); + $table->integer('projected_at')->nullable(); + $table->integer('terminal_at')->nullable(); + $table->string('discard_reason_code')->default(''); + $table->integer('created_at'); + $table->integer('updated_at'); + }); +$logSchema->create('cloud_vm_network_baselines', + static function (Blueprint $table): void { + $table->integer('instance_id')->primary(); + $table->integer('user_id'); + $table->integer('rule_version'); + $table->integer('sample_count'); + $table->string('warmup_state'); + $table->text('metrics_json'); + $table->string('last_projection_key')->nullable(); + $table->integer('last_bucket_start')->nullable(); + $table->string('projection_lease_token')->nullable(); + $table->integer('projection_lease_until')->default(0); + $table->integer('revision')->default(0); + $table->integer('created_at'); + $table->integer('updated_at'); + }); + +$main = $capsule->connection('default'); +$log = $capsule->connection('log'); +$now = 1000; +$uuid = static fn (int $id): string => sprintf( + '123e4567-e89b-42d3-a456-%012d', + $id +); +$addInstance = static function (int $id, int $user, int $host = 7) use ( + $main +): void { + $main->table('cloud_instances')->insert([ + 'id' => $id, 'host_id' => $host, 'user_id' => $user, + ]); +}; +$flow = static fn (array $values = []): array => array_merge([ + 'tcp_active_flows' => 0, 'tcp_new_flows' => 0, + 'tcp_syn_flows' => 0, 'tcp_unreplied_flows' => 0, + 'udp_active_flows' => 0, 'udp_new_flows' => 0, + 'other_active_flows' => 0, 'unique_remote_ips' => 0, + 'unique_remote_ports' => 0, 'unique_local_ports' => 0, + 'max_remote_ports_per_remote_ip' => 0, + 'max_remote_ips_per_remote_port' => 0, + 'max_local_ports_per_remote_ip' => 0, + 'max_remote_ips_per_local_port' => 0, +], $values); +$metrics = static function (array $outbound = [], array $inbound = []) use ( + $flow +): array { + return [ + 'interface' => [ + 'rx_bytes' => 1000, 'tx_bytes' => 2000, + 'rx_packets' => 10, 'tx_packets' => 20, + 'rx_errors' => 0, 'tx_errors' => 0, + 'rx_drops' => 0, 'tx_drops' => 0, + ], + 'ipv4' => ['inbound' => $flow($inbound), + 'outbound' => $flow($outbound)], + 'ipv6' => ['inbound' => $flow(), 'outbound' => $flow()], + ]; +}; +$insertBaseline = static function ( + int $instanceId, + int $userId, + array $baseline, + ?int $lastBucket = null, + ?string $lastKey = null, + ?string $lease = null, + int $leaseUntil = 0, + int $revision = 0 +) use ($log, &$now): void { + $log->table('cloud_vm_network_baselines')->insert([ + 'instance_id' => $instanceId, + 'user_id' => $userId, + 'rule_version' => CloudAbuseDetector::RULE_VERSION, + 'sample_count' => (int) ($baseline['sample_count'] ?? 0), + 'warmup_state' => (int) ($baseline['sample_count'] ?? 0) + >= CloudAbuseDetector::WARMUP_WINDOWS ? 'ready' : 'warming', + 'metrics_json' => json_encode($baseline, JSON_THROW_ON_ERROR), + 'last_projection_key' => $lastKey, + 'last_bucket_start' => $lastBucket, + 'projection_lease_token' => $lease, + 'projection_lease_until' => $leaseUntil, + 'revision' => $revision, + 'created_at' => $now, + 'updated_at' => $now, + ]); +}; +$insertSample = static function ( + int $instanceId, + int $userId, + int $bucket, + array $sampleMetrics, + bool $complete = true, + array $quality = [], + string $state = 'pending', + ?string $lease = null, + int $leaseUntil = 0, + int $attempts = 0 +) use ($log, $uuid, &$now): string { + $key = hash('sha256', $instanceId . "\n" . $bucket); + $placement = hash('sha256', 'placement-' . $instanceId); + $log->table('cloud_vm_network_observation_samples')->insert([ + 'sample_key' => $key, + 'instance_id' => $instanceId, + 'user_id' => $userId, + 'host_id' => 7, + 'vm_uuid' => $uuid($instanceId), + 'placement_key' => $placement, + 'collector_generation' => str_repeat('a', 32), + 'window_id' => substr($key, 0, 32), + 'contract_version' => 1, + 'bucket_start' => $bucket, + 'bucket_seconds' => 60, + 'complete' => $complete ? 1 : 0, + 'metrics_json' => json_encode($sampleMetrics, + JSON_THROW_ON_ERROR), + 'quality_flags_json' => json_encode($quality, + JSON_THROW_ON_ERROR), + 'received_at' => $now, + ]); + $log->table('cloud_vm_network_projection_outbox')->insert([ + 'sample_key' => $key, + 'instance_id' => $instanceId, + 'bucket_start' => $bucket, + 'placement_key' => $placement, + 'state' => $state, + 'attempts' => $attempts, + 'available_at' => $now, + 'lease_token' => $lease, + 'lease_until' => $leaseUntil, + 'last_error_code' => '', + 'projected_at' => null, + 'terminal_at' => null, + 'discard_reason_code' => '', + 'created_at' => $now, + 'updated_at' => $now, + ]); + return $key; +}; +$worker = static function (?callable $hook = null) use (&$now): object { + return new CloudAbuseProjectionService( + static function () use (&$now): int { return $now; }, + $hook + ); +}; + +// Normal success atomically creates the current state, receipt and log finality. +$addInstance(1, 11); +$key1 = $insertSample(1, 11, 120, $metrics()); +$successWorker = $worker(); +$assert($successWorker->tick(), 'A due projection was not claimed.'); +$assert($log->table('cloud_vm_network_projection_outbox') + ->where('sample_key', $key1)->value('state') === 'done' + && $main->table('cloud_abuse_projection_receipts') + ->where('sample_key', $key1)->count() === 1 + && $main->table('cloud_abuse_instance_states') + ->where('instance_id', 1)->value('risk_state') === 'observing' + && (int) $log->table('cloud_vm_network_baselines') + ->where('instance_id', 1)->value('sample_count') === 1, + 'Successful projection did not converge both databases.'); +$assert($successWorker->observationErrors() === [], + 'Successful projection retained a stale worker error.'); + +// Crash after the main commit must replay from its receipt without duplicating. +$addInstance(2, 22); +$key2 = $insertSample(2, 22, 180, $metrics()); +$crashed = false; +$worker(static function () use (&$crashed): void { + if (!$crashed) { + $crashed = true; + throw new RuntimeException('simulated cross-database crash'); + } +})->tick(); +$assert($main->table('cloud_abuse_projection_receipts') + ->where('sample_key', $key2)->count() === 1 + && $log->table('cloud_vm_network_projection_outbox') + ->where('sample_key', $key2)->value('state') === 'pending' + && (int) $log->table('cloud_vm_network_baselines') + ->where('instance_id', 2)->value('sample_count') === 0, + 'Simulated crash did not preserve the explicit replay seam.'); +$now += 15; +$assert($worker()->tick(), 'Receipt-backed projection did not replay.'); +$assert($log->table('cloud_vm_network_projection_outbox') + ->where('sample_key', $key2)->value('state') === 'done' + && $main->table('cloud_abuse_projection_receipts') + ->where('sample_key', $key2)->count() === 1 + && (int) $log->table('cloud_vm_network_baselines') + ->where('instance_id', 2)->value('sample_count') === 1, + 'Receipt replay duplicated or failed to finalize projection.'); + +// A committed main receipt is authoritative even when the cross-database +// finalization seam keeps failing beyond the ordinary retry budget. +$addInstance(12, 122); +$receiptKey = $insertSample(12, 122, 240, $metrics()); +$finalizeFailure = static function (): void { + throw new RuntimeException('simulated repeated finalization failure'); +}; +for ($attempt = 1; $attempt <= 8; ++$attempt) { + $failingWorker = $worker($finalizeFailure); + $assert($failingWorker->tick(), + 'Receipt-backed retry was not consumed.'); + $receiptOutbox = $log->table( + 'cloud_vm_network_projection_outbox' + )->where('sample_key', $receiptKey)->first(); + $assert((int) $receiptOutbox->attempts === $attempt, + 'Receipt-backed retry count was not durable.'); + $expectedError = $attempt === 8 + ? 'projection_receipt_finalize_pending' : 'projection_failed'; + $assert($failingWorker->observationErrors() === [$expectedError], + 'Receipt-backed retry exposed an unstable worker error.'); + $now += 1000; +} +$receiptOutbox = $log->table('cloud_vm_network_projection_outbox') + ->where('sample_key', $receiptKey)->first(); +$assert($receiptOutbox->state === 'pending' + && $receiptOutbox->discard_reason_code === '' + && $main->table('cloud_abuse_projection_receipts') + ->where('sample_key', $receiptKey)->count() === 1 + && (int) $log->table('cloud_vm_network_baselines') + ->where('instance_id', 12)->value('sample_count') === 0, + 'Committed receipt was discarded after finalization retries.'); +$receiptRecoveryWorker = $worker(); +$assert($receiptRecoveryWorker->tick(), + 'Receipt-backed recovery was not consumed.'); +$assert($log->table('cloud_vm_network_projection_outbox') + ->where('sample_key', $receiptKey)->value('state') === 'done' + && (int) $log->table('cloud_vm_network_baselines') + ->where('instance_id', 12)->value('sample_count') === 1 + && $main->table('cloud_abuse_projection_receipts') + ->where('sample_key', $receiptKey)->count() === 1, + 'Recovered finalization did not converge from the committed receipt.'); +$assert($receiptRecoveryWorker->observationErrors() === [], + 'Successful receipt recovery retained a worker error.'); + +// A matching receipt also protects the earlier claim/ledger-failure path. +// Because damaged identity evidence cannot reproduce the exact baseline, the +// safe convergence is to finish the already-applied outbox and warm up again. +$addInstance(13, 133); +$claimReceiptKey = $insertSample( + 13, + 133, + 300, + $metrics(), + true, + [], + 'pending', + null, + 0, + 7 +); +$claimResetFindingKey = hash('sha256', implode("\n", [ + 13, CloudAbuseDetector::RULE_VERSION, + 'outbound_tcp_syn_sweep', 'ipv4', 'outbound', +])); +$main->table('cloud_abuse_findings')->insert([ + 'finding_key' => $claimResetFindingKey, + 'instance_id' => 13, 'user_id' => 133, 'host_id' => 7, + 'rule_code' => 'outbound_tcp_syn_sweep', + 'rule_version' => CloudAbuseDetector::RULE_VERSION, + 'direction' => 'outbound', 'ip_family' => 'ipv4', + 'severity' => 'high', 'state' => 'open', + 'first_seen_at' => 120, 'last_seen_at' => 240, + 'consecutive_windows' => 2, 'evidence_json' => '{}', + 'resolved_at' => null, 'created_at' => $now, 'updated_at' => $now, +]); +$main->table('cloud_abuse_projection_receipts')->insert([ + 'sample_key' => $claimReceiptKey, + 'instance_id' => 13, + 'rule_version' => CloudAbuseDetector::RULE_VERSION, + 'bucket_start' => 300, + 'projected_at' => $now, +]); +$log->table('cloud_vm_network_projection_outbox') + ->where('sample_key', $claimReceiptKey)->update([ + 'placement_key' => str_repeat('1', 64), + ]); +$claimReceiptWorker = $worker(); +$assert($claimReceiptWorker->tick(), + 'Receipt-backed claim recovery was not consumed.'); +$claimReceiptOutbox = $log->table('cloud_vm_network_projection_outbox') + ->where('sample_key', $claimReceiptKey)->first(); +$assert($claimReceiptOutbox->state === 'done' + && $claimReceiptOutbox->discard_reason_code === '' + && (int) $claimReceiptOutbox->attempts === 8 + && $log->table('cloud_vm_network_baselines') + ->where('instance_id', 13)->count() === 1 + && (int) $log->table('cloud_vm_network_baselines') + ->where('instance_id', 13)->value('sample_count') === 0, + 'Matching claim receipt was discarded or left a divergent baseline.'); +$assert($claimReceiptWorker->observationErrors() + === ['projection_receipt_baseline_reset'], + 'Receipt-backed claim recovery exposed no bounded diagnostic.'); +$claimNextKey = $insertSample(13, 133, 360, $metrics()); +$assert($worker()->tick() + && $log->table('cloud_vm_network_projection_outbox') + ->where('sample_key', $claimNextKey)->value('state') === 'done' + && (int) $log->table('cloud_vm_network_baselines') + ->where('instance_id', 13)->value('sample_count') === 1, + 'Receipt-backed claim recovery did not restart safe warm-up.'); +$assert($main->table('cloud_abuse_findings') + ->where('finding_key', $claimResetFindingKey)->value('state') + === 'open', + 'Receipt-backed reset cleared an open finding without evidence.'); +for ($step = 1; $step <= 63; ++$step) { + $insertSample(13, 133, 360 + ($step * 60), $metrics()); + $assert($worker()->tick(), + 'Receipt-reset clear convergence did not run.'); +} +$assert($main->table('cloud_abuse_findings') + ->where('finding_key', $claimResetFindingKey)->value('state') + === 'resolved' + && $main->table('cloud_abuse_instance_states') + ->where('instance_id', 13)->value('risk_state') === 'normal', + 'Receipt reset left an open finding after verified clear evidence.'); + +// A conflicting receipt is never treated as proof that the main effect did +// not happen. It remains attributable and backed off for operator repair. +$addInstance(14, 144); +$claimConflictKey = $insertSample( + 14, + 144, + 300, + $metrics(), + true, + [], + 'pending', + null, + 0, + 7 +); +$main->table('cloud_abuse_projection_receipts')->insert([ + 'sample_key' => $claimConflictKey, + 'instance_id' => 14, + 'rule_version' => CloudAbuseDetector::RULE_VERSION, + 'bucket_start' => 301, + 'projected_at' => $now, +]); +$log->table('cloud_vm_network_projection_outbox') + ->where('sample_key', $claimConflictKey)->update([ + 'placement_key' => str_repeat('2', 64), + ]); +$claimConflictWorker = $worker(); +$assert($claimConflictWorker->tick(), + 'Conflicting claim receipt was not inspected.'); +$claimConflictOutbox = $log->table('cloud_vm_network_projection_outbox') + ->where('sample_key', $claimConflictKey)->first(); +$assert($claimConflictOutbox->state === 'pending' + && (int) $claimConflictOutbox->attempts === 8 + && $claimConflictOutbox->last_error_code + === 'projection_receipt_conflict' + && $claimConflictOutbox->discard_reason_code === '', + 'Conflicting claim receipt was incorrectly declared not applied.'); +$assert($claimConflictWorker->observationErrors() + === ['projection_receipt_conflict'], + 'Conflicting claim receipt lost its bounded claim diagnostic.'); +$log->table('cloud_vm_network_projection_outbox') + ->where('sample_key', $claimConflictKey)->delete(); +$log->table('cloud_vm_network_observation_samples') + ->where('sample_key', $claimConflictKey)->delete(); +$main->table('cloud_abuse_projection_receipts') + ->where('sample_key', $claimConflictKey)->delete(); + +// If current findings cannot be read, a matching receipt stays pending rather +// than losing the reset boundary; it converges after the read model recovers. +$addInstance(15, 155); +$seedPendingKey = $insertSample( + 15, + 155, + 300, + $metrics(), + true, + [], + 'pending', + null, + 0, + 7 +); +$main->table('cloud_abuse_projection_receipts')->insert([ + 'sample_key' => $seedPendingKey, + 'instance_id' => 15, + 'rule_version' => CloudAbuseDetector::RULE_VERSION, + 'bucket_start' => 300, + 'projected_at' => $now, +]); +$log->table('cloud_vm_network_projection_outbox') + ->where('sample_key', $seedPendingKey)->update([ + 'placement_key' => str_repeat('3', 64), + ]); +$main->statement( + 'ALTER TABLE cloud_abuse_findings ' + . 'RENAME TO cloud_abuse_findings_unavailable' +); +$seedPendingWorker = $worker(); +$assert($seedPendingWorker->tick(), + 'Unavailable baseline seed was not consumed.'); +$seedPending = $log->table('cloud_vm_network_projection_outbox') + ->where('sample_key', $seedPendingKey)->first(); +$assert($seedPending->state === 'pending' + && (int) $seedPending->attempts === 8 + && $seedPending->last_error_code + === 'projection_baseline_seed_pending' + && $seedPending->discard_reason_code === '', + 'Unavailable baseline seed was discarded or left unbounded.'); +$main->statement( + 'ALTER TABLE cloud_abuse_findings_unavailable ' + . 'RENAME TO cloud_abuse_findings' +); +$now += 1000; +$assert($worker()->tick() + && $log->table('cloud_vm_network_projection_outbox') + ->where('sample_key', $seedPendingKey)->value('state') === 'done' + && (int) $log->table('cloud_vm_network_baselines') + ->where('instance_id', 15)->value('sample_count') === 0, + 'Baseline seed did not converge after its read model recovered.'); + +// Missing owners and stale buckets end explicitly instead of locking forever. +$deletedKey = $insertSample(99, 99, 240, $metrics()); +$assert($worker()->tick(), 'Deleted-instance sample was not consumed.'); +$deleted = $log->table('cloud_vm_network_projection_outbox') + ->where('sample_key', $deletedKey)->first(); +$assert($deleted->state === 'discarded' + && $deleted->discard_reason_code === 'instance_deleted', + 'Deleted instance left a retrying or locked outbox row.'); + +$addInstance(8, 88); +$oldOwnerFindingKey = hash('sha256', implode("\n", [ + 8, CloudAbuseDetector::RULE_VERSION, + 'outbound_tcp_syn_sweep', 'ipv4', 'outbound', +])); +$main->table('cloud_abuse_instance_states')->insert([ + 'instance_id' => 8, 'user_id' => 89, 'host_id' => 7, + 'risk_state' => 'high_risk', 'collection_state' => 'healthy', + 'latest_direction' => 'outbound', + 'latest_rule_code' => 'outbound_tcp_syn_sweep', + 'latest_severity' => 'high', 'summary_json' => '{}', + 'last_projection_key' => str_repeat('8', 64), + 'last_bucket_start' => 180, 'last_sample_at' => 240, + 'last_evaluated_at' => $now, + 'created_at' => $now, 'updated_at' => $now, +]); +$main->table('cloud_abuse_findings')->insert([ + 'finding_key' => $oldOwnerFindingKey, + 'instance_id' => 8, 'user_id' => 89, 'host_id' => 7, + 'rule_code' => 'outbound_tcp_syn_sweep', + 'rule_version' => CloudAbuseDetector::RULE_VERSION, + 'direction' => 'outbound', 'ip_family' => 'ipv4', + 'severity' => 'high', 'state' => 'open', + 'first_seen_at' => 120, 'last_seen_at' => 180, + 'consecutive_windows' => 2, 'evidence_json' => '{}', + 'resolved_at' => null, 'created_at' => $now, 'updated_at' => $now, +]); +$ownerKey = $insertSample(8, 89, 240, $metrics()); +$ownerWorker = $worker(); +$assert($ownerWorker->tick(), 'Owner-mismatch sample was not consumed.'); +$ownerMismatch = $log->table('cloud_vm_network_projection_outbox') + ->where('sample_key', $ownerKey)->first(); +$assert($ownerMismatch->state === 'discarded' + && $ownerMismatch->discard_reason_code === 'instance_owner_changed' + && $log->table('cloud_vm_network_baselines') + ->where('instance_id', 8)->count() === 0 + && $main->table('cloud_abuse_instance_states') + ->where('instance_id', 8)->count() === 0 + && $main->table('cloud_abuse_findings') + ->where('finding_key', $oldOwnerFindingKey)->value('state') + === 'resolved' + && (int) $main->table('cloud_abuse_findings') + ->where('finding_key', $oldOwnerFindingKey)->value('user_id') === 89 + && (int) $main->table('cloud_instances') + ->where('id', 8)->value('user_id') === 88, + 'Owner mismatch left a retrying or locked outbox row.'); +$assert($ownerWorker->observationErrors() === ['instance_owner_changed'], + 'Owner mismatch did not expose its stable discard reason.'); +$correctOwnerKey = $insertSample(8, 88, 300, $metrics()); +$assert($worker()->tick(), 'Correct owner sample did not recover.'); +$assert($log->table('cloud_vm_network_projection_outbox') + ->where('sample_key', $correctOwnerKey)->value('state') === 'done' + && (int) $log->table('cloud_vm_network_baselines') + ->where('instance_id', 8)->value('user_id') === 88 + && (int) $main->table('cloud_abuse_instance_states') + ->where('instance_id', 8)->value('user_id') === 88, + 'Owner mismatch contaminated the next owner baseline.'); + +$addInstance(3, 33); +$lateBaseline = CloudAbuseDetector::initialBaseline(); +$lateBaseline['last_bucket_start'] = 360; +$insertBaseline(3, 33, $lateBaseline, 360, str_repeat('b', 64)); +$lateKey = $insertSample(3, 33, 300, $metrics()); +$assert($worker()->tick(), 'Out-of-order sample was not consumed.'); +$assert($log->table('cloud_vm_network_projection_outbox') + ->where('sample_key', $lateKey)->value('discard_reason_code') + === 'out_of_order_sample', + 'Out-of-order sample changed or blocked the baseline.'); + +// An expired outbox+baseline lease is safely reclaimed. +$addInstance(4, 44); +$expiredToken = str_repeat('c', 32); +$insertBaseline(4, 44, CloudAbuseDetector::initialBaseline(), + null, null, $expiredToken, $now - 1, 4); +$expiredKey = $insertSample(4, 44, 420, $metrics(), true, [], + 'leased', $expiredToken, $now - 1, 1); +$assert($worker()->tick(), 'Expired projection lease was not reclaimed.'); +$assert($log->table('cloud_vm_network_projection_outbox') + ->where('sample_key', $expiredKey)->value('state') === 'done' + && (int) $log->table('cloud_vm_network_baselines') + ->where('instance_id', 4)->value('revision') === 5, + 'Expired lease recovery did not preserve baseline CAS ordering.'); + +// Warm baseline: two high windows open a finding; five clear windows resolve it. +$addInstance(5, 55); +$warm = CloudAbuseDetector::initialBaseline(); +$warm['sample_count'] = CloudAbuseDetector::WARMUP_WINDOWS; +$warm['last_bucket_start'] = 60; +$insertBaseline(5, 55, $warm, 60, str_repeat('d', 64)); +$high = [ + 'tcp_active_flows' => 6000, 'tcp_new_flows' => 6000, + 'tcp_syn_flows' => 5000, 'tcp_unreplied_flows' => 5000, + 'unique_remote_ips' => 700, 'unique_remote_ports' => 1, + 'unique_local_ports' => 6000, + 'max_remote_ports_per_remote_ip' => 1, + 'max_remote_ips_per_remote_port' => 700, + 'max_local_ports_per_remote_ip' => 1, + 'max_remote_ips_per_local_port' => 700, +]; +foreach ([120, 180] as $bucket) { + $insertSample(5, 55, $bucket, $metrics($high)); + $assert($worker()->tick(), 'High-risk projection did not run.'); +} +$finding = $main->table('cloud_abuse_findings') + ->where('instance_id', 5)->where('state', 'open')->first(); +$assert($finding !== null && $finding->severity === 'high' + && $finding->direction === 'outbound', + 'Two high windows did not open the deterministic finding.'); +foreach ([240, 300, 360, 420, 480] as $bucket) { + $insertSample(5, 55, $bucket, $metrics()); + $assert($worker()->tick(), 'Clear projection did not run.'); +} +$finding = $main->table('cloud_abuse_findings') + ->where('instance_id', 5)->first(); +$assert($finding->state === 'resolved' + && (int) $finding->resolved_at === 480, + 'Five verified clear windows did not resolve the finding.'); + +// A panel ring gap resets its baseline but never clears an existing finding. +$addInstance(6, 66); +$gapBaseline = CloudAbuseDetector::resetBaseline([[ + 'rule_code' => 'outbound_tcp_syn_sweep', + 'direction' => 'outbound', + 'ip_family' => 'ipv4', + 'severity' => 'high', +]]); +$gapBaseline['sample_count'] = 60; +$gapBaseline['last_bucket_start'] = 60; +$gapBaseline['last_interface'] = $metrics()['interface']; +$insertBaseline(6, 66, $gapBaseline, 60, str_repeat('e', 64)); +$findingKey = hash('sha256', implode("\n", [ + 6, CloudAbuseDetector::RULE_VERSION, + 'outbound_tcp_syn_sweep', 'ipv4', 'outbound', +])); +$main->table('cloud_abuse_findings')->insert([ + 'finding_key' => $findingKey, + 'instance_id' => 6, 'user_id' => 66, 'host_id' => 7, + 'rule_code' => 'outbound_tcp_syn_sweep', + 'rule_version' => CloudAbuseDetector::RULE_VERSION, + 'direction' => 'outbound', 'ip_family' => 'ipv4', + 'severity' => 'high', 'state' => 'open', + 'first_seen_at' => 60, 'last_seen_at' => 60, + 'consecutive_windows' => 2, 'evidence_json' => '{}', + 'resolved_at' => null, 'created_at' => $now, 'updated_at' => $now, +]); +$main->table('cloud_abuse_instance_states')->insert([ + 'instance_id' => 6, 'user_id' => 66, 'host_id' => 7, + 'risk_state' => 'high_risk', 'collection_state' => 'healthy', + 'latest_direction' => 'outbound', + 'latest_rule_code' => 'outbound_tcp_syn_sweep', + 'latest_severity' => 'high', 'summary_json' => '{}', + 'last_projection_key' => str_repeat('e', 64), + 'last_bucket_start' => 60, 'last_sample_at' => 120, + 'last_evaluated_at' => $now, + 'created_at' => $now, 'updated_at' => $now, +]); +$gapKey = $insertSample(6, 66, 120, $metrics(), false, + ['incomplete', 'panel_ring_gap']); +$assert($worker()->tick(), 'Panel-gap projection did not run.'); +$gapStored = json_decode((string) $log->table( + 'cloud_vm_network_baselines' +)->where('instance_id', 6)->value('metrics_json'), true, + 32, JSON_THROW_ON_ERROR); +$state6 = $main->table('cloud_abuse_instance_states') + ->where('instance_id', 6)->first(); +$assert($log->table('cloud_vm_network_projection_outbox') + ->where('sample_key', $gapKey)->value('state') === 'done' + && $gapStored['sample_count'] === 0 + && $gapStored['last_interface'] === null + && $main->table('cloud_abuse_findings') + ->where('finding_key', $findingKey)->value('state') === 'open' + && $state6->risk_state === 'high_risk' + && $state6->collection_state === 'degraded', + 'Incomplete ring-gap evidence advanced or cleared trusted state.'); +for ($step = 1; $step <= 64; ++$step) { + $insertSample(6, 66, 120 + ($step * 60), $metrics()); + $assert($worker()->tick(), 'Panel-gap clear convergence did not run.'); +} +$assert($main->table('cloud_abuse_findings') + ->where('finding_key', $findingKey)->value('state') === 'resolved' + && $main->table('cloud_abuse_instance_states') + ->where('instance_id', 6)->value('risk_state') === 'normal', + 'Panel-gap reset left an open finding after verified clear evidence.'); + +// Corrupt/non-canonical evidence has bounded retries and an explicit terminal +// state; it can neither leak into the main read model nor hold the queue. +$addInstance(7, 77); +$invalidKey = $insertSample(7, 77, 120, $metrics()); +$invalidMetrics = $metrics(); +$invalidMetrics['ipv4']['outbound']['remote_addresses'] = ['203.0.113.8']; +$log->table('cloud_vm_network_observation_samples') + ->where('sample_key', $invalidKey)->update([ + 'metrics_json' => json_encode($invalidMetrics, + JSON_THROW_ON_ERROR), + ]); +for ($attempt = 1; $attempt <= 8; ++$attempt) { + $assert($worker()->tick(), 'Invalid sample retry was not consumed.'); + $row = $log->table('cloud_vm_network_projection_outbox') + ->where('sample_key', $invalidKey)->first(); + $assert((int) $row->attempts === $attempt, + 'Invalid sample attempt count was not durable.'); + $now += 1000; +} +$invalidOutbox = $log->table('cloud_vm_network_projection_outbox') + ->where('sample_key', $invalidKey)->first(); +$assert($invalidOutbox->state === 'discarded' + && $invalidOutbox->discard_reason_code + === 'projection_retry_exhausted' + && $main->table('cloud_abuse_projection_receipts') + ->where('sample_key', $invalidKey)->count() === 0, + 'Invalid aggregate evidence leaked or left a permanent projection lock.'); + +// A corrupt, attributable sample/outbox pair backs off and terminates without +// monopolising the globally earliest due-candidate position. +$addInstance(9, 99); +$badLedgerKey = $insertSample(9, 99, 60, $metrics()); +$log->table('cloud_vm_network_projection_outbox') + ->where('sample_key', $badLedgerKey)->update([ + 'placement_key' => str_repeat('f', 64), + ]); +$addInstance(10, 100); +$nextInstanceKey = $insertSample(10, 100, 120, $metrics()); +$badLedgerWorker = $worker(); +$assert($badLedgerWorker->tick(), 'Bad ledger candidate was not bounded.'); +$assert((int) $log->table('cloud_vm_network_projection_outbox') + ->where('sample_key', $badLedgerKey)->value('attempts') === 1, + 'Bad ledger candidate did not record its first attempt.'); +$assert($badLedgerWorker->observationErrors() + === ['projection_sample_ledger_inconsistent'], + 'Bad ledger candidate exposed no stable worker error.'); +$assert($worker()->tick(), 'Next instance did not bypass backed-off damage.'); +$assert($log->table('cloud_vm_network_projection_outbox') + ->where('sample_key', $nextInstanceKey)->value('state') === 'done', + 'Bad global candidate blocked an unrelated instance.'); +for ($attempt = 2; $attempt <= 8; ++$attempt) { + $now += 1000; + $assert($worker()->tick(), 'Bad ledger retry was not consumed.'); + $badOutbox = $log->table('cloud_vm_network_projection_outbox') + ->where('sample_key', $badLedgerKey)->first(); + $assert((int) $badOutbox->attempts === $attempt, + 'Bad ledger retry count was not durable.'); +} +$badOutbox = $log->table('cloud_vm_network_projection_outbox') + ->where('sample_key', $badLedgerKey)->first(); +$assert($badOutbox->state === 'discarded' + && $badOutbox->discard_reason_code === 'projection_retry_exhausted' + && $badOutbox->last_error_code + === 'projection_sample_ledger_inconsistent', + 'Bad ledger candidate did not reach an attributable terminal state.'); + +$idleWorker = $worker(); +$assert(!$idleWorker->tick() && $idleWorker->observationErrors() === [], + 'Idle projection worker retained an earlier observation error.'); + +$source = file_get_contents(dirname(__DIR__) + . '/app/Services/Cloud/CloudAbuseProjectionService.php'); +$assert(is_string($source) + && !str_contains($source, 'CloudJob') + && !str_contains($source, 'service_status') + && !str_contains($source, 'network_status'), + 'Projection service gained a task, lifecycle or enforcement dependency.'); + +echo "Cloud abuse projection convergence tests passed.\n";