You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1205 lines
48 KiB
PHP
1205 lines
48 KiB
PHP
<?php
|
|
|
|
namespace App\Services\Cloud;
|
|
|
|
use App\Models\User;
|
|
|
|
/**
|
|
* Idempotently projects the log-database observation outbox into the main
|
|
* database read model. The service is observe-only and never mutates an
|
|
* instance, its network, billing state or the provider-operation queue.
|
|
*/
|
|
final class CloudAbuseProjectionService
|
|
{
|
|
private const LEASE_SECONDS = 45;
|
|
private const MAX_ATTEMPTS = 8;
|
|
private const MAX_BACKOFF_SECONDS = 900;
|
|
|
|
/** @var \Closure():int */
|
|
private readonly \Closure $clock;
|
|
|
|
/** @var null|\Closure(string):void */
|
|
private readonly ?\Closure $afterMainCommit;
|
|
|
|
/** @var list<string> */
|
|
private array $errors = [];
|
|
|
|
/**
|
|
* The optional hook is a test seam for the only unavoidable cross-database
|
|
* crash window. Production callers must leave it null.
|
|
*/
|
|
public function __construct(
|
|
?callable $clock = null,
|
|
?callable $afterMainCommit = null
|
|
) {
|
|
$this->clock = \Closure::fromCallable(
|
|
$clock ?? static fn (): int => time()
|
|
);
|
|
$this->afterMainCommit = $afterMainCommit === null
|
|
? null : \Closure::fromCallable($afterMainCommit);
|
|
}
|
|
|
|
/** Project at most one ordered sample. */
|
|
public function tick(): bool
|
|
{
|
|
$this->errors = [];
|
|
$now = ($this->clock)();
|
|
if (!is_int($now) || $now < 1) {
|
|
throw new \LogicException('Abuse projection clock is invalid');
|
|
}
|
|
$claim = $this->claim($now);
|
|
if ($claim === null) {
|
|
return false;
|
|
}
|
|
if (isset($claim['claim_error_code'])) {
|
|
$this->errors[] = (string) $claim['claim_error_code'];
|
|
return true;
|
|
}
|
|
try {
|
|
$claim = self::hydrateClaim($claim);
|
|
$decision = $this->projectMain($claim, $now);
|
|
if ($this->afterMainCommit !== null) {
|
|
($this->afterMainCommit)($claim['sample_key']);
|
|
}
|
|
if ($decision['state'] === 'discarded') {
|
|
$reason = self::boundedCode(
|
|
(string) $decision['reason'],
|
|
'projection_discarded'
|
|
);
|
|
$this->discard(
|
|
$claim,
|
|
$reason,
|
|
$now
|
|
);
|
|
$this->errors[] = $reason;
|
|
} else {
|
|
$this->finalize($claim, $decision['baseline'], $now);
|
|
}
|
|
} catch (\Throwable $error) {
|
|
$this->errors[] = $this->retryOrDiscard(
|
|
$claim,
|
|
$error,
|
|
$now
|
|
);
|
|
}
|
|
return true;
|
|
}
|
|
|
|
/** @return list<string> */
|
|
public function observationErrors(): array
|
|
{
|
|
return $this->errors;
|
|
}
|
|
|
|
/** @return null|array<string,mixed> */
|
|
private function claim(int $now): ?array
|
|
{
|
|
$log = User::getConnectionResolver()->connection('log');
|
|
return $log->transaction(function () use ($log, $now): ?array {
|
|
$candidate = $log->table(
|
|
'cloud_vm_network_projection_outbox as candidate'
|
|
)->where(static function ($due) use ($now): void {
|
|
$due->where(static function ($pending) use ($now): void {
|
|
$pending->where('candidate.state', 'pending')
|
|
->where('candidate.available_at', '<=', $now);
|
|
})->orWhere(static function ($expired) use ($now): void {
|
|
$expired->where('candidate.state', 'leased')
|
|
->where('candidate.lease_until', '<=', $now);
|
|
});
|
|
})->whereNotExists(static function ($earlier): void {
|
|
$earlier->selectRaw('1')->from(
|
|
'cloud_vm_network_projection_outbox as prior'
|
|
)->whereColumn(
|
|
'prior.instance_id',
|
|
'candidate.instance_id'
|
|
)->whereIn('prior.state', ['pending', 'leased'])
|
|
->where(static function ($order): void {
|
|
$order->whereColumn(
|
|
'prior.bucket_start',
|
|
'<',
|
|
'candidate.bucket_start'
|
|
)->orWhere(static function ($same): void {
|
|
$same->whereColumn(
|
|
'prior.bucket_start',
|
|
'candidate.bucket_start'
|
|
)->whereColumn(
|
|
'prior.sample_key',
|
|
'<',
|
|
'candidate.sample_key'
|
|
);
|
|
});
|
|
});
|
|
})->orderBy('candidate.bucket_start')
|
|
->orderBy('candidate.sample_key')->first();
|
|
if ($candidate === null) {
|
|
return null;
|
|
}
|
|
$outbox = $log->table('cloud_vm_network_projection_outbox')
|
|
->where('sample_key', (string) $candidate->sample_key)
|
|
->lockForUpdate()->first();
|
|
if ($outbox === null
|
|
|| !in_array((string) $outbox->state,
|
|
['pending', 'leased'], true)
|
|
|| ((string) $outbox->state === 'pending'
|
|
&& (int) $outbox->available_at > $now)
|
|
|| ((string) $outbox->state === 'leased'
|
|
&& (int) $outbox->lease_until > $now)) {
|
|
return null;
|
|
}
|
|
$sample = $log->table(
|
|
'cloud_vm_network_observation_samples'
|
|
)->where('sample_key', (string) $outbox->sample_key)
|
|
->lockForUpdate()->first();
|
|
if ($sample === null
|
|
|| (int) $sample->instance_id !== (int) $outbox->instance_id
|
|
|| (int) $sample->bucket_start !== (int) $outbox->bucket_start
|
|
|| !hash_equals(
|
|
(string) $sample->placement_key,
|
|
(string) $outbox->placement_key
|
|
)) {
|
|
$code = 'projection_sample_ledger_inconsistent';
|
|
$reported = $this->recordClaimFailure(
|
|
$log,
|
|
$outbox,
|
|
$code,
|
|
$now
|
|
);
|
|
return ['claim_error_code' => $reported];
|
|
}
|
|
$baseline = $log->table('cloud_vm_network_baselines')
|
|
->where('instance_id', (int) $sample->instance_id)
|
|
->lockForUpdate()->first();
|
|
if ($baseline === null) {
|
|
$log->table('cloud_vm_network_baselines')->insert([
|
|
'instance_id' => (int) $sample->instance_id,
|
|
'user_id' => (int) $sample->user_id,
|
|
'rule_version' => CloudAbuseDetector::RULE_VERSION,
|
|
'sample_count' => 0,
|
|
'warmup_state' => 'warming',
|
|
'metrics_json' => self::json(
|
|
CloudAbuseDetector::initialBaseline()
|
|
),
|
|
'last_projection_key' => null,
|
|
'last_bucket_start' => null,
|
|
'projection_lease_token' => null,
|
|
'projection_lease_until' => 0,
|
|
'revision' => 0,
|
|
'created_at' => $now,
|
|
'updated_at' => $now,
|
|
]);
|
|
$baseline = $log->table('cloud_vm_network_baselines')
|
|
->where('instance_id', (int) $sample->instance_id)
|
|
->lockForUpdate()->first();
|
|
}
|
|
if ($baseline === null
|
|
|| (int) $baseline->user_id !== (int) $sample->user_id
|
|
|| (int) $baseline->rule_version
|
|
!== CloudAbuseDetector::RULE_VERSION) {
|
|
$code = 'projection_baseline_identity_inconsistent';
|
|
$reported = $this->recordClaimFailure(
|
|
$log,
|
|
$outbox,
|
|
$code,
|
|
$now
|
|
);
|
|
return ['claim_error_code' => $reported];
|
|
}
|
|
if ($baseline->projection_lease_token !== null
|
|
&& (int) $baseline->projection_lease_until > $now) {
|
|
return null;
|
|
}
|
|
$token = bin2hex(random_bytes(16));
|
|
$attempts = (int) $outbox->attempts + 1;
|
|
$outboxUpdated = $log->table(
|
|
'cloud_vm_network_projection_outbox'
|
|
)->where('sample_key', (string) $outbox->sample_key)
|
|
->where('state', (string) $outbox->state)
|
|
->update([
|
|
'state' => 'leased',
|
|
'attempts' => $attempts,
|
|
'lease_token' => $token,
|
|
'lease_until' => $now + self::LEASE_SECONDS,
|
|
'last_error_code' => '',
|
|
'updated_at' => $now,
|
|
]);
|
|
$baselineUpdated = $log->table(
|
|
'cloud_vm_network_baselines'
|
|
)->where('instance_id', (int) $sample->instance_id)
|
|
->where('revision', (int) $baseline->revision)
|
|
->update([
|
|
'projection_lease_token' => $token,
|
|
'projection_lease_until' => $now
|
|
+ self::LEASE_SECONDS,
|
|
'updated_at' => $now,
|
|
]);
|
|
if ($outboxUpdated !== 1 || $baselineUpdated !== 1) {
|
|
throw new \RuntimeException(
|
|
'Abuse projection lease compare-and-swap failed'
|
|
);
|
|
}
|
|
$complete = (int) $sample->complete === 1;
|
|
return [
|
|
'sample_key' => (string) $sample->sample_key,
|
|
'instance_id' => (int) $sample->instance_id,
|
|
'user_id' => (int) $sample->user_id,
|
|
'host_id' => (int) $sample->host_id,
|
|
'placement_key' => (string) $sample->placement_key,
|
|
'bucket_start' => (int) $sample->bucket_start,
|
|
'complete' => $complete,
|
|
'metrics_json' => (string) $sample->metrics_json,
|
|
'quality_flags_json' =>
|
|
(string) $sample->quality_flags_json,
|
|
'baseline_json' => (string) $baseline->metrics_json,
|
|
'baseline_revision' => (int) $baseline->revision,
|
|
'last_projection_key' => $baseline->last_projection_key,
|
|
'last_bucket_start' => $baseline->last_bucket_start === null
|
|
? null : (int) $baseline->last_bucket_start,
|
|
'attempts' => $attempts,
|
|
'lease_token' => $token,
|
|
];
|
|
}, 3);
|
|
}
|
|
|
|
private function recordClaimFailure(
|
|
object $log,
|
|
object $outbox,
|
|
string $code,
|
|
int $now
|
|
): string {
|
|
$code = self::boundedCode($code, 'projection_claim_failed');
|
|
$attempts = (int) $outbox->attempts + 1;
|
|
$terminal = $attempts >= self::MAX_ATTEMPTS;
|
|
if ($terminal) {
|
|
$receiptState = $this->receiptState([
|
|
'sample_key' => (string) $outbox->sample_key,
|
|
'instance_id' => (int) $outbox->instance_id,
|
|
'bucket_start' => (int) $outbox->bucket_start,
|
|
]);
|
|
if ($receiptState === 'matching') {
|
|
$baseline = $log->table('cloud_vm_network_baselines')
|
|
->where('instance_id', (int) $outbox->instance_id)
|
|
->lockForUpdate()->first();
|
|
if ($baseline !== null
|
|
&& $baseline->projection_lease_token !== null
|
|
&& (int) $baseline->projection_lease_until > $now) {
|
|
$terminal = false;
|
|
$code = 'projection_receipt_finalize_pending';
|
|
} else {
|
|
try {
|
|
$seed = $this->currentOwnerBaselineSeed(
|
|
(int) $outbox->instance_id
|
|
);
|
|
} catch (\Throwable) {
|
|
$seed = false;
|
|
}
|
|
if ($seed === false) {
|
|
$terminal = false;
|
|
$code = 'projection_baseline_seed_pending';
|
|
} else {
|
|
// The main read model already committed this sample.
|
|
// Use its current open findings to reset statistics
|
|
// without losing the evidence required for resolution.
|
|
if ($seed === null) {
|
|
$deleted = $log->table(
|
|
'cloud_vm_network_baselines'
|
|
)->where('instance_id',
|
|
(int) $outbox->instance_id)->delete();
|
|
if ($baseline !== null && $deleted !== 1) {
|
|
throw new \RuntimeException(
|
|
'Abuse baseline delete CAS failed'
|
|
);
|
|
}
|
|
} else {
|
|
$seedBaseline = $seed['baseline'];
|
|
$seedBaseline['last_bucket_start'] =
|
|
(int) $outbox->bucket_start;
|
|
$values = [
|
|
'user_id' => $seed['user_id'],
|
|
'rule_version' =>
|
|
CloudAbuseDetector::RULE_VERSION,
|
|
'sample_count' => 0,
|
|
'warmup_state' => 'warming',
|
|
'metrics_json' => self::json($seedBaseline),
|
|
'last_projection_key' =>
|
|
(string) $outbox->sample_key,
|
|
'last_bucket_start' =>
|
|
(int) $outbox->bucket_start,
|
|
'projection_lease_token' => null,
|
|
'projection_lease_until' => 0,
|
|
'updated_at' => $now,
|
|
];
|
|
if ($baseline === null) {
|
|
$log->table(
|
|
'cloud_vm_network_baselines'
|
|
)->insert([
|
|
'instance_id' =>
|
|
(int) $outbox->instance_id,
|
|
'revision' => 0,
|
|
'created_at' => $now,
|
|
] + $values);
|
|
} else {
|
|
$values['revision'] =
|
|
(int) $baseline->revision + 1;
|
|
$baselineUpdated = $log->table(
|
|
'cloud_vm_network_baselines'
|
|
)->where('instance_id',
|
|
(int) $outbox->instance_id)
|
|
->where('revision',
|
|
(int) $baseline->revision)
|
|
->update($values);
|
|
if ($baselineUpdated !== 1) {
|
|
throw new \RuntimeException(
|
|
'Abuse baseline reset CAS failed'
|
|
);
|
|
}
|
|
}
|
|
}
|
|
$updated = $log->table(
|
|
'cloud_vm_network_projection_outbox'
|
|
)->where('sample_key', (string) $outbox->sample_key)
|
|
->where('state', (string) $outbox->state)
|
|
->where('attempts', (int) $outbox->attempts)
|
|
->update([
|
|
'state' => 'done',
|
|
'attempts' => min(
|
|
$attempts,
|
|
self::MAX_ATTEMPTS
|
|
),
|
|
'lease_token' => null,
|
|
'lease_until' => 0,
|
|
'last_error_code' => '',
|
|
'projected_at' => $now,
|
|
'terminal_at' => $now,
|
|
'discard_reason_code' => '',
|
|
'updated_at' => $now,
|
|
]);
|
|
if ($updated !== 1) {
|
|
throw new \RuntimeException(
|
|
'Abuse receipt claim recovery CAS failed'
|
|
);
|
|
}
|
|
return 'projection_receipt_baseline_reset';
|
|
}
|
|
}
|
|
}
|
|
if ($receiptState === 'conflict') {
|
|
$terminal = false;
|
|
$code = 'projection_receipt_conflict';
|
|
} elseif ($receiptState === 'unknown') {
|
|
$terminal = false;
|
|
$code = 'projection_receipt_check_failed';
|
|
}
|
|
}
|
|
$values = $terminal ? [
|
|
'state' => 'discarded',
|
|
'attempts' => $attempts,
|
|
'lease_token' => null,
|
|
'lease_until' => 0,
|
|
'last_error_code' => $code,
|
|
'projected_at' => null,
|
|
'terminal_at' => $now,
|
|
'discard_reason_code' => 'projection_retry_exhausted',
|
|
'updated_at' => $now,
|
|
] : [
|
|
'state' => 'pending',
|
|
'attempts' => min($attempts, self::MAX_ATTEMPTS),
|
|
'available_at' => $now + self::retryDelay($attempts),
|
|
'lease_token' => null,
|
|
'lease_until' => 0,
|
|
'last_error_code' => $code,
|
|
'projected_at' => null,
|
|
'terminal_at' => null,
|
|
'discard_reason_code' => '',
|
|
'updated_at' => $now,
|
|
];
|
|
$updated = $log->table('cloud_vm_network_projection_outbox')
|
|
->where('sample_key', (string) $outbox->sample_key)
|
|
->where('state', (string) $outbox->state)
|
|
->where('attempts', (int) $outbox->attempts)
|
|
->update($values);
|
|
if ($updated !== 1) {
|
|
throw new \RuntimeException(
|
|
'Abuse projection claim failure compare-and-swap failed'
|
|
);
|
|
}
|
|
return $terminal ? 'projection_retry_exhausted' : $code;
|
|
}
|
|
|
|
/** @param array<string,mixed> $claim @return array<string,mixed> */
|
|
private static function hydrateClaim(array $claim): array
|
|
{
|
|
$claim['metrics'] = self::canonicalMetrics(self::decodeObject(
|
|
(string) $claim['metrics_json'],
|
|
'sample metrics'
|
|
));
|
|
$claim['quality_flags'] = self::canonicalQuality(self::decodeList(
|
|
(string) $claim['quality_flags_json'],
|
|
'sample quality flags'
|
|
));
|
|
$claim['baseline'] = self::decodeObject(
|
|
(string) $claim['baseline_json'],
|
|
'network baseline'
|
|
);
|
|
unset(
|
|
$claim['metrics_json'],
|
|
$claim['quality_flags_json'],
|
|
$claim['baseline_json']
|
|
);
|
|
if ($claim['complete'] === in_array(
|
|
'incomplete',
|
|
$claim['quality_flags'],
|
|
true
|
|
)) {
|
|
throw new \RuntimeException(
|
|
'Sample completeness and quality evidence disagree'
|
|
);
|
|
}
|
|
self::assertStoredSampleSemantics(
|
|
$claim['metrics'],
|
|
$claim['quality_flags'],
|
|
$claim['complete']
|
|
);
|
|
return $claim;
|
|
}
|
|
|
|
/**
|
|
* Recheck the semantic invariants that made the stored sample canonical.
|
|
* The log database is a durable boundary: corrupt rows must never be
|
|
* promoted to healthy evidence merely because their JSON shape is valid.
|
|
*
|
|
* @param array<string,mixed> $metrics
|
|
* @param array<int,string> $quality
|
|
*/
|
|
private static function assertStoredSampleSemantics(
|
|
array $metrics,
|
|
array $quality,
|
|
bool $complete
|
|
): void {
|
|
if ($metrics['ipv4'] === null && $metrics['ipv6'] === null) {
|
|
throw new \RuntimeException(
|
|
'Sample has no observable address family'
|
|
);
|
|
}
|
|
$interfaceUnavailable = in_array(
|
|
'interface_unavailable',
|
|
$quality,
|
|
true
|
|
);
|
|
$hasMissingCounter = in_array(
|
|
null,
|
|
$metrics['interface'],
|
|
true
|
|
);
|
|
if ($hasMissingCounter !== $interfaceUnavailable) {
|
|
throw new \RuntimeException(
|
|
'Sample interface evidence is inconsistent'
|
|
);
|
|
}
|
|
if (in_array('panel_ring_gap', $quality, true)
|
|
&& $complete) {
|
|
throw new \RuntimeException(
|
|
'Panel ring gap sample cannot be complete'
|
|
);
|
|
}
|
|
if ($complete && array_intersect($quality, [
|
|
'clock_regression', 'cold_start', 'conntrack_unavailable',
|
|
'counter_reset', 'flow_limit_reached',
|
|
'interface_unavailable',
|
|
'local_port_cardinality_saturated', 'mapping_changed',
|
|
'overflow', 'relation_cardinality_saturated',
|
|
'remote_ip_cardinality_saturated',
|
|
'remote_port_cardinality_saturated', 'sampling_gap',
|
|
]) !== []) {
|
|
throw new \RuntimeException(
|
|
'Complete sample contains risk-unsafe evidence'
|
|
);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* @param array<string,mixed> $claim
|
|
* @return array{state:string,reason?:string,baseline?:array<string,mixed>}
|
|
*/
|
|
private function projectMain(array $claim, int $now): array
|
|
{
|
|
if ($claim['last_bucket_start'] !== null
|
|
&& $claim['bucket_start'] <= $claim['last_bucket_start']) {
|
|
if (is_string($claim['last_projection_key'])
|
|
&& hash_equals(
|
|
$claim['last_projection_key'],
|
|
$claim['sample_key']
|
|
)) {
|
|
return [
|
|
'state' => 'done',
|
|
'baseline' => $claim['baseline'],
|
|
];
|
|
}
|
|
return ['state' => 'discarded',
|
|
'reason' => 'out_of_order_sample'];
|
|
}
|
|
$reset = in_array(
|
|
'panel_ring_gap',
|
|
$claim['quality_flags'],
|
|
true
|
|
);
|
|
$base = $reset
|
|
? CloudAbuseDetector::resetBaselineFromPrevious(
|
|
$claim['baseline']
|
|
) : $claim['baseline'];
|
|
$detected = CloudAbuseDetector::evaluate([
|
|
'bucket_start' => $claim['bucket_start'],
|
|
'complete' => $claim['complete'],
|
|
'quality_flags' => $claim['quality_flags'],
|
|
'metrics' => $claim['metrics'],
|
|
], $base);
|
|
$baseline = $detected['baseline'];
|
|
$evaluation = $detected['evaluation'];
|
|
if (!$claim['complete']) {
|
|
// Do not derive a per-second rate across an unobserved minute.
|
|
// Statistical history and finding streaks stay frozen, while the
|
|
// next complete sample establishes a fresh interface baseline.
|
|
$baseline['last_interface'] = null;
|
|
}
|
|
|
|
$main = User::getConnectionResolver()->connection('default');
|
|
return $main->transaction(function () use (
|
|
$main,
|
|
$claim,
|
|
$baseline,
|
|
$evaluation,
|
|
$now
|
|
): array {
|
|
$instance = $main->table('cloud_instances')
|
|
->where('id', $claim['instance_id'])
|
|
->lockForUpdate()->first();
|
|
if ($instance === null) {
|
|
return ['state' => 'discarded',
|
|
'reason' => 'instance_deleted'];
|
|
}
|
|
if ((int) $instance->user_id !== $claim['user_id']) {
|
|
$this->retireOwnerReadModel(
|
|
$main,
|
|
$claim['instance_id'],
|
|
$claim['user_id'],
|
|
$now
|
|
);
|
|
return ['state' => 'discarded',
|
|
'reason' => 'instance_owner_changed'];
|
|
}
|
|
$receipt = $main->table('cloud_abuse_projection_receipts')
|
|
->where('sample_key', $claim['sample_key'])
|
|
->lockForUpdate()->first();
|
|
if ($receipt !== null) {
|
|
if ((int) $receipt->instance_id !== $claim['instance_id']
|
|
|| (int) $receipt->bucket_start
|
|
!== $claim['bucket_start']
|
|
|| (int) $receipt->rule_version
|
|
!== CloudAbuseDetector::RULE_VERSION) {
|
|
throw new \RuntimeException(
|
|
'Abuse projection receipt identity conflict'
|
|
);
|
|
}
|
|
return ['state' => 'done', 'baseline' => $baseline];
|
|
}
|
|
$state = $main->table('cloud_abuse_instance_states')
|
|
->where('instance_id', $claim['instance_id'])
|
|
->lockForUpdate()->first();
|
|
if ($state !== null && $state->last_bucket_start !== null
|
|
&& (int) $state->last_bucket_start
|
|
>= $claim['bucket_start']) {
|
|
return ['state' => 'discarded',
|
|
'reason' => 'projection_order_conflict'];
|
|
}
|
|
|
|
$healthy = ($evaluation['collection_state'] ?? null)
|
|
=== 'healthy';
|
|
$summary = [
|
|
'rates' => $evaluation['rates'] ?? [],
|
|
'sample_count' => $evaluation['sample_count'] ?? null,
|
|
'warmup_remaining' =>
|
|
$evaluation['warmup_remaining'] ?? null,
|
|
'quality_flags' => $claim['quality_flags'],
|
|
'metrics' => $claim['metrics'],
|
|
];
|
|
$stateValues = [
|
|
'instance_id' => $claim['instance_id'],
|
|
'user_id' => $claim['user_id'],
|
|
'host_id' => (int) $instance->host_id,
|
|
'risk_state' => $healthy
|
|
? (string) $evaluation['risk_state']
|
|
: (string) ($state->risk_state ?? 'observing'),
|
|
'collection_state' => $healthy ? 'healthy' : 'degraded',
|
|
'latest_direction' => $healthy
|
|
? (string) $evaluation['direction']
|
|
: (string) ($state->latest_direction ?? 'none'),
|
|
'latest_rule_code' => $healthy
|
|
? (string) $evaluation['latest_rule_code']
|
|
: (string) ($state->latest_rule_code ?? ''),
|
|
'latest_severity' => $healthy
|
|
? (string) $evaluation['latest_severity']
|
|
: (string) ($state->latest_severity ?? 'none'),
|
|
'summary_json' => self::json($summary),
|
|
'last_projection_key' => $claim['sample_key'],
|
|
'last_bucket_start' => $claim['bucket_start'],
|
|
'last_sample_at' => $claim['bucket_start'] + 60,
|
|
'last_evaluated_at' => $healthy
|
|
? $now : ($state->last_evaluated_at ?? null),
|
|
'updated_at' => $now,
|
|
];
|
|
if ($state === null) {
|
|
$stateValues['created_at'] = $now;
|
|
$main->table('cloud_abuse_instance_states')
|
|
->insert($stateValues);
|
|
} else {
|
|
$main->table('cloud_abuse_instance_states')
|
|
->where('instance_id', $claim['instance_id'])
|
|
->update($stateValues);
|
|
}
|
|
if ($healthy) {
|
|
$this->applyFindings(
|
|
$main,
|
|
$claim,
|
|
(int) $instance->host_id,
|
|
(array) ($evaluation['findings'] ?? []),
|
|
$now
|
|
);
|
|
}
|
|
$main->table('cloud_abuse_projection_receipts')->insert([
|
|
'sample_key' => $claim['sample_key'],
|
|
'instance_id' => $claim['instance_id'],
|
|
'rule_version' => CloudAbuseDetector::RULE_VERSION,
|
|
'bucket_start' => $claim['bucket_start'],
|
|
'projected_at' => $now,
|
|
]);
|
|
return ['state' => 'done', 'baseline' => $baseline];
|
|
}, 3);
|
|
}
|
|
|
|
private function retireOwnerReadModel(
|
|
object $main,
|
|
int $instanceId,
|
|
int $oldUserId,
|
|
int $now
|
|
): void {
|
|
$main->table('cloud_abuse_instance_states')
|
|
->where('instance_id', $instanceId)
|
|
->where('user_id', $oldUserId)
|
|
->delete();
|
|
$findings = $main->table('cloud_abuse_findings')
|
|
->where('instance_id', $instanceId)
|
|
->where('user_id', $oldUserId)
|
|
->where('state', 'open')
|
|
->orderBy('id')->lockForUpdate()->get();
|
|
foreach ($findings as $finding) {
|
|
$resolvedAt = max($now, (int) $finding->last_seen_at);
|
|
$updated = $main->table('cloud_abuse_findings')
|
|
->where('id', (int) $finding->id)
|
|
->where('instance_id', $instanceId)
|
|
->where('user_id', $oldUserId)
|
|
->where('state', 'open')->update([
|
|
'state' => 'resolved',
|
|
'resolved_at' => $resolvedAt,
|
|
'updated_at' => $now,
|
|
]);
|
|
if ($updated !== 1) {
|
|
throw new \RuntimeException(
|
|
'Abuse projection owner cleanup compare-and-swap failed'
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* @return null|array{user_id:int,baseline:array<string,mixed>}
|
|
*/
|
|
private function currentOwnerBaselineSeed(int $instanceId): ?array
|
|
{
|
|
$main = User::getConnectionResolver()->connection('default');
|
|
$instance = $main->table('cloud_instances')
|
|
->where('id', $instanceId)->first(['user_id']);
|
|
if ($instance === null) {
|
|
return null;
|
|
}
|
|
$userId = (int) $instance->user_id;
|
|
$rows = $main->table('cloud_abuse_findings')
|
|
->where('instance_id', $instanceId)
|
|
->where('user_id', $userId)
|
|
->where('rule_version', CloudAbuseDetector::RULE_VERSION)
|
|
->where('state', 'open')
|
|
->orderBy('id')->get([
|
|
'rule_code', 'direction', 'ip_family', 'severity',
|
|
]);
|
|
$findings = [];
|
|
foreach ($rows as $row) {
|
|
$findings[] = [
|
|
'rule_code' => (string) $row->rule_code,
|
|
'direction' => (string) $row->direction,
|
|
'ip_family' => (string) $row->ip_family,
|
|
'severity' => (string) $row->severity,
|
|
];
|
|
}
|
|
return [
|
|
'user_id' => $userId,
|
|
'baseline' => CloudAbuseDetector::resetBaseline($findings),
|
|
];
|
|
}
|
|
|
|
/**
|
|
* @param array<string,mixed> $claim
|
|
* @param array<int,array<string,mixed>> $findings
|
|
*/
|
|
private function applyFindings(
|
|
object $main,
|
|
array $claim,
|
|
int $currentHostId,
|
|
array $findings,
|
|
int $now
|
|
): void {
|
|
foreach ($findings as $finding) {
|
|
$key = hash('sha256', implode("\n", [
|
|
$claim['instance_id'],
|
|
CloudAbuseDetector::RULE_VERSION,
|
|
(string) $finding['rule_code'],
|
|
(string) $finding['ip_family'],
|
|
(string) $finding['direction'],
|
|
]));
|
|
$row = $main->table('cloud_abuse_findings')
|
|
->where('finding_key', $key)->lockForUpdate()->first();
|
|
if (($finding['active'] ?? false) === true) {
|
|
$values = [
|
|
'instance_id' => $claim['instance_id'],
|
|
'user_id' => $claim['user_id'],
|
|
'host_id' => $currentHostId,
|
|
'rule_code' => (string) $finding['rule_code'],
|
|
'rule_version' => CloudAbuseDetector::RULE_VERSION,
|
|
'direction' => (string) $finding['direction'],
|
|
'ip_family' => (string) $finding['ip_family'],
|
|
'severity' => (string) $finding['severity'],
|
|
'state' => 'open',
|
|
'first_seen_at' => $row === null
|
|
|| (string) $row->state === 'resolved'
|
|
? $claim['bucket_start']
|
|
: (int) $row->first_seen_at,
|
|
'last_seen_at' => $claim['bucket_start'],
|
|
'consecutive_windows' => max(
|
|
1,
|
|
(int) $finding['consecutive_windows']
|
|
),
|
|
'evidence_json' => self::json(
|
|
(array) $finding['evidence']
|
|
),
|
|
'resolved_at' => null,
|
|
'updated_at' => $now,
|
|
];
|
|
if ($row === null) {
|
|
$values['finding_key'] = $key;
|
|
$values['created_at'] = $now;
|
|
$main->table('cloud_abuse_findings')->insert($values);
|
|
} else {
|
|
$main->table('cloud_abuse_findings')
|
|
->where('finding_key', $key)->update($values);
|
|
}
|
|
} elseif (($finding['transition'] ?? '') === 'resolved'
|
|
&& $row !== null && (string) $row->state === 'open') {
|
|
$main->table('cloud_abuse_findings')
|
|
->where('finding_key', $key)
|
|
->where('state', 'open')->update([
|
|
'state' => 'resolved',
|
|
'resolved_at' => $claim['bucket_start'],
|
|
'updated_at' => $now,
|
|
]);
|
|
}
|
|
}
|
|
}
|
|
|
|
/** @param array<string,mixed> $claim @param array<string,mixed> $baseline */
|
|
private function finalize(array $claim, array $baseline, int $now): void
|
|
{
|
|
$log = User::getConnectionResolver()->connection('log');
|
|
$log->transaction(function () use (
|
|
$log,
|
|
$claim,
|
|
$baseline,
|
|
$now
|
|
): void {
|
|
$base = $log->table('cloud_vm_network_baselines')
|
|
->where('instance_id', $claim['instance_id'])
|
|
->lockForUpdate()->first();
|
|
$outbox = $log->table('cloud_vm_network_projection_outbox')
|
|
->where('sample_key', $claim['sample_key'])
|
|
->lockForUpdate()->first();
|
|
if ($base === null || $outbox === null
|
|
|| (string) $base->projection_lease_token
|
|
!== $claim['lease_token']
|
|
|| (string) $outbox->lease_token !== $claim['lease_token']
|
|
|| (string) $outbox->state !== 'leased') {
|
|
throw new \RuntimeException(
|
|
'Abuse projection lease was lost before finalization'
|
|
);
|
|
}
|
|
$sampleCount = max(0, (int) ($baseline['sample_count'] ?? 0));
|
|
$baselineUpdated = $log->table('cloud_vm_network_baselines')
|
|
->where('instance_id', $claim['instance_id'])
|
|
->where('revision', $claim['baseline_revision'])
|
|
->where('projection_lease_token', $claim['lease_token'])
|
|
->update([
|
|
'sample_count' => $sampleCount,
|
|
'warmup_state' => $sampleCount
|
|
>= CloudAbuseDetector::WARMUP_WINDOWS
|
|
? 'ready' : 'warming',
|
|
'metrics_json' => self::json($baseline),
|
|
'last_projection_key' => $claim['sample_key'],
|
|
'last_bucket_start' => $claim['bucket_start'],
|
|
'projection_lease_token' => null,
|
|
'projection_lease_until' => 0,
|
|
'revision' => $claim['baseline_revision'] + 1,
|
|
'updated_at' => $now,
|
|
]);
|
|
$updated = $log->table('cloud_vm_network_projection_outbox')
|
|
->where('sample_key', $claim['sample_key'])
|
|
->where('state', 'leased')
|
|
->where('lease_token', $claim['lease_token'])
|
|
->update([
|
|
'state' => 'done',
|
|
'lease_token' => null,
|
|
'lease_until' => 0,
|
|
'last_error_code' => '',
|
|
'projected_at' => $now,
|
|
'terminal_at' => $now,
|
|
'discard_reason_code' => '',
|
|
'updated_at' => $now,
|
|
]);
|
|
if ($baselineUpdated !== 1 || $updated !== 1) {
|
|
throw new \RuntimeException(
|
|
'Abuse projection finalization compare-and-swap failed'
|
|
);
|
|
}
|
|
}, 3);
|
|
}
|
|
|
|
/** @param array<string,mixed> $claim */
|
|
private function discard(array $claim, string $reason, int $now): void
|
|
{
|
|
if (preg_match('/^[a-z0-9_]{3,64}$/D', $reason) !== 1) {
|
|
$reason = 'projection_discarded';
|
|
}
|
|
$log = User::getConnectionResolver()->connection('log');
|
|
$log->transaction(function () use ($log, $claim, $reason, $now): void {
|
|
if ($reason === 'instance_owner_changed') {
|
|
$released = $log->table('cloud_vm_network_baselines')
|
|
->where('instance_id', $claim['instance_id'])
|
|
->where('user_id', $claim['user_id'])
|
|
->where('revision', $claim['baseline_revision'])
|
|
->where('projection_lease_token', $claim['lease_token'])
|
|
->delete();
|
|
} else {
|
|
$released = $this->releaseBaselineLease(
|
|
$log,
|
|
$claim,
|
|
$now
|
|
);
|
|
}
|
|
$updated = $log->table('cloud_vm_network_projection_outbox')
|
|
->where('sample_key', $claim['sample_key'])
|
|
->where('state', 'leased')
|
|
->where('lease_token', $claim['lease_token'])
|
|
->update([
|
|
'state' => 'discarded',
|
|
'lease_token' => null,
|
|
'lease_until' => 0,
|
|
'last_error_code' => '',
|
|
'projected_at' => null,
|
|
'terminal_at' => $now,
|
|
'discard_reason_code' => $reason,
|
|
'updated_at' => $now,
|
|
]);
|
|
if ($released !== 1 || $updated !== 1) {
|
|
throw new \RuntimeException(
|
|
'Abuse projection discard finalization failed'
|
|
);
|
|
}
|
|
}, 3);
|
|
}
|
|
|
|
/** @param array<string,mixed> $claim */
|
|
private function retryOrDiscard(
|
|
array $claim,
|
|
\Throwable $error,
|
|
int $now
|
|
): string {
|
|
$code = self::boundedCode(
|
|
$error instanceof CloudAgentError
|
|
? $error->agentCode : 'projection_failed',
|
|
'projection_failed'
|
|
);
|
|
$terminal = $claim['attempts'] >= self::MAX_ATTEMPTS;
|
|
if ($terminal) {
|
|
$receiptState = $this->receiptState($claim);
|
|
if ($receiptState === 'matching') {
|
|
$terminal = false;
|
|
$code = 'projection_receipt_finalize_pending';
|
|
} elseif ($receiptState === 'conflict') {
|
|
$terminal = false;
|
|
$code = 'projection_receipt_conflict';
|
|
} elseif ($receiptState === 'unknown') {
|
|
$terminal = false;
|
|
$code = 'projection_receipt_check_failed';
|
|
}
|
|
}
|
|
$log = User::getConnectionResolver()->connection('log');
|
|
$log->transaction(function () use (
|
|
$log,
|
|
$claim,
|
|
$now,
|
|
$code,
|
|
$terminal
|
|
): void {
|
|
$this->releaseBaselineLease($log, $claim, $now);
|
|
$values = $terminal ? [
|
|
'state' => 'discarded',
|
|
'lease_token' => null,
|
|
'lease_until' => 0,
|
|
'last_error_code' => $code,
|
|
'projected_at' => null,
|
|
'terminal_at' => $now,
|
|
'discard_reason_code' => 'projection_retry_exhausted',
|
|
'updated_at' => $now,
|
|
] : [
|
|
'state' => 'pending',
|
|
'available_at' => $now
|
|
+ self::retryDelay($claim['attempts']),
|
|
'lease_token' => null,
|
|
'lease_until' => 0,
|
|
'last_error_code' => $code,
|
|
'projected_at' => null,
|
|
'terminal_at' => null,
|
|
'discard_reason_code' => '',
|
|
'updated_at' => $now,
|
|
];
|
|
$log->table('cloud_vm_network_projection_outbox')
|
|
->where('sample_key', $claim['sample_key'])
|
|
->where('state', 'leased')
|
|
->where('lease_token', $claim['lease_token'])
|
|
->update($values);
|
|
}, 3);
|
|
return $code;
|
|
}
|
|
|
|
/** @param array<string,mixed> $claim */
|
|
private function receiptState(array $claim): string
|
|
{
|
|
try {
|
|
$receipt = User::getConnectionResolver()
|
|
->connection('default')
|
|
->table('cloud_abuse_projection_receipts')
|
|
->where('sample_key', $claim['sample_key'])->first();
|
|
} catch (\Throwable) {
|
|
return 'unknown';
|
|
}
|
|
if ($receipt === null) {
|
|
return 'absent';
|
|
}
|
|
if ((int) $receipt->instance_id !== $claim['instance_id']
|
|
|| (int) $receipt->bucket_start !== $claim['bucket_start']
|
|
|| (int) $receipt->rule_version
|
|
!== CloudAbuseDetector::RULE_VERSION
|
|
|| (int) $receipt->projected_at < 1) {
|
|
return 'conflict';
|
|
}
|
|
return 'matching';
|
|
}
|
|
|
|
/** @param array<string,mixed> $claim */
|
|
private function releaseBaselineLease(
|
|
object $log,
|
|
array $claim,
|
|
int $now
|
|
): int {
|
|
return $log->table('cloud_vm_network_baselines')
|
|
->where('instance_id', $claim['instance_id'])
|
|
->where('revision', $claim['baseline_revision'])
|
|
->where('projection_lease_token', $claim['lease_token'])
|
|
->update([
|
|
'projection_lease_token' => null,
|
|
'projection_lease_until' => 0,
|
|
'updated_at' => $now,
|
|
]);
|
|
}
|
|
|
|
/** @return array<string,mixed> */
|
|
private static function decodeObject(string $json, string $name): array
|
|
{
|
|
try {
|
|
$value = json_decode($json, true, 16, JSON_THROW_ON_ERROR);
|
|
} catch (\JsonException $error) {
|
|
throw new \RuntimeException("Invalid {$name}", 0, $error);
|
|
}
|
|
if (!is_array($value) || $value === [] || array_is_list($value)) {
|
|
throw new \RuntimeException("Invalid {$name}");
|
|
}
|
|
return $value;
|
|
}
|
|
|
|
/** @return array<int,mixed> */
|
|
private static function decodeList(string $json, string $name): array
|
|
{
|
|
try {
|
|
$value = json_decode($json, true, 8, JSON_THROW_ON_ERROR);
|
|
} catch (\JsonException $error) {
|
|
throw new \RuntimeException("Invalid {$name}", 0, $error);
|
|
}
|
|
if (!is_array($value) || !array_is_list($value)) {
|
|
throw new \RuntimeException("Invalid {$name}");
|
|
}
|
|
return $value;
|
|
}
|
|
|
|
/** @param array<string,mixed> $metrics @return array<string,mixed> */
|
|
private static function canonicalMetrics(array $metrics): array
|
|
{
|
|
self::exactKeys($metrics, ['interface', 'ipv4', 'ipv6'],
|
|
'sample metrics');
|
|
$interfaceFields = [
|
|
'rx_bytes', 'tx_bytes', 'rx_packets', 'tx_packets',
|
|
'rx_errors', 'tx_errors', 'rx_drops', 'tx_drops',
|
|
];
|
|
if (!is_array($metrics['interface'])
|
|
|| array_is_list($metrics['interface'])) {
|
|
throw new \RuntimeException('Invalid sample interface metrics');
|
|
}
|
|
self::exactKeys($metrics['interface'], $interfaceFields,
|
|
'sample interface metrics');
|
|
$interface = [];
|
|
foreach ($interfaceFields as $field) {
|
|
$value = $metrics['interface'][$field];
|
|
if ($value !== null && (!is_int($value) || $value < 0)) {
|
|
throw new \RuntimeException(
|
|
'Invalid sample interface counter'
|
|
);
|
|
}
|
|
$interface[$field] = $value;
|
|
}
|
|
return [
|
|
'interface' => $interface,
|
|
'ipv4' => self::canonicalFamily($metrics['ipv4'], 'ipv4'),
|
|
'ipv6' => self::canonicalFamily($metrics['ipv6'], 'ipv6'),
|
|
];
|
|
}
|
|
|
|
private static function canonicalFamily(mixed $family, string $name): ?array
|
|
{
|
|
if ($family === null) {
|
|
return null;
|
|
}
|
|
if (!is_array($family) || array_is_list($family)) {
|
|
throw new \RuntimeException("Invalid {$name} sample metrics");
|
|
}
|
|
self::exactKeys($family, ['inbound', 'outbound'],
|
|
"{$name} sample metrics");
|
|
return [
|
|
'inbound' => self::canonicalFlow(
|
|
$family['inbound'],
|
|
"{$name} inbound"
|
|
),
|
|
'outbound' => self::canonicalFlow(
|
|
$family['outbound'],
|
|
"{$name} outbound"
|
|
),
|
|
];
|
|
}
|
|
|
|
/** @return array<string,int> */
|
|
private static function canonicalFlow(mixed $flow, string $name): array
|
|
{
|
|
$fields = [
|
|
'tcp_active_flows', 'tcp_new_flows', 'tcp_syn_flows',
|
|
'tcp_unreplied_flows', 'udp_active_flows', 'udp_new_flows',
|
|
'other_active_flows', 'unique_remote_ips',
|
|
'unique_remote_ports', 'unique_local_ports',
|
|
'max_remote_ports_per_remote_ip',
|
|
'max_remote_ips_per_remote_port',
|
|
'max_local_ports_per_remote_ip',
|
|
'max_remote_ips_per_local_port',
|
|
];
|
|
if (!is_array($flow) || array_is_list($flow)) {
|
|
throw new \RuntimeException("Invalid {$name} flow metrics");
|
|
}
|
|
self::exactKeys($flow, $fields, "{$name} flow metrics");
|
|
$result = [];
|
|
foreach ($fields as $field) {
|
|
if (!is_int($flow[$field]) || $flow[$field] < 0) {
|
|
throw new \RuntimeException("Invalid {$name} flow counter");
|
|
}
|
|
$result[$field] = $flow[$field];
|
|
}
|
|
return $result;
|
|
}
|
|
|
|
/** @param array<int,mixed> $quality @return array<int,string> */
|
|
private static function canonicalQuality(array $quality): array
|
|
{
|
|
$allowed = [
|
|
'clock_regression', 'cold_start', 'conntrack_unavailable',
|
|
'counter_reset', 'flow_limit_reached', 'incomplete',
|
|
'interface_unavailable', 'local_port_cardinality_saturated',
|
|
'mapping_changed', 'overflow', 'packet_accounting_unavailable',
|
|
'panel_ring_gap', 'relation_cardinality_saturated',
|
|
'remote_ip_cardinality_saturated',
|
|
'remote_port_cardinality_saturated', 'sampling_gap',
|
|
];
|
|
if (count($quality) > 17) {
|
|
throw new \RuntimeException('Too many sample quality flags');
|
|
}
|
|
foreach ($quality as $flag) {
|
|
if (!is_string($flag) || !in_array($flag, $allowed, true)) {
|
|
throw new \RuntimeException('Invalid sample quality flag');
|
|
}
|
|
}
|
|
$canonical = array_values(array_unique($quality));
|
|
sort($canonical, SORT_STRING);
|
|
if ($canonical !== $quality) {
|
|
throw new \RuntimeException(
|
|
'Sample quality flags are not canonical'
|
|
);
|
|
}
|
|
return $quality;
|
|
}
|
|
|
|
/** @param array<string,mixed> $value @param array<int,string> $keys */
|
|
private static function exactKeys(
|
|
array $value,
|
|
array $keys,
|
|
string $name
|
|
): void {
|
|
$actual = array_keys($value);
|
|
sort($actual, SORT_STRING);
|
|
$expected = $keys;
|
|
sort($expected, SORT_STRING);
|
|
if ($actual !== $expected) {
|
|
throw new \RuntimeException("Invalid {$name} shape");
|
|
}
|
|
}
|
|
|
|
private static function boundedCode(string $code, string $fallback): string
|
|
{
|
|
return preg_match('/^[a-z0-9_]{3,64}$/D', $code) === 1
|
|
? $code : $fallback;
|
|
}
|
|
|
|
private static function retryDelay(int $attempts): int
|
|
{
|
|
return min(
|
|
self::MAX_BACKOFF_SECONDS,
|
|
15 * (2 ** min(6, max(0, $attempts - 1)))
|
|
);
|
|
}
|
|
|
|
private static function json(array $value): string
|
|
{
|
|
return json_encode(
|
|
$value,
|
|
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
);
|
|
}
|
|
}
|