You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudAbuseProjectionService...

1388 lines
57 KiB
PHTML

<?php
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Services\Cloud\CloudAbuseDetector;
use App\Services\Cloud\CloudAbuseDetectionPolicy;
use App\Services\Cloud\CloudAbusePolicySnapshot;
use App\Services\Cloud\CloudAbuseProjectionService;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$capsule = new Capsule();
$capsule->addConnection([
'driver' => 'sqlite', 'database' => ':memory:',
], 'default');
$capsule->addConnection([
'driver' => 'sqlite', 'database' => ':memory:',
], 'log');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$mainSchema = $capsule->schema('default');
$mainSchema->create('user', static function (Blueprint $table): void {
$table->increments('id');
});
$mainSchema->create('cloud_instances', static function (Blueprint $table): void {
$table->increments('id');
$table->integer('host_id');
$table->integer('user_id');
});
$mainSchema->create('cloud_abuse_instance_states',
static function (Blueprint $table): void {
$table->integer('instance_id')->primary();
$table->integer('user_id');
$table->integer('host_id');
$table->integer('rule_version')->default(1);
$table->integer('policy_revision')->default(1);
$table->string('policy_hash')->default(
'511828bde950c504f92bb7a0603ca8b14453f575ead276c1889baa764f384dae'
);
$table->string('risk_state');
$table->string('collection_state');
$table->string('latest_direction');
$table->string('latest_rule_code');
$table->string('latest_severity');
$table->text('summary_json');
$table->string('last_projection_key')->nullable();
$table->integer('last_bucket_start')->nullable();
$table->integer('last_sample_at')->nullable();
$table->integer('last_evaluated_at')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$mainSchema->create('cloud_abuse_findings',
static function (Blueprint $table): void {
$table->increments('id');
$table->string('finding_key')->unique();
$table->integer('instance_id');
$table->integer('user_id');
$table->integer('host_id');
$table->string('rule_code');
$table->integer('rule_version');
$table->integer('policy_revision')->default(1);
$table->string('policy_hash')->default(
'511828bde950c504f92bb7a0603ca8b14453f575ead276c1889baa764f384dae'
);
$table->string('direction');
$table->string('ip_family');
$table->string('severity');
$table->string('state');
$table->integer('first_seen_at');
$table->integer('last_seen_at');
$table->integer('consecutive_windows');
$table->integer('occurrence_generation')->default(1);
$table->integer('transition_revision')->default(1);
$table->text('evidence_json');
$table->integer('resolved_at')->nullable();
$table->string('resolution_reason')->default('');
$table->integer('created_at');
$table->integer('updated_at');
});
$mainSchema->create('cloud_abuse_alert_events',
static function (Blueprint $table): void {
$table->increments('id');
$table->string('event_key')->unique();
$table->string('sample_key')->unique();
$table->integer('instance_id');
$table->integer('user_id');
$table->integer('host_id');
$table->integer('bucket_start');
$table->integer('rule_version');
$table->integer('policy_revision');
$table->string('policy_hash');
$table->string('transition_digest');
$table->integer('transition_count');
$table->string('highest_severity');
$table->integer('created_at');
});
$mainSchema->create('cloud_abuse_alert_event_items',
static function (Blueprint $table): void {
$table->increments('id');
$table->integer('event_id');
$table->string('item_key')->unique();
$table->string('finding_key');
$table->integer('occurrence_generation');
$table->integer('transition_revision');
$table->string('rule_code');
$table->string('direction');
$table->string('ip_family');
$table->string('transition_type');
$table->string('from_severity');
$table->string('to_severity');
$table->text('evidence_json');
$table->string('resolution_reason')->default('');
$table->integer('occurred_at');
$table->integer('created_at');
});
$mainSchema->create('cloud_abuse_projection_receipts',
static function (Blueprint $table): void {
$table->string('sample_key')->primary();
$table->integer('instance_id');
$table->integer('rule_version');
$table->integer('policy_revision')->default(1);
$table->string('policy_hash')->default(
'511828bde950c504f92bb7a0603ca8b14453f575ead276c1889baa764f384dae'
);
$table->integer('bucket_start');
$table->integer('projected_at');
});
$logSchema = $capsule->schema('log');
$logSchema->create('cloud_vm_network_observation_samples',
static function (Blueprint $table): void {
$table->string('sample_key')->primary();
$table->integer('instance_id');
$table->integer('user_id');
$table->integer('host_id');
$table->string('vm_uuid');
$table->string('placement_key');
$table->string('collector_generation');
$table->string('window_id');
$table->integer('contract_version');
$table->integer('bucket_start');
$table->integer('bucket_seconds');
$table->integer('complete');
$table->text('metrics_json');
$table->text('quality_flags_json');
$table->integer('received_at');
});
$logSchema->create('cloud_vm_network_projection_outbox',
static function (Blueprint $table): void {
$table->string('sample_key')->primary();
$table->integer('instance_id');
$table->integer('bucket_start');
$table->string('placement_key');
$table->integer('rule_version')->default(1);
$table->integer('policy_revision')->default(1);
$table->string('policy_hash')->default(
'511828bde950c504f92bb7a0603ca8b14453f575ead276c1889baa764f384dae'
);
$table->string('state');
$table->integer('attempts')->default(0);
$table->integer('available_at');
$table->string('lease_token')->nullable();
$table->integer('lease_until')->default(0);
$table->string('last_error_code')->default('');
$table->integer('projected_at')->nullable();
$table->integer('terminal_at')->nullable();
$table->string('discard_reason_code')->default('');
$table->integer('created_at');
$table->integer('updated_at');
});
$logSchema->create('cloud_vm_network_baselines',
static function (Blueprint $table): void {
$table->integer('instance_id')->primary();
$table->integer('user_id');
$table->integer('rule_version');
$table->integer('policy_revision')->default(1);
$table->string('policy_hash')->default(
'511828bde950c504f92bb7a0603ca8b14453f575ead276c1889baa764f384dae'
);
$table->integer('sample_count');
$table->string('warmup_state');
$table->text('metrics_json');
$table->string('last_projection_key')->nullable();
$table->integer('last_bucket_start')->nullable();
$table->string('projection_lease_token')->nullable();
$table->integer('projection_lease_until')->default(0);
$table->integer('revision')->default(0);
$table->integer('created_at');
$table->integer('updated_at');
});
$main = $capsule->connection('default');
$log = $capsule->connection('log');
$now = 1000;
$defaultPolicy = CloudAbuseDetectionPolicy::defaults();
$policySnapshot = new CloudAbusePolicySnapshot(
1,
$defaultPolicy->hash(),
$defaultPolicy,
null,
0
);
$changedDefinition = $defaultPolicy->definition();
$changedDefinition['windows']['suspicious'] = 4;
$changedDefinition['rules']['outbound_udp_fanout']['enabled'] = false;
$changedPolicy = CloudAbuseDetectionPolicy::fromDefinition(
$changedDefinition
);
$changedPolicySnapshot = new CloudAbusePolicySnapshot(
2,
$changedPolicy->hash(),
$changedPolicy,
7,
900
);
$sameDetectorSnapshot = new CloudAbusePolicySnapshot(
3,
$defaultPolicy->hash(),
$defaultPolicy,
7,
950
);
$policySnapshots = [
'1:' . $policySnapshot->policyHash => $policySnapshot,
'2:' . $changedPolicySnapshot->policyHash => $changedPolicySnapshot,
'3:' . $sameDetectorSnapshot->policyHash => $sameDetectorSnapshot,
];
$uuid = static fn (int $id): string => sprintf(
'123e4567-e89b-42d3-a456-%012d',
$id
);
$addInstance = static function (int $id, int $user, int $host = 7) use (
$main
): void {
if (!$main->table('user')->where('id', $user)->exists()) {
$main->table('user')->insert(['id' => $user]);
}
$main->table('cloud_instances')->insert([
'id' => $id, 'host_id' => $host, 'user_id' => $user,
]);
};
$flow = static fn (array $values = []): array => array_merge([
'tcp_active_flows' => 0, 'tcp_new_flows' => 0,
'tcp_syn_flows' => 0, 'tcp_unreplied_flows' => 0,
'udp_active_flows' => 0, 'udp_new_flows' => 0,
'other_active_flows' => 0, 'unique_remote_ips' => 0,
'unique_remote_ports' => 0, 'unique_local_ports' => 0,
'max_remote_ports_per_remote_ip' => 0,
'max_remote_ips_per_remote_port' => 0,
'max_local_ports_per_remote_ip' => 0,
'max_remote_ips_per_local_port' => 0,
], $values);
$metrics = static function (array $outbound = [], array $inbound = []) use (
$flow
): array {
return [
'interface' => [
'rx_bytes' => 1000, 'tx_bytes' => 2000,
'rx_packets' => 10, 'tx_packets' => 20,
'rx_errors' => 0, 'tx_errors' => 0,
'rx_drops' => 0, 'tx_drops' => 0,
],
'ipv4' => ['inbound' => $flow($inbound),
'outbound' => $flow($outbound)],
'ipv6' => ['inbound' => $flow(), 'outbound' => $flow()],
];
};
$insertBaseline = static function (
int $instanceId,
int $userId,
array $baseline,
?int $lastBucket = null,
?string $lastKey = null,
?string $lease = null,
int $leaseUntil = 0,
int $revision = 0,
?CloudAbusePolicySnapshot $snapshot = null
) use ($log, &$now, $policySnapshot): void {
$snapshot ??= $policySnapshot;
$log->table('cloud_vm_network_baselines')->insert([
'instance_id' => $instanceId,
'user_id' => $userId,
'rule_version' => $snapshot->policy->engineVersion(),
'policy_revision' => $snapshot->revision,
'policy_hash' => $snapshot->policyHash,
'sample_count' => (int) ($baseline['sample_count'] ?? 0),
'warmup_state' => (int) ($baseline['sample_count'] ?? 0)
>= CloudAbuseDetector::WARMUP_WINDOWS ? 'ready' : 'warming',
'metrics_json' => json_encode($baseline, JSON_THROW_ON_ERROR),
'last_projection_key' => $lastKey,
'last_bucket_start' => $lastBucket,
'projection_lease_token' => $lease,
'projection_lease_until' => $leaseUntil,
'revision' => $revision,
'created_at' => $now,
'updated_at' => $now,
]);
};
$insertSample = static function (
int $instanceId,
int $userId,
int $bucket,
array $sampleMetrics,
bool $complete = true,
array $quality = [],
string $state = 'pending',
?string $lease = null,
int $leaseUntil = 0,
int $attempts = 0,
?CloudAbusePolicySnapshot $snapshot = null
) use ($log, $uuid, &$now, $policySnapshot): string {
$snapshot ??= $policySnapshot;
$key = hash('sha256', $instanceId . "\n" . $bucket);
$placement = hash('sha256', 'placement-' . $instanceId);
$log->table('cloud_vm_network_observation_samples')->insert([
'sample_key' => $key,
'instance_id' => $instanceId,
'user_id' => $userId,
'host_id' => 7,
'vm_uuid' => $uuid($instanceId),
'placement_key' => $placement,
'collector_generation' => str_repeat('a', 32),
'window_id' => substr($key, 0, 32),
'contract_version' => 1,
'bucket_start' => $bucket,
'bucket_seconds' => 60,
'complete' => $complete ? 1 : 0,
'metrics_json' => json_encode($sampleMetrics,
JSON_THROW_ON_ERROR),
'quality_flags_json' => json_encode($quality,
JSON_THROW_ON_ERROR),
'received_at' => $now,
]);
$log->table('cloud_vm_network_projection_outbox')->insert([
'sample_key' => $key,
'instance_id' => $instanceId,
'bucket_start' => $bucket,
'placement_key' => $placement,
'rule_version' => $snapshot->policy->engineVersion(),
'policy_revision' => $snapshot->revision,
'policy_hash' => $snapshot->policyHash,
'state' => $state,
'attempts' => $attempts,
'available_at' => $now,
'lease_token' => $lease,
'lease_until' => $leaseUntil,
'last_error_code' => '',
'projected_at' => null,
'terminal_at' => null,
'discard_reason_code' => '',
'created_at' => $now,
'updated_at' => $now,
]);
return $key;
};
$worker = static function (?callable $hook = null) use (
&$now,
&$policySnapshots
): object {
return new CloudAbuseProjectionService(
static function () use (&$now): int { return $now; },
$hook,
static function (int $revision, string $hash) use (
&$policySnapshots
): CloudAbusePolicySnapshot {
$key = $revision . ':' . $hash;
if (!isset($policySnapshots[$key])) {
throw new RuntimeException('Unknown frozen policy identity');
}
return $policySnapshots[$key];
}
);
};
// Normal success atomically creates the current state, receipt and log finality.
$addInstance(1, 11);
$key1 = $insertSample(1, 11, 120, $metrics());
$successWorker = $worker();
$assert($successWorker->tick(), 'A due projection was not claimed.');
$assert($log->table('cloud_vm_network_projection_outbox')
->where('sample_key', $key1)->value('state') === 'done'
&& $main->table('cloud_abuse_projection_receipts')
->where('sample_key', $key1)->count() === 1
&& $main->table('cloud_abuse_instance_states')
->where('instance_id', 1)->value('risk_state') === 'observing'
&& (int) $log->table('cloud_vm_network_baselines')
->where('instance_id', 1)->value('sample_count') === 1,
'Successful projection did not converge both databases.');
$assert($successWorker->observationErrors() === [],
'Successful projection retained a stale worker error.');
// Crash after the main commit must replay from its receipt without duplicating.
$addInstance(2, 22);
$key2 = $insertSample(2, 22, 180, $metrics());
$crashed = false;
$worker(static function () use (&$crashed): void {
if (!$crashed) {
$crashed = true;
throw new RuntimeException('simulated cross-database crash');
}
})->tick();
$assert($main->table('cloud_abuse_projection_receipts')
->where('sample_key', $key2)->count() === 1
&& $log->table('cloud_vm_network_projection_outbox')
->where('sample_key', $key2)->value('state') === 'pending'
&& (int) $log->table('cloud_vm_network_baselines')
->where('instance_id', 2)->value('sample_count') === 0,
'Simulated crash did not preserve the explicit replay seam.');
$now += 15;
$assert($worker()->tick(), 'Receipt-backed projection did not replay.');
$assert($log->table('cloud_vm_network_projection_outbox')
->where('sample_key', $key2)->value('state') === 'done'
&& $main->table('cloud_abuse_projection_receipts')
->where('sample_key', $key2)->count() === 1
&& (int) $log->table('cloud_vm_network_baselines')
->where('instance_id', 2)->value('sample_count') === 1
&& (int) $main->table('cloud_abuse_projection_receipts')
->where('sample_key', $key2)->value('policy_revision') === 1
&& (int) $main->table('cloud_abuse_instance_states')
->where('instance_id', 2)->value('policy_revision') === 1,
'Receipt replay duplicated or failed to finalize projection.');
// A committed main receipt is authoritative even when the cross-database
// finalization seam keeps failing beyond the ordinary retry budget.
$addInstance(12, 122);
$receiptKey = $insertSample(12, 122, 240, $metrics());
$finalizeFailure = static function (): void {
throw new RuntimeException('simulated repeated finalization failure');
};
for ($attempt = 1; $attempt <= 8; ++$attempt) {
$failingWorker = $worker($finalizeFailure);
$assert($failingWorker->tick(),
'Receipt-backed retry was not consumed.');
$receiptOutbox = $log->table(
'cloud_vm_network_projection_outbox'
)->where('sample_key', $receiptKey)->first();
$assert((int) $receiptOutbox->attempts === $attempt,
'Receipt-backed retry count was not durable.');
$expectedError = $attempt === 8
? 'projection_receipt_finalize_pending' : 'projection_failed';
$assert($failingWorker->observationErrors() === [$expectedError],
'Receipt-backed retry exposed an unstable worker error.');
$now += 1000;
}
$receiptOutbox = $log->table('cloud_vm_network_projection_outbox')
->where('sample_key', $receiptKey)->first();
$assert($receiptOutbox->state === 'pending'
&& $receiptOutbox->discard_reason_code === ''
&& $main->table('cloud_abuse_projection_receipts')
->where('sample_key', $receiptKey)->count() === 1
&& (int) $log->table('cloud_vm_network_baselines')
->where('instance_id', 12)->value('sample_count') === 0,
'Committed receipt was discarded after finalization retries.');
$receiptRecoveryWorker = $worker();
$assert($receiptRecoveryWorker->tick(),
'Receipt-backed recovery was not consumed.');
$assert($log->table('cloud_vm_network_projection_outbox')
->where('sample_key', $receiptKey)->value('state') === 'done'
&& (int) $log->table('cloud_vm_network_baselines')
->where('instance_id', 12)->value('sample_count') === 1
&& $main->table('cloud_abuse_projection_receipts')
->where('sample_key', $receiptKey)->count() === 1,
'Recovered finalization did not converge from the committed receipt.');
$assert($receiptRecoveryWorker->observationErrors() === [],
'Successful receipt recovery retained a worker error.');
// A matching receipt also protects the earlier claim/ledger-failure path.
// Because damaged identity evidence cannot reproduce the exact baseline, the
// safe convergence is to finish the already-applied outbox and warm up again.
$addInstance(13, 133);
$claimReceiptKey = $insertSample(
13,
133,
300,
$metrics(),
true,
[],
'pending',
null,
0,
7
);
$claimResetFindingKey = hash('sha256', implode("\n", [
13, CloudAbuseDetector::RULE_VERSION,
$policySnapshot->policyHash,
'outbound_tcp_syn_sweep', 'ipv4', 'outbound',
]));
$main->table('cloud_abuse_findings')->insert([
'finding_key' => $claimResetFindingKey,
'instance_id' => 13, 'user_id' => 133, 'host_id' => 7,
'rule_code' => 'outbound_tcp_syn_sweep',
'rule_version' => CloudAbuseDetector::RULE_VERSION,
'direction' => 'outbound', 'ip_family' => 'ipv4',
'severity' => 'high', 'state' => 'open',
'first_seen_at' => 120, 'last_seen_at' => 240,
'consecutive_windows' => 2, 'evidence_json' => '{}',
'resolved_at' => null, 'created_at' => $now, 'updated_at' => $now,
]);
$main->table('cloud_abuse_projection_receipts')->insert([
'sample_key' => $claimReceiptKey,
'instance_id' => 13,
'rule_version' => CloudAbuseDetector::RULE_VERSION,
'bucket_start' => 300,
'projected_at' => $now,
]);
$log->table('cloud_vm_network_projection_outbox')
->where('sample_key', $claimReceiptKey)->update([
'placement_key' => str_repeat('1', 64),
]);
$claimReceiptWorker = $worker();
$assert($claimReceiptWorker->tick(),
'Receipt-backed claim recovery was not consumed.');
$claimReceiptOutbox = $log->table('cloud_vm_network_projection_outbox')
->where('sample_key', $claimReceiptKey)->first();
$assert($claimReceiptOutbox->state === 'done'
&& $claimReceiptOutbox->discard_reason_code === ''
&& (int) $claimReceiptOutbox->attempts === 8
&& $log->table('cloud_vm_network_baselines')
->where('instance_id', 13)->count() === 1
&& (int) $log->table('cloud_vm_network_baselines')
->where('instance_id', 13)->value('sample_count') === 0,
'Matching claim receipt was discarded or left a divergent baseline.');
$assert($claimReceiptWorker->observationErrors()
=== ['projection_receipt_baseline_reset'],
'Receipt-backed claim recovery exposed no bounded diagnostic.');
$claimNextKey = $insertSample(13, 133, 360, $metrics());
$assert($worker()->tick()
&& $log->table('cloud_vm_network_projection_outbox')
->where('sample_key', $claimNextKey)->value('state') === 'done'
&& (int) $log->table('cloud_vm_network_baselines')
->where('instance_id', 13)->value('sample_count') === 1,
'Receipt-backed claim recovery did not restart safe warm-up.');
$assert($main->table('cloud_abuse_findings')
->where('finding_key', $claimResetFindingKey)->value('state')
=== 'open',
'Receipt-backed reset cleared an open finding without evidence.');
for ($step = 1; $step <= 63; ++$step) {
$insertSample(13, 133, 360 + ($step * 60), $metrics());
$assert($worker()->tick(),
'Receipt-reset clear convergence did not run.');
}
$assert($main->table('cloud_abuse_findings')
->where('finding_key', $claimResetFindingKey)->value('state')
=== 'resolved'
&& $main->table('cloud_abuse_instance_states')
->where('instance_id', 13)->value('risk_state') === 'normal',
'Receipt reset left an open finding after verified clear evidence.');
// A conflicting receipt is never treated as proof that the main effect did
// not happen. It remains attributable and backed off for operator repair.
$addInstance(14, 144);
$claimConflictKey = $insertSample(
14,
144,
300,
$metrics(),
true,
[],
'pending',
null,
0,
7
);
$main->table('cloud_abuse_projection_receipts')->insert([
'sample_key' => $claimConflictKey,
'instance_id' => 14,
'rule_version' => CloudAbuseDetector::RULE_VERSION,
'policy_revision' => 2,
'policy_hash' => $changedPolicySnapshot->policyHash,
'bucket_start' => 300,
'projected_at' => $now,
]);
$log->table('cloud_vm_network_projection_outbox')
->where('sample_key', $claimConflictKey)->update([
'placement_key' => str_repeat('2', 64),
]);
$claimConflictWorker = $worker();
$assert($claimConflictWorker->tick(),
'Conflicting claim receipt was not inspected.');
$claimConflictOutbox = $log->table('cloud_vm_network_projection_outbox')
->where('sample_key', $claimConflictKey)->first();
$assert($claimConflictOutbox->state === 'pending'
&& (int) $claimConflictOutbox->attempts === 8
&& $claimConflictOutbox->last_error_code
=== 'projection_receipt_conflict'
&& $claimConflictOutbox->discard_reason_code === '',
'Conflicting claim receipt was incorrectly declared not applied.');
$assert($claimConflictWorker->observationErrors()
=== ['projection_receipt_conflict'],
'Conflicting claim receipt lost its bounded claim diagnostic.');
$log->table('cloud_vm_network_projection_outbox')
->where('sample_key', $claimConflictKey)->delete();
$log->table('cloud_vm_network_observation_samples')
->where('sample_key', $claimConflictKey)->delete();
$main->table('cloud_abuse_projection_receipts')
->where('sample_key', $claimConflictKey)->delete();
// If current findings cannot be read, a matching receipt stays pending rather
// than losing the reset boundary; it converges after the read model recovers.
$addInstance(15, 155);
$seedPendingKey = $insertSample(
15,
155,
300,
$metrics(),
true,
[],
'pending',
null,
0,
7
);
$main->table('cloud_abuse_projection_receipts')->insert([
'sample_key' => $seedPendingKey,
'instance_id' => 15,
'rule_version' => CloudAbuseDetector::RULE_VERSION,
'bucket_start' => 300,
'projected_at' => $now,
]);
$log->table('cloud_vm_network_projection_outbox')
->where('sample_key', $seedPendingKey)->update([
'placement_key' => str_repeat('3', 64),
]);
$main->statement(
'ALTER TABLE cloud_abuse_findings '
. 'RENAME TO cloud_abuse_findings_unavailable'
);
$seedPendingWorker = $worker();
$assert($seedPendingWorker->tick(),
'Unavailable baseline seed was not consumed.');
$seedPending = $log->table('cloud_vm_network_projection_outbox')
->where('sample_key', $seedPendingKey)->first();
$assert($seedPending->state === 'pending'
&& (int) $seedPending->attempts === 8
&& $seedPending->last_error_code
=== 'projection_baseline_seed_pending'
&& $seedPending->discard_reason_code === '',
'Unavailable baseline seed was discarded or left unbounded.');
$main->statement(
'ALTER TABLE cloud_abuse_findings_unavailable '
. 'RENAME TO cloud_abuse_findings'
);
$now += 1000;
$assert($worker()->tick()
&& $log->table('cloud_vm_network_projection_outbox')
->where('sample_key', $seedPendingKey)->value('state') === 'done'
&& (int) $log->table('cloud_vm_network_baselines')
->where('instance_id', 15)->value('sample_count') === 0,
'Baseline seed did not converge after its read model recovered.');
// Missing owners and stale buckets end explicitly instead of locking forever.
$main->table('user')->insert(['id' => 99]);
$deletedKey = $insertSample(99, 99, 240, $metrics());
$assert($worker()->tick(), 'Deleted-instance sample was not consumed.');
$deleted = $log->table('cloud_vm_network_projection_outbox')
->where('sample_key', $deletedKey)->first();
$assert($deleted->state === 'discarded'
&& $deleted->discard_reason_code === 'instance_deleted',
'Deleted instance left a retrying or locked outbox row.');
$main->table('cloud_instances')->insert([
'id' => 21, 'host_id' => 7, 'user_id' => 211,
]);
$missingOwnerKey = $insertSample(21, 211, 240, $metrics());
$assert($worker()->tick(), 'Missing-owner sample was not consumed.');
$assert($log->table('cloud_vm_network_projection_outbox')
->where('sample_key', $missingOwnerKey)
->value('discard_reason_code') === 'instance_owner_missing'
&& $main->table('cloud_abuse_instance_states')
->where('instance_id', 21)->count() === 0
&& $main->table('cloud_abuse_findings')
->where('instance_id', 21)->count() === 0
&& $main->table('cloud_abuse_alert_events')
->where('instance_id', 21)->count() === 0
&& $main->table('cloud_abuse_projection_receipts')
->where('sample_key', $missingOwnerKey)->count() === 0,
'Missing owner allowed late read-model or event insertion.');
$addInstance(8, 88);
$main->table('user')->insert(['id' => 89]);
$oldOwnerFindingKey = hash('sha256', implode("\n", [
8, CloudAbuseDetector::RULE_VERSION,
$policySnapshot->policyHash,
'outbound_tcp_syn_sweep', 'ipv4', 'outbound',
]));
$main->table('cloud_abuse_instance_states')->insert([
'instance_id' => 8, 'user_id' => 89, 'host_id' => 7,
'risk_state' => 'high_risk', 'collection_state' => 'healthy',
'latest_direction' => 'outbound',
'latest_rule_code' => 'outbound_tcp_syn_sweep',
'latest_severity' => 'high', 'summary_json' => '{}',
'last_projection_key' => str_repeat('8', 64),
'last_bucket_start' => 180, 'last_sample_at' => 240,
'last_evaluated_at' => $now,
'created_at' => $now, 'updated_at' => $now,
]);
$main->table('cloud_abuse_findings')->insert([
'finding_key' => $oldOwnerFindingKey,
'instance_id' => 8, 'user_id' => 89, 'host_id' => 7,
'rule_code' => 'outbound_tcp_syn_sweep',
'rule_version' => CloudAbuseDetector::RULE_VERSION,
'direction' => 'outbound', 'ip_family' => 'ipv4',
'severity' => 'high', 'state' => 'open',
'first_seen_at' => 120, 'last_seen_at' => 180,
'consecutive_windows' => 2, 'evidence_json' => '{}',
'resolved_at' => null, 'created_at' => $now, 'updated_at' => $now,
]);
$ownerKey = $insertSample(8, 89, 240, $metrics());
$ownerEventCount = $main->table('cloud_abuse_alert_events')->count();
$ownerWorker = $worker();
$assert($ownerWorker->tick(), 'Owner-mismatch sample was not consumed.');
$ownerMismatch = $log->table('cloud_vm_network_projection_outbox')
->where('sample_key', $ownerKey)->first();
$assert($ownerMismatch->state === 'discarded'
&& $ownerMismatch->discard_reason_code === 'instance_owner_changed'
&& $log->table('cloud_vm_network_baselines')
->where('instance_id', 8)->count() === 0
&& $main->table('cloud_abuse_instance_states')
->where('instance_id', 8)->count() === 0
&& $main->table('cloud_abuse_findings')
->where('finding_key', $oldOwnerFindingKey)->value('state')
=== 'resolved'
&& (int) $main->table('cloud_abuse_findings')
->where('finding_key', $oldOwnerFindingKey)->value('user_id') === 89
&& (int) $main->table('cloud_abuse_findings')
->where('finding_key', $oldOwnerFindingKey)
->value('transition_revision') === 2
&& $main->table('cloud_abuse_alert_events')->count()
=== $ownerEventCount
&& (int) $main->table('cloud_instances')
->where('id', 8)->value('user_id') === 88,
'Owner mismatch left a retrying or locked outbox row.');
$assert($ownerWorker->observationErrors() === ['instance_owner_changed'],
'Owner mismatch did not expose its stable discard reason.');
$correctOwnerKey = $insertSample(8, 88, 300, $metrics());
$assert($worker()->tick(), 'Correct owner sample did not recover.');
$assert($log->table('cloud_vm_network_projection_outbox')
->where('sample_key', $correctOwnerKey)->value('state') === 'done'
&& (int) $log->table('cloud_vm_network_baselines')
->where('instance_id', 8)->value('user_id') === 88
&& (int) $main->table('cloud_abuse_instance_states')
->where('instance_id', 8)->value('user_id') === 88,
'Owner mismatch contaminated the next owner baseline.');
$addInstance(3, 33);
$lateBaseline = CloudAbuseDetector::initialBaseline();
$lateBaseline['last_bucket_start'] = 360;
$insertBaseline(3, 33, $lateBaseline, 360, str_repeat('b', 64));
$lateKey = $insertSample(3, 33, 300, $metrics());
$assert($worker()->tick(), 'Out-of-order sample was not consumed.');
$assert($log->table('cloud_vm_network_projection_outbox')
->where('sample_key', $lateKey)->value('discard_reason_code')
=== 'out_of_order_sample',
'Out-of-order sample changed or blocked the baseline.');
// An expired outbox+baseline lease is safely reclaimed.
$addInstance(4, 44);
$expiredToken = str_repeat('c', 32);
$insertBaseline(4, 44, CloudAbuseDetector::initialBaseline(),
null, null, $expiredToken, $now - 1, 4);
$expiredKey = $insertSample(4, 44, 420, $metrics(), true, [],
'leased', $expiredToken, $now - 1, 1);
$assert($worker()->tick(), 'Expired projection lease was not reclaimed.');
$assert($log->table('cloud_vm_network_projection_outbox')
->where('sample_key', $expiredKey)->value('state') === 'done'
&& (int) $log->table('cloud_vm_network_baselines')
->where('instance_id', 4)->value('revision') === 5,
'Expired lease recovery did not preserve baseline CAS ordering.');
// Warm baseline: two high windows open a finding; five clear windows resolve it.
$addInstance(5, 55);
$warm = CloudAbuseDetector::initialBaseline();
$warm['sample_count'] = CloudAbuseDetector::WARMUP_WINDOWS;
$warm['last_bucket_start'] = 60;
$insertBaseline(5, 55, $warm, 60, str_repeat('d', 64));
$high = [
'tcp_active_flows' => 6000, 'tcp_new_flows' => 6000,
'tcp_syn_flows' => 5000, 'tcp_unreplied_flows' => 5000,
'unique_remote_ips' => 700, 'unique_remote_ports' => 1,
'unique_local_ports' => 6000,
'max_remote_ports_per_remote_ip' => 1,
'max_remote_ips_per_remote_port' => 700,
'max_local_ports_per_remote_ip' => 1,
'max_remote_ips_per_local_port' => 700,
];
foreach ([120, 180] as $bucket) {
$insertSample(5, 55, $bucket, $metrics($high));
$assert($worker()->tick(), 'High-risk projection did not run.');
}
$finding = $main->table('cloud_abuse_findings')
->where('instance_id', 5)->where('state', 'open')->first();
$assert($finding !== null && $finding->severity === 'high'
&& $finding->direction === 'outbound'
&& (int) $finding->occurrence_generation === 1
&& (int) $finding->transition_revision === 1,
'Two high windows did not open the deterministic finding.');
$openedEvent = $main->table('cloud_abuse_alert_events')
->where('instance_id', 5)->first();
$openedItem = $openedEvent === null ? null
: $main->table('cloud_abuse_alert_event_items')
->where('event_id', (int) $openedEvent->id)->first();
$assert($openedEvent !== null
&& (int) $openedEvent->bucket_start === 180
&& (int) $openedEvent->transition_count === 1
&& $openedEvent->highest_severity === 'high'
&& $openedItem !== null
&& $openedItem->transition_type === 'opened'
&& $openedItem->from_severity === 'none'
&& $openedItem->to_severity === 'high'
&& (int) $openedItem->occurrence_generation === 1
&& (int) $openedItem->transition_revision === 1,
'Finding open was not frozen as one immutable event item.');
$insertSample(5, 55, 240, $metrics($high));
$assert($worker()->tick(), 'Continued high-risk projection did not run.');
$assert($main->table('cloud_abuse_alert_events')
->where('instance_id', 5)->count() === 1
&& (int) $main->table('cloud_abuse_findings')
->where('instance_id', 5)->value('transition_revision') === 1,
'Continued finding created an event or advanced its transition revision.');
foreach ([300, 360, 420, 480, 540] as $bucket) {
$insertSample(5, 55, $bucket, $metrics());
$assert($worker()->tick(), 'Clear projection did not run.');
}
$finding = $main->table('cloud_abuse_findings')
->where('instance_id', 5)->first();
$assert($finding->state === 'resolved'
&& (int) $finding->resolved_at === 540
&& (int) $finding->transition_revision === 2,
'Five verified clear windows did not resolve the finding.');
$resolvedItem = $main->table('cloud_abuse_alert_event_items')
->where('finding_key', (string) $finding->finding_key)
->where('transition_type', 'resolved')->first();
$assert($main->table('cloud_abuse_alert_events')
->where('instance_id', 5)->count() === 2
&& $resolvedItem !== null
&& $resolvedItem->from_severity === 'high'
&& $resolvedItem->to_severity === 'none'
&& $resolvedItem->resolution_reason === 'evidence_clear'
&& (int) $resolvedItem->transition_revision === 2,
'Evidence-clear resolution was not frozen as an event.');
foreach ([600, 660] as $bucket) {
$insertSample(5, 55, $bucket, $metrics($high));
$assert($worker()->tick(), 'Reopen projection did not run.');
}
$reopened = $main->table('cloud_abuse_findings')
->where('instance_id', 5)->first();
$reopenedItem = $main->table('cloud_abuse_alert_event_items')
->where('finding_key', (string) $reopened->finding_key)
->where('occurrence_generation', 2)->first();
$assert($reopened->state === 'open'
&& (int) $reopened->occurrence_generation === 2
&& (int) $reopened->transition_revision === 3
&& $reopenedItem !== null
&& $reopenedItem->transition_type === 'opened'
&& (int) $reopenedItem->transition_revision === 3,
'Resolved finding did not reopen in a new occurrence generation.');
// A panel ring gap resets its baseline but never clears an existing finding.
$addInstance(6, 66);
$gapBaseline = CloudAbuseDetector::resetBaseline([[
'rule_code' => 'outbound_tcp_syn_sweep',
'direction' => 'outbound',
'ip_family' => 'ipv4',
'severity' => 'high',
]]);
$gapBaseline['sample_count'] = 60;
$gapBaseline['last_bucket_start'] = 60;
$gapBaseline['last_interface'] = $metrics()['interface'];
$insertBaseline(6, 66, $gapBaseline, 60, str_repeat('e', 64));
$findingKey = hash('sha256', implode("\n", [
6, CloudAbuseDetector::RULE_VERSION,
$policySnapshot->policyHash,
'outbound_tcp_syn_sweep', 'ipv4', 'outbound',
]));
$main->table('cloud_abuse_findings')->insert([
'finding_key' => $findingKey,
'instance_id' => 6, 'user_id' => 66, 'host_id' => 7,
'rule_code' => 'outbound_tcp_syn_sweep',
'rule_version' => CloudAbuseDetector::RULE_VERSION,
'direction' => 'outbound', 'ip_family' => 'ipv4',
'severity' => 'high', 'state' => 'open',
'first_seen_at' => 60, 'last_seen_at' => 60,
'consecutive_windows' => 2, 'evidence_json' => '{}',
'resolved_at' => null, 'created_at' => $now, 'updated_at' => $now,
]);
$main->table('cloud_abuse_instance_states')->insert([
'instance_id' => 6, 'user_id' => 66, 'host_id' => 7,
'risk_state' => 'high_risk', 'collection_state' => 'healthy',
'latest_direction' => 'outbound',
'latest_rule_code' => 'outbound_tcp_syn_sweep',
'latest_severity' => 'high', 'summary_json' => '{}',
'last_projection_key' => str_repeat('e', 64),
'last_bucket_start' => 60, 'last_sample_at' => 120,
'last_evaluated_at' => $now,
'created_at' => $now, 'updated_at' => $now,
]);
$gapKey = $insertSample(6, 66, 120, $metrics(), false,
['incomplete', 'panel_ring_gap']);
$assert($worker()->tick(), 'Panel-gap projection did not run.');
$gapStored = json_decode((string) $log->table(
'cloud_vm_network_baselines'
)->where('instance_id', 6)->value('metrics_json'), true,
32, JSON_THROW_ON_ERROR);
$state6 = $main->table('cloud_abuse_instance_states')
->where('instance_id', 6)->first();
$assert($log->table('cloud_vm_network_projection_outbox')
->where('sample_key', $gapKey)->value('state') === 'done'
&& $gapStored['sample_count'] === 0
&& $gapStored['last_interface'] === null
&& $main->table('cloud_abuse_findings')
->where('finding_key', $findingKey)->value('state') === 'open'
&& $state6->risk_state === 'high_risk'
&& $state6->collection_state === 'degraded',
'Incomplete ring-gap evidence advanced or cleared trusted state.');
for ($step = 1; $step <= 64; ++$step) {
$insertSample(6, 66, 120 + ($step * 60), $metrics());
$assert($worker()->tick(), 'Panel-gap clear convergence did not run.');
}
$assert($main->table('cloud_abuse_findings')
->where('finding_key', $findingKey)->value('state') === 'resolved'
&& $main->table('cloud_abuse_instance_states')
->where('instance_id', 6)->value('risk_state') === 'normal',
'Panel-gap reset left an open finding after verified clear evidence.');
// A changed immutable detector policy keeps historical statistics/sample
// count, clears threshold streaks, and retires old findings explicitly rather
// than pretending that new evidence cleared them.
$addInstance(16, 166);
$history = CloudAbuseDetector::evaluate([
'bucket_start' => 60,
'complete' => true,
'quality_flags' => [],
'metrics' => $metrics(),
], CloudAbuseDetector::initialBaseline(), $defaultPolicy)['baseline'];
$history['sample_count'] = 60;
$history['rules']['ipv4:outbound_tcp_syn_sweep'] = [
'active' => true,
'severity' => 'high',
'suspicious_streak' => 0,
'high_streak' => 2,
'clear_streak' => 0,
'transition' => 'continued',
];
$statsBeforeSwitch = $history['stats'];
$insertBaseline(
16,
166,
$history,
60,
str_repeat('1', 64),
null,
0,
0,
$policySnapshot
);
$replacedFindingKey = hash('sha256', implode("\n", [
16,
CloudAbuseDetector::RULE_VERSION,
$policySnapshot->policyHash,
'outbound_tcp_syn_sweep',
'ipv4',
'outbound',
]));
$main->table('cloud_abuse_findings')->insert([
'finding_key' => $replacedFindingKey,
'instance_id' => 16, 'user_id' => 166, 'host_id' => 7,
'rule_code' => 'outbound_tcp_syn_sweep',
'rule_version' => CloudAbuseDetector::RULE_VERSION,
'policy_revision' => 1,
'policy_hash' => $policySnapshot->policyHash,
'direction' => 'outbound', 'ip_family' => 'ipv4',
'severity' => 'high', 'state' => 'open',
'first_seen_at' => 60, 'last_seen_at' => 60,
'consecutive_windows' => 2,
'evidence_json' => '{"tcp_syn_flows":5000}',
'resolved_at' => null, 'resolution_reason' => '',
'created_at' => $now, 'updated_at' => $now,
]);
$main->table('cloud_abuse_instance_states')->insert([
'instance_id' => 16, 'user_id' => 166, 'host_id' => 7,
'rule_version' => CloudAbuseDetector::RULE_VERSION,
'policy_revision' => 1,
'policy_hash' => $policySnapshot->policyHash,
'risk_state' => 'high_risk', 'collection_state' => 'healthy',
'latest_direction' => 'outbound',
'latest_rule_code' => 'outbound_tcp_syn_sweep',
'latest_severity' => 'high', 'summary_json' => '{}',
'last_projection_key' => str_repeat('1', 64),
'last_bucket_start' => 60, 'last_sample_at' => 120,
'last_evaluated_at' => $now,
'created_at' => $now, 'updated_at' => $now,
]);
$switchKey = $insertSample(
16,
166,
120,
$metrics(),
false,
['incomplete', 'sampling_gap'],
'pending',
null,
0,
0,
$changedPolicySnapshot
);
$policyEventCount = $main->table('cloud_abuse_alert_events')->count();
$assert($worker()->tick(), 'Changed-policy projection did not run.');
$switchedBaseline = json_decode((string) $log->table(
'cloud_vm_network_baselines'
)->where('instance_id', 16)->value('metrics_json'), true,
32, JSON_THROW_ON_ERROR);
$replacedFinding = $main->table('cloud_abuse_findings')
->where('finding_key', $replacedFindingKey)->first();
$assert($log->table('cloud_vm_network_projection_outbox')
->where('sample_key', $switchKey)->value('state') === 'done'
&& (int) $log->table('cloud_vm_network_baselines')
->where('instance_id', 16)->value('policy_revision') === 2
&& hash_equals(
$changedPolicySnapshot->policyHash,
(string) $log->table('cloud_vm_network_baselines')
->where('instance_id', 16)->value('policy_hash')
)
&& $switchedBaseline['sample_count'] === 60
&& $switchedBaseline['stats'] == $statsBeforeSwitch
&& $switchedBaseline['rules'] === []
&& $replacedFinding->state === 'resolved'
&& $replacedFinding->resolution_reason === 'policy_replaced'
&& (int) $replacedFinding->transition_revision === 2
&& $replacedFinding->evidence_json === '{"tcp_syn_flows":5000}'
&& $main->table('cloud_abuse_alert_events')->count()
=== $policyEventCount
&& $main->table('cloud_abuse_instance_states')
->where('instance_id', 16)->value('risk_state') === 'observing',
'Detector policy switch lost statistics or forged evidence recovery.');
// A newer revision with the same detector hash may carry unrelated delivery
// metadata in a later schema. It updates identity without clearing streaks.
$addInstance(17, 177);
$sameHashBaseline = CloudAbuseDetector::resetBaseline([[
'rule_code' => 'outbound_tcp_syn_sweep',
'direction' => 'outbound',
'ip_family' => 'ipv4',
'severity' => 'high',
]], $defaultPolicy);
$sameHashBaseline['sample_count'] = 60;
$sameHashBaseline['last_bucket_start'] = 60;
$sameHashBaseline['last_interface'] = $metrics()['interface'];
$sameHashBaseline['stats'] = $statsBeforeSwitch;
$insertBaseline(
17,
177,
$sameHashBaseline,
60,
str_repeat('2', 64),
null,
0,
0,
$policySnapshot
);
$sameHashFindingKey = hash('sha256', implode("\n", [
17,
CloudAbuseDetector::RULE_VERSION,
$policySnapshot->policyHash,
'outbound_tcp_syn_sweep',
'ipv4',
'outbound',
]));
$main->table('cloud_abuse_findings')->insert([
'finding_key' => $sameHashFindingKey,
'instance_id' => 17, 'user_id' => 177, 'host_id' => 7,
'rule_code' => 'outbound_tcp_syn_sweep',
'rule_version' => CloudAbuseDetector::RULE_VERSION,
'policy_revision' => 1,
'policy_hash' => $policySnapshot->policyHash,
'direction' => 'outbound', 'ip_family' => 'ipv4',
'severity' => 'high', 'state' => 'open',
'first_seen_at' => 60, 'last_seen_at' => 60,
'consecutive_windows' => 2, 'evidence_json' => '{}',
'resolved_at' => null, 'resolution_reason' => '',
'created_at' => $now, 'updated_at' => $now,
]);
$main->table('cloud_abuse_instance_states')->insert([
'instance_id' => 17, 'user_id' => 177, 'host_id' => 7,
'rule_version' => CloudAbuseDetector::RULE_VERSION,
'policy_revision' => 1,
'policy_hash' => $policySnapshot->policyHash,
'risk_state' => 'high_risk', 'collection_state' => 'healthy',
'latest_direction' => 'outbound',
'latest_rule_code' => 'outbound_tcp_syn_sweep',
'latest_severity' => 'high', 'summary_json' => '{}',
'last_projection_key' => str_repeat('2', 64),
'last_bucket_start' => 60, 'last_sample_at' => 120,
'last_evaluated_at' => $now,
'created_at' => $now, 'updated_at' => $now,
]);
$insertSample(
17,
177,
120,
$metrics(),
false,
['incomplete', 'sampling_gap'],
'pending',
null,
0,
0,
$sameDetectorSnapshot
);
$assert($worker()->tick(), 'Same-detector revision projection did not run.');
$sameHashStored = json_decode((string) $log->table(
'cloud_vm_network_baselines'
)->where('instance_id', 17)->value('metrics_json'), true,
32, JSON_THROW_ON_ERROR);
$assert((int) $log->table('cloud_vm_network_baselines')
->where('instance_id', 17)->value('policy_revision') === 3
&& ($sameHashStored['rules']['ipv4:outbound_tcp_syn_sweep']['active']
?? false) === true
&& $main->table('cloud_abuse_findings')
->where('finding_key', $sameHashFindingKey)->value('state') === 'open'
&& $main->table('cloud_abuse_instance_states')
->where('instance_id', 17)->value('risk_state') === 'high_risk',
'Same detector hash incorrectly cleared durable finding streaks.');
// Corrupt/non-canonical evidence has bounded retries and an explicit terminal
// state; it can neither leak into the main read model nor hold the queue.
$addInstance(7, 77);
$invalidKey = $insertSample(7, 77, 120, $metrics());
$invalidMetrics = $metrics();
$invalidMetrics['ipv4']['outbound']['remote_addresses'] = ['203.0.113.8'];
$log->table('cloud_vm_network_observation_samples')
->where('sample_key', $invalidKey)->update([
'metrics_json' => json_encode($invalidMetrics,
JSON_THROW_ON_ERROR),
]);
for ($attempt = 1; $attempt <= 8; ++$attempt) {
$assert($worker()->tick(), 'Invalid sample retry was not consumed.');
$row = $log->table('cloud_vm_network_projection_outbox')
->where('sample_key', $invalidKey)->first();
$assert((int) $row->attempts === $attempt,
'Invalid sample attempt count was not durable.');
$now += 1000;
}
$invalidOutbox = $log->table('cloud_vm_network_projection_outbox')
->where('sample_key', $invalidKey)->first();
$assert($invalidOutbox->state === 'discarded'
&& $invalidOutbox->discard_reason_code
=== 'projection_retry_exhausted'
&& $main->table('cloud_abuse_projection_receipts')
->where('sample_key', $invalidKey)->count() === 0,
'Invalid aggregate evidence leaked or left a permanent projection lock.');
// A corrupt, attributable sample/outbox pair backs off and terminates without
// monopolising the globally earliest due-candidate position.
$addInstance(9, 99);
$badLedgerKey = $insertSample(9, 99, 60, $metrics());
$log->table('cloud_vm_network_projection_outbox')
->where('sample_key', $badLedgerKey)->update([
'placement_key' => str_repeat('f', 64),
]);
$addInstance(10, 100);
$nextInstanceKey = $insertSample(10, 100, 120, $metrics());
$badLedgerWorker = $worker();
$assert($badLedgerWorker->tick(), 'Bad ledger candidate was not bounded.');
$assert((int) $log->table('cloud_vm_network_projection_outbox')
->where('sample_key', $badLedgerKey)->value('attempts') === 1,
'Bad ledger candidate did not record its first attempt.');
$assert($badLedgerWorker->observationErrors()
=== ['projection_sample_ledger_inconsistent'],
'Bad ledger candidate exposed no stable worker error.');
$assert($worker()->tick(), 'Next instance did not bypass backed-off damage.');
$assert($log->table('cloud_vm_network_projection_outbox')
->where('sample_key', $nextInstanceKey)->value('state') === 'done',
'Bad global candidate blocked an unrelated instance.');
for ($attempt = 2; $attempt <= 8; ++$attempt) {
$now += 1000;
$assert($worker()->tick(), 'Bad ledger retry was not consumed.');
$badOutbox = $log->table('cloud_vm_network_projection_outbox')
->where('sample_key', $badLedgerKey)->first();
$assert((int) $badOutbox->attempts === $attempt,
'Bad ledger retry count was not durable.');
}
$badOutbox = $log->table('cloud_vm_network_projection_outbox')
->where('sample_key', $badLedgerKey)->first();
$assert($badOutbox->state === 'discarded'
&& $badOutbox->discard_reason_code === 'projection_retry_exhausted'
&& $badOutbox->last_error_code
=== 'projection_sample_ledger_inconsistent',
'Bad ledger candidate did not reach an attributable terminal state.');
// Event-item persistence is in the same main transaction as finding/state and
// receipt writes. A storage failure must roll all four back; retrying the same
// outbox after repair then creates exactly one event.
$addInstance(19, 199);
$rollbackBaseline = CloudAbuseDetector::initialBaseline();
$rollbackBaseline['sample_count'] = CloudAbuseDetector::WARMUP_WINDOWS;
$rollbackBaseline['last_bucket_start'] = 780;
$insertBaseline(19, 199, $rollbackBaseline, 780, str_repeat('4', 64));
$insertSample(19, 199, 840, $metrics($high));
$assert($worker()->tick(), 'Alert rollback setup projection did not run.');
$rollbackAlertKey = $insertSample(19, 199, 900, $metrics($high));
$main->statement(
'CREATE TRIGGER reject_abuse_alert_item '
. 'BEFORE INSERT ON cloud_abuse_alert_event_items '
. "BEGIN SELECT RAISE(ABORT, 'forced item failure'); END"
);
$failedAlertWorker = $worker();
$assert($failedAlertWorker->tick(), 'Alert item failure was not consumed.');
$assert($failedAlertWorker->observationErrors() === ['projection_failed']
&& $main->table('cloud_abuse_findings')
->where('instance_id', 19)->count() === 0
&& $main->table('cloud_abuse_alert_events')
->where('instance_id', 19)->count() === 0
&& $main->table('cloud_abuse_projection_receipts')
->where('sample_key', $rollbackAlertKey)->count() === 0
&& (int) $main->table('cloud_abuse_instance_states')
->where('instance_id', 19)->value('last_bucket_start') === 840,
'Alert item failure left partial state, finding, event or receipt data.');
$main->statement('DROP TRIGGER reject_abuse_alert_item');
$now += 1000;
$assert($worker()->tick(), 'Repaired alert event projection did not retry.');
$assert($main->table('cloud_abuse_findings')
->where('instance_id', 19)->count() === 1
&& $main->table('cloud_abuse_alert_events')
->where('instance_id', 19)->count() === 1
&& $main->table('cloud_abuse_alert_event_items')
->whereIn('event_id', $main->table('cloud_abuse_alert_events')
->where('instance_id', 19)->pluck('id')->all())->count() === 1
&& $main->table('cloud_abuse_projection_receipts')
->where('sample_key', $rollbackAlertKey)->count() === 1,
'Alert event retry did not converge to one durable batch.');
// A crash after the main commit leaves the event and receipt authoritative;
// log finalization replay must not duplicate either immutable row.
$addInstance(20, 200);
$eventReplayBaseline = CloudAbuseDetector::initialBaseline();
$eventReplayBaseline['sample_count'] = CloudAbuseDetector::WARMUP_WINDOWS;
$eventReplayBaseline['last_bucket_start'] = 960;
$insertBaseline(20, 200, $eventReplayBaseline, 960, str_repeat('5', 64));
$insertSample(20, 200, 1020, $metrics($high));
$assert($worker()->tick(), 'Event replay setup projection did not run.');
$eventReplayKey = $insertSample(20, 200, 1080, $metrics($high));
$eventCommitCrashed = false;
$worker(static function () use (&$eventCommitCrashed): void {
if (!$eventCommitCrashed) {
$eventCommitCrashed = true;
throw new RuntimeException('simulated event finalization crash');
}
})->tick();
$assert($main->table('cloud_abuse_alert_events')
->where('instance_id', 20)->count() === 1
&& $main->table('cloud_abuse_projection_receipts')
->where('sample_key', $eventReplayKey)->count() === 1
&& $log->table('cloud_vm_network_projection_outbox')
->where('sample_key', $eventReplayKey)->value('state') === 'pending',
'Post-commit crash did not retain the authoritative alert event.');
$now += 15;
$assert($worker()->tick(), 'Event receipt replay did not run.');
$replayEventId = (int) $main->table('cloud_abuse_alert_events')
->where('instance_id', 20)->value('id');
$assert($log->table('cloud_vm_network_projection_outbox')
->where('sample_key', $eventReplayKey)->value('state') === 'done'
&& $main->table('cloud_abuse_alert_events')
->where('instance_id', 20)->count() === 1
&& $main->table('cloud_abuse_alert_event_items')
->where('event_id', $replayEventId)->count() === 1
&& $main->table('cloud_abuse_projection_receipts')
->where('sample_key', $eventReplayKey)->count() === 1,
'Receipt replay duplicated the committed alert event batch.');
$idleWorker = $worker();
$assert(!$idleWorker->tick() && $idleWorker->observationErrors() === [],
'Idle projection worker retained an earlier observation error.');
// Detector reports suspicious-to-high as `continued`; the locked finding row
// is the transition authority. A later silent high-to-suspicious downgrade is
// not an alert transition and must not advance the transition revision.
$apply = new ReflectionMethod(
CloudAbuseProjectionService::class,
'applyFindings'
);
$escalationClaim = [
'sample_key' => str_repeat('9', 64),
'instance_id' => 18,
'user_id' => 188,
'host_id' => 7,
'bucket_start' => 720,
'rule_version' => CloudAbuseDetector::RULE_VERSION,
'policy_revision' => 1,
'policy_hash' => $policySnapshot->policyHash,
];
$escalationKey = hash('sha256', implode("\n", [
18,
CloudAbuseDetector::RULE_VERSION,
$policySnapshot->policyHash,
'outbound_tcp_syn_sweep',
'ipv4',
'outbound',
]));
$main->table('cloud_abuse_findings')->insert([
'finding_key' => $escalationKey,
'instance_id' => 18, 'user_id' => 188, 'host_id' => 7,
'rule_code' => 'outbound_tcp_syn_sweep',
'rule_version' => CloudAbuseDetector::RULE_VERSION,
'policy_revision' => 1,
'policy_hash' => $policySnapshot->policyHash,
'direction' => 'outbound', 'ip_family' => 'ipv4',
'severity' => 'suspicious', 'state' => 'open',
'first_seen_at' => 660, 'last_seen_at' => 660,
'consecutive_windows' => 3,
'occurrence_generation' => 1, 'transition_revision' => 1,
'evidence_json' => '{"tcp_syn_flows":900}',
'resolved_at' => null, 'resolution_reason' => '',
'created_at' => $now, 'updated_at' => $now,
]);
$continuedHigh = [[
'rule_code' => 'outbound_tcp_syn_sweep',
'direction' => 'outbound', 'ip_family' => 'ipv4',
'severity' => 'high', 'active' => true,
'transition' => 'continued', 'consecutive_windows' => 2,
'evidence' => ['tcp_syn_flows' => 5000,
'unique_remote_ips' => 700],
]];
$projectionWorker = $worker();
$escalated = $apply->invoke(
$projectionWorker,
$main,
$escalationClaim,
7,
$continuedHigh,
$now
);
$assert(count($escalated) === 1
&& $escalated[0]['transition_type'] === 'escalated'
&& $escalated[0]['from_severity'] === 'suspicious'
&& $escalated[0]['to_severity'] === 'high'
&& (int) $main->table('cloud_abuse_findings')
->where('finding_key', $escalationKey)
->value('transition_revision') === 2,
'Locked finding state did not derive suspicious-to-high escalation.');
(new App\Services\Cloud\CloudAbuseAlertEventCoordinator())->record(
$main,
$escalationClaim,
$escalated,
$now
);
$escalationClaim['sample_key'] = str_repeat('a', 64);
$escalationClaim['bucket_start'] = 780;
$silentDowngrade = $continuedHigh;
$silentDowngrade[0]['severity'] = 'suspicious';
$silentDowngrade[0]['evidence'] = ['tcp_syn_flows' => 900,
'unique_remote_ips' => 140];
$beforeDowngradeEvents = $main->table(
'cloud_abuse_alert_events'
)->count();
$downgraded = $apply->invoke(
$projectionWorker,
$main,
$escalationClaim,
7,
$silentDowngrade,
$now
);
$assert($downgraded === []
&& (int) $main->table('cloud_abuse_findings')
->where('finding_key', $escalationKey)
->value('transition_revision') === 2
&& $main->table('cloud_abuse_alert_events')->count()
=== $beforeDowngradeEvents,
'Silent severity downgrade advanced revision or created an event.');
$source = file_get_contents(dirname(__DIR__)
. '/app/Services/Cloud/CloudAbuseProjectionService.php');
$assert(is_string($source)
&& !str_contains($source, 'CloudJob')
&& !str_contains($source, 'service_status')
&& !str_contains($source, 'network_status'),
'Projection service gained a task, lifecycle or enforcement dependency.');
echo "Cloud abuse projection convergence tests passed.\n";