You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
372 lines
13 KiB
PHP
372 lines
13 KiB
PHP
<?php
|
|
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
|
|
|
use App\Services\Cloud\CloudAbuseRetentionService;
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
|
|
$assert = static function (bool $condition, string $message): void {
|
|
if (!$condition) {
|
|
throw new RuntimeException($message);
|
|
}
|
|
};
|
|
|
|
$capsule = new Capsule();
|
|
$capsule->addConnection([
|
|
'driver' => 'sqlite', 'database' => ':memory:',
|
|
], 'default');
|
|
$capsule->addConnection([
|
|
'driver' => 'sqlite', 'database' => ':memory:',
|
|
], 'log');
|
|
$capsule->setAsGlobal();
|
|
$capsule->bootEloquent();
|
|
|
|
$mainSchema = $capsule->schema('default');
|
|
$mainSchema->create('cloud_instances', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
});
|
|
$mainSchema->create(
|
|
'cloud_abuse_projection_receipts',
|
|
static function (Blueprint $table): void {
|
|
$table->string('sample_key')->primary();
|
|
$table->integer('instance_id');
|
|
$table->integer('rule_version');
|
|
$table->integer('bucket_start');
|
|
$table->integer('projected_at');
|
|
}
|
|
);
|
|
$mainSchema->create('cloud_abuse_findings',
|
|
static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('finding_key')->unique();
|
|
$table->integer('instance_id');
|
|
$table->integer('user_id');
|
|
$table->integer('host_id');
|
|
$table->string('rule_code');
|
|
$table->integer('rule_version');
|
|
$table->string('direction');
|
|
$table->string('ip_family');
|
|
$table->string('severity');
|
|
$table->string('state');
|
|
$table->integer('first_seen_at');
|
|
$table->integer('last_seen_at');
|
|
$table->integer('consecutive_windows');
|
|
$table->text('evidence_json');
|
|
$table->integer('resolved_at')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
|
|
$logSchema = $capsule->schema('log');
|
|
$logSchema->create(
|
|
'cloud_vm_network_observation_samples',
|
|
static function (Blueprint $table): void {
|
|
$table->string('sample_key')->primary();
|
|
$table->integer('instance_id');
|
|
$table->integer('bucket_start');
|
|
}
|
|
);
|
|
$logSchema->create(
|
|
'cloud_vm_network_projection_outbox',
|
|
static function (Blueprint $table): void {
|
|
$table->string('sample_key')->primary();
|
|
$table->integer('instance_id');
|
|
$table->integer('bucket_start');
|
|
$table->string('state');
|
|
$table->integer('terminal_at')->nullable();
|
|
}
|
|
);
|
|
$logSchema->create(
|
|
'cloud_vm_network_baselines',
|
|
static function (Blueprint $table): void {
|
|
$table->integer('instance_id')->primary();
|
|
$table->integer('user_id');
|
|
}
|
|
);
|
|
|
|
$main = $capsule->connection('default');
|
|
$log = $capsule->connection('log');
|
|
$now = 20_000_000;
|
|
$telemetryCutoff = $now
|
|
- CloudAbuseRetentionService::TELEMETRY_RETENTION_SECONDS;
|
|
$readCutoff = $now
|
|
- CloudAbuseRetentionService::READ_MODEL_RETENTION_SECONDS;
|
|
$oldBucket = intdiv($telemetryCutoff - 120, 60) * 60;
|
|
$recentBucket = intdiv($telemetryCutoff + 120, 60) * 60;
|
|
|
|
$key = static fn (string $name): string => hash('sha256', $name);
|
|
$sample = static function (
|
|
string $name,
|
|
int $instanceId,
|
|
int $bucket
|
|
) use ($log, $key): string {
|
|
$sampleKey = $key($name);
|
|
$log->table('cloud_vm_network_observation_samples')->insert([
|
|
'sample_key' => $sampleKey,
|
|
'instance_id' => $instanceId,
|
|
'bucket_start' => $bucket,
|
|
]);
|
|
|
|
return $sampleKey;
|
|
};
|
|
$outbox = static function (
|
|
string $sampleKey,
|
|
int $instanceId,
|
|
int $bucket,
|
|
string $state,
|
|
?int $terminalAt = null
|
|
) use ($log): void {
|
|
$log->table('cloud_vm_network_projection_outbox')->insert([
|
|
'sample_key' => $sampleKey,
|
|
'instance_id' => $instanceId,
|
|
'bucket_start' => $bucket,
|
|
'state' => $state,
|
|
'terminal_at' => $terminalAt,
|
|
]);
|
|
};
|
|
$receipt = static function (
|
|
string $sampleKey,
|
|
int $instanceId,
|
|
int $projectedAt
|
|
) use ($main): void {
|
|
$main->table('cloud_abuse_projection_receipts')->insert([
|
|
'sample_key' => $sampleKey,
|
|
'instance_id' => $instanceId,
|
|
'rule_version' => 1,
|
|
'bucket_start' => 60,
|
|
'projected_at' => $projectedAt,
|
|
]);
|
|
};
|
|
$finding = static function (
|
|
string $name,
|
|
string $state,
|
|
?int $resolvedAt
|
|
) use ($main, $key, $now): void {
|
|
$main->table('cloud_abuse_findings')->insert([
|
|
'finding_key' => $key('finding-' . $name),
|
|
'instance_id' => 1,
|
|
'user_id' => 1,
|
|
'host_id' => 1,
|
|
'rule_code' => 'test_rule',
|
|
'rule_version' => 1,
|
|
'direction' => 'outbound',
|
|
'ip_family' => 'ipv4',
|
|
'severity' => 'info',
|
|
'state' => $state,
|
|
'first_seen_at' => 60,
|
|
'last_seen_at' => 60,
|
|
'consecutive_windows' => 1,
|
|
'evidence_json' => '{}',
|
|
'resolved_at' => $resolvedAt,
|
|
'created_at' => 60,
|
|
'updated_at' => $now,
|
|
]);
|
|
};
|
|
|
|
$main->table('cloud_instances')->insert(['id' => 1]);
|
|
$log->table('cloud_vm_network_baselines')->insert([
|
|
['instance_id' => 1, 'user_id' => 1],
|
|
['instance_id' => 2, 'user_id' => 2],
|
|
['instance_id' => 3, 'user_id' => 3],
|
|
]);
|
|
|
|
$doneOld = $sample('done-old', 1, $oldBucket);
|
|
$outbox($doneOld, 1, $oldBucket, 'done', $telemetryCutoff - 1);
|
|
$receipt($doneOld, 1, $readCutoff - 1);
|
|
|
|
$pendingOld = $sample('pending-old', 3, $oldBucket);
|
|
$outbox($pendingOld, 3, $oldBucket, 'pending');
|
|
$leasedOld = $sample('leased-old', 1, $oldBucket);
|
|
$outbox($leasedOld, 1, $oldBucket, 'leased');
|
|
$doneRecent = $sample('done-recent', 1, $recentBucket);
|
|
$outbox($doneRecent, 1, $recentBucket, 'done', $telemetryCutoff + 1);
|
|
$orphanOld = $sample('orphan-old', 1, $oldBucket);
|
|
|
|
$receiptGone = $key('receipt-gone');
|
|
$receipt($receiptGone, 1, $readCutoff - 1);
|
|
$receiptRecent = $key('receipt-recent');
|
|
$receipt($receiptRecent, 1, $readCutoff + 1);
|
|
$receiptWithSample = $sample('receipt-with-sample', 1, $recentBucket);
|
|
$receipt($receiptWithSample, 1, $readCutoff - 1);
|
|
|
|
$finding('resolved-old', 'resolved', $readCutoff - 1);
|
|
$finding('resolved-recent', 'resolved', $readCutoff + 1);
|
|
$finding('open-old', 'open', null);
|
|
|
|
$service = new CloudAbuseRetentionService(static fn (): int => $now);
|
|
$result = $service->prune();
|
|
|
|
$assert($result === [
|
|
'terminal_outboxes' => 1,
|
|
'samples' => 2,
|
|
'baselines' => 1,
|
|
'receipts' => 2,
|
|
'findings' => 1,
|
|
], 'Retention pass returned unexpected bounded deletion counts.');
|
|
$assert(!$log->table('cloud_vm_network_projection_outbox')
|
|
->where('sample_key', $doneOld)->exists()
|
|
&& !$log->table('cloud_vm_network_observation_samples')
|
|
->where('sample_key', $doneOld)->exists(),
|
|
'Expired terminal projection and sample were not removed together.');
|
|
$assert(!$log->table('cloud_vm_network_observation_samples')
|
|
->where('sample_key', $orphanOld)->exists(),
|
|
'Expired orphan sample was not removed.');
|
|
foreach ([$pendingOld, $leasedOld, $doneRecent] as $protectedKey) {
|
|
$assert($log->table('cloud_vm_network_projection_outbox')
|
|
->where('sample_key', $protectedKey)->exists()
|
|
&& $log->table('cloud_vm_network_observation_samples')
|
|
->where('sample_key', $protectedKey)->exists(),
|
|
'Pending, leased or recent telemetry was removed.');
|
|
}
|
|
$assert($log->table('cloud_vm_network_baselines')
|
|
->where('instance_id', 1)->exists()
|
|
&& !$log->table('cloud_vm_network_baselines')
|
|
->where('instance_id', 2)->exists()
|
|
&& $log->table('cloud_vm_network_baselines')
|
|
->where('instance_id', 3)->exists(),
|
|
'Orphan baseline cleanup ignored live ownership or active projection.');
|
|
$assert(!$main->table('cloud_abuse_projection_receipts')
|
|
->whereIn('sample_key', [$doneOld, $receiptGone])->exists()
|
|
&& $main->table('cloud_abuse_projection_receipts')
|
|
->whereIn('sample_key', [$receiptRecent, $receiptWithSample])
|
|
->count() === 2,
|
|
'Receipt retention ignored age or retained-sample protection.');
|
|
$assert($main->table('cloud_abuse_findings')->count() === 2
|
|
&& !$main->table('cloud_abuse_findings')
|
|
->where('finding_key', $key('finding-resolved-old'))->exists(),
|
|
'Finding retention removed an open/recent finding or kept an old one.');
|
|
|
|
// A storage failure after terminal outbox deletion must roll the log
|
|
// transaction back, retaining both records for a later retry.
|
|
$rollbackKey = $sample('rollback-terminal', 1, $oldBucket);
|
|
$outbox(
|
|
$rollbackKey,
|
|
1,
|
|
$oldBucket,
|
|
'discarded',
|
|
$telemetryCutoff - 1
|
|
);
|
|
$quotedRollback = $log->getPdo()->quote($rollbackKey);
|
|
$log->statement(
|
|
'CREATE TRIGGER fail_abuse_sample_delete '
|
|
. 'BEFORE DELETE ON cloud_vm_network_observation_samples '
|
|
. 'WHEN OLD.sample_key = ' . $quotedRollback . ' '
|
|
. "BEGIN SELECT RAISE(ABORT, 'injected retention failure'); END"
|
|
);
|
|
$failed = false;
|
|
try {
|
|
$service->prune();
|
|
} catch (Throwable) {
|
|
$failed = true;
|
|
}
|
|
$assert($failed
|
|
&& $log->table('cloud_vm_network_projection_outbox')
|
|
->where('sample_key', $rollbackKey)->exists()
|
|
&& $log->table('cloud_vm_network_observation_samples')
|
|
->where('sample_key', $rollbackKey)->exists(),
|
|
'Terminal outbox deletion was not rolled back with sample failure.');
|
|
$log->statement('DROP TRIGGER fail_abuse_sample_delete');
|
|
|
|
// The configured bound applies to each independently indexed retention set.
|
|
$finding('batch-one', 'resolved', $readCutoff - 2);
|
|
$finding('batch-two', 'resolved', $readCutoff - 2);
|
|
$bounded = $service->prune(1);
|
|
$assert($bounded['findings'] === 1
|
|
&& $main->table('cloud_abuse_findings')
|
|
->whereIn('finding_key', [
|
|
$key('finding-batch-one'), $key('finding-batch-two'),
|
|
])->count() === 1,
|
|
'Retention batch size did not bound the indexed finding pass.');
|
|
$assert($service->prune() === [
|
|
'terminal_outboxes' => 0,
|
|
'samples' => 0,
|
|
'baselines' => 0,
|
|
'receipts' => 0,
|
|
'findings' => 1,
|
|
], 'A restart-safe retention pass was not idempotent.');
|
|
|
|
// Protected rows at the front of a bounded scan must not permanently starve
|
|
// an eligible row beyond the limit. The process cursor advances and wraps;
|
|
// a time-derived first anchor also varies repeated --once worker starts.
|
|
$log->table('cloud_vm_network_projection_outbox')->delete();
|
|
$log->table('cloud_vm_network_observation_samples')->delete();
|
|
$log->table('cloud_vm_network_baselines')->delete();
|
|
$main->table('cloud_abuse_projection_receipts')->delete();
|
|
$main->table('cloud_instances')->insert(['id' => 2]);
|
|
$log->table('cloud_vm_network_baselines')->insert([
|
|
['instance_id' => 1, 'user_id' => 1],
|
|
['instance_id' => 2, 'user_id' => 2],
|
|
['instance_id' => 3, 'user_id' => 3],
|
|
]);
|
|
$scanMinute = intdiv($now, 60);
|
|
$scanMinute -= $scanMinute % 4;
|
|
$scanNow = $scanMinute * 60;
|
|
$scanService = new CloudAbuseRetentionService(
|
|
static fn (): int => $scanNow
|
|
);
|
|
$firstBaselineScan = $scanService->prune(2);
|
|
$assert($firstBaselineScan['baselines'] === 0
|
|
&& $log->table('cloud_vm_network_baselines')
|
|
->where('instance_id', 3)->exists(),
|
|
'A protected baseline prefix was not exercised.');
|
|
$secondBaselineScan = $scanService->prune(2);
|
|
$assert($secondBaselineScan['baselines'] === 1
|
|
&& !$log->table('cloud_vm_network_baselines')
|
|
->where('instance_id', 3)->exists(),
|
|
'A high-ID orphan baseline remained starved after scan wraparound.');
|
|
|
|
$log->table('cloud_vm_network_projection_outbox')->delete();
|
|
$log->table('cloud_vm_network_observation_samples')->delete();
|
|
$main->table('cloud_abuse_projection_receipts')->delete();
|
|
$receiptKeys = [];
|
|
$scanReadCutoff = $scanNow
|
|
- CloudAbuseRetentionService::READ_MODEL_RETENTION_SECONDS;
|
|
foreach ([1, 2, 3] as $number) {
|
|
$receiptKeys[$number] = str_pad(
|
|
dechex($number),
|
|
64,
|
|
'0',
|
|
STR_PAD_LEFT
|
|
);
|
|
$receipt($receiptKeys[$number], 1, $scanReadCutoff - 1);
|
|
if ($number <= 2) {
|
|
$log->table('cloud_vm_network_observation_samples')->insert([
|
|
'sample_key' => $receiptKeys[$number],
|
|
'instance_id' => 1,
|
|
'bucket_start' => $oldBucket,
|
|
]);
|
|
$outbox(
|
|
$receiptKeys[$number],
|
|
1,
|
|
$oldBucket,
|
|
'pending'
|
|
);
|
|
}
|
|
}
|
|
$receiptScanService = new CloudAbuseRetentionService(
|
|
static fn (): int => $scanNow
|
|
);
|
|
$receiptScanService->prune(2);
|
|
$receiptScanService->prune(2);
|
|
$assert(!$main->table('cloud_abuse_projection_receipts')
|
|
->where('sample_key', $receiptKeys[3])->exists()
|
|
&& $main->table('cloud_abuse_projection_receipts')
|
|
->whereIn('sample_key', [$receiptKeys[1], $receiptKeys[2]])
|
|
->count() === 2,
|
|
'A protected receipt prefix starved a later orphan receipt.');
|
|
$assert($log->table('cloud_vm_network_projection_outbox')->count() === 2
|
|
&& $log->table('cloud_vm_network_observation_samples')->count() === 2,
|
|
'Receipt scan removed pending or leased projection evidence.');
|
|
|
|
foreach ([0, CloudAbuseRetentionService::DEFAULT_BATCH_SIZE + 1] as $limit) {
|
|
try {
|
|
$service->prune($limit);
|
|
throw new RuntimeException('Invalid retention batch size was accepted.');
|
|
} catch (InvalidArgumentException) {
|
|
// Expected.
|
|
}
|
|
}
|
|
|
|
echo "Cloud abuse retention service tests passed.\n";
|