You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudAbuseObservationIngest...

554 lines
20 KiB
PHP

<?php
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudInstance;
use App\Services\Cloud\CloudAbuseDetectionPolicy;
use App\Services\Cloud\CloudAbuseObservationIngestService;
use App\Services\Cloud\CloudAbusePolicySnapshot;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$capsule = new Capsule();
$capsule->addConnection([
'driver' => 'sqlite', 'database' => ':memory:',
], 'default');
$capsule->addConnection([
'driver' => 'sqlite', 'database' => ':memory:',
], 'log');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$capsule->schema('default')->create('cloud_instances',
static function (Blueprint $table): void {
$table->increments('id');
$table->integer('host_id');
$table->integer('user_id');
$table->string('uuid');
$table->string('name')->default('vm');
$table->integer('created_at');
$table->integer('updated_at');
});
$logSchema = $capsule->schema('log');
$logSchema->create('cloud_vm_network_monitor_cursors',
static function (Blueprint $table): void {
$table->integer('host_id')->primary();
$table->string('collector_generation')->nullable();
$table->integer('cursor_value')->default(0);
$table->integer('oldest_cursor')->default(0);
$table->integer('gap_before')->default(0);
$table->integer('poll_revision')->default(0);
$table->integer('next_poll_at')->default(0);
$table->string('poll_lease_token')->nullable();
$table->integer('poll_lease_until')->default(0);
$table->string('collection_state');
$table->integer('consecutive_failures')->default(0);
$table->integer('last_window_start')->nullable();
$table->integer('last_success_at')->nullable();
$table->string('last_error_code')->default('');
$table->integer('updated_at');
});
$logSchema->create('cloud_vm_network_observation_samples',
static function (Blueprint $table): void {
$table->string('sample_key')->primary();
$table->integer('instance_id');
$table->integer('user_id');
$table->integer('host_id');
$table->string('vm_uuid');
$table->string('placement_key');
$table->string('collector_generation');
$table->string('window_id');
$table->integer('contract_version');
$table->integer('bucket_start');
$table->integer('bucket_seconds');
$table->integer('complete');
$table->text('metrics_json');
$table->text('quality_flags_json');
$table->integer('received_at');
$table->unique([
'host_id', 'collector_generation', 'window_id', 'vm_uuid',
]);
});
$logSchema->create('cloud_vm_network_projection_outbox',
static function (Blueprint $table): void {
$table->string('sample_key')->primary();
$table->integer('instance_id');
$table->integer('bucket_start');
$table->string('placement_key');
$table->integer('rule_version')->default(1);
$table->integer('policy_revision')->default(1);
$table->string('policy_hash')->default(
'511828bde950c504f92bb7a0603ca8b14453f575ead276c1889baa764f384dae'
);
$table->string('state');
$table->integer('attempts');
$table->integer('available_at');
$table->string('lease_token')->nullable();
$table->integer('lease_until');
$table->string('last_error_code');
$table->integer('projected_at')->nullable();
$table->integer('terminal_at')->nullable();
$table->string('discard_reason_code');
$table->integer('created_at');
$table->integer('updated_at');
});
$instance = new CloudInstance();
$instance->host_id = 7;
$instance->user_id = 44;
$instance->uuid = '123e4567-e89b-42d3-a456-426614174000';
$instance->name = 'abuse-test';
$instance->created_at = $instance->updated_at = 100;
$instance->save();
$generation = '0123456789abcdef0123456789abcdef';
$flow = static fn (): array => [
'tcp_active_flows' => 1, 'tcp_new_flows' => 0,
'tcp_syn_flows' => 0, 'tcp_unreplied_flows' => 0,
'udp_active_flows' => 0, 'udp_new_flows' => 0,
'other_active_flows' => 0, 'unique_remote_ips' => 1,
'unique_remote_ports' => 1, 'unique_local_ports' => 1,
'max_remote_ports_per_remote_ip' => 1,
'max_remote_ips_per_remote_port' => 1,
'max_local_ports_per_remote_ip' => 1,
'max_remote_ips_per_local_port' => 1,
];
$batch = static function (
string $generation,
int $cursor,
string $uuid,
string $bucket,
bool $gap = false
) use ($flow): array {
$epoch = (new DateTimeImmutable($bucket))->getTimestamp();
return [
'contract_version' => 1,
'collector_generation' => $generation,
'oldest_cursor' => $cursor,
'gap_before' => $gap,
'next_cursor' => $cursor,
'has_more' => false,
'capability' => ['available' => true],
'windows' => [[
'cursor' => $cursor,
'window_id' => substr(hash('sha256', $generation . "\n"
. $cursor . "\n" . $uuid . "\n" . $epoch), 0, 32),
'vm_uuid' => $uuid,
'bucket_start' => $bucket,
'bucket_seconds' => 60,
'complete' => true,
'quality_flags' => [],
'interface' => [
'rx_bytes' => 100, 'tx_bytes' => 200,
'rx_packets' => 10, 'tx_packets' => 20,
'rx_errors' => 0, 'tx_errors' => 0,
'rx_drops' => 0, 'tx_drops' => 0,
],
'ipv4' => ['inbound' => $flow(), 'outbound' => $flow()],
'ipv6' => null,
]],
];
};
$empty = static fn (string $generation, int $oldest = 0): array => [
'contract_version' => 1,
'collector_generation' => $generation,
'oldest_cursor' => $oldest,
'gap_before' => false,
'next_cursor' => 0,
'has_more' => false,
'capability' => ['available' => true],
'windows' => [],
];
$client = new class() {
public array $responses = [];
public array $calls = [];
public function networkObservations(
array $actor,
?string $generation,
int $after,
int $limit
): array {
$this->calls[] = [$actor, $generation, $after, $limit];
$response = array_shift($this->responses);
if ($response instanceof Throwable) {
throw $response;
}
return $response;
}
};
$now = 1000;
$defaultPolicy = CloudAbuseDetectionPolicy::defaults();
$policySnapshot = new CloudAbusePolicySnapshot(
1,
$defaultPolicy->hash(),
$defaultPolicy,
null,
0
);
$worker = new CloudAbuseObservationIngestService(
static fn (): array => [7],
static fn (int $hostId): object => $client,
static function () use (&$now): int { return $now; },
static function () use (&$policySnapshot): CloudAbusePolicySnapshot {
return $policySnapshot;
}
);
$client->responses[] = $batch(
$generation,
1,
$instance->uuid,
'2026-11-08T01:02:00Z'
);
$assert($worker->tick(), 'Due host was not polled.');
$log = $capsule->connection('log');
$cursor = $log->table('cloud_vm_network_monitor_cursors')->first();
$sampleRow = $log->table('cloud_vm_network_observation_samples')->first();
$outbox = $log->table('cloud_vm_network_projection_outbox')->first();
$assert($cursor !== null && (int) $cursor->cursor_value === 1
&& $cursor->collection_state === 'healthy'
&& $sampleRow !== null && (int) $sampleRow->instance_id === $instance->id
&& $outbox !== null && $outbox->state === 'pending'
&& (int) $outbox->rule_version === $defaultPolicy->engineVersion()
&& (int) $outbox->policy_revision === 1
&& hash_equals($defaultPolicy->hash(), (string) $outbox->policy_hash),
'Sample, outbox and host cursor did not commit together.');
$assert(($client->calls[0][0]['is_super_admin'] ?? false) === true
&& $client->calls[0][1] === null
&& $client->calls[0][2] === 0,
'Collector did not use the dedicated signed inspection actor/cursor.');
$now += 15;
$client->responses[] = $empty($generation, 1);
$assert($worker->tick(), 'Empty same-generation poll did not run.');
$cursor = $log->table('cloud_vm_network_monitor_cursors')->first();
$assert((int) $cursor->cursor_value === 1,
'An empty same-generation page reset the durable cursor.');
$now += 15;
$gapBatch = $batch(
$generation,
3,
$instance->uuid,
'2026-11-08T01:04:00Z',
true
);
$gapBatch['has_more'] = true;
$client->responses[] = $gapBatch;
$assert($worker->tick(), 'Retained-ring gap poll did not run.');
$gapRow = $log->table('cloud_vm_network_observation_samples')
->where('window_id', $gapBatch['windows'][0]['window_id'])->first();
$flags = json_decode($gapRow->quality_flags_json, true,
8, JSON_THROW_ON_ERROR);
$assert((int) $gapRow->complete === 0
&& in_array('panel_ring_gap', $flags, true)
&& in_array('incomplete', $flags, true)
&& (int) $log->table('cloud_vm_network_monitor_cursors')
->value('gap_before') === 1,
'Ring-gap evidence remained eligible for risk classification.');
$continuedGap = $batch(
$generation,
4,
$instance->uuid,
'2026-11-08T01:05:00Z'
);
$continuedGap['oldest_cursor'] = 3;
$continuedGap['has_more'] = true;
$client->responses[] = $continuedGap;
$assert($worker->tick(), 'Pending ring gap did not continue immediately.');
$continuedRow = $log->table('cloud_vm_network_observation_samples')
->where('window_id', $continuedGap['windows'][0]['window_id'])
->first();
$continuedFlags = json_decode(
$continuedRow->quality_flags_json,
true,
8,
JSON_THROW_ON_ERROR
);
$assert((int) $continuedRow->complete === 0
&& in_array('panel_ring_gap', $continuedFlags, true)
&& (int) $log->table('cloud_vm_network_monitor_cursors')
->value('gap_before') === 1,
'A retained-ring gap was not propagated across pages.');
$resetGeneration = 'fedcba9876543210fedcba9876543210';
$reset = $batch(
$resetGeneration,
1,
$instance->uuid,
'2026-11-08T01:06:00Z'
);
$client->responses[] = $reset;
$assert($worker->tick(), 'Generation reset did not settle immediately.');
$resetRow = $log->table('cloud_vm_network_observation_samples')
->where('window_id', $reset['windows'][0]['window_id'])->first();
$resetFlags = json_decode(
$resetRow->quality_flags_json,
true,
8,
JSON_THROW_ON_ERROR
);
$resetCursor = $log->table('cloud_vm_network_monitor_cursors')->first();
$assert((int) $resetRow->complete === 1
&& !in_array('panel_ring_gap', $resetFlags, true)
&& $resetCursor->collector_generation === $resetGeneration
&& (int) $resetCursor->cursor_value === 1
&& (int) $resetCursor->gap_before === 0
&& $resetCursor->collection_state === 'healthy',
'An old-generation gap contaminated the replacement generation.');
$now += 15;
$missing = $batch(
$resetGeneration,
2,
'123e4567-e89b-42d3-a456-426614174099',
'2026-11-08T01:07:00Z'
);
$client->responses[] = $missing;
$assert($worker->tick(), 'Unknown exact placement poll did not settle.');
$cursor = $log->table('cloud_vm_network_monitor_cursors')->first();
$assert((int) $cursor->cursor_value === 2
&& $cursor->collection_state === 'degraded'
&& $cursor->last_error_code === 'instance_mapping_missing'
&& $log->table('cloud_vm_network_observation_samples')->count() === 4,
'Unknown VM was guessed, persisted or allowed to block the host cursor.');
$now += 15;
$client->responses[] = new RuntimeException('temporary transport failure');
$assert($worker->tick(), 'Failed poll did not consume its bounded attempt.');
$cursor = $log->table('cloud_vm_network_monitor_cursors')->first();
$assert($cursor->poll_lease_token === null
&& (int) $cursor->poll_lease_until === 0
&& (int) $cursor->consecutive_failures === 1
&& $cursor->collection_state === 'degraded',
'Transient failure left a permanent poll lease.');
$assert($worker->observationErrors()
=== ['network_observation_collection_failed'],
'Worker health did not retain the bounded collection failure code.');
$blocked = $batch(
$resetGeneration,
3,
$instance->uuid,
'2026-11-08T01:08:00Z'
);
$now += 15;
$beforeSamples = $log->table(
'cloud_vm_network_observation_samples'
)->count();
$beforeOutbox = $log->table(
'cloud_vm_network_projection_outbox'
)->count();
$log->unprepared(
'CREATE TRIGGER ignore_network_projection_insert '
. 'BEFORE INSERT ON cloud_vm_network_projection_outbox '
. 'BEGIN SELECT RAISE(IGNORE); END'
);
$client->responses[] = $blocked;
$assert($worker->tick(), 'Silent outbox failure was not exercised.');
$log->unprepared('DROP TRIGGER ignore_network_projection_insert');
$cursor = $log->table('cloud_vm_network_monitor_cursors')->first();
$assert((int) $cursor->cursor_value === 2
&& (int) $cursor->consecutive_failures === 2
&& $log->table('cloud_vm_network_observation_samples')->count()
=== $beforeSamples
&& $log->table('cloud_vm_network_projection_outbox')->count()
=== $beforeOutbox,
'A silently ignored outbox insert committed its sample or cursor.');
$window = $blocked['windows'][0];
$uuid = strtolower($instance->uuid);
$bucketStart = (new DateTimeImmutable(
$window['bucket_start']
))->getTimestamp();
$sampleKey = hash('sha256', implode("\n", [
7,
$resetGeneration,
$window['window_id'],
$uuid,
]));
$placementKey = hash('sha256', implode("\n", [
(int) $instance->id,
(int) $instance->user_id,
7,
$uuid,
]));
$conflictingPlacement = str_repeat('f', 64);
$assert($conflictingPlacement !== $placementKey,
'Conflict fixture accidentally matched the canonical placement.');
$log->table('cloud_vm_network_observation_samples')->insert([
'sample_key' => $sampleKey,
'instance_id' => (int) $instance->id,
'user_id' => (int) $instance->user_id,
'host_id' => 7,
'vm_uuid' => $uuid,
'placement_key' => $placementKey,
'collector_generation' => $resetGeneration,
'window_id' => $window['window_id'],
'contract_version' => 1,
'bucket_start' => $bucketStart,
'bucket_seconds' => 60,
'complete' => 1,
'metrics_json' => json_encode([
'interface' => $window['interface'],
'ipv4' => $window['ipv4'],
'ipv6' => $window['ipv6'],
], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES),
'quality_flags_json' => '[]',
'received_at' => $now,
]);
$log->table('cloud_vm_network_projection_outbox')->insert([
'sample_key' => $sampleKey,
'instance_id' => (int) $instance->id,
'bucket_start' => $bucketStart,
'placement_key' => $conflictingPlacement,
'state' => 'pending',
'attempts' => 0,
'available_at' => $now,
'lease_token' => null,
'lease_until' => 0,
'last_error_code' => '',
'projected_at' => null,
'terminal_at' => null,
'discard_reason_code' => '',
'created_at' => $now,
'updated_at' => $now,
]);
$now += 30;
$client->responses[] = $blocked;
$assert($worker->tick(), 'Conflicting outbox replay was not exercised.');
$cursor = $log->table('cloud_vm_network_monitor_cursors')->first();
$assert((int) $cursor->cursor_value === 2
&& (int) $cursor->consecutive_failures === 3
&& $cursor->collection_state === 'unavailable',
'A conflicting outbox replay advanced the durable cursor.');
$log->table('cloud_vm_network_projection_outbox')
->where('sample_key', $sampleKey)
->update(['placement_key' => $placementKey]);
$changedDefinition = $defaultPolicy->definition();
$changedDefinition['windows']['suspicious'] = 4;
$changedPolicy = CloudAbuseDetectionPolicy::fromDefinition(
$changedDefinition
);
$policySnapshot = new CloudAbusePolicySnapshot(
2,
$changedPolicy->hash(),
$changedPolicy,
7,
$now
);
$now += 60;
$client->responses[] = $blocked;
$assert($worker->tick(), 'Exact outbox replay did not recover.');
$cursor = $log->table('cloud_vm_network_monitor_cursors')->first();
$assert((int) $cursor->cursor_value === 3
&& (int) $cursor->consecutive_failures === 0
&& $cursor->collection_state === 'healthy'
&& $log->table('cloud_vm_network_observation_samples')->count()
=== $beforeSamples + 1
&& $log->table('cloud_vm_network_projection_outbox')->count()
=== $beforeOutbox + 1
&& (int) $log->table('cloud_vm_network_projection_outbox')
->where('sample_key', $sampleKey)->value('policy_revision') === 1,
'An exact sample/outbox replay was not accepted idempotently.');
$contentReplay = $batch(
$resetGeneration,
4,
$instance->uuid,
'2026-11-08T01:09:00Z'
);
$contentWindow = $contentReplay['windows'][0];
$contentBucket = (new DateTimeImmutable(
$contentWindow['bucket_start']
))->getTimestamp();
$contentSampleKey = hash('sha256', implode("\n", [
7,
$resetGeneration,
$contentWindow['window_id'],
$uuid,
]));
$expectedMetrics = [
'interface' => $contentWindow['interface'],
'ipv4' => $contentWindow['ipv4'],
'ipv6' => $contentWindow['ipv6'],
];
$conflictingMetrics = $expectedMetrics;
$conflictingMetrics['interface']['rx_bytes'] = 999999;
$log->table('cloud_vm_network_observation_samples')->insert([
'sample_key' => $contentSampleKey,
'instance_id' => (int) $instance->id,
'user_id' => (int) $instance->user_id,
'host_id' => 7,
'vm_uuid' => $uuid,
'placement_key' => $placementKey,
'collector_generation' => $resetGeneration,
'window_id' => $contentWindow['window_id'],
'contract_version' => 1,
'bucket_start' => $contentBucket,
'bucket_seconds' => 60,
'complete' => 1,
'metrics_json' => json_encode(
$conflictingMetrics,
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
),
'quality_flags_json' => '[]',
'received_at' => $now,
]);
$now += 15;
$client->responses[] = $contentReplay;
$assert($worker->tick(), 'Conflicting sample content was not exercised.');
$cursor = $log->table('cloud_vm_network_monitor_cursors')->first();
$assert((int) $cursor->cursor_value === 3
&& (int) $cursor->consecutive_failures === 1
&& !$log->table('cloud_vm_network_projection_outbox')
->where('sample_key', $contentSampleKey)->exists(),
'Conflicting sample content gained an outbox or advanced the cursor.');
$log->table('cloud_vm_network_observation_samples')
->where('sample_key', $contentSampleKey)
->update([
// Reverse object-key order to prove comparison is semantic rather than
// dependent on MySQL JSON serialization.
'metrics_json' => json_encode(
array_reverse($expectedMetrics, true),
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
),
'quality_flags_json' => '[ ]',
]);
$now += 15;
$client->responses[] = $contentReplay;
$assert($worker->tick(), 'Exact sample without outbox did not recover.');
$cursor = $log->table('cloud_vm_network_monitor_cursors')->first();
$assert((int) $cursor->cursor_value === 4
&& (int) $cursor->consecutive_failures === 0
&& $cursor->collection_state === 'healthy'
&& $log->table('cloud_vm_network_projection_outbox')
->where('sample_key', $contentSampleKey)
->where('instance_id', (int) $instance->id)
->where('bucket_start', $contentBucket)
->where('placement_key', $placementKey)
->exists(),
'An exact sample replay did not repair its missing outbox.');
$source = file_get_contents(dirname(__DIR__)
. '/app/Services/Cloud/CloudAbuseObservationIngestService.php');
$assert(is_string($source)
&& !str_contains($source, 'cloud_jobs')
&& !str_contains($source, 'service_status')
&& !str_contains($source, 'network_status'),
'Observation ingestion gained a task, lifecycle or enforcement dependency.');
echo "Cloud abuse observation ingestion tests passed.\n";