You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
368 lines
12 KiB
PHP
368 lines
12 KiB
PHP
<?php
|
|
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
|
|
|
use App\Command\CloudAbuseMonitor;
|
|
use App\Services\Cloud\CloudRuntimeHeartbeat;
|
|
|
|
$assert = static function (bool $condition, string $message): void {
|
|
if (!$condition) {
|
|
throw new RuntimeException($message);
|
|
}
|
|
};
|
|
$root = dirname(__DIR__, 2);
|
|
$read = static function (string $path) use ($root): string {
|
|
$value = file_get_contents($root . '/' . $path);
|
|
if (!is_string($value)) {
|
|
throw new RuntimeException('Unable to read ' . $path);
|
|
}
|
|
return $value;
|
|
};
|
|
$temporary = sys_get_temp_dir() . '/virt-abuse-monitor-'
|
|
. bin2hex(random_bytes(8));
|
|
if (!mkdir($temporary, 0700, true) && !is_dir($temporary)) {
|
|
throw new RuntimeException('Unable to create heartbeat fixture.');
|
|
}
|
|
$cleanup = static function (string $directory): void {
|
|
if (!is_dir($directory)) {
|
|
return;
|
|
}
|
|
foreach (new FilesystemIterator($directory) as $entry) {
|
|
if ($entry->isFile() || $entry->isLink()) {
|
|
unlink($entry->getPathname());
|
|
}
|
|
}
|
|
rmdir($directory);
|
|
};
|
|
|
|
try {
|
|
$now = 2_000_000_000;
|
|
$ingest = new class {
|
|
public int $ticks = 0;
|
|
public function tick(): bool
|
|
{
|
|
$this->ticks++;
|
|
return true;
|
|
}
|
|
public function observationErrors(): array
|
|
{
|
|
return [];
|
|
}
|
|
};
|
|
$projection = new class {
|
|
public int $ticks = 0;
|
|
public function tick(): bool
|
|
{
|
|
$this->ticks++;
|
|
return $this->ticks <= 2;
|
|
}
|
|
public function observationErrors(): array
|
|
{
|
|
return [];
|
|
}
|
|
};
|
|
$retention = new class {
|
|
public int $runs = 0;
|
|
public function prune(): array
|
|
{
|
|
$this->runs++;
|
|
return ['terminal_outboxes' => 0, 'samples' => 1,
|
|
'baselines' => 0, 'receipts' => 0, 'findings' => 0];
|
|
}
|
|
};
|
|
$heartbeat = CloudRuntimeHeartbeat::start(
|
|
'abuse_monitor',
|
|
true,
|
|
$temporary,
|
|
static fn (): int => $now
|
|
);
|
|
$status = CloudAbuseMonitor::run(
|
|
['cloudAbuseMonitor', '--once'],
|
|
$heartbeat,
|
|
$ingest,
|
|
$projection,
|
|
$retention,
|
|
static fn (): int => $now,
|
|
static function (int $_microseconds): void {
|
|
throw new RuntimeException('Once mode attempted to sleep.');
|
|
}
|
|
);
|
|
$records = iterator_to_array(new FilesystemIterator($temporary));
|
|
$record = json_decode(
|
|
file_get_contents(reset($records)->getPathname()),
|
|
true,
|
|
16,
|
|
JSON_THROW_ON_ERROR
|
|
);
|
|
$assert($status === 0
|
|
&& $ingest->ticks === 1
|
|
&& $projection->ticks === 3
|
|
&& $retention->runs === 1
|
|
&& $record['kind'] === 'abuse_monitor'
|
|
&& $record['state'] === 'stopped'
|
|
&& $record['last_success'] === $now
|
|
&& $record['unresolved_error'] === false,
|
|
'Observe-only worker did not drain its bounded burst cleanly.');
|
|
|
|
$failureDirectory = $temporary . '-failure';
|
|
mkdir($failureDirectory, 0700, true);
|
|
$failedIngest = new class {
|
|
public function tick(): bool
|
|
{
|
|
return true;
|
|
}
|
|
public function observationErrors(): array
|
|
{
|
|
return ['network_observation_collection_failed'];
|
|
}
|
|
};
|
|
$idleProjection = new class {
|
|
public function tick(): bool
|
|
{
|
|
return false;
|
|
}
|
|
public function observationErrors(): array
|
|
{
|
|
return [];
|
|
}
|
|
};
|
|
$idleRetention = new class {
|
|
public function prune(): array
|
|
{
|
|
return ['terminal_outboxes' => 0, 'samples' => 0,
|
|
'baselines' => 0, 'receipts' => 0, 'findings' => 0];
|
|
}
|
|
};
|
|
$failedHeartbeat = CloudRuntimeHeartbeat::start(
|
|
'abuse_monitor',
|
|
true,
|
|
$failureDirectory,
|
|
static fn (): int => $now
|
|
);
|
|
$failedStatus = CloudAbuseMonitor::run(
|
|
['cloudAbuseMonitor', '--once'],
|
|
$failedHeartbeat,
|
|
$failedIngest,
|
|
$idleProjection,
|
|
$idleRetention,
|
|
static fn (): int => $now
|
|
);
|
|
$failedFiles = iterator_to_array(
|
|
new FilesystemIterator($failureDirectory)
|
|
);
|
|
$failedRecord = json_decode(
|
|
file_get_contents(reset($failedFiles)->getPathname()),
|
|
true,
|
|
16,
|
|
JSON_THROW_ON_ERROR
|
|
);
|
|
$assert($failedStatus === 1
|
|
&& $failedRecord['state'] === 'stopped'
|
|
&& $failedRecord['last_error']
|
|
=== 'abuse_monitor_collection_failed'
|
|
&& $failedRecord['unresolved_error'] === true,
|
|
'A caught collection failure was falsely projected healthy.');
|
|
$cleanup($failureDirectory);
|
|
|
|
$safeDirectory = $temporary . '-safe-projection';
|
|
mkdir($safeDirectory, 0700, true);
|
|
$idleIngest = new class {
|
|
public function tick(): bool
|
|
{
|
|
return false;
|
|
}
|
|
public function observationErrors(): array
|
|
{
|
|
return [];
|
|
}
|
|
};
|
|
$safeProjection = new class {
|
|
public int $ticks = 0;
|
|
public function tick(): bool
|
|
{
|
|
return ++$this->ticks === 1;
|
|
}
|
|
public function observationErrors(): array
|
|
{
|
|
return $this->ticks === 1
|
|
? ['instance_deleted'] : [];
|
|
}
|
|
};
|
|
$safeHeartbeat = CloudRuntimeHeartbeat::start(
|
|
'abuse_monitor',
|
|
true,
|
|
$safeDirectory,
|
|
static fn (): int => $now
|
|
);
|
|
$safeStatus = CloudAbuseMonitor::run(
|
|
['cloudAbuseMonitor', '--once'],
|
|
$safeHeartbeat,
|
|
$idleIngest,
|
|
$safeProjection,
|
|
$idleRetention,
|
|
static fn (): int => $now
|
|
);
|
|
$safeFiles = iterator_to_array(new FilesystemIterator($safeDirectory));
|
|
$safeRecord = json_decode(
|
|
file_get_contents(reset($safeFiles)->getPathname()),
|
|
true,
|
|
16,
|
|
JSON_THROW_ON_ERROR
|
|
);
|
|
$assert($safeStatus === 0
|
|
&& $safeRecord['unresolved_error'] === false,
|
|
'A safely converged projection discard raised a worker failure.');
|
|
$cleanup($safeDirectory);
|
|
|
|
$conflictDirectory = $temporary . '-projection-conflict';
|
|
mkdir($conflictDirectory, 0700, true);
|
|
$conflictProjection = new class {
|
|
public int $ticks = 0;
|
|
public function tick(): bool
|
|
{
|
|
return ++$this->ticks === 1;
|
|
}
|
|
public function observationErrors(): array
|
|
{
|
|
return $this->ticks === 1
|
|
? ['projection_order_conflict'] : [];
|
|
}
|
|
};
|
|
$conflictHeartbeat = CloudRuntimeHeartbeat::start(
|
|
'abuse_monitor',
|
|
true,
|
|
$conflictDirectory,
|
|
static fn (): int => $now
|
|
);
|
|
$conflictStatus = CloudAbuseMonitor::run(
|
|
['cloudAbuseMonitor', '--once'],
|
|
$conflictHeartbeat,
|
|
$idleIngest,
|
|
$conflictProjection,
|
|
$idleRetention,
|
|
static fn (): int => $now
|
|
);
|
|
$conflictFiles = iterator_to_array(
|
|
new FilesystemIterator($conflictDirectory)
|
|
);
|
|
$conflictRecord = json_decode(
|
|
file_get_contents(reset($conflictFiles)->getPathname()),
|
|
true,
|
|
16,
|
|
JSON_THROW_ON_ERROR
|
|
);
|
|
$assert($conflictStatus === 1
|
|
&& $conflictRecord['last_error']
|
|
=== 'abuse_monitor_projection_failed'
|
|
&& $conflictRecord['unresolved_error'] === true,
|
|
'A projection ordering conflict was silently treated as safe.');
|
|
$cleanup($conflictDirectory);
|
|
|
|
// A stage clears its own warning after a successful zero-work retry; it
|
|
// must not depend on unrelated collection or projection work.
|
|
$recoveryDirectory = $temporary . '-retention-recovery';
|
|
mkdir($recoveryDirectory, 0700, true);
|
|
$recoveringRetention = new class {
|
|
public int $runs = 0;
|
|
public function prune(): array
|
|
{
|
|
if (++$this->runs === 1) {
|
|
throw new RuntimeException('simulated retention outage');
|
|
}
|
|
return ['terminal_outboxes' => 0, 'samples' => 0,
|
|
'baselines' => 0, 'receipts' => 0, 'findings' => 0];
|
|
}
|
|
};
|
|
$recoveryNow = $now;
|
|
$sleepCalls = 0;
|
|
$recoveryHeartbeat = CloudRuntimeHeartbeat::start(
|
|
'abuse_monitor',
|
|
false,
|
|
$recoveryDirectory,
|
|
static function () use (&$recoveryNow): int {
|
|
return $recoveryNow;
|
|
}
|
|
);
|
|
try {
|
|
CloudAbuseMonitor::run(
|
|
['cloudAbuseMonitor'],
|
|
$recoveryHeartbeat,
|
|
$idleIngest,
|
|
$idleProjection,
|
|
$recoveringRetention,
|
|
static function () use (&$recoveryNow): int {
|
|
return $recoveryNow;
|
|
},
|
|
static function (int $_microseconds) use (
|
|
&$sleepCalls,
|
|
&$recoveryNow,
|
|
$recoveryDirectory,
|
|
$assert
|
|
): void {
|
|
++$sleepCalls;
|
|
$recoveryNow += 301;
|
|
if ($sleepCalls < 2) {
|
|
return;
|
|
}
|
|
$files = iterator_to_array(
|
|
new FilesystemIterator($recoveryDirectory)
|
|
);
|
|
$row = json_decode(
|
|
file_get_contents(reset($files)->getPathname()),
|
|
true,
|
|
16,
|
|
JSON_THROW_ON_ERROR
|
|
);
|
|
$assert($row['state'] === 'idle'
|
|
&& $row['unresolved_error'] === false
|
|
&& $row['last_success'] === $recoveryNow - 301,
|
|
'Successful zero-work retention did not clear its warning.');
|
|
throw new RuntimeException('stop daemon fixture');
|
|
}
|
|
);
|
|
throw new RuntimeException('Daemon fixture did not stop.');
|
|
} catch (RuntimeException $error) {
|
|
if ($error->getMessage() !== 'stop daemon fixture') {
|
|
throw $error;
|
|
}
|
|
}
|
|
$cleanup($recoveryDirectory);
|
|
|
|
$command = $read('web/app/Command/CloudAbuseMonitor.php');
|
|
$xcat = $read('web/app/Command/XCat.php');
|
|
$heartbeatSource = $read(
|
|
'web/app/Services/Cloud/CloudRuntimeHeartbeat.php'
|
|
);
|
|
$runtime = $read(
|
|
'web/app/Services/Cloud/CloudRuntimeHealthService.php'
|
|
);
|
|
$unit = $read('deploy/virt-cloud-abuse-monitor.service');
|
|
$deployment = $read('docs/DEPLOYMENT.md');
|
|
$assert(str_contains($xcat, "'cloudAbuseMonitor'")
|
|
&& str_contains($xcat, 'CloudAbuseMonitor::run')
|
|
&& str_contains($command, 'PROJECTION_BURST = 64')
|
|
&& str_contains($command, 'CloudAbuseObservationIngestService')
|
|
&& str_contains($command, 'CloudAbuseProjectionService')
|
|
&& str_contains($command, 'CloudAbuseRetentionService')
|
|
&& !str_contains($command, "table('cloud_jobs')")
|
|
&& !str_contains($command, 'CloudJob'),
|
|
'Monitor command is unreachable, unbounded, or coupled to jobs.');
|
|
$assert(str_contains($heartbeatSource, "'abuse_monitor'")
|
|
&& str_contains($heartbeatSource, "'abuse_monitor_ingest'")
|
|
&& str_contains($runtime,
|
|
"'abuse_monitor' => '网络滥用监测'"),
|
|
'Runtime observation does not expose the independent monitor.');
|
|
$assert(str_contains($unit,
|
|
'ExecStart=/usr/bin/php /home/virt-cloud/web/xcat cloudAbuseMonitor')
|
|
&& str_contains($unit, 'NoNewPrivileges=true')
|
|
&& str_contains($unit, 'ProtectSystem=strict')
|
|
&& str_contains($unit, 'LimitCORE=0')
|
|
&& str_contains($deployment,
|
|
'virt-cloud-abuse-monitor.service'),
|
|
'Monitor systemd isolation or deployment documentation is incomplete.');
|
|
} finally {
|
|
$cleanup($temporary);
|
|
}
|
|
|
|
echo "Observe-only abuse monitor worker contract tests passed.\n";
|