feat(cloud): run bounded abuse monitoring worker
parent
7a9a364a66
commit
bd103fc64b
@ -0,0 +1,25 @@
|
||||
[Unit]
|
||||
Description=Virt Cloud observe-only network abuse monitor
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=apache
|
||||
Group=apache
|
||||
WorkingDirectory=/home/virt-cloud/web
|
||||
ExecStart=/usr/bin/php /home/virt-cloud/web/xcat cloudAbuseMonitor
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
LimitCORE=0
|
||||
UMask=0077
|
||||
NoNewPrivileges=true
|
||||
PrivateTmp=true
|
||||
ProtectHome=false
|
||||
ProtectSystem=strict
|
||||
ReadWritePaths=-/home/virt-cloud/runtime/php
|
||||
ReadWritePaths=-/home/virt-cloud/web/storage
|
||||
TimeoutStopSec=30
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@ -0,0 +1,198 @@
|
||||
<?php
|
||||
|
||||
namespace App\Command;
|
||||
|
||||
use App\Services\Cloud\CloudAbuseObservationIngestService;
|
||||
use App\Services\Cloud\CloudAbuseProjectionService;
|
||||
use App\Services\Cloud\CloudAbuseRetentionService;
|
||||
use App\Services\Cloud\CloudRuntimeHeartbeat;
|
||||
|
||||
/**
|
||||
* Independent, observe-only network-abuse monitor.
|
||||
*
|
||||
* Collection and projection deliberately stay outside cloud_jobs: every host
|
||||
* call is read-only and every database write is an idempotent telemetry or
|
||||
* read-model update. No lifecycle, billing, firewall or task mutation belongs
|
||||
* in this loop.
|
||||
*/
|
||||
final class CloudAbuseMonitor
|
||||
{
|
||||
private const PROJECTION_BURST = 64;
|
||||
private const RETENTION_INTERVAL_SECONDS = 300;
|
||||
private const IDLE_SLEEP_MICROSECONDS = 1_000_000;
|
||||
private const SAFE_PROJECTION_DIAGNOSTICS = [
|
||||
'instance_deleted',
|
||||
'instance_owner_changed',
|
||||
'projection_receipt_baseline_reset',
|
||||
];
|
||||
|
||||
/**
|
||||
* Injectable objects and clock keep the command testable without turning
|
||||
* the production services into interfaces solely for the command layer.
|
||||
*
|
||||
* @param null|object $ingest must expose tick() and observationErrors()
|
||||
* @param null|object $projection must expose tick() and observationErrors()
|
||||
* @param null|object $retention must expose prune()
|
||||
* @param null|callable():int $clock
|
||||
* @param null|callable(int):void $sleep
|
||||
*/
|
||||
public static function run(
|
||||
array $argv = [],
|
||||
?CloudRuntimeHeartbeat $heartbeat = null,
|
||||
?object $ingest = null,
|
||||
?object $projection = null,
|
||||
?object $retention = null,
|
||||
?callable $clock = null,
|
||||
?callable $sleep = null
|
||||
): int {
|
||||
$once = in_array('--once', $argv, true);
|
||||
$heartbeat ??= CloudRuntimeHeartbeat::start(
|
||||
'abuse_monitor',
|
||||
$once
|
||||
);
|
||||
$ingest ??= new CloudAbuseObservationIngestService();
|
||||
$projection ??= new CloudAbuseProjectionService();
|
||||
$retention ??= new CloudAbuseRetentionService();
|
||||
$clock ??= static fn (): int => time();
|
||||
$sleep ??= static function (int $microseconds): void {
|
||||
usleep($microseconds);
|
||||
};
|
||||
self::assertWorker($ingest, ['tick', 'observationErrors']);
|
||||
self::assertWorker($projection, ['tick', 'observationErrors']);
|
||||
self::assertWorker($retention, ['prune']);
|
||||
|
||||
$nextRetentionAt = 0;
|
||||
$unresolved = [
|
||||
'collection' => '',
|
||||
'projection' => '',
|
||||
'retention' => '',
|
||||
];
|
||||
try {
|
||||
do {
|
||||
$worked = false;
|
||||
|
||||
$heartbeat->busy('abuse_monitor_ingest');
|
||||
try {
|
||||
$worked = $ingest->tick() === true || $worked;
|
||||
$unresolved['collection'] =
|
||||
$ingest->observationErrors() === []
|
||||
? '' : 'abuse_monitor_collection_failed';
|
||||
} catch (\Throwable $error) {
|
||||
$unresolved['collection'] =
|
||||
'abuse_monitor_collection_failed';
|
||||
self::writeFailure('collection', $error);
|
||||
}
|
||||
|
||||
$heartbeat->busy('abuse_monitor_projection');
|
||||
try {
|
||||
$projectionFailed = false;
|
||||
for ($projected = 0;
|
||||
$projected < self::PROJECTION_BURST;
|
||||
$projected++
|
||||
) {
|
||||
if ($projection->tick() !== true) {
|
||||
break;
|
||||
}
|
||||
$worked = true;
|
||||
$projectionFailed = self::hasProjectionFailure(
|
||||
$projection->observationErrors()
|
||||
) || $projectionFailed;
|
||||
}
|
||||
$unresolved['projection'] = $projectionFailed
|
||||
? 'abuse_monitor_projection_failed' : '';
|
||||
} catch (\Throwable $error) {
|
||||
$unresolved['projection'] =
|
||||
'abuse_monitor_projection_failed';
|
||||
self::writeFailure('projection', $error);
|
||||
}
|
||||
|
||||
$now = $clock();
|
||||
if (!is_int($now) || $now < 1) {
|
||||
throw new \LogicException(
|
||||
'Abuse monitor command clock is invalid'
|
||||
);
|
||||
}
|
||||
if ($now >= $nextRetentionAt) {
|
||||
$heartbeat->busy('abuse_monitor_retention');
|
||||
try {
|
||||
$pruned = $retention->prune();
|
||||
if (!is_array($pruned)) {
|
||||
throw new \RuntimeException(
|
||||
'Abuse retention returned an invalid result'
|
||||
);
|
||||
}
|
||||
$worked = array_sum(array_map(
|
||||
static fn (mixed $value): int =>
|
||||
is_int($value) && $value > 0 ? $value : 0,
|
||||
$pruned
|
||||
)) > 0 || $worked;
|
||||
$unresolved['retention'] = '';
|
||||
} catch (\Throwable $error) {
|
||||
$unresolved['retention'] =
|
||||
'abuse_monitor_retention_failed';
|
||||
self::writeFailure('retention', $error);
|
||||
}
|
||||
$nextRetentionAt = $now
|
||||
+ self::RETENTION_INTERVAL_SECONDS;
|
||||
}
|
||||
|
||||
$errors = array_values(array_filter($unresolved));
|
||||
if ($errors !== []) {
|
||||
$heartbeat->error($errors[0]);
|
||||
} else {
|
||||
$heartbeat->success();
|
||||
}
|
||||
if ($once) {
|
||||
return $errors === [] ? 0 : 1;
|
||||
}
|
||||
if (!$worked) {
|
||||
$sleep(self::IDLE_SLEEP_MICROSECONDS);
|
||||
}
|
||||
} while (true);
|
||||
} catch (\Throwable $error) {
|
||||
$heartbeat->error('abuse_monitor_runtime_failed');
|
||||
throw $error;
|
||||
} finally {
|
||||
$heartbeat->stopped();
|
||||
}
|
||||
}
|
||||
|
||||
/** @param list<string> $methods */
|
||||
private static function assertWorker(object $worker, array $methods): void
|
||||
{
|
||||
foreach ($methods as $method) {
|
||||
if (!is_callable([$worker, $method])) {
|
||||
throw new \LogicException(
|
||||
'Abuse monitor dependency is invalid'
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static function writeFailure(
|
||||
string $stage,
|
||||
\Throwable $error
|
||||
): void {
|
||||
fwrite(
|
||||
STDERR,
|
||||
'abuse monitor ' . $stage . ' temporarily unavailable: '
|
||||
. get_class($error) . PHP_EOL
|
||||
);
|
||||
}
|
||||
|
||||
/** @param list<mixed> $diagnostics */
|
||||
private static function hasProjectionFailure(array $diagnostics): bool
|
||||
{
|
||||
foreach ($diagnostics as $diagnostic) {
|
||||
if (!is_string($diagnostic)
|
||||
|| !in_array(
|
||||
$diagnostic,
|
||||
self::SAFE_PROJECTION_DIAGNOSTICS,
|
||||
true
|
||||
)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,367 @@
|
||||
<?php
|
||||
|
||||
require dirname(__DIR__) . '/vendor/autoload.php';
|
||||
|
||||
use App\Command\CloudAbuseMonitor;
|
||||
use App\Services\Cloud\CloudRuntimeHeartbeat;
|
||||
|
||||
$assert = static function (bool $condition, string $message): void {
|
||||
if (!$condition) {
|
||||
throw new RuntimeException($message);
|
||||
}
|
||||
};
|
||||
$root = dirname(__DIR__, 2);
|
||||
$read = static function (string $path) use ($root): string {
|
||||
$value = file_get_contents($root . '/' . $path);
|
||||
if (!is_string($value)) {
|
||||
throw new RuntimeException('Unable to read ' . $path);
|
||||
}
|
||||
return $value;
|
||||
};
|
||||
$temporary = sys_get_temp_dir() . '/virt-abuse-monitor-'
|
||||
. bin2hex(random_bytes(8));
|
||||
if (!mkdir($temporary, 0700, true) && !is_dir($temporary)) {
|
||||
throw new RuntimeException('Unable to create heartbeat fixture.');
|
||||
}
|
||||
$cleanup = static function (string $directory): void {
|
||||
if (!is_dir($directory)) {
|
||||
return;
|
||||
}
|
||||
foreach (new FilesystemIterator($directory) as $entry) {
|
||||
if ($entry->isFile() || $entry->isLink()) {
|
||||
unlink($entry->getPathname());
|
||||
}
|
||||
}
|
||||
rmdir($directory);
|
||||
};
|
||||
|
||||
try {
|
||||
$now = 2_000_000_000;
|
||||
$ingest = new class {
|
||||
public int $ticks = 0;
|
||||
public function tick(): bool
|
||||
{
|
||||
$this->ticks++;
|
||||
return true;
|
||||
}
|
||||
public function observationErrors(): array
|
||||
{
|
||||
return [];
|
||||
}
|
||||
};
|
||||
$projection = new class {
|
||||
public int $ticks = 0;
|
||||
public function tick(): bool
|
||||
{
|
||||
$this->ticks++;
|
||||
return $this->ticks <= 2;
|
||||
}
|
||||
public function observationErrors(): array
|
||||
{
|
||||
return [];
|
||||
}
|
||||
};
|
||||
$retention = new class {
|
||||
public int $runs = 0;
|
||||
public function prune(): array
|
||||
{
|
||||
$this->runs++;
|
||||
return ['terminal_outboxes' => 0, 'samples' => 1,
|
||||
'baselines' => 0, 'receipts' => 0, 'findings' => 0];
|
||||
}
|
||||
};
|
||||
$heartbeat = CloudRuntimeHeartbeat::start(
|
||||
'abuse_monitor',
|
||||
true,
|
||||
$temporary,
|
||||
static fn (): int => $now
|
||||
);
|
||||
$status = CloudAbuseMonitor::run(
|
||||
['cloudAbuseMonitor', '--once'],
|
||||
$heartbeat,
|
||||
$ingest,
|
||||
$projection,
|
||||
$retention,
|
||||
static fn (): int => $now,
|
||||
static function (int $_microseconds): void {
|
||||
throw new RuntimeException('Once mode attempted to sleep.');
|
||||
}
|
||||
);
|
||||
$records = iterator_to_array(new FilesystemIterator($temporary));
|
||||
$record = json_decode(
|
||||
file_get_contents(reset($records)->getPathname()),
|
||||
true,
|
||||
16,
|
||||
JSON_THROW_ON_ERROR
|
||||
);
|
||||
$assert($status === 0
|
||||
&& $ingest->ticks === 1
|
||||
&& $projection->ticks === 3
|
||||
&& $retention->runs === 1
|
||||
&& $record['kind'] === 'abuse_monitor'
|
||||
&& $record['state'] === 'stopped'
|
||||
&& $record['last_success'] === $now
|
||||
&& $record['unresolved_error'] === false,
|
||||
'Observe-only worker did not drain its bounded burst cleanly.');
|
||||
|
||||
$failureDirectory = $temporary . '-failure';
|
||||
mkdir($failureDirectory, 0700, true);
|
||||
$failedIngest = new class {
|
||||
public function tick(): bool
|
||||
{
|
||||
return true;
|
||||
}
|
||||
public function observationErrors(): array
|
||||
{
|
||||
return ['network_observation_collection_failed'];
|
||||
}
|
||||
};
|
||||
$idleProjection = new class {
|
||||
public function tick(): bool
|
||||
{
|
||||
return false;
|
||||
}
|
||||
public function observationErrors(): array
|
||||
{
|
||||
return [];
|
||||
}
|
||||
};
|
||||
$idleRetention = new class {
|
||||
public function prune(): array
|
||||
{
|
||||
return ['terminal_outboxes' => 0, 'samples' => 0,
|
||||
'baselines' => 0, 'receipts' => 0, 'findings' => 0];
|
||||
}
|
||||
};
|
||||
$failedHeartbeat = CloudRuntimeHeartbeat::start(
|
||||
'abuse_monitor',
|
||||
true,
|
||||
$failureDirectory,
|
||||
static fn (): int => $now
|
||||
);
|
||||
$failedStatus = CloudAbuseMonitor::run(
|
||||
['cloudAbuseMonitor', '--once'],
|
||||
$failedHeartbeat,
|
||||
$failedIngest,
|
||||
$idleProjection,
|
||||
$idleRetention,
|
||||
static fn (): int => $now
|
||||
);
|
||||
$failedFiles = iterator_to_array(
|
||||
new FilesystemIterator($failureDirectory)
|
||||
);
|
||||
$failedRecord = json_decode(
|
||||
file_get_contents(reset($failedFiles)->getPathname()),
|
||||
true,
|
||||
16,
|
||||
JSON_THROW_ON_ERROR
|
||||
);
|
||||
$assert($failedStatus === 1
|
||||
&& $failedRecord['state'] === 'stopped'
|
||||
&& $failedRecord['last_error']
|
||||
=== 'abuse_monitor_collection_failed'
|
||||
&& $failedRecord['unresolved_error'] === true,
|
||||
'A caught collection failure was falsely projected healthy.');
|
||||
$cleanup($failureDirectory);
|
||||
|
||||
$safeDirectory = $temporary . '-safe-projection';
|
||||
mkdir($safeDirectory, 0700, true);
|
||||
$idleIngest = new class {
|
||||
public function tick(): bool
|
||||
{
|
||||
return false;
|
||||
}
|
||||
public function observationErrors(): array
|
||||
{
|
||||
return [];
|
||||
}
|
||||
};
|
||||
$safeProjection = new class {
|
||||
public int $ticks = 0;
|
||||
public function tick(): bool
|
||||
{
|
||||
return ++$this->ticks === 1;
|
||||
}
|
||||
public function observationErrors(): array
|
||||
{
|
||||
return $this->ticks === 1
|
||||
? ['instance_deleted'] : [];
|
||||
}
|
||||
};
|
||||
$safeHeartbeat = CloudRuntimeHeartbeat::start(
|
||||
'abuse_monitor',
|
||||
true,
|
||||
$safeDirectory,
|
||||
static fn (): int => $now
|
||||
);
|
||||
$safeStatus = CloudAbuseMonitor::run(
|
||||
['cloudAbuseMonitor', '--once'],
|
||||
$safeHeartbeat,
|
||||
$idleIngest,
|
||||
$safeProjection,
|
||||
$idleRetention,
|
||||
static fn (): int => $now
|
||||
);
|
||||
$safeFiles = iterator_to_array(new FilesystemIterator($safeDirectory));
|
||||
$safeRecord = json_decode(
|
||||
file_get_contents(reset($safeFiles)->getPathname()),
|
||||
true,
|
||||
16,
|
||||
JSON_THROW_ON_ERROR
|
||||
);
|
||||
$assert($safeStatus === 0
|
||||
&& $safeRecord['unresolved_error'] === false,
|
||||
'A safely converged projection discard raised a worker failure.');
|
||||
$cleanup($safeDirectory);
|
||||
|
||||
$conflictDirectory = $temporary . '-projection-conflict';
|
||||
mkdir($conflictDirectory, 0700, true);
|
||||
$conflictProjection = new class {
|
||||
public int $ticks = 0;
|
||||
public function tick(): bool
|
||||
{
|
||||
return ++$this->ticks === 1;
|
||||
}
|
||||
public function observationErrors(): array
|
||||
{
|
||||
return $this->ticks === 1
|
||||
? ['projection_order_conflict'] : [];
|
||||
}
|
||||
};
|
||||
$conflictHeartbeat = CloudRuntimeHeartbeat::start(
|
||||
'abuse_monitor',
|
||||
true,
|
||||
$conflictDirectory,
|
||||
static fn (): int => $now
|
||||
);
|
||||
$conflictStatus = CloudAbuseMonitor::run(
|
||||
['cloudAbuseMonitor', '--once'],
|
||||
$conflictHeartbeat,
|
||||
$idleIngest,
|
||||
$conflictProjection,
|
||||
$idleRetention,
|
||||
static fn (): int => $now
|
||||
);
|
||||
$conflictFiles = iterator_to_array(
|
||||
new FilesystemIterator($conflictDirectory)
|
||||
);
|
||||
$conflictRecord = json_decode(
|
||||
file_get_contents(reset($conflictFiles)->getPathname()),
|
||||
true,
|
||||
16,
|
||||
JSON_THROW_ON_ERROR
|
||||
);
|
||||
$assert($conflictStatus === 1
|
||||
&& $conflictRecord['last_error']
|
||||
=== 'abuse_monitor_projection_failed'
|
||||
&& $conflictRecord['unresolved_error'] === true,
|
||||
'A projection ordering conflict was silently treated as safe.');
|
||||
$cleanup($conflictDirectory);
|
||||
|
||||
// A stage clears its own warning after a successful zero-work retry; it
|
||||
// must not depend on unrelated collection or projection work.
|
||||
$recoveryDirectory = $temporary . '-retention-recovery';
|
||||
mkdir($recoveryDirectory, 0700, true);
|
||||
$recoveringRetention = new class {
|
||||
public int $runs = 0;
|
||||
public function prune(): array
|
||||
{
|
||||
if (++$this->runs === 1) {
|
||||
throw new RuntimeException('simulated retention outage');
|
||||
}
|
||||
return ['terminal_outboxes' => 0, 'samples' => 0,
|
||||
'baselines' => 0, 'receipts' => 0, 'findings' => 0];
|
||||
}
|
||||
};
|
||||
$recoveryNow = $now;
|
||||
$sleepCalls = 0;
|
||||
$recoveryHeartbeat = CloudRuntimeHeartbeat::start(
|
||||
'abuse_monitor',
|
||||
false,
|
||||
$recoveryDirectory,
|
||||
static function () use (&$recoveryNow): int {
|
||||
return $recoveryNow;
|
||||
}
|
||||
);
|
||||
try {
|
||||
CloudAbuseMonitor::run(
|
||||
['cloudAbuseMonitor'],
|
||||
$recoveryHeartbeat,
|
||||
$idleIngest,
|
||||
$idleProjection,
|
||||
$recoveringRetention,
|
||||
static function () use (&$recoveryNow): int {
|
||||
return $recoveryNow;
|
||||
},
|
||||
static function (int $_microseconds) use (
|
||||
&$sleepCalls,
|
||||
&$recoveryNow,
|
||||
$recoveryDirectory,
|
||||
$assert
|
||||
): void {
|
||||
++$sleepCalls;
|
||||
$recoveryNow += 301;
|
||||
if ($sleepCalls < 2) {
|
||||
return;
|
||||
}
|
||||
$files = iterator_to_array(
|
||||
new FilesystemIterator($recoveryDirectory)
|
||||
);
|
||||
$row = json_decode(
|
||||
file_get_contents(reset($files)->getPathname()),
|
||||
true,
|
||||
16,
|
||||
JSON_THROW_ON_ERROR
|
||||
);
|
||||
$assert($row['state'] === 'idle'
|
||||
&& $row['unresolved_error'] === false
|
||||
&& $row['last_success'] === $recoveryNow - 301,
|
||||
'Successful zero-work retention did not clear its warning.');
|
||||
throw new RuntimeException('stop daemon fixture');
|
||||
}
|
||||
);
|
||||
throw new RuntimeException('Daemon fixture did not stop.');
|
||||
} catch (RuntimeException $error) {
|
||||
if ($error->getMessage() !== 'stop daemon fixture') {
|
||||
throw $error;
|
||||
}
|
||||
}
|
||||
$cleanup($recoveryDirectory);
|
||||
|
||||
$command = $read('web/app/Command/CloudAbuseMonitor.php');
|
||||
$xcat = $read('web/app/Command/XCat.php');
|
||||
$heartbeatSource = $read(
|
||||
'web/app/Services/Cloud/CloudRuntimeHeartbeat.php'
|
||||
);
|
||||
$runtime = $read(
|
||||
'web/app/Services/Cloud/CloudRuntimeHealthService.php'
|
||||
);
|
||||
$unit = $read('deploy/virt-cloud-abuse-monitor.service');
|
||||
$deployment = $read('docs/DEPLOYMENT.md');
|
||||
$assert(str_contains($xcat, "'cloudAbuseMonitor'")
|
||||
&& str_contains($xcat, 'CloudAbuseMonitor::run')
|
||||
&& str_contains($command, 'PROJECTION_BURST = 64')
|
||||
&& str_contains($command, 'CloudAbuseObservationIngestService')
|
||||
&& str_contains($command, 'CloudAbuseProjectionService')
|
||||
&& str_contains($command, 'CloudAbuseRetentionService')
|
||||
&& !str_contains($command, "table('cloud_jobs')")
|
||||
&& !str_contains($command, 'CloudJob'),
|
||||
'Monitor command is unreachable, unbounded, or coupled to jobs.');
|
||||
$assert(str_contains($heartbeatSource, "'abuse_monitor'")
|
||||
&& str_contains($heartbeatSource, "'abuse_monitor_ingest'")
|
||||
&& str_contains($runtime,
|
||||
"'abuse_monitor' => '网络滥用监测'"),
|
||||
'Runtime observation does not expose the independent monitor.');
|
||||
$assert(str_contains($unit,
|
||||
'ExecStart=/usr/bin/php /home/virt-cloud/web/xcat cloudAbuseMonitor')
|
||||
&& str_contains($unit, 'NoNewPrivileges=true')
|
||||
&& str_contains($unit, 'ProtectSystem=strict')
|
||||
&& str_contains($unit, 'LimitCORE=0')
|
||||
&& str_contains($deployment,
|
||||
'virt-cloud-abuse-monitor.service'),
|
||||
'Monitor systemd isolation or deployment documentation is incomplete.');
|
||||
} finally {
|
||||
$cleanup($temporary);
|
||||
}
|
||||
|
||||
echo "Observe-only abuse monitor worker contract tests passed.\n";
|
||||
Loading…
Reference in New Issue