feat(cloud): run bounded abuse monitoring worker

go
codex 1 week ago
parent 7a9a364a66
commit bd103fc64b

@ -0,0 +1,25 @@
[Unit]
Description=Virt Cloud observe-only network abuse monitor
After=network-online.target
Wants=network-online.target
[Service]
Type=simple
User=apache
Group=apache
WorkingDirectory=/home/virt-cloud/web
ExecStart=/usr/bin/php /home/virt-cloud/web/xcat cloudAbuseMonitor
Restart=on-failure
RestartSec=5
LimitCORE=0
UMask=0077
NoNewPrivileges=true
PrivateTmp=true
ProtectHome=false
ProtectSystem=strict
ReadWritePaths=-/home/virt-cloud/runtime/php
ReadWritePaths=-/home/virt-cloud/web/storage
TimeoutStopSec=30
[Install]
WantedBy=multi-user.target

@ -232,6 +232,7 @@ cp /home/virt-cloud/deploy/virt-cloud-notification-worker.service /etc/systemd/s
cp /home/virt-cloud/deploy/virt-cloud-storage-replica-worker.service /etc/systemd/system/
cp /home/virt-cloud/deploy/virt-cloud-storage-publication-worker.service /etc/systemd/system/
cp /home/virt-cloud/deploy/virt-cloud-storage-health-worker.service /etc/systemd/system/
cp /home/virt-cloud/deploy/virt-cloud-abuse-monitor.service /etc/systemd/system/
cd /home/virt-cloud/agent
go build -trimpath -o /home/virt-cloud/bin/virt-cloud-storage-replicator \
./cmd/virt-cloud-storage-replicator
@ -243,6 +244,7 @@ systemctl enable --now virt-cloud-worker.service \
virt-cloud-storage-replica-worker.service \
virt-cloud-storage-publication-worker.service \
virt-cloud-storage-health-worker.service \
virt-cloud-abuse-monitor.service \
virt-cloud-lifecycle.service \
virt-cloud-policy-worker@1.service virt-cloud-policy-worker@2.service \
virt-cloud-vpc-worker@{1..8}.service
@ -258,6 +260,8 @@ systemctl enable --now virt-cloud-worker.service \
`virt-cloud-storage-health-worker.service` 与副本复制 worker 彻底隔离:它每次只领取一个到期后端,在数据库事务外执行最长 15 秒的 `HeadBucket`,并且只能按领取时冻结的配置版本结算。成功只恢复健康证据,不会把人工设置的 `read_only`、`draining` 或 `disabled` 改回 `active`;失败使正在 `active` 的后端转为 `read_only`、按 60 秒到 1 小时退避,新写入随即停止选择它。这个 worker 不执行 PUT、COPY、DELETE 或 Abort,也不把凭据写入心跳、日志或命令行。
`virt-cloud-abuse-monitor.service` 是独立、只读宿主机的网络滥用观测进程。Agent 仅发布每分钟公网接口与 conntrack 的有界聚合计数,不采集报文正文、远端 IP 列表或远端端口列表;PHP 将样本、投影意图与宿主机游标原子写入日志库,再用幂等 receipt 投影管理员只读状态。该服务不进入 `cloud_jobs`,不会自动断网、限速、修改防火墙、实例生命周期或账单。采集缺口、Agent generation 变化及不完整样本均按异常证据显示,不会当作正常零值参与判定。正式启用前需确认宿主机 conntrack 可读、Agent 网络观测能力可用,并同时检查 `abuse_monitor` 心跳与后台采集健康状态。
实例补偿的批次、明细和收件人账本已包含在当前 v42 完整 schema 中。`php xcat cloudInit` 只能为全新空数据库创建和验证当前全部结构,不能冒充既有数据库的增量升级。既有安装发布前必须备份数据库,使用经过审核的一次性迁移落地完整 v42 结构并核对 schema 指纹;升级成功后删除迁移脚本。然后按本节重启普通、通知及对象存储 workers。补偿本身不调用 Agent/Gateway,因此仅补偿功能不要求更新或重新编译宿主机 Agent 和面板 Gateway。完整语义见 [`SERVICE_COMPENSATION.md`](SERVICE_COMPENSATION.md)。
共享流量池必须严格按 `disabled -> shadow -> enforced` 上线,并在强制执行前通过全部账本、采样和网络策略门禁。现有数据库不能使用 `php xcat cloudInit` 冒充增量升级,旧服务周期也不能通过 SQL 回填到流量池;完整步骤、验收项和故障处置见 [`TRAFFIC_POOL_ROLLOUT.md`](TRAFFIC_POOL_ROLLOUT.md)。
@ -450,6 +454,7 @@ journalctl -u virt-cloud-worker.service -u virt-cloud-recovery-worker.service \
-u virt-cloud-storage-replica-worker.service \
-u virt-cloud-storage-publication-worker.service \
-u virt-cloud-storage-health-worker.service \
-u virt-cloud-abuse-monitor.service \
-u virt-cloud-lifecycle.service \
-u 'virt-cloud-policy-worker@*' -u 'virt-cloud-vpc-worker@*' \
-n 200 --no-pager

@ -0,0 +1,198 @@
<?php
namespace App\Command;
use App\Services\Cloud\CloudAbuseObservationIngestService;
use App\Services\Cloud\CloudAbuseProjectionService;
use App\Services\Cloud\CloudAbuseRetentionService;
use App\Services\Cloud\CloudRuntimeHeartbeat;
/**
* Independent, observe-only network-abuse monitor.
*
* Collection and projection deliberately stay outside cloud_jobs: every host
* call is read-only and every database write is an idempotent telemetry or
* read-model update. No lifecycle, billing, firewall or task mutation belongs
* in this loop.
*/
final class CloudAbuseMonitor
{
private const PROJECTION_BURST = 64;
private const RETENTION_INTERVAL_SECONDS = 300;
private const IDLE_SLEEP_MICROSECONDS = 1_000_000;
private const SAFE_PROJECTION_DIAGNOSTICS = [
'instance_deleted',
'instance_owner_changed',
'projection_receipt_baseline_reset',
];
/**
* Injectable objects and clock keep the command testable without turning
* the production services into interfaces solely for the command layer.
*
* @param null|object $ingest must expose tick() and observationErrors()
* @param null|object $projection must expose tick() and observationErrors()
* @param null|object $retention must expose prune()
* @param null|callable():int $clock
* @param null|callable(int):void $sleep
*/
public static function run(
array $argv = [],
?CloudRuntimeHeartbeat $heartbeat = null,
?object $ingest = null,
?object $projection = null,
?object $retention = null,
?callable $clock = null,
?callable $sleep = null
): int {
$once = in_array('--once', $argv, true);
$heartbeat ??= CloudRuntimeHeartbeat::start(
'abuse_monitor',
$once
);
$ingest ??= new CloudAbuseObservationIngestService();
$projection ??= new CloudAbuseProjectionService();
$retention ??= new CloudAbuseRetentionService();
$clock ??= static fn (): int => time();
$sleep ??= static function (int $microseconds): void {
usleep($microseconds);
};
self::assertWorker($ingest, ['tick', 'observationErrors']);
self::assertWorker($projection, ['tick', 'observationErrors']);
self::assertWorker($retention, ['prune']);
$nextRetentionAt = 0;
$unresolved = [
'collection' => '',
'projection' => '',
'retention' => '',
];
try {
do {
$worked = false;
$heartbeat->busy('abuse_monitor_ingest');
try {
$worked = $ingest->tick() === true || $worked;
$unresolved['collection'] =
$ingest->observationErrors() === []
? '' : 'abuse_monitor_collection_failed';
} catch (\Throwable $error) {
$unresolved['collection'] =
'abuse_monitor_collection_failed';
self::writeFailure('collection', $error);
}
$heartbeat->busy('abuse_monitor_projection');
try {
$projectionFailed = false;
for ($projected = 0;
$projected < self::PROJECTION_BURST;
$projected++
) {
if ($projection->tick() !== true) {
break;
}
$worked = true;
$projectionFailed = self::hasProjectionFailure(
$projection->observationErrors()
) || $projectionFailed;
}
$unresolved['projection'] = $projectionFailed
? 'abuse_monitor_projection_failed' : '';
} catch (\Throwable $error) {
$unresolved['projection'] =
'abuse_monitor_projection_failed';
self::writeFailure('projection', $error);
}
$now = $clock();
if (!is_int($now) || $now < 1) {
throw new \LogicException(
'Abuse monitor command clock is invalid'
);
}
if ($now >= $nextRetentionAt) {
$heartbeat->busy('abuse_monitor_retention');
try {
$pruned = $retention->prune();
if (!is_array($pruned)) {
throw new \RuntimeException(
'Abuse retention returned an invalid result'
);
}
$worked = array_sum(array_map(
static fn (mixed $value): int =>
is_int($value) && $value > 0 ? $value : 0,
$pruned
)) > 0 || $worked;
$unresolved['retention'] = '';
} catch (\Throwable $error) {
$unresolved['retention'] =
'abuse_monitor_retention_failed';
self::writeFailure('retention', $error);
}
$nextRetentionAt = $now
+ self::RETENTION_INTERVAL_SECONDS;
}
$errors = array_values(array_filter($unresolved));
if ($errors !== []) {
$heartbeat->error($errors[0]);
} else {
$heartbeat->success();
}
if ($once) {
return $errors === [] ? 0 : 1;
}
if (!$worked) {
$sleep(self::IDLE_SLEEP_MICROSECONDS);
}
} while (true);
} catch (\Throwable $error) {
$heartbeat->error('abuse_monitor_runtime_failed');
throw $error;
} finally {
$heartbeat->stopped();
}
}
/** @param list<string> $methods */
private static function assertWorker(object $worker, array $methods): void
{
foreach ($methods as $method) {
if (!is_callable([$worker, $method])) {
throw new \LogicException(
'Abuse monitor dependency is invalid'
);
}
}
}
private static function writeFailure(
string $stage,
\Throwable $error
): void {
fwrite(
STDERR,
'abuse monitor ' . $stage . ' temporarily unavailable: '
. get_class($error) . PHP_EOL
);
}
/** @param list<mixed> $diagnostics */
private static function hasProjectionFailure(array $diagnostics): bool
{
foreach ($diagnostics as $diagnostic) {
if (!is_string($diagnostic)
|| !in_array(
$diagnostic,
self::SAFE_PROJECTION_DIAGNOSTICS,
true
)) {
return true;
}
}
return false;
}
}

@ -28,6 +28,8 @@ class XCat
CloudStoragePublicationWorker::run($this->argv) === 0,
'cloudStorageHealthWorker' =>
CloudStorageHealthWorker::run($this->argv) === 0,
'cloudAbuseMonitor' =>
CloudAbuseMonitor::run($this->argv) === 0,
'cloudLifecycle' => CloudLifecycle::run($this->argv) === 0,
'cloudTrafficPoolAttention' =>
CloudTrafficPoolAttention::run($this->argv) === 0,
@ -110,6 +112,7 @@ class XCat
. " php xcat cloudStorageReplicaWorker [--once]\n"
. " php xcat cloudStoragePublicationWorker [--once]\n"
. " php xcat cloudStorageHealthWorker [--once]\n"
. " php xcat cloudAbuseMonitor [--once]\n"
. " php xcat cloudLifecycle [--once]\n"
. " php xcat cloudTrafficPoolAttention inspect --pool-id=N\n"
. " php xcat cloudTrafficPoolAttention inspect-resolution --resolution-key=UUID\n"

@ -95,6 +95,7 @@ final class CloudRuntimeHealthService
'storage_replica' => '对象存储副本复制',
'storage_publication' => '对象存储业务发布',
'storage_health' => '对象存储健康探测',
'abuse_monitor' => '网络滥用监测',
'lifecycle' => '到期与流量调度'] as $key => $label) {
$found = null;
foreach (is_array($heartbeats) ? $heartbeats : [] as $row) {

@ -14,6 +14,7 @@ final class CloudRuntimeHeartbeat
'storage_publication', '对象存储业务发布 worker',
],
'storage_health' => ['storage_health', '对象存储健康探测 worker'],
'abuse_monitor' => ['abuse_monitor', '网络滥用监测 worker'],
'lifecycle' => ['lifecycle', '生命周期调度']];
private const PHASES = ['startup' => '启动', 'worker_tick' => '任务消费或核对',
'recovery_tick' => '任务安全收敛',
@ -29,6 +30,9 @@ final class CloudRuntimeHeartbeat
'storage_replica_tick' => '对象存储副本复制或核对',
'storage_publication_tick' => '对象存储业务目录最终化',
'storage_health_probe' => '对象存储健康探测',
'abuse_monitor_ingest' => '网络观测采集',
'abuse_monitor_projection' => '网络风险投影',
'abuse_monitor_retention' => '网络观测保留清理',
'traffic_sync' => '流量同步',
'runtime_status_sync' => '实例状态同步',
'instance_reconcile' => '实例策略核对',
@ -45,6 +49,10 @@ final class CloudRuntimeHeartbeat
'storage_replica_failed',
'storage_publication_failed',
'storage_health_failed', 'storage_backend_unhealthy',
'abuse_monitor_collection_failed',
'abuse_monitor_projection_failed',
'abuse_monitor_retention_failed',
'abuse_monitor_runtime_failed',
'traffic_sync_failed', 'runtime_status_sync_failed',
'runtime_status_sync_unconfigured', 'traffic_collection_unconfigured',
'traffic_log_schema_missing', 'traffic_agent_unavailable',
@ -234,7 +242,7 @@ final class CloudRuntimeHeartbeat
if (++$scanned > self::SCAN_LIMIT || $removed >= 32) {
break;
}
if (preg_match('/^(normal|policy|vpc|recovery|notification|storage_replica|storage_publication|storage_health|lifecycle)-[1-9][0-9]*-[a-f0-9]{24}\.json(?:\.tmp)?$/D',
if (preg_match('/^(normal|policy|vpc|recovery|notification|storage_replica|storage_publication|storage_health|abuse_monitor|lifecycle)-[1-9][0-9]*-[a-f0-9]{24}\.json(?:\.tmp)?$/D',
$entry->getFilename()) !== 1) {
continue;
}
@ -324,7 +332,7 @@ final class CloudRuntimeHeartbeat
$incomplete = true;
break;
}
if (preg_match('/^(normal|policy|vpc|recovery|notification|storage_replica|storage_publication|storage_health|lifecycle)-[1-9][0-9]*-[a-f0-9]{24}\.json$/D',
if (preg_match('/^(normal|policy|vpc|recovery|notification|storage_replica|storage_publication|storage_health|abuse_monitor|lifecycle)-[1-9][0-9]*-[a-f0-9]{24}\.json$/D',
$entry->getFilename(), $match) !== 1) {
continue;
}

@ -0,0 +1,367 @@
<?php
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Command\CloudAbuseMonitor;
use App\Services\Cloud\CloudRuntimeHeartbeat;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$root = dirname(__DIR__, 2);
$read = static function (string $path) use ($root): string {
$value = file_get_contents($root . '/' . $path);
if (!is_string($value)) {
throw new RuntimeException('Unable to read ' . $path);
}
return $value;
};
$temporary = sys_get_temp_dir() . '/virt-abuse-monitor-'
. bin2hex(random_bytes(8));
if (!mkdir($temporary, 0700, true) && !is_dir($temporary)) {
throw new RuntimeException('Unable to create heartbeat fixture.');
}
$cleanup = static function (string $directory): void {
if (!is_dir($directory)) {
return;
}
foreach (new FilesystemIterator($directory) as $entry) {
if ($entry->isFile() || $entry->isLink()) {
unlink($entry->getPathname());
}
}
rmdir($directory);
};
try {
$now = 2_000_000_000;
$ingest = new class {
public int $ticks = 0;
public function tick(): bool
{
$this->ticks++;
return true;
}
public function observationErrors(): array
{
return [];
}
};
$projection = new class {
public int $ticks = 0;
public function tick(): bool
{
$this->ticks++;
return $this->ticks <= 2;
}
public function observationErrors(): array
{
return [];
}
};
$retention = new class {
public int $runs = 0;
public function prune(): array
{
$this->runs++;
return ['terminal_outboxes' => 0, 'samples' => 1,
'baselines' => 0, 'receipts' => 0, 'findings' => 0];
}
};
$heartbeat = CloudRuntimeHeartbeat::start(
'abuse_monitor',
true,
$temporary,
static fn (): int => $now
);
$status = CloudAbuseMonitor::run(
['cloudAbuseMonitor', '--once'],
$heartbeat,
$ingest,
$projection,
$retention,
static fn (): int => $now,
static function (int $_microseconds): void {
throw new RuntimeException('Once mode attempted to sleep.');
}
);
$records = iterator_to_array(new FilesystemIterator($temporary));
$record = json_decode(
file_get_contents(reset($records)->getPathname()),
true,
16,
JSON_THROW_ON_ERROR
);
$assert($status === 0
&& $ingest->ticks === 1
&& $projection->ticks === 3
&& $retention->runs === 1
&& $record['kind'] === 'abuse_monitor'
&& $record['state'] === 'stopped'
&& $record['last_success'] === $now
&& $record['unresolved_error'] === false,
'Observe-only worker did not drain its bounded burst cleanly.');
$failureDirectory = $temporary . '-failure';
mkdir($failureDirectory, 0700, true);
$failedIngest = new class {
public function tick(): bool
{
return true;
}
public function observationErrors(): array
{
return ['network_observation_collection_failed'];
}
};
$idleProjection = new class {
public function tick(): bool
{
return false;
}
public function observationErrors(): array
{
return [];
}
};
$idleRetention = new class {
public function prune(): array
{
return ['terminal_outboxes' => 0, 'samples' => 0,
'baselines' => 0, 'receipts' => 0, 'findings' => 0];
}
};
$failedHeartbeat = CloudRuntimeHeartbeat::start(
'abuse_monitor',
true,
$failureDirectory,
static fn (): int => $now
);
$failedStatus = CloudAbuseMonitor::run(
['cloudAbuseMonitor', '--once'],
$failedHeartbeat,
$failedIngest,
$idleProjection,
$idleRetention,
static fn (): int => $now
);
$failedFiles = iterator_to_array(
new FilesystemIterator($failureDirectory)
);
$failedRecord = json_decode(
file_get_contents(reset($failedFiles)->getPathname()),
true,
16,
JSON_THROW_ON_ERROR
);
$assert($failedStatus === 1
&& $failedRecord['state'] === 'stopped'
&& $failedRecord['last_error']
=== 'abuse_monitor_collection_failed'
&& $failedRecord['unresolved_error'] === true,
'A caught collection failure was falsely projected healthy.');
$cleanup($failureDirectory);
$safeDirectory = $temporary . '-safe-projection';
mkdir($safeDirectory, 0700, true);
$idleIngest = new class {
public function tick(): bool
{
return false;
}
public function observationErrors(): array
{
return [];
}
};
$safeProjection = new class {
public int $ticks = 0;
public function tick(): bool
{
return ++$this->ticks === 1;
}
public function observationErrors(): array
{
return $this->ticks === 1
? ['instance_deleted'] : [];
}
};
$safeHeartbeat = CloudRuntimeHeartbeat::start(
'abuse_monitor',
true,
$safeDirectory,
static fn (): int => $now
);
$safeStatus = CloudAbuseMonitor::run(
['cloudAbuseMonitor', '--once'],
$safeHeartbeat,
$idleIngest,
$safeProjection,
$idleRetention,
static fn (): int => $now
);
$safeFiles = iterator_to_array(new FilesystemIterator($safeDirectory));
$safeRecord = json_decode(
file_get_contents(reset($safeFiles)->getPathname()),
true,
16,
JSON_THROW_ON_ERROR
);
$assert($safeStatus === 0
&& $safeRecord['unresolved_error'] === false,
'A safely converged projection discard raised a worker failure.');
$cleanup($safeDirectory);
$conflictDirectory = $temporary . '-projection-conflict';
mkdir($conflictDirectory, 0700, true);
$conflictProjection = new class {
public int $ticks = 0;
public function tick(): bool
{
return ++$this->ticks === 1;
}
public function observationErrors(): array
{
return $this->ticks === 1
? ['projection_order_conflict'] : [];
}
};
$conflictHeartbeat = CloudRuntimeHeartbeat::start(
'abuse_monitor',
true,
$conflictDirectory,
static fn (): int => $now
);
$conflictStatus = CloudAbuseMonitor::run(
['cloudAbuseMonitor', '--once'],
$conflictHeartbeat,
$idleIngest,
$conflictProjection,
$idleRetention,
static fn (): int => $now
);
$conflictFiles = iterator_to_array(
new FilesystemIterator($conflictDirectory)
);
$conflictRecord = json_decode(
file_get_contents(reset($conflictFiles)->getPathname()),
true,
16,
JSON_THROW_ON_ERROR
);
$assert($conflictStatus === 1
&& $conflictRecord['last_error']
=== 'abuse_monitor_projection_failed'
&& $conflictRecord['unresolved_error'] === true,
'A projection ordering conflict was silently treated as safe.');
$cleanup($conflictDirectory);
// A stage clears its own warning after a successful zero-work retry; it
// must not depend on unrelated collection or projection work.
$recoveryDirectory = $temporary . '-retention-recovery';
mkdir($recoveryDirectory, 0700, true);
$recoveringRetention = new class {
public int $runs = 0;
public function prune(): array
{
if (++$this->runs === 1) {
throw new RuntimeException('simulated retention outage');
}
return ['terminal_outboxes' => 0, 'samples' => 0,
'baselines' => 0, 'receipts' => 0, 'findings' => 0];
}
};
$recoveryNow = $now;
$sleepCalls = 0;
$recoveryHeartbeat = CloudRuntimeHeartbeat::start(
'abuse_monitor',
false,
$recoveryDirectory,
static function () use (&$recoveryNow): int {
return $recoveryNow;
}
);
try {
CloudAbuseMonitor::run(
['cloudAbuseMonitor'],
$recoveryHeartbeat,
$idleIngest,
$idleProjection,
$recoveringRetention,
static function () use (&$recoveryNow): int {
return $recoveryNow;
},
static function (int $_microseconds) use (
&$sleepCalls,
&$recoveryNow,
$recoveryDirectory,
$assert
): void {
++$sleepCalls;
$recoveryNow += 301;
if ($sleepCalls < 2) {
return;
}
$files = iterator_to_array(
new FilesystemIterator($recoveryDirectory)
);
$row = json_decode(
file_get_contents(reset($files)->getPathname()),
true,
16,
JSON_THROW_ON_ERROR
);
$assert($row['state'] === 'idle'
&& $row['unresolved_error'] === false
&& $row['last_success'] === $recoveryNow - 301,
'Successful zero-work retention did not clear its warning.');
throw new RuntimeException('stop daemon fixture');
}
);
throw new RuntimeException('Daemon fixture did not stop.');
} catch (RuntimeException $error) {
if ($error->getMessage() !== 'stop daemon fixture') {
throw $error;
}
}
$cleanup($recoveryDirectory);
$command = $read('web/app/Command/CloudAbuseMonitor.php');
$xcat = $read('web/app/Command/XCat.php');
$heartbeatSource = $read(
'web/app/Services/Cloud/CloudRuntimeHeartbeat.php'
);
$runtime = $read(
'web/app/Services/Cloud/CloudRuntimeHealthService.php'
);
$unit = $read('deploy/virt-cloud-abuse-monitor.service');
$deployment = $read('docs/DEPLOYMENT.md');
$assert(str_contains($xcat, "'cloudAbuseMonitor'")
&& str_contains($xcat, 'CloudAbuseMonitor::run')
&& str_contains($command, 'PROJECTION_BURST = 64')
&& str_contains($command, 'CloudAbuseObservationIngestService')
&& str_contains($command, 'CloudAbuseProjectionService')
&& str_contains($command, 'CloudAbuseRetentionService')
&& !str_contains($command, "table('cloud_jobs')")
&& !str_contains($command, 'CloudJob'),
'Monitor command is unreachable, unbounded, or coupled to jobs.');
$assert(str_contains($heartbeatSource, "'abuse_monitor'")
&& str_contains($heartbeatSource, "'abuse_monitor_ingest'")
&& str_contains($runtime,
"'abuse_monitor' => '网络滥用监测'"),
'Runtime observation does not expose the independent monitor.');
$assert(str_contains($unit,
'ExecStart=/usr/bin/php /home/virt-cloud/web/xcat cloudAbuseMonitor')
&& str_contains($unit, 'NoNewPrivileges=true')
&& str_contains($unit, 'ProtectSystem=strict')
&& str_contains($unit, 'LimitCORE=0')
&& str_contains($deployment,
'virt-cloud-abuse-monitor.service'),
'Monitor systemd isolation or deployment documentation is incomplete.');
} finally {
$cleanup($temporary);
}
echo "Observe-only abuse monitor worker contract tests passed.\n";

@ -52,7 +52,7 @@ foreach ([['user_id' => 42, 'is_admin' => false], ['user_id' => 0, 'is_admin' =>
$assert($calls === [], 'Unauthorized requests performed probes.');
$payload = $service->snapshot($actor);
$checks = array_column($payload['checks'], null, 'key');
$assert(count($checks) === 13
$assert(count($checks) === 14
&& $calls === ['default', 'log', 'agents', 'redis'],
'Missing or duplicate probes.');
$assert($checks['agents']['status'] === 'ok' && $checks['worker']['status'] === 'busy'
@ -62,6 +62,7 @@ $assert($checks['agents']['status'] === 'ok' && $checks['worker']['status'] ===
&& $checks['storage_replica']['status'] === 'unknown'
&& $checks['storage_publication']['status'] === 'unknown'
&& $checks['storage_health']['status'] === 'unknown'
&& $checks['abuse_monitor']['status'] === 'unknown'
&& $checks['lifecycle']['status'] === 'warning',
'Health or heartbeat semantics changed.');
$assert(!str_contains(json_encode($payload), 'MUST_NOT_EXPOSE'), 'Heartbeat internal fields leaked.');
@ -186,7 +187,7 @@ $response = (new CloudAdminRuntimeController())->index($request, new Response())
$assert($response->getStatusCode() === 200
&& $response->getHeaderLine('Cache-Control') === 'private, no-store', 'Runtime response cached or unavailable.');
$decoded = json_decode((string) $response->getBody(), true);
$assert(count($decoded['checks']) === 13
$assert(count($decoded['checks']) === 14
&& array_column($decoded['checks'], 'status', 'key')['agents'] === 'error',
'Partial failure lost other checks or fabricated Agent configuration health.');
$routes = file_get_contents(dirname(__DIR__) . '/config/routes_cloud.php');

@ -49,7 +49,7 @@ try {
$assert(array_keys($empty) === [
'worker', 'policy', 'vpc', 'recovery', 'notification',
'storage_replica', 'storage_publication', 'storage_health',
'lifecycle',
'abuse_monitor', 'lifecycle',
]
&& array_unique(array_column($empty, 'status')) === ['unknown'] && !file_exists($missing),
'Missing snapshot must be unknown, cover all kinds, and never create runtime files.');
@ -71,7 +71,10 @@ try {
$storageHealth = CloudRuntimeHeartbeat::start(
'storage_health', false, $directory, $clock
);
$assert(count($files($directory)) === 8,
$abuseMonitor = CloudRuntimeHeartbeat::start(
'abuse_monitor', false, $directory, $clock
);
$assert(count($files($directory)) === 9,
'Concurrent worker identities overwrite one another.');
$assert(is_readable($directory) && is_writable($directory),
'Runtime directory is not readable and writable.');
@ -90,7 +93,9 @@ try {
&& $checks($directory)['storage_publication']['label']
=== '对象存储业务发布 worker'
&& $checks($directory)['storage_health']['active_count'] === 1
&& $checks($directory)['storage_health']['label'] === '对象存储健康探测 worker',
&& $checks($directory)['storage_health']['label'] === '对象存储健康探测 worker'
&& $checks($directory)['abuse_monitor']['active_count'] === 1
&& $checks($directory)['abuse_monitor']['label'] === '网络滥用监测 worker',
'Starting workers or multiple policy reporters were not represented.');
$notification->busy('notification');
$assert(str_contains(
@ -116,6 +121,15 @@ try {
$checks($directory)['storage_health']['message'],
'对象存储健康探测'
), 'Object storage health probes did not receive an isolated runtime phase.');
$abuseMonitor->busy('abuse_monitor_projection');
$assert(str_contains(
$checks($directory)['abuse_monitor']['message'],
'网络风险投影'
), 'Abuse projection did not receive an isolated runtime phase.');
$now++;
$abuseMonitor->error('abuse_monitor_projection_failed');
$assert($checks($directory)['abuse_monitor']['status'] === 'warning',
'Abuse-monitor failure was not preserved as a stable warning.');
$normal->busy('worker_tick');
$assert(str_contains($checks($directory)['worker']['message'], '任务消费或核对'),
@ -158,7 +172,10 @@ try {
'traffic_agent_unavailable', 'traffic_metric_unavailable',
'traffic_collection_failed', 'traffic_retention_failed'] as $collectorError) {
$normal->error($collectorError);
$record = $read($directory)[0];
$record = array_values(array_filter(
$read($directory),
static fn (array $row): bool => $row['kind'] === 'normal'
))[0];
$assert($record['last_error'] === $collectorError
&& $record['unresolved_error'] === true,
'Stable traffic collector observation was collapsed to runtime_error: '

Loading…
Cancel
Save