feat: deliver maintenance notice batches

go
codex 2 weeks ago
parent cbd4798f4c
commit 0375c0fa4c

@ -0,0 +1,24 @@
[Unit]
Description=Virt Cloud maintenance notification worker
After=network-online.target
Wants=network-online.target
[Service]
Type=simple
User=apache
Group=apache
WorkingDirectory=/home/virt-cloud/web
ExecStart=/usr/bin/php /home/virt-cloud/web/xcat cloudWorker --notification-only
Restart=on-failure
RestartSec=5
UMask=0077
NoNewPrivileges=true
PrivateTmp=true
ProtectHome=false
ProtectSystem=strict
ReadWritePaths=-/home/virt-cloud/runtime/php
ReadWritePaths=-/home/virt-cloud/web/storage
TimeoutStopSec=30
[Install]
WantedBy=multi-user.target

@ -228,15 +228,19 @@ cp /home/virt-cloud/deploy/virt-cloud-lifecycle.service /etc/systemd/system/
cp /home/virt-cloud/deploy/virt-cloud-policy-worker@.service /etc/systemd/system/
cp /home/virt-cloud/deploy/virt-cloud-vpc-worker@.service /etc/systemd/system/
cp /home/virt-cloud/deploy/virt-cloud-recovery-worker.service /etc/systemd/system/
cp /home/virt-cloud/deploy/virt-cloud-notification-worker.service /etc/systemd/system/
systemctl daemon-reload
systemctl enable --now virt-cloud-worker.service \
virt-cloud-recovery-worker.service virt-cloud-lifecycle.service \
virt-cloud-recovery-worker.service virt-cloud-notification-worker.service \
virt-cloud-lifecycle.service \
virt-cloud-policy-worker@1.service virt-cloud-policy-worker@2.service \
virt-cloud-vpc-worker@{1..8}.service
```
普通 worker 负责购买、创建、重装、实例升级、邮件和最终删除;至少两个策略 worker 处理到期断网、网络恢复与删除前策略;lifecycle 生成续费账单并推进账期、流量、宽限和删除状态。VPC 批量加入/移出没有实例数量上限:面板会在一个数据库事务中释放同一批次的全部 VM 子任务,实际同一时刻运行多少台由 `virt-cloud-vpc-worker@N.service` 的实例数和各宿主机 Agent 的安全 admission 决定,超出 worker 并发度的任务仍已持久接收并等待空闲进程,不依赖浏览器。上例启用 8 个专用 worker;可按面板 CPU、数据库连接和宿主机冷启动能力增减实例数,而不是在页面设置固定的“最多 4 台”。未安装专用 unit 时普通 worker 仍会安全消费这些任务,但吞吐量较低。
`virt-cloud-notification-worker.service` 只消费管理员维护通知,每轮最多领取一封,不占用虚拟机任务 worker。投递使用 300 秒租约;邮件传输一旦开始后返回失败、异常或租约过期都记为“效果未知”,不会自动重发。该服务是维护通知功能的必需组件;只更新 PHP 而不安装并启动该 unit 时,通知会安全保留在待发送状态。
共享流量池必须严格按 `disabled -> shadow -> enforced` 上线,并在强制执行前通过全部账本、采样和网络策略门禁。现有数据库不能使用 `php xcat cloudInit` 冒充增量升级,旧服务周期也不能通过 SQL 回填到流量池;完整步骤、验收项和故障处置见 [`TRAFFIC_POOL_ROLLOUT.md`](TRAFFIC_POOL_ROLLOUT.md)。
`virt-cloud-recovery-worker.service` 是必需的独立 operation 恢复进程,覆盖策略表中的实例电源、重装/救援、公网 IP、实例设置、实例升级、快照、防火墙、删除、生命周期网络、模板与 ISO Agent 叶子任务。它只做 operation-bound inspection/栅栏审计、持久晚到回执的 PHP 收敛或聚合父任务的本地汇总,不投递邮件,也绝不重放原 mutation。离线、身份冲突或账本异常按固定低频间隔持续核对,没有次数或总时长上限。它与普通队列隔离,避免某个离线 Agent 的核对占住正常 worker。

@ -2,6 +2,7 @@
namespace App\Command;
use App\Services\Cloud\CloudInvoiceService;
use App\Services\Cloud\CloudMaintenanceNoticeDeliveryService;
use App\Services\Cloud\CloudAutomaticOperationRecoveryService;
use App\Services\Cloud\CloudRuntimeHeartbeat;
use App\Services\Cloud\CloudWorkerService;
@ -14,34 +15,41 @@ final class CloudWorker
$policyOnly = in_array('--policy-only', $argv, true);
$vpcBatchOnly = in_array('--vpc-batch-only', $argv, true);
$recoveryOnly = in_array('--recovery-only', $argv, true);
$notificationOnly = in_array('--notification-only', $argv, true);
if (($policyOnly ? 1 : 0) + ($vpcBatchOnly ? 1 : 0)
+ ($recoveryOnly ? 1 : 0) > 1) {
+ ($recoveryOnly ? 1 : 0) + ($notificationOnly ? 1 : 0) > 1) {
throw new \InvalidArgumentException(
'worker 专用模式不能同时使用。'
);
}
$heartbeat ??= CloudRuntimeHeartbeat::start(
$recoveryOnly ? 'recovery' : ($policyOnly ? 'policy'
: ($vpcBatchOnly ? 'vpc' : 'normal')),
$notificationOnly ? 'notification'
: ($recoveryOnly ? 'recovery' : ($policyOnly ? 'policy'
: ($vpcBatchOnly ? 'vpc' : 'normal'))),
$once
);
try {
$worker = $recoveryOnly
? new CloudAutomaticOperationRecoveryService()
: new CloudWorkerService(null, $policyOnly, $vpcBatchOnly);
$worker = $notificationOnly
? new CloudMaintenanceNoticeDeliveryService()
: ($recoveryOnly
? new CloudAutomaticOperationRecoveryService()
: new CloudWorkerService(null, $policyOnly, $vpcBatchOnly));
do {
$errors = [];
$heartbeat->busy(
$recoveryOnly ? 'recovery_tick' : 'worker_tick'
$notificationOnly ? 'maintenance_notification'
: ($recoveryOnly ? 'recovery_tick' : 'worker_tick')
);
try {
$worked = $worker->tick();
if (!$recoveryOnly) {
if (!$recoveryOnly && !$notificationOnly) {
$errors = $worker->observationErrors();
}
} catch (\Throwable $error) {
$code = $recoveryOnly
? 'recovery_tick_failed' : 'worker_tick_failed';
$code = $notificationOnly
? 'maintenance_notification_failed'
: ($recoveryOnly
? 'recovery_tick_failed' : 'worker_tick_failed');
$errors[] = $code;
$heartbeat->error($code);
fwrite(STDERR, 'cloud worker temporarily unavailable: ' . get_class($error) . PHP_EOL);
@ -54,7 +62,8 @@ final class CloudWorker
$heartbeat->error($errors[0]);
}
// SMTP remains isolated from lifecycle and policy workers.
if (!$policyOnly && !$vpcBatchOnly && !$recoveryOnly) {
if (!$policyOnly && !$vpcBatchOnly && !$recoveryOnly
&& !$notificationOnly) {
$errors = array_merge($errors, self::deliverNotifications($heartbeat));
}
if ($errors === []) {
@ -68,8 +77,9 @@ final class CloudWorker
} while (!$once);
} catch (\Throwable $error) {
$heartbeat->error(
$recoveryOnly
? 'recovery_tick_failed' : 'worker_tick_failed'
$notificationOnly ? 'maintenance_notification_failed'
: ($recoveryOnly
? 'recovery_tick_failed' : 'worker_tick_failed')
);
throw $error;
} finally {

@ -100,7 +100,7 @@ class XCat
echo "Virt Cloud CLI\n"
. " php xcat createAdmin <email> [name]\n"
. " php xcat cloudInit\n"
. " php xcat cloudWorker [--once] [--policy-only|--vpc-batch-only|--recovery-only]\n"
. " php xcat cloudWorker [--once] [--policy-only|--vpc-batch-only|--recovery-only|--notification-only]\n"
. " php xcat cloudLifecycle [--once]\n"
. " php xcat cloudTrafficPoolAttention inspect --pool-id=N\n"
. " php xcat cloudTrafficPoolAttention inspect-resolution --resolution-key=UUID\n"

@ -0,0 +1,255 @@
<?php
namespace App\Services\Cloud;
use App\Models\CloudMaintenanceNotice;
use App\Models\CloudMaintenanceNoticeInstance;
use App\Models\CloudMaintenanceNoticeRecipient;
use App\Models\User;
use App\Services\Mail;
use Illuminate\Database\Capsule\Manager as DB;
/** Leased, one-at-a-time maintenance mail delivery with no unknown replay. */
final class CloudMaintenanceNoticeDeliveryService
{
private const LEASE_SECONDS = 300;
private const EXPIRED_LEASE_LIMIT = 10;
private readonly \Closure $clock;
private readonly \Closure $sender;
private readonly \Closure $blacklist;
public function __construct(?callable $sender = null,
?callable $blacklist = null, ?callable $clock = null,
private readonly CloudMaintenanceNoticeService $notices =
new CloudMaintenanceNoticeService())
{
$this->clock = \Closure::fromCallable($clock
?? static fn (): int => time());
$this->sender = \Closure::fromCallable($sender
?? static fn (string $to, string $subject, array $variables): bool =>
Mail::send(
$to,
$subject,
'cloud/maintenance-notice.tpl',
$variables,
[],
'maintenance_notice'
));
$this->blacklist = \Closure::fromCallable($blacklist
?? static fn (string $email): bool =>
Mail::isRecipientBlacklisted($email));
}
public function tick(): bool
{
$worked = $this->convergeExpiredLeases();
$claim = $this->claim();
if ($claim === null) {
return $worked;
}
if (isset($claim['settled_notice_id'])) {
$this->notices->recalculate((int) $claim['settled_notice_id']);
return true;
}
$recipient = $claim['recipient'];
$token = $claim['lease_token'];
$recipientId = (int) $recipient->id;
$noticeId = (int) $recipient->notice_id;
try {
if (!User::where('id', (int) $recipient->user_id)->exists()) {
$this->finalize($recipientId, $noticeId, $token,
'skipped', 'user_deleted', '');
return true;
}
if (($this->blacklist)((string) $recipient->email_snapshot)) {
$this->finalize($recipientId, $noticeId, $token,
'skipped', 'recipient_blacklisted', '');
return true;
}
$mail = $this->mail($recipient);
} catch (\Throwable $error) {
$this->finalize($recipientId, $noticeId, $token,
'failed', 'delivery_preflight_failed', get_class($error));
return true;
}
try {
$accepted = ($this->sender)(
(string) $recipient->email_snapshot,
(string) $mail['subject'],
$mail['variables']
) === true;
} catch (\Throwable $error) {
$this->finalize($recipientId, $noticeId, $token,
'unknown', 'delivery_effect_unknown', get_class($error));
return true;
}
$this->finalize(
$recipientId,
$noticeId,
$token,
$accepted ? 'sent' : 'unknown',
$accepted ? '' : 'delivery_effect_unknown',
''
);
return true;
}
private function claim(): ?array
{
return DB::connection('default')->transaction(function (): ?array {
$now = $this->now();
$recipient = CloudMaintenanceNoticeRecipient::where(
'status', 'pending'
)->where('available_at', '<=', $now)->orderBy('available_at')
->orderBy('id')->lockForUpdate()->first();
if ($recipient === null) {
return null;
}
$notice = CloudMaintenanceNotice::where(
'id', $recipient->notice_id
)->lockForUpdate()->first();
if ($notice === null || !in_array((string) $notice->status,
['scheduled', 'sending'], true)) {
$recipient->status = 'cancelled';
$recipient->last_error_code = 'notice_not_deliverable';
$recipient->last_error = '';
$recipient->updated_at = $now;
$recipient->save();
return ['settled_notice_id' => (int) $recipient->notice_id];
}
$token = bin2hex(random_bytes(16));
$recipient->status = 'sending';
$recipient->attempts = (int) $recipient->attempts + 1;
$recipient->lease_token = $token;
$recipient->lease_until = $now + self::LEASE_SECONDS;
$recipient->last_error_code = '';
$recipient->last_error = '';
$recipient->updated_at = $now;
if (!$recipient->save()) {
throw new \RuntimeException('Maintenance recipient claim failed.');
}
if ((string) $notice->status === 'scheduled') {
$notice->status = 'sending';
$notice->updated_at = $now;
$notice->save();
}
return ['recipient' => $recipient, 'lease_token' => $token];
}, 3);
}
private function convergeExpiredLeases(): bool
{
$noticeIds = DB::connection('default')->transaction(function (): array {
$now = $this->now();
$rows = CloudMaintenanceNoticeRecipient::where('status', 'sending')
->where('lease_until', '>', 0)
->where('lease_until', '<=', $now)
->orderBy('lease_until')->orderBy('id')
->limit(self::EXPIRED_LEASE_LIMIT)->lockForUpdate()->get();
$noticeIds = [];
foreach ($rows as $row) {
$updated = CloudMaintenanceNoticeRecipient::where('id', $row->id)
->where('status', 'sending')
->where('lease_token', $row->lease_token)->update([
'status' => 'unknown',
'lease_token' => null,
'lease_until' => 0,
'last_error_code' => 'delivery_lease_expired',
'last_error' => '',
'updated_at' => $now,
]);
if ($updated === 1) {
$noticeIds[(int) $row->notice_id] = true;
}
}
return array_keys($noticeIds);
}, 3);
foreach ($noticeIds as $noticeId) {
$this->notices->recalculate((int) $noticeId);
}
return $noticeIds !== [];
}
private function mail(CloudMaintenanceNoticeRecipient $recipient): array
{
$notice = CloudMaintenanceNotice::where(
'id', $recipient->notice_id
)->first();
if ($notice === null) {
throw new \RuntimeException('Maintenance notice is missing.');
}
$scope = json_decode((string) $notice->scope_snapshot_json,
true, 32, JSON_THROW_ON_ERROR);
if (!is_array($scope) || !is_string($scope['label'] ?? null)) {
throw new \RuntimeException('Maintenance scope is invalid.');
}
$instances = CloudMaintenanceNoticeInstance::where(
'notice_id', $notice->id
)->where('user_id', $recipient->user_id)->orderBy('id')->get()
->map(static fn ($instance): array => [
'name' => (string) $instance->instance_name,
'ipv4' => (string) $instance->ipv4,
'ipv6' => (string) $instance->ipv6,
])->all();
if ($instances === []) {
throw new \RuntimeException('Maintenance instance list is empty.');
}
$impacts = [
'network_interruption' => '网络可能短时中断',
'instance_restart' => '实例将重新启动',
'service_interruption' => '服务可能短时中断',
'no_interruption' => '预计不会中断服务',
];
$impact = $impacts[(string) $notice->impact] ?? null;
if ($impact === null) {
throw new \RuntimeException('Maintenance impact is invalid.');
}
return [
'subject' => (string) $notice->subject,
'variables' => [
'notice_subject' => (string) $notice->subject,
'message_text' => (string) $notice->message_text,
'scope_label' => (string) $scope['label'],
'impact_label' => $impact,
'maintenance_starts_utc' => gmdate(
'Y-m-d H:i:s \\U\\T\\C',
(int) $notice->maintenance_starts_at
),
'maintenance_ends_utc' => gmdate(
'Y-m-d H:i:s \\U\\T\\C',
(int) $notice->maintenance_ends_at
),
'instances' => $instances,
],
];
}
private function finalize(int $recipientId, int $noticeId,
string $token, string $status, string $errorCode,
string $error): void
{
$now = $this->now();
$updated = CloudMaintenanceNoticeRecipient::where('id', $recipientId)
->where('status', 'sending')->where('lease_token', $token)
->update([
'status' => $status,
'lease_token' => null,
'lease_until' => 0,
'sent_at' => $status === 'sent' ? $now : null,
'last_error_code' => $errorCode,
'last_error' => mb_substr($error, 0, 500),
'updated_at' => $now,
]);
if ($updated === 1) {
$this->notices->recalculate($noticeId);
}
}
private function now(): int
{
return (int) ($this->clock)();
}
}

@ -349,7 +349,12 @@ final class CloudMaintenanceNoticeService
$unknown = $count('unknown');
$failed = $count('failed');
$cancelled = $count('cancelled');
if ($pending > 0 || $sending > 0) {
$total = array_sum(array_map('intval', $counts));
if ($total === 0) {
$status = 'cancelled';
$terminalAt = $now;
$errorCode = 'audience_deleted';
} elseif ($pending > 0 || $sending > 0) {
$status = ($sending > 0 || $sent > 0
|| $unknown > 0 || $failed > 0) ? 'sending' : 'scheduled';
$terminalAt = null;
@ -364,10 +369,14 @@ final class CloudMaintenanceNoticeService
$status = 'completed';
$terminalAt = $now;
$errorCode = '';
} else {
} elseif ($cancelled > 0) {
$status = 'cancelled';
$terminalAt = $now;
$errorCode = 'admin_cancelled';
} else {
$status = 'completed';
$terminalAt = $now;
$errorCode = '';
}
$notice->status = $status;
$notice->terminal_at = $terminalAt;

@ -91,6 +91,7 @@ final class CloudRuntimeHealthService
foreach (['worker' => '普通任务处理', 'policy' => '策略任务处理',
'vpc' => 'VPC 批量任务处理',
'recovery' => '任务安全收敛',
'notification' => '维护邮件投递',
'lifecycle' => '到期与流量调度'] as $key => $label) {
$found = null;
foreach (is_array($heartbeats) ? $heartbeats : [] as $row) {

@ -8,9 +8,11 @@ final class CloudRuntimeHeartbeat
'policy' => ['policy', '策略任务 worker'],
'vpc' => ['vpc', 'VPC 批量任务 worker'],
'recovery' => ['recovery', '任务安全收敛 worker'],
'notification' => ['notification', '维护通知 worker'],
'lifecycle' => ['lifecycle', '生命周期调度']];
private const PHASES = ['startup' => '启动', 'worker_tick' => '任务消费或核对',
'recovery_tick' => '任务安全收敛',
'maintenance_notification' => '维护邮件投递',
'notification' => '邮件通知', 'lifecycle_tick' => '生命周期巡检',
'provision_expiry' => '开通账单过期处理', 'invoice_generation' => '续费账单生成',
'automatic_backup_schedule' => '自动备份调度',
@ -24,7 +26,7 @@ final class CloudRuntimeHeartbeat
'instance_reconcile' => '实例策略核对',
'idle' => '等待下一轮', 'stopped' => '已退出'];
private const ERRORS = ['worker_tick_failed', 'recovery_tick_failed',
'notification_failed',
'notification_failed', 'maintenance_notification_failed',
'lifecycle_tick_failed', 'provision_expiry_failed', 'invoice_generation_failed',
'automatic_backup_schedule_failed',
'automatic_backup_retention_failed',
@ -221,7 +223,7 @@ final class CloudRuntimeHeartbeat
if (++$scanned > self::SCAN_LIMIT || $removed >= 32) {
break;
}
if (preg_match('/^(normal|policy|vpc|recovery|lifecycle)-[1-9][0-9]*-[a-f0-9]{24}\.json(?:\.tmp)?$/D',
if (preg_match('/^(normal|policy|vpc|recovery|notification|lifecycle)-[1-9][0-9]*-[a-f0-9]{24}\.json(?:\.tmp)?$/D',
$entry->getFilename()) !== 1) {
continue;
}
@ -311,7 +313,7 @@ final class CloudRuntimeHeartbeat
$incomplete = true;
break;
}
if (preg_match('/^(normal|policy|vpc|recovery|lifecycle)-[1-9][0-9]*-[a-f0-9]{24}\.json$/D',
if (preg_match('/^(normal|policy|vpc|recovery|notification|lifecycle)-[1-9][0-9]*-[a-f0-9]{24}\.json$/D',
$entry->getFilename(), $match) !== 1) {
continue;
}

@ -4,6 +4,7 @@ namespace App\Services;
use App\Models\EmailLog;
use App\Services\TicketAttachmentService;
use App\Services\Cloud\CloudMaintenanceNoticeService;
use App\Models\User;
class UserDataDeletionService
@ -23,6 +24,8 @@ class UserDataDeletionService
// remove. cloud_jobs may contain encrypted provisioning credentials.
['cloud_vpc_membership_batch_items', 'user_id'],
['cloud_vpc_membership_batches', 'user_id'],
['cloud_maintenance_notice_recipients', 'user_id'],
['cloud_maintenance_notice_instances', 'user_id'],
['cloud_jobs', 'user_id'],
['cloud_instance_snapshots', 'user_id'],
['cloud_instance_firewalls', 'user_id'],
@ -97,6 +100,25 @@ class UserDataDeletionService
'用户仍有支付安全栅栏,请先完成网关或链上状态处理');
}
}
$maintenanceNoticeIds = [];
if ($this->hasColumns(
$connection,
'cloud_maintenance_notice_recipients',
['notice_id', 'user_id', 'status']
)) {
$maintenanceRecipients = $connection->table(
'cloud_maintenance_notice_recipients'
)->whereIn('user_id', $ids)->orderBy('id')
->lockForUpdate()->get(['notice_id', 'status']);
foreach ($maintenanceRecipients as $recipient) {
if ((string) $recipient->status === 'sending') {
throw new \RuntimeException(
'用户仍有投递中的维护通知,请等待投递结果核对后再删除账户'
);
}
$maintenanceNoticeIds[(int) $recipient->notice_id] = true;
}
}
foreach ($users as $user) {
LoginSessionRevocationService::revokeUser($user);
}
@ -140,6 +162,12 @@ class UserDataDeletionService
$emails,
$ticketAttachmentStorageKeys
);
if ($maintenanceNoticeIds !== []) {
$noticeService = new CloudMaintenanceNoticeService();
foreach (array_keys($maintenanceNoticeIds) as $noticeId) {
$noticeService->recalculate((int) $noticeId);
}
}
$trafficScanRows = $this->deleteCloudTrafficScanFailures(
$connection,
$trafficInstanceIds,

@ -0,0 +1,27 @@
<?php
$root = dirname(__DIR__);
$deletion = file_get_contents(
$root . '/app/Services/UserDataDeletionService.php'
);
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$assert(str_contains($deletion,
"['cloud_maintenance_notice_recipients', 'user_id']")
&& str_contains($deletion,
"['cloud_maintenance_notice_instances', 'user_id']"),
'Account deletion does not remove frozen maintenance recipient data.');
$assert(str_contains($deletion,
"(string) \$recipient->status === 'sending'")
&& str_contains($deletion,
'投递中的维护通知'),
'Account deletion can erase an in-flight maintenance delivery.');
$assert(str_contains($deletion,
'$noticeService->recalculate((int) $noticeId);'),
'Account deletion does not reconcile affected notice aggregates.');
echo "Maintenance notice account-deletion contracts passed.\n";

@ -0,0 +1,224 @@
<?php
declare(strict_types=1);
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Services\Cloud\CloudMaintenanceNoticeDeliveryService;
use App\Services\Cloud\CloudMaintenanceNoticeAudience;
use App\Services\Cloud\CloudMaintenanceNoticeService;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$capsule = new Capsule();
$capsule->addConnection([
'driver' => 'sqlite', 'database' => ':memory:',
], 'default');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$schema = $capsule->schema('default');
$schema->create('user', static function (Blueprint $table): void {
$table->increments('id');
$table->string('email');
});
$schema->create('cloud_maintenance_notices', static function (Blueprint $table): void {
$table->increments('id');
$table->string('notice_key');
$table->integer('requested_by_user_id');
$table->string('request_idempotency_key');
$table->string('request_fingerprint');
$table->string('scope_type');
$table->integer('scope_id');
$table->text('scope_snapshot_json');
$table->string('subject');
$table->text('message_text');
$table->string('impact');
$table->integer('maintenance_starts_at');
$table->integer('maintenance_ends_at');
$table->integer('scheduled_at');
$table->string('audience_digest');
$table->integer('instance_count');
$table->integer('recipient_count');
$table->integer('skipped_count');
$table->string('status');
$table->string('last_error_code');
$table->integer('terminal_at')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_maintenance_notice_instances', static function (Blueprint $table): void {
$table->increments('id');
$table->integer('notice_id');
$table->integer('instance_id');
$table->integer('user_id');
$table->integer('host_id');
$table->integer('public_ip_pool_id')->nullable();
$table->string('instance_name');
$table->string('ipv4');
$table->string('ipv6');
$table->text('evidence_json');
$table->integer('created_at');
});
$schema->create('cloud_maintenance_notice_recipients', static function (Blueprint $table): void {
$table->increments('id');
$table->integer('notice_id');
$table->integer('user_id');
$table->string('email_snapshot');
$table->string('status');
$table->integer('attempts');
$table->integer('available_at');
$table->string('lease_token')->nullable();
$table->integer('lease_until');
$table->integer('sent_at')->nullable();
$table->string('last_error_code');
$table->string('last_error');
$table->integer('created_at');
$table->integer('updated_at');
});
$now = 1793900000;
Capsule::table('user')->insert([
['id' => 1, 'email' => 'ok@example.com'],
['id' => 2, 'email' => 'unknown@example.com'],
['id' => 3, 'email' => 'blocked@example.com'],
]);
$insertNotice = static function (int $id, string $status = 'scheduled') use ($now): void {
Capsule::table('cloud_maintenance_notices')->insert([
'id' => $id, 'notice_key' => str_pad(dechex($id), 32, '0', STR_PAD_LEFT),
'requested_by_user_id' => 9,
'request_idempotency_key' => 'delivery-test-' . str_pad((string) $id, 4, '0', STR_PAD_LEFT),
'request_fingerprint' => str_repeat('a', 64),
'scope_type' => 'host', 'scope_id' => 1,
'scope_snapshot_json' => '{"label":"HKG-A"}',
'subject' => 'Maintenance <notice>', 'message_text' => 'Please save.',
'impact' => 'network_interruption',
'maintenance_starts_at' => $now + 3600,
'maintenance_ends_at' => $now + 7200,
'scheduled_at' => $now, 'audience_digest' => str_repeat('b', 64),
'instance_count' => 1, 'recipient_count' => 1, 'skipped_count' => 0,
'status' => $status, 'last_error_code' => '', 'terminal_at' => null,
'created_at' => $now, 'updated_at' => $now,
]);
};
$insertRecipient = static function (int $id, int $noticeId, int $userId,
string $email, string $status, int $availableAt,
?string $token = null, int $leaseUntil = 0) use ($now): void {
Capsule::table('cloud_maintenance_notice_recipients')->insert([
'id' => $id, 'notice_id' => $noticeId, 'user_id' => $userId,
'email_snapshot' => $email, 'status' => $status,
'attempts' => $status === 'sending' ? 1 : 0,
'available_at' => $availableAt, 'lease_token' => $token,
'lease_until' => $leaseUntil, 'sent_at' => null,
'last_error_code' => '', 'last_error' => '',
'created_at' => $now, 'updated_at' => $now,
]);
Capsule::table('cloud_maintenance_notice_instances')->insert([
'notice_id' => $noticeId, 'instance_id' => 100 + $noticeId,
'user_id' => $userId, 'host_id' => 1,
'public_ip_pool_id' => null, 'instance_name' => 'vm-' . $noticeId,
'ipv4' => '203.0.113.' . $noticeId,
'ipv6' => '2001:db8::' . $noticeId,
'evidence_json' => '{}', 'created_at' => $now,
]);
};
$insertNotice(1);
$insertRecipient(1, 1, 1, 'ok@example.com', 'pending', $now);
$insertRecipient(2, 1, 2, 'unknown@example.com', 'pending', $now + 60);
$sent = [];
$delivery = new CloudMaintenanceNoticeDeliveryService(
static function (string $to, string $subject, array $variables) use (&$sent): bool {
$sent[] = compact('to', 'subject', 'variables');
return $to === 'ok@example.com';
},
static fn (string $email): bool => $email === 'blocked@example.com',
static fn (): int => $now,
new CloudMaintenanceNoticeService(
new CloudMaintenanceNoticeAudience(),
static fn (): int => $now
)
);
$assert($delivery->tick() === true && count($sent) === 1,
'One due recipient must be delivered per tick.');
$row = Capsule::table('cloud_maintenance_notice_recipients')->find(1);
$assert($row->status === 'sent' && (int) $row->attempts === 1
&& (int) $row->sent_at === $now,
'Accepted mail was not finalized exactly once.');
$assert($sent[0]['variables']['maintenance_starts_utc']
=== gmdate('Y-m-d H:i:s \\U\\T\\C', $now + 3600)
&& $sent[0]['variables']['instances'][0]['ipv6'] === '2001:db8::1',
'Delivery lost the frozen UTC window or recipient instance list.');
$assert($delivery->tick() === false && count($sent) === 1,
'A future recipient was delivered early.');
Capsule::table('cloud_maintenance_notice_recipients')->where('id', 2)
->update(['available_at' => $now]);
$assert($delivery->tick() === true && count($sent) === 2,
'The second due recipient was not attempted.');
$row = Capsule::table('cloud_maintenance_notice_recipients')->find(2);
$notice = Capsule::table('cloud_maintenance_notices')->find(1);
$assert($row->status === 'unknown'
&& $row->last_error_code === 'delivery_effect_unknown'
&& $notice->status === 'attention',
'A transport false result was not preserved as unknown attention.');
$assert($delivery->tick() === false && count($sent) === 2,
'An unknown delivery was replayed automatically.');
$insertNotice(2, 'sending');
$insertRecipient(3, 2, 1, 'expired@example.com', 'sending',
$now - 400, str_repeat('c', 32), $now - 1);
$assert($delivery->tick() === true && count($sent) === 2,
'An expired sending lease was not converged without transport.');
$row = Capsule::table('cloud_maintenance_notice_recipients')->find(3);
$assert($row->status === 'unknown'
&& $row->last_error_code === 'delivery_lease_expired',
'Expired lease was retried or lost its unknown-effect evidence.');
$insertNotice(3);
$insertRecipient(4, 3, 3, 'blocked@example.com', 'pending', $now);
$assert($delivery->tick() === true && count($sent) === 2,
'A blacklisted recipient did not settle without transport.');
$row = Capsule::table('cloud_maintenance_notice_recipients')->find(4);
$notice = Capsule::table('cloud_maintenance_notices')->find(3);
$assert($row->status === 'skipped'
&& $row->last_error_code === 'recipient_blacklisted'
&& $notice->status === 'completed',
'Blacklisted delivery was sent or aggregated as a failure.');
$insertNotice(4);
$insertRecipient(5, 4, 99, 'deleted@example.com', 'pending', $now);
$assert($delivery->tick() === true && count($sent) === 2
&& Capsule::table('cloud_maintenance_notice_recipients')->find(5)
->status === 'skipped',
'A deleted user still reached the transport.');
$insertNotice(5);
$insertRecipient(6, 5, 1, 'fenced@example.com', 'pending', $now);
$fenced = new CloudMaintenanceNoticeDeliveryService(
static function (): bool {
Capsule::table('cloud_maintenance_notice_recipients')
->where('id', 6)->update([
'status' => 'unknown', 'lease_token' => null,
'lease_until' => 0,
'last_error_code' => 'external_reconciliation',
]);
return true;
},
static fn (): bool => false,
static fn (): int => $now,
new CloudMaintenanceNoticeService(
new CloudMaintenanceNoticeAudience(),
static fn (): int => $now
)
);
$assert($fenced->tick() === true
&& Capsule::table('cloud_maintenance_notice_recipients')->find(6)
->status === 'unknown',
'A stale lease token overwrote a reconciled terminal outcome.');
echo "Maintenance notice delivery tests passed.\n";

@ -27,6 +27,8 @@ $heartbeats = [
'message' => '未收到运行记录', 'checked_at' => time()],
['key' => 'recovery', 'label' => '任务安全收敛', 'status' => 'unknown',
'message' => '未收到运行记录', 'checked_at' => time()],
['key' => 'notification', 'label' => '维护邮件投递', 'status' => 'unknown',
'message' => '未收到运行记录', 'checked_at' => time()],
['key' => 'lifecycle', 'label' => '到期与流量调度', 'status' => 'warning',
'message' => '本轮部分检查未完成', 'checked_at' => time()],
];
@ -50,10 +52,11 @@ foreach ([['user_id' => 42, 'is_admin' => false], ['user_id' => 0, 'is_admin' =>
$assert($calls === [], 'Unauthorized requests performed probes.');
$payload = $service->snapshot($actor);
$checks = array_column($payload['checks'], null, 'key');
$assert(count($checks) === 9 && $calls === ['default', 'log', 'agents', 'redis'], 'Missing or duplicate probes.');
$assert(count($checks) === 10 && $calls === ['default', 'log', 'agents', 'redis'], 'Missing or duplicate probes.');
$assert($checks['agents']['status'] === 'ok' && $checks['worker']['status'] === 'busy'
&& $checks['policy']['status'] === 'unknown'
&& $checks['vpc']['status'] === 'unknown'
&& $checks['notification']['status'] === 'unknown'
&& $checks['lifecycle']['status'] === 'warning',
'Health or heartbeat semantics changed.');
$assert(!str_contains(json_encode($payload), 'MUST_NOT_EXPOSE'), 'Heartbeat internal fields leaked.');
@ -178,7 +181,7 @@ $response = (new CloudAdminRuntimeController())->index($request, new Response())
$assert($response->getStatusCode() === 200
&& $response->getHeaderLine('Cache-Control') === 'private, no-store', 'Runtime response cached or unavailable.');
$decoded = json_decode((string) $response->getBody(), true);
$assert(count($decoded['checks']) === 9
$assert(count($decoded['checks']) === 10
&& array_column($decoded['checks'], 'status', 'key')['agents'] === 'error',
'Partial failure lost other checks or fabricated Agent configuration health.');
$routes = file_get_contents(dirname(__DIR__) . '/config/routes_cloud.php');

@ -46,7 +46,9 @@ $removeTree = static function (string $directory) use (&$removeTree): void {
try {
$missing = $testRoot . '/not-created';
$empty = $checks($missing);
$assert(array_keys($empty) === ['worker', 'policy', 'vpc', 'recovery', 'lifecycle']
$assert(array_keys($empty) === [
'worker', 'policy', 'vpc', 'recovery', 'notification', 'lifecycle',
]
&& array_unique(array_column($empty, 'status')) === ['unknown'] && !file_exists($missing),
'Missing snapshot must be unknown, cover all kinds, and never create runtime files.');
@ -55,7 +57,10 @@ try {
$policy = CloudRuntimeHeartbeat::start('policy', false, $directory, $clock);
$policy2 = CloudRuntimeHeartbeat::start('policy', false, $directory, $clock);
$recovery = CloudRuntimeHeartbeat::start('recovery', false, $directory, $clock);
$assert(count($files($directory)) === 4, 'Concurrent worker identities overwrite one another.');
$notification = CloudRuntimeHeartbeat::start(
'notification', false, $directory, $clock
);
$assert(count($files($directory)) === 5, 'Concurrent worker identities overwrite one another.');
$assert(is_readable($directory) && is_writable($directory),
'Runtime directory is not readable and writable.');
foreach ($files($directory) as $path) {
@ -64,8 +69,14 @@ try {
}
$assert($checks($directory)['worker']['status'] === 'busy'
&& $checks($directory)['policy']['active_count'] === 2
&& $checks($directory)['recovery']['active_count'] === 1,
&& $checks($directory)['recovery']['active_count'] === 1
&& $checks($directory)['notification']['active_count'] === 1,
'Starting workers or multiple policy reporters were not represented.');
$notification->busy('maintenance_notification');
$assert(str_contains(
$checks($directory)['notification']['message'],
'维护邮件投递'
), 'Maintenance delivery did not receive an isolated runtime phase.');
$normal->busy('worker_tick');
$assert(str_contains($checks($directory)['worker']['message'], '任务消费或核对'),
@ -459,16 +470,24 @@ try {
'A retired global engine-readiness gate still blocks per-host Agent work.');
$workerUnit = file_get_contents(dirname(__DIR__, 2)
. '/deploy/virt-cloud-recovery-worker.service');
$notificationUnit = file_get_contents(dirname(__DIR__, 2)
. '/deploy/virt-cloud-notification-worker.service');
$xcat = file_get_contents(dirname(__DIR__) . '/app/Command/XCat.php');
$assert(str_contains($workerCommand, "'--recovery-only'")
&& str_contains($workerCommand, 'CloudAutomaticOperationRecoveryService')
&& str_contains($workerCommand,
"if (!\$policyOnly && !\$vpcBatchOnly && !\$recoveryOnly)")
&& str_contains($workerCommand,
'!$policyOnly && !$vpcBatchOnly && !$recoveryOnly')
&& str_contains($workerCommand, '&& !$notificationOnly')
&& str_contains($xcat, '--recovery-only')
&& str_contains($workerUnit, 'cloudWorker --recovery-only'),
'The dedicated recovery worker is missing, sends mail, or is not deployable.');
$assert(str_contains($workerCommand, "'--notification-only'")
&& str_contains($workerCommand,
'CloudMaintenanceNoticeDeliveryService')
&& str_contains($xcat, '--notification-only')
&& str_contains($notificationUnit,
'cloudWorker --notification-only'),
'The isolated maintenance notification worker is not deployable.');
Capsule::table('cloud_instances')->insert(['id' => 10, 'status' => 'active',
'service_status' => 'active', 'expires_at' => time() + 3600,

Loading…
Cancel
Save