diff --git a/deploy/virt-cloud-notification-worker.service b/deploy/virt-cloud-notification-worker.service new file mode 100644 index 00000000..17ea5d98 --- /dev/null +++ b/deploy/virt-cloud-notification-worker.service @@ -0,0 +1,24 @@ +[Unit] +Description=Virt Cloud maintenance notification worker +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=apache +Group=apache +WorkingDirectory=/home/virt-cloud/web +ExecStart=/usr/bin/php /home/virt-cloud/web/xcat cloudWorker --notification-only +Restart=on-failure +RestartSec=5 +UMask=0077 +NoNewPrivileges=true +PrivateTmp=true +ProtectHome=false +ProtectSystem=strict +ReadWritePaths=-/home/virt-cloud/runtime/php +ReadWritePaths=-/home/virt-cloud/web/storage +TimeoutStopSec=30 + +[Install] +WantedBy=multi-user.target diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 3e0559f2..cb713d5c 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -228,15 +228,19 @@ cp /home/virt-cloud/deploy/virt-cloud-lifecycle.service /etc/systemd/system/ cp /home/virt-cloud/deploy/virt-cloud-policy-worker@.service /etc/systemd/system/ cp /home/virt-cloud/deploy/virt-cloud-vpc-worker@.service /etc/systemd/system/ cp /home/virt-cloud/deploy/virt-cloud-recovery-worker.service /etc/systemd/system/ +cp /home/virt-cloud/deploy/virt-cloud-notification-worker.service /etc/systemd/system/ systemctl daemon-reload systemctl enable --now virt-cloud-worker.service \ - virt-cloud-recovery-worker.service virt-cloud-lifecycle.service \ + virt-cloud-recovery-worker.service virt-cloud-notification-worker.service \ + virt-cloud-lifecycle.service \ virt-cloud-policy-worker@1.service virt-cloud-policy-worker@2.service \ virt-cloud-vpc-worker@{1..8}.service ``` 普通 worker 负责购买、创建、重装、实例升级、邮件和最终删除;至少两个策略 worker 处理到期断网、网络恢复与删除前策略;lifecycle 生成续费账单并推进账期、流量、宽限和删除状态。VPC 批量加入/移出没有实例数量上限:面板会在一个数据库事务中释放同一批次的全部 VM 子任务,实际同一时刻运行多少台由 `virt-cloud-vpc-worker@N.service` 的实例数和各宿主机 Agent 的安全 admission 决定,超出 worker 并发度的任务仍已持久接收并等待空闲进程,不依赖浏览器。上例启用 8 个专用 worker;可按面板 CPU、数据库连接和宿主机冷启动能力增减实例数,而不是在页面设置固定的“最多 4 台”。未安装专用 unit 时普通 worker 仍会安全消费这些任务,但吞吐量较低。 +`virt-cloud-notification-worker.service` 只消费管理员维护通知,每轮最多领取一封,不占用虚拟机任务 worker。投递使用 300 秒租约;邮件传输一旦开始后返回失败、异常或租约过期都记为“效果未知”,不会自动重发。该服务是维护通知功能的必需组件;只更新 PHP 而不安装并启动该 unit 时,通知会安全保留在待发送状态。 + 共享流量池必须严格按 `disabled -> shadow -> enforced` 上线,并在强制执行前通过全部账本、采样和网络策略门禁。现有数据库不能使用 `php xcat cloudInit` 冒充增量升级,旧服务周期也不能通过 SQL 回填到流量池;完整步骤、验收项和故障处置见 [`TRAFFIC_POOL_ROLLOUT.md`](TRAFFIC_POOL_ROLLOUT.md)。 `virt-cloud-recovery-worker.service` 是必需的独立 operation 恢复进程,覆盖策略表中的实例电源、重装/救援、公网 IP、实例设置、实例升级、快照、防火墙、删除、生命周期网络、模板与 ISO Agent 叶子任务。它只做 operation-bound inspection/栅栏审计、持久晚到回执的 PHP 收敛或聚合父任务的本地汇总,不投递邮件,也绝不重放原 mutation。离线、身份冲突或账本异常按固定低频间隔持续核对,没有次数或总时长上限。它与普通队列隔离,避免某个离线 Agent 的核对占住正常 worker。 diff --git a/web/app/Command/CloudWorker.php b/web/app/Command/CloudWorker.php index 25110fcd..9ba871ae 100644 --- a/web/app/Command/CloudWorker.php +++ b/web/app/Command/CloudWorker.php @@ -2,6 +2,7 @@ namespace App\Command; use App\Services\Cloud\CloudInvoiceService; +use App\Services\Cloud\CloudMaintenanceNoticeDeliveryService; use App\Services\Cloud\CloudAutomaticOperationRecoveryService; use App\Services\Cloud\CloudRuntimeHeartbeat; use App\Services\Cloud\CloudWorkerService; @@ -14,34 +15,41 @@ final class CloudWorker $policyOnly = in_array('--policy-only', $argv, true); $vpcBatchOnly = in_array('--vpc-batch-only', $argv, true); $recoveryOnly = in_array('--recovery-only', $argv, true); + $notificationOnly = in_array('--notification-only', $argv, true); if (($policyOnly ? 1 : 0) + ($vpcBatchOnly ? 1 : 0) - + ($recoveryOnly ? 1 : 0) > 1) { + + ($recoveryOnly ? 1 : 0) + ($notificationOnly ? 1 : 0) > 1) { throw new \InvalidArgumentException( 'worker 专用模式不能同时使用。' ); } $heartbeat ??= CloudRuntimeHeartbeat::start( - $recoveryOnly ? 'recovery' : ($policyOnly ? 'policy' - : ($vpcBatchOnly ? 'vpc' : 'normal')), + $notificationOnly ? 'notification' + : ($recoveryOnly ? 'recovery' : ($policyOnly ? 'policy' + : ($vpcBatchOnly ? 'vpc' : 'normal'))), $once ); try { - $worker = $recoveryOnly - ? new CloudAutomaticOperationRecoveryService() - : new CloudWorkerService(null, $policyOnly, $vpcBatchOnly); + $worker = $notificationOnly + ? new CloudMaintenanceNoticeDeliveryService() + : ($recoveryOnly + ? new CloudAutomaticOperationRecoveryService() + : new CloudWorkerService(null, $policyOnly, $vpcBatchOnly)); do { $errors = []; $heartbeat->busy( - $recoveryOnly ? 'recovery_tick' : 'worker_tick' + $notificationOnly ? 'maintenance_notification' + : ($recoveryOnly ? 'recovery_tick' : 'worker_tick') ); try { $worked = $worker->tick(); - if (!$recoveryOnly) { + if (!$recoveryOnly && !$notificationOnly) { $errors = $worker->observationErrors(); } } catch (\Throwable $error) { - $code = $recoveryOnly - ? 'recovery_tick_failed' : 'worker_tick_failed'; + $code = $notificationOnly + ? 'maintenance_notification_failed' + : ($recoveryOnly + ? 'recovery_tick_failed' : 'worker_tick_failed'); $errors[] = $code; $heartbeat->error($code); fwrite(STDERR, 'cloud worker temporarily unavailable: ' . get_class($error) . PHP_EOL); @@ -54,7 +62,8 @@ final class CloudWorker $heartbeat->error($errors[0]); } // SMTP remains isolated from lifecycle and policy workers. - if (!$policyOnly && !$vpcBatchOnly && !$recoveryOnly) { + if (!$policyOnly && !$vpcBatchOnly && !$recoveryOnly + && !$notificationOnly) { $errors = array_merge($errors, self::deliverNotifications($heartbeat)); } if ($errors === []) { @@ -68,8 +77,9 @@ final class CloudWorker } while (!$once); } catch (\Throwable $error) { $heartbeat->error( - $recoveryOnly - ? 'recovery_tick_failed' : 'worker_tick_failed' + $notificationOnly ? 'maintenance_notification_failed' + : ($recoveryOnly + ? 'recovery_tick_failed' : 'worker_tick_failed') ); throw $error; } finally { diff --git a/web/app/Command/XCat.php b/web/app/Command/XCat.php index bef15158..1919fe17 100644 --- a/web/app/Command/XCat.php +++ b/web/app/Command/XCat.php @@ -100,7 +100,7 @@ class XCat echo "Virt Cloud CLI\n" . " php xcat createAdmin [name]\n" . " php xcat cloudInit\n" - . " php xcat cloudWorker [--once] [--policy-only|--vpc-batch-only|--recovery-only]\n" + . " php xcat cloudWorker [--once] [--policy-only|--vpc-batch-only|--recovery-only|--notification-only]\n" . " php xcat cloudLifecycle [--once]\n" . " php xcat cloudTrafficPoolAttention inspect --pool-id=N\n" . " php xcat cloudTrafficPoolAttention inspect-resolution --resolution-key=UUID\n" diff --git a/web/app/Services/Cloud/CloudMaintenanceNoticeDeliveryService.php b/web/app/Services/Cloud/CloudMaintenanceNoticeDeliveryService.php new file mode 100644 index 00000000..d354457f --- /dev/null +++ b/web/app/Services/Cloud/CloudMaintenanceNoticeDeliveryService.php @@ -0,0 +1,255 @@ +clock = \Closure::fromCallable($clock + ?? static fn (): int => time()); + $this->sender = \Closure::fromCallable($sender + ?? static fn (string $to, string $subject, array $variables): bool => + Mail::send( + $to, + $subject, + 'cloud/maintenance-notice.tpl', + $variables, + [], + 'maintenance_notice' + )); + $this->blacklist = \Closure::fromCallable($blacklist + ?? static fn (string $email): bool => + Mail::isRecipientBlacklisted($email)); + } + + public function tick(): bool + { + $worked = $this->convergeExpiredLeases(); + $claim = $this->claim(); + if ($claim === null) { + return $worked; + } + if (isset($claim['settled_notice_id'])) { + $this->notices->recalculate((int) $claim['settled_notice_id']); + return true; + } + + $recipient = $claim['recipient']; + $token = $claim['lease_token']; + $recipientId = (int) $recipient->id; + $noticeId = (int) $recipient->notice_id; + try { + if (!User::where('id', (int) $recipient->user_id)->exists()) { + $this->finalize($recipientId, $noticeId, $token, + 'skipped', 'user_deleted', ''); + return true; + } + if (($this->blacklist)((string) $recipient->email_snapshot)) { + $this->finalize($recipientId, $noticeId, $token, + 'skipped', 'recipient_blacklisted', ''); + return true; + } + $mail = $this->mail($recipient); + } catch (\Throwable $error) { + $this->finalize($recipientId, $noticeId, $token, + 'failed', 'delivery_preflight_failed', get_class($error)); + return true; + } + + try { + $accepted = ($this->sender)( + (string) $recipient->email_snapshot, + (string) $mail['subject'], + $mail['variables'] + ) === true; + } catch (\Throwable $error) { + $this->finalize($recipientId, $noticeId, $token, + 'unknown', 'delivery_effect_unknown', get_class($error)); + return true; + } + $this->finalize( + $recipientId, + $noticeId, + $token, + $accepted ? 'sent' : 'unknown', + $accepted ? '' : 'delivery_effect_unknown', + '' + ); + return true; + } + + private function claim(): ?array + { + return DB::connection('default')->transaction(function (): ?array { + $now = $this->now(); + $recipient = CloudMaintenanceNoticeRecipient::where( + 'status', 'pending' + )->where('available_at', '<=', $now)->orderBy('available_at') + ->orderBy('id')->lockForUpdate()->first(); + if ($recipient === null) { + return null; + } + $notice = CloudMaintenanceNotice::where( + 'id', $recipient->notice_id + )->lockForUpdate()->first(); + if ($notice === null || !in_array((string) $notice->status, + ['scheduled', 'sending'], true)) { + $recipient->status = 'cancelled'; + $recipient->last_error_code = 'notice_not_deliverable'; + $recipient->last_error = ''; + $recipient->updated_at = $now; + $recipient->save(); + return ['settled_notice_id' => (int) $recipient->notice_id]; + } + $token = bin2hex(random_bytes(16)); + $recipient->status = 'sending'; + $recipient->attempts = (int) $recipient->attempts + 1; + $recipient->lease_token = $token; + $recipient->lease_until = $now + self::LEASE_SECONDS; + $recipient->last_error_code = ''; + $recipient->last_error = ''; + $recipient->updated_at = $now; + if (!$recipient->save()) { + throw new \RuntimeException('Maintenance recipient claim failed.'); + } + if ((string) $notice->status === 'scheduled') { + $notice->status = 'sending'; + $notice->updated_at = $now; + $notice->save(); + } + return ['recipient' => $recipient, 'lease_token' => $token]; + }, 3); + } + + private function convergeExpiredLeases(): bool + { + $noticeIds = DB::connection('default')->transaction(function (): array { + $now = $this->now(); + $rows = CloudMaintenanceNoticeRecipient::where('status', 'sending') + ->where('lease_until', '>', 0) + ->where('lease_until', '<=', $now) + ->orderBy('lease_until')->orderBy('id') + ->limit(self::EXPIRED_LEASE_LIMIT)->lockForUpdate()->get(); + $noticeIds = []; + foreach ($rows as $row) { + $updated = CloudMaintenanceNoticeRecipient::where('id', $row->id) + ->where('status', 'sending') + ->where('lease_token', $row->lease_token)->update([ + 'status' => 'unknown', + 'lease_token' => null, + 'lease_until' => 0, + 'last_error_code' => 'delivery_lease_expired', + 'last_error' => '', + 'updated_at' => $now, + ]); + if ($updated === 1) { + $noticeIds[(int) $row->notice_id] = true; + } + } + return array_keys($noticeIds); + }, 3); + foreach ($noticeIds as $noticeId) { + $this->notices->recalculate((int) $noticeId); + } + return $noticeIds !== []; + } + + private function mail(CloudMaintenanceNoticeRecipient $recipient): array + { + $notice = CloudMaintenanceNotice::where( + 'id', $recipient->notice_id + )->first(); + if ($notice === null) { + throw new \RuntimeException('Maintenance notice is missing.'); + } + $scope = json_decode((string) $notice->scope_snapshot_json, + true, 32, JSON_THROW_ON_ERROR); + if (!is_array($scope) || !is_string($scope['label'] ?? null)) { + throw new \RuntimeException('Maintenance scope is invalid.'); + } + $instances = CloudMaintenanceNoticeInstance::where( + 'notice_id', $notice->id + )->where('user_id', $recipient->user_id)->orderBy('id')->get() + ->map(static fn ($instance): array => [ + 'name' => (string) $instance->instance_name, + 'ipv4' => (string) $instance->ipv4, + 'ipv6' => (string) $instance->ipv6, + ])->all(); + if ($instances === []) { + throw new \RuntimeException('Maintenance instance list is empty.'); + } + $impacts = [ + 'network_interruption' => '网络可能短时中断', + 'instance_restart' => '实例将重新启动', + 'service_interruption' => '服务可能短时中断', + 'no_interruption' => '预计不会中断服务', + ]; + $impact = $impacts[(string) $notice->impact] ?? null; + if ($impact === null) { + throw new \RuntimeException('Maintenance impact is invalid.'); + } + return [ + 'subject' => (string) $notice->subject, + 'variables' => [ + 'notice_subject' => (string) $notice->subject, + 'message_text' => (string) $notice->message_text, + 'scope_label' => (string) $scope['label'], + 'impact_label' => $impact, + 'maintenance_starts_utc' => gmdate( + 'Y-m-d H:i:s \\U\\T\\C', + (int) $notice->maintenance_starts_at + ), + 'maintenance_ends_utc' => gmdate( + 'Y-m-d H:i:s \\U\\T\\C', + (int) $notice->maintenance_ends_at + ), + 'instances' => $instances, + ], + ]; + } + + private function finalize(int $recipientId, int $noticeId, + string $token, string $status, string $errorCode, + string $error): void + { + $now = $this->now(); + $updated = CloudMaintenanceNoticeRecipient::where('id', $recipientId) + ->where('status', 'sending')->where('lease_token', $token) + ->update([ + 'status' => $status, + 'lease_token' => null, + 'lease_until' => 0, + 'sent_at' => $status === 'sent' ? $now : null, + 'last_error_code' => $errorCode, + 'last_error' => mb_substr($error, 0, 500), + 'updated_at' => $now, + ]); + if ($updated === 1) { + $this->notices->recalculate($noticeId); + } + } + + private function now(): int + { + return (int) ($this->clock)(); + } +} diff --git a/web/app/Services/Cloud/CloudMaintenanceNoticeService.php b/web/app/Services/Cloud/CloudMaintenanceNoticeService.php index 896c07c5..8347fe4a 100644 --- a/web/app/Services/Cloud/CloudMaintenanceNoticeService.php +++ b/web/app/Services/Cloud/CloudMaintenanceNoticeService.php @@ -349,7 +349,12 @@ final class CloudMaintenanceNoticeService $unknown = $count('unknown'); $failed = $count('failed'); $cancelled = $count('cancelled'); - if ($pending > 0 || $sending > 0) { + $total = array_sum(array_map('intval', $counts)); + if ($total === 0) { + $status = 'cancelled'; + $terminalAt = $now; + $errorCode = 'audience_deleted'; + } elseif ($pending > 0 || $sending > 0) { $status = ($sending > 0 || $sent > 0 || $unknown > 0 || $failed > 0) ? 'sending' : 'scheduled'; $terminalAt = null; @@ -364,10 +369,14 @@ final class CloudMaintenanceNoticeService $status = 'completed'; $terminalAt = $now; $errorCode = ''; - } else { + } elseif ($cancelled > 0) { $status = 'cancelled'; $terminalAt = $now; $errorCode = 'admin_cancelled'; + } else { + $status = 'completed'; + $terminalAt = $now; + $errorCode = ''; } $notice->status = $status; $notice->terminal_at = $terminalAt; diff --git a/web/app/Services/Cloud/CloudRuntimeHealthService.php b/web/app/Services/Cloud/CloudRuntimeHealthService.php index 65681e9e..f194fa51 100644 --- a/web/app/Services/Cloud/CloudRuntimeHealthService.php +++ b/web/app/Services/Cloud/CloudRuntimeHealthService.php @@ -91,6 +91,7 @@ final class CloudRuntimeHealthService foreach (['worker' => '普通任务处理', 'policy' => '策略任务处理', 'vpc' => 'VPC 批量任务处理', 'recovery' => '任务安全收敛', + 'notification' => '维护邮件投递', 'lifecycle' => '到期与流量调度'] as $key => $label) { $found = null; foreach (is_array($heartbeats) ? $heartbeats : [] as $row) { diff --git a/web/app/Services/Cloud/CloudRuntimeHeartbeat.php b/web/app/Services/Cloud/CloudRuntimeHeartbeat.php index b0ace9b1..7d79ab39 100644 --- a/web/app/Services/Cloud/CloudRuntimeHeartbeat.php +++ b/web/app/Services/Cloud/CloudRuntimeHeartbeat.php @@ -8,9 +8,11 @@ final class CloudRuntimeHeartbeat 'policy' => ['policy', '策略任务 worker'], 'vpc' => ['vpc', 'VPC 批量任务 worker'], 'recovery' => ['recovery', '任务安全收敛 worker'], + 'notification' => ['notification', '维护通知 worker'], 'lifecycle' => ['lifecycle', '生命周期调度']]; private const PHASES = ['startup' => '启动', 'worker_tick' => '任务消费或核对', 'recovery_tick' => '任务安全收敛', + 'maintenance_notification' => '维护邮件投递', 'notification' => '邮件通知', 'lifecycle_tick' => '生命周期巡检', 'provision_expiry' => '开通账单过期处理', 'invoice_generation' => '续费账单生成', 'automatic_backup_schedule' => '自动备份调度', @@ -24,7 +26,7 @@ final class CloudRuntimeHeartbeat 'instance_reconcile' => '实例策略核对', 'idle' => '等待下一轮', 'stopped' => '已退出']; private const ERRORS = ['worker_tick_failed', 'recovery_tick_failed', - 'notification_failed', + 'notification_failed', 'maintenance_notification_failed', 'lifecycle_tick_failed', 'provision_expiry_failed', 'invoice_generation_failed', 'automatic_backup_schedule_failed', 'automatic_backup_retention_failed', @@ -221,7 +223,7 @@ final class CloudRuntimeHeartbeat if (++$scanned > self::SCAN_LIMIT || $removed >= 32) { break; } - if (preg_match('/^(normal|policy|vpc|recovery|lifecycle)-[1-9][0-9]*-[a-f0-9]{24}\.json(?:\.tmp)?$/D', + if (preg_match('/^(normal|policy|vpc|recovery|notification|lifecycle)-[1-9][0-9]*-[a-f0-9]{24}\.json(?:\.tmp)?$/D', $entry->getFilename()) !== 1) { continue; } @@ -311,7 +313,7 @@ final class CloudRuntimeHeartbeat $incomplete = true; break; } - if (preg_match('/^(normal|policy|vpc|recovery|lifecycle)-[1-9][0-9]*-[a-f0-9]{24}\.json$/D', + if (preg_match('/^(normal|policy|vpc|recovery|notification|lifecycle)-[1-9][0-9]*-[a-f0-9]{24}\.json$/D', $entry->getFilename(), $match) !== 1) { continue; } diff --git a/web/app/Services/UserDataDeletionService.php b/web/app/Services/UserDataDeletionService.php index 0368c4c9..0e97f4b9 100644 --- a/web/app/Services/UserDataDeletionService.php +++ b/web/app/Services/UserDataDeletionService.php @@ -4,6 +4,7 @@ namespace App\Services; use App\Models\EmailLog; use App\Services\TicketAttachmentService; +use App\Services\Cloud\CloudMaintenanceNoticeService; use App\Models\User; class UserDataDeletionService @@ -23,6 +24,8 @@ class UserDataDeletionService // remove. cloud_jobs may contain encrypted provisioning credentials. ['cloud_vpc_membership_batch_items', 'user_id'], ['cloud_vpc_membership_batches', 'user_id'], + ['cloud_maintenance_notice_recipients', 'user_id'], + ['cloud_maintenance_notice_instances', 'user_id'], ['cloud_jobs', 'user_id'], ['cloud_instance_snapshots', 'user_id'], ['cloud_instance_firewalls', 'user_id'], @@ -97,6 +100,25 @@ class UserDataDeletionService '用户仍有支付安全栅栏,请先完成网关或链上状态处理'); } } + $maintenanceNoticeIds = []; + if ($this->hasColumns( + $connection, + 'cloud_maintenance_notice_recipients', + ['notice_id', 'user_id', 'status'] + )) { + $maintenanceRecipients = $connection->table( + 'cloud_maintenance_notice_recipients' + )->whereIn('user_id', $ids)->orderBy('id') + ->lockForUpdate()->get(['notice_id', 'status']); + foreach ($maintenanceRecipients as $recipient) { + if ((string) $recipient->status === 'sending') { + throw new \RuntimeException( + '用户仍有投递中的维护通知,请等待投递结果核对后再删除账户' + ); + } + $maintenanceNoticeIds[(int) $recipient->notice_id] = true; + } + } foreach ($users as $user) { LoginSessionRevocationService::revokeUser($user); } @@ -140,6 +162,12 @@ class UserDataDeletionService $emails, $ticketAttachmentStorageKeys ); + if ($maintenanceNoticeIds !== []) { + $noticeService = new CloudMaintenanceNoticeService(); + foreach (array_keys($maintenanceNoticeIds) as $noticeId) { + $noticeService->recalculate((int) $noticeId); + } + } $trafficScanRows = $this->deleteCloudTrafficScanFailures( $connection, $trafficInstanceIds, diff --git a/web/tests/CloudMaintenanceNoticeAccountDeletionContractTest.php b/web/tests/CloudMaintenanceNoticeAccountDeletionContractTest.php new file mode 100644 index 00000000..5db4f4e7 --- /dev/null +++ b/web/tests/CloudMaintenanceNoticeAccountDeletionContractTest.php @@ -0,0 +1,27 @@ +status === 'sending'") + && str_contains($deletion, + '投递中的维护通知'), + 'Account deletion can erase an in-flight maintenance delivery.'); +$assert(str_contains($deletion, + '$noticeService->recalculate((int) $noticeId);'), + 'Account deletion does not reconcile affected notice aggregates.'); + +echo "Maintenance notice account-deletion contracts passed.\n"; diff --git a/web/tests/CloudMaintenanceNoticeDeliveryTest.php b/web/tests/CloudMaintenanceNoticeDeliveryTest.php new file mode 100644 index 00000000..e851f595 --- /dev/null +++ b/web/tests/CloudMaintenanceNoticeDeliveryTest.php @@ -0,0 +1,224 @@ +addConnection([ + 'driver' => 'sqlite', 'database' => ':memory:', +], 'default'); +$capsule->setAsGlobal(); +$capsule->bootEloquent(); +$schema = $capsule->schema('default'); + +$schema->create('user', static function (Blueprint $table): void { + $table->increments('id'); + $table->string('email'); +}); +$schema->create('cloud_maintenance_notices', static function (Blueprint $table): void { + $table->increments('id'); + $table->string('notice_key'); + $table->integer('requested_by_user_id'); + $table->string('request_idempotency_key'); + $table->string('request_fingerprint'); + $table->string('scope_type'); + $table->integer('scope_id'); + $table->text('scope_snapshot_json'); + $table->string('subject'); + $table->text('message_text'); + $table->string('impact'); + $table->integer('maintenance_starts_at'); + $table->integer('maintenance_ends_at'); + $table->integer('scheduled_at'); + $table->string('audience_digest'); + $table->integer('instance_count'); + $table->integer('recipient_count'); + $table->integer('skipped_count'); + $table->string('status'); + $table->string('last_error_code'); + $table->integer('terminal_at')->nullable(); + $table->integer('created_at'); + $table->integer('updated_at'); +}); +$schema->create('cloud_maintenance_notice_instances', static function (Blueprint $table): void { + $table->increments('id'); + $table->integer('notice_id'); + $table->integer('instance_id'); + $table->integer('user_id'); + $table->integer('host_id'); + $table->integer('public_ip_pool_id')->nullable(); + $table->string('instance_name'); + $table->string('ipv4'); + $table->string('ipv6'); + $table->text('evidence_json'); + $table->integer('created_at'); +}); +$schema->create('cloud_maintenance_notice_recipients', static function (Blueprint $table): void { + $table->increments('id'); + $table->integer('notice_id'); + $table->integer('user_id'); + $table->string('email_snapshot'); + $table->string('status'); + $table->integer('attempts'); + $table->integer('available_at'); + $table->string('lease_token')->nullable(); + $table->integer('lease_until'); + $table->integer('sent_at')->nullable(); + $table->string('last_error_code'); + $table->string('last_error'); + $table->integer('created_at'); + $table->integer('updated_at'); +}); + +$now = 1793900000; +Capsule::table('user')->insert([ + ['id' => 1, 'email' => 'ok@example.com'], + ['id' => 2, 'email' => 'unknown@example.com'], + ['id' => 3, 'email' => 'blocked@example.com'], +]); +$insertNotice = static function (int $id, string $status = 'scheduled') use ($now): void { + Capsule::table('cloud_maintenance_notices')->insert([ + 'id' => $id, 'notice_key' => str_pad(dechex($id), 32, '0', STR_PAD_LEFT), + 'requested_by_user_id' => 9, + 'request_idempotency_key' => 'delivery-test-' . str_pad((string) $id, 4, '0', STR_PAD_LEFT), + 'request_fingerprint' => str_repeat('a', 64), + 'scope_type' => 'host', 'scope_id' => 1, + 'scope_snapshot_json' => '{"label":"HKG-A"}', + 'subject' => 'Maintenance ', 'message_text' => 'Please save.', + 'impact' => 'network_interruption', + 'maintenance_starts_at' => $now + 3600, + 'maintenance_ends_at' => $now + 7200, + 'scheduled_at' => $now, 'audience_digest' => str_repeat('b', 64), + 'instance_count' => 1, 'recipient_count' => 1, 'skipped_count' => 0, + 'status' => $status, 'last_error_code' => '', 'terminal_at' => null, + 'created_at' => $now, 'updated_at' => $now, + ]); +}; +$insertRecipient = static function (int $id, int $noticeId, int $userId, + string $email, string $status, int $availableAt, + ?string $token = null, int $leaseUntil = 0) use ($now): void { + Capsule::table('cloud_maintenance_notice_recipients')->insert([ + 'id' => $id, 'notice_id' => $noticeId, 'user_id' => $userId, + 'email_snapshot' => $email, 'status' => $status, + 'attempts' => $status === 'sending' ? 1 : 0, + 'available_at' => $availableAt, 'lease_token' => $token, + 'lease_until' => $leaseUntil, 'sent_at' => null, + 'last_error_code' => '', 'last_error' => '', + 'created_at' => $now, 'updated_at' => $now, + ]); + Capsule::table('cloud_maintenance_notice_instances')->insert([ + 'notice_id' => $noticeId, 'instance_id' => 100 + $noticeId, + 'user_id' => $userId, 'host_id' => 1, + 'public_ip_pool_id' => null, 'instance_name' => 'vm-' . $noticeId, + 'ipv4' => '203.0.113.' . $noticeId, + 'ipv6' => '2001:db8::' . $noticeId, + 'evidence_json' => '{}', 'created_at' => $now, + ]); +}; + +$insertNotice(1); +$insertRecipient(1, 1, 1, 'ok@example.com', 'pending', $now); +$insertRecipient(2, 1, 2, 'unknown@example.com', 'pending', $now + 60); +$sent = []; +$delivery = new CloudMaintenanceNoticeDeliveryService( + static function (string $to, string $subject, array $variables) use (&$sent): bool { + $sent[] = compact('to', 'subject', 'variables'); + return $to === 'ok@example.com'; + }, + static fn (string $email): bool => $email === 'blocked@example.com', + static fn (): int => $now, + new CloudMaintenanceNoticeService( + new CloudMaintenanceNoticeAudience(), + static fn (): int => $now + ) +); +$assert($delivery->tick() === true && count($sent) === 1, + 'One due recipient must be delivered per tick.'); +$row = Capsule::table('cloud_maintenance_notice_recipients')->find(1); +$assert($row->status === 'sent' && (int) $row->attempts === 1 + && (int) $row->sent_at === $now, + 'Accepted mail was not finalized exactly once.'); +$assert($sent[0]['variables']['maintenance_starts_utc'] + === gmdate('Y-m-d H:i:s \\U\\T\\C', $now + 3600) + && $sent[0]['variables']['instances'][0]['ipv6'] === '2001:db8::1', + 'Delivery lost the frozen UTC window or recipient instance list.'); +$assert($delivery->tick() === false && count($sent) === 1, + 'A future recipient was delivered early.'); + +Capsule::table('cloud_maintenance_notice_recipients')->where('id', 2) + ->update(['available_at' => $now]); +$assert($delivery->tick() === true && count($sent) === 2, + 'The second due recipient was not attempted.'); +$row = Capsule::table('cloud_maintenance_notice_recipients')->find(2); +$notice = Capsule::table('cloud_maintenance_notices')->find(1); +$assert($row->status === 'unknown' + && $row->last_error_code === 'delivery_effect_unknown' + && $notice->status === 'attention', + 'A transport false result was not preserved as unknown attention.'); +$assert($delivery->tick() === false && count($sent) === 2, + 'An unknown delivery was replayed automatically.'); + +$insertNotice(2, 'sending'); +$insertRecipient(3, 2, 1, 'expired@example.com', 'sending', + $now - 400, str_repeat('c', 32), $now - 1); +$assert($delivery->tick() === true && count($sent) === 2, + 'An expired sending lease was not converged without transport.'); +$row = Capsule::table('cloud_maintenance_notice_recipients')->find(3); +$assert($row->status === 'unknown' + && $row->last_error_code === 'delivery_lease_expired', + 'Expired lease was retried or lost its unknown-effect evidence.'); + +$insertNotice(3); +$insertRecipient(4, 3, 3, 'blocked@example.com', 'pending', $now); +$assert($delivery->tick() === true && count($sent) === 2, + 'A blacklisted recipient did not settle without transport.'); +$row = Capsule::table('cloud_maintenance_notice_recipients')->find(4); +$notice = Capsule::table('cloud_maintenance_notices')->find(3); +$assert($row->status === 'skipped' + && $row->last_error_code === 'recipient_blacklisted' + && $notice->status === 'completed', + 'Blacklisted delivery was sent or aggregated as a failure.'); + +$insertNotice(4); +$insertRecipient(5, 4, 99, 'deleted@example.com', 'pending', $now); +$assert($delivery->tick() === true && count($sent) === 2 + && Capsule::table('cloud_maintenance_notice_recipients')->find(5) + ->status === 'skipped', + 'A deleted user still reached the transport.'); + +$insertNotice(5); +$insertRecipient(6, 5, 1, 'fenced@example.com', 'pending', $now); +$fenced = new CloudMaintenanceNoticeDeliveryService( + static function (): bool { + Capsule::table('cloud_maintenance_notice_recipients') + ->where('id', 6)->update([ + 'status' => 'unknown', 'lease_token' => null, + 'lease_until' => 0, + 'last_error_code' => 'external_reconciliation', + ]); + return true; + }, + static fn (): bool => false, + static fn (): int => $now, + new CloudMaintenanceNoticeService( + new CloudMaintenanceNoticeAudience(), + static fn (): int => $now + ) +); +$assert($fenced->tick() === true + && Capsule::table('cloud_maintenance_notice_recipients')->find(6) + ->status === 'unknown', + 'A stale lease token overwrote a reconciled terminal outcome.'); + +echo "Maintenance notice delivery tests passed.\n"; diff --git a/web/tests/CloudRuntimeHealthTest.php b/web/tests/CloudRuntimeHealthTest.php index d1c72418..28d43cd8 100644 --- a/web/tests/CloudRuntimeHealthTest.php +++ b/web/tests/CloudRuntimeHealthTest.php @@ -27,6 +27,8 @@ $heartbeats = [ 'message' => '未收到运行记录', 'checked_at' => time()], ['key' => 'recovery', 'label' => '任务安全收敛', 'status' => 'unknown', 'message' => '未收到运行记录', 'checked_at' => time()], + ['key' => 'notification', 'label' => '维护邮件投递', 'status' => 'unknown', + 'message' => '未收到运行记录', 'checked_at' => time()], ['key' => 'lifecycle', 'label' => '到期与流量调度', 'status' => 'warning', 'message' => '本轮部分检查未完成', 'checked_at' => time()], ]; @@ -50,10 +52,11 @@ foreach ([['user_id' => 42, 'is_admin' => false], ['user_id' => 0, 'is_admin' => $assert($calls === [], 'Unauthorized requests performed probes.'); $payload = $service->snapshot($actor); $checks = array_column($payload['checks'], null, 'key'); -$assert(count($checks) === 9 && $calls === ['default', 'log', 'agents', 'redis'], 'Missing or duplicate probes.'); +$assert(count($checks) === 10 && $calls === ['default', 'log', 'agents', 'redis'], 'Missing or duplicate probes.'); $assert($checks['agents']['status'] === 'ok' && $checks['worker']['status'] === 'busy' && $checks['policy']['status'] === 'unknown' && $checks['vpc']['status'] === 'unknown' + && $checks['notification']['status'] === 'unknown' && $checks['lifecycle']['status'] === 'warning', 'Health or heartbeat semantics changed.'); $assert(!str_contains(json_encode($payload), 'MUST_NOT_EXPOSE'), 'Heartbeat internal fields leaked.'); @@ -178,7 +181,7 @@ $response = (new CloudAdminRuntimeController())->index($request, new Response()) $assert($response->getStatusCode() === 200 && $response->getHeaderLine('Cache-Control') === 'private, no-store', 'Runtime response cached or unavailable.'); $decoded = json_decode((string) $response->getBody(), true); -$assert(count($decoded['checks']) === 9 + $assert(count($decoded['checks']) === 10 && array_column($decoded['checks'], 'status', 'key')['agents'] === 'error', 'Partial failure lost other checks or fabricated Agent configuration health.'); $routes = file_get_contents(dirname(__DIR__) . '/config/routes_cloud.php'); diff --git a/web/tests/CloudRuntimeHeartbeatTest.php b/web/tests/CloudRuntimeHeartbeatTest.php index af4531de..85af4693 100644 --- a/web/tests/CloudRuntimeHeartbeatTest.php +++ b/web/tests/CloudRuntimeHeartbeatTest.php @@ -46,7 +46,9 @@ $removeTree = static function (string $directory) use (&$removeTree): void { try { $missing = $testRoot . '/not-created'; $empty = $checks($missing); - $assert(array_keys($empty) === ['worker', 'policy', 'vpc', 'recovery', 'lifecycle'] + $assert(array_keys($empty) === [ + 'worker', 'policy', 'vpc', 'recovery', 'notification', 'lifecycle', + ] && array_unique(array_column($empty, 'status')) === ['unknown'] && !file_exists($missing), 'Missing snapshot must be unknown, cover all kinds, and never create runtime files.'); @@ -55,7 +57,10 @@ try { $policy = CloudRuntimeHeartbeat::start('policy', false, $directory, $clock); $policy2 = CloudRuntimeHeartbeat::start('policy', false, $directory, $clock); $recovery = CloudRuntimeHeartbeat::start('recovery', false, $directory, $clock); - $assert(count($files($directory)) === 4, 'Concurrent worker identities overwrite one another.'); + $notification = CloudRuntimeHeartbeat::start( + 'notification', false, $directory, $clock + ); + $assert(count($files($directory)) === 5, 'Concurrent worker identities overwrite one another.'); $assert(is_readable($directory) && is_writable($directory), 'Runtime directory is not readable and writable.'); foreach ($files($directory) as $path) { @@ -64,8 +69,14 @@ try { } $assert($checks($directory)['worker']['status'] === 'busy' && $checks($directory)['policy']['active_count'] === 2 - && $checks($directory)['recovery']['active_count'] === 1, + && $checks($directory)['recovery']['active_count'] === 1 + && $checks($directory)['notification']['active_count'] === 1, 'Starting workers or multiple policy reporters were not represented.'); + $notification->busy('maintenance_notification'); + $assert(str_contains( + $checks($directory)['notification']['message'], + '维护邮件投递' + ), 'Maintenance delivery did not receive an isolated runtime phase.'); $normal->busy('worker_tick'); $assert(str_contains($checks($directory)['worker']['message'], '任务消费或核对'), @@ -459,16 +470,24 @@ try { 'A retired global engine-readiness gate still blocks per-host Agent work.'); $workerUnit = file_get_contents(dirname(__DIR__, 2) . '/deploy/virt-cloud-recovery-worker.service'); + $notificationUnit = file_get_contents(dirname(__DIR__, 2) + . '/deploy/virt-cloud-notification-worker.service'); $xcat = file_get_contents(dirname(__DIR__) . '/app/Command/XCat.php'); $assert(str_contains($workerCommand, "'--recovery-only'") && str_contains($workerCommand, 'CloudAutomaticOperationRecoveryService') - && str_contains($workerCommand, - "if (!\$policyOnly && !\$vpcBatchOnly && !\$recoveryOnly)") && str_contains($workerCommand, '!$policyOnly && !$vpcBatchOnly && !$recoveryOnly') + && str_contains($workerCommand, '&& !$notificationOnly') && str_contains($xcat, '--recovery-only') && str_contains($workerUnit, 'cloudWorker --recovery-only'), 'The dedicated recovery worker is missing, sends mail, or is not deployable.'); + $assert(str_contains($workerCommand, "'--notification-only'") + && str_contains($workerCommand, + 'CloudMaintenanceNoticeDeliveryService') + && str_contains($xcat, '--notification-only') + && str_contains($notificationUnit, + 'cloudWorker --notification-only'), + 'The isolated maintenance notification worker is not deployable.'); Capsule::table('cloud_instances')->insert(['id' => 10, 'status' => 'active', 'service_status' => 'active', 'expires_at' => time() + 3600,