diff --git a/deploy/virt-cloud-notification-worker.service b/deploy/virt-cloud-notification-worker.service index 17ea5d98..69a75e94 100644 --- a/deploy/virt-cloud-notification-worker.service +++ b/deploy/virt-cloud-notification-worker.service @@ -1,5 +1,5 @@ [Unit] -Description=Virt Cloud maintenance notification worker +Description=Virt Cloud customer notification worker After=network-online.target Wants=network-online.target diff --git a/web/app/Command/CloudWorker.php b/web/app/Command/CloudWorker.php index 9ba871ae..ee8a5bb8 100644 --- a/web/app/Command/CloudWorker.php +++ b/web/app/Command/CloudWorker.php @@ -2,7 +2,7 @@ namespace App\Command; use App\Services\Cloud\CloudInvoiceService; -use App\Services\Cloud\CloudMaintenanceNoticeDeliveryService; +use App\Services\Cloud\CloudNotificationDeliveryService; use App\Services\Cloud\CloudAutomaticOperationRecoveryService; use App\Services\Cloud\CloudRuntimeHeartbeat; use App\Services\Cloud\CloudWorkerService; @@ -30,24 +30,24 @@ final class CloudWorker ); try { $worker = $notificationOnly - ? new CloudMaintenanceNoticeDeliveryService() + ? new CloudNotificationDeliveryService() : ($recoveryOnly ? new CloudAutomaticOperationRecoveryService() : new CloudWorkerService(null, $policyOnly, $vpcBatchOnly)); do { $errors = []; $heartbeat->busy( - $notificationOnly ? 'maintenance_notification' + $notificationOnly ? 'notification' : ($recoveryOnly ? 'recovery_tick' : 'worker_tick') ); try { $worked = $worker->tick(); - if (!$recoveryOnly && !$notificationOnly) { + if (!$recoveryOnly) { $errors = $worker->observationErrors(); } } catch (\Throwable $error) { $code = $notificationOnly - ? 'maintenance_notification_failed' + ? 'notification_failed' : ($recoveryOnly ? 'recovery_tick_failed' : 'worker_tick_failed'); $errors[] = $code; @@ -77,7 +77,7 @@ final class CloudWorker } while (!$once); } catch (\Throwable $error) { $heartbeat->error( - $notificationOnly ? 'maintenance_notification_failed' + $notificationOnly ? 'notification_failed' : ($recoveryOnly ? 'recovery_tick_failed' : 'worker_tick_failed') ); diff --git a/web/app/Services/Cloud/CloudNotificationDeliveryService.php b/web/app/Services/Cloud/CloudNotificationDeliveryService.php new file mode 100644 index 00000000..c573d727 --- /dev/null +++ b/web/app/Services/Cloud/CloudNotificationDeliveryService.php @@ -0,0 +1,64 @@ +maintenanceTick = \Closure::fromCallable($maintenanceTick); + $this->compensationTick = \Closure::fromCallable($compensationTick); + } + + public function tick(): bool + { + $this->errors = []; + $channels = $this->maintenanceFirst + ? [ + ['maintenance_notification_failed', $this->maintenanceTick], + ['notification_failed', $this->compensationTick], + ] + : [ + ['notification_failed', $this->compensationTick], + ['maintenance_notification_failed', $this->maintenanceTick], + ]; + $this->maintenanceFirst = !$this->maintenanceFirst; + + $worked = false; + foreach ($channels as [$errorCode, $tick]) { + try { + $worked = $tick() === true || $worked; + } catch (\Throwable $error) { + $this->errors[] = $errorCode; + fwrite( + STDERR, + 'cloud notification channel temporarily unavailable: ' + . $errorCode . PHP_EOL + ); + } + } + return $worked; + } + + /** @return list */ + public function observationErrors(): array + { + return $this->errors; + } +} diff --git a/web/tests/CloudNotificationDeliveryServiceTest.php b/web/tests/CloudNotificationDeliveryServiceTest.php new file mode 100644 index 00000000..473196b2 --- /dev/null +++ b/web/tests/CloudNotificationDeliveryServiceTest.php @@ -0,0 +1,71 @@ +tick() === true + && $order === ['maintenance', 'compensation'] + && $maintenanceCalls === 1 && $compensationCalls === 1 + && $service->observationErrors() === [], + 'First notification pass was not bounded or did not report work.'); +$order = []; +$assert($service->tick() === true + && $order === ['compensation', 'maintenance'] + && $maintenanceCalls === 2 && $compensationCalls === 2, + 'Notification channel priority did not alternate fairly.'); + +$order = []; +$isolated = new CloudNotificationDeliveryService( + static function () use (&$order): bool { + $order[] = 'maintenance'; + throw new RuntimeException('expected maintenance failure'); + }, + static function () use (&$order): bool { + $order[] = 'compensation'; + return true; + } +); +$assert($isolated->tick() === true + && $order === ['maintenance', 'compensation'] + && $isolated->observationErrors() === [ + 'maintenance_notification_failed', + ], 'A maintenance failure blocked compensation delivery or was hidden.'); + +$bothFailed = new CloudNotificationDeliveryService( + static function (): bool { + throw new RuntimeException('expected maintenance failure'); + }, + static function (): bool { + throw new RuntimeException('expected compensation failure'); + } +); +$assert($bothFailed->tick() === false + && $bothFailed->observationErrors() === [ + 'maintenance_notification_failed', + 'notification_failed', + ], 'Multiple notification channel failures were not retained.'); + +echo "Cloud notification delivery service test passed.\n"; diff --git a/web/tests/CloudRuntimeHeartbeatTest.php b/web/tests/CloudRuntimeHeartbeatTest.php index 85d8419d..00e20665 100644 --- a/web/tests/CloudRuntimeHeartbeatTest.php +++ b/web/tests/CloudRuntimeHeartbeatTest.php @@ -72,11 +72,11 @@ try { && $checks($directory)['recovery']['active_count'] === 1 && $checks($directory)['notification']['active_count'] === 1, 'Starting workers or multiple policy reporters were not represented.'); - $notification->busy('maintenance_notification'); + $notification->busy('notification'); $assert(str_contains( $checks($directory)['notification']['message'], - '维护邮件投递' - ), 'Maintenance delivery did not receive an isolated runtime phase.'); + '邮件通知' + ), 'Customer notification delivery did not receive an isolated runtime phase.'); $normal->busy('worker_tick'); $assert(str_contains($checks($directory)['worker']['message'], '任务消费或核对'), @@ -520,11 +520,13 @@ try { 'The dedicated recovery worker is missing, sends mail, or is not deployable.'); $assert(str_contains($workerCommand, "'--notification-only'") && str_contains($workerCommand, - 'CloudMaintenanceNoticeDeliveryService') + 'CloudNotificationDeliveryService') && str_contains($xcat, '--notification-only') && str_contains($notificationUnit, - 'cloudWorker --notification-only'), - 'The isolated maintenance notification worker is not deployable.'); + 'cloudWorker --notification-only') + && str_contains($notificationUnit, + 'customer notification worker'), + 'The isolated customer notification worker is not deployable.'); Capsule::table('cloud_instances')->insert(['id' => 10, 'status' => 'active', 'service_status' => 'active', 'expires_at' => time() + 3600,