feat(cloud): deliver service compensation notifications

go
codex 2 weeks ago
parent 50e054ab86
commit f628d8bd7b

@ -1,5 +1,5 @@
[Unit]
Description=Virt Cloud maintenance notification worker
Description=Virt Cloud customer notification worker
After=network-online.target
Wants=network-online.target

@ -2,7 +2,7 @@
namespace App\Command;
use App\Services\Cloud\CloudInvoiceService;
use App\Services\Cloud\CloudMaintenanceNoticeDeliveryService;
use App\Services\Cloud\CloudNotificationDeliveryService;
use App\Services\Cloud\CloudAutomaticOperationRecoveryService;
use App\Services\Cloud\CloudRuntimeHeartbeat;
use App\Services\Cloud\CloudWorkerService;
@ -30,24 +30,24 @@ final class CloudWorker
);
try {
$worker = $notificationOnly
? new CloudMaintenanceNoticeDeliveryService()
? new CloudNotificationDeliveryService()
: ($recoveryOnly
? new CloudAutomaticOperationRecoveryService()
: new CloudWorkerService(null, $policyOnly, $vpcBatchOnly));
do {
$errors = [];
$heartbeat->busy(
$notificationOnly ? 'maintenance_notification'
$notificationOnly ? 'notification'
: ($recoveryOnly ? 'recovery_tick' : 'worker_tick')
);
try {
$worked = $worker->tick();
if (!$recoveryOnly && !$notificationOnly) {
if (!$recoveryOnly) {
$errors = $worker->observationErrors();
}
} catch (\Throwable $error) {
$code = $notificationOnly
? 'maintenance_notification_failed'
? 'notification_failed'
: ($recoveryOnly
? 'recovery_tick_failed' : 'worker_tick_failed');
$errors[] = $code;
@ -77,7 +77,7 @@ final class CloudWorker
} while (!$once);
} catch (\Throwable $error) {
$heartbeat->error(
$notificationOnly ? 'maintenance_notification_failed'
$notificationOnly ? 'notification_failed'
: ($recoveryOnly
? 'recovery_tick_failed' : 'worker_tick_failed')
);

@ -0,0 +1,64 @@
<?php
namespace App\Services\Cloud;
/** Fair, failure-isolated dispatcher for customer notification outboxes. */
final class CloudNotificationDeliveryService
{
private readonly \Closure $maintenanceTick;
private readonly \Closure $compensationTick;
private bool $maintenanceFirst = true;
private array $errors = [];
public function __construct(
?callable $maintenanceTick = null,
?callable $compensationTick = null
) {
if ($maintenanceTick === null) {
$maintenance = new CloudMaintenanceNoticeDeliveryService();
$maintenanceTick = [$maintenance, 'tick'];
}
if ($compensationTick === null) {
$compensation = new CloudServiceCompensationNotificationService();
$compensationTick = [$compensation, 'tick'];
}
$this->maintenanceTick = \Closure::fromCallable($maintenanceTick);
$this->compensationTick = \Closure::fromCallable($compensationTick);
}
public function tick(): bool
{
$this->errors = [];
$channels = $this->maintenanceFirst
? [
['maintenance_notification_failed', $this->maintenanceTick],
['notification_failed', $this->compensationTick],
]
: [
['notification_failed', $this->compensationTick],
['maintenance_notification_failed', $this->maintenanceTick],
];
$this->maintenanceFirst = !$this->maintenanceFirst;
$worked = false;
foreach ($channels as [$errorCode, $tick]) {
try {
$worked = $tick() === true || $worked;
} catch (\Throwable $error) {
$this->errors[] = $errorCode;
fwrite(
STDERR,
'cloud notification channel temporarily unavailable: '
. $errorCode . PHP_EOL
);
}
}
return $worked;
}
/** @return list<string> */
public function observationErrors(): array
{
return $this->errors;
}
}

@ -0,0 +1,71 @@
<?php
declare(strict_types=1);
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Services\Cloud\CloudNotificationDeliveryService;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$order = [];
$maintenanceCalls = 0;
$compensationCalls = 0;
$service = new CloudNotificationDeliveryService(
static function () use (&$order, &$maintenanceCalls): bool {
$order[] = 'maintenance';
$maintenanceCalls++;
return false;
},
static function () use (&$order, &$compensationCalls): bool {
$order[] = 'compensation';
$compensationCalls++;
return true;
}
);
$assert($service->tick() === true
&& $order === ['maintenance', 'compensation']
&& $maintenanceCalls === 1 && $compensationCalls === 1
&& $service->observationErrors() === [],
'First notification pass was not bounded or did not report work.');
$order = [];
$assert($service->tick() === true
&& $order === ['compensation', 'maintenance']
&& $maintenanceCalls === 2 && $compensationCalls === 2,
'Notification channel priority did not alternate fairly.');
$order = [];
$isolated = new CloudNotificationDeliveryService(
static function () use (&$order): bool {
$order[] = 'maintenance';
throw new RuntimeException('expected maintenance failure');
},
static function () use (&$order): bool {
$order[] = 'compensation';
return true;
}
);
$assert($isolated->tick() === true
&& $order === ['maintenance', 'compensation']
&& $isolated->observationErrors() === [
'maintenance_notification_failed',
], 'A maintenance failure blocked compensation delivery or was hidden.');
$bothFailed = new CloudNotificationDeliveryService(
static function (): bool {
throw new RuntimeException('expected maintenance failure');
},
static function (): bool {
throw new RuntimeException('expected compensation failure');
}
);
$assert($bothFailed->tick() === false
&& $bothFailed->observationErrors() === [
'maintenance_notification_failed',
'notification_failed',
], 'Multiple notification channel failures were not retained.');
echo "Cloud notification delivery service test passed.\n";

@ -72,11 +72,11 @@ try {
&& $checks($directory)['recovery']['active_count'] === 1
&& $checks($directory)['notification']['active_count'] === 1,
'Starting workers or multiple policy reporters were not represented.');
$notification->busy('maintenance_notification');
$notification->busy('notification');
$assert(str_contains(
$checks($directory)['notification']['message'],
'维护邮件投递'
), 'Maintenance delivery did not receive an isolated runtime phase.');
'邮件通知'
), 'Customer notification delivery did not receive an isolated runtime phase.');
$normal->busy('worker_tick');
$assert(str_contains($checks($directory)['worker']['message'], '任务消费或核对'),
@ -520,11 +520,13 @@ try {
'The dedicated recovery worker is missing, sends mail, or is not deployable.');
$assert(str_contains($workerCommand, "'--notification-only'")
&& str_contains($workerCommand,
'CloudMaintenanceNoticeDeliveryService')
'CloudNotificationDeliveryService')
&& str_contains($xcat, '--notification-only')
&& str_contains($notificationUnit,
'cloudWorker --notification-only'),
'The isolated maintenance notification worker is not deployable.');
'cloudWorker --notification-only')
&& str_contains($notificationUnit,
'customer notification worker'),
'The isolated customer notification worker is not deployable.');
Capsule::table('cloud_instances')->insert(['id' => 10, 'status' => 'active',
'service_status' => 'active', 'expires_at' => time() + 3600,

Loading…
Cancel
Save