You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/app/Command/CloudWorker.php

91 lines
3.6 KiB
PHP

<?php
namespace App\Command;
use App\Services\Cloud\CloudEngineReadiness;
use App\Services\Cloud\CloudInvoiceService;
use App\Services\Cloud\CloudRuntimeHeartbeat;
use App\Services\Cloud\CloudWorkerService;
final class CloudWorker
{
public static function run(array $argv = [], ?CloudRuntimeHeartbeat $heartbeat = null,
?callable $engineReadiness = null): int
{
$once = in_array('--once', $argv, true);
$policyOnly = in_array('--policy-only', $argv, true);
$heartbeat ??= CloudRuntimeHeartbeat::start($policyOnly ? 'policy' : 'normal', $once);
$engineReadiness ??= static fn (): bool => CloudEngineReadiness::wait();
try {
// Do not claim a durable VM job during the brief interval where
// systemd reports the Type=exec engine active but its signed API
// is not ready. The service manager will retry this daemon if the
// bounded readiness window expires.
if (!$once) {
$heartbeat->busy('engine_readiness');
if (!$engineReadiness()) {
$heartbeat->error('engine_readiness_timeout');
fwrite(STDERR, "cloud worker engine readiness timed out\n");
if (!$policyOnly) {
self::deliverNotifications($heartbeat);
// Notification telemetry must not hide why this VM
// worker is exiting for systemd to retry.
$heartbeat->error('engine_readiness_timeout');
}
return 1;
}
}
$worker = new CloudWorkerService(null, $policyOnly);
do {
$errors = [];
$heartbeat->busy('worker_tick');
try {
$worked = $worker->tick();
$errors = $worker->observationErrors();
} catch (\Throwable $error) {
$errors[] = 'worker_tick_failed';
$heartbeat->error('worker_tick_failed');
fwrite(STDERR, 'cloud worker temporarily unavailable: ' . get_class($error) . PHP_EOL);
if ($once) {
return 1;
}
$worked = false;
}
if ($errors !== []) {
$heartbeat->error($errors[0]);
}
// SMTP remains isolated from lifecycle and policy workers.
if (!$policyOnly) {
$errors = array_merge($errors, self::deliverNotifications($heartbeat));
}
if ($errors === []) {
$heartbeat->success();
} else {
$heartbeat->error($errors[0]);
}
if (!$once && !$worked) {
usleep(1000000);
}
} while (!$once);
} catch (\Throwable $error) {
$heartbeat->error('worker_tick_failed');
throw $error;
} finally {
$heartbeat->stopped();
}
return 0;
}
private static function deliverNotifications(CloudRuntimeHeartbeat $heartbeat): array
{
$heartbeat->busy('notification');
try {
CloudInvoiceService::deliverReminders(1);
return CloudInvoiceService::observationErrors('notifications');
} catch (\Throwable $error) {
fwrite(STDERR, 'cloud notification mail temporarily unavailable: '
. get_class($error) . PHP_EOL);
return ['notification_failed'];
}
}
}