You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
91 lines
3.6 KiB
PHP
91 lines
3.6 KiB
PHP
<?php
|
|
namespace App\Command;
|
|
|
|
use App\Services\Cloud\CloudEngineReadiness;
|
|
use App\Services\Cloud\CloudInvoiceService;
|
|
use App\Services\Cloud\CloudRuntimeHeartbeat;
|
|
use App\Services\Cloud\CloudWorkerService;
|
|
|
|
final class CloudWorker
|
|
{
|
|
public static function run(array $argv = [], ?CloudRuntimeHeartbeat $heartbeat = null,
|
|
?callable $engineReadiness = null): int
|
|
{
|
|
$once = in_array('--once', $argv, true);
|
|
$policyOnly = in_array('--policy-only', $argv, true);
|
|
$heartbeat ??= CloudRuntimeHeartbeat::start($policyOnly ? 'policy' : 'normal', $once);
|
|
$engineReadiness ??= static fn (): bool => CloudEngineReadiness::wait();
|
|
try {
|
|
// Do not claim a durable VM job during the brief interval where
|
|
// systemd reports the Type=exec engine active but its signed API
|
|
// is not ready. The service manager will retry this daemon if the
|
|
// bounded readiness window expires.
|
|
if (!$once) {
|
|
$heartbeat->busy('engine_readiness');
|
|
if (!$engineReadiness()) {
|
|
$heartbeat->error('engine_readiness_timeout');
|
|
fwrite(STDERR, "cloud worker engine readiness timed out\n");
|
|
if (!$policyOnly) {
|
|
self::deliverNotifications($heartbeat);
|
|
// Notification telemetry must not hide why this VM
|
|
// worker is exiting for systemd to retry.
|
|
$heartbeat->error('engine_readiness_timeout');
|
|
}
|
|
return 1;
|
|
}
|
|
}
|
|
$worker = new CloudWorkerService(null, $policyOnly);
|
|
do {
|
|
$errors = [];
|
|
$heartbeat->busy('worker_tick');
|
|
try {
|
|
$worked = $worker->tick();
|
|
$errors = $worker->observationErrors();
|
|
} catch (\Throwable $error) {
|
|
$errors[] = 'worker_tick_failed';
|
|
$heartbeat->error('worker_tick_failed');
|
|
fwrite(STDERR, 'cloud worker temporarily unavailable: ' . get_class($error) . PHP_EOL);
|
|
if ($once) {
|
|
return 1;
|
|
}
|
|
$worked = false;
|
|
}
|
|
if ($errors !== []) {
|
|
$heartbeat->error($errors[0]);
|
|
}
|
|
// SMTP remains isolated from lifecycle and policy workers.
|
|
if (!$policyOnly) {
|
|
$errors = array_merge($errors, self::deliverNotifications($heartbeat));
|
|
}
|
|
if ($errors === []) {
|
|
$heartbeat->success();
|
|
} else {
|
|
$heartbeat->error($errors[0]);
|
|
}
|
|
if (!$once && !$worked) {
|
|
usleep(1000000);
|
|
}
|
|
} while (!$once);
|
|
} catch (\Throwable $error) {
|
|
$heartbeat->error('worker_tick_failed');
|
|
throw $error;
|
|
} finally {
|
|
$heartbeat->stopped();
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
private static function deliverNotifications(CloudRuntimeHeartbeat $heartbeat): array
|
|
{
|
|
$heartbeat->busy('notification');
|
|
try {
|
|
CloudInvoiceService::deliverReminders(1);
|
|
return CloudInvoiceService::observationErrors('notifications');
|
|
} catch (\Throwable $error) {
|
|
fwrite(STDERR, 'cloud notification mail temporarily unavailable: '
|
|
. get_class($error) . PHP_EOL);
|
|
return ['notification_failed'];
|
|
}
|
|
}
|
|
}
|