|
|
|
|
<?php
|
|
|
|
|
namespace App\Command;
|
|
|
|
|
|
|
|
|
|
use App\Services\Cloud\CloudAgentReadiness;
|
|
|
|
|
use App\Services\Cloud\CloudInvoiceService;
|
|
|
|
|
use App\Services\Cloud\CloudRuntimeHeartbeat;
|
|
|
|
|
use App\Services\Cloud\CloudWorkerService;
|
|
|
|
|
|
|
|
|
|
final class CloudWorker
|
|
|
|
|
{
|
|
|
|
|
public static function run(array $argv = [], ?CloudRuntimeHeartbeat $heartbeat = null,
|
|
|
|
|
?callable $engineReadiness = null): int
|
|
|
|
|
{
|
|
|
|
|
$once = in_array('--once', $argv, true);
|
|
|
|
|
$policyOnly = in_array('--policy-only', $argv, true);
|
|
|
|
|
$heartbeat ??= CloudRuntimeHeartbeat::start($policyOnly ? 'policy' : 'normal', $once);
|
|
|
|
|
$engineReadiness ??= static fn (): bool => CloudAgentReadiness::wait();
|
|
|
|
|
try {
|
|
|
|
|
// Do not claim a durable VM job while the enrolled Go Agent
|
|
|
|
|
// control plane is not ready. The service manager will retry this
|
|
|
|
|
// daemon if the bounded liveness window expires.
|
|
|
|
|
if (!$once) {
|
|
|
|
|
$heartbeat->busy('engine_readiness');
|
|
|
|
|
if (!$engineReadiness()) {
|
|
|
|
|
$heartbeat->error('engine_readiness_timeout');
|
|
|
|
|
fwrite(STDERR, "cloud worker Go Agent readiness timed out\n");
|
|
|
|
|
if (!$policyOnly) {
|
|
|
|
|
self::deliverNotifications($heartbeat);
|
|
|
|
|
// Notification telemetry must not hide why this VM
|
|
|
|
|
// worker is exiting for systemd to retry.
|
|
|
|
|
$heartbeat->error('engine_readiness_timeout');
|
|
|
|
|
}
|
|
|
|
|
return 1;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
$worker = new CloudWorkerService(null, $policyOnly);
|
|
|
|
|
do {
|
|
|
|
|
$errors = [];
|
|
|
|
|
$heartbeat->busy('worker_tick');
|
|
|
|
|
try {
|
|
|
|
|
$worked = $worker->tick();
|
|
|
|
|
$errors = $worker->observationErrors();
|
|
|
|
|
} catch (\Throwable $error) {
|
|
|
|
|
$errors[] = 'worker_tick_failed';
|
|
|
|
|
$heartbeat->error('worker_tick_failed');
|
|
|
|
|
fwrite(STDERR, 'cloud worker temporarily unavailable: ' . get_class($error) . PHP_EOL);
|
|
|
|
|
if ($once) {
|
|
|
|
|
return 1;
|
|
|
|
|
}
|
|
|
|
|
$worked = false;
|
|
|
|
|
}
|
|
|
|
|
if ($errors !== []) {
|
|
|
|
|
$heartbeat->error($errors[0]);
|
|
|
|
|
}
|
|
|
|
|
// SMTP remains isolated from lifecycle and policy workers.
|
|
|
|
|
if (!$policyOnly) {
|
|
|
|
|
$errors = array_merge($errors, self::deliverNotifications($heartbeat));
|
|
|
|
|
}
|
|
|
|
|
if ($errors === []) {
|
|
|
|
|
$heartbeat->success();
|
|
|
|
|
} else {
|
|
|
|
|
$heartbeat->error($errors[0]);
|
|
|
|
|
}
|
|
|
|
|
if (!$once && !$worked) {
|
|
|
|
|
usleep(1000000);
|
|
|
|
|
}
|
|
|
|
|
} while (!$once);
|
|
|
|
|
} catch (\Throwable $error) {
|
|
|
|
|
$heartbeat->error('worker_tick_failed');
|
|
|
|
|
throw $error;
|
|
|
|
|
} finally {
|
|
|
|
|
$heartbeat->stopped();
|
|
|
|
|
}
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static function deliverNotifications(CloudRuntimeHeartbeat $heartbeat): array
|
|
|
|
|
{
|
|
|
|
|
$heartbeat->busy('notification');
|
|
|
|
|
try {
|
|
|
|
|
CloudInvoiceService::deliverReminders(1);
|
|
|
|
|
return CloudInvoiceService::observationErrors('notifications');
|
|
|
|
|
} catch (\Throwable $error) {
|
|
|
|
|
fwrite(STDERR, 'cloud notification mail temporarily unavailable: '
|
|
|
|
|
. get_class($error) . PHP_EOL);
|
|
|
|
|
return ['notification_failed'];
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|