You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
119 lines
4.8 KiB
PHP
119 lines
4.8 KiB
PHP
<?php
|
|
namespace App\Command;
|
|
|
|
use App\Services\Cloud\CloudInvoiceService;
|
|
use App\Services\Cloud\CloudAutomaticOperationRecoveryService;
|
|
use App\Services\Cloud\CloudMigrationBundleService;
|
|
use App\Services\Cloud\CloudObjectStorageTransferError;
|
|
use App\Services\Cloud\CloudRuntimeHeartbeat;
|
|
use App\Services\Cloud\CloudWorkerService;
|
|
|
|
final class CloudWorker
|
|
{
|
|
private const CLEANUP_IDLE_SECONDS = 30;
|
|
private const CLEANUP_ERROR_SECONDS = 60;
|
|
|
|
public static function run(array $argv = [], ?CloudRuntimeHeartbeat $heartbeat = null): int
|
|
{
|
|
$once = in_array('--once', $argv, true);
|
|
$policyOnly = in_array('--policy-only', $argv, true);
|
|
$recoveryOnly = in_array('--recovery-only', $argv, true);
|
|
if ($policyOnly && $recoveryOnly) {
|
|
throw new \InvalidArgumentException(
|
|
'--policy-only 与 --recovery-only 不能同时使用。'
|
|
);
|
|
}
|
|
$heartbeat ??= CloudRuntimeHeartbeat::start(
|
|
$recoveryOnly ? 'recovery' : ($policyOnly ? 'policy' : 'normal'),
|
|
$once
|
|
);
|
|
try {
|
|
$worker = $recoveryOnly
|
|
? new CloudAutomaticOperationRecoveryService()
|
|
: new CloudWorkerService(null, $policyOnly);
|
|
$nextMigrationBundleCleanupAt = 0;
|
|
do {
|
|
$errors = [];
|
|
$heartbeat->busy(
|
|
$recoveryOnly ? 'recovery_tick' : 'worker_tick'
|
|
);
|
|
try {
|
|
$worked = $worker->tick();
|
|
if (!$recoveryOnly) {
|
|
$errors = $worker->observationErrors();
|
|
}
|
|
} catch (\Throwable $error) {
|
|
$code = $recoveryOnly
|
|
? 'recovery_tick_failed' : 'worker_tick_failed';
|
|
$errors[] = $code;
|
|
$heartbeat->error($code);
|
|
fwrite(STDERR, 'cloud worker temporarily unavailable: ' . get_class($error) . PHP_EOL);
|
|
if ($once) {
|
|
return 1;
|
|
}
|
|
$worked = false;
|
|
}
|
|
$cleanupNow = time();
|
|
if (!$policyOnly && !$recoveryOnly
|
|
&& $cleanupNow >= $nextMigrationBundleCleanupAt) {
|
|
$heartbeat->busy('migration_bundle_cleanup');
|
|
try {
|
|
$cleaned = CloudMigrationBundleService::cleanupOneEligible();
|
|
$worked = $cleaned || $worked;
|
|
$nextMigrationBundleCleanupAt = $cleaned
|
|
? 0 : $cleanupNow + self::CLEANUP_IDLE_SECONDS;
|
|
} catch (\Throwable $error) {
|
|
$nextMigrationBundleCleanupAt = $cleanupNow
|
|
+ self::CLEANUP_ERROR_SECONDS;
|
|
$errors[] = 'migration_bundle_cleanup_failed';
|
|
$heartbeat->error('migration_bundle_cleanup_failed');
|
|
$detail = $error instanceof CloudObjectStorageTransferError
|
|
? ':' . $error->publicCode : '';
|
|
fwrite(STDERR,
|
|
'migration bundle cleanup temporarily unavailable: '
|
|
. get_class($error) . $detail . PHP_EOL
|
|
);
|
|
}
|
|
}
|
|
if ($errors !== []) {
|
|
$heartbeat->error($errors[0]);
|
|
}
|
|
// SMTP remains isolated from lifecycle and policy workers.
|
|
if (!$policyOnly && !$recoveryOnly) {
|
|
$errors = array_merge($errors, self::deliverNotifications($heartbeat));
|
|
}
|
|
if ($errors === []) {
|
|
$heartbeat->success();
|
|
} else {
|
|
$heartbeat->error($errors[0]);
|
|
}
|
|
if (!$once && !$worked) {
|
|
usleep(1000000);
|
|
}
|
|
} while (!$once);
|
|
} catch (\Throwable $error) {
|
|
$heartbeat->error(
|
|
$recoveryOnly
|
|
? 'recovery_tick_failed' : 'worker_tick_failed'
|
|
);
|
|
throw $error;
|
|
} finally {
|
|
$heartbeat->stopped();
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
private static function deliverNotifications(CloudRuntimeHeartbeat $heartbeat): array
|
|
{
|
|
$heartbeat->busy('notification');
|
|
try {
|
|
CloudInvoiceService::deliverReminders(1);
|
|
return CloudInvoiceService::observationErrors('notifications');
|
|
} catch (\Throwable $error) {
|
|
fwrite(STDERR, 'cloud notification mail temporarily unavailable: '
|
|
. get_class($error) . PHP_EOL);
|
|
return ['notification_failed'];
|
|
}
|
|
}
|
|
}
|