You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
246 lines
9.9 KiB
PHP
246 lines
9.9 KiB
PHP
<?php
|
|
|
|
namespace App\Services\Cloud;
|
|
|
|
use App\Models\CloudInstance;
|
|
use App\Models\CloudInstanceRuntime;
|
|
|
|
/**
|
|
* Refresh the instance runtime observation cache from host batch state.
|
|
*
|
|
* This service is intentionally separate from page rendering and mutation
|
|
* endpoints. A failed host request only marks its observations unavailable;
|
|
* it never changes the last known power state to "stopped".
|
|
*/
|
|
final class CloudInstanceRuntimeSyncService
|
|
{
|
|
private const ENGINE_TIMEOUT_SECONDS = 3.0;
|
|
|
|
public function __construct(private ?CloudEngineClient $engine = null)
|
|
{
|
|
$this->engine ??= new CloudEngineClient(null, self::ENGINE_TIMEOUT_SECONDS);
|
|
}
|
|
|
|
/**
|
|
* Poll active instances in host batches. The returned counters contain
|
|
* no engine-controlled text and are suitable for heartbeat diagnostics.
|
|
*/
|
|
public function sync(int $limit = 500): array
|
|
{
|
|
$result = ['observed' => 0, 'unavailable' => 0, 'skipped' => 0, 'errors' => []];
|
|
if (!$this->runtimeTableAvailable()) {
|
|
// Rolling upgrades may start the daemon before cloudInit has run.
|
|
// Keep billing/lifecycle work healthy until the table is present.
|
|
$result['skipped'] = 1;
|
|
return $result;
|
|
}
|
|
|
|
$actor = CloudDaemonActor::resolve('cloud-runtime-sync');
|
|
if ($actor === null) {
|
|
$result['errors'][] = 'runtime_status_sync_unconfigured';
|
|
return $result;
|
|
}
|
|
|
|
try {
|
|
// Select the least recently attempted observations first. This
|
|
// gives a bounded daemon pass a rotating cursor without storing a
|
|
// second global cursor: every successful or failed host batch
|
|
// advances last_attempt_at, so high instance IDs cannot starve
|
|
// behind a permanently busy low-ID prefix.
|
|
$instanceQuery = CloudInstance::leftJoin(
|
|
'cloud_instance_runtime as runtime',
|
|
'runtime.instance_id', '=', 'cloud_instances.id'
|
|
)->where('cloud_instances.status', '<>', 'deleted')
|
|
->orderByRaw('COALESCE(runtime.last_attempt_at, runtime.last_success_at, 0) ASC')
|
|
->orderBy('cloud_instances.id')
|
|
->limit(max(1, min(2000, $limit)));
|
|
// Production uses MariaDB, where filtering malformed legacy UUIDs
|
|
// at the database boundary prevents them from occupying every
|
|
// bounded pass. Keep the query usable in SQLite-based contract
|
|
// tests and local diagnostics; the application-level validation
|
|
// below remains the final guard in either driver.
|
|
if ((new CloudInstance())->getConnection()->getDriverName() === 'mysql') {
|
|
$instanceQuery->whereRaw(
|
|
"cloud_instances.uuid REGEXP ?",
|
|
['^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$']
|
|
);
|
|
}
|
|
$instances = $instanceQuery->get([
|
|
'cloud_instances.id', 'cloud_instances.host_id', 'cloud_instances.uuid',
|
|
]);
|
|
} catch (\Throwable $error) {
|
|
$result['errors'][] = 'runtime_status_sync_failed';
|
|
return $result;
|
|
}
|
|
if ($instances->isEmpty()) {
|
|
return $result;
|
|
}
|
|
|
|
$groups = [];
|
|
foreach ($instances as $instance) {
|
|
$hostId = (int) $instance->host_id;
|
|
$uuid = strtolower(trim((string) $instance->uuid));
|
|
if ($hostId < 1 || preg_match(
|
|
'/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/D',
|
|
$uuid
|
|
) !== 1) {
|
|
$result['skipped']++;
|
|
continue;
|
|
}
|
|
$groups[$hostId][] = [
|
|
'id' => (int) $instance->id,
|
|
'host_id' => $hostId,
|
|
'uuid' => $uuid,
|
|
];
|
|
// Legacy instances may predate the independent cache. Seeding a
|
|
// row does not claim a power state; the batch response still must
|
|
// provide the first successful observation.
|
|
try {
|
|
CloudInstanceRuntimeService::seed(
|
|
(int) $instance->id, $hostId, $uuid
|
|
);
|
|
} catch (\Throwable $error) {
|
|
$result['skipped']++;
|
|
// Do not leave a row that failed identity seeding in the
|
|
// host group; otherwise syncHost() would claim it again and
|
|
// count the same failure twice in one pass.
|
|
array_pop($groups[$hostId]);
|
|
}
|
|
}
|
|
|
|
foreach ($groups as $hostId => $group) {
|
|
$this->syncHost((int) $hostId, $group, $actor, $result);
|
|
}
|
|
return $result;
|
|
}
|
|
|
|
private function syncHost(int $hostId, array $group, array $actor, array &$result): void
|
|
{
|
|
// Claim every row immediately before the host request. A successful
|
|
// worker operation advances the same revision, making this batch
|
|
// stale without allowing its eventual response to overwrite the
|
|
// worker's fresher state.
|
|
$active = [];
|
|
$revisions = [];
|
|
foreach ($group as $row) {
|
|
try {
|
|
$revisions[(int) $row['id']] = CloudInstanceRuntimeService::beginObservation(
|
|
(int) $row['id'], $hostId, (string) $row['uuid']
|
|
);
|
|
$active[] = $row;
|
|
} catch (\Throwable $error) {
|
|
$result['skipped']++;
|
|
}
|
|
}
|
|
if ($active === []) {
|
|
return;
|
|
}
|
|
// Keep the host request bounded and pass the explicit UUID set to the
|
|
// engine. The old no-argument vm-states endpoint enumerated every
|
|
// domain and is reserved for legacy private UI compatibility only.
|
|
foreach (array_chunk($active, 100) as $chunk) {
|
|
$allowed = [];
|
|
foreach ($chunk as $row) {
|
|
$allowed[(string) $row['uuid']] = $row;
|
|
}
|
|
$query = rawurlencode(implode(',', array_keys($allowed)));
|
|
$chunkIds = array_map(
|
|
static fn (array $row): int => (int) $row['id'], $chunk
|
|
);
|
|
try {
|
|
$payload = $this->engine->host($hostId, 'GET',
|
|
'vm-states?ids=' . $query, $actor);
|
|
} catch (CloudError $error) {
|
|
$chunkRevisions = [];
|
|
foreach ($chunkIds as $id) {
|
|
if (array_key_exists($id, $revisions)) {
|
|
$chunkRevisions[$id] = $revisions[$id];
|
|
}
|
|
}
|
|
CloudInstanceRuntimeService::markUnavailable(
|
|
$chunkIds, $hostId, time(), self::errorCode($error), $chunkRevisions
|
|
);
|
|
$result['unavailable'] += count($chunkIds);
|
|
$result['errors'][] = 'runtime_status_sync_failed';
|
|
continue;
|
|
} catch (\Throwable $error) {
|
|
$chunkRevisions = [];
|
|
foreach ($chunkIds as $id) {
|
|
if (array_key_exists($id, $revisions)) {
|
|
$chunkRevisions[$id] = $revisions[$id];
|
|
}
|
|
}
|
|
CloudInstanceRuntimeService::markUnavailable(
|
|
$chunkIds, $hostId, time(), 'host_unavailable', $chunkRevisions
|
|
);
|
|
$result['unavailable'] += count($chunkIds);
|
|
$result['errors'][] = 'runtime_status_sync_failed';
|
|
continue;
|
|
}
|
|
|
|
$seen = [];
|
|
$items = is_array($payload['items'] ?? null) ? $payload['items'] : [];
|
|
foreach ($items as $item) {
|
|
if (!is_array($item) || !is_string($item['uuid'] ?? null)) {
|
|
continue;
|
|
}
|
|
$uuid = strtolower(trim($item['uuid']));
|
|
if (!isset($allowed[$uuid])) {
|
|
continue;
|
|
}
|
|
$row = $allowed[$uuid];
|
|
$state = $item['state'] ?? ($item['power_state'] ?? null);
|
|
$seen[$uuid] = true;
|
|
try {
|
|
CloudInstanceRuntimeService::markObserved(
|
|
(int) $row['id'], $hostId, $uuid, $state, null,
|
|
$revisions[(int) $row['id']] ?? null
|
|
);
|
|
if (CloudInstanceRuntimeService::normalizeState($state) !== 'unknown') {
|
|
$result['observed']++;
|
|
} else {
|
|
$result['unavailable']++;
|
|
}
|
|
} catch (\Throwable $error) {
|
|
$result['skipped']++;
|
|
}
|
|
}
|
|
|
|
$missing = [];
|
|
$missingRevisions = [];
|
|
foreach ($allowed as $uuid => $row) {
|
|
if (!isset($seen[$uuid])) {
|
|
$missing[] = (int) $row['id'];
|
|
$id = (int) $row['id'];
|
|
if (array_key_exists($id, $revisions)) {
|
|
$missingRevisions[$id] = $revisions[$id];
|
|
}
|
|
}
|
|
}
|
|
if ($missing !== []) {
|
|
CloudInstanceRuntimeService::markUnavailable(
|
|
$missing, $hostId, time(), 'invalid_response', $missingRevisions
|
|
);
|
|
$result['unavailable'] += count($missing);
|
|
$result['errors'][] = 'runtime_status_sync_failed';
|
|
}
|
|
}
|
|
}
|
|
|
|
private function runtimeTableAvailable(): bool
|
|
{
|
|
try {
|
|
return (new CloudInstanceRuntime())->getConnection()
|
|
->getSchemaBuilder()->hasTable('cloud_instance_runtime');
|
|
} catch (\Throwable $error) {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
private static function errorCode(CloudError $error): string
|
|
{
|
|
return in_array($error->httpStatus, [408, 429, 502, 503, 504], true)
|
|
? 'host_unavailable' : 'engine_error';
|
|
}
|
|
}
|