You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/app/Services/Cloud/CloudInstanceRuntimeSyncSer...

246 lines
9.9 KiB
PHP

<?php
namespace App\Services\Cloud;
use App\Models\CloudInstance;
use App\Models\CloudInstanceRuntime;
/**
* Refresh the instance runtime observation cache from host batch state.
*
* This service is intentionally separate from page rendering and mutation
* endpoints. A failed host request only marks its observations unavailable;
* it never changes the last known power state to "stopped".
*/
final class CloudInstanceRuntimeSyncService
{
private const ENGINE_TIMEOUT_SECONDS = 3.0;
public function __construct(private ?CloudEngineClient $engine = null)
{
$this->engine ??= new CloudEngineClient(null, self::ENGINE_TIMEOUT_SECONDS);
}
/**
* Poll active instances in host batches. The returned counters contain
* no engine-controlled text and are suitable for heartbeat diagnostics.
*/
public function sync(int $limit = 500): array
{
$result = ['observed' => 0, 'unavailable' => 0, 'skipped' => 0, 'errors' => []];
if (!$this->runtimeTableAvailable()) {
// Rolling upgrades may start the daemon before cloudInit has run.
// Keep billing/lifecycle work healthy until the table is present.
$result['skipped'] = 1;
return $result;
}
$actor = CloudDaemonActor::resolve('cloud-runtime-sync');
if ($actor === null) {
$result['errors'][] = 'runtime_status_sync_unconfigured';
return $result;
}
try {
// Select the least recently attempted observations first. This
// gives a bounded daemon pass a rotating cursor without storing a
// second global cursor: every successful or failed host batch
// advances last_attempt_at, so high instance IDs cannot starve
// behind a permanently busy low-ID prefix.
$instanceQuery = CloudInstance::leftJoin(
'cloud_instance_runtime as runtime',
'runtime.instance_id', '=', 'cloud_instances.id'
)->where('cloud_instances.status', '<>', 'deleted')
->orderByRaw('COALESCE(runtime.last_attempt_at, runtime.last_success_at, 0) ASC')
->orderBy('cloud_instances.id')
->limit(max(1, min(2000, $limit)));
// Production uses MariaDB, where filtering malformed legacy UUIDs
// at the database boundary prevents them from occupying every
// bounded pass. Keep the query usable in SQLite-based contract
// tests and local diagnostics; the application-level validation
// below remains the final guard in either driver.
if ((new CloudInstance())->getConnection()->getDriverName() === 'mysql') {
$instanceQuery->whereRaw(
"cloud_instances.uuid REGEXP ?",
['^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$']
);
}
$instances = $instanceQuery->get([
'cloud_instances.id', 'cloud_instances.host_id', 'cloud_instances.uuid',
]);
} catch (\Throwable $error) {
$result['errors'][] = 'runtime_status_sync_failed';
return $result;
}
if ($instances->isEmpty()) {
return $result;
}
$groups = [];
foreach ($instances as $instance) {
$hostId = (int) $instance->host_id;
$uuid = strtolower(trim((string) $instance->uuid));
if ($hostId < 1 || preg_match(
'/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/D',
$uuid
) !== 1) {
$result['skipped']++;
continue;
}
$groups[$hostId][] = [
'id' => (int) $instance->id,
'host_id' => $hostId,
'uuid' => $uuid,
];
// Legacy instances may predate the independent cache. Seeding a
// row does not claim a power state; the batch response still must
// provide the first successful observation.
try {
CloudInstanceRuntimeService::seed(
(int) $instance->id, $hostId, $uuid
);
} catch (\Throwable $error) {
$result['skipped']++;
// Do not leave a row that failed identity seeding in the
// host group; otherwise syncHost() would claim it again and
// count the same failure twice in one pass.
array_pop($groups[$hostId]);
}
}
foreach ($groups as $hostId => $group) {
$this->syncHost((int) $hostId, $group, $actor, $result);
}
return $result;
}
private function syncHost(int $hostId, array $group, array $actor, array &$result): void
{
// Claim every row immediately before the host request. A successful
// worker operation advances the same revision, making this batch
// stale without allowing its eventual response to overwrite the
// worker's fresher state.
$active = [];
$revisions = [];
foreach ($group as $row) {
try {
$revisions[(int) $row['id']] = CloudInstanceRuntimeService::beginObservation(
(int) $row['id'], $hostId, (string) $row['uuid']
);
$active[] = $row;
} catch (\Throwable $error) {
$result['skipped']++;
}
}
if ($active === []) {
return;
}
// Keep the host request bounded and pass the explicit UUID set to the
// engine. The old no-argument vm-states endpoint enumerated every
// domain and is reserved for legacy private UI compatibility only.
foreach (array_chunk($active, 100) as $chunk) {
$allowed = [];
foreach ($chunk as $row) {
$allowed[(string) $row['uuid']] = $row;
}
$query = rawurlencode(implode(',', array_keys($allowed)));
$chunkIds = array_map(
static fn (array $row): int => (int) $row['id'], $chunk
);
try {
$payload = $this->engine->host($hostId, 'GET',
'vm-states?ids=' . $query, $actor);
} catch (CloudError $error) {
$chunkRevisions = [];
foreach ($chunkIds as $id) {
if (array_key_exists($id, $revisions)) {
$chunkRevisions[$id] = $revisions[$id];
}
}
CloudInstanceRuntimeService::markUnavailable(
$chunkIds, $hostId, time(), self::errorCode($error), $chunkRevisions
);
$result['unavailable'] += count($chunkIds);
$result['errors'][] = 'runtime_status_sync_failed';
continue;
} catch (\Throwable $error) {
$chunkRevisions = [];
foreach ($chunkIds as $id) {
if (array_key_exists($id, $revisions)) {
$chunkRevisions[$id] = $revisions[$id];
}
}
CloudInstanceRuntimeService::markUnavailable(
$chunkIds, $hostId, time(), 'host_unavailable', $chunkRevisions
);
$result['unavailable'] += count($chunkIds);
$result['errors'][] = 'runtime_status_sync_failed';
continue;
}
$seen = [];
$items = is_array($payload['items'] ?? null) ? $payload['items'] : [];
foreach ($items as $item) {
if (!is_array($item) || !is_string($item['uuid'] ?? null)) {
continue;
}
$uuid = strtolower(trim($item['uuid']));
if (!isset($allowed[$uuid])) {
continue;
}
$row = $allowed[$uuid];
$state = $item['state'] ?? ($item['power_state'] ?? null);
$seen[$uuid] = true;
try {
CloudInstanceRuntimeService::markObserved(
(int) $row['id'], $hostId, $uuid, $state, null,
$revisions[(int) $row['id']] ?? null
);
if (CloudInstanceRuntimeService::normalizeState($state) !== 'unknown') {
$result['observed']++;
} else {
$result['unavailable']++;
}
} catch (\Throwable $error) {
$result['skipped']++;
}
}
$missing = [];
$missingRevisions = [];
foreach ($allowed as $uuid => $row) {
if (!isset($seen[$uuid])) {
$missing[] = (int) $row['id'];
$id = (int) $row['id'];
if (array_key_exists($id, $revisions)) {
$missingRevisions[$id] = $revisions[$id];
}
}
}
if ($missing !== []) {
CloudInstanceRuntimeService::markUnavailable(
$missing, $hostId, time(), 'invalid_response', $missingRevisions
);
$result['unavailable'] += count($missing);
$result['errors'][] = 'runtime_status_sync_failed';
}
}
}
private function runtimeTableAvailable(): bool
{
try {
return (new CloudInstanceRuntime())->getConnection()
->getSchemaBuilder()->hasTable('cloud_instance_runtime');
} catch (\Throwable $error) {
return false;
}
}
private static function errorCode(CloudError $error): string
{
return in_array($error->httpStatus, [408, 429, 502, 503, 504], true)
? 'host_unavailable' : 'engine_error';
}
}