engine ??= new CloudEngineClient(null, self::ENGINE_TIMEOUT_SECONDS); } /** * Poll active instances in host batches. The returned counters contain * no engine-controlled text and are suitable for heartbeat diagnostics. */ public function sync(int $limit = 500): array { $result = ['observed' => 0, 'unavailable' => 0, 'skipped' => 0, 'errors' => []]; if (!$this->runtimeTableAvailable()) { // Rolling upgrades may start the daemon before cloudInit has run. // Keep billing/lifecycle work healthy until the table is present. $result['skipped'] = 1; return $result; } $actor = CloudDaemonActor::resolve('cloud-runtime-sync'); if ($actor === null) { $result['errors'][] = 'runtime_status_sync_unconfigured'; return $result; } try { // Select the least recently attempted observations first. This // gives a bounded daemon pass a rotating cursor without storing a // second global cursor: every successful or failed host batch // advances last_attempt_at, so high instance IDs cannot starve // behind a permanently busy low-ID prefix. $instanceQuery = CloudInstance::leftJoin( 'cloud_instance_runtime as runtime', 'runtime.instance_id', '=', 'cloud_instances.id' )->where('cloud_instances.status', '<>', 'deleted') ->orderByRaw('COALESCE(runtime.last_attempt_at, runtime.last_success_at, 0) ASC') ->orderBy('cloud_instances.id') ->limit(max(1, min(2000, $limit))); // Production uses MariaDB, where filtering malformed legacy UUIDs // at the database boundary prevents them from occupying every // bounded pass. Keep the query usable in SQLite-based contract // tests and local diagnostics; the application-level validation // below remains the final guard in either driver. if ((new CloudInstance())->getConnection()->getDriverName() === 'mysql') { $instanceQuery->whereRaw( "cloud_instances.uuid REGEXP ?", ['^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$'] ); } $instances = $instanceQuery->get([ 'cloud_instances.id', 'cloud_instances.host_id', 'cloud_instances.uuid', ]); } catch (\Throwable $error) { $result['errors'][] = 'runtime_status_sync_failed'; return $result; } if ($instances->isEmpty()) { return $result; } $groups = []; foreach ($instances as $instance) { $hostId = (int) $instance->host_id; $uuid = strtolower(trim((string) $instance->uuid)); if ($hostId < 1 || preg_match( '/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/D', $uuid ) !== 1) { $result['skipped']++; continue; } $groups[$hostId][] = [ 'id' => (int) $instance->id, 'host_id' => $hostId, 'uuid' => $uuid, ]; // Legacy instances may predate the independent cache. Seeding a // row does not claim a power state; the batch response still must // provide the first successful observation. try { CloudInstanceRuntimeService::seed( (int) $instance->id, $hostId, $uuid ); } catch (\Throwable $error) { $result['skipped']++; // Do not leave a row that failed identity seeding in the // host group; otherwise syncHost() would claim it again and // count the same failure twice in one pass. array_pop($groups[$hostId]); } } foreach ($groups as $hostId => $group) { $this->syncHost((int) $hostId, $group, $actor, $result); } return $result; } private function syncHost(int $hostId, array $group, array $actor, array &$result): void { // Claim every row immediately before the host request. A successful // worker operation advances the same revision, making this batch // stale without allowing its eventual response to overwrite the // worker's fresher state. $active = []; $revisions = []; foreach ($group as $row) { try { $revisions[(int) $row['id']] = CloudInstanceRuntimeService::beginObservation( (int) $row['id'], $hostId, (string) $row['uuid'] ); $active[] = $row; } catch (\Throwable $error) { $result['skipped']++; } } if ($active === []) { return; } // Keep the host request bounded and pass the explicit UUID set to the // engine. The old no-argument vm-states endpoint enumerated every // domain and is reserved for legacy private UI compatibility only. foreach (array_chunk($active, 100) as $chunk) { $allowed = []; foreach ($chunk as $row) { $allowed[(string) $row['uuid']] = $row; } $query = rawurlencode(implode(',', array_keys($allowed))); $chunkIds = array_map( static fn (array $row): int => (int) $row['id'], $chunk ); try { $payload = $this->engine->host($hostId, 'GET', 'vm-states?ids=' . $query, $actor); } catch (CloudError $error) { $chunkRevisions = []; foreach ($chunkIds as $id) { if (array_key_exists($id, $revisions)) { $chunkRevisions[$id] = $revisions[$id]; } } CloudInstanceRuntimeService::markUnavailable( $chunkIds, $hostId, time(), self::errorCode($error), $chunkRevisions ); $result['unavailable'] += count($chunkIds); $result['errors'][] = 'runtime_status_sync_failed'; continue; } catch (\Throwable $error) { $chunkRevisions = []; foreach ($chunkIds as $id) { if (array_key_exists($id, $revisions)) { $chunkRevisions[$id] = $revisions[$id]; } } CloudInstanceRuntimeService::markUnavailable( $chunkIds, $hostId, time(), 'host_unavailable', $chunkRevisions ); $result['unavailable'] += count($chunkIds); $result['errors'][] = 'runtime_status_sync_failed'; continue; } $seen = []; $items = is_array($payload['items'] ?? null) ? $payload['items'] : []; foreach ($items as $item) { if (!is_array($item) || !is_string($item['uuid'] ?? null)) { continue; } $uuid = strtolower(trim($item['uuid'])); if (!isset($allowed[$uuid])) { continue; } $row = $allowed[$uuid]; $state = $item['state'] ?? ($item['power_state'] ?? null); $seen[$uuid] = true; try { CloudInstanceRuntimeService::markObserved( (int) $row['id'], $hostId, $uuid, $state, null, $revisions[(int) $row['id']] ?? null ); if (CloudInstanceRuntimeService::normalizeState($state) !== 'unknown') { $result['observed']++; } else { $result['unavailable']++; } } catch (\Throwable $error) { $result['skipped']++; } } $missing = []; $missingRevisions = []; foreach ($allowed as $uuid => $row) { if (!isset($seen[$uuid])) { $missing[] = (int) $row['id']; $id = (int) $row['id']; if (array_key_exists($id, $revisions)) { $missingRevisions[$id] = $revisions[$id]; } } } if ($missing !== []) { CloudInstanceRuntimeService::markUnavailable( $missing, $hostId, time(), 'invalid_response', $missingRevisions ); $result['unavailable'] += count($missing); $result['errors'][] = 'runtime_status_sync_failed'; } } } private function runtimeTableAvailable(): bool { try { return (new CloudInstanceRuntime())->getConnection() ->getSchemaBuilder()->hasTable('cloud_instance_runtime'); } catch (\Throwable $error) { return false; } } private static function errorCode(CloudError $error): string { return in_array($error->httpStatus, [408, 429, 502, 503, 504], true) ? 'host_unavailable' : 'engine_error'; } }