You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/app/Services/Cloud/CloudInstanceRuntimeSyncSer...

290 lines
11 KiB
PHP

<?php
namespace App\Services\Cloud;
use App\Models\CloudInstance;
use App\Models\CloudInstanceRuntime;
/**
* Refresh the instance runtime observation cache through the Go Agent.
*
* This service is intentionally separate from page rendering and mutation
* endpoints. Each request carries one explicit host/UUID identity to the
* typed Agent observation operation. A failed or uncertain Agent request
* only marks that cache row unavailable; it never changes the last known
* power state to "stopped".
*/
final class CloudInstanceRuntimeSyncService
{
// A lifecycle tick must not spend hours serially waiting on an offline
// host. The cursor remains in the runtime table, so the next tick can
// resume the bounded rotation.
private const MAX_SYNC_SECONDS = 25.0;
/** @var \Closure():CloudAgentObservationService */
private \Closure $agentFactory;
public function __construct(
private ?CloudAgentObservationService $agent = null,
?callable $agentFactory = null,
) {
$this->agentFactory = \Closure::fromCallable(
$agentFactory ?? static fn (): CloudAgentObservationService => new CloudAgentObservationService(
CloudAgentHostClientResolverFactory::fromConfig(),
new CloudAgentOperationRepository(),
),
);
}
/**
* Poll active instances with bounded, explicit Agent observations. The
* returned counters contain no host-controlled text and are suitable for
* heartbeat diagnostics.
*/
public function sync(int $limit = 500): array
{
$result = ['observed' => 0, 'unavailable' => 0, 'skipped' => 0, 'errors' => []];
if (!$this->runtimeTableAvailable()) {
// Rolling upgrades may start the daemon before cloudInit has run.
// Keep billing/lifecycle work healthy until the table is present.
$result['skipped'] = 1;
return $result;
}
$actor = CloudDaemonActor::resolve('cloud-runtime-sync');
if ($actor === null) {
$result['errors'][] = 'runtime_status_sync_unconfigured';
return $result;
}
try {
// Select the least recently attempted observations first. This
// gives a bounded daemon pass a rotating cursor without storing a
// second global cursor: every successful or failed Agent request
// advances last_attempt_at, so high instance IDs cannot starve
// behind a permanently busy low-ID prefix.
$instanceQuery = CloudInstance::leftJoin(
'cloud_instance_runtime as runtime',
'runtime.instance_id', '=', 'cloud_instances.id'
)->where('cloud_instances.status', '<>', 'deleted')
->orderByRaw('COALESCE(runtime.last_attempt_at, runtime.last_success_at, 0) ASC')
->orderBy('cloud_instances.id')
->limit(max(1, min(2000, $limit)));
// Production uses MariaDB, where filtering malformed UUIDs
// at the database boundary prevents them from occupying every
// bounded pass. Keep the query usable in SQLite-based contract
// tests and local diagnostics; the application-level validation
// below remains the final guard in either driver.
if ((new CloudInstance())->getConnection()->getDriverName() === 'mysql') {
$instanceQuery->whereRaw(
"cloud_instances.uuid REGEXP ?",
['^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$']
);
}
$instances = $instanceQuery->get([
'cloud_instances.id', 'cloud_instances.host_id', 'cloud_instances.uuid',
]);
} catch (\Throwable $error) {
$result['errors'][] = 'runtime_status_sync_failed';
return $result;
}
if ($instances->isEmpty()) {
return $result;
}
$groups = [];
foreach ($instances as $instance) {
$hostId = (int) $instance->host_id;
$uuid = strtolower(trim((string) $instance->uuid));
if ($hostId < 1 || preg_match(
'/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/D',
$uuid
) !== 1) {
$result['skipped']++;
continue;
}
$groups[$hostId][] = [
'id' => (int) $instance->id,
'host_id' => $hostId,
'uuid' => $uuid,
];
// Instances may predate the independent cache. Seeding a row does
// not claim a power state; the Agent response must still provide
// the first successful observation.
try {
CloudInstanceRuntimeService::seed(
(int) $instance->id, $hostId, $uuid
);
} catch (\Throwable $error) {
$result['skipped']++;
// Do not leave a row that failed identity seeding in the
// host group; otherwise syncHost() would claim it again and
// count the same failure twice in one pass.
array_pop($groups[$hostId]);
}
}
$deadline = microtime(true) + self::MAX_SYNC_SECONDS;
foreach ($groups as $hostId => $group) {
if (microtime(true) >= $deadline) {
foreach ($group as $row) {
$this->defer($row);
}
$result['skipped'] += count($group);
continue;
}
$this->syncHost((int) $hostId, $group, $actor, $result, $deadline);
}
// Routine observations are high-frequency internal readbacks, not
// tenant-visible operation history. Keep a bounded diagnostic window
// while retaining every pending/running/uncertain row for explicit
// reconciliation.
(new CloudAgentOperationRepository())->pruneReadOnly(
time() - 86400,
2000,
);
return $result;
}
private function syncHost(
int $hostId,
array $group,
array $actor,
array &$result,
float $deadline,
): void
{
// Claim each row immediately before its Agent request. A successful
// worker operation advances the same revision, making the response
// stale without allowing it to overwrite fresher state.
foreach ($group as $index => $row) {
if (microtime(true) >= $deadline) {
foreach (array_slice($group, (int) $index) as $deferred) {
$this->defer($deferred);
}
$result['skipped'] += max(1, count($group) - (int) $index);
break;
}
$instanceId = (int) $row['id'];
$uuid = (string) $row['uuid'];
try {
$revision = CloudInstanceRuntimeService::beginObservation(
$instanceId, $hostId, $uuid
);
} catch (\Throwable $error) {
$result['skipped']++;
continue;
}
try {
$observed = $this->agent()->observe(
$hostId,
$uuid,
$actor,
null,
null,
$instanceId,
);
$record = is_array($observed['record'] ?? null)
? $observed['record'] : [];
$candidate = is_array($observed['observation'] ?? null)
? $observed['observation'] : [];
$candidateUuid = is_string($candidate['uuid'] ?? null)
? strtolower(trim($candidate['uuid'])) : '';
$state = CloudInstanceRuntimeService::normalizeState(
$candidate['state'] ?? null
);
if (($record['status'] ?? null) !== 'succeeded'
|| $candidateUuid === ''
|| !hash_equals($uuid, $candidateUuid)
|| $state === 'unknown') {
$this->markUnavailable(
$instanceId, $hostId, $revision, 'invalid_response', $result
);
continue;
}
try {
CloudInstanceRuntimeService::markObserved(
$instanceId, $hostId, $uuid, $state, null, $revision
);
$result['observed']++;
} catch (\Throwable $error) {
// A concurrent lifecycle writer may have won the CAS.
// Preserve its fresher value and do not turn that race
// into a host outage.
$result['skipped']++;
}
} catch (CloudAgentOperationException $error) {
$this->markUnavailable(
$instanceId, $hostId, $revision, self::agentErrorCode($error), $result
);
} catch (CloudError $error) {
$this->markUnavailable(
$instanceId, $hostId, $revision, self::agentErrorCode($error), $result
);
} catch (\Throwable $error) {
$this->markUnavailable(
$instanceId, $hostId, $revision, 'host_unavailable', $result
);
}
}
}
private function agent(): CloudAgentObservationService
{
return $this->agent ??= ($this->agentFactory)();
}
/** @param array{id:int,host_id:int,uuid:string} $row */
private function defer(array $row): void
{
try {
CloudInstanceRuntimeService::deferObservation(
(int) ($row['id'] ?? 0),
(int) ($row['host_id'] ?? 0),
(string) ($row['uuid'] ?? ''),
);
} catch (\Throwable) {
// Scheduling metadata is best-effort; never turn a soft deadline
// into a lifecycle failure or recreate a deleted runtime row.
}
}
/** @param array{observed:int,unavailable:int,skipped:int,errors:list<string>} $result */
private function markUnavailable(
int $instanceId,
int $hostId,
int $revision,
string $errorCode,
array &$result,
): void {
try {
CloudInstanceRuntimeService::markUnavailable(
[$instanceId], $hostId, time(), $errorCode, [$instanceId => $revision]
);
} catch (\Throwable $error) {
// The observation row may have been removed or replaced while
// the Agent request was in flight. Either way, never recreate it.
}
$result['unavailable']++;
$result['errors'][] = 'runtime_status_sync_failed';
}
private function runtimeTableAvailable(): bool
{
try {
return (new CloudInstanceRuntime())->getConnection()
->getSchemaBuilder()->hasTable('cloud_instance_runtime');
} catch (\Throwable $error) {
return false;
}
}
private static function agentErrorCode(\Throwable $error): string
{
$status = $error instanceof CloudError || $error instanceof CloudAgentOperationException
? $error->httpStatus : 503;
return in_array($status, [408, 429, 502, 503, 504], true)
? 'host_unavailable' : 'invalid_response';
}
}