You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
290 lines
11 KiB
PHP
290 lines
11 KiB
PHP
<?php
|
|
|
|
namespace App\Services\Cloud;
|
|
|
|
use App\Models\CloudInstance;
|
|
use App\Models\CloudInstanceRuntime;
|
|
|
|
/**
|
|
* Refresh the instance runtime observation cache through the Go Agent.
|
|
*
|
|
* This service is intentionally separate from page rendering and mutation
|
|
* endpoints. Each request carries one explicit host/UUID identity to the
|
|
* typed Agent observation operation. A failed or uncertain Agent request
|
|
* only marks that cache row unavailable; it never changes the last known
|
|
* power state to "stopped".
|
|
*/
|
|
final class CloudInstanceRuntimeSyncService
|
|
{
|
|
// A lifecycle tick must not spend hours serially waiting on an offline
|
|
// host. The cursor remains in the runtime table, so the next tick can
|
|
// resume the bounded rotation.
|
|
private const MAX_SYNC_SECONDS = 25.0;
|
|
|
|
/** @var \Closure():CloudAgentObservationService */
|
|
private \Closure $agentFactory;
|
|
|
|
public function __construct(
|
|
private ?CloudAgentObservationService $agent = null,
|
|
?callable $agentFactory = null,
|
|
) {
|
|
$this->agentFactory = \Closure::fromCallable(
|
|
$agentFactory ?? static fn (): CloudAgentObservationService => new CloudAgentObservationService(
|
|
CloudAgentHostClientResolverFactory::fromConfig(),
|
|
new CloudAgentOperationRepository(),
|
|
),
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Poll active instances with bounded, explicit Agent observations. The
|
|
* returned counters contain no host-controlled text and are suitable for
|
|
* heartbeat diagnostics.
|
|
*/
|
|
public function sync(int $limit = 500): array
|
|
{
|
|
$result = ['observed' => 0, 'unavailable' => 0, 'skipped' => 0, 'errors' => []];
|
|
if (!$this->runtimeTableAvailable()) {
|
|
// Rolling upgrades may start the daemon before cloudInit has run.
|
|
// Keep billing/lifecycle work healthy until the table is present.
|
|
$result['skipped'] = 1;
|
|
return $result;
|
|
}
|
|
|
|
$actor = CloudDaemonActor::resolve('cloud-runtime-sync');
|
|
if ($actor === null) {
|
|
$result['errors'][] = 'runtime_status_sync_unconfigured';
|
|
return $result;
|
|
}
|
|
|
|
try {
|
|
// Select the least recently attempted observations first. This
|
|
// gives a bounded daemon pass a rotating cursor without storing a
|
|
// second global cursor: every successful or failed Agent request
|
|
// advances last_attempt_at, so high instance IDs cannot starve
|
|
// behind a permanently busy low-ID prefix.
|
|
$instanceQuery = CloudInstance::leftJoin(
|
|
'cloud_instance_runtime as runtime',
|
|
'runtime.instance_id', '=', 'cloud_instances.id'
|
|
)->where('cloud_instances.status', '<>', 'deleted')
|
|
->orderByRaw('COALESCE(runtime.last_attempt_at, runtime.last_success_at, 0) ASC')
|
|
->orderBy('cloud_instances.id')
|
|
->limit(max(1, min(2000, $limit)));
|
|
// Production uses MariaDB, where filtering malformed UUIDs
|
|
// at the database boundary prevents them from occupying every
|
|
// bounded pass. Keep the query usable in SQLite-based contract
|
|
// tests and local diagnostics; the application-level validation
|
|
// below remains the final guard in either driver.
|
|
if ((new CloudInstance())->getConnection()->getDriverName() === 'mysql') {
|
|
$instanceQuery->whereRaw(
|
|
"cloud_instances.uuid REGEXP ?",
|
|
['^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$']
|
|
);
|
|
}
|
|
$instances = $instanceQuery->get([
|
|
'cloud_instances.id', 'cloud_instances.host_id', 'cloud_instances.uuid',
|
|
]);
|
|
} catch (\Throwable $error) {
|
|
$result['errors'][] = 'runtime_status_sync_failed';
|
|
return $result;
|
|
}
|
|
if ($instances->isEmpty()) {
|
|
return $result;
|
|
}
|
|
|
|
$groups = [];
|
|
foreach ($instances as $instance) {
|
|
$hostId = (int) $instance->host_id;
|
|
$uuid = strtolower(trim((string) $instance->uuid));
|
|
if ($hostId < 1 || preg_match(
|
|
'/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/D',
|
|
$uuid
|
|
) !== 1) {
|
|
$result['skipped']++;
|
|
continue;
|
|
}
|
|
$groups[$hostId][] = [
|
|
'id' => (int) $instance->id,
|
|
'host_id' => $hostId,
|
|
'uuid' => $uuid,
|
|
];
|
|
// Instances may predate the independent cache. Seeding a row does
|
|
// not claim a power state; the Agent response must still provide
|
|
// the first successful observation.
|
|
try {
|
|
CloudInstanceRuntimeService::seed(
|
|
(int) $instance->id, $hostId, $uuid
|
|
);
|
|
} catch (\Throwable $error) {
|
|
$result['skipped']++;
|
|
// Do not leave a row that failed identity seeding in the
|
|
// host group; otherwise syncHost() would claim it again and
|
|
// count the same failure twice in one pass.
|
|
array_pop($groups[$hostId]);
|
|
}
|
|
}
|
|
|
|
$deadline = microtime(true) + self::MAX_SYNC_SECONDS;
|
|
foreach ($groups as $hostId => $group) {
|
|
if (microtime(true) >= $deadline) {
|
|
foreach ($group as $row) {
|
|
$this->defer($row);
|
|
}
|
|
$result['skipped'] += count($group);
|
|
continue;
|
|
}
|
|
$this->syncHost((int) $hostId, $group, $actor, $result, $deadline);
|
|
}
|
|
// Routine observations are high-frequency internal readbacks, not
|
|
// tenant-visible operation history. Keep a bounded diagnostic window
|
|
// while retaining every pending/running/uncertain row for explicit
|
|
// reconciliation.
|
|
(new CloudAgentOperationRepository())->pruneReadOnly(
|
|
time() - 86400,
|
|
2000,
|
|
);
|
|
return $result;
|
|
}
|
|
|
|
private function syncHost(
|
|
int $hostId,
|
|
array $group,
|
|
array $actor,
|
|
array &$result,
|
|
float $deadline,
|
|
): void
|
|
{
|
|
// Claim each row immediately before its Agent request. A successful
|
|
// worker operation advances the same revision, making the response
|
|
// stale without allowing it to overwrite fresher state.
|
|
foreach ($group as $index => $row) {
|
|
if (microtime(true) >= $deadline) {
|
|
foreach (array_slice($group, (int) $index) as $deferred) {
|
|
$this->defer($deferred);
|
|
}
|
|
$result['skipped'] += max(1, count($group) - (int) $index);
|
|
break;
|
|
}
|
|
$instanceId = (int) $row['id'];
|
|
$uuid = (string) $row['uuid'];
|
|
try {
|
|
$revision = CloudInstanceRuntimeService::beginObservation(
|
|
$instanceId, $hostId, $uuid
|
|
);
|
|
} catch (\Throwable $error) {
|
|
$result['skipped']++;
|
|
continue;
|
|
}
|
|
try {
|
|
$observed = $this->agent()->observe(
|
|
$hostId,
|
|
$uuid,
|
|
$actor,
|
|
null,
|
|
null,
|
|
$instanceId,
|
|
);
|
|
$record = is_array($observed['record'] ?? null)
|
|
? $observed['record'] : [];
|
|
$candidate = is_array($observed['observation'] ?? null)
|
|
? $observed['observation'] : [];
|
|
$candidateUuid = is_string($candidate['uuid'] ?? null)
|
|
? strtolower(trim($candidate['uuid'])) : '';
|
|
$state = CloudInstanceRuntimeService::normalizeState(
|
|
$candidate['state'] ?? null
|
|
);
|
|
if (($record['status'] ?? null) !== 'succeeded'
|
|
|| $candidateUuid === ''
|
|
|| !hash_equals($uuid, $candidateUuid)
|
|
|| $state === 'unknown') {
|
|
$this->markUnavailable(
|
|
$instanceId, $hostId, $revision, 'invalid_response', $result
|
|
);
|
|
continue;
|
|
}
|
|
try {
|
|
CloudInstanceRuntimeService::markObserved(
|
|
$instanceId, $hostId, $uuid, $state, null, $revision
|
|
);
|
|
$result['observed']++;
|
|
} catch (\Throwable $error) {
|
|
// A concurrent lifecycle writer may have won the CAS.
|
|
// Preserve its fresher value and do not turn that race
|
|
// into a host outage.
|
|
$result['skipped']++;
|
|
}
|
|
} catch (CloudAgentOperationException $error) {
|
|
$this->markUnavailable(
|
|
$instanceId, $hostId, $revision, self::agentErrorCode($error), $result
|
|
);
|
|
} catch (CloudError $error) {
|
|
$this->markUnavailable(
|
|
$instanceId, $hostId, $revision, self::agentErrorCode($error), $result
|
|
);
|
|
} catch (\Throwable $error) {
|
|
$this->markUnavailable(
|
|
$instanceId, $hostId, $revision, 'host_unavailable', $result
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
private function agent(): CloudAgentObservationService
|
|
{
|
|
return $this->agent ??= ($this->agentFactory)();
|
|
}
|
|
|
|
/** @param array{id:int,host_id:int,uuid:string} $row */
|
|
private function defer(array $row): void
|
|
{
|
|
try {
|
|
CloudInstanceRuntimeService::deferObservation(
|
|
(int) ($row['id'] ?? 0),
|
|
(int) ($row['host_id'] ?? 0),
|
|
(string) ($row['uuid'] ?? ''),
|
|
);
|
|
} catch (\Throwable) {
|
|
// Scheduling metadata is best-effort; never turn a soft deadline
|
|
// into a lifecycle failure or recreate a deleted runtime row.
|
|
}
|
|
}
|
|
|
|
/** @param array{observed:int,unavailable:int,skipped:int,errors:list<string>} $result */
|
|
private function markUnavailable(
|
|
int $instanceId,
|
|
int $hostId,
|
|
int $revision,
|
|
string $errorCode,
|
|
array &$result,
|
|
): void {
|
|
try {
|
|
CloudInstanceRuntimeService::markUnavailable(
|
|
[$instanceId], $hostId, time(), $errorCode, [$instanceId => $revision]
|
|
);
|
|
} catch (\Throwable $error) {
|
|
// The observation row may have been removed or replaced while
|
|
// the Agent request was in flight. Either way, never recreate it.
|
|
}
|
|
$result['unavailable']++;
|
|
$result['errors'][] = 'runtime_status_sync_failed';
|
|
}
|
|
|
|
private function runtimeTableAvailable(): bool
|
|
{
|
|
try {
|
|
return (new CloudInstanceRuntime())->getConnection()
|
|
->getSchemaBuilder()->hasTable('cloud_instance_runtime');
|
|
} catch (\Throwable $error) {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
private static function agentErrorCode(\Throwable $error): string
|
|
{
|
|
$status = $error instanceof CloudError || $error instanceof CloudAgentOperationException
|
|
? $error->httpStatus : 503;
|
|
return in_array($status, [408, 429, 502, 503, 504], true)
|
|
? 'host_unavailable' : 'invalid_response';
|
|
}
|
|
}
|