You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/app/Services/Cloud/CloudInstanceRuntimeSyncSer...

232 lines
9.2 KiB
PHP

<?php
namespace App\Services\Cloud;
use App\Models\CloudInstance;
use App\Models\CloudInstanceRuntime;
/**
* Refresh the instance runtime observation cache from host batch state.
*
* This service is intentionally separate from page rendering and mutation
* endpoints. A failed host request only marks its observations unavailable;
* it never changes the last known power state to "stopped".
*/
final class CloudInstanceRuntimeSyncService
{
private const AGENT_TIMEOUT_SECONDS = 3.0;
public function __construct(private ?CloudAgentClient $agent = null) {}
private function agentForHost(int $hostId): CloudAgentClient
{
return $this->agent ?? CloudAgentSettingsService::clientForHost(
$hostId,
self::AGENT_TIMEOUT_SECONDS
);
}
/**
* Poll active instances in host batches. The returned counters contain
* no Agent-controlled text and are suitable for heartbeat diagnostics.
*/
public function sync(int $limit = 500): array
{
$result = ['observed' => 0, 'unavailable' => 0, 'skipped' => 0, 'errors' => []];
if (!$this->runtimeTableAvailable()) {
// Rolling upgrades may start the daemon before cloudInit has run.
// Keep billing/lifecycle work healthy until the table is present.
$result['skipped'] = 1;
return $result;
}
$actor = CloudDaemonActor::resolve('cloud-runtime-sync');
if ($actor === null) {
$result['errors'][] = 'runtime_status_sync_unconfigured';
return $result;
}
try {
// Select the least recently attempted observations first. This
// gives a bounded daemon pass a rotating cursor without storing a
// second global cursor: every successful or failed host batch
// advances last_attempt_at, so high instance IDs cannot starve
// behind a permanently busy low-ID prefix.
$instanceQuery = CloudInstance::leftJoin(
'cloud_instance_runtime as runtime',
'runtime.instance_id', '=', 'cloud_instances.id'
)->where('cloud_instances.status', '<>', 'deleted')
->orderByRaw('COALESCE(runtime.last_attempt_at, runtime.last_success_at, 0) ASC')
->orderBy('cloud_instances.id')
->limit(max(1, min(2000, $limit)));
// Production uses MariaDB; filtering malformed UUIDs at the
// database boundary prevents them from occupying every bounded
// pass. The application-level validation remains the final guard.
if ((new CloudInstance())->getConnection()->getDriverName() === 'mysql') {
$instanceQuery->whereRaw(
"cloud_instances.uuid REGEXP ?",
['^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$']
);
}
$instances = $instanceQuery->get([
'cloud_instances.id', 'cloud_instances.host_id', 'cloud_instances.uuid',
]);
} catch (\Throwable $error) {
$result['errors'][] = 'runtime_status_sync_failed';
return $result;
}
if ($instances->isEmpty()) {
return $result;
}
$groups = [];
foreach ($instances as $instance) {
$hostId = (int) $instance->host_id;
$uuid = strtolower(trim((string) $instance->uuid));
if ($hostId < 1 || preg_match(
'/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/D',
$uuid
) !== 1) {
$result['skipped']++;
continue;
}
$groups[$hostId][] = [
'id' => (int) $instance->id,
'host_id' => $hostId,
'uuid' => $uuid,
];
// Seeding a row does not claim a power state; the Agent response
// still must provide the first successful observation.
try {
CloudInstanceRuntimeService::seed(
(int) $instance->id, $hostId, $uuid
);
} catch (\Throwable $error) {
$result['skipped']++;
// Do not leave a row that failed identity seeding in the
// host group; otherwise syncHost() would claim it again and
// count the same failure twice in one pass.
array_pop($groups[$hostId]);
}
}
foreach ($groups as $hostId => $group) {
$this->syncHost((int) $hostId, $group, $actor, $result);
}
return $result;
}
private function syncHost(int $hostId, array $group, array $actor, array &$result): void
{
// Claim every row immediately before the host request. A successful
// worker operation advances the same revision, making this batch
// stale without allowing its eventual response to overwrite the
// worker's fresher state.
$active = [];
$revisions = [];
foreach ($group as $row) {
try {
$revisions[(int) $row['id']] = CloudInstanceRuntimeService::beginObservation(
(int) $row['id'], $hostId, (string) $row['uuid']
);
$active[] = $row;
} catch (\Throwable $error) {
$result['skipped']++;
}
}
if ($active === []) {
return;
}
// Query only UUIDs already selected from the actor-independent panel
// database; the Agent API intentionally exposes per-VM state reads.
foreach (array_chunk($active, 100) as $chunk) {
$chunkIds = array_map(
static fn (array $row): int => (int) $row['id'], $chunk
);
try {
$agent = $this->agentForHost($hostId);
} catch (CloudError $error) {
$chunkRevisions = [];
foreach ($chunkIds as $id) {
if (array_key_exists($id, $revisions)) {
$chunkRevisions[$id] = $revisions[$id];
}
}
CloudInstanceRuntimeService::markUnavailable(
$chunkIds, $hostId, time(), self::errorCode($error), $chunkRevisions
);
$result['unavailable'] += count($chunkIds);
$result['errors'][] = 'runtime_status_sync_failed';
continue;
} catch (\Throwable $error) {
$chunkRevisions = [];
foreach ($chunkIds as $id) {
if (array_key_exists($id, $revisions)) {
$chunkRevisions[$id] = $revisions[$id];
}
}
CloudInstanceRuntimeService::markUnavailable(
$chunkIds, $hostId, time(), 'host_unavailable', $chunkRevisions
);
$result['unavailable'] += count($chunkIds);
$result['errors'][] = 'runtime_status_sync_failed';
continue;
}
foreach ($chunk as $row) {
$uuid = (string) $row['uuid'];
try {
$item = $agent->request(
'GET',
'/v1/vms/' . rawurlencode($uuid) . '/state',
$actor
);
if (!is_string($item['uuid'] ?? null)
|| !hash_equals($uuid,
strtolower(trim($item['uuid'])))) {
throw new CloudError('Agent 状态响应与实例不匹配', 502);
}
$state = $item['state'] ?? null;
CloudInstanceRuntimeService::markObserved(
(int) $row['id'], $hostId, $uuid, $state, null,
$revisions[(int) $row['id']] ?? null
);
if (CloudInstanceRuntimeService::normalizeState($state) !== 'unknown') {
$result['observed']++;
} else {
$result['unavailable']++;
}
} catch (CloudError $error) {
$id = (int) $row['id'];
CloudInstanceRuntimeService::markUnavailable(
[$id],
$hostId,
time(),
self::errorCode($error),
isset($revisions[$id]) ? [$id => $revisions[$id]] : []
);
$result['unavailable']++;
$result['errors'][] = 'runtime_status_sync_failed';
} catch (\Throwable $error) {
$result['skipped']++;
}
}
}
}
private function runtimeTableAvailable(): bool
{
try {
return (new CloudInstanceRuntime())->getConnection()
->getSchemaBuilder()->hasTable('cloud_instance_runtime');
} catch (\Throwable $error) {
return false;
}
}
private static function errorCode(CloudError $error): string
{
return in_array($error->httpStatus, [408, 429, 502, 503, 504], true)
? 'host_unavailable' : 'agent_error';
}
}