From 9dbc6b2c84f6428540bc63d507036f71ec8532dc Mon Sep 17 00:00:00 2001 From: admin Date: Fri, 16 Oct 2026 02:55:17 +0800 Subject: [PATCH] feat: sync runtime cache through Go Agent --- docs/GO_AGENT_RUNTIME_SYNC.md | 32 +++ .../Cloud/CloudInstanceRuntimeSyncService.php | 202 ++++++++---------- ...udInstanceRuntimeSyncAgentContractTest.php | 37 ++++ 3 files changed, 164 insertions(+), 107 deletions(-) create mode 100644 docs/GO_AGENT_RUNTIME_SYNC.md create mode 100644 web/tests/CloudInstanceRuntimeSyncAgentContractTest.php diff --git a/docs/GO_AGENT_RUNTIME_SYNC.md b/docs/GO_AGENT_RUNTIME_SYNC.md new file mode 100644 index 00000000..bb6487dc --- /dev/null +++ b/docs/GO_AGENT_RUNTIME_SYNC.md @@ -0,0 +1,32 @@ +# Go Agent runtime observation sync + +`CloudInstanceRuntimeSyncService` now refreshes the lifecycle power-state cache +through the fresh Go Agent protocol. It no longer calls the retired Python +Engine or the host-wide `vm-states` endpoint. + +The scheduler first selects a bounded, actor-independent set of non-deleted +database instances. Every row must have a valid `host_id` and canonical VM UUID; +the pair is retained as the only target identity. Before making a host request, +PHP claims that runtime row's revision with `beginObservation()`. It then sends +one signed `domain_observe` operation through `CloudAgentObservationService` to +that exact host and UUID. + +Only a succeeded operation whose returned UUID matches the requested UUID and +whose state is in the panel's bounded vocabulary may call `markObserved()` with +the claim revision. A transport timeout, uncertain operation, deterministic +Agent rejection, malformed response, unknown state, or UUID mismatch calls +`markUnavailable()` with the same revision. The last successful power state is +therefore retained; an unavailable observation is never interpreted as +`stopped`. A concurrent lifecycle/action writer can win the revision CAS, in +which case the stale poll is counted as skipped and cannot overwrite it. + +The returned counters remain `observed`, `unavailable`, `skipped`, and a +bounded error-code list so existing lifecycle heartbeat diagnostics and page +contracts remain unchanged. Each Agent observation is journaled with its +operation ID; uncertain transport outcomes are not replayed by the scheduler. +They require the normal operation-status/reconcile path. + +This is a narrow readback cutover, not approval to delete all Python code. VM +creation, reinstall/delete, network, settings, snapshots, firewall, rescue, +console, metrics, and remaining host administration still need typed Agent +operations and real-host acceptance before Python service units can be removed. diff --git a/web/app/Services/Cloud/CloudInstanceRuntimeSyncService.php b/web/app/Services/Cloud/CloudInstanceRuntimeSyncService.php index 2cef5b04..112a2385 100644 --- a/web/app/Services/Cloud/CloudInstanceRuntimeSyncService.php +++ b/web/app/Services/Cloud/CloudInstanceRuntimeSyncService.php @@ -6,24 +6,35 @@ use App\Models\CloudInstance; use App\Models\CloudInstanceRuntime; /** - * Refresh the instance runtime observation cache from host batch state. + * Refresh the instance runtime observation cache through the Go Agent. * * This service is intentionally separate from page rendering and mutation - * endpoints. A failed host request only marks its observations unavailable; - * it never changes the last known power state to "stopped". + * endpoints. Each request carries one explicit host/UUID identity to the + * typed Agent observation operation. A failed or uncertain Agent request + * only marks that cache row unavailable; it never changes the last known + * power state to "stopped". */ final class CloudInstanceRuntimeSyncService { - private const ENGINE_TIMEOUT_SECONDS = 3.0; + /** @var \Closure():CloudAgentObservationService */ + private \Closure $agentFactory; - public function __construct(private ?CloudEngineClient $engine = null) - { - $this->engine ??= new CloudEngineClient(null, self::ENGINE_TIMEOUT_SECONDS); + public function __construct( + private ?CloudAgentObservationService $agent = null, + ?callable $agentFactory = null, + ) { + $this->agentFactory = \Closure::fromCallable( + $agentFactory ?? static fn (): CloudAgentObservationService => new CloudAgentObservationService( + CloudAgentHostClientResolverFactory::fromConfig(), + new CloudAgentOperationRepository(), + ), + ); } /** - * Poll active instances in host batches. The returned counters contain - * no engine-controlled text and are suitable for heartbeat diagnostics. + * Poll active instances with bounded, explicit Agent observations. The + * returned counters contain no host-controlled text and are suitable for + * heartbeat diagnostics. */ public function sync(int $limit = 500): array { @@ -44,7 +55,7 @@ final class CloudInstanceRuntimeSyncService try { // Select the least recently attempted observations first. This // gives a bounded daemon pass a rotating cursor without storing a - // second global cursor: every successful or failed host batch + // second global cursor: every successful or failed Agent request // advances last_attempt_at, so high instance IDs cannot starve // behind a permanently busy low-ID prefix. $instanceQuery = CloudInstance::leftJoin( @@ -54,7 +65,7 @@ final class CloudInstanceRuntimeSyncService ->orderByRaw('COALESCE(runtime.last_attempt_at, runtime.last_success_at, 0) ASC') ->orderBy('cloud_instances.id') ->limit(max(1, min(2000, $limit))); - // Production uses MariaDB, where filtering malformed legacy UUIDs + // Production uses MariaDB, where filtering malformed UUIDs // at the database boundary prevents them from occupying every // bounded pass. Keep the query usable in SQLite-based contract // tests and local diagnostics; the application-level validation @@ -92,9 +103,9 @@ final class CloudInstanceRuntimeSyncService 'host_id' => $hostId, 'uuid' => $uuid, ]; - // Legacy instances may predate the independent cache. Seeding a - // row does not claim a power state; the batch response still must - // provide the first successful observation. + // Instances may predate the independent cache. Seeding a row does + // not claim a power state; the Agent response must still provide + // the first successful observation. try { CloudInstanceRuntimeService::seed( (int) $instance->id, $hostId, $uuid @@ -116,117 +127,92 @@ final class CloudInstanceRuntimeSyncService private function syncHost(int $hostId, array $group, array $actor, array &$result): void { - // Claim every row immediately before the host request. A successful - // worker operation advances the same revision, making this batch - // stale without allowing its eventual response to overwrite the - // worker's fresher state. - $active = []; - $revisions = []; + // Claim each row immediately before its Agent request. A successful + // worker operation advances the same revision, making the response + // stale without allowing it to overwrite fresher state. foreach ($group as $row) { + $instanceId = (int) $row['id']; + $uuid = (string) $row['uuid']; try { - $revisions[(int) $row['id']] = CloudInstanceRuntimeService::beginObservation( - (int) $row['id'], $hostId, (string) $row['uuid'] + $revision = CloudInstanceRuntimeService::beginObservation( + $instanceId, $hostId, $uuid ); - $active[] = $row; } catch (\Throwable $error) { $result['skipped']++; + continue; } - } - if ($active === []) { - return; - } - // Keep the host request bounded and pass the explicit UUID set to the - // engine. The old no-argument vm-states endpoint enumerated every - // domain and is reserved for legacy private UI compatibility only. - foreach (array_chunk($active, 100) as $chunk) { - $allowed = []; - foreach ($chunk as $row) { - $allowed[(string) $row['uuid']] = $row; - } - $query = rawurlencode(implode(',', array_keys($allowed))); - $chunkIds = array_map( - static fn (array $row): int => (int) $row['id'], $chunk - ); try { - $payload = $this->engine->host($hostId, 'GET', - 'vm-states?ids=' . $query, $actor); - } catch (CloudError $error) { - $chunkRevisions = []; - foreach ($chunkIds as $id) { - if (array_key_exists($id, $revisions)) { - $chunkRevisions[$id] = $revisions[$id]; - } - } - CloudInstanceRuntimeService::markUnavailable( - $chunkIds, $hostId, time(), self::errorCode($error), $chunkRevisions - ); - $result['unavailable'] += count($chunkIds); - $result['errors'][] = 'runtime_status_sync_failed'; - continue; - } catch (\Throwable $error) { - $chunkRevisions = []; - foreach ($chunkIds as $id) { - if (array_key_exists($id, $revisions)) { - $chunkRevisions[$id] = $revisions[$id]; - } - } - CloudInstanceRuntimeService::markUnavailable( - $chunkIds, $hostId, time(), 'host_unavailable', $chunkRevisions + $observed = $this->agent()->observe($hostId, $uuid, $actor); + $record = is_array($observed['record'] ?? null) + ? $observed['record'] : []; + $candidate = is_array($observed['observation'] ?? null) + ? $observed['observation'] : []; + $candidateUuid = is_string($candidate['uuid'] ?? null) + ? strtolower(trim($candidate['uuid'])) : ''; + $state = CloudInstanceRuntimeService::normalizeState( + $candidate['state'] ?? null ); - $result['unavailable'] += count($chunkIds); - $result['errors'][] = 'runtime_status_sync_failed'; - continue; - } - - $seen = []; - $items = is_array($payload['items'] ?? null) ? $payload['items'] : []; - foreach ($items as $item) { - if (!is_array($item) || !is_string($item['uuid'] ?? null)) { - continue; - } - $uuid = strtolower(trim($item['uuid'])); - if (!isset($allowed[$uuid])) { + if (($record['status'] ?? null) !== 'succeeded' + || $candidateUuid === '' + || !hash_equals($uuid, $candidateUuid) + || $state === 'unknown') { + $this->markUnavailable( + $instanceId, $hostId, $revision, 'invalid_response', $result + ); continue; } - $row = $allowed[$uuid]; - $state = $item['state'] ?? ($item['power_state'] ?? null); - $seen[$uuid] = true; try { CloudInstanceRuntimeService::markObserved( - (int) $row['id'], $hostId, $uuid, $state, null, - $revisions[(int) $row['id']] ?? null + $instanceId, $hostId, $uuid, $state, null, $revision ); - if (CloudInstanceRuntimeService::normalizeState($state) !== 'unknown') { - $result['observed']++; - } else { - $result['unavailable']++; - } + $result['observed']++; } catch (\Throwable $error) { + // A concurrent lifecycle writer may have won the CAS. + // Preserve its fresher value and do not turn that race + // into a host outage. $result['skipped']++; } - } - - $missing = []; - $missingRevisions = []; - foreach ($allowed as $uuid => $row) { - if (!isset($seen[$uuid])) { - $missing[] = (int) $row['id']; - $id = (int) $row['id']; - if (array_key_exists($id, $revisions)) { - $missingRevisions[$id] = $revisions[$id]; - } - } - } - if ($missing !== []) { - CloudInstanceRuntimeService::markUnavailable( - $missing, $hostId, time(), 'invalid_response', $missingRevisions + } catch (CloudAgentOperationException $error) { + $this->markUnavailable( + $instanceId, $hostId, $revision, self::agentErrorCode($error), $result + ); + } catch (CloudError $error) { + $this->markUnavailable( + $instanceId, $hostId, $revision, self::agentErrorCode($error), $result + ); + } catch (\Throwable $error) { + $this->markUnavailable( + $instanceId, $hostId, $revision, 'host_unavailable', $result ); - $result['unavailable'] += count($missing); - $result['errors'][] = 'runtime_status_sync_failed'; } } } + private function agent(): CloudAgentObservationService + { + return $this->agent ??= ($this->agentFactory)(); + } + + /** @param array{observed:int,unavailable:int,skipped:int,errors:list} $result */ + private function markUnavailable( + int $instanceId, + int $hostId, + int $revision, + string $errorCode, + array &$result, + ): void { + try { + CloudInstanceRuntimeService::markUnavailable( + [$instanceId], $hostId, time(), $errorCode, [$instanceId => $revision] + ); + } catch (\Throwable $error) { + // The observation row may have been removed or replaced while + // the Agent request was in flight. Either way, never recreate it. + } + $result['unavailable']++; + $result['errors'][] = 'runtime_status_sync_failed'; + } + private function runtimeTableAvailable(): bool { try { @@ -237,9 +223,11 @@ final class CloudInstanceRuntimeSyncService } } - private static function errorCode(CloudError $error): string + private static function agentErrorCode(\Throwable $error): string { - return in_array($error->httpStatus, [408, 429, 502, 503, 504], true) - ? 'host_unavailable' : 'engine_error'; + $status = $error instanceof CloudError || $error instanceof CloudAgentOperationException + ? $error->httpStatus : 503; + return in_array($status, [408, 429, 502, 503, 504], true) + ? 'host_unavailable' : 'invalid_response'; } } diff --git a/web/tests/CloudInstanceRuntimeSyncAgentContractTest.php b/web/tests/CloudInstanceRuntimeSyncAgentContractTest.php new file mode 100644 index 00000000..59338209 --- /dev/null +++ b/web/tests/CloudInstanceRuntimeSyncAgentContractTest.php @@ -0,0 +1,37 @@ +observe($hostId, $uuid, $actor)') + && str_contains($service, 'CloudInstanceRuntimeService::beginObservation(') + && str_contains($service, 'CloudInstanceRuntimeService::markObserved(') + && str_contains($service, 'CloudInstanceRuntimeService::markUnavailable(') + && str_contains($service, 'hash_equals($uuid, $candidateUuid)') + && !str_contains($service, 'CloudEngineClient') + && !str_contains($service, 'vm-states') + && !str_contains($service, 'shell_exec') + && !str_contains($service, 'ssh'), + 'Runtime sync must use typed Agent observations and no Python/SSH or host-wide vm-states fallback.'); + +$assert(str_contains($observation, 'resolveReadOnly($hostId)') + && str_contains($observation, "'domain_observe'") + && str_contains($observation, 'markUncertain') + && str_contains($observation, 'observeMany'), + 'Agent observation service must keep read-only resolver, typed operation and uncertain journal semantics.'); + +echo "Cloud Agent runtime sync contract passed.\n";