feat: sync runtime cache through Go Agent

go-agent
admin 1 month ago
parent 210a1ec244
commit 9dbc6b2c84

@ -0,0 +1,32 @@
# Go Agent runtime observation sync
`CloudInstanceRuntimeSyncService` now refreshes the lifecycle power-state cache
through the fresh Go Agent protocol. It no longer calls the retired Python
Engine or the host-wide `vm-states` endpoint.
The scheduler first selects a bounded, actor-independent set of non-deleted
database instances. Every row must have a valid `host_id` and canonical VM UUID;
the pair is retained as the only target identity. Before making a host request,
PHP claims that runtime row's revision with `beginObservation()`. It then sends
one signed `domain_observe` operation through `CloudAgentObservationService` to
that exact host and UUID.
Only a succeeded operation whose returned UUID matches the requested UUID and
whose state is in the panel's bounded vocabulary may call `markObserved()` with
the claim revision. A transport timeout, uncertain operation, deterministic
Agent rejection, malformed response, unknown state, or UUID mismatch calls
`markUnavailable()` with the same revision. The last successful power state is
therefore retained; an unavailable observation is never interpreted as
`stopped`. A concurrent lifecycle/action writer can win the revision CAS, in
which case the stale poll is counted as skipped and cannot overwrite it.
The returned counters remain `observed`, `unavailable`, `skipped`, and a
bounded error-code list so existing lifecycle heartbeat diagnostics and page
contracts remain unchanged. Each Agent observation is journaled with its
operation ID; uncertain transport outcomes are not replayed by the scheduler.
They require the normal operation-status/reconcile path.
This is a narrow readback cutover, not approval to delete all Python code. VM
creation, reinstall/delete, network, settings, snapshots, firewall, rescue,
console, metrics, and remaining host administration still need typed Agent
operations and real-host acceptance before Python service units can be removed.

@ -6,24 +6,35 @@ use App\Models\CloudInstance;
use App\Models\CloudInstanceRuntime;
/**
* Refresh the instance runtime observation cache from host batch state.
* Refresh the instance runtime observation cache through the Go Agent.
*
* This service is intentionally separate from page rendering and mutation
* endpoints. A failed host request only marks its observations unavailable;
* it never changes the last known power state to "stopped".
* endpoints. Each request carries one explicit host/UUID identity to the
* typed Agent observation operation. A failed or uncertain Agent request
* only marks that cache row unavailable; it never changes the last known
* power state to "stopped".
*/
final class CloudInstanceRuntimeSyncService
{
private const ENGINE_TIMEOUT_SECONDS = 3.0;
/** @var \Closure():CloudAgentObservationService */
private \Closure $agentFactory;
public function __construct(private ?CloudEngineClient $engine = null)
{
$this->engine ??= new CloudEngineClient(null, self::ENGINE_TIMEOUT_SECONDS);
public function __construct(
private ?CloudAgentObservationService $agent = null,
?callable $agentFactory = null,
) {
$this->agentFactory = \Closure::fromCallable(
$agentFactory ?? static fn (): CloudAgentObservationService => new CloudAgentObservationService(
CloudAgentHostClientResolverFactory::fromConfig(),
new CloudAgentOperationRepository(),
),
);
}
/**
* Poll active instances in host batches. The returned counters contain
* no engine-controlled text and are suitable for heartbeat diagnostics.
* Poll active instances with bounded, explicit Agent observations. The
* returned counters contain no host-controlled text and are suitable for
* heartbeat diagnostics.
*/
public function sync(int $limit = 500): array
{
@ -44,7 +55,7 @@ final class CloudInstanceRuntimeSyncService
try {
// Select the least recently attempted observations first. This
// gives a bounded daemon pass a rotating cursor without storing a
// second global cursor: every successful or failed host batch
// second global cursor: every successful or failed Agent request
// advances last_attempt_at, so high instance IDs cannot starve
// behind a permanently busy low-ID prefix.
$instanceQuery = CloudInstance::leftJoin(
@ -54,7 +65,7 @@ final class CloudInstanceRuntimeSyncService
->orderByRaw('COALESCE(runtime.last_attempt_at, runtime.last_success_at, 0) ASC')
->orderBy('cloud_instances.id')
->limit(max(1, min(2000, $limit)));
// Production uses MariaDB, where filtering malformed legacy UUIDs
// Production uses MariaDB, where filtering malformed UUIDs
// at the database boundary prevents them from occupying every
// bounded pass. Keep the query usable in SQLite-based contract
// tests and local diagnostics; the application-level validation
@ -92,9 +103,9 @@ final class CloudInstanceRuntimeSyncService
'host_id' => $hostId,
'uuid' => $uuid,
];
// Legacy instances may predate the independent cache. Seeding a
// row does not claim a power state; the batch response still must
// provide the first successful observation.
// Instances may predate the independent cache. Seeding a row does
// not claim a power state; the Agent response must still provide
// the first successful observation.
try {
CloudInstanceRuntimeService::seed(
(int) $instance->id, $hostId, $uuid
@ -116,117 +127,92 @@ final class CloudInstanceRuntimeSyncService
private function syncHost(int $hostId, array $group, array $actor, array &$result): void
{
// Claim every row immediately before the host request. A successful
// worker operation advances the same revision, making this batch
// stale without allowing its eventual response to overwrite the
// worker's fresher state.
$active = [];
$revisions = [];
// Claim each row immediately before its Agent request. A successful
// worker operation advances the same revision, making the response
// stale without allowing it to overwrite fresher state.
foreach ($group as $row) {
$instanceId = (int) $row['id'];
$uuid = (string) $row['uuid'];
try {
$revisions[(int) $row['id']] = CloudInstanceRuntimeService::beginObservation(
(int) $row['id'], $hostId, (string) $row['uuid']
$revision = CloudInstanceRuntimeService::beginObservation(
$instanceId, $hostId, $uuid
);
$active[] = $row;
} catch (\Throwable $error) {
$result['skipped']++;
continue;
}
}
if ($active === []) {
return;
}
// Keep the host request bounded and pass the explicit UUID set to the
// engine. The old no-argument vm-states endpoint enumerated every
// domain and is reserved for legacy private UI compatibility only.
foreach (array_chunk($active, 100) as $chunk) {
$allowed = [];
foreach ($chunk as $row) {
$allowed[(string) $row['uuid']] = $row;
}
$query = rawurlencode(implode(',', array_keys($allowed)));
$chunkIds = array_map(
static fn (array $row): int => (int) $row['id'], $chunk
);
try {
$payload = $this->engine->host($hostId, 'GET',
'vm-states?ids=' . $query, $actor);
} catch (CloudError $error) {
$chunkRevisions = [];
foreach ($chunkIds as $id) {
if (array_key_exists($id, $revisions)) {
$chunkRevisions[$id] = $revisions[$id];
}
}
CloudInstanceRuntimeService::markUnavailable(
$chunkIds, $hostId, time(), self::errorCode($error), $chunkRevisions
);
$result['unavailable'] += count($chunkIds);
$result['errors'][] = 'runtime_status_sync_failed';
continue;
} catch (\Throwable $error) {
$chunkRevisions = [];
foreach ($chunkIds as $id) {
if (array_key_exists($id, $revisions)) {
$chunkRevisions[$id] = $revisions[$id];
}
}
CloudInstanceRuntimeService::markUnavailable(
$chunkIds, $hostId, time(), 'host_unavailable', $chunkRevisions
$observed = $this->agent()->observe($hostId, $uuid, $actor);
$record = is_array($observed['record'] ?? null)
? $observed['record'] : [];
$candidate = is_array($observed['observation'] ?? null)
? $observed['observation'] : [];
$candidateUuid = is_string($candidate['uuid'] ?? null)
? strtolower(trim($candidate['uuid'])) : '';
$state = CloudInstanceRuntimeService::normalizeState(
$candidate['state'] ?? null
);
$result['unavailable'] += count($chunkIds);
$result['errors'][] = 'runtime_status_sync_failed';
continue;
}
$seen = [];
$items = is_array($payload['items'] ?? null) ? $payload['items'] : [];
foreach ($items as $item) {
if (!is_array($item) || !is_string($item['uuid'] ?? null)) {
continue;
}
$uuid = strtolower(trim($item['uuid']));
if (!isset($allowed[$uuid])) {
if (($record['status'] ?? null) !== 'succeeded'
|| $candidateUuid === ''
|| !hash_equals($uuid, $candidateUuid)
|| $state === 'unknown') {
$this->markUnavailable(
$instanceId, $hostId, $revision, 'invalid_response', $result
);
continue;
}
$row = $allowed[$uuid];
$state = $item['state'] ?? ($item['power_state'] ?? null);
$seen[$uuid] = true;
try {
CloudInstanceRuntimeService::markObserved(
(int) $row['id'], $hostId, $uuid, $state, null,
$revisions[(int) $row['id']] ?? null
$instanceId, $hostId, $uuid, $state, null, $revision
);
if (CloudInstanceRuntimeService::normalizeState($state) !== 'unknown') {
$result['observed']++;
} else {
$result['unavailable']++;
}
$result['observed']++;
} catch (\Throwable $error) {
// A concurrent lifecycle writer may have won the CAS.
// Preserve its fresher value and do not turn that race
// into a host outage.
$result['skipped']++;
}
}
$missing = [];
$missingRevisions = [];
foreach ($allowed as $uuid => $row) {
if (!isset($seen[$uuid])) {
$missing[] = (int) $row['id'];
$id = (int) $row['id'];
if (array_key_exists($id, $revisions)) {
$missingRevisions[$id] = $revisions[$id];
}
}
}
if ($missing !== []) {
CloudInstanceRuntimeService::markUnavailable(
$missing, $hostId, time(), 'invalid_response', $missingRevisions
} catch (CloudAgentOperationException $error) {
$this->markUnavailable(
$instanceId, $hostId, $revision, self::agentErrorCode($error), $result
);
} catch (CloudError $error) {
$this->markUnavailable(
$instanceId, $hostId, $revision, self::agentErrorCode($error), $result
);
} catch (\Throwable $error) {
$this->markUnavailable(
$instanceId, $hostId, $revision, 'host_unavailable', $result
);
$result['unavailable'] += count($missing);
$result['errors'][] = 'runtime_status_sync_failed';
}
}
}
private function agent(): CloudAgentObservationService
{
return $this->agent ??= ($this->agentFactory)();
}
/** @param array{observed:int,unavailable:int,skipped:int,errors:list<string>} $result */
private function markUnavailable(
int $instanceId,
int $hostId,
int $revision,
string $errorCode,
array &$result,
): void {
try {
CloudInstanceRuntimeService::markUnavailable(
[$instanceId], $hostId, time(), $errorCode, [$instanceId => $revision]
);
} catch (\Throwable $error) {
// The observation row may have been removed or replaced while
// the Agent request was in flight. Either way, never recreate it.
}
$result['unavailable']++;
$result['errors'][] = 'runtime_status_sync_failed';
}
private function runtimeTableAvailable(): bool
{
try {
@ -237,9 +223,11 @@ final class CloudInstanceRuntimeSyncService
}
}
private static function errorCode(CloudError $error): string
private static function agentErrorCode(\Throwable $error): string
{
return in_array($error->httpStatus, [408, 429, 502, 503, 504], true)
? 'host_unavailable' : 'engine_error';
$status = $error instanceof CloudError || $error instanceof CloudAgentOperationException
? $error->httpStatus : 503;
return in_array($status, [408, 429, 502, 503, 504], true)
? 'host_unavailable' : 'invalid_response';
}
}

@ -0,0 +1,37 @@
<?php
// Static migration contract for the lifecycle runtime poller. The scheduler
// must use one typed, explicitly addressed Agent observation per VM; a batch
// inventory response or the retired Python/SSH transport would reintroduce a
// host-wide authority boundary and cannot preserve per-row CAS fencing.
$root = dirname(__DIR__);
$assert = static function ($condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$service = file_get_contents($root . '/app/Services/Cloud/CloudInstanceRuntimeSyncService.php');
$observation = file_get_contents($root . '/app/Services/Cloud/CloudAgentObservationService.php');
$assert(is_string($service) && is_string($observation), 'Runtime sync sources are unreadable.');
$assert(str_contains($service, 'CloudAgentObservationService')
&& str_contains($service, 'CloudAgentHostClientResolverFactory::fromConfig()')
&& str_contains($service, '->observe($hostId, $uuid, $actor)')
&& str_contains($service, 'CloudInstanceRuntimeService::beginObservation(')
&& str_contains($service, 'CloudInstanceRuntimeService::markObserved(')
&& str_contains($service, 'CloudInstanceRuntimeService::markUnavailable(')
&& str_contains($service, 'hash_equals($uuid, $candidateUuid)')
&& !str_contains($service, 'CloudEngineClient')
&& !str_contains($service, 'vm-states')
&& !str_contains($service, 'shell_exec')
&& !str_contains($service, 'ssh'),
'Runtime sync must use typed Agent observations and no Python/SSH or host-wide vm-states fallback.');
$assert(str_contains($observation, 'resolveReadOnly($hostId)')
&& str_contains($observation, "'domain_observe'")
&& str_contains($observation, 'markUncertain')
&& str_contains($observation, 'observeMany'),
'Agent observation service must keep read-only resolver, typed operation and uncertain journal semantics.');
echo "Cloud Agent runtime sync contract passed.\n";
Loading…
Cancel
Save