feat: cache VM runtime status for instant list rendering
parent
b2c19d18bb
commit
cf2a4e3ec4
@ -0,0 +1,21 @@
|
||||
<?php
|
||||
|
||||
namespace App\Models;
|
||||
|
||||
final class CloudInstanceRuntime extends Model
|
||||
{
|
||||
protected $connection = 'default';
|
||||
protected $table = 'cloud_instance_runtime';
|
||||
protected $primaryKey = 'instance_id';
|
||||
public $incrementing = false;
|
||||
protected $keyType = 'int';
|
||||
protected $casts = [
|
||||
'instance_id' => 'integer',
|
||||
'host_id' => 'integer',
|
||||
'last_success_at' => 'integer',
|
||||
'last_attempt_at' => 'integer',
|
||||
'revision' => 'integer',
|
||||
'created_at' => 'integer',
|
||||
'updated_at' => 'integer',
|
||||
];
|
||||
}
|
||||
@ -0,0 +1,441 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Cloud;
|
||||
|
||||
use App\Models\CloudInstanceRuntime;
|
||||
|
||||
/**
|
||||
* Database-backed last-known VM power observations.
|
||||
*
|
||||
* This service is deliberately independent of the engine client. A worker
|
||||
* records a successful batch observation with markObserved(), or records a
|
||||
* host-level failure with markUnavailable(). Page services can then render a
|
||||
* bounded first paint from forIds() without treating a stale value as live.
|
||||
*/
|
||||
final class CloudInstanceRuntimeService
|
||||
{
|
||||
public const FRESHNESS_SECONDS = 30;
|
||||
|
||||
private const ERROR_CODES = [
|
||||
'host_unavailable', 'engine_timeout', 'engine_error', 'invalid_response',
|
||||
];
|
||||
|
||||
/** Create the initial observation row after provisioning succeeds. */
|
||||
public static function seed(int $instanceId, int $hostId, string $uuid,
|
||||
?int $now = null): CloudInstanceRuntime
|
||||
{
|
||||
$now = self::now($now);
|
||||
[$instanceId, $hostId, $uuid] = self::identity($instanceId, $hostId, $uuid);
|
||||
$runtime = CloudInstanceRuntime::where('instance_id', $instanceId)->first();
|
||||
if ($runtime === null) {
|
||||
$runtime = new CloudInstanceRuntime();
|
||||
$runtime->instance_id = $instanceId;
|
||||
$runtime->host_id = $hostId;
|
||||
$runtime->uuid = $uuid;
|
||||
$runtime->power_state = 'unknown';
|
||||
$runtime->sync_state = 'never';
|
||||
$runtime->last_success_at = 0;
|
||||
$runtime->last_attempt_at = 0;
|
||||
$runtime->revision = 0;
|
||||
$runtime->last_error_code = '';
|
||||
$runtime->created_at = $now;
|
||||
$runtime->updated_at = $now;
|
||||
$runtime->save();
|
||||
return $runtime;
|
||||
}
|
||||
|
||||
if (!self::sameIdentity($runtime, $hostId, $uuid)) {
|
||||
self::reset($runtime, $hostId, $uuid, $now);
|
||||
$runtime->save();
|
||||
return $runtime;
|
||||
}
|
||||
return $runtime;
|
||||
}
|
||||
|
||||
/**
|
||||
* Save one trusted engine observation. Unknown engine states are never
|
||||
* promoted to running or stopped; they leave the cached state unchanged.
|
||||
*/
|
||||
public static function markObserved(int $instanceId, int $hostId, string $uuid,
|
||||
$powerState, ?int $now = null, ?int $revision = null): CloudInstanceRuntime
|
||||
{
|
||||
$now = self::now($now);
|
||||
[$instanceId, $hostId, $uuid] = self::identity($instanceId, $hostId, $uuid);
|
||||
$state = self::normalizeState($powerState);
|
||||
// Resolve the identity before writing, but apply the write through a
|
||||
// conditional revision update. A worker write (which omits the
|
||||
// token) advances the revision and therefore invalidates any in-flight
|
||||
// background poll carrying an older token.
|
||||
// A tokenized background response must never recreate a row that a
|
||||
// completed delete already removed. Immediate worker writes may seed
|
||||
// a legacy/missing row, but only when no batch token is present.
|
||||
self::identityRow($instanceId, $hostId, $uuid, $now, $revision === null);
|
||||
$values = [
|
||||
'last_attempt_at' => $now,
|
||||
'updated_at' => $now,
|
||||
];
|
||||
if ($state !== 'unknown') {
|
||||
$values += [
|
||||
'power_state' => $state,
|
||||
'last_success_at' => $now,
|
||||
'sync_state' => 'ok',
|
||||
'last_error_code' => '',
|
||||
];
|
||||
} else {
|
||||
// A malformed or unsupported engine item is a failed observation;
|
||||
// retaining the previous state avoids falsely reporting shutdown.
|
||||
$values += [
|
||||
'sync_state' => 'unavailable',
|
||||
'last_error_code' => 'invalid_response',
|
||||
];
|
||||
}
|
||||
return self::writeObservation(
|
||||
$instanceId, $hostId, $uuid, $values, $revision
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Claim one host-batch item before issuing vm-states. The returned token
|
||||
* is valid only until another writer advances the row revision.
|
||||
*/
|
||||
public static function beginObservation(int $instanceId, int $hostId, string $uuid,
|
||||
?int $now = null): int
|
||||
{
|
||||
$now = self::now($now);
|
||||
[$instanceId, $hostId, $uuid] = self::identity($instanceId, $hostId, $uuid);
|
||||
for ($attempt = 0; $attempt < 3; $attempt++) {
|
||||
// The sync pass seeds rows before claiming them. Do not recreate
|
||||
// one that a concurrent delete has already forgotten.
|
||||
$runtime = self::identityRow($instanceId, $hostId, $uuid, $now, false);
|
||||
$current = max(0, (int) $runtime->revision);
|
||||
$next = $current + 1;
|
||||
$updated = CloudInstanceRuntime::where('instance_id', $instanceId)
|
||||
->where('host_id', $hostId)
|
||||
->where('uuid', $uuid)
|
||||
->where('revision', $current)
|
||||
->increment('revision', 1, [
|
||||
'last_attempt_at' => $now,
|
||||
'updated_at' => $now,
|
||||
]);
|
||||
if ((int) $updated === 1) {
|
||||
return $next;
|
||||
}
|
||||
}
|
||||
throw new \RuntimeException('Runtime observation token unavailable');
|
||||
}
|
||||
|
||||
/** Mark a host batch unavailable while preserving its last good state. */
|
||||
public static function markUnavailable(array $instanceIds, ?int $hostId = null,
|
||||
?int $now = null, string $errorCode = 'host_unavailable', array $revisions = []): int
|
||||
{
|
||||
$now = self::now($now);
|
||||
$ids = self::ids($instanceIds);
|
||||
if ($ids === []) {
|
||||
return 0;
|
||||
}
|
||||
$errorCode = self::errorCode($errorCode);
|
||||
if ($hostId !== null) {
|
||||
if ($hostId < 1) {
|
||||
throw new \InvalidArgumentException('Host ID must be positive');
|
||||
}
|
||||
}
|
||||
$values = [
|
||||
'sync_state' => 'unavailable',
|
||||
'last_attempt_at' => $now,
|
||||
'last_error_code' => $errorCode,
|
||||
'updated_at' => $now,
|
||||
];
|
||||
// Tokenized writes are intentionally per-row: each row may have a
|
||||
// different claim revision. A stale token simply affects zero rows.
|
||||
if ($revisions !== []) {
|
||||
$updated = 0;
|
||||
foreach ($ids as $id) {
|
||||
if (!array_key_exists($id, $revisions)) {
|
||||
continue;
|
||||
}
|
||||
$query = CloudInstanceRuntime::where('instance_id', $id);
|
||||
if ($hostId !== null) {
|
||||
$query->where('host_id', $hostId);
|
||||
}
|
||||
$query->where('revision', (int) $revisions[$id]);
|
||||
$updated += (int) $query->increment('revision', 1, $values);
|
||||
}
|
||||
return $updated;
|
||||
}
|
||||
$query = CloudInstanceRuntime::whereIn('instance_id', $ids);
|
||||
if ($hostId !== null) {
|
||||
$query->where('host_id', $hostId);
|
||||
}
|
||||
return (int) $query->increment('revision', 1, $values);
|
||||
}
|
||||
|
||||
/** Remove an observation when its panel instance is irreversibly deleted. */
|
||||
public static function forget(int $instanceId): int
|
||||
{
|
||||
if ($instanceId < 1) {
|
||||
return 0;
|
||||
}
|
||||
return (int) CloudInstanceRuntime::where('instance_id', $instanceId)->delete();
|
||||
}
|
||||
|
||||
/**
|
||||
* Return rows keyed by instance ID. Missing rows are represented as a
|
||||
* pending observation, allowing callers to render deployment/syncing
|
||||
* without inventing a power state.
|
||||
*/
|
||||
public static function forIds(array $instanceIds, ?int $now = null): array
|
||||
{
|
||||
$now = self::now($now);
|
||||
$ids = self::ids($instanceIds);
|
||||
if ($ids === []) {
|
||||
return [];
|
||||
}
|
||||
$rows = [];
|
||||
foreach (CloudInstanceRuntime::whereIn('instance_id', $ids)->get() as $runtime) {
|
||||
$rows[(int) $runtime->instance_id] = self::view($runtime, $now);
|
||||
}
|
||||
foreach ($ids as $id) {
|
||||
if (!isset($rows[$id])) {
|
||||
$rows[$id] = self::pendingView($id, $now);
|
||||
}
|
||||
}
|
||||
// Preserve the caller's de-duplicated ID order for predictable JSON.
|
||||
$ordered = [];
|
||||
foreach ($ids as $id) {
|
||||
$ordered[(string) $id] = $rows[$id];
|
||||
}
|
||||
return $ordered;
|
||||
}
|
||||
|
||||
/** Return observations for an iterable of CloudInstance-like records. */
|
||||
public static function forInstances(iterable $instances, ?int $now = null): array
|
||||
{
|
||||
$items = [];
|
||||
foreach ($instances as $instance) {
|
||||
$id = self::value($instance, 'id');
|
||||
if ($id === null) {
|
||||
$id = self::value($instance, 'instance_id');
|
||||
}
|
||||
$id = filter_var($id, FILTER_VALIDATE_INT,
|
||||
['options' => ['min_range' => 1]]);
|
||||
if ($id === false) {
|
||||
continue;
|
||||
}
|
||||
$items[(int) $id] = [
|
||||
'host_id' => self::value($instance, 'host_id'),
|
||||
'uuid' => self::value($instance, 'uuid'),
|
||||
];
|
||||
}
|
||||
$views = self::forIds(array_keys($items), $now);
|
||||
foreach ($items as $id => $identity) {
|
||||
$currentHost = max(0, (int) $identity['host_id']);
|
||||
$currentUuid = is_string($identity['uuid'])
|
||||
? strtolower(trim($identity['uuid'])) : '';
|
||||
$cachedHost = (int) ($views[(string) $id]['host_id'] ?? 0);
|
||||
$cachedUuid = strtolower(trim((string) ($views[(string) $id]['uuid'] ?? '')));
|
||||
// A reused instance row must never inherit the previous VM's
|
||||
// power state. Treat a partial or mismatched identity as a fresh
|
||||
// pending observation and expose only the current DB identity.
|
||||
$identityMismatch = ($cachedHost !== 0 || $cachedUuid !== '')
|
||||
&& ($cachedHost !== $currentHost || $cachedUuid !== $currentUuid);
|
||||
if ($identityMismatch || $currentHost < 1 || !self::isUuid($currentUuid)) {
|
||||
$views[(string) $id] = self::pendingView((int) $id, self::now($now));
|
||||
}
|
||||
$views[(string) $id]['host_id'] = $currentHost;
|
||||
$views[(string) $id]['uuid'] = $currentUuid;
|
||||
}
|
||||
return $views;
|
||||
}
|
||||
|
||||
/** Normalize engine values to the panel's bounded state vocabulary. */
|
||||
public static function normalizeState($value): string
|
||||
{
|
||||
$state = CloudInstanceMetadata::state($value);
|
||||
return match ($state) {
|
||||
'shutoff', 'shut_off', 'stopped' => 'stopped',
|
||||
'running' => 'running',
|
||||
'paused' => 'paused',
|
||||
'crashed' => 'crashed',
|
||||
'pmsuspended' => 'pmsuspended',
|
||||
'blocked' => 'blocked',
|
||||
default => 'unknown',
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply a cache write while atomically advancing its revision. When a
|
||||
* token is supplied, the row must still carry exactly that revision;
|
||||
* otherwise this is a worker/immediate write that supersedes any poll.
|
||||
*/
|
||||
private static function writeObservation(int $instanceId, int $hostId, string $uuid,
|
||||
array $values, ?int $revision): CloudInstanceRuntime
|
||||
{
|
||||
$query = CloudInstanceRuntime::where('instance_id', $instanceId)
|
||||
->where('host_id', $hostId)
|
||||
->where('uuid', $uuid);
|
||||
if ($revision !== null) {
|
||||
$query->where('revision', max(0, $revision));
|
||||
}
|
||||
$updated = $query->increment('revision', 1, $values);
|
||||
$runtime = CloudInstanceRuntime::where('instance_id', $instanceId)->first();
|
||||
if ($runtime !== null) {
|
||||
// A zero-row conditional update means a newer writer won the
|
||||
// race. Return that current row without overwriting it.
|
||||
return $runtime;
|
||||
}
|
||||
// The row can disappear during an irreversible delete race. Do not
|
||||
// recreate a cache row for a VM that was just forgotten; return an
|
||||
// ephemeral identity-only value to preserve the historical contract.
|
||||
$runtime = new CloudInstanceRuntime();
|
||||
$runtime->instance_id = $instanceId;
|
||||
$runtime->host_id = $hostId;
|
||||
$runtime->uuid = $uuid;
|
||||
return $runtime;
|
||||
}
|
||||
|
||||
private static function identityRow(int $instanceId, int $hostId, string $uuid,
|
||||
int $now, bool $create = true): CloudInstanceRuntime
|
||||
{
|
||||
$runtime = CloudInstanceRuntime::where('instance_id', $instanceId)->first();
|
||||
if ($runtime === null) {
|
||||
if (!$create) {
|
||||
throw new \RuntimeException('Runtime observation was removed');
|
||||
}
|
||||
return self::seed($instanceId, $hostId, $uuid, $now);
|
||||
}
|
||||
if (!self::sameIdentity($runtime, $hostId, $uuid)) {
|
||||
throw new \RuntimeException('Runtime observation identity mismatch');
|
||||
}
|
||||
return $runtime;
|
||||
}
|
||||
|
||||
private static function reset(CloudInstanceRuntime $runtime, int $hostId,
|
||||
string $uuid, int $now): void
|
||||
{
|
||||
$runtime->host_id = $hostId;
|
||||
$runtime->uuid = $uuid;
|
||||
$runtime->power_state = 'unknown';
|
||||
$runtime->sync_state = 'never';
|
||||
$runtime->last_success_at = 0;
|
||||
$runtime->last_attempt_at = 0;
|
||||
// Bump the generation when an instance ID is reused, invalidating any
|
||||
// response still in flight for the previous host/UUID identity.
|
||||
$runtime->revision = max(0, (int) $runtime->revision) + 1;
|
||||
$runtime->last_error_code = '';
|
||||
$runtime->updated_at = $now;
|
||||
}
|
||||
|
||||
private static function view(CloudInstanceRuntime $runtime, int $now): array
|
||||
{
|
||||
$success = max(0, (int) $runtime->last_success_at);
|
||||
$age = $success > 0 ? max(0, $now - $success) : null;
|
||||
$future = $success > $now;
|
||||
$fresh = !$future && $success > 0 && $age !== null && $age <= self::FRESHNESS_SECONDS
|
||||
&& (string) $runtime->sync_state === 'ok';
|
||||
$sync = match ((string) $runtime->sync_state) {
|
||||
'unavailable' => 'unavailable',
|
||||
'never' => 'pending',
|
||||
default => $success === 0 || $future ? 'pending' : ($fresh ? 'fresh' : 'syncing'),
|
||||
};
|
||||
return [
|
||||
'instance_id' => (string) $runtime->instance_id,
|
||||
'host_id' => (int) $runtime->host_id,
|
||||
'uuid' => (string) $runtime->uuid,
|
||||
'state' => self::normalizeState($runtime->power_state),
|
||||
'power_state' => self::normalizeState($runtime->power_state),
|
||||
'fresh' => $fresh,
|
||||
'state_fresh' => $fresh,
|
||||
'freshness' => $success === 0 || $future ? 'never' : ($fresh ? 'fresh' : 'stale'),
|
||||
'sync_status' => $sync,
|
||||
'state_sync_status' => $sync === 'unavailable' ? 'offline' : $sync,
|
||||
'last_success_at' => $success > 0 ? $success : null,
|
||||
'state_observed_at' => $success > 0 && !$future ? $success : null,
|
||||
'last_attempt_at' => max(0, (int) $runtime->last_attempt_at) ?: null,
|
||||
'age_seconds' => $age,
|
||||
'state_label_hint' => $sync === 'unavailable'
|
||||
? '暂时无法同步' : ($success === 0 || $future ? '等待宿主机同步' : ''),
|
||||
];
|
||||
}
|
||||
|
||||
private static function pendingView(int $instanceId, int $now): array
|
||||
{
|
||||
return [
|
||||
'instance_id' => (string) $instanceId,
|
||||
'host_id' => 0,
|
||||
'uuid' => '',
|
||||
'state' => 'unknown',
|
||||
'power_state' => 'unknown',
|
||||
'fresh' => false,
|
||||
'state_fresh' => false,
|
||||
'freshness' => 'never',
|
||||
'sync_status' => 'pending',
|
||||
'state_sync_status' => 'pending',
|
||||
'last_success_at' => null,
|
||||
'state_observed_at' => null,
|
||||
'last_attempt_at' => null,
|
||||
'age_seconds' => null,
|
||||
'state_label_hint' => '等待宿主机同步',
|
||||
];
|
||||
}
|
||||
|
||||
private static function sameIdentity(CloudInstanceRuntime $runtime, int $hostId,
|
||||
string $uuid): bool
|
||||
{
|
||||
return (int) $runtime->host_id === $hostId
|
||||
&& hash_equals(strtolower((string) $runtime->uuid), $uuid);
|
||||
}
|
||||
|
||||
private static function identity(int $instanceId, int $hostId, string $uuid): array
|
||||
{
|
||||
if ($instanceId < 1 || $hostId < 1) {
|
||||
throw new \InvalidArgumentException('Instance and host IDs must be positive');
|
||||
}
|
||||
$uuid = strtolower(trim($uuid));
|
||||
// Libvirt accepts operator-supplied UUIDs whose version/variant
|
||||
// nibbles are not RFC 4122 values. Match the existing panel identity
|
||||
// boundary and require only the canonical five-group shape.
|
||||
if (!self::isUuid($uuid)) {
|
||||
throw new \InvalidArgumentException('Invalid runtime UUID');
|
||||
}
|
||||
return [$instanceId, $hostId, $uuid];
|
||||
}
|
||||
|
||||
private static function isUuid(string $uuid): bool
|
||||
{
|
||||
return preg_match(
|
||||
'/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/D',
|
||||
strtolower(trim($uuid))
|
||||
) === 1;
|
||||
}
|
||||
|
||||
private static function ids(array $values): array
|
||||
{
|
||||
$ids = [];
|
||||
foreach ($values as $value) {
|
||||
$id = filter_var($value, FILTER_VALIDATE_INT,
|
||||
['options' => ['min_range' => 1]]);
|
||||
if ($id !== false) {
|
||||
$ids[(int) $id] = true;
|
||||
}
|
||||
}
|
||||
return array_keys($ids);
|
||||
}
|
||||
|
||||
private static function value($instance, string $key)
|
||||
{
|
||||
if (is_array($instance)) {
|
||||
return $instance[$key] ?? null;
|
||||
}
|
||||
return is_object($instance) ? ($instance->{$key} ?? null) : null;
|
||||
}
|
||||
|
||||
private static function errorCode(string $value): string
|
||||
{
|
||||
return in_array($value, self::ERROR_CODES, true) ? $value : 'host_unavailable';
|
||||
}
|
||||
|
||||
private static function now(?int $now): int
|
||||
{
|
||||
return max(1, $now ?? time());
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,259 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Cloud;
|
||||
|
||||
use App\Models\CloudInstance;
|
||||
use App\Models\CloudInstanceRuntime;
|
||||
use App\Models\User;
|
||||
use App\Services\Config;
|
||||
|
||||
/**
|
||||
* Refresh the instance runtime observation cache from host batch state.
|
||||
*
|
||||
* This service is intentionally separate from page rendering and mutation
|
||||
* endpoints. A failed host request only marks its observations unavailable;
|
||||
* it never changes the last known power state to "stopped".
|
||||
*/
|
||||
final class CloudInstanceRuntimeSyncService
|
||||
{
|
||||
private const ENGINE_TIMEOUT_SECONDS = 3.0;
|
||||
|
||||
public function __construct(private ?CloudEngineClient $engine = null)
|
||||
{
|
||||
$this->engine ??= new CloudEngineClient(null, self::ENGINE_TIMEOUT_SECONDS);
|
||||
}
|
||||
|
||||
/**
|
||||
* Poll active instances in host batches. The returned counters contain
|
||||
* no engine-controlled text and are suitable for heartbeat diagnostics.
|
||||
*/
|
||||
public function sync(int $limit = 500): array
|
||||
{
|
||||
$result = ['observed' => 0, 'unavailable' => 0, 'skipped' => 0, 'errors' => []];
|
||||
if (!$this->runtimeTableAvailable()) {
|
||||
// Rolling upgrades may start the daemon before cloudInit has run.
|
||||
// Keep billing/lifecycle work healthy until the table is present.
|
||||
$result['skipped'] = 1;
|
||||
return $result;
|
||||
}
|
||||
|
||||
$actor = $this->systemActor();
|
||||
if ($actor === null) {
|
||||
$result['errors'][] = 'runtime_status_sync_unconfigured';
|
||||
return $result;
|
||||
}
|
||||
|
||||
try {
|
||||
// Select the least recently attempted observations first. This
|
||||
// gives a bounded daemon pass a rotating cursor without storing a
|
||||
// second global cursor: every successful or failed host batch
|
||||
// advances last_attempt_at, so high instance IDs cannot starve
|
||||
// behind a permanently busy low-ID prefix.
|
||||
$instanceQuery = CloudInstance::leftJoin(
|
||||
'cloud_instance_runtime as runtime',
|
||||
'runtime.instance_id', '=', 'cloud_instances.id'
|
||||
)->where('cloud_instances.status', '<>', 'deleted')
|
||||
->orderByRaw('COALESCE(runtime.last_attempt_at, runtime.last_success_at, 0) ASC')
|
||||
->orderBy('cloud_instances.id')
|
||||
->limit(max(1, min(2000, $limit)));
|
||||
// Production uses MariaDB, where filtering malformed legacy UUIDs
|
||||
// at the database boundary prevents them from occupying every
|
||||
// bounded pass. Keep the query usable in SQLite-based contract
|
||||
// tests and local diagnostics; the application-level validation
|
||||
// below remains the final guard in either driver.
|
||||
if ((new CloudInstance())->getConnection()->getDriverName() === 'mysql') {
|
||||
$instanceQuery->whereRaw(
|
||||
"cloud_instances.uuid REGEXP ?",
|
||||
['^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$']
|
||||
);
|
||||
}
|
||||
$instances = $instanceQuery->get([
|
||||
'cloud_instances.id', 'cloud_instances.host_id', 'cloud_instances.uuid',
|
||||
]);
|
||||
} catch (\Throwable $error) {
|
||||
$result['errors'][] = 'runtime_status_sync_failed';
|
||||
return $result;
|
||||
}
|
||||
if ($instances->isEmpty()) {
|
||||
return $result;
|
||||
}
|
||||
|
||||
$groups = [];
|
||||
foreach ($instances as $instance) {
|
||||
$hostId = (int) $instance->host_id;
|
||||
$uuid = strtolower(trim((string) $instance->uuid));
|
||||
if ($hostId < 1 || preg_match(
|
||||
'/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/D',
|
||||
$uuid
|
||||
) !== 1) {
|
||||
$result['skipped']++;
|
||||
continue;
|
||||
}
|
||||
$groups[$hostId][] = [
|
||||
'id' => (int) $instance->id,
|
||||
'host_id' => $hostId,
|
||||
'uuid' => $uuid,
|
||||
];
|
||||
// Legacy instances may predate the independent cache. Seeding a
|
||||
// row does not claim a power state; the batch response still must
|
||||
// provide the first successful observation.
|
||||
try {
|
||||
CloudInstanceRuntimeService::seed(
|
||||
(int) $instance->id, $hostId, $uuid
|
||||
);
|
||||
} catch (\Throwable $error) {
|
||||
$result['skipped']++;
|
||||
// Do not leave a row that failed identity seeding in the
|
||||
// host group; otherwise syncHost() would claim it again and
|
||||
// count the same failure twice in one pass.
|
||||
array_pop($groups[$hostId]);
|
||||
}
|
||||
}
|
||||
|
||||
foreach ($groups as $hostId => $group) {
|
||||
$this->syncHost((int) $hostId, $group, $actor, $result);
|
||||
}
|
||||
return $result;
|
||||
}
|
||||
|
||||
private function syncHost(int $hostId, array $group, array $actor, array &$result): void
|
||||
{
|
||||
// Claim every row immediately before the host request. A successful
|
||||
// worker operation advances the same revision, making this batch
|
||||
// stale without allowing its eventual response to overwrite the
|
||||
// worker's fresher state.
|
||||
$active = [];
|
||||
$revisions = [];
|
||||
foreach ($group as $row) {
|
||||
try {
|
||||
$revisions[(int) $row['id']] = CloudInstanceRuntimeService::beginObservation(
|
||||
(int) $row['id'], $hostId, (string) $row['uuid']
|
||||
);
|
||||
$active[] = $row;
|
||||
} catch (\Throwable $error) {
|
||||
$result['skipped']++;
|
||||
}
|
||||
}
|
||||
if ($active === []) {
|
||||
return;
|
||||
}
|
||||
$group = $active;
|
||||
$ids = array_map(static fn (array $row): int => (int) $row['id'], $group);
|
||||
$allowed = [];
|
||||
foreach ($group as $row) {
|
||||
$allowed[(string) $row['uuid']] = $row;
|
||||
}
|
||||
try {
|
||||
$payload = $this->engine->host($hostId, 'GET', 'vm-states', $actor);
|
||||
} catch (CloudError $error) {
|
||||
CloudInstanceRuntimeService::markUnavailable(
|
||||
$ids, $hostId, time(), self::errorCode($error), $revisions
|
||||
);
|
||||
$result['unavailable'] += count($ids);
|
||||
$result['errors'][] = 'runtime_status_sync_failed';
|
||||
return;
|
||||
} catch (\Throwable $error) {
|
||||
CloudInstanceRuntimeService::markUnavailable(
|
||||
$ids, $hostId, time(), 'host_unavailable', $revisions
|
||||
);
|
||||
$result['unavailable'] += count($ids);
|
||||
$result['errors'][] = 'runtime_status_sync_failed';
|
||||
return;
|
||||
}
|
||||
|
||||
$seen = [];
|
||||
$items = is_array($payload['items'] ?? null) ? $payload['items'] : [];
|
||||
foreach ($items as $item) {
|
||||
if (!is_array($item) || !is_string($item['uuid'] ?? null)) {
|
||||
continue;
|
||||
}
|
||||
$uuid = strtolower(trim($item['uuid']));
|
||||
if (!isset($allowed[$uuid])) {
|
||||
continue;
|
||||
}
|
||||
$row = $allowed[$uuid];
|
||||
$state = $item['state'] ?? ($item['power_state'] ?? null);
|
||||
$seen[$uuid] = true;
|
||||
try {
|
||||
CloudInstanceRuntimeService::markObserved(
|
||||
(int) $row['id'], $hostId, $uuid, $state, null,
|
||||
$revisions[(int) $row['id']] ?? null
|
||||
);
|
||||
if (CloudInstanceRuntimeService::normalizeState($state) !== 'unknown') {
|
||||
$result['observed']++;
|
||||
} else {
|
||||
$result['unavailable']++;
|
||||
}
|
||||
} catch (\Throwable $error) {
|
||||
$result['skipped']++;
|
||||
}
|
||||
}
|
||||
|
||||
$missing = [];
|
||||
foreach ($allowed as $uuid => $row) {
|
||||
if (!isset($seen[$uuid])) {
|
||||
$missing[] = (int) $row['id'];
|
||||
}
|
||||
}
|
||||
if ($missing !== []) {
|
||||
$missingRevisions = [];
|
||||
foreach ($missing as $missingId) {
|
||||
if (array_key_exists($missingId, $revisions)) {
|
||||
$missingRevisions[$missingId] = $revisions[$missingId];
|
||||
}
|
||||
}
|
||||
CloudInstanceRuntimeService::markUnavailable(
|
||||
$missing, $hostId, time(), 'invalid_response', $missingRevisions
|
||||
);
|
||||
$result['unavailable'] += count($missing);
|
||||
$result['errors'][] = 'runtime_status_sync_failed';
|
||||
}
|
||||
}
|
||||
|
||||
private function systemActor(): ?array
|
||||
{
|
||||
$ids = [];
|
||||
foreach (explode(',', (string) Config::get('super_admin', '')) as $value) {
|
||||
$id = filter_var(trim($value), FILTER_VALIDATE_INT,
|
||||
['options' => ['min_range' => 1]]);
|
||||
if ($id !== false) {
|
||||
$ids[(int) $id] = true;
|
||||
}
|
||||
}
|
||||
if ($ids === []) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
$user = User::whereIn('id', array_keys($ids))
|
||||
->where('enable', 1)->where('is_admin', 1)->first(['id']);
|
||||
} catch (\Throwable $error) {
|
||||
return null;
|
||||
}
|
||||
if ($user === null || (int) $user->id < 1) {
|
||||
return null;
|
||||
}
|
||||
return [
|
||||
'user_id' => (int) $user->id,
|
||||
'is_admin' => true,
|
||||
'is_super_admin' => true,
|
||||
'can_manage' => true,
|
||||
'session_id' => 'cloud-runtime-sync',
|
||||
];
|
||||
}
|
||||
|
||||
private function runtimeTableAvailable(): bool
|
||||
{
|
||||
try {
|
||||
return (new CloudInstanceRuntime())->getConnection()
|
||||
->getSchemaBuilder()->hasTable('cloud_instance_runtime');
|
||||
} catch (\Throwable $error) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private static function errorCode(CloudError $error): string
|
||||
{
|
||||
return in_array($error->httpStatus, [408, 429, 502, 503, 504], true)
|
||||
? 'host_unavailable' : 'engine_error';
|
||||
}
|
||||
}
|
||||
@ -1,2 +1,2 @@
|
||||
{* Shared Venix CSS/JS cache key. Increment whenever a shared browser asset changes. *}
|
||||
{assign var=venix_asset_version value='2026100205' scope='global'}
|
||||
{assign var=venix_asset_version value='2026100206' scope='global'}
|
||||
|
||||
Loading…
Reference in New Issue