diff --git a/docs/VALIDATION.md b/docs/VALIDATION.md index 54aee6cc..2b499704 100644 --- a/docs/VALIDATION.md +++ b/docs/VALIDATION.md @@ -208,3 +208,21 @@ authorized host mutation test: 待部署后在用户指定的测试环境按 ope browser/payment/SMTP test: 自动化 PHP/Node 契约测试通过;真实创建付款需在最终提交动作前取得用户即时确认 known skips or risks: libvirt 调用已执行但结果不可证明时会安全保留 uncertain,必须复用原 operation_id 核对,禁止自动重放;macOS libvirt 链接仅有重复库警告 ``` + +### 2026-10-16 持久化目录任务收敛验收 + +```text +commit: 本记录所在 go 分支提交(基线 e7fa625) +Go version / libvirt version / PHP version / MariaDB version: Go 1.27.1 / 本机动态链接测试沿用已安装 libvirt / PHP 8.5.11 / MariaDB 未在本机运行 +go vet: Agent 默认及 libvirt tag、Gateway 均通过 +go test: Agent 默认及 libvirt tag、Gateway 均通过;两个 Go 模块的 mod verify 均通过 +go test -race: Agent libvirt tag 与 Gateway 均通过;Gateway race 测试获准绑定本机临时回环端口 +Linux AMD64 build: 本提交未修改 Go 源码,继续使用 e7fa625 对应的已验证 Agent 与 Gateway 二进制 +PHP tests: 392 个 PHP 文件逐文件 lint 通过,119 个测试文件通过;SourceSyntax 独立检查 396 个文件通过;仅输出既有测试 warning 与第三方依赖 deprecation,退出码为 0 +Node tests: 89 个 JavaScript 文件语法检查通过,4 个行为测试通过;部署 JSON 示例有效 +systemd-analyze: 本提交未修改 service 文件,未重复执行 +read-only host capability check: HKG-A 出站 WSS 已稳定连接;Agent 报告 14 项必要能力、UEFI 可用且 Secure Boot 关闭 +authorized host mutation test: HTTPS Debian 13 云镜像导入在 Agent 端确定成功,436207616 字节,SHA-256 为 1010eb3215be632548cfb6793b53bd8243064d8820b5106f2becfcd179099b2f;本提交新增后台同 operation_id 收敛,禁止重放原始导入 POST +browser/payment/SMTP test: 管理端网络选择器已由错误的 bridge:br0 修正为 Agent 声明的 public;旧创建任务已按原 operation_id 核对并明确失败退款;新的真实 10 USD 创建提交仍等待用户即时确认 +known skips or risks: worker 每 tick 最多处理一个普通任务和一个目录核对;真实创建、重装、快照恢复和删除属于外部或破坏性操作,必须在动作前取得确认后再完成联调 +``` diff --git a/web/app/Services/Cloud/CloudRescueTemplateImportService.php b/web/app/Services/Cloud/CloudRescueTemplateImportService.php index 7d12ec37..3e28cb3b 100644 --- a/web/app/Services/Cloud/CloudRescueTemplateImportService.php +++ b/web/app/Services/Cloud/CloudRescueTemplateImportService.php @@ -10,6 +10,8 @@ use App\Models\CloudRescueTemplateImport; */ final class CloudRescueTemplateImportService { + private const IMPORT_POLL_INTERVAL_SECONDS = 5; + public static function start( int $hostId, string $importType, @@ -86,6 +88,101 @@ final class CloudRescueTemplateImportService return self::pollIfAllowed($record, $actor); } + /** + * Converge one already-dispatched rescue import without replaying it. + * The persisted operation ID is the only capability available here. + */ + public static function reconcileOnePendingImport(): bool + { + $connection = (new CloudRescueTemplateImport())->getConnection(); + if (!$connection->getSchemaBuilder()->hasTable( + 'cloud_rescue_template_imports' + )) { + return false; + } + + $now = time(); + $record = CloudRescueTemplateImport::query() + ->where(function ($query) use ($now): void { + $query->where(function ($running) use ($now): void { + $running->where('state', 'running') + ->where('updated_at', '<=', $now + - self::IMPORT_POLL_INTERVAL_SECONDS); + })->orWhere(function ($crossing) use ($now): void { + $crossing->whereIn('state', ['dispatching', 'sending']) + ->where('updated_at', '<=', $now - 90); + }); + })->orderBy('updated_at')->orderBy('operation_id')->first(); + if ($record === null) { + return false; + } + + $state = (string) $record->state; + $operationId = (string) $record->operation_id; + $claimed = CloudRescueTemplateImport::query() + ->where('operation_id', $operationId) + ->where('host_id', (int) $record->host_id) + ->where('template_id', (string) $record->template_id) + ->where('state', $state) + ->where('updated_at', (int) $record->updated_at) + ->update(['updated_at' => $now]); + if ($claimed !== 1) { + return false; + } + + if ($state === 'dispatching') { + self::mark($record, 'uncertain', 'import_dispatch_unconfirmed'); + return true; + } + if ($state === 'sending') { + self::mark($record, 'running', 'import_dispatch_unconfirmed'); + $record = self::reload($record); + } + if (preg_match('/^[A-Za-z0-9_-]{8,80}$/D', $operationId) !== 1) { + self::mark($record, 'uncertain', 'operation_id_invalid'); + throw new CloudError( + '救援镜像导入操作标识无效,必须人工核查', 503 + ); + } + + $actor = CloudDaemonActor::resolve( + 'cloud-rescue-template-import-worker' + ); + if ($actor === null) { + self::mark($record, 'running', 'daemon_actor_unavailable'); + throw new CloudError('救援镜像导入后台核对未配置管理员身份', 503); + } + $actor['operation_id'] = $operationId; + + try { + $client = CloudAgentSettingsService::clientForHost( + (int) $record->host_id + ); + $operation = $client->request('GET', '/v1/operations/' + . rawurlencode($operationId), $actor); + $operation = CloudAgentOperationReconciler::ifUncertain( + $client, $operationId, $actor, $operation + ); + } catch (CloudAgentError $error) { + $missing = $error->httpStatus === 404 + || $error->agentHttpStatus === 404; + self::mark($record, $missing ? 'uncertain' : 'running', + $missing ? 'operation_not_found' : 'agent_unavailable'); + throw $error; + } catch (\Throwable $error) { + self::mark($record, 'running', 'agent_unavailable'); + throw $error; + } + + try { + self::applyOperation($record, $operation); + } catch (\Throwable $error) { + self::mark($record, 'uncertain', 'panel_persist_failed'); + throw $error; + } + return true; + } + private static function reserve(string $operationId, int $hostId, string $templateId, string $importType): CloudRescueTemplateImport { diff --git a/web/app/Services/Cloud/CloudTemplateCatalogService.php b/web/app/Services/Cloud/CloudTemplateCatalogService.php index a7583629..2d7c2220 100644 --- a/web/app/Services/Cloud/CloudTemplateCatalogService.php +++ b/web/app/Services/Cloud/CloudTemplateCatalogService.php @@ -11,6 +11,7 @@ use App\Models\CloudOrder; final class CloudTemplateCatalogService { public const STARTUP_SCRIPT_MAX_BYTES = 32768; + private const IMPORT_POLL_INTERVAL_SECONDS = 5; public static function adminCatalog(): array { @@ -49,8 +50,9 @@ final class CloudTemplateCatalogService if ($hostIds === []) { throw new CloudError('没有已启用且已配置 Agent 的宿主机', 409); } + $hostIds = array_values(array_unique(array_map('intval', $hostIds))); $template = self::newImport($name, $family, $version, 'remote', - $sourceUrl, 0, $sha256); + $sourceUrl, 0, $sha256, $hostIds); return self::dispatchImport($template, $hostIds, $actor, static fn (int $hostId): array => [ 'source_url' => $sourceUrl, @@ -72,6 +74,7 @@ final class CloudTemplateCatalogService if ($hostIds === []) { throw new CloudError('没有已启用且已配置 Agent 的宿主机', 409); } + $hostIds = array_values(array_unique(array_map('intval', $hostIds))); $digest = strtolower((string) ($file['sha256'] ?? '')); $bytes = filter_var($file['size_bytes'] ?? null, FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]); @@ -80,7 +83,7 @@ final class CloudTemplateCatalogService throw new CloudError('本地云镜像校验结果无效', 503); } $template = self::newImport($name, $family, $version, 'upload', - '', (int) $bytes, $digest); + '', (int) $bytes, $digest, $hostIds); return self::dispatchImport($template, $hostIds, $actor, static function (int $hostId) use ($file): array { $staging = CloudUploadStagingService::issue( @@ -147,6 +150,144 @@ final class CloudTemplateCatalogService return self::importJobDto($template); } + /** + * Converge one durable template import while the cloud_jobs queue is idle. + * The original source request is intentionally unavailable here: only the + * already-persisted Agent operation ID may be read or reconciled. + */ + public static function reconcileOnePendingImport(): bool + { + $connection = (new CloudTemplateReplica())->getConnection(); + $schema = $connection->getSchemaBuilder(); + if (!$schema->hasTable('cloud_templates') + || !$schema->hasTable('cloud_template_replicas')) { + return false; + } + + $now = time(); + $replica = CloudTemplateReplica::query() + ->select('cloud_template_replicas.*') + ->join('cloud_templates', 'cloud_templates.id', '=', + 'cloud_template_replicas.template_id') + ->where('cloud_template_replicas.state', 'pending') + ->whereIn('cloud_templates.state', ['importing', 'error']) + ->where('cloud_template_replicas.updated_at', '<=', + $now - self::IMPORT_POLL_INTERVAL_SECONDS) + ->orderBy('cloud_template_replicas.updated_at') + ->orderBy('cloud_template_replicas.host_id') + ->orderBy('cloud_template_replicas.template_id')->first(); + + if ($replica === null) { + // Versions predating atomic parent/replica reservation could + // expose a parent row before the first host operation existed. + // There is no saved mutation to replay, so age it into an error. + $orphan = CloudTemplate::query()->where('state', 'importing') + ->where('updated_at', '<=', $now - 90) + ->whereNotExists(static function ($query): void { + $query->selectRaw('1')->from('cloud_template_replicas') + ->whereColumn('cloud_template_replicas.template_id', + 'cloud_templates.id'); + })->orderBy('updated_at')->orderBy('id')->first(); + if ($orphan !== null) { + return CloudTemplate::query() + ->where('id', (string) $orphan->id) + ->where('state', 'importing') + ->where('updated_at', (int) $orphan->updated_at) + ->update([ + 'state' => 'error', + 'updated_at' => $now, + ]) === 1; + } + + // Repair the narrow crash window where every replica committed but + // the parent aggregate did not. Failed or uncertain replicas are + // excluded so an idle worker cannot spin on a terminal import. + $template = CloudTemplate::query() + ->whereIn('state', ['importing', 'error']) + ->whereExists(static function ($query): void { + $query->selectRaw('1')->from('cloud_template_replicas') + ->whereColumn('cloud_template_replicas.template_id', + 'cloud_templates.id'); + })->whereNotExists(static function ($query): void { + $query->selectRaw('1')->from('cloud_template_replicas') + ->whereColumn('cloud_template_replicas.template_id', + 'cloud_templates.id') + ->where('cloud_template_replicas.state', '<>', 'active'); + })->orderBy('updated_at')->orderBy('id')->first(); + if ($template === null) { + return false; + } + self::refreshImportState($template); + return true; + } + + $hostId = (int) $replica->host_id; + $templateId = (string) $replica->template_id; + $operationId = (string) $replica->operation_id; + // updated_at is a schema-free claim and backoff. Concurrent workers + // can only duplicate a read, never the original import mutation. + $claimed = CloudTemplateReplica::query()->where('host_id', $hostId) + ->where('template_id', $templateId) + ->where('state', 'pending')->where('operation_id', $operationId) + ->where('updated_at', (int) $replica->updated_at) + ->update(['updated_at' => $now]); + if ($claimed !== 1) { + return false; + } + + $template = CloudTemplate::query()->where('id', $templateId) + ->whereIn('state', ['importing', 'error'])->first(); + if ($template === null) { + return true; + } + if (preg_match('/^[A-Za-z0-9_-]{8,80}$/D', $operationId) !== 1) { + self::markReplica($hostId, $templateId, $operationId, + ['pending'], 'uncertain', 'operation_id_invalid'); + self::refreshImportState($template); + throw new CloudError('云镜像导入操作标识无效,必须人工核查', 503); + } + $actor = CloudDaemonActor::resolve('cloud-template-import-worker'); + if ($actor === null) { + self::markReplica($hostId, $templateId, $operationId, + ['pending'], 'pending', 'daemon_actor_unavailable'); + throw new CloudError('云镜像导入后台核对未配置管理员身份', 503); + } + $actor['operation_id'] = $operationId; + + try { + $client = CloudAgentSettingsService::clientForHost($hostId); + $operation = $client->request('GET', '/v1/operations/' + . rawurlencode($operationId), $actor); + $operation = CloudAgentOperationReconciler::ifUncertain( + $client, $operationId, $actor, $operation + ); + } catch (CloudAgentError $error) { + $missing = $error->httpStatus === 404 + || $error->agentHttpStatus === 404; + self::markReplica($hostId, $templateId, $operationId, + ['pending'], $missing ? 'uncertain' : 'pending', + $missing ? 'operation_not_found' : 'agent_unavailable'); + self::refreshImportState($template); + throw $error; + } catch (\Throwable $error) { + self::markReplica($hostId, $templateId, $operationId, + ['pending'], 'pending', 'agent_unavailable'); + throw $error; + } + + try { + self::applyImportOperation($template, $hostId, $operationId, + $operation); + self::refreshImportState($template); + } catch (\Throwable $error) { + self::markReplica($hostId, $templateId, $operationId, + ['pending'], 'uncertain', 'panel_persist_failed'); + self::refreshImportState($template); + throw $error; + } + return true; + } + /** Push the current PHP name/revision to every existing host replica. */ public static function synchronizeMetadata(array $template, array $actor): array @@ -160,8 +301,18 @@ final class CloudTemplateCatalogService || preg_match('/^[a-f0-9]{32}$/D', $generation) !== 1) { throw new CloudError('云镜像版本无效', 409); } + $authority = CloudTemplate::query()->where('id', $id)->first([ + 'id', 'name', 'revision', 'generation_token', 'state', + ]); + if ($authority === null || (string) $authority->state !== 'active' + || (int) $authority->revision !== (int) $revision + || !hash_equals((string) $authority->generation_token, $generation) + || (string) $authority->name !== $name) { + throw new CloudError('仅已完成导入的当前云镜像版本可以同步元数据', 409); + } $pending = []; foreach (CloudTemplateReplica::query()->where('template_id', $id) + ->whereIn('state', ['active', 'stale']) ->orderBy('host_id')->get() as $replica) { $hostId = (int) $replica->host_id; $operationId = bin2hex(random_bytes(16)); @@ -191,6 +342,7 @@ final class CloudTemplateCatalogService CloudTemplateReplica::query()->where('host_id', $hostId) ->where('template_id', $id) ->where('generation_token', $generation) + ->whereIn('state', ['active', 'stale']) // A slower response for revision N must never overwrite a // concurrent revision N+1 response. ->where('revision', '<=', (int) $revision)->update([ @@ -205,6 +357,7 @@ final class CloudTemplateCatalogService CloudTemplateReplica::query()->where('host_id', $hostId) ->where('template_id', $id) ->where('generation_token', $generation) + ->whereIn('state', ['active', 'stale']) // Do not let a late failure regress a newer successful // metadata synchronization. ->where('revision', '<=', (int) $revision)->update([ @@ -516,6 +669,9 @@ final class CloudTemplateCatalogService if ($template === null || (string) $template->state === 'retired') { throw new CloudError('云镜像模板不存在', 404); } + if ((string) $template->state !== 'active') { + throw new CloudError('云镜像尚未完成导入,暂不能修改启动脚本', 409); + } if (!hash_equals((string) $template->generation_token, $generationToken) || (int) $template->revision !== $expectedRevision) { throw new CloudError('初始化脚本已被修改,请重新打开对话框', 409); @@ -760,6 +916,9 @@ final class CloudTemplateCatalogService if ($template === null || (string) $template->state === 'retired') { throw new CloudError('云镜像模板不存在', 404); } + if ((string) $template->state !== 'active') { + throw new CloudError('云镜像尚未完成导入,暂不能修改元数据', 409); + } if ((int) $template->revision !== $expectedRevision || !hash_equals((string) $template->generation_token, $generationToken)) { @@ -784,6 +943,21 @@ final class CloudTemplateCatalogService private static function adminDto(CloudTemplate $template): array { + $replicas = []; + $connection = (new CloudTemplateReplica())->getConnection(); + if ($connection->getSchemaBuilder()->hasTable( + 'cloud_template_replicas')) { + $replicas = CloudTemplateReplica::query() + ->where('template_id', (string) $template->id) + ->orderBy('host_id')->get([ + 'host_id', 'state', 'error_code', 'operation_id', + ])->map(static fn (CloudTemplateReplica $replica): array => [ + 'host_id' => (int) $replica->host_id, + 'state' => (string) $replica->state, + 'error_code' => (string) $replica->error_code, + 'operation_id' => (string) $replica->operation_id, + ])->all(); + } return [ 'id' => (string) $template->id, 'name' => (string) $template->name, @@ -798,62 +972,57 @@ final class CloudTemplateCatalogService 'generation_token' => (string) $template->generation_token, 'created_at' => (int) $template->created_at, 'updated_at' => (int) $template->updated_at, + 'replicas' => $replicas, ]; } - private static function writeReplica(int $hostId, CloudTemplate $template, - string $state, string $operationId, string $errorCode, int $bytes, - string $sha256): void - { - $values = [ - 'revision' => (int) $template->revision, - 'generation_token' => (string) $template->generation_token, - 'state' => $state, - 'image_bytes' => $bytes, - 'sha256' => $sha256, - 'operation_id' => $operationId, - 'error_code' => $errorCode, - 'updated_at' => time(), - ]; - $query = CloudTemplateReplica::query()->where('host_id', $hostId) - ->where('template_id', (string) $template->id); - if ($query->exists()) { - $query->update($values); - return; - } - $replica = new CloudTemplateReplica(); - foreach ($values + [ - 'host_id' => $hostId, - 'template_id' => (string) $template->id, - 'created_at' => time(), - ] as $field => $value) { - $replica->{$field} = $value; - } - $replica->save(); - } - private static function newImport(string $name, string $family, string $version, string $sourceType, string $sourceUrl, int $bytes, - string $sha256): CloudTemplate + string $sha256, array $hostIds): CloudTemplate { - $template = new CloudTemplate(); - $template->id = self::newTemplateId($family, $version); - $template->name = $name; - $template->os_family = $family; - $template->os_version = $version; - $template->source_type = $sourceType; - $template->sealed_source_url = CloudSecrets::encrypt( - ['url' => $sourceUrl], 'template-source' - ); - $template->image_bytes = $bytes; - $template->sha256 = $sha256; - $template->startup_script = ''; - $template->revision = 1; - $template->generation_token = bin2hex(random_bytes(16)); - $template->state = 'importing'; - $template->created_at = $template->updated_at = time(); - $template->save(); - return $template; + $connection = (new CloudTemplate())->getConnection(); + return $connection->transaction(static function () use ( + $name, $family, $version, $sourceType, $sourceUrl, $bytes, + $sha256, $hostIds + ): CloudTemplate { + $template = new CloudTemplate(); + $template->id = self::newTemplateId($family, $version); + $template->name = $name; + $template->os_family = $family; + $template->os_version = $version; + $template->source_type = $sourceType; + $template->sealed_source_url = CloudSecrets::encrypt( + ['url' => $sourceUrl], 'template-source' + ); + $template->image_bytes = $bytes; + $template->sha256 = $sha256; + $template->startup_script = ''; + $template->revision = 1; + $template->generation_token = bin2hex(random_bytes(16)); + $template->state = 'importing'; + $template->created_at = $template->updated_at = time(); + $template->save(); + + $replicas = []; + foreach ($hostIds as $hostId) { + $replicas[] = [ + 'host_id' => $hostId, + 'template_id' => (string) $template->id, + 'revision' => (int) $template->revision, + 'generation_token' => + (string) $template->generation_token, + 'state' => 'pending', + 'image_bytes' => 0, + 'sha256' => (string) $template->sha256, + 'operation_id' => bin2hex(random_bytes(16)), + 'error_code' => '', + 'created_at' => (int) $template->created_at, + 'updated_at' => (int) $template->updated_at, + ]; + } + CloudTemplateReplica::query()->insert($replicas); + return $template; + }, 3); } private static function dispatchImport(CloudTemplate $template, @@ -861,11 +1030,20 @@ final class CloudTemplateCatalogService { $operations = []; foreach ($hostIds as $hostId) { - $hostId = (int) $hostId; - $operationId = bin2hex(random_bytes(16)); - $operations[$hostId] = $operationId; - self::writeReplica($hostId, $template, 'pending', $operationId, - '', 0, (string) $template->sha256); + $replica = CloudTemplateReplica::query() + ->where('host_id', $hostId) + ->where('template_id', (string) $template->id) + ->where('state', 'pending')->first(); + if ($replica === null) { + CloudTemplate::query()->where('id', (string) $template->id) + ->where('state', 'importing')->update([ + 'state' => 'error', + 'updated_at' => time(), + ]); + $template->refresh(); + return self::importJobDto($template); + } + $operations[$hostId] = (string) $replica->operation_id; } foreach ($operations as $hostId => $operationId) { try { @@ -1027,9 +1205,16 @@ final class CloudTemplateCatalogService 'agent_result_mismatch'); return; } + $metadataChanged = false; if ((string) $locked->sha256 === '') { $locked->sha256 = $digest; + $metadataChanged = true; + } + if ((int) $locked->image_bytes === 0) { $locked->image_bytes = (int) $bytes; + $metadataChanged = true; + } + if ($metadataChanged) { $locked->updated_at = time(); $locked->save(); } diff --git a/web/app/Services/Cloud/CloudUserIsoService.php b/web/app/Services/Cloud/CloudUserIsoService.php index ebbb588e..180bba4a 100644 --- a/web/app/Services/Cloud/CloudUserIsoService.php +++ b/web/app/Services/Cloud/CloudUserIsoService.php @@ -10,6 +10,7 @@ final class CloudUserIsoService { private const MAX_ISOS = 5; private const MAX_TOTAL_BYTES = 10 * 1024 * 1024 * 1024; + private const IMPORT_POLL_INTERVAL_SECONDS = 5; public static function listing(int $userId): array { @@ -37,9 +38,10 @@ final class CloudUserIsoService if ($hosts === []) { throw new CloudError('没有可用的宿主机 Agent', 503); } + $hosts = array_values(array_unique(array_map('intval', $hosts))); $connection = (new CloudUserIso())->getConnection(); $iso = $connection->transaction(static function () use ( - $userId, $name, $url + $userId, $name, $url, $hosts ): CloudUserIso { $rows = CloudUserIso::query()->where('user_id', $userId) ->where('state', '<>', 'retired') @@ -63,14 +65,48 @@ final class CloudUserIsoService $iso->import_operation_id = bin2hex(random_bytes(16)); $iso->created_at = $iso->updated_at = time(); $iso->save(); + + // Persist the complete target-host snapshot before the first + // Agent mutation. A process crash can therefore leave only a + // fully enumerable set of pending operations, never a partial + // replica set that could be mistaken for a completed import. + $replicas = []; + foreach ($hosts as $hostId) { + $replicas[] = [ + 'host_id' => $hostId, + 'user_iso_id' => (string) $iso->id, + 'user_id' => $userId, + 'state' => 'pending', + 'size_bytes' => 0, + 'sha256' => '', + 'operation_id' => bin2hex(random_bytes(16)), + 'error_code' => '', + 'created_at' => (int) $iso->created_at, + 'updated_at' => (int) $iso->updated_at, + ]; + } + CloudUserIsoReplica::query()->insert($replicas); return $iso; }, 3); + $operations = []; foreach ($hosts as $hostId) { - $hostId = (int) $hostId; - $operationId = bin2hex(random_bytes(16)); - self::writeReplica($hostId, $iso, 'pending', $operationId, - '', 0, ''); + $replica = CloudUserIsoReplica::query() + ->where('host_id', $hostId) + ->where('user_iso_id', (string) $iso->id) + ->where('state', 'pending')->first(); + if ($replica === null) { + CloudUserIso::query()->where('id', (string) $iso->id) + ->where('state', 'importing')->update([ + 'state' => 'failed', + 'updated_at' => time(), + ]); + $iso->refresh(); + return self::jobDto($iso, self::jobStatus($iso)); + } + $operations[$hostId] = (string) $replica->operation_id; + } + foreach ($operations as $hostId => $operationId) { try { $requestActor = $actor; $requestActor['can_manage'] = true; @@ -190,6 +226,149 @@ final class CloudUserIsoService return self::jobDto($iso, self::jobStatus($iso)); } + /** + * Converge one durable ISO replica using only its saved Agent operation. + * Source URLs are intentionally unavailable, so the import POST cannot be + * replayed by the background worker. + */ + public static function reconcileOnePendingImport(): bool + { + $connection = (new CloudUserIsoReplica())->getConnection(); + $schema = $connection->getSchemaBuilder(); + if (!$schema->hasTable('cloud_user_isos') + || !$schema->hasTable('cloud_user_iso_replicas')) { + return false; + } + + $now = time(); + $replica = CloudUserIsoReplica::query() + ->select('cloud_user_iso_replicas.*') + ->join('cloud_user_isos', 'cloud_user_isos.id', '=', + 'cloud_user_iso_replicas.user_iso_id') + ->where('cloud_user_iso_replicas.state', 'pending') + ->whereIn('cloud_user_isos.state', ['importing', 'failed']) + ->where('cloud_user_iso_replicas.updated_at', '<=', + $now - self::IMPORT_POLL_INTERVAL_SECONDS) + ->orderBy('cloud_user_iso_replicas.updated_at') + ->orderBy('cloud_user_iso_replicas.host_id') + ->orderBy('cloud_user_iso_replicas.user_iso_id')->first(); + + if ($replica === null) { + // Rows written by versions predating atomic replica reservation + // may have crashed after the parent commit but before the first + // replica. They contain no operation that could be safely + // replayed, so age them into a visible terminal failure. + $orphan = CloudUserIso::query()->where('state', 'importing') + ->where('updated_at', '<=', $now - 90) + ->whereNotExists(static function ($query): void { + $query->selectRaw('1')->from('cloud_user_iso_replicas') + ->whereColumn('cloud_user_iso_replicas.user_iso_id', + 'cloud_user_isos.id'); + })->orderBy('updated_at')->orderBy('id')->first(); + if ($orphan !== null) { + return CloudUserIso::query() + ->where('id', (string) $orphan->id) + ->where('state', 'importing') + ->where('updated_at', (int) $orphan->updated_at) + ->update([ + 'state' => 'failed', + 'updated_at' => $now, + ]) === 1; + } + + // Repair the crash window after every replica committed but before + // the aggregate catalog row was refreshed. Terminal replica sets + // are excluded so an idle worker cannot spin indefinitely. + $iso = CloudUserIso::query() + // A failed all-active aggregate may be a durable quota + // rejection. Only importing is an unambiguous crash window; + // otherwise the idle worker would select the same row forever. + ->where('state', 'importing') + ->whereExists(static function ($query): void { + $query->selectRaw('1')->from('cloud_user_iso_replicas') + ->whereColumn('cloud_user_iso_replicas.user_iso_id', + 'cloud_user_isos.id'); + })->whereNotExists(static function ($query): void { + $query->selectRaw('1')->from('cloud_user_iso_replicas') + ->whereColumn('cloud_user_iso_replicas.user_iso_id', + 'cloud_user_isos.id') + ->where('cloud_user_iso_replicas.state', '<>', + 'active'); + })->orderBy('updated_at')->orderBy('id')->first(); + if ($iso === null) { + return false; + } + self::refreshCatalogState($iso); + return true; + } + + $hostId = (int) $replica->host_id; + $isoId = (string) $replica->user_iso_id; + $operationId = (string) $replica->operation_id; + $claimed = CloudUserIsoReplica::query()->where('host_id', $hostId) + ->where('user_iso_id', $isoId)->where('state', 'pending') + ->where('operation_id', $operationId) + ->where('updated_at', (int) $replica->updated_at) + ->update(['updated_at' => $now]); + if ($claimed !== 1) { + return false; + } + + $iso = CloudUserIso::query()->where('id', $isoId) + ->whereIn('state', ['importing', 'failed'])->first(); + if ($iso === null) { + return true; + } + if (preg_match('/^[A-Za-z0-9_-]{8,80}$/D', $operationId) !== 1) { + self::markReplica($hostId, $isoId, $operationId, ['pending'], + 'uncertain', 'operation_id_invalid'); + self::refreshCatalogState($iso); + throw new CloudError('ISO 导入操作标识无效,必须人工核查', 503); + } + + $actor = CloudDaemonActor::resolve('cloud-user-iso-import-worker'); + if ($actor === null) { + self::markReplica($hostId, $isoId, $operationId, ['pending'], + 'pending', 'daemon_actor_unavailable'); + throw new CloudError('ISO 导入后台核对未配置管理员身份', 503); + } + $actor['operation_id'] = $operationId; + + try { + $client = CloudAgentSettingsService::clientForHost($hostId); + $operation = $client->request('GET', '/v1/operations/' + . rawurlencode($operationId), $actor); + $operation = CloudAgentOperationReconciler::ifUncertain( + $client, $operationId, $actor, $operation + ); + } catch (CloudAgentError $error) { + $missing = $error->httpStatus === 404 + || $error->agentHttpStatus === 404; + self::markReplica($hostId, $isoId, $operationId, ['pending'], + $missing ? 'uncertain' : 'pending', + $missing ? 'operation_not_found' : 'agent_unavailable'); + self::refreshCatalogState($iso); + throw $error; + } catch (\Throwable $error) { + self::markReplica($hostId, $isoId, $operationId, ['pending'], + 'pending', 'agent_unavailable'); + self::refreshCatalogState($iso); + throw $error; + } + + try { + self::applyImportOperation($iso, $hostId, $operationId, + $operation); + self::refreshCatalogState($iso); + } catch (\Throwable $error) { + self::markReplica($hostId, $isoId, $operationId, ['pending'], + 'uncertain', 'panel_persist_failed'); + self::refreshCatalogState($iso); + throw $error; + } + return true; + } + public static function delete(string $id, array $actor): array { if (preg_match('/^[a-f0-9]{32}$/D', $id) !== 1) { @@ -656,36 +835,6 @@ final class CloudUserIsoService ]); } - private static function writeReplica(int $hostId, CloudUserIso $iso, - string $state, string $operationId, string $errorCode, int $bytes, - string $sha256): void - { - $values = [ - 'user_id' => (int) $iso->user_id, - 'state' => $state, - 'size_bytes' => $bytes, - 'sha256' => $sha256, - 'operation_id' => $operationId, - 'error_code' => $errorCode, - 'updated_at' => time(), - ]; - $query = CloudUserIsoReplica::query()->where('host_id', $hostId) - ->where('user_iso_id', (string) $iso->id); - if ($query->exists()) { - $query->update($values); - return; - } - $replica = new CloudUserIsoReplica(); - foreach ($values + [ - 'host_id' => $hostId, - 'user_iso_id' => (string) $iso->id, - 'created_at' => time(), - ] as $field => $value) { - $replica->{$field} = $value; - } - $replica->save(); - } - private static function markReplica(int $hostId, string $isoId, string $operationId, array $allowedStates, string $state, string $errorCode): bool diff --git a/web/app/Services/Cloud/CloudWorkerService.php b/web/app/Services/Cloud/CloudWorkerService.php index 64841960..01c4fd90 100644 --- a/web/app/Services/Cloud/CloudWorkerService.php +++ b/web/app/Services/Cloud/CloudWorkerService.php @@ -21,6 +21,7 @@ final class CloudWorkerService private bool $policyOnly; private array $observedErrors = []; + private int $catalogCursor = 0; public function observationErrors(): array { @@ -127,6 +128,17 @@ final class CloudWorkerService public function tick(): bool { $this->observedErrors = []; + $worked = $this->tickJob(); + if ($this->policyOnly) { + return $worked; + } + $catalogWorked = $this->convergeOneCatalogImport(); + return $worked || $catalogWorked; + } + + /** Claim and advance at most one durable cloud job. */ + private function tickJob(): bool + { $job = (new CloudJob())->getConnection()->transaction(function () { $query = CloudJob::whereIn('status', ['queued', 'running', 'uncertain']) ->where('available_at', '<=', time())->where('lease_until', '<=', time()); @@ -1059,6 +1071,49 @@ final class CloudWorkerService return $operation; } + /** + * Poll at most one due operation across the PHP-owned catalogs. Each + * service can only read or reconcile the operation ID persisted before + * dispatch; none has the source payload needed to replay an import. + */ + private function convergeOneCatalogImport(): bool + { + $catalogs = [ + ['template', [CloudTemplateCatalogService::class, + 'reconcileOnePendingImport']], + ['rescue_template', [CloudRescueTemplateImportService::class, + 'reconcileOnePendingImport']], + ['user_iso', [CloudUserIsoService::class, + 'reconcileOnePendingImport']], + ]; + $count = count($catalogs); + for ($offset = 0; $offset < $count; $offset++) { + $index = ($this->catalogCursor + $offset) % $count; + [$label, $converge] = $catalogs[$index]; + try { + if (!$converge()) { + continue; + } + $this->catalogCursor = ($index + 1) % $count; + return true; + } catch (\Throwable $error) { + // The callback may have claimed or polled an operation before + // failing, so do not run a second catalog action in this tick. + // An error alone is not "work": an idle worker must retain its + // normal sleep instead of spinning against a broken catalog. + $this->catalogCursor = ($index + 1) % $count; + $this->observedErrors[] = 'worker_tick_failed'; + error_log('cloud worker ' . $label + . ' import reconciliation failed: ' . get_class($error)); + return false; + } + } + // Shift the first probe even when every catalog is idle, so a catalog + // that becomes ready while cloud_jobs stays busy gets the next chance. + $this->catalogCursor = ($this->catalogCursor + 1) % $count; + return false; + } + private static function normalizeAgentOperation(array $operation): array { $error = $operation['error'] ?? null; diff --git a/web/public/cloud/admin.js b/web/public/cloud/admin.js index 63ced39e..05446a8b 100644 --- a/web/public/cloud/admin.js +++ b/web/public/cloud/admin.js @@ -463,12 +463,20 @@ async function loadCatalog() { const url = rescue ? `${catalogPath()}?host_id=${encodeURIComponent(hostId)}` : catalogPath(); try { const entries = collection(await api(url), 'templates'); if (epoch !== state.epoch) return; if (!entries.length) list.append(element('p', 'text-muted', '暂无模版。')); - for (const entry of entries) { const row = element('form', 'cloud-admin-row'); const name = element('input', 'form-control'); name.value = entry.name || ''; name.maxLength = 64; name.required = true; name.setAttribute('aria-label', '模版名称'); name.disabled = !canManage; const save = element('button', 'btn btn-outline-primary', '保存名称'); save.type = 'submit'; save.disabled = !canManage; const remove = element('button', 'btn btn-outline-danger', '删除'); remove.type = 'button'; remove.disabled = !canManage; const imageBytes = Number(entry.image_bytes || entry.size_bytes || 0); + for (const entry of entries) { const row = element('form', 'cloud-admin-row'); const catalogState = String(entry.state || '').toLowerCase(); const active = rescue || catalogState === 'active'; const name = element('input', 'form-control'); name.value = entry.name || ''; name.maxLength = 64; name.required = true; name.setAttribute('aria-label', '模版名称'); name.disabled = !canManage || !active; const save = element('button', 'btn btn-outline-primary', '保存名称'); save.type = 'submit'; save.disabled = !canManage || !active; const remove = element('button', 'btn btn-outline-danger', '删除'); remove.type = 'button'; remove.disabled = !canManage; const imageBytes = Number(entry.image_bytes || entry.size_bytes || 0); row.append(name, save, remove); - if (!rescue) { const startup = element('button', 'btn btn-outline-secondary cloud-admin-startup-button', canManage ? '设置启动脚本' : '查看启动脚本'); startup.type = 'button'; startup.addEventListener('click', () => openStartupScript(entry, startup)); row.append(startup); } - row.append(element('small', 'cloud-admin-meta', `${entry.os_family || '镜像'} ${entry.os_version || ''} · ${imageBytes > 0 ? (imageBytes / 1024 ** 3).toFixed(2) + ' GiB' : entry.id}`)); + if (!rescue) { const startup = element('button', 'btn btn-outline-secondary cloud-admin-startup-button', canManage ? '设置启动脚本' : '查看启动脚本'); startup.type = 'button'; startup.disabled = !active; startup.addEventListener('click', () => openStartupScript(entry, startup)); row.append(startup); } + const stateLabel = rescue ? '' : ({ active: '已就绪', importing: '导入中', error: '导入失败' }[catalogState] || '状态未知') + ' · '; + row.dataset.catalogState = catalogState || 'unknown'; + row.append(element('small', 'cloud-admin-meta', `${stateLabel}${entry.os_family || '镜像'} ${entry.os_version || ''} · ${imageBytes > 0 ? (imageBytes / 1024 ** 3).toFixed(2) + ' GiB' : entry.id}`)); + if (!rescue && Array.isArray(entry.replicas)) for (const replica of entry.replicas) { + const replicaState = String(replica?.state || 'unknown').toLowerCase(); + const replicaLabel = ({ active: '已就绪', pending: '核对中', failed: '失败', uncertain: '待人工核对', stale: '等待同步' }[replicaState] || '状态未知'); + const details = [replica?.error_code, replica?.operation_id].filter(value => String(value || '').trim() !== '').map(String); + row.append(element('small', 'cloud-admin-meta', `宿主机 #${Number(replica?.host_id || 0)} · ${replicaLabel}${details.length ? ' · ' + details.join(' · ') : ''}`)); + } if (rescue) { name.readOnly = true; save.hidden = true; save.disabled = true; } - row.addEventListener('submit', async event => { event.preventDefault(); if (rescue) return; if (state.busy) return notice('请等待当前导入任务完成。', true); save.disabled = true; try { await api(`${catalogPath()}/${encodeURIComponent(entry.id)}`, { method: 'PATCH', body: { name: name.value.trim(), expected_revision: Number(entry.revision), generation_token: String(entry.generation_token || '') } }); await loadCatalog(); notice('名称已保存。'); } catch (error) { notice(error.message, true); } finally { save.disabled = !canManage; } }); + row.addEventListener('submit', async event => { event.preventDefault(); if (rescue) return; if (!active) return notice('镜像尚未就绪,不能修改名称。', true); if (state.busy) return notice('请等待当前导入任务完成。', true); save.disabled = true; try { await api(`${catalogPath()}/${encodeURIComponent(entry.id)}`, { method: 'PATCH', body: { name: name.value.trim(), expected_revision: Number(entry.revision), generation_token: String(entry.generation_token || '') } }); await loadCatalog(); notice('名称已保存。'); } catch (error) { notice(error.message, true); } finally { save.disabled = !canManage || !active; } }); remove.addEventListener('click', async () => { if (state.busy) return notice('请等待当前导入任务完成。', true); const message = rescue @@ -502,7 +510,7 @@ async function importProgress(result) { const suffix = isRescueCatalog() ? `?host_id=${encodeURIComponent(state.selectedHost)}` : ''; const fallback = `${catalogPath()}/import-jobs/${encodeURIComponent(result.job_id)}${suffix}`; const url = String(result.status_url || '').startsWith('/api/v1/') ? result.status_url : fallback; - for (let i = 0; i < 1800; i++) { const job = await api(url); notice(job.phase || `导入状态:${job.status}`); if (['succeeded', 'completed', 'done'].includes(job.status)) return; if (['failed', 'uncertain', 'cancelled'].includes(job.status)) throw new Error(job.error || '导入未完成,请核查宿主机状态后再操作。'); await new Promise(resolve => setTimeout(resolve, 2000)); } throw new Error('导入仍在进行,请稍后刷新查看。不要重复提交。'); + for (let i = 0; i < 3600; i++) { const job = await api(url); notice(job.phase || `导入状态:${job.status}`); if (['succeeded', 'completed', 'done'].includes(job.status)) return; if (['failed', 'uncertain', 'cancelled'].includes(job.status)) throw new Error(job.error || '导入未完成,请核查宿主机状态后再操作。'); await new Promise(resolve => setTimeout(resolve, 2000)); } throw new Error('导入仍在进行,请稍后刷新查看。不要重复提交。'); } async function importTemplate(event) { event.preventDefault(); if (state.busy) return; diff --git a/web/tests/AdminTemplateStartupScriptBehaviorTest.mjs b/web/tests/AdminTemplateStartupScriptBehaviorTest.mjs index c6f18846..ffa136cf 100644 --- a/web/tests/AdminTemplateStartupScriptBehaviorTest.mjs +++ b/web/tests/AdminTemplateStartupScriptBehaviorTest.mjs @@ -23,7 +23,7 @@ class Element { const flush = async () => { for (let turn = 0; turn < 20; turn += 1) await Promise.resolve(); }; const generation = '0123456789abcdef0123456789abcdef'; -const entry = { id: 'debian-12-template', name: '', revision: 1, generation_token: generation }; +const entry = { id: 'debian-12-template', name: '', state: 'active', image_bytes: 1073741824, revision: 1, generation_token: generation }; const payload = { id: entry.id, startup_script: '#!/bin/bash\necho ready\n', revision: 2, generation_token: generation }; const submit = { preventDefault() {} }; @@ -57,6 +57,9 @@ function fixture(canManage = true) { const button = test.get('catalog-list').children[0].children .find(item => item.textContent === '设置启动脚本'); assert.ok(button, 'Cloud template row has no startup-script control.'); + const metadata = test.get('catalog-list').children[0].children + .find(item => item.className === 'cloud-admin-meta'); + assert.match(metadata.textContent, /^已就绪 · .*1\.00 GiB$/); const opening = button.listeners.get('click')(); assert.equal(test.get('template-startup-title').textContent, `启动脚本 · ${entry.name}`); assert.equal(test.get('template-startup-save').disabled, true); @@ -83,6 +86,29 @@ function fixture(canManage = true) { assert.equal(button.focused, true); } +{ + const test = fixture(); + const loading = test.loadCatalog(); + await test.respond(test.requests[0], { templates: [{ + ...entry, + state: 'importing', + image_bytes: 0, + replicas: [{ host_id: 7, state: 'pending', error_code: 'agent_unavailable', operation_id: 'operation-1234' }], + }] }); + await loading; + const row = test.get('catalog-list').children[0]; + const save = row.children.find(item => item.textContent === '保存名称'); + const remove = row.children.find(item => item.textContent === '删除'); + const startup = row.children.find(item => item.textContent === '设置启动脚本'); + const metadata = row.children.find(item => item.className === 'cloud-admin-meta'); + assert.equal(row.dataset.catalogState, 'importing'); + assert.equal(save.disabled, true); + assert.equal(startup.disabled, true); + assert.equal(remove.disabled, false, 'An operator must retain access to the catalog deletion workflow.'); + assert.match(metadata.textContent, /^导入中 · .*debian-12-template$/); + assert.match(row.children.at(-1).textContent, /^宿主机 #7 · 核对中 · agent_unavailable · operation-1234$/); +} + { const test = fixture(false); const opening = test.openStartupScript(entry, new Element()); diff --git a/web/tests/CloudAuxCatalogWorkerConvergenceTest.php b/web/tests/CloudAuxCatalogWorkerConvergenceTest.php new file mode 100644 index 00000000..ebd7e864 --- /dev/null +++ b/web/tests/CloudAuxCatalogWorkerConvergenceTest.php @@ -0,0 +1,484 @@ + 1, + 'is_admin' => true, + 'is_super_admin' => true, + 'can_manage' => true, + 'session_id' => $sessionId, + ]; + } + } + + final class AuxCatalogWorkerFakeAgent + { + public array $requests = []; + public array $getResponses = []; + public array $reconcileResponses = []; + public array $getErrors = []; + public $onRequest = null; + + public function request(string $method, string $target, array $actor, + ?array $payload = null): array + { + $this->requests[] = compact('method', 'target', 'actor', 'payload'); + if (is_callable($this->onRequest)) { + ($this->onRequest)($method, $target, $actor, $payload); + } + if (preg_match('#^/v1/operations/([A-Za-z0-9_-]+)$#D', + $target, $match) === 1 && $method === 'GET') { + $operationId = rawurldecode($match[1]); + if (isset($this->getErrors[$operationId])) { + throw $this->getErrors[$operationId]; + } + return ['operation_id' => $operationId] + + ($this->getResponses[$operationId] + ?? ['status' => 'running']); + } + if (preg_match( + '#^/v1/operations/([A-Za-z0-9_-]+)/reconcile$#D', + $target, $match) === 1 && $method === 'POST') { + $operationId = rawurldecode($match[1]); + return ['operation_id' => $operationId] + + ($this->reconcileResponses[$operationId] + ?? ['status' => 'uncertain']); + } + if ($method === 'POST' && $target === '/v1/templates') { + return [ + 'operation_id' => (string) $payload['operation_id'], + 'status' => 'succeeded', + 'result' => [ + 'id' => (string) $payload['id'], + 'revision' => (int) $payload['revision'], + 'generation_token' => + (string) $payload['generation_token'], + 'size_bytes' => 4096, + 'sha256' => str_repeat('a', 64), + ], + ]; + } + if ($method === 'POST' && $target === '/v1/user-isos') { + return [ + 'operation_id' => (string) $payload['operation_id'], + 'status' => 'succeeded', + 'result' => [ + 'id' => (string) $payload['id'], + 'owner_id' => (int) $payload['owner_id'], + 'size_bytes' => 8192, + 'sha256' => str_repeat('b', 64), + ], + ]; + } + throw new \RuntimeException('Background worker replayed an import'); + } + } + + final class CloudAgentSettingsService + { + public static ?AuxCatalogWorkerFakeAgent $client = null; + public static array $hostIds = [11]; + + public static function enabledHostIds(): array + { + return self::$hostIds; + } + + public static function clientForHost(int $hostId, + float $timeoutSeconds = 20.0): AuxCatalogWorkerFakeAgent + { + if (!in_array($hostId, self::$hostIds, true) + || self::$client === null) { + throw new \RuntimeException('Missing fake Agent'); + } + return self::$client; + } + } +} + +namespace { + require dirname(__DIR__) . '/vendor/autoload.php'; + + use App\Models\CloudRescueTemplateImport; + use App\Models\CloudTemplate; + use App\Models\CloudTemplateReplica; + use App\Models\CloudUserIso; + use App\Models\CloudUserIsoReplica; + use App\Services\Cloud\AuxCatalogWorkerFakeAgent; + use App\Services\Cloud\CloudAgentError; + use App\Services\Cloud\CloudAgentSettingsService; + use App\Services\Cloud\CloudRescueTemplateImportService; + use App\Services\Cloud\CloudTemplateCatalogService; + use App\Services\Cloud\CloudUserIsoService; + use Illuminate\Database\Capsule\Manager as Capsule; + use Illuminate\Database\Schema\Blueprint; + + $assert = static function (bool $condition, string $message): void { + if (!$condition) { + throw new RuntimeException($message); + } + }; + + $capsule = new Capsule(); + $capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'], + 'default'); + $capsule->setAsGlobal(); + $capsule->bootEloquent(); + $schema = $capsule->schema('default'); + $schema->create('cloud_templates', static function (Blueprint $table): void { + $table->string('id')->primary(); + $table->string('name'); + $table->string('os_family'); + $table->string('os_version'); + $table->string('source_type'); + $table->text('sealed_source_url'); + $table->integer('image_bytes')->default(0); + $table->string('sha256')->default(''); + $table->text('startup_script')->default(''); + $table->integer('revision'); + $table->string('generation_token'); + $table->string('state'); + $table->integer('created_at'); + $table->integer('updated_at'); + }); + $schema->create('cloud_template_replicas', + static function (Blueprint $table): void { + $table->integer('host_id'); + $table->string('template_id'); + $table->integer('revision'); + $table->string('generation_token'); + $table->string('state'); + $table->integer('image_bytes')->default(0); + $table->string('sha256')->default(''); + $table->string('operation_id')->default(''); + $table->string('error_code')->default(''); + $table->integer('created_at'); + $table->integer('updated_at'); + $table->primary(['host_id', 'template_id']); + }); + $schema->create('cloud_rescue_template_imports', + static function (Blueprint $table): void { + $table->string('operation_id')->primary(); + $table->integer('host_id'); + $table->string('template_id'); + $table->string('import_type'); + $table->string('state'); + $table->string('error_code')->default(''); + $table->integer('created_at'); + $table->integer('updated_at'); + $table->unique(['host_id', 'template_id']); + }); + $schema->create('cloud_user_isos', static function (Blueprint $table): void { + $table->string('id')->primary(); + $table->integer('user_id'); + $table->string('name'); + $table->text('sealed_source_url'); + $table->integer('size_bytes')->default(0); + $table->string('sha256')->default(''); + $table->string('state'); + $table->string('import_operation_id'); + $table->integer('created_at'); + $table->integer('updated_at'); + }); + $schema->create('cloud_user_iso_replicas', + static function (Blueprint $table): void { + $table->integer('host_id'); + $table->string('user_iso_id'); + $table->integer('user_id'); + $table->string('state'); + $table->integer('size_bytes')->default(0); + $table->string('sha256')->default(''); + $table->string('operation_id')->default(''); + $table->string('error_code')->default(''); + $table->integer('created_at'); + $table->integer('updated_at'); + $table->primary(['host_id', 'user_iso_id']); + }); + + $agent = new AuxCatalogWorkerFakeAgent(); + CloudAgentSettingsService::$client = $agent; + $now = time(); + + // Both global catalogs must persist their complete target-host snapshot + // before the first host mutation. A crash at any dispatch boundary then + // leaves one durable operation per expected host, never a partial set. + CloudAgentSettingsService::$hostIds = [11, 12]; + $agent->onRequest = static function (string $method, string $target, + array $actor, ?array $payload) use ($assert): void { + if ($method !== 'POST') { + return; + } + if ($target === '/v1/templates') { + $hostIds = CloudTemplateReplica::query() + ->where('template_id', (string) $payload['id']) + ->orderBy('host_id')->pluck('host_id')->map( + static fn ($value): int => (int) $value + )->all(); + $assert($hostIds === [11, 12], + 'Template dispatch began before every replica was reserved.'); + } + if ($target === '/v1/user-isos') { + $hostIds = CloudUserIsoReplica::query() + ->where('user_iso_id', (string) $payload['id']) + ->orderBy('host_id')->pluck('host_id')->map( + static fn ($value): int => (int) $value + )->all(); + $assert($hostIds === [11, 12], + 'ISO dispatch began before every replica was reserved.'); + } + }; + $templateReservation = CloudTemplateCatalogService::importRemote([ + 'name' => 'Debian 13 atomic reservation', + 'url' => 'https://images.example/debian-13.qcow2', + 'sha256' => '', + ], ['user_id' => 1, 'is_admin' => true, 'can_manage' => true]); + $isoReservation = CloudUserIsoService::import([ + 'name' => 'Atomic reservation ISO', + 'url' => 'https://images.example/atomic-reservation.iso', + ], ['user_id' => 90, 'is_admin' => false, 'can_manage' => false]); + $assert($templateReservation['status'] === 'succeeded' + && $isoReservation['status'] === 'succeeded', + 'Atomic catalog reservations did not complete in the test fixture.'); + $agent->onRequest = null; + $agent->requests = []; + CloudAgentSettingsService::$hostIds = [11]; + + $rescueOperation = str_repeat('a', 32); + $freshRescueOperation = str_repeat('b', 32); + $rescueTemplate = str_repeat('1', 32); + $freshRescueTemplate = str_repeat('2', 32); + Capsule::table('cloud_rescue_template_imports')->insert([ + [ + 'operation_id' => $rescueOperation, + 'host_id' => 11, + 'template_id' => $rescueTemplate, + 'import_type' => 'remote', + 'state' => 'running', + 'error_code' => '', + 'created_at' => $now - 30, + 'updated_at' => $now - 30, + ], + [ + 'operation_id' => $freshRescueOperation, + 'host_id' => 11, + 'template_id' => $freshRescueTemplate, + 'import_type' => 'remote', + 'state' => 'running', + 'error_code' => '', + 'created_at' => $now, + 'updated_at' => $now, + ], + ]); + $agent->getResponses[$rescueOperation] = [ + 'status' => 'uncertain', + 'error' => ['code' => 'operation_uncertain'], + ]; + $agent->reconcileResponses[$rescueOperation] = [ + 'status' => 'succeeded', + 'result' => ['id' => $rescueTemplate], + ]; + + $assert(CloudRescueTemplateImportService::reconcileOnePendingImport(), + 'Rescue import worker did not claim one due operation.'); + $rescue = CloudRescueTemplateImport::find($rescueOperation); + $freshRescue = CloudRescueTemplateImport::find($freshRescueOperation); + $assert((string) $rescue->state === 'succeeded' + && (string) $freshRescue->state === 'running' + && count($agent->requests) === 2 + && $agent->requests[0]['method'] === 'GET' + && $agent->requests[0]['target'] + === '/v1/operations/' . $rescueOperation + && $agent->requests[1]['method'] === 'POST' + && $agent->requests[1]['target'] + === '/v1/operations/' . $rescueOperation . '/reconcile' + && ($agent->requests[0]['actor']['session_id'] ?? null) + === 'cloud-rescue-template-import-worker' + && ($agent->requests[0]['actor']['operation_id'] ?? null) + === $rescueOperation, + 'Rescue worker did not preserve the saved operation boundary.'); + $assert(!CloudRescueTemplateImportService::reconcileOnePendingImport(), + 'Fresh rescue import bypassed the five-second poll backoff.'); + + Capsule::table('cloud_rescue_template_imports') + ->where('operation_id', $freshRescueOperation) + ->update(['updated_at' => $now - 30]); + $agent->getErrors[$freshRescueOperation] = new CloudAgentError( + 'temporary failure', 503, 'agent_transport_error', true, 503 + ); + try { + CloudRescueTemplateImportService::reconcileOnePendingImport(); + throw new RuntimeException('Rescue transport error was swallowed.'); + } catch (CloudAgentError $error) { + $assert($error->agentCode === 'agent_transport_error', + 'Rescue worker threw the wrong transport error.'); + } + $freshRescue->refresh(); + $assert((string) $freshRescue->state === 'running' + && (string) $freshRescue->error_code === 'agent_unavailable' + && !CloudRescueTemplateImportService::reconcileOnePendingImport(), + 'Rescue transport failure was not persisted with poll backoff.'); + + $insertIso = static function (string $id, string $jobId, int $userId, + string $state, int $updatedAt): void { + Capsule::table('cloud_user_isos')->insert([ + 'id' => $id, + 'user_id' => $userId, + 'name' => $id, + 'sealed_source_url' => 'test-sealed-source', + 'size_bytes' => 0, + 'sha256' => '', + 'state' => $state, + 'import_operation_id' => $jobId, + 'created_at' => $updatedAt, + 'updated_at' => $updatedAt, + ]); + }; + $insertIsoReplica = static function (string $id, int $userId, + string $operationId, string $state, int $updatedAt, + int $bytes = 0, string $sha256 = ''): void { + Capsule::table('cloud_user_iso_replicas')->insert([ + 'host_id' => 11, + 'user_iso_id' => $id, + 'user_id' => $userId, + 'state' => $state, + 'size_bytes' => $bytes, + 'sha256' => $sha256, + 'operation_id' => $operationId, + 'error_code' => '', + 'created_at' => $updatedAt, + 'updated_at' => $updatedAt, + ]); + }; + + $isoId = str_repeat('3', 32); + $freshIsoId = str_repeat('4', 32); + $healIsoId = str_repeat('5', 32); + $isoOperation = str_repeat('c', 32); + $freshIsoOperation = str_repeat('d', 32); + $digest = str_repeat('e', 64); + $insertIso($isoId, str_repeat('6', 32), 42, 'importing', $now - 30); + $insertIsoReplica($isoId, 42, $isoOperation, 'pending', $now - 30); + $insertIso($freshIsoId, str_repeat('7', 32), 43, 'importing', $now); + $insertIsoReplica($freshIsoId, 43, $freshIsoOperation, 'pending', $now); + $insertIso($healIsoId, str_repeat('8', 32), 44, 'importing', $now - 20); + $insertIsoReplica($healIsoId, 44, str_repeat('f', 32), 'active', + $now - 20, 4096, $digest); + Capsule::table('cloud_user_isos')->where('id', $healIsoId)->update([ + 'size_bytes' => 4096, + 'sha256' => $digest, + ]); + $agent->getResponses[$isoOperation] = [ + 'status' => 'uncertain', + 'error' => ['code' => 'operation_uncertain'], + ]; + $agent->reconcileResponses[$isoOperation] = [ + 'status' => 'succeeded', + 'result' => [ + 'id' => $isoId, + 'owner_id' => 42, + 'size_bytes' => 8192, + 'sha256' => $digest, + ], + ]; + $requestsBeforeIso = count($agent->requests); + $assert(CloudUserIsoService::reconcileOnePendingImport(), + 'ISO worker did not claim one due replica.'); + $iso = CloudUserIso::find($isoId); + $isoReplica = CloudUserIsoReplica::query()->where('host_id', 11) + ->where('user_iso_id', $isoId)->first(); + $freshIsoReplica = CloudUserIsoReplica::query()->where('host_id', 11) + ->where('user_iso_id', $freshIsoId)->first(); + $isoRequests = array_slice($agent->requests, $requestsBeforeIso); + $assert((string) $iso->state === 'active' + && (string) $isoReplica->state === 'active' + && (string) $freshIsoReplica->state === 'pending' + && count($isoRequests) === 2 + && $isoRequests[0]['method'] === 'GET' + && $isoRequests[0]['target'] === '/v1/operations/' . $isoOperation + && $isoRequests[1]['method'] === 'POST' + && $isoRequests[1]['target'] + === '/v1/operations/' . $isoOperation . '/reconcile' + && ($isoRequests[0]['actor']['session_id'] ?? null) + === 'cloud-user-iso-import-worker' + && ($isoRequests[0]['actor']['operation_id'] ?? null) + === $isoOperation, + 'ISO worker did not preserve one saved operation boundary.'); + + $requestsBeforeHeal = count($agent->requests); + $assert(CloudUserIsoService::reconcileOnePendingImport() + && (string) CloudUserIso::find($healIsoId)->state === 'active' + && count($agent->requests) === $requestsBeforeHeal, + 'ISO worker did not repair an all-active parent locally.'); + $assert(!CloudUserIsoService::reconcileOnePendingImport(), + 'Fresh ISO replica bypassed the five-second poll backoff.'); + + Capsule::table('cloud_user_iso_replicas') + ->where('user_iso_id', $freshIsoId) + ->update(['updated_at' => $now - 30]); + $agent->getErrors[$freshIsoOperation] = new CloudAgentError( + 'temporary failure', 503, 'agent_transport_error', true, 503 + ); + try { + CloudUserIsoService::reconcileOnePendingImport(); + throw new RuntimeException('ISO transport error was swallowed.'); + } catch (CloudAgentError $error) { + $assert($error->agentCode === 'agent_transport_error', + 'ISO worker threw the wrong transport error.'); + } + $freshIso = CloudUserIso::find($freshIsoId); + $freshIsoReplica->refresh(); + $assert((string) $freshIso->state === 'importing' + && (string) $freshIsoReplica->state === 'pending' + && (string) $freshIsoReplica->error_code === 'agent_unavailable' + && !CloudUserIsoService::reconcileOnePendingImport(), + 'ISO transport failure was not persisted with poll backoff.'); + + $orphanTemplateId = 'debian-orphan-parent'; + Capsule::table('cloud_templates')->insert([ + 'id' => $orphanTemplateId, + 'name' => 'Orphan template', + 'os_family' => 'Debian', + 'os_version' => '13', + 'source_type' => 'remote', + 'sealed_source_url' => 'test-sealed:template-source', + 'image_bytes' => 0, + 'sha256' => '', + 'startup_script' => '', + 'revision' => 1, + 'generation_token' => str_repeat('0', 32), + 'state' => 'importing', + 'created_at' => $now - 180, + 'updated_at' => $now - 180, + ]); + $orphanIsoId = str_repeat('9', 32); + $insertIso($orphanIsoId, str_repeat('0', 32), 45, 'importing', + $now - 180); + $requestsBeforeOrphans = count($agent->requests); + $assert(CloudTemplateCatalogService::reconcileOnePendingImport() + && (string) CloudTemplate::find($orphanTemplateId)->state === 'error' + && CloudUserIsoService::reconcileOnePendingImport() + && (string) CloudUserIso::find($orphanIsoId)->state === 'failed' + && count($agent->requests) === $requestsBeforeOrphans, + 'Replica-free legacy parents were not failed without replay.'); + + foreach ($agent->requests as $request) { + $assert(!in_array($request['target'], + ['/v1/rescue-templates', '/v1/user-isos'], true), + 'Background convergence replayed an original import POST.'); + } + + echo "Rescue and user ISO background import convergence passed.\n"; +} diff --git a/web/tests/CloudTemplateCatalogTest.php b/web/tests/CloudTemplateCatalogTest.php index 5c6b10e0..fd4f81f4 100644 --- a/web/tests/CloudTemplateCatalogTest.php +++ b/web/tests/CloudTemplateCatalogTest.php @@ -146,16 +146,46 @@ $cleared = CloudTemplateCatalogService::saveStartupScript( $assert($cleared['startup_script'] === '' && $cleared['revision'] === 3, 'An empty script did not disable first-boot execution.'); +$operationBeforeBlockedMutation = (string) CloudTemplateReplica::query() + ->where('host_id', 1)->where('template_id', 'debian-12-template') + ->value('operation_id'); +foreach (['importing', 'error'] as $blockedState) { + Capsule::table('cloud_templates')->where('id', 'debian-12-template') + ->update(['state' => $blockedState]); + $expect(static fn () => CloudTemplateCatalogService::rename( + 'debian-12-template', 'Blocked rename', 3, $generation + ), 409, 'A non-active template was renamed.'); + $expect(static fn () => CloudTemplateCatalogService::saveStartupScript( + 'debian-12-template', 'blocked script', 3, $generation + ), 409, 'A non-active template startup script was changed.'); + $blocked = CloudTemplate::find('debian-12-template'); + $assert((int) $blocked->revision === 3 + && (string) $blocked->name === 'Debian 12' + && (string) $blocked->startup_script === '' + && (string) CloudTemplateReplica::query()->where('host_id', 1) + ->where('template_id', 'debian-12-template') + ->value('operation_id') === $operationBeforeBlockedMutation, + 'A blocked metadata mutation changed template or replica identity.'); +} +Capsule::table('cloud_templates')->where('id', 'debian-12-template') + ->update(['state' => 'active']); + $public = CloudTemplateCatalogService::publicCatalog()['templates'][0]; $admin = CloudTemplateCatalogService::adminCatalog()['templates'][0]; foreach (['startup_script', 'startup_script_configured', 'revision', - 'generation_token', 'source_url', 'sealed_source_url', 'sha256'] + 'generation_token', 'source_url', 'sealed_source_url', 'sha256', 'replicas'] as $privateField) { $assert(!array_key_exists($privateField, $public), 'Tenant catalog exposed private template field: ' . $privateField); } $assert($admin['startup_script_configured'] === false - && !array_key_exists('startup_script', $admin), + && !array_key_exists('startup_script', $admin) + && $admin['replicas'] === [[ + 'host_id' => 1, + 'state' => 'active', + 'error_code' => '', + 'operation_id' => $operationBeforeBlockedMutation, + ]], 'Admin catalog must show script state without returning script content.'); $row = CloudTemplate::find('debian-12-template'); $replica = CloudTemplateReplica::query()->where('host_id', 1) diff --git a/web/tests/CloudTemplateMetadataSyncStateMachineTest.php b/web/tests/CloudTemplateMetadataSyncStateMachineTest.php index 7fa7a89a..b21b7c02 100644 --- a/web/tests/CloudTemplateMetadataSyncStateMachineTest.php +++ b/web/tests/CloudTemplateMetadataSyncStateMachineTest.php @@ -56,6 +56,7 @@ namespace { use App\Models\CloudTemplateReplica; use App\Services\Cloud\CloudAgentSettingsService; + use App\Services\Cloud\CloudError; use App\Services\Cloud\CloudTemplateCatalogService; use App\Services\Cloud\TemplateMetadataSyncFakeAgent; use Illuminate\Database\Capsule\Manager as Capsule; @@ -66,12 +67,31 @@ namespace { throw new RuntimeException($message); } }; + $expect = static function (callable $callable, int $status, + string $message) use ($assert): void { + try { + $callable(); + } catch (CloudError $error) { + $assert($error->httpStatus === $status, + $message . ' Unexpected status: ' . $error->httpStatus); + return; + } + throw new RuntimeException($message); + }; $capsule = new Capsule(); $capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'], 'default'); $capsule->setAsGlobal(); $capsule->bootEloquent(); + $capsule->schema('default')->create('cloud_templates', + static function (Blueprint $table): void { + $table->string('id')->primary(); + $table->string('name'); + $table->integer('revision'); + $table->string('generation_token'); + $table->string('state'); + }); $capsule->schema('default')->create('cloud_template_replicas', static function (Blueprint $table): void { $table->integer('host_id'); @@ -91,6 +111,13 @@ namespace { $generation = str_repeat('a', 32); $id = 'debian-13-metadata-sync'; $now = time(); + Capsule::table('cloud_templates')->insert([ + 'id' => $id, + 'name' => 'Debian 13 renamed', + 'revision' => 2, + 'generation_token' => $generation, + 'state' => 'active', + ]); Capsule::table('cloud_template_replicas')->insert([ 'host_id' => 7, 'template_id' => $id, @@ -118,6 +145,14 @@ namespace { 'generation_token' => $generation, ]; $actor = ['user_id' => 1, 'can_manage' => true]; + $setAuthority = static function (int $revision, string $name, + string $state = 'active') use ($id): void { + Capsule::table('cloud_templates')->where('id', $id)->update([ + 'name' => $name, + 'revision' => $revision, + 'state' => $state, + ]); + }; $agent = new TemplateMetadataSyncFakeAgent(); CloudAgentSettingsService::$client = $agent; @@ -139,6 +174,7 @@ namespace { $agent->mode = 'wrong_operation'; $agent->result = $result(3, 'Debian 13 wrong operation'); + $setAuthority(3, 'Debian 13 wrong operation'); $mismatch = CloudTemplateCatalogService::synchronizeMetadata( $snapshot(3, 'Debian 13 wrong operation'), $actor ); @@ -163,6 +199,7 @@ namespace { ]); }; $agent->result = $result(4, 'Debian 13 older response'); + $setAuthority(4, 'Debian 13 older response'); CloudTemplateCatalogService::synchronizeMetadata( $snapshot(4, 'Debian 13 older response'), $actor ); @@ -175,6 +212,7 @@ namespace { $agent->mode = 'uncertain'; $agent->beforeResponse = null; $agent->result = $result(6, 'Debian 13 reconciled'); + $setAuthority(6, 'Debian 13 reconciled'); $before = count($agent->requests); $reconciled = CloudTemplateCatalogService::synchronizeMetadata( $snapshot(6, 'Debian 13 reconciled'), $actor @@ -189,5 +227,16 @@ namespace { && str_ends_with($requests[1]['target'], '/reconcile'), 'Uncertain metadata synchronization was replayed or not reconciled.'); + $setAuthority(7, 'Debian 13 blocked', 'importing'); + $requestsBeforeBlocked = count($agent->requests); + $operationBeforeBlocked = (string) $row->operation_id; + $expect(static fn () => CloudTemplateCatalogService::synchronizeMetadata( + $snapshot(7, 'Debian 13 blocked'), $actor + ), 409, 'A non-active template dispatched metadata synchronization.'); + $row->refresh(); + $assert(count($agent->requests) === $requestsBeforeBlocked + && (string) $row->operation_id === $operationBeforeBlocked, + 'Blocked metadata synchronization changed the replica operation ID.'); + echo "Template metadata operation binding and CAS tests passed.\n"; } diff --git a/web/tests/CloudTemplateWorkerConvergenceTest.php b/web/tests/CloudTemplateWorkerConvergenceTest.php new file mode 100644 index 00000000..c8bdfb62 --- /dev/null +++ b/web/tests/CloudTemplateWorkerConvergenceTest.php @@ -0,0 +1,240 @@ + 1, + 'is_admin' => true, + 'is_super_admin' => true, + 'can_manage' => true, + 'session_id' => $sessionId, + ]; + } + } + + final class TemplateWorkerFakeAgent + { + public array $requests = []; + public array $results = []; + + public function request(string $method, string $target, array $actor, + ?array $payload = null): array + { + $this->requests[] = compact('method', 'target', 'actor', 'payload'); + if (preg_match('#^/v1/operations/([A-Za-z0-9_-]+)$#D', + $target, $match) === 1 && $method === 'GET') { + $operationId = rawurldecode($match[1]); + return [ + 'operation_id' => $operationId, + 'status' => 'uncertain', + 'error' => ['code' => 'operation_uncertain'], + ]; + } + if (preg_match('#^/v1/operations/([A-Za-z0-9_-]+)/reconcile$#D', + $target, $match) === 1 && $method === 'POST') { + $operationId = rawurldecode($match[1]); + return [ + 'operation_id' => $operationId, + 'status' => 'succeeded', + 'result' => $this->results[$operationId], + ]; + } + throw new \RuntimeException('Template worker replayed an import'); + } + } + + final class CloudAgentSettingsService + { + public static ?TemplateWorkerFakeAgent $client = null; + + public static function clientForHost(int $hostId, + float $timeoutSeconds = 20.0): TemplateWorkerFakeAgent + { + if ($hostId !== 11 || self::$client === null) { + throw new \RuntimeException('Missing fake Agent'); + } + return self::$client; + } + } +} + +namespace { + require dirname(__DIR__) . '/vendor/autoload.php'; + + use App\Models\CloudTemplate; + use App\Models\CloudTemplateReplica; + use App\Services\Cloud\CloudAgentSettingsService; + use App\Services\Cloud\CloudWorkerService; + use App\Services\Cloud\TemplateWorkerFakeAgent; + use Illuminate\Database\Capsule\Manager as Capsule; + use Illuminate\Database\Schema\Blueprint; + + $assert = static function (bool $condition, string $message): void { + if (!$condition) { + throw new RuntimeException($message); + } + }; + + $capsule = new Capsule(); + $capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'], + 'default'); + $capsule->setAsGlobal(); + $capsule->bootEloquent(); + $schema = $capsule->schema('default'); + $schema->create('cloud_jobs', static function (Blueprint $table): void { + $table->increments('id'); + $table->string('kind'); + $table->string('status'); + $table->integer('available_at'); + $table->integer('lease_until'); + }); + $schema->create('cloud_templates', static function (Blueprint $table): void { + $table->string('id')->primary(); + $table->string('name'); + $table->string('os_family'); + $table->string('os_version'); + $table->string('source_type'); + $table->text('sealed_source_url'); + $table->integer('image_bytes')->default(0); + $table->string('sha256')->default(''); + $table->text('startup_script')->default(''); + $table->integer('revision'); + $table->string('generation_token'); + $table->string('state'); + $table->integer('created_at'); + $table->integer('updated_at'); + }); + $schema->create('cloud_template_replicas', + static function (Blueprint $table): void { + $table->integer('host_id'); + $table->string('template_id'); + $table->integer('revision'); + $table->string('generation_token'); + $table->string('state'); + $table->integer('image_bytes')->default(0); + $table->string('sha256')->default(''); + $table->string('operation_id')->default(''); + $table->string('error_code')->default(''); + $table->integer('created_at'); + $table->integer('updated_at'); + $table->primary(['host_id', 'template_id']); + }); + + $now = time(); + $digest = str_repeat('c', 64); + $operationA = str_repeat('a', 32); + $operationB = str_repeat('b', 32); + $generationA = str_repeat('1', 32); + $generationB = str_repeat('2', 32); + $insertTemplate = static function (string $id, string $generation, + string $digest, string $state, int $updatedAt) use ($now): void { + Capsule::table('cloud_templates')->insert([ + 'id' => $id, + 'name' => $id, + 'os_family' => 'Debian', + 'os_version' => '13', + 'source_type' => 'remote', + 'sealed_source_url' => 'test-sealed-source', + 'image_bytes' => 0, + 'sha256' => $digest, + 'startup_script' => '', + 'revision' => 1, + 'generation_token' => $generation, + 'state' => $state, + 'created_at' => $now, + 'updated_at' => $updatedAt, + ]); + }; + $insertReplica = static function (string $id, string $generation, + string $digest, string $operationId, string $state, + int $updatedAt): void { + Capsule::table('cloud_template_replicas')->insert([ + 'host_id' => 11, + 'template_id' => $id, + 'revision' => 1, + 'generation_token' => $generation, + 'state' => $state, + 'image_bytes' => $state === 'active' ? 8192 : 0, + 'sha256' => $digest, + 'operation_id' => $operationId, + 'error_code' => '', + 'created_at' => $updatedAt, + 'updated_at' => $updatedAt, + ]); + }; + + $insertTemplate('debian-a', $generationA, $digest, 'importing', $now - 30); + $insertReplica('debian-a', $generationA, $digest, $operationA, + 'pending', $now - 30); + $insertTemplate('debian-b', $generationB, $digest, 'importing', $now - 20); + $insertReplica('debian-b', $generationB, $digest, $operationB, + 'pending', $now - 20); + + $agent = new TemplateWorkerFakeAgent(); + $agent->results[$operationA] = [ + 'id' => 'debian-a', + 'revision' => 1, + 'generation_token' => $generationA, + 'size_bytes' => 8192, + 'sha256' => $digest, + ]; + CloudAgentSettingsService::$client = $agent; + + $policy = new CloudWorkerService(null, true); + $assert($policy->tick() === false && $agent->requests === [], + 'The policy-only worker consumed a template import.'); + + $worker = new CloudWorkerService(); + $assert($worker->tick() === true && $worker->observationErrors() === [], + 'The ordinary idle worker did not converge one template import.'); + $templateA = CloudTemplate::find('debian-a'); + $replicaA = CloudTemplateReplica::query()->where('host_id', 11) + ->where('template_id', 'debian-a')->first(); + $templateB = CloudTemplate::find('debian-b'); + $replicaB = CloudTemplateReplica::query()->where('host_id', 11) + ->where('template_id', 'debian-b')->first(); + $assert((string) $templateA->state === 'active' + && (int) $templateA->image_bytes === 8192 + && (string) $templateA->sha256 === $digest + && (string) $replicaA->state === 'active' + && (string) $templateB->state === 'importing' + && (string) $replicaB->state === 'pending', + 'One bounded worker pass changed the wrong import or lost image size.'); + $assert(count($agent->requests) === 2 + && $agent->requests[0]['method'] === 'GET' + && $agent->requests[0]['target'] === '/v1/operations/' . $operationA + && $agent->requests[1]['method'] === 'POST' + && $agent->requests[1]['target'] + === '/v1/operations/' . $operationA . '/reconcile' + && ($agent->requests[0]['actor']['session_id'] ?? null) + === 'cloud-template-import-worker' + && ($agent->requests[0]['actor']['operation_id'] ?? null) + === $operationA, + 'Worker lookup/reconciliation did not preserve the saved operation ID.'); + foreach ($agent->requests as $request) { + $assert($request['target'] !== '/v1/templates', + 'The worker replayed the original template import POST.'); + } + + Capsule::table('cloud_template_replicas')->where('template_id', 'debian-b') + ->update(['state' => 'failed', 'error_code' => 'test_terminal']); + Capsule::table('cloud_templates')->where('id', 'debian-b') + ->update(['state' => 'error']); + $generationC = str_repeat('3', 32); + $insertTemplate('debian-c', $generationC, $digest, 'importing', $now - 10); + $insertReplica('debian-c', $generationC, $digest, str_repeat('d', 32), + 'active', $now - 10); + $requestsBeforeAggregateRepair = count($agent->requests); + $assert($worker->tick() === true + && (string) CloudTemplate::find('debian-c')->state === 'active' + && count($agent->requests) === $requestsBeforeAggregateRepair, + 'Worker did not repair an all-active template aggregate locally.'); + $assert($worker->tick() === false, + 'A terminal failed template kept the idle worker in a busy loop.'); + + echo "Idle worker template import convergence and no-replay tests passed.\n"; +} diff --git a/web/tests/CloudWorkerCatalogFairnessTest.php b/web/tests/CloudWorkerCatalogFairnessTest.php new file mode 100644 index 00000000..af759833 --- /dev/null +++ b/web/tests/CloudWorkerCatalogFairnessTest.php @@ -0,0 +1,213 @@ +addConnection(['driver' => 'sqlite', 'database' => ':memory:'], + 'default'); + $capsule->setAsGlobal(); + $capsule->bootEloquent(); + $schema = $capsule->schema('default'); + $schema->create('user', static function (Blueprint $table): void { + $table->increments('id'); + $table->integer('enable')->default(1); + $table->integer('is_admin')->default(0); + }); + $schema->create('cloud_jobs', static function (Blueprint $table): void { + $table->increments('id'); + $table->string('operation_id')->unique(); + $table->integer('order_id')->nullable(); + $table->integer('instance_id')->nullable(); + $table->integer('snapshot_id')->nullable(); + $table->integer('user_id'); + $table->integer('host_id'); + $table->string('kind'); + $table->string('status'); + $table->text('sealed_payload'); + $table->integer('attempts')->default(0); + $table->integer('dispatched_at')->nullable(); + $table->string('lease_token')->nullable(); + $table->integer('lease_until')->default(0); + $table->integer('available_at'); + $table->string('error_code')->nullable(); + $table->text('result_json')->nullable(); + $table->integer('created_at'); + $table->integer('updated_at'); + }); + Capsule::table('user')->insert(['id' => 1, 'enable' => 1, 'is_admin' => 0]); + + $responses = []; + $now = time(); + for ($number = 1; $number <= 3; $number++) { + $operationId = str_repeat((string) $number, 32); + Capsule::table('cloud_jobs')->insert([ + 'operation_id' => $operationId, + 'order_id' => null, + 'instance_id' => null, + 'snapshot_id' => null, + 'user_id' => 1, + 'host_id' => 7, + 'kind' => 'action', + 'status' => 'queued', + 'sealed_payload' => CloudSecrets::encrypt([ + 'method' => 'POST', + 'request_target' => '/v1/vms/test-vm/actions', + 'body' => [ + 'operation_id' => $operationId, + 'action' => 'start', + ], + 'owner_id' => 1, + ], 'job-payload'), + 'attempts' => 0, + 'dispatched_at' => null, + 'lease_token' => null, + 'lease_until' => 0, + 'available_at' => 0, + 'error_code' => null, + 'result_json' => null, + 'created_at' => $now, + 'updated_at' => $now, + ]); + $responses[] = new Response(200, [], json_encode([ + 'operation_id' => $operationId, + 'status' => 'dispatching', + ], JSON_THROW_ON_ERROR)); + } + + $client = new CloudAgentClient( + '7', + 'test-agent-secret-at-least-32-bytes', + new Client(['handler' => HandlerStack::create(new MockHandler($responses))]), + 20.0, + 'http://127.0.0.1:8001/internal/agent/request', + 'test-gateway-secret-at-least-32-bytes' + ); + $worker = new CloudWorkerService($client); + + for ($tick = 1; $tick <= 3; $tick++) { + $assert($worker->tick(), 'A due cloud job was not processed.'); + $assert(CloudJob::query()->where('status', 'running')->count() === $tick, + 'The persistent cloud_jobs backlog did not advance one row.'); + $assert(CloudTemplateCatalogService::$calls + + CloudRescueTemplateImportService::$calls + + CloudUserIsoService::$calls === $tick, + 'One worker tick ran more than one catalog action.'); + } + $assert(CloudTemplateCatalogService::$calls === 1 + && CloudRescueTemplateImportService::$calls === 1 + && CloudUserIsoService::$calls === 1, + 'A continuously non-empty cloud_jobs queue starved a catalog.'); + + // A catalog exception is observable but is not work. Returning false keeps + // the command loop's one-second idle backoff and prevents a log/DB hot loop. + CloudTemplateCatalogService::$fail = true; + $assert($worker->tick() === false, + 'A catalog exception incorrectly reported successful work.'); + $assert($worker->observationErrors() === ['worker_tick_failed'], + 'A catalog exception was not exposed through worker health.'); + $assert(CloudTemplateCatalogService::$calls === 2 + && CloudRescueTemplateImportService::$calls === 1 + && CloudUserIsoService::$calls === 1, + 'The worker ran a second catalog after an ambiguous catalog failure.'); + + $beforePolicy = [ + CloudTemplateCatalogService::$calls, + CloudRescueTemplateImportService::$calls, + CloudUserIsoService::$calls, + ]; + $policyWorker = new CloudWorkerService(null, true); + $assert($policyWorker->tick() === false + && $beforePolicy === [ + CloudTemplateCatalogService::$calls, + CloudRescueTemplateImportService::$calls, + CloudUserIsoService::$calls, + ], 'The policy-only worker was delayed by catalog convergence.'); + + echo "Cloud worker catalog fairness and failure backoff tests passed.\n"; +}