diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 20c24847..18d9292d 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -117,7 +117,7 @@ Agent 配置字段: 先在宿主机创建五个互相独立的真实目录,并为 libvirt/QEMU 配置正确的属主、权限、挂载和剩余空间;Agent 不会创建目录、挂载文件系统或删除其中的数据。保存并启用 Agent 连接后,在后台“虚拟机管理 -> 宿主机 -> 存储目录”填写系统盘与快照、云系统模板、cloud-init Seed、用户 ISO、救援 ISO 五类目录。Agent 校验目录是 `storage_path_roots` 的真实子目录后,分别注册固定名称的 persistent `dir` pool:`virt-cloud-managed`、`virt-cloud-templates`、`virt-cloud-seeds`、`virt-cloud-user-isos`、`virt-cloud-rescue-isos`,并启动、刷新和启用 autostart。若固定名称已被其他目录占用,Agent 会拒绝注册;不会扫描、复用或迁移其他 pool。 -libvirt 的持久 pool 定义就是 Agent 后续使用的映射,不另写存储状态文件。系统盘与快照、模板、Seed、用户 ISO 和救援 ISO 后续只使用各自固定 pool。当前实现不兼容旧存储布局,也不提供自动迁移或回退读取;已有数据必须在启用本功能前另行完成受控迁移。首次成功应用后网页不允许更换路径。注册成功后回到 `/admin/agents` 重新执行连接测试;`storage_configuration` capability 可用前,宿主机不会进入调度。 +libvirt 的持久 pool 定义就是 Agent 后续使用的映射,不另写存储状态文件。系统盘与快照、模板、Seed、用户 ISO 和救援 ISO 后续只使用各自固定 pool。当前实现不兼容旧存储布局,也不提供自动迁移或回退读取;已有数据必须在启用本功能前另行完成受控迁移。首次成功应用后网页不允许更换路径。若任务以失败终态结束,旧 CloudJob 和 operation ID 保留为审计记录且绝不重放;管理员可修正目录后显式提交一个新 CloudJob 和新 operation ID。这个新任务只会幂等地核验并补全固定 pool;既不更换已有 pool 的路径,也不迁移或删除数据。注册成功后回到 `/admin/agents` 重新执行连接测试;`storage_configuration` capability 可用前,宿主机不会进入调度。 网络接入统一在宿主机页面填写“接入类型”和“网桥 / 网络名称”,新建默认 Linux 网桥 `br0`。面板保存为 `bridge:br0` 或 `network:default` 这样的 `default_network`;名称以字母或数字开头,后续只能包含字母、数字、点、下划线和短横线,bridge 名称为 1~15 字符,libvirt network 名称为 1~64 字符。宿主机上的网桥或 libvirt network 必须由运维预先准备。Agent 只选择、验证和使用现有网络,不创建网桥、不启动 libvirt network,也不修改宿主机系统网卡、路由或挂载。 diff --git a/web/app/Services/Cloud/CloudHostStorageConfigurationService.php b/web/app/Services/Cloud/CloudHostStorageConfigurationService.php index 27b8a71b..b80d983f 100644 --- a/web/app/Services/Cloud/CloudHostStorageConfigurationService.php +++ b/web/app/Services/Cloud/CloudHostStorageConfigurationService.php @@ -7,7 +7,7 @@ use App\Models\CloudHostAgent; use App\Models\CloudHostStorageConfiguration; use App\Models\CloudJob; -/** One persisted, single-dispatch registration of a host's storage layout. */ +/** Current desired host storage layout plus its latest single-dispatch task. */ final class CloudHostStorageConfigurationService { private const DIRECTORY_FIELDS = [ @@ -45,20 +45,39 @@ final class CloudHostStorageConfigurationService $row = CloudHostStorageConfiguration::query() ->where('host_id', $hostId)->lockForUpdate()->first(); $actualRevision = $row === null ? 0 : (int) $row->revision; + $samePaths = $row !== null && self::samePaths($row, $paths); if ($actualRevision !== $expectedRevision) { throw new CloudError('存储配置已被修改,请刷新页面后重试', 409); } if ($row !== null) { - if (self::samePaths($row, $paths) - && (int) ($row->job_id ?? 0) > 0) { - $prior = CloudJob::find((int) $row->job_id); - if ($prior !== null) { + $prior = self::linkedJob($row); + $state = (string) $row->state; + if ($state === 'applied') { + if ($samePaths && $prior !== null + && (string) $prior->phase === 'terminal' + && (string) $prior->outcome === 'succeeded') { return $prior; } + throw new CloudError( + '宿主机存储配置已成功应用,不允许更换路径', 409 + ); + } + if ($state === 'applying' + && $prior !== null + && (string) $prior->phase !== 'terminal') { + if ($samePaths) { + return $prior; + } + throw new CloudError( + '宿主机存储配置任务正在执行,请等待终态回执', 409 + ); + } + if (!in_array($state, ['applying', 'failed'], true) + || !self::isTerminalFailure($prior)) { + throw new CloudError( + '宿主机存储配置状态与任务账本不一致,请联系管理员核查', 409 + ); } - throw new CloudError( - '宿主机存储配置已有任务或终态记录,禁止重复下发', 409 - ); } $setting = CloudHostAgent::query()->where('host_id', $hostId) ->lockForUpdate()->first(); @@ -66,11 +85,16 @@ final class CloudHostStorageConfigurationService || trim((string) $setting->sealed_api_secret) === '') { throw new CloudError('请先启用并配置宿主机 Agent API', 409); } + // The revision describes the desired paths, not the number of + // attempts. Re-applying the same failed layout gets a new task and + // operation ID without pretending that the configuration changed. + $nextRevision = $row === null || !$samePaths + ? $actualRevision + 1 : $actualRevision; $operationId = bin2hex(random_bytes(16)); [$job, $created] = CloudJobService::reserveStandaloneAgentMutation( $actor, $hostId, 'host_storage_configure', 'POST', '/v1/storage/configuration', $paths, - ['can_manage' => true, 'storage_revision' => 1], + ['can_manage' => true, 'storage_revision' => $nextRevision], [ 'host_id' => $hostId, 'expected_revision' => $expectedRevision, @@ -81,14 +105,17 @@ final class CloudHostStorageConfigurationService if (!$created) { return $job; } - $row = new CloudHostStorageConfiguration(); - $row->host_id = $hostId; - $row->revision = 1; + if ($row === null) { + $row = new CloudHostStorageConfiguration(); + $row->host_id = $hostId; + $row->created_at = time(); + } + $row->revision = $nextRevision; self::assignPaths($row, $paths); $row->state = 'applying'; $row->operation_id = $operationId; $row->job_id = (int) $job->id; - $row->created_at = $row->updated_at = time(); + $row->updated_at = time(); $row->save(); CloudAgentSettingsService::invalidateReadiness($setting); return $job; @@ -107,14 +134,26 @@ final class CloudHostStorageConfigurationService } $row = CloudHostStorageConfiguration::query() ->where('host_id', (int) $job->host_id) - ->where('operation_id', (string) $job->operation_id) ->lockForUpdate()->first(); if ($row === null) { throw new CloudError('存储配置任务账本不存在', 409); } + if (!hash_equals((string) $row->operation_id, + (string) $job->operation_id)) { + // A terminally failed generation may be superseded by an explicit + // new request. A late receipt from that old generation still + // closes its CloudJob, but must never overwrite the current row. + return; + } + if ((int) ($row->job_id ?? 0) !== (int) $job->id) { + throw new CloudError('存储配置任务账本关联不一致', 409); + } if (in_array((string) $row->state, ['applied', 'failed'], true)) { return; } + if ((string) $row->state !== 'applying') { + throw new CloudError('存储配置状态无效', 409); + } if ($status === 'succeeded') { if (!is_array($result) || array_is_list($result) || ($result['configured'] ?? null) !== true) { @@ -180,6 +219,30 @@ final class CloudHostStorageConfigurationService return true; } + private static function linkedJob($row): ?CloudJob + { + $jobId = (int) ($row->job_id ?? 0); + if ($jobId < 1 || trim((string) $row->operation_id) === '') { + return null; + } + $job = CloudJob::find($jobId); + if ($job === null + || (int) $job->host_id !== (int) $row->host_id + || (string) $job->kind !== 'host_storage_configure' + || !hash_equals((string) $row->operation_id, + (string) $job->operation_id)) { + return null; + } + return $job; + } + + private static function isTerminalFailure(?CloudJob $job): bool + { + return $job !== null + && (string) $job->phase === 'terminal' + && (string) $job->outcome === 'failed'; + } + private static function assignPaths($row, array $paths): void { foreach (self::DIRECTORY_FIELDS as $field) { @@ -199,12 +262,24 @@ final class CloudHostStorageConfigurationService { $setting = CloudHostAgent::find((int) $host->id); $state = $row === null ? 'unconfigured' : (string) $row->state; + $terminalFailure = $row !== null + && self::isTerminalFailure(self::linkedJob($row)); + // A worker may terminalize its CloudJob after a local finalization + // incident left this projection at applying. The immutable task ledger + // is authoritative, so present that condition as failed and recoverable. + if ($state === 'applying' && $terminalFailure) { + $state = 'failed'; + } + $canSubmit = $row === null || ($state === 'failed' && $terminalFailure); $result = [ 'host_id' => (int) $host->id, 'host_name' => (string) $host->name, 'revision' => $row === null ? 0 : (int) $row->revision, 'state' => $state, 'configured' => $state === 'applied', + 'can_submit' => $canSubmit, + 'operation_id' => $row === null + ? null : (string) $row->operation_id, 'requires_agent_retest' => $state === 'applied' && ($setting === null || !$setting->isReady() || !CloudNetworkSelector::isValid($host->default_network)), diff --git a/web/public/cloud/admin-operations.js b/web/public/cloud/admin-operations.js index 57e4fb7b..267c44c9 100644 --- a/web/public/cloud/admin-operations.js +++ b/web/public/cloud/admin-operations.js @@ -171,6 +171,9 @@ function shownFilters() { function failureText(job, view) { if (view.protocolError) return '任务数据不符合公开协议;资源按可能已生效处理并保持锁定。'; if (view.status !== 'failed') return ''; + if (job.kind === 'host_storage_configure') { + return '旧存储任务已终止且不会重放;部分固定存储池可能已应用。可到宿主机存储目录页修正后提交新的独立任务。'; + } if (job.effect_may_have_applied === true && job.effect_resolution === 'unknown') { return '操作效果可能已发生;资源已安全锁定,需管理员人工处理。'; } @@ -376,10 +379,14 @@ function renderDetail(payload) { ['创建时间', date(job.created_at)], ['更新时间', date(job.updated_at)], ])); const failure = failureText(job, view); + const storageFailure = job.kind === 'host_storage_configure' + && view.status === 'failed'; if (failure) body.append(el('h3', 'h6 mt-4', '失败详情'), el('p', 'text-danger text-break', failure)); if (job.finalization_incident === true) { body.append(el('h3', 'h6 mt-4', '业务账本收尾事件'), - el('p', 'text-danger text-break', `任务已终态失败,但业务账本未能安全收敛;资金和资源已保持锁定,不会自动重试。${job.finalization_error_code ? ` 事件代码:${job.finalization_error_code}` : ''}`)); + el('p', 'text-danger text-break', storageFailure + ? `旧存储任务已终态失败,收尾事件仅保留审计且不会重放;存储目录页仍可以终态任务账本为准提交新的独立任务。${job.finalization_error_code ? ` 事件代码:${job.finalization_error_code}` : ''}` + : `任务已终态失败,但业务账本未能安全收敛;资金和资源已保持锁定,不会自动重试。${job.finalization_error_code ? ` 事件代码:${job.finalization_error_code}` : ''}`)); } if (state.inspection) { const inspectedAt = state.inspection.checked_at || state.inspection.inspected_at; @@ -387,7 +394,9 @@ function renderDetail(payload) { body.append(el('h3', 'h6 mt-4', paymentEvidence ? '支付账本审计' : '宿主机只读检查'), el('p', 'text-muted fs-13', paymentEvidence ? `已记录外部支付平台核验结论${inspectedAt ? ` · ${date(inspectedAt)}` : ''}。原任务仍保持失败,且没有重新下单。` - : `已完成只读检查${inspectedAt ? ` · ${date(inspectedAt)}` : ''}。检查不会执行 Agent 操作;任务仍以失败详情和资源安全栅栏为准。`)); + : storageFailure + ? `已完成只读检查${inspectedAt ? ` · ${date(inspectedAt)}` : ''}。检查不会执行 Agent 操作;旧任务仍保持失败,也不会影响存储目录页提交新的独立任务。` + : `已完成只读检查${inspectedAt ? ` · ${date(inspectedAt)}` : ''}。检查不会执行 Agent 操作;任务仍以失败详情和资源安全栅栏为准。`)); } for (const note of payload.notes || []) body.append(el('p', 'text-muted fs-13 mt-3 mb-0', note)); const links = byId('operation-detail-links'); links.replaceChildren(); diff --git a/web/public/cloud/admin.js b/web/public/cloud/admin.js index 5eb3408c..7df0d8c2 100644 --- a/web/public/cloud/admin.js +++ b/web/public/cloud/admin.js @@ -183,20 +183,20 @@ function renderHostStorage(payload) { byId('host-storage-host-id').value = payload.host_id; byId('host-storage-revision').value = Number(payload.revision || 0); const validState = ['unconfigured', 'applying', 'failed', 'applied'].includes(payload.state); - const locked = payload.configured === true || payload.state === 'applying' - || !validState || (payload.state === 'failed' && payload.effect_may_have_applied === true); + const canSubmit = validState && payload.can_submit === true; + const locked = !canSubmit; for (const [field, id] of Object.entries(storageFields)) { const input = byId(id); input.value = payload[field] || ''; input.readOnly = locked; input.disabled = !canManage; } - const labels = { unconfigured: '尚未配置', applying: '正在等待 Agent 完成', failed: '应用失败', applied: '已应用' }; + const labels = { unconfigured: '尚未配置', applying: '正在等待 Agent 完成', failed: '上一次任务已失败且不会自动重试;可修正目录后提交新的独立任务', applied: '已应用' }; let message = labels[payload.state] || '状态协议无效,资源已安全锁定,需管理员人工处理'; - if (payload.state === 'failed' && payload.effect_may_have_applied === true) { - message = '应用失败,但操作效果可能已经发生。资源已安全锁定,需管理员人工处理'; + if (payload.state === 'failed' && !canSubmit) { + message = '应用失败,但任务账本状态不完整,请联系管理员核查'; } if (payload.operation_id && payload.state !== 'applied') message += ` · operation_id ${payload.operation_id}`; if (payload.requires_agent_retest) message += ' · 请前往 Agent API 重新测试后再启用调度'; hostStorageNotice(message, payload.state === 'failed' || !validState); - const save = byId('host-storage-save'); if (save) { save.dataset.applied = payload.configured === true ? '1' : '0'; save.dataset.taskPending = '0'; save.disabled = !canManage || payload.configured === true; } + const save = byId('host-storage-save'); if (save) { save.dataset.applied = payload.configured === true ? '1' : '0'; save.dataset.taskPending = '0'; save.disabled = !canManage || !canSubmit; } } function lockHostStorageForTask(locked) { for (const id of Object.values(storageFields)) byId(id).readOnly = locked; @@ -240,8 +240,19 @@ async function saveHostStorage(event) { .catch(error => hostStorageNotice(`任务 #${task.id} 已成功,但刷新存储配置失败:${error.message}`, true)); return; } - lockHostStorageForTask(task.safe_to_resubmit !== true); - hostStorageNotice(taskFailureMessage(task, '宿主机存储配置失败。'), true); + const failure = task?.error || '宿主机存储配置失败。'; + const partial = task?.effect_may_have_applied === true + ? ' 部分固定存储池可能已经应用;新任务只会核验并补全,不会迁移或删除数据。' : ''; + void api(`/api/v1/admin/hosts/${encodeURIComponent(hostId)}/storage-configuration`) + .then(payload => { + if (!currentHost()) return; + renderHostStorage(payload); + const next = payload.state === 'failed' && payload.can_submit === true + ? ' 旧任务已终止且不会重放;修正目录后可提交新的独立任务。' + : ' 当前任务账本尚未进入可提交状态,请联系管理员核查。'; + hostStorageNotice(`${failure}${partial}${next}`, true); + }) + .catch(error => hostStorageNotice(`${failure} 刷新存储状态失败:${error.message}`, true)); }, onStop: reason => { if (currentHost() && reason === 'deadline') hostStorageNotice('自动观察已到达截止时间。任务可能仍在后台执行;当前配置保持锁定,请从任务记录核对结果。', true); diff --git a/web/public/cloud/task-center.js b/web/public/cloud/task-center.js index 65b69c4d..436c9cf9 100644 --- a/web/public/cloud/task-center.js +++ b/web/public/cloud/task-center.js @@ -189,7 +189,9 @@ function renderItems(emptyMessage = '暂无任务记录。') { if (meta.children.length) article.append(meta); if (task.status === 'failed') { article.append(element('p', 'panel-task-error', task.error || '任务执行失败。')); - if (task.effect_may_have_applied) { + if (raw?.kind === 'host_storage_configure') { + article.append(element('p', 'panel-task-warning', '旧存储任务已终止且不会重放;可到宿主机存储目录页修正后提交新的独立任务。')); + } else if (task.effect_may_have_applied) { article.append(element('p', 'panel-task-warning', '操作效果可能已经发生。为避免重复执行,请勿再次提交;请将任务编号提供给管理员核查。')); } else if (task.safe_to_resubmit === false) { article.append(element('p', 'panel-task-warning', '此任务不可再次提交,请联系管理员核查。')); diff --git a/web/sql/cloud.sql b/web/sql/cloud.sql index 37311948..b7ab5b9b 100644 --- a/web/sql/cloud.sql +++ b/web/sql/cloud.sql @@ -81,8 +81,9 @@ CREATE TABLE cloud_host_agents ( ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- The panel stores only the requested host-local directories; the Agent owns --- validation and registration of the five fixed libvirt pools. A dispatched --- operation_id is retained as an immutable audit link to its one CloudJob. +-- validation and registration of the five fixed libvirt pools. This row +-- points at the latest requested generation; superseded failed generations +-- remain immutable in cloud_jobs under their original operation IDs. CREATE TABLE cloud_host_storage_configurations ( host_id BIGINT UNSIGNED NOT NULL PRIMARY KEY, revision BIGINT UNSIGNED NOT NULL DEFAULT 0, diff --git a/web/tests/AdminAcceptedTaskBehaviorTest.mjs b/web/tests/AdminAcceptedTaskBehaviorTest.mjs index 7ceb8480..0b6e3bce 100644 --- a/web/tests/AdminAcceptedTaskBehaviorTest.mjs +++ b/web/tests/AdminAcceptedTaskBehaviorTest.mjs @@ -25,8 +25,10 @@ assert.match(storage, /observeTask\(receipt/); assert.match(storage, /task\.status === 'succeeded'/); assert.match(storage, /api\(`\/api\/v1\/admin\/hosts\/\$\{encodeURIComponent\(hostId\)\}\/storage-configuration`\)/); assert.match(storage, /renderHostStorage\(payload\)/); -assert.match(storage, /taskFailureMessage\(task/); -assert.match(storage, /task\.safe_to_resubmit !== true/); +assert.match(storage, /task\?\.effect_may_have_applied === true/); +assert.match(storage, /payload\.state === 'failed' && payload\.can_submit === true/); +assert.match(storage, /旧任务已终止且不会重放/); +assert.doesNotMatch(storage, /task\.safe_to_resubmit/); assert.doesNotMatch(storage, /setInterval\(/); const catalog = source.slice(source.indexOf('async function loadCatalog'), source.indexOf('function taskFailureMessage')); diff --git a/web/tests/AdminOperationsCancelBehaviorTest.mjs b/web/tests/AdminOperationsCancelBehaviorTest.mjs index 09fa0d03..bd2c3494 100644 --- a/web/tests/AdminOperationsCancelBehaviorTest.mjs +++ b/web/tests/AdminOperationsCancelBehaviorTest.mjs @@ -22,6 +22,11 @@ assert.match(source, /label: '协议错误'/); assert.match(source, /effect_may_have_applied === true/); assert.match(source, /unresolved_effect/); assert.match(source, /资源已安全锁定,需管理员人工处理/); +assert.match(source, /job\.kind === 'host_storage_configure'/); +assert.match(source, /可到宿主机存储目录页修正后提交新的独立任务/); +assert.match(source, /storageFailure/); +assert.match(source, /收尾事件仅保留审计且不会重放/); +assert.match(source, /不会影响存储目录页提交新的独立任务/); // The one retained high-risk action creates a distinct task once. The UI // validates the fresh receipt, immediately acknowledges it, and observes the diff --git a/web/tests/CloudHostStorageConfigurationContractTest.php b/web/tests/CloudHostStorageConfigurationContractTest.php index ad6633ea..20294d2b 100644 --- a/web/tests/CloudHostStorageConfigurationContractTest.php +++ b/web/tests/CloudHostStorageConfigurationContractTest.php @@ -46,8 +46,11 @@ $assert(str_contains($service, "'host_storage_configure', 'POST'") && str_contains($service, "'/v1/storage/configuration', \$paths") && str_contains($service, 'CloudJobService::acknowledgement($job)') + && str_contains($service, 'isTerminalFailure($prior)') + && str_contains($service, "'can_submit' => \$canSubmit") + && str_contains($service, '!hash_equals((string) $row->operation_id') && !str_contains($service, "'GET', '/v1/storage/configuration'"), - 'Storage registration lost its one-shot durable task contract.'); + 'Storage registration lost its single-dispatch or failure-supersession contract.'); $assert(str_contains($agentSettings, 'cloud_host_storage_configurations as host_storage') && str_contains($placement, diff --git a/web/tests/CloudHostStorageConfigurationTest.php b/web/tests/CloudHostStorageConfigurationTest.php index c5bb1160..dca7e882 100644 --- a/web/tests/CloudHostStorageConfigurationTest.php +++ b/web/tests/CloudHostStorageConfigurationTest.php @@ -1,5 +1,315 @@ httpStatus === $status, + $message . ' Unexpected status: ' . $error->httpStatus); + return; + } + throw new RuntimeException($message); + }; + $terminalize = static function (CloudJob $job, string $outcome, + bool $effectMayHaveApplied = false): void { + $now = time(); + $job->phase = 'terminal'; + $job->outcome = $outcome; + $job->status = $outcome; + $job->attempts = 1; + $job->dispatch_attempts = 1; + $job->dispatch_started_at = $now - 2; + $job->dispatched_at = $now - 2; + $job->agent_accepted_at = $now - 1; + $job->receipt_deadline_at = $now + 900; + $job->terminal_at = $now; + $job->effect_may_have_applied = $effectMayHaveApplied ? 1 : 0; + $job->safe_to_resubmit = 0; + $job->effect_resolution = $outcome === 'succeeded' + ? 'applied' : ($effectMayHaveApplied ? 'unknown' : 'not_applied'); + $job->lease_token = null; + $job->lease_until = 0; + $job->available_at = 0; + $job->updated_at = $now; + $job->save(); + }; + + $capsule = new Capsule(); + $capsule->addConnection([ + 'driver' => 'sqlite', 'database' => ':memory:', + ], 'default'); + $capsule->setAsGlobal(); + $capsule->bootEloquent(); + $schema = $capsule->schema('default'); + + $schema->create('cloud_hosts', static function (Blueprint $table): void { + $table->increments('id'); + $table->string('name'); + $table->string('region'); + $table->string('country_code', 2); + $table->boolean('enabled'); + $table->string('default_network'); + $table->integer('created_at'); + $table->integer('updated_at'); + }); + $schema->create('cloud_host_agents', static function (Blueprint $table): void { + $table->integer('host_id')->primary(); + $table->text('sealed_api_secret'); + $table->boolean('enabled')->default(false); + $table->integer('revision')->default(1); + $table->integer('tested_revision')->default(0); + $table->string('last_test_status')->default('never'); + $table->string('last_error_code')->default(''); + $table->string('last_agent_version')->default(''); + $table->integer('last_tested_at')->default(0); + $table->integer('created_at'); + $table->integer('updated_at'); + }); + $schema->create('cloud_host_storage_configurations', + static function (Blueprint $table): void { + $table->integer('host_id')->primary(); + $table->integer('revision')->default(0); + foreach (['managed', 'template', 'seed', 'user_iso', 'rescue'] as $role) { + $table->text($role . '_directory'); + } + $table->string('state')->default('applying'); + $table->string('operation_id')->nullable()->unique(); + $table->integer('job_id')->nullable(); + $table->integer('created_at'); + $table->integer('updated_at'); + }); + $schema->create('cloud_jobs', static function (Blueprint $table): void { + $table->increments('id'); + $table->string('operation_id')->unique(); + $table->string('request_idempotency_key'); + $table->string('request_fingerprint'); + $table->integer('parent_job_id')->nullable(); + $table->integer('order_id')->nullable(); + $table->integer('invoice_id')->nullable(); + $table->integer('instance_id')->nullable(); + $table->integer('snapshot_id')->nullable(); + $table->integer('user_id'); + $table->integer('host_id')->nullable(); + $table->string('kind'); + $table->string('execution_mode'); + $table->string('phase'); + $table->string('outcome')->nullable(); + $table->string('status'); + $table->text('sealed_payload'); + $table->integer('attempts'); + $table->integer('dispatch_attempts'); + $table->integer('dispatch_deadline_at'); + $table->integer('dispatch_started_at')->nullable(); + $table->integer('agent_accepted_at')->nullable(); + $table->integer('receipt_deadline_at')->nullable(); + $table->integer('terminal_at')->nullable(); + $table->boolean('effect_may_have_applied')->nullable(); + $table->boolean('safe_to_resubmit')->nullable(); + $table->string('effect_resolution'); + $table->integer('effect_resolved_at')->nullable(); + $table->integer('effect_resolved_by')->nullable(); + $table->text('effect_resolution_reason')->nullable(); + $table->string('completion_event_id')->nullable(); + $table->integer('read_at')->nullable(); + $table->integer('dispatched_at')->nullable(); + $table->string('lease_token')->nullable(); + $table->integer('lease_until')->default(0); + $table->integer('available_at'); + $table->string('error_code')->nullable(); + $table->boolean('finalization_incident')->default(0); + $table->string('finalization_error_code')->nullable(); + $table->text('result_json')->nullable(); + $table->integer('created_at'); + $table->integer('updated_at'); + }); + + foreach ([1, 2] as $hostId) { + Capsule::table('cloud_hosts')->insert([ + 'id' => $hostId, 'name' => 'Host ' . $hostId, + 'region' => 'HongKong', 'country_code' => 'HK', 'enabled' => 1, + 'default_network' => 'bridge:br0', + 'created_at' => 1, 'updated_at' => 1, + ]); + Capsule::table('cloud_host_agents')->insert([ + 'host_id' => $hostId, 'sealed_api_secret' => 'test-secret', + 'enabled' => 1, 'revision' => 1, 'tested_revision' => 1, + 'last_test_status' => 'healthy', 'last_error_code' => '', + 'last_agent_version' => 'test', 'last_tested_at' => 1, + 'created_at' => 1, 'updated_at' => 1, + ]); + } + + $actor = [ + 'user_id' => 1, 'is_admin' => true, 'is_super_admin' => true, + 'can_manage' => true, 'session_id' => 'storage-test', + ]; + $first = [ + 'expected_revision' => 0, + 'managed_directory' => '/home/libvirt/kvm', + 'template_directory' => '/home/libvirt/images', + 'seed_directory' => '/home/libvirt/seed', + 'user_iso_directory' => '/home/libvirt/iso', + 'rescue_directory' => '/home/libvirt/rescue-iso', + ]; + + $empty = CloudHostStorageConfigurationService::read(1); + $assert($empty['state'] === 'unconfigured' + && $empty['revision'] === 0 && $empty['can_submit'] === true, + 'An unconfigured host was not writable.'); + + $receipt1 = CloudHostStorageConfigurationService::save(1, $first, $actor); + $job1 = CloudJob::find((int) $receipt1['task_id']); + $row = CloudHostStorageConfiguration::find(1); + $assert($job1 !== null && $row !== null + && (string) $row->operation_id === (string) $job1->operation_id + && (int) $row->job_id === (int) $job1->id + && (int) $row->revision === 1 && (string) $row->state === 'applying', + 'Initial storage submission did not create one linked task.'); + + $sameActive = $first; + $sameActive['expected_revision'] = 1; + $activeReceipt = CloudHostStorageConfigurationService::save( + 1, $sameActive, $actor + ); + $assert($activeReceipt['task_id'] === $receipt1['task_id'] + && CloudJob::count() === 1, + 'An active same-path request created another task.'); + $changedActive = $sameActive; + $changedActive['rescue_directory'] = '/home/libvirt/rescue-corrected'; + $expect(static fn () => CloudHostStorageConfigurationService::save( + 1, $changedActive, $actor + ), 409, 'An active task allowed concurrent path replacement.'); + + CloudHostStorageConfigurationService::finishJobLocked( + $job1, 'failed', null, ['effect_may_have_applied' => true] + ); + $terminalize($job1, 'failed', true); + $failed = CloudHostStorageConfigurationService::read(1); + $assert($failed['state'] === 'failed' + && $failed['can_submit'] === true + && $failed['operation_id'] === (string) $job1->operation_id, + 'A terminal failure remained blocked.'); + + $receipt2 = CloudHostStorageConfigurationService::save( + 1, $sameActive, $actor + ); + $job2 = CloudJob::find((int) $receipt2['task_id']); + $row = CloudHostStorageConfiguration::find(1); + $assert($job2 !== null + && (int) $job2->id !== (int) $job1->id + && (string) $job2->operation_id !== (string) $job1->operation_id + && (int) $row->revision === 1 + && (string) $row->operation_id === (string) $job2->operation_id + && (string) $row->state === 'applying' + && (string) $job1->phase === 'terminal' + && (string) $job1->outcome === 'failed', + 'A same-path correction did not create a new operation while preserving the failed audit.'); + + CloudHostStorageConfigurationService::finishJobLocked( + $job1, 'succeeded', ['configured' => true], [] + ); + $row->refresh(); + $assert((string) $row->operation_id === (string) $job2->operation_id + && (string) $row->state === 'applying', + 'A late superseded receipt overwrote the current generation.'); + + CloudHostStorageConfigurationService::finishJobLocked( + $job2, 'failed', null, [] + ); + $terminalize($job2, 'failed'); + $changedAfterFailure = $changedActive; + $receipt3 = CloudHostStorageConfigurationService::save( + 1, $changedAfterFailure, $actor + ); + $job3 = CloudJob::find((int) $receipt3['task_id']); + $row = CloudHostStorageConfiguration::find(1); + $assert($job3 !== null + && (string) $job3->operation_id !== (string) $job2->operation_id + && (int) $row->revision === 2 + && (string) $row->rescue_directory + === '/home/libvirt/rescue-corrected', + 'A corrected path after terminal failure did not create the next revision.'); + + CloudHostStorageConfigurationService::finishJobLocked( + $job3, 'succeeded', ['configured' => true], [] + ); + $terminalize($job3, 'succeeded'); + $applied = CloudHostStorageConfigurationService::read(1); + $assert($applied['state'] === 'applied' + && $applied['configured'] === true + && $applied['can_submit'] === false, + 'A confirmed storage layout was not locked as applied.'); + $changedAfterFailure['expected_revision'] = 2; + $sameApplied = CloudHostStorageConfigurationService::save( + 1, $changedAfterFailure, $actor + ); + $assert($sameApplied['task_id'] === $receipt3['task_id'] + && CloudJob::count() === 3, + 'An exact applied request was not idempotent.'); + $differentApplied = $changedAfterFailure; + $differentApplied['managed_directory'] = '/home/libvirt/other-kvm'; + $expect(static fn () => CloudHostStorageConfigurationService::save( + 1, $differentApplied, $actor + ), 409, 'An applied storage layout was mutable.'); + + // A worker-finalization incident must not turn a known terminal failure + // into a permanent "applying" fence. + $receipt4 = CloudHostStorageConfigurationService::save(2, $first, $actor); + $job4 = CloudJob::find((int) $receipt4['task_id']); + $terminalize($job4, 'failed', true); + $staleProjection = CloudHostStorageConfigurationService::read(2); + $assert($staleProjection['state'] === 'failed' + && $staleProjection['can_submit'] === true, + 'A terminally failed job remained visually locked behind stale applying state.'); + $staleApplying = $first; + $staleApplying['expected_revision'] = 1; + $receipt5 = CloudHostStorageConfigurationService::save( + 2, $staleApplying, $actor + ); + $job5 = CloudJob::find((int) $receipt5['task_id']); + $row2 = CloudHostStorageConfiguration::find(2); + $assert($job5 !== null + && (int) $job5->id !== (int) $job4->id + && (string) $row2->operation_id === (string) $job5->operation_id + && (int) $row2->revision === 1, + 'A linked terminal failure with a stale applying row remained blocked.'); + + echo "Host storage failure supersession state machine passed.\n"; +} diff --git a/web/tests/CloudTaskObserverBehaviorTest.mjs b/web/tests/CloudTaskObserverBehaviorTest.mjs index 17d24787..d0236723 100644 --- a/web/tests/CloudTaskObserverBehaviorTest.mjs +++ b/web/tests/CloudTaskObserverBehaviorTest.mjs @@ -179,6 +179,8 @@ const protocolFailure = task => { assert.match(source, /\{ title: '进行中'/); assert.match(source, /\{ title: '已完成'/); assert.match(source, /if \(task\.terminal && task\.id/); + assert.match(source, /raw\?\.kind === 'host_storage_configure'/); + assert.match(source, /旧存储任务已终止且不会重放/); assert.doesNotMatch(source, /重试任务|重新提交任务|data-task-retry/); }