From 99c03d7e39cc909f1238fa25befb0c2bb520d1b9 Mon Sep 17 00:00:00 2001 From: codex Date: Sat, 31 Oct 2026 14:48:20 +0800 Subject: [PATCH] feat(template): resume and reap multipart uploads --- docs/DEPLOYMENT.md | 4 +- web/app/Controllers/CloudAdminController.php | 11 + .../Cloud/CloudObjectStorageS3Client.php | 30 +- .../Cloud/CloudTemplateImportStateMachine.php | 2 +- .../Cloud/CloudTemplateImportWorkflow.php | 331 +++++++++++++++++- web/app/Services/Cloud/CloudWorkerService.php | 17 +- web/config/routes_cloud.php | 2 + web/public/cloud/admin-template-upload.js | 143 +++++++- web/public/cloud/admin.css | 57 +++ web/public/cloud/admin.js | 2 +- .../views/venix/admin/cloud/templates.tpl | 5 + web/resources/views/venix/layout_assets.tpl | 2 +- web/tests/AdminActionRuntimeContractTest.php | 2 +- ...minTemplateMultipartResumeBehaviorTest.mjs | 54 +++ .../CloudTemplateImportIntentContractTest.php | 6 + .../CloudTemplateMultipartImportTest.php | 170 +++++++++ 16 files changed, 806 insertions(+), 32 deletions(-) create mode 100644 web/tests/AdminTemplateMultipartResumeBehaviorTest.mjs diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 4eb098cd..ae408e7b 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -97,7 +97,9 @@ php xcat cloudRepairInstanceMetadata --apply Access Key ID 与 Secret Access Key 只在超级管理员提交时进入 PHP,使用 `cloud_data_key` 的独立用途密钥密封后保存,读取页面不会回显。修改任一配置或停用应用都会清除原就绪标记;必须重新执行“验证连接并启用”。连接验证只发送签名的 `HeadBucket` 请求,需要目标 Bucket 的 `s3:ListBucket` 权限,不上传、覆盖或删除对象。管理员虚拟机管理已接入完整系统盘备份和新实例恢复,目标可以是原宿主机或其他合格宿主机;用户自动备份使用同一对象存储,在实例设置中按每天或每周于 UTC 00:00 调度并保留最多两份。云镜像导入还会先把经过校验的内容发布到 `cloud-templates/` 不可变主副本,再向宿主机签发该精确版本的只读地址;没有旧本地模板路径或直接暂存分发回退。运行账户需要在配置前缀下同时授予 `vm-backups/` 与 `cloud-templates/` 的上传、`GetObject`、`HeadObject` 和 `DeleteObject` 权限;开启 Bucket 版本控制时还必须授予 `s3:GetObjectVersion` 与 `s3:DeleteObjectVersion`,让 HEAD 与删除准确绑定 `versionId`。维护迁移使用宿主机间 PeerMigration V3 直传,不创建 S3 迁移对象,也不需要 `vm-migrations/` 权限。当前不包含增量链和管理员手工备份的自动保留策略,具体安全边界见 `docs/PORTABLE_VM_BACKUPS.md`。 -生产 Bucket 必须把生命周期规则作为控制面之外的最后兜底:在短周期内终止未完成 multipart upload;开启版本控制时,按业务保留要求清理 `vm-backups/` 的非当前版本和过期删除标记。生命周期不得提前删除仍由面板备份或云镜像目录引用的当前版本。面板会保存 HEAD 返回的 ETag 与 `version_id`,删除时精确定位该版本;删除结果不明时先进行间隔稳定 HEAD,只有对象身份完全一致才创建新的删除代次,每个自动周期最多三次。达到上限的对象进入后台人工核对,不再自动删除;对象存储页面的“重新核对”只恢复稳定 HEAD 证据流程,不会重放原 DELETE。云镜像的人工重试同样只恢复对象存储核对,不会再次删除宿主机副本。 +生产 Bucket 必须把生命周期规则作为控制面之外的最后兜底:建议在 1 天后终止未完成 multipart upload;开启版本控制时,按业务保留要求清理 `vm-backups/` 的非当前版本和过期删除标记。生命周期不得提前删除仍由面板备份或云镜像目录引用的当前版本。管理员本地上传云镜像时,浏览器直接向对象存储执行分片 `PUT`,Bucket CORS 必须只允许面板 HTTPS origin,允许 `PUT`,并在 `ExposeHeaders` 中公开 `ETag`;不要使用 `*` origin 搭配管理站点。面板保存每个已确认分片,刷新后重新选择同一文件会跳过这些分片;取消或 24 小时过期后,普通 worker 会中止仍可确认的 multipart。若创建 multipart 的响应在 upload ID 返回前丢失,面板只能放弃该上传记录,因此上述生命周期规则是清除提供商残留分片的必需兜底。 + +面板会保存 HEAD 返回的 ETag 与 `version_id`,删除时精确定位该版本;删除结果不明时先进行间隔稳定 HEAD,只有对象身份完全一致才创建新的删除代次,每个自动周期最多三次。达到上限的对象进入后台人工核对,不再自动删除;对象存储页面的“重新核对”只恢复稳定 HEAD 证据流程,不会重放原 DELETE。云镜像的人工重试同样只恢复对象存储核对,不会再次删除宿主机副本。 生产环境应为面板创建独立服务账户,把权限限制到配置的 Bucket 和备份前缀。数据库备份必须与原始 `cloud_data_key` 一起保管,重新生成主密钥无法恢复全局配置及各备份冻结的对象存储凭据。 diff --git a/web/app/Controllers/CloudAdminController.php b/web/app/Controllers/CloudAdminController.php index 91e63a47..39f36a5d 100644 --- a/web/app/Controllers/CloudAdminController.php +++ b/web/app/Controllers/CloudAdminController.php @@ -437,6 +437,17 @@ final class CloudAdminController )->withHeader('Cache-Control', 'no-store'); } + public function cloudTemplateImportCancel($request, $response, $args) + { + $actor = $this->actor($request, true); + return $response->withJson( + (new CloudTemplateImportWorkflow())->cancel( + (string) ($args['id'] ?? ''), + $actor + ) + )->withHeader('Cache-Control', 'no-store'); + } + public function cloudTemplateImportUpload($request, $response, $args) { $actor = $this->actor($request, true); diff --git a/web/app/Services/Cloud/CloudObjectStorageS3Client.php b/web/app/Services/Cloud/CloudObjectStorageS3Client.php index 29c40e38..ed332e21 100644 --- a/web/app/Services/Cloud/CloudObjectStorageS3Client.php +++ b/web/app/Services/Cloud/CloudObjectStorageS3Client.php @@ -240,6 +240,12 @@ final class CloudObjectStorageS3Client 'multipart_parts_unavailable' ); } + if ($response->getStatusCode() === 404) { + throw new CloudObjectStorageTransferError( + '对象存储分段上传已经不存在。', + 'multipart_upload_absent' + ); + } if ($response->getStatusCode() < 200 || $response->getStatusCode() >= 300) { throw new CloudObjectStorageTransferError( @@ -852,19 +858,37 @@ final class CloudObjectStorageS3Client } $status = $response->getStatusCode(); if ($status < 200 || $status >= 300) { + $unknown = in_array($status, [408, 425, 429], true) + || $status >= 500; throw new CloudObjectStorageTransferError( - '对象存储拒绝创建分段上传。', 'multipart_create_rejected' + $unknown + ? '无法确认对象存储是否已开始分段上传。' + : '对象存储拒绝创建分段上传。', + $unknown + ? 'multipart_create_unknown' + : 'multipart_create_rejected', + $unknown ); } $body = (string) $response->getBody(); if (strlen($body) > 65536 || preg_match('/([^<]+)<\/UploadId>/i', $body, $match) !== 1) { throw new CloudObjectStorageTransferError( - '对象存储返回了无效的分段上传标识。', 'multipart_create_invalid' + '对象存储返回了无效的分段上传标识。', + 'multipart_create_invalid', + true ); } $uploadId = html_entity_decode($match[1], ENT_QUOTES | ENT_XML1, 'UTF-8'); - self::assertUploadId($uploadId); + try { + self::assertUploadId($uploadId); + } catch (CloudObjectStorageTransferError $error) { + throw new CloudObjectStorageTransferError( + '对象存储返回了无效的分段上传标识。', + 'multipart_create_invalid', + true + ); + } return $uploadId; } diff --git a/web/app/Services/Cloud/CloudTemplateImportStateMachine.php b/web/app/Services/Cloud/CloudTemplateImportStateMachine.php index 47231dcb..2d050acf 100644 --- a/web/app/Services/Cloud/CloudTemplateImportStateMachine.php +++ b/web/app/Services/Cloud/CloudTemplateImportStateMachine.php @@ -24,7 +24,7 @@ final class CloudTemplateImportStateMachine ], 'source_ready' => ['distributing', 'cancel_pending'], 'distributing' => ['ready', 'distribution_failed'], - 'upload_failed' => ['uploading', 'cancelled'], + 'upload_failed' => ['uploading', 'cancel_pending', 'cancelled'], 'source_failed' => ['uploading', 'fetching', 'cancelled'], 'source_unknown' => ['source_verifying', 'cancel_pending'], 'distribution_failed' => ['distributing', 'cancel_pending'], diff --git a/web/app/Services/Cloud/CloudTemplateImportWorkflow.php b/web/app/Services/Cloud/CloudTemplateImportWorkflow.php index c6757994..23965cae 100644 --- a/web/app/Services/Cloud/CloudTemplateImportWorkflow.php +++ b/web/app/Services/Cloud/CloudTemplateImportWorkflow.php @@ -187,6 +187,7 @@ final class CloudTemplateImportWorkflow self::markUploadFailure( $id, $error->publicCode, + $error->effectMayHaveApplied, $now ); } @@ -774,12 +775,14 @@ final class CloudTemplateImportWorkflow private static function markUploadFailure( string $id, string $code, + bool $unknown, int $now ): void { $connection = (new CloudTemplateImportIntent())->getConnection(); $connection->transaction(static function () use ( $id, $code, + $unknown, $now ): void { $intent = self::lockIntent($id); @@ -787,7 +790,10 @@ final class CloudTemplateImportWorkflow || (string) $intent->multipart_upload_id !== '') { return; } - CloudTemplateImportStateMachine::move($intent, 'upload_failed'); + CloudTemplateImportStateMachine::move( + $intent, + $unknown ? 'source_unknown' : 'upload_failed' + ); $intent->error_code = substr($code, 0, 128); $intent->updated_at = $now; $intent->save(); @@ -1004,6 +1010,290 @@ final class CloudTemplateImportWorkflow return ['intent' => self::dto($intent)]; } + /** + * Cancel one local multipart upload. A persisted cancel_pending row is + * reconciled with read-only provider evidence before another Abort is + * allowed, so a lost DELETE response is never blindly replayed. + * + * @return array + */ + public function cancel( + string $id, + array $actor, + ?CloudObjectStorageS3Client $client = null + ): array { + CloudAccess::manage($actor); + return $this->cancelIntent( + self::intentId($id), + $client ?? new CloudObjectStorageS3Client(), + false + ); + } + + /** + * Reap at most one expired local upload. Bucket lifecycle remains the + * final safety net when CreateMultipartUpload succeeded but its response + * was lost before PHP learned the provider upload ID. + */ + public function cleanupOneExpired( + ?CloudObjectStorageS3Client $client = null + ): bool { + $model = new CloudTemplateImportIntent(); + $connection = $model->getConnection(); + if (!$connection->getSchemaBuilder()->hasTable( + $model->getTable() + )) { + return false; + } + $now = ($this->clock)(); + $candidate = CloudTemplateImportIntent::query() + ->where('source_kind', 'upload') + ->where('expires_at', '<=', $now) + ->where(static function ($query): void { + $query->whereIn('state', [ + 'prepared', 'uploading', 'upload_failed', + 'cancel_pending', + ])->orWhere(static function ($unknown): void { + $unknown->where('state', 'source_unknown') + ->where(static function ($recoverable): void { + $recoverable->where('multipart_upload_id', '') + ->orWhereIn('error_code', [ + 'multipart_abort_rejected', + 'multipart_abort_not_applied', + ]); + }); + }); + }) + ->orderBy('expires_at')->orderBy('id')->first(); + if ($candidate === null) { + return false; + } + $this->cancelIntent( + (string) $candidate->id, + $client ?? new CloudObjectStorageS3Client(), + true + ); + return true; + } + + /** @return array */ + private function cancelIntent( + string $id, + CloudObjectStorageS3Client $client, + bool $expiredOnly + ): array { + $now = ($this->clock)(); + $connection = (new CloudTemplateImportIntent())->getConnection(); + $prepared = $connection->transaction(static function () use ( + $id, + $expiredOnly, + $now + ): array { + $intent = self::lockIntent($id); + if ((string) $intent->source_kind !== 'upload') { + throw new CloudError('该导入记录不是本地上传', 409); + } + if ($expiredOnly && (int) $intent->expires_at > $now) { + return ['done' => true, 'intent' => $intent]; + } + $state = (string) $intent->state; + if ($state === 'cancelled') { + return ['done' => true, 'intent' => $intent]; + } + if ($state === 'ready') { + throw new CloudError('已发布的云镜像不能作为上传取消', 409); + } + $uploadId = (string) $intent->multipart_upload_id; + if ($uploadId === '') { + if ($state === 'uploading' && !$expiredOnly + && (int) $intent->updated_at > $now - 120) { + throw new CloudError( + '对象存储上传会话正在创建,请稍后再取消', + 409 + ); + } + if (!in_array($state, [ + 'prepared', 'uploading', 'upload_failed', + 'source_unknown', 'cancel_pending', + ], true)) { + throw new CloudError('当前导入状态不能取消', 409); + } + if (in_array($state, ['prepared', 'upload_failed'], true)) { + CloudTemplateImportStateMachine::move( + $intent, + 'cancelled' + ); + $intent->error_code = ''; + } else { + if ($state !== 'cancel_pending') { + CloudTemplateImportStateMachine::move( + $intent, + 'cancel_pending' + ); + } + CloudTemplateImportStateMachine::move( + $intent, + 'cancelled' + ); + $intent->error_code = 'multipart_abandoned_lifecycle'; + } + $intent->completed_at = $now; + $intent->updated_at = $now; + $intent->save(); + return ['done' => true, 'intent' => $intent]; + } + $retryableUnknown = $state === 'source_unknown' + && in_array((string) $intent->error_code, [ + 'multipart_abort_rejected', + 'multipart_abort_not_applied', + ], true); + if (!in_array($state, + ['uploading', 'upload_failed', 'cancel_pending'], true) + && !$retryableUnknown) { + throw new CloudError( + '当前上传结果尚未确认,不能终止原分段上传', + 409 + ); + } + $reconcileFirst = $state === 'cancel_pending'; + if (!$reconcileFirst) { + CloudTemplateImportStateMachine::move( + $intent, + 'cancel_pending' + ); + $intent->error_code = 'multipart_abort_pending'; + // If PHP stops after this commit, the worker can immediately + // reconcile the pending cancellation from provider evidence. + $intent->expires_at = $now; + $intent->updated_at = $now; + $intent->save(); + } + return [ + 'done' => false, + 'intent' => $intent, + 'reconcile_first' => $reconcileFirst, + ]; + }, 3); + /** @var CloudTemplateImportIntent $intent */ + $intent = $prepared['intent']; + if (($prepared['done'] ?? false) === true) { + return self::dto($intent); + } + if (($prepared['reconcile_first'] ?? false) === true) { + $evidence = $this->cancellationEvidence($intent, $client); + if ($evidence === 'absent') { + return self::settleCancelled($id, $now); + } + if ($evidence !== 'active') { + self::settleCancelUnknown($id, $evidence, $now); + throw new CloudError( + '无法确认原分段上传是否已终止,请勿重复取消', + 502 + ); + } + } + $runtime = self::runtime($intent); + try { + $client->abortMultipartUpload( + $runtime['config'], + $runtime['credentials'], + (string) $intent->object_key, + (string) $intent->multipart_upload_id + ); + return self::settleCancelled($id, $now); + } catch (CloudObjectStorageTransferError $error) { + $evidence = $this->cancellationEvidence($intent, $client); + if ($evidence === 'absent') { + return self::settleCancelled($id, $now); + } + $code = $evidence === 'active' + ? 'multipart_abort_not_applied' : $evidence; + self::settleCancelUnknown($id, $code, $now); + throw new CloudError( + $evidence === 'active' + ? '对象存储尚未终止分段上传,可以稍后重新取消' + : '无法确认原分段上传是否已终止,请勿重复取消', + $evidence === 'active' ? 409 : 502 + ); + } + } + + /** @return 'active'|'absent'|string */ + private function cancellationEvidence( + CloudTemplateImportIntent $intent, + CloudObjectStorageS3Client $client + ): string { + $runtime = self::runtime($intent); + try { + if ($client->inspectObject( + $runtime['config'], + $runtime['credentials'], + (string) $intent->object_key + ) !== null) { + return 'multipart_abort_object_present'; + } + } catch (CloudObjectStorageTransferError $error) { + return $error->publicCode; + } + try { + $client->listMultipartParts( + $runtime['config'], + $runtime['credentials'], + (string) $intent->object_key, + (string) $intent->multipart_upload_id + ); + return 'active'; + } catch (CloudObjectStorageTransferError $error) { + return $error->publicCode === 'multipart_upload_absent' + ? 'absent' : $error->publicCode; + } + } + + /** @return array */ + private static function settleCancelled(string $id, int $now): array + { + $connection = (new CloudTemplateImportIntent())->getConnection(); + return $connection->transaction(static function () use ( + $id, + $now + ): array { + $intent = self::lockIntent($id); + if ((string) $intent->state !== 'cancelled') { + if ((string) $intent->state !== 'cancel_pending') { + throw new CloudError('上传取消状态已变化,请刷新页面', 409); + } + CloudTemplateImportStateMachine::move($intent, 'cancelled'); + } + $intent->error_code = ''; + $intent->completed_at = $now; + $intent->updated_at = $now; + $intent->save(); + return self::dto($intent); + }, 3); + } + + private static function settleCancelUnknown( + string $id, + string $code, + int $now + ): void { + $connection = (new CloudTemplateImportIntent())->getConnection(); + $connection->transaction(static function () use ( + $id, + $code, + $now + ): void { + $intent = self::lockIntent($id); + if ((string) $intent->state !== 'cancel_pending') { + return; + } + CloudTemplateImportStateMachine::move($intent, 'source_unknown'); + $intent->error_code = substr($code, 0, 128); + $intent->updated_at = $now; + $intent->save(); + }, 3); + } + /** @return array{created:bool,intent:array} */ private function reserveUploadIntent( array $actor, @@ -1108,10 +1398,16 @@ final class CloudTemplateImportWorkflow /** @return array */ public static function dto(CloudTemplateImportIntent $intent): array { - $partQuery = CloudTemplateImportPart::query() - ->where('intent_id', (string) $intent->id); - $uploadedParts = (int) (clone $partQuery)->count(); - $uploadedBytes = (int) (clone $partQuery)->sum('size_bytes'); + $parts = CloudTemplateImportPart::query() + ->where('intent_id', (string) $intent->id) + ->orderBy('part_number')->get()->map( + static fn (CloudTemplateImportPart $part): array => [ + 'part_number' => (int) $part->part_number, + 'size_bytes' => (int) $part->size_bytes, + ] + )->all(); + $uploadedParts = count($parts); + $uploadedBytes = array_sum(array_column($parts, 'size_bytes')); $state = (string) $intent->state; return [ 'id' => (string) $intent->id, @@ -1127,10 +1423,11 @@ final class CloudTemplateImportWorkflow 'state' => $state, 'uploaded_parts' => $uploadedParts, 'uploaded_bytes' => $uploadedBytes, + 'parts' => $parts, 'part_count' => (int) $intent->multipart_part_count, 'part_size_bytes' => (int) $intent->multipart_part_size_bytes, 'can_resume' => CloudTemplateImportStateMachine::resumable($state), - 'can_cancel' => !CloudTemplateImportStateMachine::terminal($state), + 'can_cancel' => self::cancellable($intent), 'task_id' => $intent->active_job_id === null ? null : (int) $intent->active_job_id, 'error_code' => (string) $intent->error_code, @@ -1142,6 +1439,28 @@ final class CloudTemplateImportWorkflow ]; } + private static function cancellable( + CloudTemplateImportIntent $intent + ): bool { + if ((string) $intent->source_kind !== 'upload') { + return false; + } + $state = (string) $intent->state; + if (in_array($state, [ + 'prepared', 'uploading', 'upload_failed', + ], true)) { + return true; + } + if ($state !== 'source_unknown') { + return false; + } + return (string) $intent->multipart_upload_id === '' + || in_array((string) $intent->error_code, [ + 'multipart_abort_rejected', + 'multipart_abort_not_applied', + ], true); + } + private static function assertAdministrator(array $actor): void { if (($actor['is_admin'] ?? false) !== true) { diff --git a/web/app/Services/Cloud/CloudWorkerService.php b/web/app/Services/Cloud/CloudWorkerService.php index d5db53d3..b926bc9c 100644 --- a/web/app/Services/Cloud/CloudWorkerService.php +++ b/web/app/Services/Cloud/CloudWorkerService.php @@ -139,10 +139,21 @@ final class CloudWorkerService // Deadline convergence is always first. Recovery inspection runs in // the dedicated --recovery-only worker, so this worker never waits on // an offline recovery target before dispatching ordinary work. - if ($quarantined || $this->tickJob()) { - return true; + $jobWorked = !$quarantined && $this->tickJob(); + $templateCleanup = false; + if (!$this->policyOnly && !$this->vpcBatchOnly) { + try { + $templateCleanup = (new CloudTemplateImportWorkflow()) + ->cleanupOneExpired(); + } catch (\Throwable $error) { + // The intent has already moved to source_unknown when an S3 + // abort cannot be proven. Surface one worker observation and + // never loop a destructive provider request blindly. + $this->observedErrors[] = + 'template_import_cleanup_unresolved'; + } } - return $expired > 0; + return $quarantined || $jobWorked || $templateCleanup || $expired > 0; } /** diff --git a/web/config/routes_cloud.php b/web/config/routes_cloud.php index 60102b64..38382ab6 100644 --- a/web/config/routes_cloud.php +++ b/web/config/routes_cloud.php @@ -220,6 +220,8 @@ $app->group('/api/v1', function () { $this->get('/admin/cloud-template-imports', 'App\Controllers\CloudAdminController:cloudTemplateImportIntents'); $this->post('/admin/cloud-template-imports', 'App\Controllers\CloudAdminController:cloudTemplateImportIntents'); $this->get('/admin/cloud-template-imports/{id}', 'App\Controllers\CloudAdminController:cloudTemplateImportIntent'); + $this->delete('/admin/cloud-template-imports/{id}', + 'App\Controllers\CloudAdminController:cloudTemplateImportCancel'); $this->post('/admin/cloud-template-imports/{id}/upload', 'App\Controllers\CloudAdminController:cloudTemplateImportUpload'); $this->put('/admin/cloud-template-imports/{id}/parts/{part:[1-9][0-9]*}', diff --git a/web/public/cloud/admin-template-upload.js b/web/public/cloud/admin-template-upload.js index 3dcce528..e7b271fe 100644 --- a/web/public/cloud/admin-template-upload.js +++ b/web/public/cloud/admin-template-upload.js @@ -150,7 +150,7 @@ export function putMultipartPart(url, blob, onProgress, }); } -function validatePlan(plan, intentId, fileSize) { +export function multipartPartSchedule(plan, intentId, fileSize) { const partSize = Number(plan?.part_size_bytes); const partCount = Number(plan?.part_count); const urls = plan?.part_urls; @@ -162,7 +162,32 @@ function validatePlan(plan, intentId, fileSize) { || urls.some(url => typeof url !== 'string' || !url)) { throw new Error('面板返回了无效的对象存储上传计划。'); } - return { partSize, partCount, urls }; + const completed = new Map(); + if (!Array.isArray(plan.intent.parts)) { + throw new Error('面板返回的上传分片账本无效。'); + } + for (const part of plan.intent.parts) { + const partNumber = Number(part?.part_number); + const size = Number(part?.size_bytes); + const start = (partNumber - 1) * partSize; + const expected = Math.min(fileSize, start + partSize) - start; + if (!Number.isSafeInteger(partNumber) || partNumber < 1 + || partNumber > partCount || !Number.isSafeInteger(size) + || size !== expected || completed.has(partNumber)) { + throw new Error('面板返回的上传分片账本无效。'); + } + completed.set(partNumber, size); + } + return urls.map((url, index) => { + const partNumber = index + 1; + const start = index * partSize; + const end = Math.min(fileSize, start + partSize); + return Object.freeze({ + partNumber, start, end, url, + size: end - start, + completed: completed.has(partNumber), + }); + }); } function percentage(value, total) { @@ -180,16 +205,21 @@ export function mountTemplateUpload(options) { const config = options && typeof options === 'object' ? options : {}; const documentImpl = config.documentImpl || globalThis.document; const cryptoImpl = config.cryptoImpl || globalThis.crypto; - const storage = config.storage || globalThis.localStorage; + let storage = config.storage; + if (storage === undefined) { + try { storage = globalThis.localStorage; } catch { storage = null; } + } const WorkerImpl = config.WorkerImpl || globalThis.Worker; const XMLHttpRequestImpl = config.XMLHttpRequestImpl || globalThis.XMLHttpRequest; const requestRuntime = config.requestRuntime; const notice = config.notice; const onTask = config.onTask; + const confirmDanger = config.confirmDanger; const canManage = config.canManage === true; const form = documentImpl.getElementById('template-upload'); if (!form || !requestRuntime || typeof requestRuntime.mutate !== 'function' + || typeof requestRuntime.read !== 'function' || typeof notice !== 'function' || typeof onTask !== 'function') { throw new TypeError('Template upload module dependencies are incomplete.'); } @@ -201,8 +231,10 @@ export function mountTemplateUpload(options) { const progressBar = documentImpl.getElementById('template-upload-progress-bar'); const progressLabel = documentImpl.getElementById('template-upload-progress-label'); const hashLabel = documentImpl.getElementById('template-upload-hash-label'); + const activeList = documentImpl.getElementById('template-upload-active-list'); + const activeEmpty = documentImpl.getElementById('template-upload-active-empty'); if ([nameInput, systemInput, fileInput, button, progressBox, progressBar, - progressLabel, hashLabel].some(value => !value)) { + progressLabel, hashLabel, activeList, activeEmpty].some(value => !value)) { throw new TypeError('Template upload controls are incomplete.'); } @@ -228,6 +260,77 @@ export function mountTemplateUpload(options) { hashLabel.textContent = `SHA-256 校验 ${hashPercent.toFixed(1)}%`; }; + const stateLabels = Object.freeze({ + prepared: '等待上传', uploading: '上传中', upload_failed: '可继续', + source_verifying: '正在核对', source_unknown: '需要核查', + source_ready: '等待分发', distributing: '正在分发', + distribution_failed: '分发失败', cancel_pending: '正在取消', + }); + const renderActive = intents => { + activeList.replaceChildren(); + activeEmpty.hidden = intents.length > 0; + for (const intent of intents) { + if (!HEX_32.test(intent?.id || '')) continue; + const row = documentImpl.createElement('div'); + row.className = 'cloud-template-upload-active-row'; + const identity = documentImpl.createElement('div'); + const title = documentImpl.createElement('strong'); + title.textContent = String(intent.name || intent.source_filename || intent.id); + const detail = documentImpl.createElement('small'); + const total = Number(intent.source_size_bytes || 0); + const uploaded = Number(intent.uploaded_bytes || 0); + detail.textContent = `${stateLabels[intent.state] || '状态待核查'} · ` + + `${(uploaded / 1024 ** 2).toFixed(1)} / ` + + `${(total / 1024 ** 2).toFixed(1)} MiB`; + identity.append(title, detail); + row.append(identity); + if (intent.can_cancel === true && canManage) { + const cancel = documentImpl.createElement('button'); + cancel.type = 'button'; + cancel.className = 'btn btn-sm btn-outline-danger'; + cancel.textContent = '取消上传'; + cancel.addEventListener('click', async () => { + if (busy() || activeController) return; + if (typeof confirmDanger === 'function' && !await confirmDanger({ + tone: 'warning', + title: `取消上传 · ${String(intent.name || intent.source_filename)}`, + message: '已上传但尚未发布的对象存储分片会被终止;此操作不会删除已经发布的云镜像。', + confirmText: '取消上传', + })) return; + setBusy(true); cancel.disabled = true; + try { + await requestRuntime.mutate( + `/api/v1/admin/cloud-template-imports/${encodeURIComponent(intent.id)}`, + { method: 'DELETE', body: {}, + idempotencyKey: `template-upload-cancel-${intent.id}` } + ); + notice('未完成的云镜像上传已取消。'); + } catch (error) { + notice(error instanceof Error ? error.message : '取消上传失败。', true); + } finally { + setBusy(false); + await loadActive(); + } + }); + row.append(cancel); + } + activeList.append(row); + } + }; + const loadActive = async () => { + try { + const payload = await requestRuntime.read( + '/api/v1/admin/cloud-template-imports' + ); + renderActive(Array.isArray(payload?.intents) ? payload.intents : []); + } catch (error) { + activeList.replaceChildren(); + activeEmpty.hidden = false; + activeEmpty.textContent = error instanceof Error + ? error.message : '无法读取未完成上传。'; + } + }; + const submit = async event => { event.preventDefault(); if (!canManage || busy() || activeController) return; @@ -305,29 +408,36 @@ export function mountTemplateUpload(options) { idempotencyKey: `${record.requestKey}.start`, } ); - const validated = validatePlan(plan, intent.id, file.size); - for (let index = 0; index < validated.partCount; index += 1) { - const start = index * validated.partSize; - const end = Math.min(file.size, start + validated.partSize); - const part = file.slice(start, end); + const schedule = multipartPartSchedule(plan, intent.id, file.size); + for (const item of schedule) { + if (item.completed) { + uploadedBytes += item.size; + renderProgress(uploadedBytes, file.size, hashedBytes); + continue; + } + const part = file.slice(item.start, item.end); + const confirmedBeforePart = uploadedBytes; const etag = await putMultipartPart( - validated.urls[index], + item.url, part, loaded => { - uploadedBytes = start + loaded; - renderProgress(uploadedBytes, file.size, hashedBytes); + renderProgress( + confirmedBeforePart + loaded, + file.size, + hashedBytes + ); }, XMLHttpRequestImpl, activeController.signal ); - uploadedBytes = end; + uploadedBytes = confirmedBeforePart + item.size; renderProgress(uploadedBytes, file.size, hashedBytes); await requestRuntime.mutate( - `/api/v1/admin/cloud-template-imports/${encodeURIComponent(intent.id)}/parts/${index + 1}`, { + `/api/v1/admin/cloud-template-imports/${encodeURIComponent(intent.id)}/parts/${item.partNumber}`, { method: 'PUT', body: { size_bytes: part.size, etag }, signal: activeController.signal, - idempotencyKey: `${record.requestKey}.part.${index + 1}`, + idempotencyKey: `${record.requestKey}.part.${item.partNumber}`, } ); } @@ -360,12 +470,15 @@ export function mountTemplateUpload(options) { activeController = null; setBusy(false); setControls(false); + await loadActive(); } }; form.addEventListener('submit', submit); setControls(false); + void loadActive(); return Object.freeze({ + refresh: loadActive, destroy() { form.removeEventListener('submit', submit); activeController?.abort(); diff --git a/web/public/cloud/admin.css b/web/public/cloud/admin.css index f7bc89c5..7536ed5c 100644 --- a/web/public/cloud/admin.css +++ b/web/public/cloud/admin.css @@ -337,6 +337,53 @@ background: #16a3ad; } +.cloud-template-upload-active { + display: grid; + gap: 9px; + margin-top: 14px; + padding-top: 14px; + border-top: 1px solid #e4ebf1; +} + +.cloud-template-upload-active-head, +.cloud-template-upload-active-row { + display: flex; + align-items: center; + justify-content: space-between; + gap: 12px; +} + +.cloud-template-upload-active-head span, +.cloud-template-upload-active-row small { + color: #7b8da0; + font-size: 12px; +} + +.cloud-template-upload-active-list { + display: grid; + gap: 8px; +} + +.cloud-template-upload-active-row { + padding: 10px 11px; + border: 1px solid #dce6ec; + border-radius: 9px; + background: #f8fbfc; +} + +.cloud-template-upload-active-row > div { + display: grid; + min-width: 0; + gap: 2px; +} + +.cloud-template-upload-active-row strong, +.cloud-template-upload-active-row small { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + .cloud-template-catalog-actions { display: flex; align-items: center; @@ -1833,6 +1880,16 @@ gap: 4px; } + .cloud-template-upload-active-head, + .cloud-template-upload-active-row { + align-items: stretch; + flex-direction: column; + } + + .cloud-template-upload-active-row .btn { + width: 100%; + } + .cloud-template-catalog-actions { width: 100%; justify-content: space-between; diff --git a/web/public/cloud/admin.js b/web/public/cloud/admin.js index bae6780c..a864d1de 100644 --- a/web/public/cloud/admin.js +++ b/web/public/cloud/admin.js @@ -1438,7 +1438,7 @@ async function initialize() { else if (root.dataset.cloudAdmin === 'templates') { const { mountTemplateUpload } = await import(versionedModuleUrl('./admin-template-upload.js')); templateUploadController = mountTemplateUpload({ - requestRuntime, notice, canManage, + requestRuntime, notice, confirmDanger, canManage, isBusy: () => state.busy, setBusy: value => { state.busy = value; }, onTask: task => { diff --git a/web/resources/views/venix/admin/cloud/templates.tpl b/web/resources/views/venix/admin/cloud/templates.tpl index 340d40cd..9d094c56 100644 --- a/web/resources/views/venix/admin/cloud/templates.tpl +++ b/web/resources/views/venix/admin/cloud/templates.tpl @@ -65,6 +65,11 @@
准备上传…SHA-256 校验 0.0%
0% +
+
未完成上传重新选择同一文件会跳过已确认分片
+

当前没有未完成上传。

+
+
diff --git a/web/resources/views/venix/layout_assets.tpl b/web/resources/views/venix/layout_assets.tpl index 7d84bf4c..5743246d 100644 --- a/web/resources/views/venix/layout_assets.tpl +++ b/web/resources/views/venix/layout_assets.tpl @@ -1,2 +1,2 @@ {* Shared Venix CSS/JS cache key. Increment whenever a shared browser asset changes. *} -{assign var=venix_asset_version value='2026103119' scope='global'} +{assign var=venix_asset_version value='2026103120' scope='global'} diff --git a/web/tests/AdminActionRuntimeContractTest.php b/web/tests/AdminActionRuntimeContractTest.php index 7173067f..75187f06 100644 --- a/web/tests/AdminActionRuntimeContractTest.php +++ b/web/tests/AdminActionRuntimeContractTest.php @@ -78,7 +78,7 @@ $assert(str_contains($combined, "request.open('PUT', parsed.href, true)") "'/api/v1/admin/cloud-template-imports'") && !str_contains($combined, '/admin/cloud-templates/upload?'), 'The cloud-template file upload must use credential-free direct multipart PUT.'); -$assert(preg_match("/venix_asset_version value='2026103119'/", $assets) === 1, +$assert(preg_match("/venix_asset_version value='2026103120'/", $assets) === 1, 'Shared administrator assets must use the current cache version.'); echo "Administrator shared action runtime contract passed.\n"; diff --git a/web/tests/AdminTemplateMultipartResumeBehaviorTest.mjs b/web/tests/AdminTemplateMultipartResumeBehaviorTest.mjs new file mode 100644 index 00000000..58753526 --- /dev/null +++ b/web/tests/AdminTemplateMultipartResumeBehaviorTest.mjs @@ -0,0 +1,54 @@ +import { multipartPartSchedule } from + '../public/cloud/admin-template-upload.js'; + +const intentId = '0123456789abcdef0123456789abcdef'; +const mib = 1024 ** 2; +const schedule = multipartPartSchedule({ + intent: { + id: intentId, + parts: [{ part_number: 1, size_bytes: 64 * mib }], + }, + part_size_bytes: 64 * mib, + part_count: 2, + part_urls: [ + 'https://objects.example.test/part-1', + 'https://objects.example.test/part-2', + ], +}, intentId, 70 * mib); + +if (schedule.length !== 2 + || schedule[0].completed !== true + || schedule[0].size !== 64 * mib + || schedule[1].completed !== false + || schedule[1].size !== 6 * mib + || schedule[1].partNumber !== 2) { + throw new Error('Confirmed multipart parts were not safely skipped.'); +} + +for (const invalidParts of [ + [{ part_number: 1, size_bytes: 63 * mib }], + [{ part_number: 3, size_bytes: 6 * mib }], + [ + { part_number: 1, size_bytes: 64 * mib }, + { part_number: 1, size_bytes: 64 * mib }, + ], +]) { + try { + multipartPartSchedule({ + intent: { id: intentId, parts: invalidParts }, + part_size_bytes: 64 * mib, + part_count: 2, + part_urls: [ + 'https://objects.example.test/part-1', + 'https://objects.example.test/part-2', + ], + }, intentId, 70 * mib); + throw new Error('An invalid server part ledger was accepted.'); + } catch (error) { + if (!String(error.message).includes('分片账本')) throw error; + } +} + +(globalThis.print || console.log)( + 'Administrator multipart resume behavior passed.' +); diff --git a/web/tests/CloudTemplateImportIntentContractTest.php b/web/tests/CloudTemplateImportIntentContractTest.php index 53b8135e..5886dcf5 100644 --- a/web/tests/CloudTemplateImportIntentContractTest.php +++ b/web/tests/CloudTemplateImportIntentContractTest.php @@ -53,10 +53,14 @@ $assert(str_contains($routes, "'/admin/cloud-template-imports'") "'/admin/cloud-template-imports/{id}/parts/{part:[1-9][0-9]*}'") && str_contains($routes, "'/admin/cloud-template-imports/{id}/complete'") + && str_contains($routes, + "delete('/admin/cloud-template-imports/{id}'") && str_contains($controller, 'CloudTemplateImportWorkflow') && str_contains($controller, 'beginUpload(') && str_contains($controller, 'startUpload(') && str_contains($controller, 'recordPart(') + && str_contains($controller, 'cloudTemplateImportCancel(') + && str_contains($controller, '->cancel(') && str_contains($controller, 'completeUpload('), 'Administrator template-import intent routes are not wired.'); $assert(str_contains($browser, @@ -65,6 +69,8 @@ $assert(str_contains($browser, && str_contains($browser, "request.getResponseHeader('ETag')") && str_contains($browser, '/api/v1/admin/cloud-template-imports') && str_contains($browser, 'idempotencyKey: record.requestKey') + && str_contains($browser, 'multipartPartSchedule(') + && str_contains($browser, 'template-upload-active-list') && str_contains($worker, 'hashBlob(file') && str_contains($template, 'id="template-upload-progress"') && str_contains($template, '文件内容不经过面板服务器'), diff --git a/web/tests/CloudTemplateMultipartImportTest.php b/web/tests/CloudTemplateMultipartImportTest.php index 287f4b89..2c3ebd31 100644 --- a/web/tests/CloudTemplateMultipartImportTest.php +++ b/web/tests/CloudTemplateMultipartImportTest.php @@ -16,6 +16,7 @@ namespace { require dirname(__DIR__) . '/vendor/autoload.php'; use App\Models\CloudTemplateImportIntent; + use App\Services\Cloud\CloudError; use App\Services\Cloud\CloudObjectStorageS3Client; use App\Services\Cloud\CloudSecrets; use App\Services\Cloud\CloudTemplateImportWorkflow; @@ -394,5 +395,174 @@ namespace { ], 'A lost completion response was replayed instead of reconciled.'); + // Refreshing the page must expose the exact server-confirmed part ledger; + // re-signing an existing upload must not allocate a second upload ID. + $actor['request_idempotency_key'] = 'multipart-template-intent-0003'; + $resumable = $workflow->beginUpload(array_replace($input, [ + 'name' => 'Debian 13 resumable', + 'filename' => 'debian-13-resumable.qcow2', + ]), $actor)['intent']; + $resumeHistory = []; + $resumeStack = HandlerStack::create(new MockHandler([ + new Response(200, [], '' + . 'intent-upload-3' + . ''), + new Response(204), + ])); + $resumeStack->push(Middleware::history($resumeHistory)); + $resumeClient = new CloudObjectStorageS3Client(new Client([ + 'handler' => $resumeStack, + ])); + $workflow->startUpload((string) $resumable['id'], $actor, $resumeClient); + $workflow->recordPart((string) $resumable['id'], 1, [ + 'size_bytes' => 64 * 1024 * 1024, + 'etag' => $partEtag, + ], $actor); + $resigned = $workflow->startUpload( + (string) $resumable['id'], $actor, $resumeClient + ); + $assert(($resigned['intent']['parts'] ?? null) === [[ + 'part_number' => 1, + 'size_bytes' => 64 * 1024 * 1024, + ]] + && array_map(static fn (array $item): string => + $item['request']->getMethod(), $resumeHistory) === ['POST'], + 'Resume did not expose the exact part ledger or created a new upload.'); + $cancelled = $workflow->cancel( + (string) $resumable['id'], $actor, $resumeClient + ); + $assert(($cancelled['state'] ?? null) === 'cancelled' + && array_map(static fn (array $item): string => + $item['request']->getMethod(), $resumeHistory) === [ + 'POST', 'DELETE', + ], + 'A confirmed multipart abort did not settle the intent once.'); + + // A lost Abort response is reconciled with HEAD + ListParts. A 404 for + // the exact upload ID proves cancellation; DELETE must not be replayed. + $actor['request_idempotency_key'] = 'multipart-template-intent-0004'; + $abortLost = $workflow->beginUpload(array_replace($input, [ + 'name' => 'Debian 13 abort lost', + 'filename' => 'debian-13-abort-lost.qcow2', + ]), $actor)['intent']; + $abortHistory = []; + $abortStack = HandlerStack::create(new MockHandler([ + new Response(200, [], '' + . 'intent-upload-4' + . ''), + new ConnectException('abort response lost', + new Request('DELETE', 'https://objects.example.test')), + new Response(404), + new Response(404), + ])); + $abortStack->push(Middleware::history($abortHistory)); + $abortClient = new CloudObjectStorageS3Client(new Client([ + 'handler' => $abortStack, + ])); + $workflow->startUpload((string) $abortLost['id'], $actor, $abortClient); + $abortResult = $workflow->cancel( + (string) $abortLost['id'], $actor, $abortClient + ); + $assert(($abortResult['state'] ?? null) === 'cancelled' + && array_map(static fn (array $item): string => + $item['request']->getMethod(), $abortHistory) === [ + 'POST', 'DELETE', 'HEAD', 'GET', + ], + 'A lost Abort response was replayed instead of read-only reconciled.'); + + // If read-only evidence proves the upload still exists, cancellation is + // retryable with a new explicit action rather than reported as complete. + $actor['request_idempotency_key'] = 'multipart-template-intent-0005'; + $abortActive = $workflow->beginUpload(array_replace($input, [ + 'name' => 'Debian 13 abort active', + 'filename' => 'debian-13-abort-active.qcow2', + ]), $actor)['intent']; + $activeStack = HandlerStack::create(new MockHandler([ + new Response(200, [], '' + . 'intent-upload-5' + . ''), + new ConnectException('abort response lost', + new Request('DELETE', 'https://objects.example.test')), + new Response(404), + new Response(200, [], 'false' + . ''), + ])); + $activeClient = new CloudObjectStorageS3Client(new Client([ + 'handler' => $activeStack, + ])); + $workflow->startUpload((string) $abortActive['id'], $actor, $activeClient); + try { + $workflow->cancel((string) $abortActive['id'], $actor, $activeClient); + throw new \RuntimeException('A still-active abort was reported complete.'); + } catch (CloudError $error) { + $assert($error->httpStatus === 409, + 'A retryable abort failure returned the wrong HTTP status.'); + } + $activeStored = CloudTemplateImportIntent::findOrFail( + (string) $abortActive['id'] + ); + $assert((string) $activeStored->state === 'source_unknown' + && (string) $activeStored->error_code + === 'multipart_abort_not_applied' + && CloudTemplateImportWorkflow::dto($activeStored)['can_cancel'] + === true, + 'Provider proof that Abort did not apply was not made retryable.'); + + // Losing the CreateMultipartUpload response must never be treated as a + // proven no-effect failure. The panel can abandon the unique object key, + // while bucket lifecycle removes any provider-side orphan. + $actor['request_idempotency_key'] = 'multipart-template-intent-0006'; + $createUnknown = $workflow->beginUpload(array_replace($input, [ + 'name' => 'Debian 13 create unknown', + 'filename' => 'debian-13-create-unknown.qcow2', + ]), $actor)['intent']; + $createUnknownClient = new CloudObjectStorageS3Client(new Client([ + 'handler' => HandlerStack::create(new MockHandler([ + new ConnectException('create response lost', + new Request('POST', 'https://objects.example.test')), + ])), + ])); + try { + $workflow->startUpload( + (string) $createUnknown['id'], $actor, $createUnknownClient + ); + throw new \RuntimeException('Unknown multipart create was retryable.'); + } catch (CloudError $error) { + $assert($error->httpStatus === 502, + 'Unknown multipart create returned the wrong HTTP status.'); + } + $unknownStored = CloudTemplateImportIntent::findOrFail( + (string) $createUnknown['id'] + ); + $assert((string) $unknownStored->state === 'source_unknown' + && CloudTemplateImportWorkflow::dto($unknownStored)['can_cancel'] + === true, + 'Unknown multipart create was not fenced for lifecycle abandonment.'); + $abandoned = $workflow->cancel( + (string) $createUnknown['id'], $actor, $createUnknownClient + ); + $assert(($abandoned['state'] ?? null) === 'cancelled' + && ($abandoned['error_code'] ?? null) + === 'multipart_abandoned_lifecycle', + 'Unknown multipart create was not abandoned for lifecycle cleanup.'); + + // An expired intent that never crossed multipart creation is cleaned + // locally without contacting object storage. + $actor['request_idempotency_key'] = 'multipart-template-intent-0007'; + $expired = $workflow->beginUpload(array_replace($input, [ + 'name' => 'Debian 13 expired', + 'filename' => 'debian-13-expired.qcow2', + ]), $actor)['intent']; + Capsule::table('cloud_template_import_intents') + ->where('id', (string) $expired['id']) + ->update(['expires_at' => $clock - 1]); + $assert($workflow->cleanupOneExpired(new CloudObjectStorageS3Client( + new Client(['handler' => HandlerStack::create(new MockHandler([]))]) + )) + && (string) CloudTemplateImportIntent::findOrFail( + (string) $expired['id'] + )->state === 'cancelled', + 'Expired prepared upload was not cleaned without provider mutation.'); + echo "Cloud-template resumable multipart import behavior passed.\n"; }