From adbaa7539ef236d11aa6d5f075f467d716e09f6f Mon Sep 17 00:00:00 2001 From: codex Date: Fri, 6 Nov 2026 20:33:28 +0800 Subject: [PATCH] fix(rescue): recover object deletion attention --- web/app/Controllers/CloudAdminController.php | 25 +++ .../Cloud/CloudRescueCatalogService.php | 5 + .../CloudRescueTemplateRetirementService.php | 78 ++++++++ web/config/routes_cloud.php | 4 + web/public/cloud/admin-catalog-runtime.js | 31 +++- web/tests/CloudRescueReplicaWorkflowTest.php | 170 ++++++++++++++++++ web/tests/VirtualizationAssetsTest.php | 8 + 7 files changed, 318 insertions(+), 3 deletions(-) diff --git a/web/app/Controllers/CloudAdminController.php b/web/app/Controllers/CloudAdminController.php index 825cc774..c79f90d9 100644 --- a/web/app/Controllers/CloudAdminController.php +++ b/web/app/Controllers/CloudAdminController.php @@ -296,6 +296,31 @@ final class CloudAdminController )->withHeader('Cache-Control', 'no-store'); } + public function rescueTemplateDeletionRetry( + $request, + $response, + $args = [] + ) { + $actor = $this->actor($request, true); + $body = CloudInput::object($request); + CloudInput::only($body, [ + 'expected_revision', 'generation_token', + ]); + $templateId = strtolower((string) ($args['id'] ?? '')); + if (preg_match('/^[a-f0-9]{32}$/D', $templateId) !== 1) { + throw new CloudError('救援 ISO 标识无效'); + } + return $response->withJson( + CloudRescueTemplateRetirementService::retryObjectDeletion( + $templateId, + $body['expected_revision'] ?? null, + $body['generation_token'] ?? null, + $actor + ), + 202 + )->withHeader('Cache-Control', 'no-store'); + } + public function templates($request, $response, $args = []) { $method = $request->getMethod(); diff --git a/web/app/Services/Cloud/CloudRescueCatalogService.php b/web/app/Services/Cloud/CloudRescueCatalogService.php index 69aab509..7a476a63 100644 --- a/web/app/Services/Cloud/CloudRescueCatalogService.php +++ b/web/app/Services/Cloud/CloudRescueCatalogService.php @@ -406,6 +406,11 @@ final class CloudRescueCatalogService 'delete_attention_at' => $artifact->delete_attention_at === null ? null : (int) $artifact->delete_attention_at, + 'recovery_actions' => + CloudRescueCatalogStateMachine::recoveryActions( + 'artifact', + (string) $artifact->state + ), ], ]; } diff --git a/web/app/Services/Cloud/CloudRescueTemplateRetirementService.php b/web/app/Services/Cloud/CloudRescueTemplateRetirementService.php index 9b773fba..d641b11c 100644 --- a/web/app/Services/Cloud/CloudRescueTemplateRetirementService.php +++ b/web/app/Services/Cloud/CloudRescueTemplateRetirementService.php @@ -323,6 +323,84 @@ final class CloudRescueTemplateRetirementService ]; } + /** + * Re-open only the exact provider deletion intent that exhausted its + * bounded reconciliation cycle. Host deletion operations remain settled + * and are never submitted again by this action. + * + * @return array + */ + public static function retryObjectDeletion( + string $templateId, + mixed $expectedRevision, + mixed $generationToken, + array $actor + ): array { + CloudAccess::manage($actor); + self::assertExpectedIdentity( + $templateId, + $expectedRevision, + $generationToken + ); + $connection = (new CloudRescueTemplate())->getConnection(); + return $connection->transaction(static function () use ( + $templateId, + $expectedRevision, + $generationToken + ): array { + $template = CloudRescueTemplate::query() + ->where('id', $templateId)->lockForUpdate()->first(); + $artifact = CloudRescueTemplateArtifact::query() + ->where('template_id', $templateId) + ->lockForUpdate()->first(); + if ($template === null || $artifact === null) { + throw new CloudError('救援 ISO 不存在', 404); + } + self::assertTemplateIdentity( + $template, + (int) $expectedRevision, + (string) $generationToken + ); + if ((string) $template->state !== 'retiring' + || (string) $artifact->state !== 'delete_failed' + || $artifact->delete_attention_at === null) { + throw new CloudError( + '救援 ISO 对象不在待人工核对的删除状态', + 409 + ); + } + $storageArtifactId = (int) ($artifact->storage_artifact_id ?? 0); + if ($storageArtifactId < 1) { + throw new CloudError( + '旧救援 ISO 不支持双对象存储退役', + 409 + ); + } + (new CloudStorageArtifactDeletionService())->reopenAttention( + $storageArtifactId, + ['rescue_template_id' => $templateId] + ); + CloudRescueCatalogStateMachine::assertTransition( + 'artifact', + 'delete_failed', + 'delete_pending' + ); + $now = time(); + $artifact->state = 'delete_pending'; + $artifact->error_code = ''; + $artifact->delete_next_check_at = $now; + $artifact->delete_attention_at = null; + $artifact->updated_at = $now; + $artifact->save(); + return [ + 'template_id' => $templateId, + 'state' => (string) $template->state, + 'artifact_state' => (string) $artifact->state, + 'retry_scheduled' => true, + ]; + }, 3); + } + /** @return list> */ private static function scheduleReplicaCleanupLocked( CloudRescueTemplate $template, diff --git a/web/config/routes_cloud.php b/web/config/routes_cloud.php index a71850ba..900d98ad 100644 --- a/web/config/routes_cloud.php +++ b/web/config/routes_cloud.php @@ -266,6 +266,10 @@ $app->group('/api/v1', function () { '/admin/rescue-templates/{id}/replicas/{host_id:[1-9][0-9]*}/repair', 'App\Controllers\CloudAdminController:rescueTemplateReplicaRepair' ); + $this->post( + '/admin/rescue-templates/{id}/deletion/retry', + 'App\Controllers\CloudAdminController:rescueTemplateDeletionRetry' + ); $this->get('/admin/cloud-template-imports', 'App\Controllers\CloudAdminController:cloudTemplateImportIntents'); $this->post('/admin/cloud-template-imports', 'App\Controllers\CloudAdminController:cloudTemplateImportIntents'); $this->post('/admin/cloud-template-imports/remote', diff --git a/web/public/cloud/admin-catalog-runtime.js b/web/public/cloud/admin-catalog-runtime.js index cec88821..0cce2745 100644 --- a/web/public/cloud/admin-catalog-runtime.js +++ b/web/public/cloud/admin-catalog-runtime.js @@ -552,8 +552,15 @@ function rescueTemplateCard(entry) { ); header.append(identity, metadata); const actionBar = element('div', 'cloud-rescue-version-actions'); - const warning = element('p', 'cloud-rescue-version-note', String(entry.state || '') === 'retiring' - ? `正在清理:${Number(entry.replica_summary?.cleanup_debt || 0)} 台宿主机仍有清理债务。` + const artifactRecoveryActions = Array.isArray(entry.artifact?.recovery_actions) + ? entry.artifact.recovery_actions.map(String) : []; + const canRetryObjectDeletion = String(entry.state || '') === 'retiring' + && artifactRecoveryActions.includes('retry_delete') + && Number(entry.artifact?.delete_attention_at || 0) > 0; + const warning = element('p', 'cloud-rescue-version-note', canRetryObjectDeletion + ? '对象存储删除已停止自动重试;继续前会沿用原删除标识并先核对精确对象状态。' + : String(entry.state || '') === 'retiring' + ? `正在清理:${Number(entry.replica_summary?.cleanup_debt || 0)} 台宿主机仍有清理债务。` : '删除后会立即停止新挂载;已经挂载救援介质的实例必须先退出救援。'); const remove = element('button', 'btn btn-outline-danger', String(entry.state || '') === 'retiring' ? '正在退役' : '退役并清理'); remove.type = 'button'; remove.disabled = !canManage || String(entry.state || '') === 'retiring'; @@ -576,7 +583,25 @@ function rescueTemplateCard(entry) { } catch (error) { notice(error.message, true); remove.disabled = !canManage; } finally { state.busy = false; } }); - actionBar.append(warning, remove); + if (canRetryObjectDeletion) { + const retry = element('button', 'btn btn-outline-warning', '核对并继续清理'); + retry.type = 'button'; retry.disabled = !canManage; + retry.addEventListener('click', async () => { + if (state.busy) return; + state.busy = true; retry.disabled = true; + notice('正在重开原对象删除核对,不会创建新的对象删除意图…'); + try { + await api(`${catalogPath()}/${encodeURIComponent(entry.id)}/deletion/retry`, { + method: 'POST', + body: { expected_revision: Number(entry.revision), generation_token: String(entry.generation_token || '') }, + }); + await loadCatalog(); + notice('已恢复原对象删除核对,后台会继续收敛清理结果。'); + } catch (error) { notice(error.message, true); retry.disabled = !canManage; } + finally { state.busy = false; } + }); + actionBar.append(warning, retry); + } else actionBar.append(warning, remove); card.append(header, actionBar, rescueReplicaMatrix(entry)); return card; } diff --git a/web/tests/CloudRescueReplicaWorkflowTest.php b/web/tests/CloudRescueReplicaWorkflowTest.php index 2ba4bf61..f5cce00c 100644 --- a/web/tests/CloudRescueReplicaWorkflowTest.php +++ b/web/tests/CloudRescueReplicaWorkflowTest.php @@ -622,6 +622,176 @@ namespace { ->where('template_id', $templateId)->count() === 3, 'Exact object deletion did not retain a settled retirement tombstone.'); + $attentionId = str_repeat('f', 32); + $attentionGeneration = str_repeat('1', 32); + Capsule::table('cloud_rescue_templates')->insert([ + 'id' => $attentionId, + 'name' => 'Attention delete rescue', + 'size_bytes' => $bytes, + 'sha256' => $sha256, + 'revision' => 1, + 'generation_token' => $attentionGeneration, + 'state' => 'active', + 'active_job_id' => null, + 'created_at' => $now, + 'updated_at' => $now, + ]); + $attentionStorageArtifactId = $seedStorage( + $attentionId, + $attentionGeneration, + $bytes, + $sha256 + ); + Capsule::table('cloud_rescue_template_artifacts')->insert([ + 'template_id' => $attentionId, + 'storage_artifact_id' => $attentionStorageArtifactId, + 'storage_revision' => 0, + 'sealed_storage_runtime' => CloudSecrets::encrypt( + $runtime, + 'rescue-artifact-runtime' + ), + 'object_key' => 'virt/rescue-isos/' . $attentionId . '/' + . $attentionGeneration . '/rescue.iso', + 'object_version_id' => 'version-1', + 'object_etag' => '"etag-1"', + 'size_bytes' => $bytes, + 'sha256' => $sha256, + 'state' => 'ready', + 'created_at' => $now, + 'updated_at' => $now, + ]); + $attentionRetirement = CloudRescueTemplateRetirementService::retire( + $attentionId, + 1, + $attentionGeneration, + $actor + ); + $assert($attentionRetirement['tasks'] === [] + && (string) CloudRescueTemplateArtifact::findOrFail( + $attentionId + )->state === 'delete_pending', + 'Host-free rescue retirement did not arm provider deletion.'); + $attentionReplicas = Capsule::table('cloud_storage_replicas') + ->where('artifact_id', $attentionStorageArtifactId) + ->orderBy('id')->get(); + $attentionDeleteOperation = str_repeat('9', 32); + Capsule::table('cloud_storage_operations')->insert([ + 'operation_id' => $attentionDeleteOperation, + 'request_idempotency_key' => 'rescue-attention-delete-1', + 'request_fingerprint' => hash( + 'sha256', + $attentionId . '|delete|1' + ), + 'artifact_id' => $attentionStorageArtifactId, + 'replica_id' => (int) $attentionReplicas[0]->id, + 'source_backend_id' => null, + 'target_backend_id' => (int) $attentionReplicas[0]->backend_id, + 'kind' => 'delete', + 'phase' => 'terminal', + 'outcome' => 'failed', + 'effect_resolution' => 'unknown', + 'effect_may_have_applied' => 1, + 'safe_to_resubmit' => 0, + 'attempts' => 6, + 'lease_token' => null, + 'lease_until' => 0, + 'available_at' => 0, + 'next_check_at' => 0, + 'error_code' => 'object_delete_reconcile_exhausted', + 'result_json' => null, + 'terminal_at' => $now, + 'created_at' => $now, + 'updated_at' => $now, + ]); + Capsule::table('cloud_storage_replicas') + ->where('id', (int) $attentionReplicas[0]->id)->update([ + 'state' => 'delete_attention', + 'delete_operation_id' => $attentionDeleteOperation, + 'delete_generation' => 1, + 'delete_cycle_attempts' => 6, + 'delete_presence_confirmations' => 0, + 'delete_next_check_at' => 0, + 'delete_attention_at' => $now, + 'error_code' => 'object_delete_reconcile_exhausted', + 'updated_at' => $now, + ]); + Capsule::table('cloud_storage_replicas') + ->where('id', (int) $attentionReplicas[1]->id)->update([ + 'state' => 'deleted', + 'delete_next_check_at' => 0, + 'delete_attention_at' => null, + 'error_code' => '', + 'updated_at' => $now, + ]); + Capsule::table('cloud_storage_artifacts') + ->where('id', $attentionStorageArtifactId)->update([ + 'state' => 'delete_attention', + 'last_operation_id' => $attentionDeleteOperation, + 'updated_at' => $now, + ]); + Capsule::table('cloud_rescue_template_artifacts') + ->where('template_id', $attentionId)->update([ + 'state' => 'delete_failed', + 'error_code' => 'rescue_storage_delete_attention', + 'delete_next_check_at' => 0, + 'delete_attention_at' => $now, + 'updated_at' => $now, + ]); + + $attentionCatalogEntry = collect( + CloudRescueCatalogService::adminCatalog()['templates'] + )->firstWhere('id', $attentionId); + $assert(is_array($attentionCatalogEntry) + && ($attentionCatalogEntry['artifact']['delete_attention_at'] ?? null) + !== null + && in_array( + 'retry_delete', + $attentionCatalogEntry['artifact']['recovery_actions'] ?? [], + true + ), + 'Administrator rescue DTO did not expose the executable recovery action.'); + + $retry = CloudRescueTemplateRetirementService::retryObjectDeletion( + $attentionId, + 1, + $attentionGeneration, + $actor + ); + $attentionOperation = Capsule::table('cloud_storage_operations') + ->where('operation_id', $attentionDeleteOperation)->first(); + $attentionReplica = Capsule::table('cloud_storage_replicas') + ->where('id', (int) $attentionReplicas[0]->id)->first(); + $attentionArtifact = CloudRescueTemplateArtifact::findOrFail( + $attentionId + ); + $assert(($retry['retry_scheduled'] ?? false) === true + && (string) $attentionArtifact->state === 'delete_pending' + && $attentionArtifact->delete_attention_at === null + && (string) $attentionOperation->operation_id + === $attentionDeleteOperation + && (string) $attentionOperation->phase === 'reconciling' + && (string) $attentionOperation->effect_resolution === 'unknown' + && (string) $attentionReplica->delete_operation_id + === $attentionDeleteOperation + && (string) $attentionReplica->state === 'delete_uncertain' + && Capsule::table('cloud_storage_operations') + ->where('artifact_id', $attentionStorageArtifactId) + ->where('kind', 'delete')->count() === 1, + 'Manual rescue cleanup recovery did not resume the exact delete intent.'); + $retryRejected = false; + try { + CloudRescueTemplateRetirementService::retryObjectDeletion( + $attentionId, + 1, + $attentionGeneration, + $actor + ); + } catch (\App\Services\Cloud\CloudError $error) { + $retryRejected = $error->httpStatus === 409; + } + $assert($retryRejected, + 'Rescue cleanup recovery accepted an artifact outside attention.'); + $unknownId = str_repeat('d', 32); $unknownGeneration = str_repeat('e', 32); Capsule::table('cloud_rescue_templates')->insert([ diff --git a/web/tests/VirtualizationAssetsTest.php b/web/tests/VirtualizationAssetsTest.php index 0beaa049..7bbe6009 100644 --- a/web/tests/VirtualizationAssetsTest.php +++ b/web/tests/VirtualizationAssetsTest.php @@ -746,8 +746,16 @@ $assert(str_contains($cloudAdminController, 'CloudRescueTemplateReplicaCoordinator::syncHost(') && str_contains($cloudRoutes, '/admin/rescue-templates/{id}/replicas/{host_id:[1-9][0-9]*}/repair') + && str_contains($cloudRoutes, + '/admin/rescue-templates/{id}/deletion/retry') + && str_contains($cloudAdminController, + 'public function rescueTemplateDeletionRetry(') && str_contains($adminScript, '/api/v1/admin/rescue-templates/${encodeURIComponent(entry.id)}/replicas/') + && str_contains($adminScript, + 'artifactRecoveryActions.includes(\'retry_delete\')') + && str_contains($adminScript, + '/deletion/retry`') && str_contains($adminScript, 'expected_revision: Number(entry.revision)') && str_contains($adminScript,