fix(rescue): recover object deletion attention

go
codex 2 weeks ago
parent 2bec19bd9f
commit adbaa7539e

@ -296,6 +296,31 @@ final class CloudAdminController
)->withHeader('Cache-Control', 'no-store');
}
public function rescueTemplateDeletionRetry(
$request,
$response,
$args = []
) {
$actor = $this->actor($request, true);
$body = CloudInput::object($request);
CloudInput::only($body, [
'expected_revision', 'generation_token',
]);
$templateId = strtolower((string) ($args['id'] ?? ''));
if (preg_match('/^[a-f0-9]{32}$/D', $templateId) !== 1) {
throw new CloudError('救援 ISO 标识无效');
}
return $response->withJson(
CloudRescueTemplateRetirementService::retryObjectDeletion(
$templateId,
$body['expected_revision'] ?? null,
$body['generation_token'] ?? null,
$actor
),
202
)->withHeader('Cache-Control', 'no-store');
}
public function templates($request, $response, $args = [])
{
$method = $request->getMethod();

@ -406,6 +406,11 @@ final class CloudRescueCatalogService
'delete_attention_at' =>
$artifact->delete_attention_at === null
? null : (int) $artifact->delete_attention_at,
'recovery_actions' =>
CloudRescueCatalogStateMachine::recoveryActions(
'artifact',
(string) $artifact->state
),
],
];
}

@ -323,6 +323,84 @@ final class CloudRescueTemplateRetirementService
];
}
/**
* Re-open only the exact provider deletion intent that exhausted its
* bounded reconciliation cycle. Host deletion operations remain settled
* and are never submitted again by this action.
*
* @return array<string,mixed>
*/
public static function retryObjectDeletion(
string $templateId,
mixed $expectedRevision,
mixed $generationToken,
array $actor
): array {
CloudAccess::manage($actor);
self::assertExpectedIdentity(
$templateId,
$expectedRevision,
$generationToken
);
$connection = (new CloudRescueTemplate())->getConnection();
return $connection->transaction(static function () use (
$templateId,
$expectedRevision,
$generationToken
): array {
$template = CloudRescueTemplate::query()
->where('id', $templateId)->lockForUpdate()->first();
$artifact = CloudRescueTemplateArtifact::query()
->where('template_id', $templateId)
->lockForUpdate()->first();
if ($template === null || $artifact === null) {
throw new CloudError('救援 ISO 不存在', 404);
}
self::assertTemplateIdentity(
$template,
(int) $expectedRevision,
(string) $generationToken
);
if ((string) $template->state !== 'retiring'
|| (string) $artifact->state !== 'delete_failed'
|| $artifact->delete_attention_at === null) {
throw new CloudError(
'救援 ISO 对象不在待人工核对的删除状态',
409
);
}
$storageArtifactId = (int) ($artifact->storage_artifact_id ?? 0);
if ($storageArtifactId < 1) {
throw new CloudError(
'旧救援 ISO 不支持双对象存储退役',
409
);
}
(new CloudStorageArtifactDeletionService())->reopenAttention(
$storageArtifactId,
['rescue_template_id' => $templateId]
);
CloudRescueCatalogStateMachine::assertTransition(
'artifact',
'delete_failed',
'delete_pending'
);
$now = time();
$artifact->state = 'delete_pending';
$artifact->error_code = '';
$artifact->delete_next_check_at = $now;
$artifact->delete_attention_at = null;
$artifact->updated_at = $now;
$artifact->save();
return [
'template_id' => $templateId,
'state' => (string) $template->state,
'artifact_state' => (string) $artifact->state,
'retry_scheduled' => true,
];
}, 3);
}
/** @return list<array<string,mixed>> */
private static function scheduleReplicaCleanupLocked(
CloudRescueTemplate $template,

@ -266,6 +266,10 @@ $app->group('/api/v1', function () {
'/admin/rescue-templates/{id}/replicas/{host_id:[1-9][0-9]*}/repair',
'App\Controllers\CloudAdminController:rescueTemplateReplicaRepair'
);
$this->post(
'/admin/rescue-templates/{id}/deletion/retry',
'App\Controllers\CloudAdminController:rescueTemplateDeletionRetry'
);
$this->get('/admin/cloud-template-imports', 'App\Controllers\CloudAdminController:cloudTemplateImportIntents');
$this->post('/admin/cloud-template-imports', 'App\Controllers\CloudAdminController:cloudTemplateImportIntents');
$this->post('/admin/cloud-template-imports/remote',

@ -552,8 +552,15 @@ function rescueTemplateCard(entry) {
);
header.append(identity, metadata);
const actionBar = element('div', 'cloud-rescue-version-actions');
const warning = element('p', 'cloud-rescue-version-note', String(entry.state || '') === 'retiring'
? `正在清理:${Number(entry.replica_summary?.cleanup_debt || 0)} 台宿主机仍有清理债务。`
const artifactRecoveryActions = Array.isArray(entry.artifact?.recovery_actions)
? entry.artifact.recovery_actions.map(String) : [];
const canRetryObjectDeletion = String(entry.state || '') === 'retiring'
&& artifactRecoveryActions.includes('retry_delete')
&& Number(entry.artifact?.delete_attention_at || 0) > 0;
const warning = element('p', 'cloud-rescue-version-note', canRetryObjectDeletion
? '对象存储删除已停止自动重试;继续前会沿用原删除标识并先核对精确对象状态。'
: String(entry.state || '') === 'retiring'
? `正在清理:${Number(entry.replica_summary?.cleanup_debt || 0)} 台宿主机仍有清理债务。`
: '删除后会立即停止新挂载;已经挂载救援介质的实例必须先退出救援。');
const remove = element('button', 'btn btn-outline-danger', String(entry.state || '') === 'retiring' ? '正在退役' : '退役并清理');
remove.type = 'button'; remove.disabled = !canManage || String(entry.state || '') === 'retiring';
@ -576,7 +583,25 @@ function rescueTemplateCard(entry) {
} catch (error) { notice(error.message, true); remove.disabled = !canManage; }
finally { state.busy = false; }
});
actionBar.append(warning, remove);
if (canRetryObjectDeletion) {
const retry = element('button', 'btn btn-outline-warning', '核对并继续清理');
retry.type = 'button'; retry.disabled = !canManage;
retry.addEventListener('click', async () => {
if (state.busy) return;
state.busy = true; retry.disabled = true;
notice('正在重开原对象删除核对,不会创建新的对象删除意图…');
try {
await api(`${catalogPath()}/${encodeURIComponent(entry.id)}/deletion/retry`, {
method: 'POST',
body: { expected_revision: Number(entry.revision), generation_token: String(entry.generation_token || '') },
});
await loadCatalog();
notice('已恢复原对象删除核对,后台会继续收敛清理结果。');
} catch (error) { notice(error.message, true); retry.disabled = !canManage; }
finally { state.busy = false; }
});
actionBar.append(warning, retry);
} else actionBar.append(warning, remove);
card.append(header, actionBar, rescueReplicaMatrix(entry));
return card;
}

@ -622,6 +622,176 @@ namespace {
->where('template_id', $templateId)->count() === 3,
'Exact object deletion did not retain a settled retirement tombstone.');
$attentionId = str_repeat('f', 32);
$attentionGeneration = str_repeat('1', 32);
Capsule::table('cloud_rescue_templates')->insert([
'id' => $attentionId,
'name' => 'Attention delete rescue',
'size_bytes' => $bytes,
'sha256' => $sha256,
'revision' => 1,
'generation_token' => $attentionGeneration,
'state' => 'active',
'active_job_id' => null,
'created_at' => $now,
'updated_at' => $now,
]);
$attentionStorageArtifactId = $seedStorage(
$attentionId,
$attentionGeneration,
$bytes,
$sha256
);
Capsule::table('cloud_rescue_template_artifacts')->insert([
'template_id' => $attentionId,
'storage_artifact_id' => $attentionStorageArtifactId,
'storage_revision' => 0,
'sealed_storage_runtime' => CloudSecrets::encrypt(
$runtime,
'rescue-artifact-runtime'
),
'object_key' => 'virt/rescue-isos/' . $attentionId . '/'
. $attentionGeneration . '/rescue.iso',
'object_version_id' => 'version-1',
'object_etag' => '"etag-1"',
'size_bytes' => $bytes,
'sha256' => $sha256,
'state' => 'ready',
'created_at' => $now,
'updated_at' => $now,
]);
$attentionRetirement = CloudRescueTemplateRetirementService::retire(
$attentionId,
1,
$attentionGeneration,
$actor
);
$assert($attentionRetirement['tasks'] === []
&& (string) CloudRescueTemplateArtifact::findOrFail(
$attentionId
)->state === 'delete_pending',
'Host-free rescue retirement did not arm provider deletion.');
$attentionReplicas = Capsule::table('cloud_storage_replicas')
->where('artifact_id', $attentionStorageArtifactId)
->orderBy('id')->get();
$attentionDeleteOperation = str_repeat('9', 32);
Capsule::table('cloud_storage_operations')->insert([
'operation_id' => $attentionDeleteOperation,
'request_idempotency_key' => 'rescue-attention-delete-1',
'request_fingerprint' => hash(
'sha256',
$attentionId . '|delete|1'
),
'artifact_id' => $attentionStorageArtifactId,
'replica_id' => (int) $attentionReplicas[0]->id,
'source_backend_id' => null,
'target_backend_id' => (int) $attentionReplicas[0]->backend_id,
'kind' => 'delete',
'phase' => 'terminal',
'outcome' => 'failed',
'effect_resolution' => 'unknown',
'effect_may_have_applied' => 1,
'safe_to_resubmit' => 0,
'attempts' => 6,
'lease_token' => null,
'lease_until' => 0,
'available_at' => 0,
'next_check_at' => 0,
'error_code' => 'object_delete_reconcile_exhausted',
'result_json' => null,
'terminal_at' => $now,
'created_at' => $now,
'updated_at' => $now,
]);
Capsule::table('cloud_storage_replicas')
->where('id', (int) $attentionReplicas[0]->id)->update([
'state' => 'delete_attention',
'delete_operation_id' => $attentionDeleteOperation,
'delete_generation' => 1,
'delete_cycle_attempts' => 6,
'delete_presence_confirmations' => 0,
'delete_next_check_at' => 0,
'delete_attention_at' => $now,
'error_code' => 'object_delete_reconcile_exhausted',
'updated_at' => $now,
]);
Capsule::table('cloud_storage_replicas')
->where('id', (int) $attentionReplicas[1]->id)->update([
'state' => 'deleted',
'delete_next_check_at' => 0,
'delete_attention_at' => null,
'error_code' => '',
'updated_at' => $now,
]);
Capsule::table('cloud_storage_artifacts')
->where('id', $attentionStorageArtifactId)->update([
'state' => 'delete_attention',
'last_operation_id' => $attentionDeleteOperation,
'updated_at' => $now,
]);
Capsule::table('cloud_rescue_template_artifacts')
->where('template_id', $attentionId)->update([
'state' => 'delete_failed',
'error_code' => 'rescue_storage_delete_attention',
'delete_next_check_at' => 0,
'delete_attention_at' => $now,
'updated_at' => $now,
]);
$attentionCatalogEntry = collect(
CloudRescueCatalogService::adminCatalog()['templates']
)->firstWhere('id', $attentionId);
$assert(is_array($attentionCatalogEntry)
&& ($attentionCatalogEntry['artifact']['delete_attention_at'] ?? null)
!== null
&& in_array(
'retry_delete',
$attentionCatalogEntry['artifact']['recovery_actions'] ?? [],
true
),
'Administrator rescue DTO did not expose the executable recovery action.');
$retry = CloudRescueTemplateRetirementService::retryObjectDeletion(
$attentionId,
1,
$attentionGeneration,
$actor
);
$attentionOperation = Capsule::table('cloud_storage_operations')
->where('operation_id', $attentionDeleteOperation)->first();
$attentionReplica = Capsule::table('cloud_storage_replicas')
->where('id', (int) $attentionReplicas[0]->id)->first();
$attentionArtifact = CloudRescueTemplateArtifact::findOrFail(
$attentionId
);
$assert(($retry['retry_scheduled'] ?? false) === true
&& (string) $attentionArtifact->state === 'delete_pending'
&& $attentionArtifact->delete_attention_at === null
&& (string) $attentionOperation->operation_id
=== $attentionDeleteOperation
&& (string) $attentionOperation->phase === 'reconciling'
&& (string) $attentionOperation->effect_resolution === 'unknown'
&& (string) $attentionReplica->delete_operation_id
=== $attentionDeleteOperation
&& (string) $attentionReplica->state === 'delete_uncertain'
&& Capsule::table('cloud_storage_operations')
->where('artifact_id', $attentionStorageArtifactId)
->where('kind', 'delete')->count() === 1,
'Manual rescue cleanup recovery did not resume the exact delete intent.');
$retryRejected = false;
try {
CloudRescueTemplateRetirementService::retryObjectDeletion(
$attentionId,
1,
$attentionGeneration,
$actor
);
} catch (\App\Services\Cloud\CloudError $error) {
$retryRejected = $error->httpStatus === 409;
}
$assert($retryRejected,
'Rescue cleanup recovery accepted an artifact outside attention.');
$unknownId = str_repeat('d', 32);
$unknownGeneration = str_repeat('e', 32);
Capsule::table('cloud_rescue_templates')->insert([

@ -746,8 +746,16 @@ $assert(str_contains($cloudAdminController,
'CloudRescueTemplateReplicaCoordinator::syncHost(')
&& str_contains($cloudRoutes,
'/admin/rescue-templates/{id}/replicas/{host_id:[1-9][0-9]*}/repair')
&& str_contains($cloudRoutes,
'/admin/rescue-templates/{id}/deletion/retry')
&& str_contains($cloudAdminController,
'public function rescueTemplateDeletionRetry(')
&& str_contains($adminScript,
'/api/v1/admin/rescue-templates/${encodeURIComponent(entry.id)}/replicas/')
&& str_contains($adminScript,
'artifactRecoveryActions.includes(\'retry_delete\')')
&& str_contains($adminScript,
'/deletion/retry`')
&& str_contains($adminScript,
'expected_revision: Number(entry.revision)')
&& str_contains($adminScript,

Loading…
Cancel
Save