|
|
|
|
@ -622,6 +622,176 @@ namespace {
|
|
|
|
|
->where('template_id', $templateId)->count() === 3,
|
|
|
|
|
'Exact object deletion did not retain a settled retirement tombstone.');
|
|
|
|
|
|
|
|
|
|
$attentionId = str_repeat('f', 32);
|
|
|
|
|
$attentionGeneration = str_repeat('1', 32);
|
|
|
|
|
Capsule::table('cloud_rescue_templates')->insert([
|
|
|
|
|
'id' => $attentionId,
|
|
|
|
|
'name' => 'Attention delete rescue',
|
|
|
|
|
'size_bytes' => $bytes,
|
|
|
|
|
'sha256' => $sha256,
|
|
|
|
|
'revision' => 1,
|
|
|
|
|
'generation_token' => $attentionGeneration,
|
|
|
|
|
'state' => 'active',
|
|
|
|
|
'active_job_id' => null,
|
|
|
|
|
'created_at' => $now,
|
|
|
|
|
'updated_at' => $now,
|
|
|
|
|
]);
|
|
|
|
|
$attentionStorageArtifactId = $seedStorage(
|
|
|
|
|
$attentionId,
|
|
|
|
|
$attentionGeneration,
|
|
|
|
|
$bytes,
|
|
|
|
|
$sha256
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_rescue_template_artifacts')->insert([
|
|
|
|
|
'template_id' => $attentionId,
|
|
|
|
|
'storage_artifact_id' => $attentionStorageArtifactId,
|
|
|
|
|
'storage_revision' => 0,
|
|
|
|
|
'sealed_storage_runtime' => CloudSecrets::encrypt(
|
|
|
|
|
$runtime,
|
|
|
|
|
'rescue-artifact-runtime'
|
|
|
|
|
),
|
|
|
|
|
'object_key' => 'virt/rescue-isos/' . $attentionId . '/'
|
|
|
|
|
. $attentionGeneration . '/rescue.iso',
|
|
|
|
|
'object_version_id' => 'version-1',
|
|
|
|
|
'object_etag' => '"etag-1"',
|
|
|
|
|
'size_bytes' => $bytes,
|
|
|
|
|
'sha256' => $sha256,
|
|
|
|
|
'state' => 'ready',
|
|
|
|
|
'created_at' => $now,
|
|
|
|
|
'updated_at' => $now,
|
|
|
|
|
]);
|
|
|
|
|
$attentionRetirement = CloudRescueTemplateRetirementService::retire(
|
|
|
|
|
$attentionId,
|
|
|
|
|
1,
|
|
|
|
|
$attentionGeneration,
|
|
|
|
|
$actor
|
|
|
|
|
);
|
|
|
|
|
$assert($attentionRetirement['tasks'] === []
|
|
|
|
|
&& (string) CloudRescueTemplateArtifact::findOrFail(
|
|
|
|
|
$attentionId
|
|
|
|
|
)->state === 'delete_pending',
|
|
|
|
|
'Host-free rescue retirement did not arm provider deletion.');
|
|
|
|
|
$attentionReplicas = Capsule::table('cloud_storage_replicas')
|
|
|
|
|
->where('artifact_id', $attentionStorageArtifactId)
|
|
|
|
|
->orderBy('id')->get();
|
|
|
|
|
$attentionDeleteOperation = str_repeat('9', 32);
|
|
|
|
|
Capsule::table('cloud_storage_operations')->insert([
|
|
|
|
|
'operation_id' => $attentionDeleteOperation,
|
|
|
|
|
'request_idempotency_key' => 'rescue-attention-delete-1',
|
|
|
|
|
'request_fingerprint' => hash(
|
|
|
|
|
'sha256',
|
|
|
|
|
$attentionId . '|delete|1'
|
|
|
|
|
),
|
|
|
|
|
'artifact_id' => $attentionStorageArtifactId,
|
|
|
|
|
'replica_id' => (int) $attentionReplicas[0]->id,
|
|
|
|
|
'source_backend_id' => null,
|
|
|
|
|
'target_backend_id' => (int) $attentionReplicas[0]->backend_id,
|
|
|
|
|
'kind' => 'delete',
|
|
|
|
|
'phase' => 'terminal',
|
|
|
|
|
'outcome' => 'failed',
|
|
|
|
|
'effect_resolution' => 'unknown',
|
|
|
|
|
'effect_may_have_applied' => 1,
|
|
|
|
|
'safe_to_resubmit' => 0,
|
|
|
|
|
'attempts' => 6,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'next_check_at' => 0,
|
|
|
|
|
'error_code' => 'object_delete_reconcile_exhausted',
|
|
|
|
|
'result_json' => null,
|
|
|
|
|
'terminal_at' => $now,
|
|
|
|
|
'created_at' => $now,
|
|
|
|
|
'updated_at' => $now,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_storage_replicas')
|
|
|
|
|
->where('id', (int) $attentionReplicas[0]->id)->update([
|
|
|
|
|
'state' => 'delete_attention',
|
|
|
|
|
'delete_operation_id' => $attentionDeleteOperation,
|
|
|
|
|
'delete_generation' => 1,
|
|
|
|
|
'delete_cycle_attempts' => 6,
|
|
|
|
|
'delete_presence_confirmations' => 0,
|
|
|
|
|
'delete_next_check_at' => 0,
|
|
|
|
|
'delete_attention_at' => $now,
|
|
|
|
|
'error_code' => 'object_delete_reconcile_exhausted',
|
|
|
|
|
'updated_at' => $now,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_storage_replicas')
|
|
|
|
|
->where('id', (int) $attentionReplicas[1]->id)->update([
|
|
|
|
|
'state' => 'deleted',
|
|
|
|
|
'delete_next_check_at' => 0,
|
|
|
|
|
'delete_attention_at' => null,
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
'updated_at' => $now,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_storage_artifacts')
|
|
|
|
|
->where('id', $attentionStorageArtifactId)->update([
|
|
|
|
|
'state' => 'delete_attention',
|
|
|
|
|
'last_operation_id' => $attentionDeleteOperation,
|
|
|
|
|
'updated_at' => $now,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_rescue_template_artifacts')
|
|
|
|
|
->where('template_id', $attentionId)->update([
|
|
|
|
|
'state' => 'delete_failed',
|
|
|
|
|
'error_code' => 'rescue_storage_delete_attention',
|
|
|
|
|
'delete_next_check_at' => 0,
|
|
|
|
|
'delete_attention_at' => $now,
|
|
|
|
|
'updated_at' => $now,
|
|
|
|
|
]);
|
|
|
|
|
|
|
|
|
|
$attentionCatalogEntry = collect(
|
|
|
|
|
CloudRescueCatalogService::adminCatalog()['templates']
|
|
|
|
|
)->firstWhere('id', $attentionId);
|
|
|
|
|
$assert(is_array($attentionCatalogEntry)
|
|
|
|
|
&& ($attentionCatalogEntry['artifact']['delete_attention_at'] ?? null)
|
|
|
|
|
!== null
|
|
|
|
|
&& in_array(
|
|
|
|
|
'retry_delete',
|
|
|
|
|
$attentionCatalogEntry['artifact']['recovery_actions'] ?? [],
|
|
|
|
|
true
|
|
|
|
|
),
|
|
|
|
|
'Administrator rescue DTO did not expose the executable recovery action.');
|
|
|
|
|
|
|
|
|
|
$retry = CloudRescueTemplateRetirementService::retryObjectDeletion(
|
|
|
|
|
$attentionId,
|
|
|
|
|
1,
|
|
|
|
|
$attentionGeneration,
|
|
|
|
|
$actor
|
|
|
|
|
);
|
|
|
|
|
$attentionOperation = Capsule::table('cloud_storage_operations')
|
|
|
|
|
->where('operation_id', $attentionDeleteOperation)->first();
|
|
|
|
|
$attentionReplica = Capsule::table('cloud_storage_replicas')
|
|
|
|
|
->where('id', (int) $attentionReplicas[0]->id)->first();
|
|
|
|
|
$attentionArtifact = CloudRescueTemplateArtifact::findOrFail(
|
|
|
|
|
$attentionId
|
|
|
|
|
);
|
|
|
|
|
$assert(($retry['retry_scheduled'] ?? false) === true
|
|
|
|
|
&& (string) $attentionArtifact->state === 'delete_pending'
|
|
|
|
|
&& $attentionArtifact->delete_attention_at === null
|
|
|
|
|
&& (string) $attentionOperation->operation_id
|
|
|
|
|
=== $attentionDeleteOperation
|
|
|
|
|
&& (string) $attentionOperation->phase === 'reconciling'
|
|
|
|
|
&& (string) $attentionOperation->effect_resolution === 'unknown'
|
|
|
|
|
&& (string) $attentionReplica->delete_operation_id
|
|
|
|
|
=== $attentionDeleteOperation
|
|
|
|
|
&& (string) $attentionReplica->state === 'delete_uncertain'
|
|
|
|
|
&& Capsule::table('cloud_storage_operations')
|
|
|
|
|
->where('artifact_id', $attentionStorageArtifactId)
|
|
|
|
|
->where('kind', 'delete')->count() === 1,
|
|
|
|
|
'Manual rescue cleanup recovery did not resume the exact delete intent.');
|
|
|
|
|
$retryRejected = false;
|
|
|
|
|
try {
|
|
|
|
|
CloudRescueTemplateRetirementService::retryObjectDeletion(
|
|
|
|
|
$attentionId,
|
|
|
|
|
1,
|
|
|
|
|
$attentionGeneration,
|
|
|
|
|
$actor
|
|
|
|
|
);
|
|
|
|
|
} catch (\App\Services\Cloud\CloudError $error) {
|
|
|
|
|
$retryRejected = $error->httpStatus === 409;
|
|
|
|
|
}
|
|
|
|
|
$assert($retryRejected,
|
|
|
|
|
'Rescue cleanup recovery accepted an artifact outside attention.');
|
|
|
|
|
|
|
|
|
|
$unknownId = str_repeat('d', 32);
|
|
|
|
|
$unknownGeneration = str_repeat('e', 32);
|
|
|
|
|
Capsule::table('cloud_rescue_templates')->insert([
|
|
|
|
|
|