httpStatus === 409, 'Deletion blocker returned an unexpected status.'); return; } throw new RuntimeException('Expected deletion blocker was not raised.'); }; $db = new Capsule(); $db->addConnection(['driver' => 'sqlite', 'database' => ':memory:']); $db->setAsGlobal(); $db->bootEloquent(); $schema = $db->schema(); $schema->create('cloud_instance_backups', static function (Blueprint $t): void { $t->increments('id'); $t->string('backup_key')->unique(); $t->integer('source_instance_id'); $t->integer('source_host_id'); $t->integer('user_id'); $t->string('source_uuid'); $t->string('name'); $t->string('purpose')->default('portable'); $t->string('source')->default('manual'); $t->integer('subscription_id')->nullable(); $t->string('state'); $t->string('consistency'); $t->string('object_key')->unique(); $t->integer('object_size_bytes'); $t->integer('virtual_size_bytes'); $t->string('sha256'); $t->string('object_etag')->default(''); $t->string('object_version_id')->default(''); $t->text('spec_json'); $t->text('sealed_credential')->nullable(); $t->integer('storage_revision'); $t->integer('storage_artifact_id')->nullable(); $t->integer('active_job_id')->nullable(); $t->string('last_operation_id')->nullable(); $t->string('delete_operation_id')->nullable(); $t->integer('delete_generation')->default(0); $t->integer('delete_cycle_attempts')->default(0); $t->integer('delete_presence_confirmations')->default(0); $t->integer('delete_requested_at')->nullable(); $t->integer('delete_last_checked_at')->nullable(); $t->integer('delete_next_check_at')->default(0); $t->integer('delete_attention_at')->nullable(); $t->string('error_code')->default(''); $t->integer('ready_at')->nullable(); $t->integer('created_at'); $t->integer('updated_at'); }); $schema->create('cloud_deployment_snapshots', static function (Blueprint $t): void { $t->increments('id'); $t->integer('backup_id')->unique(); $t->string('state'); $t->string('error_code')->default(''); $t->integer('deleted_at')->nullable(); $t->integer('updated_at')->default(0); }); $schema->create('cloud_backup_restores', static function (Blueprint $t): void { $t->increments('id'); $t->integer('backup_id'); $t->string('state'); }); $schema->create('cloud_backup_restore_attempts', static function (Blueprint $t): void { $t->increments('id'); $t->integer('backup_restore_id'); $t->string('state'); $t->integer('reservation_id')->nullable(); }); $schema->create('cloud_backup_recoveries', static function (Blueprint $t): void { $t->increments('id'); $t->integer('backup_id'); $t->string('state'); }); $schema->create('cloud_jobs', static function (Blueprint $t): void { $t->increments('id'); $t->integer('backup_id')->nullable(); $t->string('phase'); $t->string('effect_resolution'); $t->integer('finalization_incident')->default(0); }); $schema->create('cloud_resource_reservations', static function (Blueprint $t): void { $t->increments('id'); $t->string('state'); $t->integer('released_at')->nullable(); }); $schema->create('cloud_resource_reservation_ips', static function (Blueprint $t): void { $t->increments('id'); $t->integer('reservation_id'); }); $schema->create('cloud_storage_artifacts', static function (Blueprint $t): void { $t->increments('id'); $t->string('artifact_key')->unique(); $t->string('kind'); $t->integer('maximum_size_bytes'); $t->integer('size_bytes'); $t->string('sha256'); $t->string('durability_policy'); $t->integer('desired_copies'); $t->integer('required_copies'); $t->string('state'); $t->string('last_operation_id')->nullable(); $t->integer('verified_at'); $t->integer('created_at'); $t->integer('updated_at'); }); $schema->create('cloud_storage_replicas', static function (Blueprint $t): void { $t->increments('id'); $t->string('replica_key')->unique(); $t->integer('artifact_id'); $t->integer('backend_id'); $t->string('role'); $t->string('object_key'); $t->string('object_version_id')->default(''); $t->string('object_etag')->default(''); $t->integer('size_bytes'); $t->string('sha256'); $t->string('state'); $t->string('publish_operation_id')->nullable(); $t->string('delete_operation_id')->nullable(); $t->integer('delete_generation')->default(0); $t->integer('delete_cycle_attempts')->default(0); $t->integer('delete_presence_confirmations')->default(0); $t->integer('delete_last_checked_at')->default(0); $t->integer('delete_next_check_at')->default(0); $t->integer('delete_attention_at')->nullable(); $t->string('lease_token')->nullable(); $t->integer('lease_until')->default(0); $t->string('error_code')->default(''); $t->integer('verified_at')->default(0); $t->integer('created_at'); $t->integer('updated_at'); }); $schema->create('cloud_storage_operations', static function (Blueprint $t): void { $t->increments('id'); $t->string('operation_id')->unique(); $t->string('request_idempotency_key')->unique(); $t->string('request_fingerprint'); $t->integer('artifact_id'); $t->integer('replica_id')->nullable(); $t->integer('source_backend_id')->nullable(); $t->integer('target_backend_id')->nullable(); $t->string('kind'); $t->string('phase'); $t->string('outcome')->nullable(); $t->string('effect_resolution'); $t->integer('effect_may_have_applied')->nullable(); $t->integer('safe_to_resubmit')->nullable(); $t->integer('attempts'); $t->string('lease_token')->nullable(); $t->integer('lease_until'); $t->integer('available_at'); $t->integer('next_check_at'); $t->string('error_code'); $t->text('result_json')->nullable(); $t->integer('terminal_at')->nullable(); $t->integer('created_at'); $t->integer('updated_at'); }); $schema->create('cloud_storage_publications', static function (Blueprint $t): void { $t->increments('id'); $t->string('publication_key')->unique(); $t->integer('artifact_id')->unique(); $t->integer('backup_id')->nullable(); $t->string('template_import_intent_id')->nullable(); $t->string('rescue_template_id')->nullable(); $t->string('state'); $t->integer('attempts'); $t->integer('available_at'); $t->string('lease_token')->nullable(); $t->integer('lease_until'); $t->string('last_error_code'); $t->integer('coverage_ready_at')->nullable(); $t->integer('finalized_at')->nullable(); $t->integer('created_at'); $t->integer('updated_at'); }); $now = time(); $actor = ['user_id' => 1, 'is_admin' => true, 'is_super_admin' => true, 'can_manage' => true]; $newBackup = static function (int $sourceId, string $suffix) use ($now): CloudInstanceBackup { $row = new CloudInstanceBackup(); $row->backup_key = str_pad($suffix, 32, '0'); $row->source_instance_id = $sourceId; $row->source_host_id = 3; $row->user_id = 9; $row->source_uuid = '11111111-1111-4111-8111-111111111111'; $row->name = 'backup-' . $suffix; $row->purpose = 'portable'; $row->source = 'manual'; $row->subscription_id = null; $row->state = 'ready'; $row->consistency = 'crash'; $row->object_key = 'placeholder/' . $suffix; $row->object_size_bytes = 1024; $row->virtual_size_bytes = 2048; $row->sha256 = str_repeat('a', 64); $row->object_etag = ''; $row->object_version_id = ''; $row->spec_json = '{}'; $row->sealed_credential = 'guest-secret'; $row->storage_revision = 0; $row->storage_artifact_id = null; $row->active_job_id = null; $row->last_operation_id = null; $row->delete_operation_id = null; $row->delete_generation = 0; $row->delete_cycle_attempts = 0; $row->delete_presence_confirmations = 0; $row->delete_requested_at = null; $row->delete_last_checked_at = null; $row->delete_next_check_at = 0; $row->delete_attention_at = null; $row->error_code = ''; $row->ready_at = $now; $row->created_at = $row->updated_at = $now; $row->save(); return $row; }; $bindArtifact = static function ( CloudInstanceBackup $owner, string $seed, int $copies = 2 ) use ($now): CloudStorageArtifact { $artifact = new CloudStorageArtifact(); $artifact->artifact_key = str_repeat($seed, 32); $artifact->kind = 'backup'; $artifact->maximum_size_bytes = 1024; $artifact->size_bytes = 1024; $artifact->sha256 = str_repeat('a', 64); $artifact->durability_policy = 'mirror_async_2'; $artifact->desired_copies = $copies; $artifact->required_copies = 1; $artifact->state = 'protected'; $artifact->last_operation_id = null; $artifact->verified_at = $now; $artifact->created_at = $artifact->updated_at = $now; $artifact->save(); $publication = new CloudStoragePublication(); $publication->publication_key = str_repeat(chr(ord($seed) + 1), 32); $publication->artifact_id = (int) $artifact->id; $publication->backup_id = (int) $owner->id; $publication->template_import_intent_id = null; $publication->rescue_template_id = null; $publication->state = 'completed'; $publication->attempts = 0; $publication->available_at = 0; $publication->lease_token = null; $publication->lease_until = 0; $publication->last_error_code = ''; $publication->coverage_ready_at = $now; $publication->finalized_at = $now; $publication->created_at = $publication->updated_at = $now; $publication->save(); $roles = [1 => 'primary', 2 => 'secondary']; if ($copies === 1) { unset($roles[2]); } foreach ($roles as $backendId => $role) { $replica = new CloudStorageReplica(); $replica->replica_key = str_repeat($seed . $backendId, 16); $replica->artifact_id = (int) $artifact->id; $replica->backend_id = $backendId; $replica->role = $role; $replica->object_key = 'dual/' . $seed . '/' . $backendId; $replica->object_version_id = 'version-' . $backendId; $replica->object_etag = '"etag-' . $backendId . '"'; $replica->size_bytes = 1024; $replica->sha256 = str_repeat('a', 64); $replica->state = 'ready'; $replica->publish_operation_id = null; $replica->delete_operation_id = null; $replica->delete_generation = 0; $replica->delete_cycle_attempts = 0; $replica->delete_presence_confirmations = 0; $replica->delete_last_checked_at = 0; $replica->delete_next_check_at = 0; $replica->delete_attention_at = null; $replica->lease_token = null; $replica->lease_until = 0; $replica->error_code = ''; $replica->verified_at = $now; $replica->created_at = $replica->updated_at = $now; $replica->save(); } $owner->storage_artifact_id = (int) $artifact->id; $owner->save(); return $artifact; }; $client = static function (array $responses, array &$history): CloudObjectStorageS3Client { $stack = HandlerStack::create(new MockHandler($responses)); $stack->push(Middleware::history($history)); return new CloudObjectStorageS3Client(new Client(['handler' => $stack])); }; $runtime = ['config' => ['endpoint' => 'https://objects.example.test', 'region' => 'us-east-1', 'bucket' => 'bucket', 'prefix' => '', 'path_style' => true], 'credentials' => ['access_key_id' => 'KEY', 'secret_access_key' => 'secret'], 'revision' => 1, 'backend_id' => 1]; $legacy = $newBackup(10, 'legacy'); $legacyHistory = []; $expectConflict(static fn () => CloudBackupService::delete( $legacy, $actor, $client([], $legacyHistory) )); $assert($legacyHistory === [] && (string) $legacy->fresh()->state === 'ready', 'A backup without a replica ledger attempted provider deletion.'); $backup = $newBackup(11, 'artifact'); $artifact = $bindArtifact($backup, 'a'); $history = []; $artifactClient = $client([ new Response(200, ['Content-Length' => '1024', 'ETag' => '"etag-1"', 'x-amz-version-id' => 'version-1']), new Response(204), new Response(200, ['Content-Length' => '1024', 'ETag' => '"etag-2"', 'x-amz-version-id' => 'version-2']), new Response(204), ], $history); $deletion = new CloudStorageArtifactDeletionService( $artifactClient, static fn (int $backendId, bool $healthy): array => $runtime ); $result = CloudBackupService::delete($backup, $actor, $artifactClient, null, $deletion); $backup->refresh(); $artifact->refresh(); $assert(($result['deleted'] ?? false) === true && (string) $backup->state === 'deleted' && (string) $artifact->state === 'deleted' && $backup->sealed_credential === null && count($history) === 4 && CloudStorageReplica::where('artifact_id', $artifact->id) ->where('state', 'deleted')->count() === 2, 'Artifact-backed backup did not delete every exact replica and tombstone its owner.'); $blocked = $newBackup(12, 'blocked'); $blockedArtifact = $bindArtifact($blocked, 'c'); $restore = new CloudBackupRestore(); $restore->backup_id = (int) $blocked->id; $restore->state = 'restoring'; $restore->save(); $blockedHistory = []; $expectConflict(static fn () => CloudBackupService::delete( $blocked, $actor, $client([], $blockedHistory) )); $blocked->refresh(); $blockedArtifact->refresh(); $assert($blockedHistory === [] && (string) $blocked->state === 'ready' && (string) $blockedArtifact->state === 'protected', 'An active restore did not preserve its replicated source backup.'); $unknown = $newBackup(13, 'unknown'); $unknownArtifact = $bindArtifact($unknown, 'e', 1); $unknownHistory = []; $unknownClient = $client([ new Response(200, ['Content-Length' => '1024', 'ETag' => '"etag-1"', 'x-amz-version-id' => 'version-1']), new ConnectException( 'delete response lost', new Request('DELETE', 'https://objects.example.test') ), new Response(404), ], $unknownHistory); $clock = 1_000; $unknownDeletion = new CloudStorageArtifactDeletionService( $unknownClient, static fn (int $backendId, bool $healthy): array => $runtime, static function () use (&$clock): int { return $clock; } ); $unknownResult = CloudBackupService::delete( $unknown, $actor, $unknownClient, null, $unknownDeletion ); $unknownOperation = Capsule::table('cloud_storage_operations') ->where('artifact_id', (int) $unknownArtifact->id) ->where('kind', 'delete')->first(); $unknownReplica = CloudStorageReplica::where( 'artifact_id', (int) $unknownArtifact->id )->firstOrFail(); $assert(($unknownResult['requires_reconciliation'] ?? false) === true && $unknownOperation !== null && $unknownOperation->effect_resolution === 'unknown' && $unknownOperation->phase === 'reconciling' && $unknownOperation->outcome === null, 'Ambiguous replica deletion lost its original durable operation.'); // Model an administrator-attention projection after the same unknown // operation exhausted its bounded automatic observations. Reopening must // continue that operation with read-only evidence, not create another DELETE. $unknownArtifact->state = 'delete_attention'; $unknownArtifact->save(); $operationId = (string) $unknownOperation->operation_id; Capsule::table('cloud_storage_operations') ->where('operation_id', $operationId)->update([ 'phase' => 'terminal', 'outcome' => 'failed', 'effect_resolution' => 'unknown', 'effect_may_have_applied' => 1, 'safe_to_resubmit' => 0, 'next_check_at' => 0, 'terminal_at' => $clock, ]); $unknownReplica->state = 'delete_attention'; $unknownReplica->delete_attention_at = $clock; $unknownReplica->delete_presence_confirmations = 0; $unknownReplica->save(); $unknown->state = 'delete_attention'; $unknown->delete_attention_at = $clock; $unknown->save(); $operationCount = Capsule::table('cloud_storage_operations') ->where('artifact_id', (int) $unknownArtifact->id) ->where('kind', 'delete')->count(); $clock = 1_061; $recovered = CloudBackupService::retryCleanup( $unknown, $actor, $unknownClient, $unknownDeletion ); $unknown->refresh(); $unknownArtifact->refresh(); $unknownReplica->refresh(); $assert(($recovered['deleted'] ?? false) === true && (string) $unknown->state === 'deleted' && (string) $unknownArtifact->state === 'deleted' && (string) $unknownReplica->state === 'deleted' && Capsule::table('cloud_storage_operations') ->where('artifact_id', (int) $unknownArtifact->id) ->where('kind', 'delete')->count() === $operationCount && Capsule::table('cloud_storage_operations') ->where('operation_id', $operationId)->exists() && array_map(static fn (array $entry): string => $entry['request']->getMethod(), $unknownHistory) === [ 'HEAD', 'DELETE', 'HEAD', ], 'Manual recovery did not reconcile the original ambiguous deletion.'); echo "Portable backup replicated deletion and clean-break tests passed.\n";