addConnection([ 'driver' => 'sqlite', 'database' => ':memory:', ], 'default'); $capsule->setAsGlobal(); $capsule->bootEloquent(); $schema = $capsule->schema('default'); $schema->create('cloud_instance_backups', static function (Blueprint $table): void { $table->increments('id'); $table->string('purpose'); $table->string('state'); $table->string('object_key'); $table->text('sealed_storage_runtime'); $table->text('sealed_credential')->nullable(); $table->integer('storage_revision'); $table->integer('active_job_id')->nullable(); $table->string('error_code')->default(''); $table->integer('updated_at'); }); $schema->create('cloud_instance_migrations', static function (Blueprint $table): void { $table->increments('id'); $table->integer('backup_id'); $table->string('state'); $table->string('error_code')->default(''); $table->integer('updated_at'); }); $schema->create('cloud_migration_bundle_artifacts', static function (Blueprint $table): void { $table->increments('id'); $table->integer('migration_id'); $table->string('object_key'); $table->string('state'); $table->integer('updated_at'); }); Config::$dataKey = base64_encode(str_repeat('m', 32)); $runtime = [ 'config' => [ 'endpoint' => 'https://objects.example.com', 'region' => 'us-east-1', 'bucket' => 'backup-bucket', 'prefix' => 'virt', 'path_style' => true, ], 'credentials' => [ 'access_key_id' => 'MIGRATIONACCESS', 'secret_access_key' => 'migration-secret-value', ], 'revision' => 9, ]; $sealedRuntime = CloudSecrets::encrypt( $runtime, 'object-storage-backup-runtime' ); $client = static function (array $responses, array &$history): CloudObjectStorageS3Client { $stack = HandlerStack::create(new MockHandler($responses)); $stack->push(Middleware::history($history)); return new CloudObjectStorageS3Client(new Client(['handler' => $stack])); }; $newMigration = static function (int $id, string $state, int $artifactCount = 2) use ($sealedRuntime, $capsule): void { $now = time() - 600; $capsule->table('cloud_instance_backups')->insert([ 'id' => 100 + $id, 'purpose' => 'migration_bundle', 'state' => 'ready', 'object_key' => 'virt/vm-migrations/bundle-' . $id . '/system.qcow2', 'sealed_storage_runtime' => $sealedRuntime, 'sealed_credential' => 'sealed-guest-credential', 'storage_revision' => 9, 'active_job_id' => null, 'error_code' => '', 'updated_at' => $now, ]); $capsule->table('cloud_instance_migrations')->insert([ 'id' => $id, 'backup_id' => 100 + $id, 'state' => $state, 'error_code' => '', 'updated_at' => $now, ]); for ($index = 1; $index <= $artifactCount; $index++) { $capsule->table('cloud_migration_bundle_artifacts')->insert([ 'id' => $id * 10 + $index, 'migration_id' => $id, 'object_key' => 'virt/vm-migrations/bundle-' . $id . '/snapshots/snapshot-' . $index . '.qcow2', 'state' => 'restored', 'updated_at' => $now, ]); } }; // A completed migration must delete every snapshot transport before the // root object, then remove its frozen object-store credentials. $newMigration(1, 'completed'); $history = []; $storage = $client([ new Response(204), new Response(204), new Response(204), ], $history); for ($step = 0; $step < 3; $step++) { $assert(CloudMigrationBundleService::cleanupOneEligible( $storage, time() + $step ), 'Successful migration bundle cleanup did not advance.'); } $backup = CloudInstanceBackup::findOrFail(101); $assert(CloudMigrationBundleArtifact::where('migration_id', 1) ->where('state', 'deleted')->count() === 2 && (string) $backup->state === 'deleted' && (string) $backup->sealed_storage_runtime === '' && $backup->sealed_credential === null, 'Successful migration retained object-store artifacts or credentials.'); $assert(array_map( static fn (array $entry): string => $entry['request']->getMethod() . ' ' . $entry['request']->getUri()->getPath(), $history ) === [ 'DELETE /backup-bucket/virt/vm-migrations/bundle-1/snapshots/snapshot-1.qcow2', 'DELETE /backup-bucket/virt/vm-migrations/bundle-1/snapshots/snapshot-2.qcow2', 'DELETE /backup-bucket/virt/vm-migrations/bundle-1/system.qcow2', ], 'Migration transport objects were not deleted snapshot-first/root-last.'); // A deliberately retained source is still a successful target migration; // its object-store transport must not become a permanent hidden backup. $newMigration(2, 'source_retained', 0); $retainedHistory = []; $assert(CloudMigrationBundleService::cleanupOneEligible( $client([new Response(204)], $retainedHistory), time() ), 'Source-retained migration did not clean its transport object.'); $assert((string) CloudInstanceBackup::findOrFail(102)->state === 'deleted' && (string) CloudInstanceMigration::findOrFail(2)->state === 'source_retained', 'Transport cleanup changed the retained-source migration contract.'); // Once DeleteObject has an unknown effect, every automatic follow-up must // be read-only HEAD reconciliation; it must never replay DELETE. $newMigration(3, 'completed', 1); $unknownHistory = []; $expectCleanupFailure(static function () use ( $client, &$unknownHistory ): void { CloudMigrationBundleService::cleanupOneEligible($client([ new ConnectException('transport lost', new Request( 'DELETE', 'https://objects.example.com' )), ], $unknownHistory), time()); }, 'Unknown DeleteObject effect was not reported.'); $artifact = CloudMigrationBundleArtifact::findOrFail(31); $assert((string) $artifact->state === 'delete_uncertain' && count($unknownHistory) === 1 && $unknownHistory[0]['request']->getMethod() === 'DELETE', 'Unknown object deletion did not retain its effect fence.'); CloudInstanceBackup::where('id', 103)->update([ 'sealed_storage_runtime' => 'invalid-ciphertext', ]); $invalidFenceHistory = []; $expectCleanupFailure(static function () use ( $client, &$invalidFenceHistory ): void { CloudMigrationBundleService::cleanupOneEligible( $client([], $invalidFenceHistory), time() + 301 ); }, 'Invalid runtime did not surface during ambiguous reconciliation.'); $assert($invalidFenceHistory === [] && (string) CloudMigrationBundleArtifact::findOrFail(31)->state === 'delete_uncertain', 'Invalid runtime downgraded an unknown-effect deletion fence.'); CloudInstanceBackup::where('id', 103)->update([ 'sealed_storage_runtime' => $sealedRuntime, ]); $presentHistory = []; $expectCleanupFailure(static function () use ( $client, &$presentHistory ): void { CloudMigrationBundleService::cleanupOneEligible( $client([new Response(200)], $presentHistory), time() + 602 ); }, 'Still-present ambiguous object was not reported.'); $assert((string) CloudMigrationBundleArtifact::findOrFail(31)->state === 'delete_uncertain' && count($presentHistory) === 1 && $presentHistory[0]['request']->getMethod() === 'HEAD', 'A present ambiguous object caused DeleteObject to be replayed.'); $absentHistory = []; $assert(CloudMigrationBundleService::cleanupOneEligible( $client([new Response(404)], $absentHistory), time() + 903 ), 'Absent ambiguous object did not converge to deleted.'); $assert((string) CloudMigrationBundleArtifact::findOrFail(31)->state === 'deleted' && $absentHistory[0]['request']->getMethod() === 'HEAD', 'HEAD 404 did not safely finish ambiguous object cleanup.'); // Finish migration 3's root before proving that a later failed migration // cannot become eligible by accident. $thirdRootHistory = []; $assert(CloudMigrationBundleService::cleanupOneEligible( $client([new Response(204)], $thirdRootHistory), time() + 904 ) && (string) CloudInstanceBackup::findOrFail(103)->state === 'deleted', 'A reconciled bundle did not finish by deleting its root object.'); // A failed migration is not a successful migration and must retain its // bundle for rollback/recovery investigation. $newMigration(4, 'rolled_back', 0); $failedHistory = []; $assert(!CloudMigrationBundleService::cleanupOneEligible( $client([], $failedHistory), time() + 1000 ) && $failedHistory === [] && (string) CloudInstanceBackup::findOrFail(104)->state === 'ready', 'A non-successful migration lost its recovery bundle.'); // Successful migration alone is insufficient: the target must have // restored every snapshot and the root backup must remain ready. $newMigration(5, 'completed', 1); CloudMigrationBundleArtifact::where('migration_id', 5)->update([ 'state' => 'ready', ]); $newMigration(6, 'completed', 0); CloudInstanceBackup::where('id', 106)->update(['state' => 'failed']); $newMigration(7, 'cleanup_uncertain', 0); $failClosedHistory = []; $assert(!CloudMigrationBundleService::cleanupOneEligible( $client([], $failClosedHistory), time() + 1100 ) && $failClosedHistory === [], 'An incomplete or uncertain migration bundle became deletable.'); // A fresh deleting claim belongs to another worker. Once stale, it is an // unknown effect and must be reconciled with HEAD instead of replaying it. $newMigration(8, 'completed', 1); $leaseNow = time() + 2000; CloudMigrationBundleArtifact::where('migration_id', 8)->update([ 'state' => 'deleting', 'updated_at' => $leaseNow, ]); $freshHistory = []; $assert(!CloudMigrationBundleService::cleanupOneEligible( $client([], $freshHistory), $leaseNow + 59 ) && $freshHistory === [], 'A fresh object-delete lease was stolen by another worker.'); CloudInstanceBackup::where('id', 108)->update([ 'sealed_storage_runtime' => 'invalid-ciphertext', ]); $staleInvalidHistory = []; $expectCleanupFailure(static function () use ( $client, &$staleInvalidHistory, $leaseNow ): void { CloudMigrationBundleService::cleanupOneEligible( $client([], $staleInvalidHistory), $leaseNow + 60 ); }, 'Invalid runtime did not surface for a stale delete claim.'); $assert($staleInvalidHistory === [] && (string) CloudMigrationBundleArtifact::where( 'migration_id', 8 )->value('state') === 'delete_uncertain', 'A stale unknown-effect claim was downgraded to a replayable state.'); CloudInstanceBackup::where('id', 108)->update([ 'sealed_storage_runtime' => $sealedRuntime, ]); $staleHistory = []; $assert(CloudMigrationBundleService::cleanupOneEligible( $client([new Response(404)], $staleHistory), $leaseNow + 361 ) && $staleHistory[0]['request']->getMethod() === 'HEAD', 'A stale object-delete claim was replayed instead of reconciled.'); $staleRootHistory = []; $assert(CloudMigrationBundleService::cleanupOneEligible( $client([new Response(204)], $staleRootHistory), $leaseNow + 362 ) && (string) CloudInstanceBackup::findOrFail(108)->state === 'deleted', 'The reconciled stale bundle did not delete its root last.'); // A definitive provider rejection may be retried, but only after the // backoff window; this must not form a hot loop in the worker. $newMigration(9, 'completed', 1); $rejectedHistory = []; $expectCleanupFailure(static function () use ( $client, &$rejectedHistory, $leaseNow ): void { CloudMigrationBundleService::cleanupOneEligible( $client([new Response(403)], $rejectedHistory), $leaseNow + 400 ); }, 'A definitive DeleteObject rejection was not reported.'); $assert((string) CloudMigrationBundleArtifact::where( 'migration_id', 9 )->value('state') === 'delete_rejected', 'A definitive DeleteObject rejection was not persisted.'); $backoffHistory = []; $assert(!CloudMigrationBundleService::cleanupOneEligible( $client([], $backoffHistory), $leaseNow + 699 ) && $backoffHistory === [], 'A rejected DeleteObject entered a retry hot loop.'); $retryHistory = []; $assert(CloudMigrationBundleService::cleanupOneEligible( $client([new Response(204)], $retryHistory), $leaseNow + 700 ) && $retryHistory[0]['request']->getMethod() === 'DELETE', 'A definitively rejected delete was not retried after backoff.'); $retryRootHistory = []; $assert(CloudMigrationBundleService::cleanupOneEligible( $client([new Response(204)], $retryRootHistory), $leaseNow + 701 ), 'The retry test bundle root was not deleted.'); // Root deletion has the same unknown-effect fence: runtime damage after a // lost DELETE response cannot turn it back into a replayable operation. $newMigration(10, 'completed', 0); $rootUnknownHistory = []; $expectCleanupFailure(static function () use ( $client, &$rootUnknownHistory, $leaseNow ): void { CloudMigrationBundleService::cleanupOneEligible($client([ new ConnectException('transport lost', new Request( 'DELETE', 'https://objects.example.com' )), ], $rootUnknownHistory), $leaseNow + 800); }, 'Unknown root DeleteObject effect was not reported.'); $assert((string) CloudInstanceBackup::findOrFail(110)->state === 'delete_uncertain' && $rootUnknownHistory[0]['request']->getMethod() === 'DELETE', 'Unknown root deletion did not retain its effect fence.'); CloudInstanceBackup::where('id', 110)->update([ 'sealed_storage_runtime' => 'invalid-ciphertext', ]); $rootInvalidHistory = []; $expectCleanupFailure(static function () use ( $client, &$rootInvalidHistory, $leaseNow ): void { CloudMigrationBundleService::cleanupOneEligible( $client([], $rootInvalidHistory), $leaseNow + 1101 ); }, 'Invalid runtime did not surface for ambiguous root cleanup.'); $assert($rootInvalidHistory === [] && (string) CloudInstanceBackup::findOrFail(110)->state === 'delete_uncertain', 'Invalid runtime downgraded an ambiguous root deletion fence.'); CloudInstanceBackup::where('id', 110)->update([ 'sealed_storage_runtime' => $sealedRuntime, ]); $rootAbsentHistory = []; $assert(CloudMigrationBundleService::cleanupOneEligible( $client([new Response(404)], $rootAbsentHistory), $leaseNow + 1402 ) && (string) CloudInstanceBackup::findOrFail(110)->state === 'deleted' && $rootAbsentHistory[0]['request']->getMethod() === 'HEAD', 'Ambiguous root deletion did not converge with HEAD only.'); // A corrupt frozen runtime is isolated with backoff and cannot starve a // later valid bundle forever. $newMigration(11, 'completed', 0); CloudInstanceBackup::where('id', 111)->update([ 'sealed_storage_runtime' => 'invalid-ciphertext', ]); $newMigration(12, 'completed', 0); $invalidHistory = []; $expectCleanupFailure(static function () use ( $client, &$invalidHistory, $leaseNow ): void { CloudMigrationBundleService::cleanupOneEligible( $client([], $invalidHistory), $leaseNow + 1500 ); }, 'An invalid frozen runtime was not reported.'); $assert($invalidHistory === [] && (string) CloudInstanceBackup::findOrFail(111)->state === 'delete_rejected', 'An invalid frozen runtime was not isolated behind backoff.'); $laterHistory = []; $assert(CloudMigrationBundleService::cleanupOneEligible( $client([new Response(204)], $laterHistory), $leaseNow + 1501 ) && (string) CloudInstanceBackup::findOrFail(112)->state === 'deleted', 'One invalid runtime starved a later valid migration bundle.'); echo "Maintenance migration bundle cleanup and reconciliation passed.\n"; }