addConnection([ 'driver' => 'sqlite', 'database' => ':memory:', ], 'default'); $capsule->setAsGlobal(); $capsule->bootEloquent(); $schema = $capsule->schema('default'); $schema->create('cloud_storage_backends', static function ( Blueprint $table ): void { $table->increments('id'); $table->string('backend_key'); $table->string('identity_sha256'); $table->string('name'); $table->string('driver'); $table->string('endpoint'); $table->string('region'); $table->string('bucket'); $table->string('prefix'); $table->boolean('path_style'); $table->string('failure_domain'); $table->text('sealed_credentials'); $table->integer('config_revision'); $table->integer('tested_revision'); $table->string('admin_state'); $table->string('health_state'); $table->integer('write_priority'); $table->integer('consecutive_failures')->default(0); $table->integer('consecutive_successes')->default(0); $table->string('last_error_code')->default(''); $table->integer('last_tested_at')->default(0); $table->integer('last_healthy_at')->default(0); $table->integer('next_probe_at')->default(0); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_storage_artifacts', static function ( Blueprint $table ): void { $table->increments('id'); $table->string('artifact_key')->unique(); $table->string('kind'); $table->integer('maximum_size_bytes'); $table->integer('size_bytes'); $table->string('sha256'); $table->string('durability_policy'); $table->integer('desired_copies'); $table->integer('required_copies'); $table->string('state'); $table->string('last_operation_id')->nullable(); $table->integer('verified_at'); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_storage_replicas', static function ( Blueprint $table ): void { $table->increments('id'); $table->string('replica_key')->unique(); $table->integer('artifact_id'); $table->integer('backend_id'); $table->string('role'); $table->string('object_key'); $table->string('object_version_id')->default(''); $table->string('object_etag')->default(''); $table->integer('size_bytes')->default(0); $table->string('sha256')->default(''); $table->string('state'); $table->string('publish_operation_id')->nullable(); $table->string('delete_operation_id')->nullable(); $table->integer('delete_generation')->default(0); $table->integer('delete_cycle_attempts')->default(0); $table->integer('delete_presence_confirmations')->default(0); $table->integer('delete_last_checked_at')->default(0); $table->integer('delete_next_check_at')->default(0); $table->integer('delete_attention_at')->nullable(); $table->string('lease_token')->nullable(); $table->integer('lease_until')->default(0); $table->string('error_code')->default(''); $table->integer('verified_at')->default(0); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_storage_operations', static function ( Blueprint $table ): void { $table->increments('id'); $table->string('operation_id')->unique(); $table->string('request_idempotency_key')->unique(); $table->string('request_fingerprint'); $table->integer('artifact_id'); $table->integer('replica_id')->nullable(); $table->integer('source_backend_id')->nullable(); $table->integer('target_backend_id')->nullable(); $table->string('kind'); $table->string('phase'); $table->string('outcome')->nullable(); $table->string('effect_resolution'); $table->boolean('effect_may_have_applied')->nullable(); $table->boolean('safe_to_resubmit')->nullable(); $table->integer('attempts'); $table->string('lease_token')->nullable(); $table->integer('lease_until'); $table->integer('available_at'); $table->integer('next_check_at'); $table->string('error_code'); $table->text('result_json')->nullable(); $table->integer('terminal_at')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_storage_publications', static function ( Blueprint $table ): void { $table->increments('id'); $table->string('publication_key')->unique(); $table->integer('artifact_id')->unique(); $table->integer('backup_id')->nullable()->unique(); $table->string('template_import_intent_id')->nullable()->unique(); $table->string('rescue_template_id')->nullable()->unique(); $table->string('state'); $table->integer('attempts')->default(0); $table->integer('available_at')->default(0); $table->string('lease_token')->nullable(); $table->integer('lease_until')->default(0); $table->string('last_error_code')->default(''); $table->integer('coverage_ready_at')->nullable(); $table->integer('finalized_at')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); $capsule->schema('default')->create('cloud_rescue_templates', static function (Blueprint $table): void { $table->string('id')->primary(); $table->string('name'); $table->integer('size_bytes'); $table->string('sha256'); $table->integer('revision'); $table->string('generation_token')->unique(); $table->string('state'); $table->integer('active_job_id')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); $capsule->schema('default')->create( 'cloud_rescue_template_artifacts', static function (Blueprint $table): void { $table->string('template_id')->primary(); $table->integer('storage_artifact_id')->nullable()->unique(); $table->integer('storage_revision'); $table->string('object_key')->unique(); $table->string('object_version_id')->default(''); $table->string('object_etag')->default(''); $table->integer('size_bytes'); $table->string('sha256'); $table->string('state'); $table->string('error_code')->default(''); $table->string('delete_operation_id')->default(''); $table->integer('delete_generation')->default(0); $table->integer('delete_cycle_attempts')->default(0); $table->integer('delete_presence_confirmations')->default(0); $table->integer('delete_last_checked_at')->default(0); $table->integer('delete_next_check_at')->default(0); $table->integer('delete_attention_at')->nullable(); $table->integer('verified_at')->default(0); $table->integer('created_at'); $table->integer('updated_at'); }); $body = str_repeat('bootable-rescue-iso-', 65536); $digest = hash('sha256', $body); $fetchHistory = []; $fetchStack = HandlerStack::create(new MockHandler([ new Response(200, [ 'Content-Length' => (string) strlen($body), 'Content-Encoding' => 'identity', ], $body), ])); $fetchStack->push(Middleware::history($fetchHistory)); $fetcher = new CloudRemoteArtifactFetcher( '救援 ISO', 1, 8 * 1024 * 1024 * 1024, 'virt-rescue-test-', new Client(['handler' => $fetchStack]), static fn (string $host): array => ['93.184.216.34'] ); $storageHistory = []; $storageStack = HandlerStack::create(new MockHandler([ new Response(200, [ 'ETag' => '"rescue-etag"', 'x-amz-version-id' => 'rescue-version-1', ]), ])); $storageStack->push(Middleware::history($storageHistory)); $client = new CloudObjectStorageS3Client(new Client([ 'handler' => $storageStack, ])); $runtime = [ 'config' => [ 'endpoint' => 'https://objects.example.com', 'region' => 'ewr', 'bucket' => 'rescue-bucket', 'prefix' => 'virt-cloud', 'path_style' => true, ], 'credentials' => [ 'access_key_id' => 'RESCUEACCESSKEY', 'secret_access_key' => 'rescue-secret', ], 'revision' => 9, ]; $backendNow = time(); foreach ([ [1, 'primary', 'objects.example.com', 'rescue-bucket', 'virt-cloud', 'provider-a', 10], [2, 'secondary', 'objects-b.example.com', 'rescue-bucket-b', 'virt-cloud', 'provider-b', 20], ] as [$id, $backendName, $host, $bucket, $prefix, $domain, $priority]) { $endpoint = 'https://' . $host; Capsule::table('cloud_storage_backends')->insert([ 'id' => $id, 'backend_key' => str_repeat((string) $id, 32), 'identity_sha256' => hash('sha256', $endpoint . '|' . $bucket), 'name' => $backendName, 'driver' => 's3', 'endpoint' => $endpoint, 'region' => 'ewr', 'bucket' => $bucket, 'prefix' => $prefix, 'path_style' => 1, 'failure_domain' => $domain, 'sealed_credentials' => CloudSecrets::encrypt( $runtime['credentials'], 'object-storage-backend-credentials' ), 'config_revision' => 9, 'tested_revision' => 9, 'admin_state' => 'active', 'health_state' => 'healthy', 'write_priority' => $priority, 'consecutive_failures' => 0, 'consecutive_successes' => 1, 'last_error_code' => '', 'last_tested_at' => $backendNow, 'last_healthy_at' => $backendNow, 'next_probe_at' => $backendNow + 300, 'created_at' => $backendNow, 'updated_at' => $backendNow, ]); } $result = CloudRescueCatalogService::importRemote([ 'name' => 'SystemRescue', 'url' => 'https://images.example.com/systemrescue.iso', 'sha256' => $digest, ], [ 'is_super_admin' => true, 'can_manage' => true, 'user_id' => 1, ], $fetcher, $client, [32, 31]); $template = CloudRescueTemplate::find($result['template_id']); $artifact = CloudRescueTemplateArtifact::find($result['template_id']); $request = $storageHistory[0]['request']; $assert($template !== null && $artifact !== null && (string) $template->state === 'preparing' && (string) $artifact->state === 'publishing' && (int) $artifact->storage_artifact_id > 0 && (int) $artifact->storage_revision === 0 && (string) $artifact->object_version_id === '' && (string) $template->sha256 === $digest && (int) $template->size_bytes === strlen($body) && $result['validation_host_id'] === 31, 'Rescue import did not freeze the quarantine identity and host order.'); $assert($request->getMethod() === 'PUT' && str_contains( (string) $request->getUri(), '/rescue-bucket/virt-cloud/rescue-isos/' . (string) $template->id . '/' . (string) $template->generation_token . '/rescue.iso' ) && $request->getHeaderLine('If-None-Match') === '*' && $request->getHeaderLine('X-Amz-Content-Sha256') === $digest, 'Rescue master was not published to an immutable exact object key.'); $assert(count($fetchHistory) === 1 && count($storageHistory) === 1, 'Rescue import unexpectedly replayed a remote side effect.'); $storageArtifact = CloudStorageArtifact::findOrFail( (int) $artifact->storage_artifact_id ); $publication = CloudStoragePublication::query() ->where('artifact_id', (int) $storageArtifact->id)->firstOrFail(); $assert((string) $publication->state === 'waiting_protection' && (string) $template->state === 'preparing', 'A single rescue storage replica prematurely released validation.'); $replicas = CloudStorageReplica::query() ->where('artifact_id', (int) $storageArtifact->id) ->orderBy('id')->get(); $secondary = $replicas->firstWhere('role', 'secondary'); $assert($secondary instanceof CloudStorageReplica, 'Rescue publication did not reserve its secondary replica.'); $secondary->state = 'ready'; $secondary->size_bytes = strlen($body); $secondary->sha256 = $digest; $secondary->object_etag = '"rescue-secondary"'; $secondary->object_version_id = 'rescue-secondary-version'; $secondary->verified_at = time(); $secondary->save(); $project = (new ReflectionClass(CloudStorageArtifactService::class)) ->getMethod('projectArtifactLocked'); $project->invoke( null, $storageArtifact, CloudStorageReplica::query() ->where('artifact_id', (int) $storageArtifact->id) ->orderBy('id')->get()->all(), time() ); $assert(CloudRescueStoragePublicationService::finalizeReadyOnly( (string) $template->id ), 'Protected rescue publication was not consumed.'); $template->refresh(); $artifact->refresh(); $publication->refresh(); $assert((string) $template->state === 'validating' && (string) $artifact->state === 'validating' && (string) $publication->state === 'completed', 'Storage and rescue business finalization did not commit together.'); Capsule::table('cloud_storage_backends')->where('id', 1)->update([ 'health_state' => 'unreachable', ]); $secondarySource = CloudRescueTemplateArtifactManager::downloadSource( $template, $client ); $assert(str_contains( (string) $secondarySource['source_url'], 'objects-b.example.com' ), 'Rescue read routing did not fail over to the exact secondary replica.'); Capsule::table('cloud_storage_backends')->where('id', 1)->update([ 'health_state' => 'healthy', ]); $unknownFetcher = new CloudRemoteArtifactFetcher( '救援 ISO', 1, 8 * 1024 * 1024 * 1024, 'virt-rescue-test-', new Client(['handler' => HandlerStack::create(new MockHandler([ new Response(200, [ 'Content-Length' => (string) strlen($body), ], $body), ]))]), static fn (string $host): array => ['93.184.216.34'] ); $unknownClient = new CloudObjectStorageS3Client(new Client([ 'handler' => HandlerStack::create(new MockHandler([ new ConnectException( 'simulated lost rescue PUT response', new Request('PUT', 'https://objects.example.com') ), new ConnectException( 'simulated unavailable rescue HEAD', new Request('HEAD', 'https://objects.example.com') ), ])), ])); CloudRescueCatalogService::importRemote([ 'name' => 'Unknown Rescue', 'url' => 'https://images.example.com/unknown.iso', 'sha256' => $digest, ], [ 'is_super_admin' => true, 'can_manage' => true, 'user_id' => 1, ], $unknownFetcher, $unknownClient, [31]); $unknownTemplate = CloudRescueTemplate::query() ->where('name', 'Unknown Rescue')->first(); $unknownArtifact = $unknownTemplate === null ? null : CloudRescueTemplateArtifact::find((string) $unknownTemplate->id); $assert($unknownTemplate !== null && $unknownArtifact !== null && (string) $unknownTemplate->state === 'preparing' && (string) $unknownArtifact->state === 'publishing' && (int) $unknownArtifact->storage_artifact_id > 0, 'Unknown rescue PUT effect was not durably fenced for reconciliation.'); $unknownTemplate->state = 'retiring'; $unknownTemplate->save(); $unknownArtifact->state = 'delete_pending'; $unknownArtifact->delete_next_check_at = time(); $unknownArtifact->save(); CloudStorageOperation::query() ->where('artifact_id', (int) $unknownArtifact->storage_artifact_id) ->where('kind', 'publish')->update(['next_check_at' => 0]); $assert(!CloudRescueStoragePublicationService::prepareRetirement( (string) $unknownTemplate->id, $unknownClient ) && (string) CloudStorageArtifact::findOrFail( (int) $unknownArtifact->storage_artifact_id )->state === 'publishing', 'Retirement fenced an unknown primary before its original HEAD path.'); $gapId = str_repeat('a', 32); $gapGeneration = str_repeat('b', 32); (new CloudRescueTemplate())->getConnection()->transaction( static function () use ( $gapId, $gapGeneration, $body, $digest ): void { $now = time(); $gap = new CloudRescueTemplate(); $gap->id = $gapId; $gap->name = 'Reserved source lost'; $gap->size_bytes = strlen($body); $gap->sha256 = $digest; $gap->revision = 1; $gap->generation_token = $gapGeneration; $gap->state = 'preparing'; $gap->active_job_id = null; $gap->created_at = $gap->updated_at = $now; $gap->save(); CloudRescueStoragePublicationService::reserveLocked($gap); } ); $requestCount = count($storageHistory); $assert(!CloudRescueStoragePublicationService::resume($gapId, $client), 'A reservation without its temporary source entered validation.'); $gapArtifact = CloudRescueTemplateArtifact::findOrFail($gapId); $gapPublication = CloudStoragePublication::query() ->where('artifact_id', (int) $gapArtifact->storage_artifact_id) ->firstOrFail(); $gapOperation = CloudStorageOperation::query() ->where('artifact_id', (int) $gapArtifact->storage_artifact_id) ->where('kind', 'publish')->firstOrFail(); $assert(count($storageHistory) === $requestCount && (string) CloudRescueTemplate::findOrFail($gapId)->state === 'source_failed' && (string) $gapArtifact->state === 'publish_failed' && (string) $gapPublication->state === 'failed' && (string) $gapOperation->phase === 'terminal' && (string) $gapOperation->outcome === 'failed' && (string) $gapOperation->effect_resolution === 'not_applied', 'The reserve-to-PUT crash gap was not closed without provider replay.'); CloudRescueTemplate::query()->where('id', $gapId)->update([ 'state' => 'preparing', ]); CloudRescueTemplateArtifact::query()->where('template_id', $gapId)->update([ 'state' => 'publishing', 'error_code' => '', ]); $assert(!CloudRescueStoragePublicationService::resume($gapId, $client) && count($storageHistory) === $requestCount && (string) CloudRescueTemplate::findOrFail($gapId)->state === 'source_failed' && (string) CloudRescueTemplateArtifact::findOrFail($gapId)->state === 'publish_failed', 'A committed storage failure was not projected after a crash gap.'); $legacyId = str_repeat('f', 32); $legacyGeneration = str_repeat('e', 32); $now = time(); Capsule::table('cloud_rescue_templates')->insert([ 'id' => $legacyId, 'name' => 'Legacy rescue', 'size_bytes' => strlen($body), 'sha256' => $digest, 'revision' => 1, 'generation_token' => $legacyGeneration, 'state' => 'validating', 'active_job_id' => null, 'created_at' => $now, 'updated_at' => $now, ]); Capsule::table('cloud_rescue_template_artifacts')->insert([ 'template_id' => $legacyId, 'storage_artifact_id' => null, 'storage_revision' => 1, 'object_key' => 'legacy/rescue.iso', 'object_version_id' => 'legacy-version', 'object_etag' => 'legacy-etag', 'size_bytes' => strlen($body), 'sha256' => $digest, 'state' => 'validating', 'created_at' => $now, 'updated_at' => $now, ]); $legacyRejected = false; try { CloudRescueTemplateArtifactManager::downloadSource( CloudRescueTemplate::findOrFail($legacyId), $client ); } catch (CloudError $error) { $legacyRejected = $error->httpStatus === 409; } $assert($legacyRejected, 'A legacy rescue artifact without shared storage identity was readable.'); echo "Global rescue ISO quarantine import passed.\n"; }