httpStatus === $status, 'Unexpected CloudError status: ' . $error->httpStatus); return; } throw new RuntimeException('Expected CloudError was not raised.'); }; $db = new Capsule(); $db->addConnection(['driver' => 'sqlite', 'database' => ':memory:']); $db->setAsGlobal(); $db->bootEloquent(); $db->schema()->create('cloud_storage_backends', static function ( Blueprint $table ): void { $table->increments('id'); $table->string('backend_key'); $table->string('identity_sha256'); $table->string('name'); $table->string('driver'); $table->string('endpoint'); $table->string('region'); $table->string('bucket'); $table->string('prefix'); $table->boolean('path_style'); $table->string('failure_domain'); $table->text('sealed_credentials'); $table->unsignedBigInteger('config_revision'); $table->unsignedBigInteger('tested_revision'); $table->string('admin_state'); $table->string('health_state'); $table->unsignedInteger('write_priority'); $table->unsignedInteger('consecutive_failures')->default(0); $table->unsignedInteger('consecutive_successes')->default(0); $table->string('last_error_code')->default(''); $table->unsignedBigInteger('last_tested_at')->default(0); $table->unsignedBigInteger('last_healthy_at')->default(0); $table->unsignedBigInteger('next_probe_at')->default(0); $table->unsignedBigInteger('created_at'); $table->unsignedBigInteger('updated_at'); }); $db->schema()->create('cloud_storage_artifacts', static function ( Blueprint $table ): void { $table->increments('id'); $table->string('artifact_key')->unique(); $table->string('kind'); $table->unsignedBigInteger('maximum_size_bytes'); $table->unsignedBigInteger('size_bytes'); $table->string('sha256'); $table->string('durability_policy'); $table->unsignedTinyInteger('desired_copies'); $table->unsignedTinyInteger('required_copies'); $table->string('state'); $table->string('last_operation_id')->nullable(); $table->unsignedBigInteger('verified_at'); $table->unsignedBigInteger('created_at'); $table->unsignedBigInteger('updated_at'); }); $db->schema()->create('cloud_storage_replicas', static function ( Blueprint $table ): void { $table->increments('id'); $table->string('replica_key')->unique(); $table->unsignedBigInteger('artifact_id'); $table->unsignedBigInteger('backend_id'); $table->string('role'); $table->string('object_key'); $table->string('object_version_id')->default(''); $table->string('object_etag')->default(''); $table->unsignedBigInteger('size_bytes')->default(0); $table->string('sha256')->default(''); $table->string('state'); $table->string('publish_operation_id')->nullable(); $table->string('delete_operation_id')->nullable(); $table->unsignedInteger('delete_generation')->default(0); $table->unsignedInteger('delete_cycle_attempts')->default(0); $table->unsignedInteger('delete_presence_confirmations')->default(0); $table->unsignedBigInteger('delete_last_checked_at')->default(0); $table->unsignedBigInteger('delete_next_check_at')->default(0); $table->unsignedBigInteger('delete_attention_at')->nullable(); $table->string('lease_token')->nullable(); $table->unsignedBigInteger('lease_until')->default(0); $table->string('error_code')->default(''); $table->unsignedBigInteger('verified_at')->default(0); $table->unsignedBigInteger('created_at'); $table->unsignedBigInteger('updated_at'); }); $db->schema()->create('cloud_storage_operations', static function ( Blueprint $table ): void { $table->increments('id'); $table->string('operation_id')->unique(); $table->string('request_idempotency_key')->unique(); $table->string('request_fingerprint'); $table->unsignedBigInteger('artifact_id'); $table->unsignedBigInteger('replica_id')->nullable(); $table->unsignedBigInteger('source_backend_id')->nullable(); $table->unsignedBigInteger('target_backend_id')->nullable(); $table->string('kind'); $table->string('phase'); $table->string('outcome')->nullable(); $table->string('effect_resolution'); $table->boolean('effect_may_have_applied')->nullable(); $table->boolean('safe_to_resubmit')->nullable(); $table->unsignedInteger('attempts'); $table->string('lease_token')->nullable(); $table->unsignedBigInteger('lease_until'); $table->unsignedBigInteger('available_at'); $table->unsignedBigInteger('next_check_at'); $table->string('error_code'); $table->text('result_json')->nullable(); $table->unsignedBigInteger('terminal_at')->nullable(); $table->unsignedBigInteger('created_at'); $table->unsignedBigInteger('updated_at'); }); $db->schema()->create('cloud_storage_publications', static function ( Blueprint $table ): void { $table->increments('id'); $table->string('publication_key')->unique(); $table->unsignedBigInteger('artifact_id')->unique(); $table->unsignedBigInteger('backup_id')->nullable()->unique(); $table->string('template_import_intent_id')->nullable()->unique(); $table->string('rescue_template_id')->nullable()->unique(); $table->string('state'); $table->unsignedInteger('attempts')->default(0); $table->unsignedBigInteger('available_at')->default(0); $table->string('lease_token')->nullable(); $table->unsignedBigInteger('lease_until')->default(0); $table->string('last_error_code')->default(''); $table->unsignedBigInteger('coverage_ready_at')->nullable(); $table->unsignedBigInteger('finalized_at')->nullable(); $table->unsignedBigInteger('created_at'); $table->unsignedBigInteger('updated_at'); }); $databaseNow = time(); foreach ([ [1, 'primary', 'provider-a', 10], [2, 'secondary', 'provider-b', 20], ] as [$id, $name, $domain, $priority]) { Capsule::table('cloud_storage_backends')->insert([ 'id' => $id, 'backend_key' => str_repeat((string) $id, 32), 'identity_sha256' => str_repeat((string) $id, 64), 'name' => $name, 'driver' => 's3', 'endpoint' => 'https://' . $name . '.example.test', 'region' => 'us-east-1', 'bucket' => $name . '-bucket', 'prefix' => $name, 'path_style' => 1, 'failure_domain' => $domain, 'sealed_credentials' => 'sealed', 'config_revision' => 1, 'tested_revision' => 1, 'admin_state' => 'active', 'health_state' => 'healthy', 'write_priority' => $priority, 'consecutive_failures' => 0, 'consecutive_successes' => 1, 'last_error_code' => '', 'last_tested_at' => $databaseNow, 'last_healthy_at' => $databaseNow, 'next_probe_at' => $databaseNow + 300, 'created_at' => $databaseNow, 'updated_at' => $databaseNow, ]); } $temporaryPaths = []; register_shutdown_function(static function () use (&$temporaryPaths): void { foreach ($temporaryPaths as $path) { @unlink($path); } }); $fixtureSequence = 0; $fixture = static function ( string $kind = 'template' ) use (&$fixtureSequence, &$temporaryPaths): array { $fixtureSequence++; $path = tempnam(sys_get_temp_dir(), 'virt-primary-publication-'); if ($path === false) { throw new RuntimeException('Cannot create publication fixture.'); } file_put_contents( $path, str_repeat('immutable-primary-' . $fixtureSequence . '-', 64) ); $temporaryPaths[] = $path; $size = filesize($path); $sha256 = hash_file('sha256', $path); if (!is_int($size) || !is_string($sha256)) { throw new RuntimeException('Cannot identify publication fixture.'); } $ownerId = str_pad(dechex($fixtureSequence), 32, '0', STR_PAD_LEFT); $owner = $kind === 'rescue' ? ['rescue_template_id' => $ownerId] : ['template_import_intent_id' => $ownerId]; $view = CloudStorageArtifactService::beginPublication( $kind, $kind . '/fixture-' . $fixtureSequence . '/system.qcow2', $size, $sha256, 'immutable-primary-' . str_pad( (string) $fixtureSequence, 4, '0', STR_PAD_LEFT ), hash('sha256', $kind . '|' . $fixtureSequence), $owner ); return [ 'path' => $path, 'size_bytes' => $size, 'sha256' => $sha256, 'operation_id' => (string) $view['operation_id'], 'artifact_id' => (int) $view['artifact_id'], 'replica_id' => (int) $view['replica_id'], 'object_key' => (string) $view['object_key'], ]; }; $publisher = static function ( array &$responses, array &$requests, array &$transactionLevels, int &$clock ): CloudStorageImmutablePrimaryPublication { $handler = static function ( RequestInterface $request, array $options ) use (&$responses, &$requests, &$transactionLevels) { $transactionLevels[] = Capsule::connection()->transactionLevel(); $requests[] = $request; if ($responses === []) { throw new RuntimeException('Unexpected object-storage request.'); } $response = array_shift($responses); if (!$response instanceof Response) { throw new RuntimeException('Invalid object-storage response fixture.'); } return Create::promiseFor($response); }; $client = new CloudObjectStorageS3Client(new Client([ 'handler' => HandlerStack::create($handler), ])); return new CloudStorageImmutablePrimaryPublication( $client, static fn (int $backendId, bool $healthy): array => [ 'backend_id' => $backendId, 'revision' => 1, 'config' => [ 'endpoint' => 'https://objects.example.test', 'region' => 'us-east-1', 'bucket' => 'immutable-fixtures', 'prefix' => 'unused-runtime-prefix', 'path_style' => true, ], 'credentials' => [ 'access_key_id' => 'IMMUTABLEACCESSKEY', 'secret_access_key' => 'immutable-secret-must-not-persist', ], ], static function () use (&$clock): int { return $clock; } ); }; // A successful immutable PUT settles only the primary. Template publication // remains behind the required-two protection gate until its copy completes. $applied = $fixture('template'); $clock = time() + 10; $responses = [new Response(200, [ 'ETag' => '"applied-etag"', 'x-amz-version-id' => 'applied-version', ])]; $requests = []; $transactionLevels = []; $module = $publisher($responses, $requests, $transactionLevels, $clock); $appliedResult = $module->publish( $applied['operation_id'], $applied['artifact_id'], $applied ); $assert($appliedResult['primary_completed'] === true && $appliedResult['publication_state'] === 'waiting_protection' && $appliedResult['ready_to_finalize'] === false, 'Primary success bypassed the template publication quorum.'); $assert(count($requests) === 1 && $requests[0]->getMethod() === 'PUT' && $requests[0]->getHeaderLine('If-None-Match') === '*' && $requests[0]->getHeaderLine('x-amz-meta-virt-sha256') === $applied['sha256'] && str_contains( $requests[0]->getHeaderLine('Authorization'), 'x-amz-meta-virt-sha256' ), 'Immutable primary PUT lost its conditional signed identity.'); $assert($transactionLevels === [0], 'Immutable primary PUT ran inside a database transaction.'); // A provider rejection proves non-application and schedules a bounded retry; // it never recursively replays inside the same call. $rejected = $fixture('rescue'); $clock += 1000; $responses = [new Response(403)]; $requests = []; $transactionLevels = []; $module = $publisher($responses, $requests, $transactionLevels, $clock); $rejectedResult = $module->publish( $rejected['operation_id'], $rejected['artifact_id'], $rejected ); $rejectedOperation = CloudStorageOperation::where( 'operation_id', $rejected['operation_id'] )->firstOrFail(); $rejectedReplica = CloudStorageReplica::whereKey( $rejected['replica_id'] )->firstOrFail(); $assert($rejectedResult['phase'] === 'accepted' && (int) $rejectedOperation->attempts === 1 && $rejectedOperation->outcome === null && (int) $rejectedOperation->available_at > $clock && (string) $rejectedReplica->state === 'reserved' && count($requests) === 1, 'Proven non-application was not delayed without replay.'); // A lost PUT receipt is settled only by exact-key size plus signed SHA // metadata. The temporary local file is deliberately removed before HEAD. $unknownApplied = $fixture('template'); $clock += 1000; $responses = [ new Response(503), new Response(200, [ 'Content-Length' => (string) $unknownApplied['size_bytes'], 'ETag' => '"unknown-applied-etag"', 'x-amz-version-id' => 'unknown-applied-version', 'x-amz-meta-virt-sha256' => $unknownApplied['sha256'], ]), ]; $requests = []; $transactionLevels = []; $module = $publisher($responses, $requests, $transactionLevels, $clock); $unknownResult = $module->publish( $unknownApplied['operation_id'], $unknownApplied['artifact_id'], $unknownApplied ); $assert($unknownResult['phase'] === 'reconciling', 'Unknown PUT did not enter read-only reconciliation.'); @unlink($unknownApplied['path']); $clock += 60; $unknownResult = $module->publish( $unknownApplied['operation_id'], $unknownApplied['artifact_id'], $unknownApplied ); $assert($unknownResult['primary_completed'] === true && array_map( static fn (RequestInterface $request): string => $request->getMethod(), $requests ) === ['PUT', 'HEAD'] && (string) $requests[0]->getUri() === (string) $requests[1]->getUri(), 'Unknown PUT was replayed or reconciled against a different key.'); $assert($transactionLevels === [0, 0], 'Immutable PUT reconciliation ran remote I/O inside a transaction.'); // Two spaced exact-key absences are required before the original operation, // artifact and key may become dispatchable again. $absent = $fixture('template'); $clock += 1000; $responses = [ new Response(503), new Response(404), new Response(404), new Response(200, [ 'ETag' => '"retried-etag"', 'x-amz-version-id' => 'retried-version', ]), ]; $requests = []; $transactionLevels = []; $module = $publisher($responses, $requests, $transactionLevels, $clock); $artifactCount = CloudStorageArtifact::count(); $operationCount = CloudStorageOperation::count(); $module->publish($absent['operation_id'], $absent['artifact_id'], $absent); $clock += 60; $firstAbsent = $module->publish( $absent['operation_id'], $absent['artifact_id'], $absent ); $assert($firstAbsent['phase'] === 'reconciling', 'One absence observation incorrectly authorized a retry.'); $clock += 60; $secondAbsent = $module->publish( $absent['operation_id'], $absent['artifact_id'], $absent ); $assert($secondAbsent['phase'] === 'accepted', 'Two stable absence observations did not release the original intent.'); $clock += 60; $retried = $module->publish( $absent['operation_id'], $absent['artifact_id'], $absent ); $methods = array_map( static fn (RequestInterface $request): string => $request->getMethod(), $requests ); $assert($retried['primary_completed'] === true && $methods === ['PUT', 'HEAD', 'HEAD', 'PUT'] && (string) $requests[0]->getUri() === (string) $requests[1]->getUri() && (string) $requests[0]->getUri() === (string) $requests[2]->getUri() && (string) $requests[0]->getUri() === (string) $requests[3]->getUri() && (int) CloudStorageOperation::where( 'operation_id', $absent['operation_id'] )->value('attempts') === 2 && CloudStorageArtifact::count() === $artifactCount && CloudStorageOperation::count() === $operationCount, 'Stable absence did not retry the same frozen publication intent.'); // Existing content with a different signed identity is an attention state, // never evidence for success and never permission to overwrite the key. $conflict = $fixture('rescue'); $clock += 1000; $responses = [ new Response(503), new Response(200, [ 'Content-Length' => (string) $conflict['size_bytes'], 'ETag' => '"conflicting-etag"', 'x-amz-version-id' => 'conflicting-version', 'x-amz-meta-virt-sha256' => str_repeat('f', 64), ]), ]; $requests = []; $transactionLevels = []; $module = $publisher($responses, $requests, $transactionLevels, $clock); $module->publish($conflict['operation_id'], $conflict['artifact_id'], $conflict); $clock += 60; $conflictResult = $module->publish( $conflict['operation_id'], $conflict['artifact_id'], $conflict ); $conflictOperation = CloudStorageOperation::where( 'operation_id', $conflict['operation_id'] )->firstOrFail(); $assert($conflictResult['requires_attention'] === true && (string) $conflictOperation->phase === 'terminal' && (string) $conflictOperation->outcome === 'failed' && (string) $conflictOperation->effect_resolution === 'unknown' && (bool) $conflictOperation->safe_to_resubmit === false && (string) CloudStorageArtifact::whereKey( $conflict['artifact_id'] )->value('state') === 'degraded' && (string) CloudStoragePublication::where( 'artifact_id', $conflict['artifact_id'] )->value('state') === 'failed' && count($requests) === 2, 'Conflicting immutable identity did not stop in operator attention.'); // An administrator may reopen only the same unknown-effect operation for // read-only HEAD reconciliation. Secondary work stopped by the primary fence // is restored but remains blocked until that exact primary succeeds. $artifactCountBeforeReopen = CloudStorageArtifact::count(); $operationCountBeforeReopen = CloudStorageOperation::count(); $clock += 1; $attention = CloudStorageReplicaAttentionService::listView(); $attentionItem = collect($attention['items'])->firstWhere( 'operation_id', $conflict['operation_id'] ); $assert(is_array($attentionItem) && ($attentionItem['operation_type'] ?? null) === '不可变主副本发布' && ($attentionItem['owner_type'] ?? null) === 'rescue' && ($attentionItem['can_retry'] ?? false) === true, 'Immutable primary attention was not exposed to administrators.'); $attentionResult = CloudStorageReplicaAttentionService::retry( $conflict['operation_id'], 'publish' ); $assert(($attentionResult['phase'] ?? null) === 'reconciling', 'Immutable primary attention did not reopen through the admin service.'); $reopenedConflict = CloudStorageOperation::where( 'operation_id', $conflict['operation_id'] )->firstOrFail(); $reopenedCopy = CloudStorageOperation::where( 'artifact_id', $conflict['artifact_id'] )->where('kind', 'copy')->firstOrFail(); $reopenedPrimary = CloudStorageReplica::whereKey( $conflict['replica_id'] )->firstOrFail(); $reopenedSecondary = CloudStorageReplica::whereKey( (int) $reopenedCopy->replica_id )->firstOrFail(); $assert((string) $reopenedConflict->phase === 'reconciling' && (string) $reopenedConflict->effect_resolution === 'unknown' && $reopenedConflict->outcome === null && (string) $reopenedPrimary->state === 'publishing' && (string) $reopenedCopy->phase === 'accepted' && (int) $reopenedCopy->available_at === 253402300799 && (string) $reopenedSecondary->state === 'reserved' && (string) CloudStorageArtifact::whereKey( $conflict['artifact_id'] )->value('state') === 'publishing' && (string) CloudStoragePublication::where( 'artifact_id', $conflict['artifact_id'] )->value('state') === 'waiting_primary' && CloudStorageArtifact::count() === $artifactCountBeforeReopen && CloudStorageOperation::count() === $operationCountBeforeReopen, 'Administrator reconciliation did not reopen the original fenced ledger.'); $responses = [new Response(404), new Response(404)]; $requests = []; $transactionLevels = []; $module = $publisher($responses, $requests, $transactionLevels, $clock); $module->publish( $conflict['operation_id'], $conflict['artifact_id'], $conflict ); $clock += 60; $reconciledAbsence = $module->publish( $conflict['operation_id'], $conflict['artifact_id'], $conflict ); $assert($reconciledAbsence['phase'] === 'accepted' && array_map( static fn (RequestInterface $request): string => $request->getMethod(), $requests ) === ['HEAD', 'HEAD'], 'Administrator recovery replayed a write before stable absence evidence.'); // Repeated proven non-application also stops for operator review. Reopening // authorizes one new dispatch of the same operation and immutable key only. $manualRetry = $fixture('template'); $clock += 1000; $responses = [new Response(403), new Response(403), new Response(403)]; $requests = []; $transactionLevels = []; $module = $publisher($responses, $requests, $transactionLevels, $clock); for ($attempt = 0; $attempt < 3; $attempt++) { $module->publish( $manualRetry['operation_id'], $manualRetry['artifact_id'], $manualRetry ); $clock += 300; } $manualRetryOperation = CloudStorageOperation::where( 'operation_id', $manualRetry['operation_id'] )->firstOrFail(); $assert((string) $manualRetryOperation->phase === 'terminal' && (string) $manualRetryOperation->effect_resolution === 'not_applied' && (string) $manualRetryOperation->error_code === 'immutable_primary_publish_retry_exhausted', 'Bounded proven failures did not enter manual retry attention.'); $manualAttention = CloudStorageReplicaAttentionService::listView(); $manualAttentionItem = collect($manualAttention['items'])->firstWhere( 'operation_id', $manualRetry['operation_id'] ); $assert(is_array($manualAttentionItem) && ($manualAttentionItem['can_retry'] ?? false) === true, 'Proven-not-applied primary attention was not exposed for retry.'); CloudStorageReplicaAttentionService::retry( $manualRetry['operation_id'], 'publish' ); $responses = [new Response(200, [ 'ETag' => '"manual-retry-etag"', 'x-amz-version-id' => 'manual-retry-version', ])]; $requests = []; $transactionLevels = []; $module = $publisher($responses, $requests, $transactionLevels, $clock); $manualRetryResult = $module->publish( $manualRetry['operation_id'], $manualRetry['artifact_id'], $manualRetry ); $assert($manualRetryResult['primary_completed'] === true && count($requests) === 1 && $requests[0]->getMethod() === 'PUT' && (int) CloudStorageOperation::where( 'operation_id', $manualRetry['operation_id'] )->value('attempts') === 4, 'Manual retry did not reuse the exact proven-not-applied operation.'); // The expected artifact fence runs before source validation or provider I/O. $fencedA = $fixture('template'); $fencedB = $fixture('rescue'); $clock += 1000; $responses = [new Response(200)]; $requests = []; $transactionLevels = []; $module = $publisher($responses, $requests, $transactionLevels, $clock); $beforeA = CloudStorageOperation::where( 'operation_id', $fencedA['operation_id'] )->firstOrFail()->toArray(); $expectCloudError(static fn () => $module->publish( $fencedA['operation_id'], $fencedB['artifact_id'], $fencedA ), 409); $afterA = CloudStorageOperation::where( 'operation_id', $fencedA['operation_id'] )->firstOrFail()->toArray(); $assert($requests === [] && $beforeA === $afterA, 'Wrong expected artifact crossed the durable or remote mutation fence.'); // A settled publication is idempotent even after its local source is gone. @unlink($applied['path']); $requestsBefore = count($requests); $terminalAgain = new CloudStorageImmutablePrimaryPublication( new CloudObjectStorageS3Client(new Client([ 'handler' => HandlerStack::create(static function () { throw new RuntimeException('Terminal publication used HTTP.'); }), ])), static function (): array { throw new RuntimeException('Terminal publication resolved runtime.'); }, static fn (): int => time() ); $idempotent = $terminalAgain->publish( $applied['operation_id'], $applied['artifact_id'], $applied ); $assert($idempotent['primary_completed'] === true && (int) CloudStorageOperation::where( 'operation_id', $applied['operation_id'] )->value('attempts') === 1 && count($requests) === $requestsBefore, 'Terminal publication was not idempotent without its local file.'); foreach (CloudStorageOperation::query()->get() as $operation) { $stored = (string) $operation->result_json; $assert(!str_contains($stored, 'https://') && !str_contains($stored, 'IMMUTABLEACCESSKEY') && !str_contains($stored, 'immutable-secret-must-not-persist'), 'Publication ledger persisted a credential or capability URL.'); } echo "Immutable primary publication behavior passed.\n";