httpStatus === $status, 'Unexpected CloudError status: ' . $error->httpStatus); return; } throw new RuntimeException('Expected CloudError was not raised.'); }; $db = new Capsule(); $db->addConnection(['driver' => 'sqlite', 'database' => ':memory:']); $db->setAsGlobal(); $db->bootEloquent(); $db->schema()->create('cloud_storage_backends', static function (Blueprint $t): void { $t->increments('id'); $t->string('backend_key'); $t->string('identity_sha256'); $t->string('name'); $t->string('driver'); $t->string('endpoint'); $t->string('region'); $t->string('bucket'); $t->string('prefix'); $t->boolean('path_style'); $t->string('failure_domain'); $t->text('sealed_credentials'); $t->unsignedBigInteger('config_revision'); $t->unsignedBigInteger('tested_revision'); $t->string('admin_state'); $t->string('health_state'); $t->unsignedInteger('write_priority'); $t->unsignedInteger('consecutive_failures')->default(0); $t->unsignedInteger('consecutive_successes')->default(0); $t->string('last_error_code')->default(''); $t->unsignedBigInteger('last_tested_at')->default(0); $t->unsignedBigInteger('last_healthy_at')->default(0); $t->unsignedBigInteger('next_probe_at')->default(0); $t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at'); }); $db->schema()->create('cloud_storage_artifacts', static function (Blueprint $t): void { $t->increments('id'); $t->string('artifact_key')->unique(); $t->string('kind'); $t->unsignedBigInteger('maximum_size_bytes'); $t->unsignedBigInteger('size_bytes'); $t->string('sha256'); $t->string('durability_policy'); $t->unsignedTinyInteger('desired_copies'); $t->unsignedTinyInteger('required_copies'); $t->string('state'); $t->string('last_operation_id')->nullable(); $t->unsignedBigInteger('verified_at'); $t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at'); }); $db->schema()->create('cloud_storage_replicas', static function (Blueprint $t): void { $t->increments('id'); $t->string('replica_key')->unique(); $t->unsignedBigInteger('artifact_id'); $t->unsignedBigInteger('backend_id'); $t->string('role'); $t->string('object_key'); $t->string('object_version_id')->default(''); $t->string('object_etag')->default(''); $t->unsignedBigInteger('size_bytes')->default(0); $t->string('sha256')->default(''); $t->string('state'); $t->string('publish_operation_id')->nullable(); $t->string('delete_operation_id')->nullable(); $t->unsignedInteger('delete_generation')->default(0); $t->unsignedInteger('delete_cycle_attempts')->default(0); $t->unsignedInteger('delete_presence_confirmations')->default(0); $t->unsignedBigInteger('delete_last_checked_at')->default(0); $t->unsignedBigInteger('delete_next_check_at')->default(0); $t->unsignedBigInteger('delete_attention_at')->nullable(); $t->string('lease_token')->nullable(); $t->unsignedBigInteger('lease_until')->default(0); $t->string('error_code')->default(''); $t->unsignedBigInteger('verified_at')->default(0); $t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at'); }); $db->schema()->create('cloud_storage_operations', static function (Blueprint $t): void { $t->increments('id'); $t->string('operation_id')->unique(); $t->string('request_idempotency_key')->unique(); $t->string('request_fingerprint'); $t->unsignedBigInteger('artifact_id'); $t->unsignedBigInteger('replica_id')->nullable(); $t->unsignedBigInteger('source_backend_id')->nullable(); $t->unsignedBigInteger('target_backend_id')->nullable(); $t->string('kind'); $t->string('phase'); $t->string('outcome')->nullable(); $t->string('effect_resolution'); $t->boolean('effect_may_have_applied')->nullable(); $t->boolean('safe_to_resubmit')->nullable(); $t->unsignedInteger('attempts'); $t->string('lease_token')->nullable(); $t->unsignedBigInteger('lease_until'); $t->unsignedBigInteger('available_at'); $t->unsignedBigInteger('next_check_at'); $t->string('error_code'); $t->text('result_json')->nullable(); $t->unsignedBigInteger('terminal_at')->nullable(); $t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at'); }); $db->schema()->create('cloud_storage_publications', static function (Blueprint $t): void { $t->increments('id'); $t->string('publication_key')->unique(); $t->unsignedBigInteger('artifact_id')->unique(); $t->unsignedBigInteger('backup_id')->nullable()->unique(); $t->string('template_import_intent_id')->nullable()->unique(); $t->string('rescue_template_id')->nullable()->unique(); $t->string('state'); $t->unsignedInteger('attempts')->default(0); $t->unsignedBigInteger('available_at')->default(0); $t->string('lease_token')->nullable(); $t->unsignedBigInteger('lease_until')->default(0); $t->string('last_error_code')->default(''); $t->unsignedBigInteger('coverage_ready_at')->nullable(); $t->unsignedBigInteger('finalized_at')->nullable(); $t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at'); }); $now = time(); foreach ([ [1, 'provider-a', 10, 'primary'], [2, 'provider-b', 20, 'secondary'], ] as [$id, $domain, $priority, $name]) { Capsule::table('cloud_storage_backends')->insert([ 'id' => $id, 'backend_key' => str_repeat((string) $id, 32), 'identity_sha256' => str_repeat((string) $id, 64), 'name' => $name, 'driver' => 's3', 'endpoint' => 'https://' . $name . '.example.test', 'region' => 'us-east-1', 'bucket' => $name . '-bucket', 'prefix' => $name, 'path_style' => 1, 'failure_domain' => $domain, 'sealed_credentials' => 'sealed', 'config_revision' => 1, 'tested_revision' => 1, 'admin_state' => 'active', 'health_state' => 'healthy', 'write_priority' => $priority, 'consecutive_failures' => 0, 'consecutive_successes' => 1, 'last_error_code' => '', 'last_tested_at' => $now, 'last_healthy_at' => $now, 'next_probe_at' => $now + 300, 'created_at' => $now, 'updated_at' => $now, ]); } $requestKey = 'backup-publication-0001'; $fingerprint = hash('sha256', 'fixture'); $view = CloudStorageArtifactService::beginPublication( 'backup', 'backups/fixture/system.qcow2', 4096, str_repeat('a', 64), $requestKey, $fingerprint, ['backup_id' => 1] ); $assert($view['can_publish'] && $view['backend_id'] === 1 && CloudStorageArtifact::count() === 1 && CloudStorageReplica::count() === 2 && CloudStorageOperation::where('kind', 'copy')->count() === 1, 'Publication did not reserve one primary and one secondary operation.'); $again = CloudStorageArtifactService::beginPublication( 'backup', 'backups/fixture/system.qcow2', 4096, str_repeat('a', 64), $requestKey, $fingerprint, ['backup_id' => 1] ); $assert($again['operation_id'] === $view['operation_id'] && CloudStorageArtifact::count() === 1, 'Idempotent publication allocated a second artifact.'); $expect(static fn () => CloudStorageArtifactService::beginPublication( 'backup', 'backups/other/system.qcow2', 4096, str_repeat('a', 64), $requestKey, hash('sha256', 'different'), ['backup_id' => 1] ), 409); $assert(CloudStorageArtifactService::claimReplicaWork(time()) === null, 'Secondary copy became claimable before the primary was verified.'); CloudStorageArtifactService::markPrimaryDispatching($view['operation_id']); $expect(static fn () => CloudStorageArtifactService::markPrimaryDispatching( $view['operation_id'] ), 409); $complete = CloudStorageArtifactService::completePrimary( $view['operation_id'], (int) $view['artifact_id'], [ 'size_bytes' => 4096, 'sha256' => str_repeat('a', 64), 'etag' => '"etag"', 'version_id' => '', ] ); $artifact = CloudStorageArtifact::firstOrFail(); $primary = CloudStorageReplica::where('role', 'primary')->firstOrFail(); $copy = CloudStorageOperation::where('kind', 'copy')->firstOrFail(); $assert($complete['outcome'] === 'succeeded' && (string) $artifact->state === 'available' && (string) $primary->state === 'ready' && (string) $copy->phase === 'accepted', 'Primary settlement did not expose an async backup or retain copy work.'); CloudStorageArtifactService::completePrimary( $view['operation_id'], (int) $view['artifact_id'], [ 'size_bytes' => 4096, 'sha256' => str_repeat('a', 64), 'etag' => '"etag"', 'version_id' => '', ]); $expect(static fn () => CloudStorageArtifactService::completePrimary( $view['operation_id'], (int) $view['artifact_id'], [ 'size_bytes' => 4096, 'sha256' => str_repeat('a', 64), 'etag' => '"different"', 'version_id' => '', ] ), 409); $publication = CloudStoragePublication::firstOrFail(); $assert((string) $publication->state === 'ready_to_finalize' && (int) $publication->coverage_ready_at > 0, 'Required primary coverage did not release the durable finalizer signal.'); $publicationBeforeReject = $publication->getAttributes(); $expect(static fn () => CloudStoragePublicationFinalizerService::complete( 'backup', 999, (int) $artifact->id, $view['operation_id'] ), 409); $expect(static fn () => CloudStoragePublicationFinalizerService::complete( 'backup', 1, 999, $view['operation_id'] ), 409); $expect(static fn () => CloudStoragePublicationFinalizerService::complete( 'backup', 1, (int) $artifact->id, (string) $copy->operation_id ), 409); $publication->refresh(); $assert($publication->getAttributes() === $publicationBeforeReject, 'Rejected publication finalization changed the durable outbox.'); $finalized = CloudStoragePublicationFinalizerService::complete( 'backup', 1, (int) $artifact->id, $view['operation_id'] ); $publication->refresh(); $finalizedAt = (int) $publication->finalized_at; $assert($finalized['state'] === 'completed' && (string) $publication->state === 'completed' && (int) $publication->attempts === 1 && $finalizedAt > 0, 'Owner finalization did not complete the durable publication outbox.'); $againFinalized = CloudStoragePublicationFinalizerService::complete( 'backup', 1, (int) $artifact->id, $view['operation_id'] ); $publication->refresh(); $assert($againFinalized['state'] === 'completed' && (int) $publication->attempts === 1 && (int) $publication->finalized_at === $finalizedAt, 'Repeated owner finalization was not idempotent.'); $claimed = CloudStorageArtifactService::claimReplicaWork(time()); $assert(is_array($claimed) && $claimed['mode'] === 'dispatch' && $claimed['source_version_id'] === '', 'Versionless secondary copy was not leased from the durable operation.'); $assert(CloudStorageArtifactService::claimReplicaWork(time() + 21600) === null, 'A second worker reclaimed a copy before the helper deadline elapsed.'); $metadata = [ 'upload_id' => 'upload-1', 'part_size_bytes' => 67108864, 'part_count' => 1, 'maximum_size_bytes' => 4096, 'absence_confirmations' => 0, ]; CloudStorageArtifactService::markReplicaAwaitingReceipt( $claimed['operation_id'], $claimed['lease_token'], $metadata, time() ); $stored = (string) CloudStorageOperation::where( 'operation_id', $claimed['operation_id'] )->value('result_json'); $assert(!str_contains($stored, 'http') && str_contains($stored, 'upload-1'), 'Durable operation persisted a capability URL or lost its upload identity.'); CloudStorageArtifactService::completeReplica( $claimed['operation_id'], $claimed['lease_token'], [ 'size_bytes' => 4096, 'sha256' => str_repeat('a', 64), 'etag' => '"secondary"', 'version_id' => '', ], time() ); CloudStorageArtifactService::completeReplica( $claimed['operation_id'], $claimed['lease_token'], [ 'size_bytes' => 4096, 'sha256' => str_repeat('a', 64), 'etag' => '"secondary"', 'version_id' => '', ], time() ); $expect(static fn () => CloudStorageArtifactService::completeReplica( $claimed['operation_id'], $claimed['lease_token'], [ 'size_bytes' => 4096, 'sha256' => str_repeat('a', 64), 'etag' => '"secondary"', 'version_id' => 'unexpected-version', ], time() ), 409); $assert((string) CloudStorageArtifact::firstOrFail()->state === 'protected' && CloudStorageReplica::where('state', 'ready')->count() === 2, 'Versionless secondary receipt did not protect the artifact.'); $agentView = CloudStorageArtifactService::beginAgentPublication( 'backup', 'backups/agent/system.qcow2', 8192, 'backup-publication-0002', hash('sha256', 'agent'), ['backup_id' => 2] ); CloudStorageArtifactService::markPrimaryDispatching($agentView['operation_id']); CloudStorageArtifactService::completePrimary( $agentView['operation_id'], (int) $agentView['artifact_id'], [ 'size_bytes' => 6144, 'sha256' => str_repeat('c', 64), 'etag' => '"agent"', 'version_id' => '', ]); $agentArtifact = CloudStorageArtifact::whereKey( $agentView['artifact_id'] )->firstOrFail(); $assert((int) $agentArtifact->maximum_size_bytes === 8192 && (int) $agentArtifact->size_bytes === 6144 && (string) $agentArtifact->sha256 === str_repeat('c', 64), 'Agent publication did not freeze its bounded receipt identity.'); $capturedRequest = null; $mock = new MockHandler([ new Response(200, [], '' . 'worker-upload' . ''), new Response(200, [ 'Content-Length' => '6144', 'ETag' => '"worker"', ]), ]); $s3 = new CloudObjectStorageS3Client(new Client([ 'handler' => HandlerStack::create($mock), ])); $helper = new CloudStorageReplicaHelper(null, static function (array $request) use (&$capturedRequest): array { $capturedRequest = $request; return [ 'protocol' => $request['protocol'], 'operation_id' => $request['operation_id'], 'outcome' => 'succeeded', 'effect_resolution' => 'applied', 'size_bytes' => 6144, 'sha256' => str_repeat('c', 64), 'etag' => '"worker"', 'version_id' => '', ]; } ); $runtime = static fn (int $backendId, bool $healthy): array => [ 'config' => [ 'endpoint' => 'https://storage-' . $backendId . '.example.test', 'region' => 'us-east-1', 'bucket' => 'bucket-' . $backendId, 'path_style' => true, 'prefix' => '', ], 'credentials' => [ 'access_key_id' => 'access-key', 'secret_access_key' => 'secret-key', ], 'revision' => 1, 'backend_id' => $backendId, ]; $worker = new CloudStorageReplicaWorkerService($helper, $s3, $runtime); $assert($worker->tick() && is_array($capturedRequest) && $capturedRequest['protocol'] === 'object-replica-v1' && !str_contains($capturedRequest['source']['get_url'], 'versionId='), 'Replica worker did not stream a versionless exact-key source.'); $agentCopy = CloudStorageOperation::where('artifact_id', $agentArtifact->id) ->where('kind', 'copy')->firstOrFail(); $assert((string) $agentCopy->phase === 'terminal' && (string) $agentCopy->outcome === 'succeeded' && !str_contains((string) $agentCopy->result_json, 'https://'), 'Replica worker did not settle safely or persisted a capability URL.'); $unknownView = CloudStorageArtifactService::beginPublication( 'template', 'cloud-templates/unknown/system.qcow2', 4096, str_repeat('e', 64), 'template-publication-0003', hash('sha256', 'unknown'), [ 'template_import_intent_id' => str_repeat('e', 32), ] ); CloudStorageArtifactService::markPrimaryDispatching( $unknownView['operation_id'] ); CloudStorageArtifactService::completePrimary( $unknownView['operation_id'], (int) $unknownView['artifact_id'], [ 'size_bytes' => 4096, 'sha256' => str_repeat('e', 64), 'etag' => '"unknown-primary"', 'version_id' => '', ]); $copyCalls = 0; $unknownHelper = new CloudStorageReplicaHelper(null, static function (array $request) use (&$copyCalls): array { $copyCalls++; return [ 'protocol' => $request['protocol'], // A mismatched receipt must become unknown, never success. 'operation_id' => str_repeat('f', 32), 'outcome' => 'succeeded', 'effect_resolution' => 'applied', 'size_bytes' => 4096, 'sha256' => str_repeat('e', 64), 'etag' => '"wrong-operation"', 'version_id' => '', ]; } ); $unknownHTTP = new MockHandler([ new Response(200, [], '' . 'unknown-upload' . ''), new Response(404), new Response(200, [], '' . 'false' . ''), new Response(404), new Response(200, [], '' . 'false' . ''), ]); $unknownWorker = new CloudStorageReplicaWorkerService( $unknownHelper, new CloudObjectStorageS3Client(new Client([ 'handler' => HandlerStack::create($unknownHTTP), ])), $runtime ); $unknownWorker->tick(); $unknownCopy = CloudStorageOperation::where( 'artifact_id', $unknownView['artifact_id'] )->where('kind', 'copy')->firstOrFail(); $assert((string) $unknownCopy->phase === 'reconciling' && (string) $unknownCopy->effect_resolution === 'unknown', 'Mismatched helper receipt did not preserve unknown effect.'); $unknownCopy->next_check_at = 0; $unknownCopy->save(); $unknownWorker->tick(); $unknownCopy->refresh(); $unknownMetadata = json_decode((string) $unknownCopy->result_json, true); $assert($copyCalls === 1 && (string) $unknownCopy->phase === 'reconciling' && ($unknownMetadata['multipart_absence_confirmations'] ?? null) === 1, 'Unknown copy was replayed instead of read-only reconciliation.'); $unknownCopy->next_check_at = 0; $unknownCopy->save(); $unknownWorker->tick(); $unknownCopy->refresh(); $unknownPublication = CloudStoragePublication::where( 'artifact_id', $unknownView['artifact_id'] )->firstOrFail(); $assert($copyCalls === 1 && (string) $unknownCopy->phase === 'accepted' && (int) $unknownCopy->attempts === 1 && $unknownCopy->result_json === null && in_array( 'replica_retry_scheduled', $unknownWorker->observationErrors(), true ) && (string) $unknownPublication->state === 'waiting_protection', 'Required-two copy was not safely rescheduled after read-only absence proof: ' . json_encode([ 'calls' => $copyCalls, 'phase' => $unknownCopy->phase, 'attempts' => $unknownCopy->attempts, 'result' => $unknownCopy->result_json, 'publication' => $unknownPublication->state, 'error' => $unknownCopy->error_code, ])); $unknownCopy->available_at = 253402300799; $unknownCopy->save(); $retryView = CloudStorageArtifactService::beginPublication( 'backup', 'backups/retry/system.qcow2', 4096, str_repeat('f', 64), 'backup-publication-0005', hash('sha256', 'retry'), ['backup_id' => 5] ); CloudStorageArtifactService::markPrimaryDispatching( $retryView['operation_id'] ); CloudStorageArtifactService::completePrimary( $retryView['operation_id'], (int) $retryView['artifact_id'], [ 'size_bytes' => 4096, 'sha256' => str_repeat('f', 64), 'etag' => '"retry-primary"', 'version_id' => '', ]); $retryClaim = CloudStorageArtifactService::claimReplicaWork(time()); $assert(is_array($retryClaim) && (int) $retryClaim['artifact_id'] === (int) $retryView['artifact_id'], 'Recoverable copy fixture was not claimable.'); CloudStorageArtifactService::rescheduleReplicaNotApplied( $retryClaim['operation_id'], $retryClaim['lease_token'], 'replica_source_download_failed', time(), 60 ); $retryOperation = CloudStorageOperation::where( 'operation_id', $retryClaim['operation_id'] )->firstOrFail(); $assert((string) $retryOperation->phase === 'accepted' && $retryOperation->outcome === null && $retryOperation->terminal_at === null && $retryOperation->result_json === null, 'Definite no-effect copy failure exhausted the only copy intent.'); $retryOperation->available_at = 253402300799; $retryOperation->save(); $adoptView = CloudStorageArtifactService::beginPublication( 'backup', 'backups/adopt/system.qcow2', 4096, str_repeat('1', 64), 'backup-publication-0006', hash('sha256', 'adopt'), ['backup_id' => 6] ); CloudStorageArtifactService::markPrimaryDispatching( $adoptView['operation_id'] ); CloudStorageArtifactService::completePrimary( $adoptView['operation_id'], (int) $adoptView['artifact_id'], [ 'size_bytes' => 4096, 'sha256' => str_repeat('1', 64), 'etag' => '"adopt-primary"', 'version_id' => '', ]); $adoptCopy = CloudStorageOperation::where( 'artifact_id', $adoptView['artifact_id'] )->where('kind', 'copy')->firstOrFail(); $adoptTarget = CloudStorageReplica::whereKey( (int) $adoptCopy->replica_id )->firstOrFail(); $adoptCalls = 0; $adoptHelper = new CloudStorageReplicaHelper(null, static function (array $request) use (&$adoptCalls): array { $adoptCalls++; return [ 'protocol' => $request['protocol'], 'operation_id' => $request['operation_id'], 'outcome' => 'succeeded', 'effect_resolution' => 'applied', 'size_bytes' => 4096, 'sha256' => str_repeat('1', 64), 'etag' => '"adopted"', 'version_id' => '', ]; } ); $adoptHTTP = new MockHandler([ // The provider may create the session even though the response is lost. new Response(500), new Response(404), new Response(200, [], '' . 'false' . '' . htmlspecialchars( (string) $adoptTarget->object_key, ENT_XML1 | ENT_QUOTES, 'UTF-8' ) . 'adopt-upload' . gmdate('Y-m-d\\TH:i:s\\Z', time()) . ''), new Response(200, [ 'Content-Length' => '4096', 'ETag' => '"adopted"', ]), ]); $adoptWorker = new CloudStorageReplicaWorkerService( $adoptHelper, new CloudObjectStorageS3Client(new Client([ 'handler' => HandlerStack::create($adoptHTTP), ])), $runtime ); $adoptWorker->tick(); $adoptCopy->refresh(); $assert((string) $adoptCopy->phase === 'reconciling' && $adoptCopy->result_json === '[]' && $adoptCalls === 0, 'Ambiguous multipart creation was replayed before read-only lookup.'); $adoptCopy->next_check_at = 0; $adoptCopy->save(); $adoptWorker->tick(); $adoptCopy->refresh(); $assert($adoptCalls === 1 && (string) $adoptCopy->phase === 'terminal' && (string) $adoptCopy->outcome === 'succeeded', 'One exact multipart session was not adopted exactly once.'); // Crash after the upload_id is durable but before helper startup. The same // open session is safe to resume only after two stable read-only observations; // no new multipart session or Abort is allowed. $resumeView = CloudStorageArtifactService::beginPublication( 'backup', 'backups/resume/system.qcow2', 4096, str_repeat('2', 64), 'backup-publication-0007', hash('sha256', 'resume'), ['backup_id' => 7] ); CloudStorageArtifactService::markPrimaryDispatching( $resumeView['operation_id'] ); CloudStorageArtifactService::completePrimary( $resumeView['operation_id'], (int) $resumeView['artifact_id'], [ 'size_bytes' => 4096, 'sha256' => str_repeat('2', 64), 'etag' => '"resume-primary"', 'version_id' => '', ]); $resumeWork = CloudStorageArtifactService::claimReplicaWork(time()); $assert(is_array($resumeWork), 'Resume crash fixture was not claimable.'); CloudStorageArtifactService::markReplicaAwaitingReceipt( $resumeWork['operation_id'], $resumeWork['lease_token'], [ 'upload_id' => 'resume-upload', 'part_size_bytes' => 67108864, 'part_count' => 1, 'maximum_size_bytes' => 4096, 'multipart_absence_confirmations' => 0, 'multipart_presence_confirmations' => 0, ], time() ); $resumeOperation = CloudStorageOperation::where( 'operation_id', $resumeWork['operation_id'] )->firstOrFail(); $resumeOperation->lease_until = 0; $resumeOperation->save(); $resumeTarget = CloudStorageReplica::whereKey( (int) $resumeOperation->replica_id )->firstOrFail(); $resumeTarget->lease_until = 0; $resumeTarget->save(); $resumeXML = 'false' . '' . htmlspecialchars( (string) $resumeTarget->object_key, ENT_XML1 | ENT_QUOTES, 'UTF-8' ) . 'resume-upload' . gmdate('Y-m-d\\TH:i:s\\Z', time()) . ''; $resumeHTTP = new MockHandler([ new Response(404), new Response(200, [], $resumeXML), new Response(404), new Response(200, [], $resumeXML), new Response(200, [ 'Content-Length' => '4096', 'ETag' => '"resumed"', ]), ]); $resumeCalls = 0; $resumeHelper = new CloudStorageReplicaHelper(null, static function (array $request) use (&$resumeCalls): array { $resumeCalls++; return [ 'protocol' => $request['protocol'], 'operation_id' => $request['operation_id'], 'outcome' => 'succeeded', 'effect_resolution' => 'applied', 'size_bytes' => 4096, 'sha256' => str_repeat('2', 64), 'etag' => '"resumed"', 'version_id' => '', ]; } ); $resumeWorker = new CloudStorageReplicaWorkerService( $resumeHelper, new CloudObjectStorageS3Client(new Client([ 'handler' => HandlerStack::create($resumeHTTP), ])), $runtime ); $resumeWorker->tick(); $resumeOperation->refresh(); $resumeEvidence = json_decode((string) $resumeOperation->result_json, true); $assert($resumeCalls === 0 && ($resumeEvidence['multipart_presence_confirmations'] ?? 0) === 1, 'A persisted multipart session resumed without two stable observations.'); $resumeOperation->next_check_at = 0; $resumeOperation->save(); $resumeWorker->tick(); $resumeOperation->refresh(); $assert($resumeCalls === 1 && (string) $resumeOperation->phase === 'terminal' && (string) $resumeOperation->outcome === 'succeeded', 'A stable persisted multipart session did not resume exactly once.'); // A successful helper receipt is not authoritative until target HEAD agrees // on both ETag and version identity. $mixedView = CloudStorageArtifactService::beginPublication( 'backup', 'backups/mixed-version/system.qcow2', 4096, str_repeat('3', 64), 'backup-publication-0008', hash('sha256', 'mixed-version'), ['backup_id' => 8] ); CloudStorageArtifactService::markPrimaryDispatching( $mixedView['operation_id'] ); CloudStorageArtifactService::completePrimary( $mixedView['operation_id'], (int) $mixedView['artifact_id'], [ 'size_bytes' => 4096, 'sha256' => str_repeat('3', 64), 'etag' => '"mixed-primary"', 'version_id' => '', ]); $mixedHelper = new CloudStorageReplicaHelper(null, static fn (array $request): array => [ 'protocol' => $request['protocol'], 'operation_id' => $request['operation_id'], 'outcome' => 'succeeded', 'effect_resolution' => 'applied', 'size_bytes' => 4096, 'sha256' => str_repeat('3', 64), 'etag' => '"mixed"', 'version_id' => '', ] ); $mixedHTTP = new MockHandler([ new Response(200, [], '' . 'mixed-upload' . ''), new Response(200, [ 'Content-Length' => '4096', 'ETag' => '"mixed"', 'x-amz-version-id' => 'versioned-object', ]), ]); $mixedWorker = new CloudStorageReplicaWorkerService( $mixedHelper, new CloudObjectStorageS3Client(new Client([ 'handler' => HandlerStack::create($mixedHTTP), ])), $runtime ); $mixedWorker->tick(); $mixedCopy = CloudStorageOperation::where( 'artifact_id', $mixedView['artifact_id'] )->where('kind', 'copy')->firstOrFail(); $assert((string) $mixedCopy->phase === 'terminal' && (string) $mixedCopy->outcome === 'failed' && (string) $mixedCopy->error_code === 'replica_identity_changed', 'Mixed versioned/versionless completion was accepted.'); $assert(in_array( 'replica_identity_changed', $mixedWorker->observationErrors(), true ), 'A terminal identity mismatch was hidden from worker health.'); CloudStorageArtifactService::failReplicaAppliedMismatch( (string) $mixedCopy->operation_id, str_repeat('0', 32), 'replica_identity_changed', time() ); $expect(static fn () => CloudStorageArtifactService::failReplicaAppliedMismatch( (string) $mixedCopy->operation_id, str_repeat('0', 32), 'replica_identity_changed_again', time() ), 409); $expect(static fn () => CloudStorageArtifactService::failReplicaNotApplied( (string) $mixedCopy->operation_id, str_repeat('0', 32), 'replica_identity_changed', time() ), 409); // Stable multipart identity ambiguity must stop automatic polling and expose // one administrator recovery entry for the original operation only. $attentionView = CloudStorageArtifactService::beginPublication( 'template', 'cloud-templates/attention/system.qcow2', 4096, str_repeat('4', 64), 'template-publication-attention', hash('sha256', 'attention'), [ 'template_import_intent_id' => str_repeat('4', 32), ] ); CloudStorageArtifactService::markPrimaryDispatching( $attentionView['operation_id'] ); CloudStorageArtifactService::completePrimary( $attentionView['operation_id'], (int) $attentionView['artifact_id'], [ 'size_bytes' => 4096, 'sha256' => str_repeat('4', 64), 'etag' => '"attention-primary"', 'version_id' => '', ]); $attentionCopy = CloudStorageOperation::where( 'artifact_id', $attentionView['artifact_id'] )->where('kind', 'copy')->firstOrFail(); $attentionTarget = CloudStorageReplica::whereKey( (int) $attentionCopy->replica_id )->firstOrFail(); $attentionKey = htmlspecialchars( (string) $attentionTarget->object_key, ENT_XML1 | ENT_QUOTES, 'UTF-8' ); $attentionSessions = '' . 'false' . '' . $attentionKey . 'attention-a' . '' . gmdate('Y-m-d\\TH:i:s\\Z', time()) . '' . '' . $attentionKey . 'attention-b' . '' . gmdate('Y-m-d\\TH:i:s\\Z', time()) . ''; $attentionHTTP = new MockHandler([ new Response(500), new Response(404), new Response(200, [], $attentionSessions), new Response(404), new Response(200, [], $attentionSessions), ]); $attentionHelperCalls = 0; $attentionWorker = new CloudStorageReplicaWorkerService( new CloudStorageReplicaHelper(null, static function (array $request) use (&$attentionHelperCalls): array { $attentionHelperCalls++; throw new RuntimeException('Ambiguous upload must not start helper'); } ), new CloudObjectStorageS3Client(new Client([ 'handler' => HandlerStack::create($attentionHTTP), ])), $runtime ); $attentionWorker->tick(); $attentionCopy->refresh(); $attentionCopy->next_check_at = 0; $attentionCopy->save(); $attentionWorker->tick(); $attentionCopy->refresh(); $firstAttentionEvidence = json_decode( (string) $attentionCopy->result_json, true ); $assert((string) $attentionCopy->phase === 'reconciling' && ($firstAttentionEvidence['multipart_attention_confirmations'] ?? 0) === 1, 'First multipart ambiguity observation did not remain reconciling.'); $attentionCopy->next_check_at = 0; $attentionCopy->save(); $attentionWorker->tick(); $attentionCopy->refresh(); $attentionEvidence = json_decode((string) $attentionCopy->result_json, true); $attentionList = CloudStorageReplicaAttentionService::listView(); $operationCountBeforeAttentionRetry = CloudStorageOperation::count(); $assert($attentionHelperCalls === 0 && (string) $attentionCopy->phase === 'terminal' && (string) $attentionCopy->outcome === 'failed' && (string) $attentionCopy->effect_resolution === 'unknown' && (bool) $attentionCopy->effect_may_have_applied && !(bool) $attentionCopy->safe_to_resubmit && ($attentionEvidence['multipart_attention_confirmations'] ?? 0) === 2 && ($attentionList['summary']['count'] ?? 0) === 1 && !str_contains(json_encode($attentionList), $attentionKey) && !str_contains(json_encode($attentionList), 'attention-a') && !str_contains(json_encode($attentionList), 'attention-b'), 'Stable multipart ambiguity did not converge to administrator attention.'); $attentionTarget->refresh(); $attentionTarget->state = 'ready'; $attentionTarget->size_bytes = 4096; $attentionTarget->sha256 = str_repeat('4', 64); $attentionTarget->verified_at = time(); $attentionTarget->error_code = ''; $attentionTarget->save(); $staleAttentionList = CloudStorageReplicaAttentionService::listView(); $assert(($staleAttentionList['items'][0]['can_retry'] ?? true) === false, 'A stale attention operation remained retryable after target repair.'); $expect(static fn () => CloudStorageReplicaAttentionService::retry( (string) $attentionCopy->operation_id ), 409); $attentionTarget->state = 'degraded'; $attentionTarget->error_code = 'replica_multipart_identity_ambiguous'; $attentionTarget->save(); $attentionArtifact = CloudStorageArtifact::whereKey( (int) $attentionView['artifact_id'] )->firstOrFail(); $attentionArtifact->state = 'delete_attention'; $attentionArtifact->save(); $deleteAttentionList = CloudStorageReplicaAttentionService::listView(); $deleteAttentionItem = $deleteAttentionList['items'][0] ?? []; $assert(($deleteAttentionItem['operation_id'] ?? '') === (string) $attentionCopy->operation_id && ($deleteAttentionItem['can_retry'] ?? false) === true, 'Delete-attention copy did not expose read-only reconciliation.'); CloudStorageReplicaAttentionService::retry( (string) $attentionCopy->operation_id ); $attentionCopy->refresh(); $attentionArtifact->refresh(); $reopenedEvidence = json_decode((string) $attentionCopy->result_json, true); $assert((string) $attentionCopy->phase === 'reconciling' && (string) $attentionCopy->error_code === 'admin_reconcile_requested' && ($reopenedEvidence['multipart_attention_confirmations'] ?? -1) === 0 && (string) $attentionArtifact->state === 'delete_attention' && CloudStorageOperation::count() === $operationCountBeforeAttentionRetry && $attentionHelperCalls === 0, 'Delete-attention retry replaced or dispatched the original copy intent.'); $attentionCopy->next_check_at = 253402300799; $attentionCopy->save(); // A definite no-effect failure is idempotent only for the exact same terminal // identity; conflicting late receipts must be rejected. $failedCopyView = CloudStorageArtifactService::beginPublication( 'backup', 'backups/failed-copy/system.qcow2', 4096, str_repeat('5', 64), 'backup-publication-failed-copy', hash('sha256', 'failed-copy'), ['backup_id' => 9] ); CloudStorageArtifactService::markPrimaryDispatching( $failedCopyView['operation_id'] ); CloudStorageArtifactService::completePrimary( $failedCopyView['operation_id'], (int) $failedCopyView['artifact_id'], [ 'size_bytes' => 4096, 'sha256' => str_repeat('5', 64), 'etag' => '"failed-copy-primary"', 'version_id' => '', ]); $failedCopyClaim = CloudStorageArtifactService::claimReplicaWork(time()); $assert(is_array($failedCopyClaim), 'Failed-copy fixture was not claimable.'); CloudStorageArtifactService::failReplicaNotApplied( $failedCopyClaim['operation_id'], $failedCopyClaim['lease_token'], 'replica_copy_rejected', time() ); CloudStorageArtifactService::failReplicaNotApplied( $failedCopyClaim['operation_id'], $failedCopyClaim['lease_token'], 'replica_copy_rejected', time() ); $expect(static fn () => CloudStorageArtifactService::failReplicaNotApplied( $failedCopyClaim['operation_id'], $failedCopyClaim['lease_token'], 'replica_copy_rejected_again', time() ), 409); $pendingView = CloudStorageArtifactService::beginAgentPublication( 'backup', 'backups/pending/system.qcow2', 8192, 'backup-publication-0004', hash('sha256', 'pending'), ['backup_id' => 4] ); CloudStorageArtifactService::markPrimaryDispatching( $pendingView['operation_id'] ); CloudStorageArtifactService::failPrimary( $pendingView['operation_id'], (int) $pendingView['artifact_id'], 'not_applied', 'primary_upload_rejected' ); CloudStorageArtifactService::failPrimary( $pendingView['operation_id'], (int) $pendingView['artifact_id'], 'not_applied', 'primary_upload_rejected' ); $expect(static fn () => CloudStorageArtifactService::failPrimary( $pendingView['operation_id'], (int) $pendingView['artifact_id'], 'not_applied', 'primary_upload_rejected_again' ), 409); $pendingArtifact = CloudStorageArtifact::whereKey( $pendingView['artifact_id'] )->firstOrFail(); $pendingPublication = CloudStoragePublication::where( 'artifact_id', $pendingArtifact->id )->firstOrFail(); $pendingCopy = CloudStorageOperation::where( 'artifact_id', $pendingArtifact->id )->where('kind', 'copy')->firstOrFail(); $assert((string) $pendingArtifact->state === 'publishing' && (int) $pendingArtifact->size_bytes === 0 && (string) $pendingArtifact->sha256 === '' && (string) $pendingPublication->state === 'failed' && (string) $pendingCopy->phase === 'terminal' && (string) $pendingCopy->effect_resolution === 'not_applied', 'Failed pending publication violated the blank-identity state contract.'); // Copy settlement must never overwrite a deletion projection or deletion // worker lease, even if an older accepted copy operation remains claimable. $deletedView = CloudStorageArtifactService::beginPublication( 'backup', 'backups/deleted-copy/system.qcow2', 4096, str_repeat('6', 64), 'backup-publication-deleted-copy', hash('sha256', 'deleted-copy'), ['backup_id' => 10] ); CloudStorageArtifactService::markPrimaryDispatching( $deletedView['operation_id'] ); CloudStorageArtifactService::completePrimary( $deletedView['operation_id'], (int) $deletedView['artifact_id'], [ 'size_bytes' => 4096, 'sha256' => str_repeat('6', 64), 'etag' => '"deleted-copy-primary"', 'version_id' => '', ]); $deletedArtifact = CloudStorageArtifact::whereKey( $deletedView['artifact_id'] )->firstOrFail(); $deletedCopy = CloudStorageOperation::where( 'artifact_id', $deletedArtifact->id )->where('kind', 'copy')->firstOrFail(); $deletedReplica = CloudStorageReplica::whereKey( (int) $deletedCopy->replica_id )->firstOrFail(); $deletedArtifact->state = 'deleted'; $deletedArtifact->save(); $deletedReplica->state = 'deleted'; $deletedReplica->lease_token = str_repeat('d', 32); $deletedReplica->lease_until = time() + 3600; $deletedReplica->error_code = 'delete_projection_sentinel'; $deletedReplica->save(); $assert(CloudStorageArtifactService::claimReplicaWork(time()) === null, 'Deleted artifact unexpectedly returned copy work.'); $deletedCopy->refresh(); $deletedReplica->refresh(); $deletedArtifact->refresh(); $assert((string) $deletedCopy->phase === 'terminal' && (string) $deletedCopy->effect_resolution === 'not_applied' && (string) $deletedArtifact->state === 'deleted' && (string) $deletedReplica->state === 'deleted' && (string) $deletedReplica->lease_token === str_repeat('d', 32) && (string) $deletedReplica->error_code === 'delete_projection_sentinel', 'Copy fence overwrote an existing deletion projection.'); // A receipt must be bound to the caller's frozen artifact before any row is // changed, even when two artifacts intentionally share the same bytes. $crossA = CloudStorageArtifactService::beginPublication( 'backup', 'backups/cross-a/system.qcow2', 4096, str_repeat('7', 64), 'backup-publication-cross-a', hash('sha256', 'cross-a'), ['backup_id' => 110] ); $crossB = CloudStorageArtifactService::beginPublication( 'backup', 'backups/cross-b/system.qcow2', 4096, str_repeat('7', 64), 'backup-publication-cross-b', hash('sha256', 'cross-b'), ['backup_id' => 111] ); CloudStorageArtifactService::markPrimaryDispatching( $crossA['operation_id'] ); CloudStorageArtifactService::markPrimaryDispatching( $crossB['operation_id'] ); $expect(static fn () => CloudStorageArtifactService::completePrimary( $crossA['operation_id'], (int) $crossB['artifact_id'], [ 'size_bytes' => 4096, 'sha256' => str_repeat('7', 64), 'etag' => '"cross-a"', 'version_id' => '', ] ), 409); $crossAOperation = CloudStorageOperation::where( 'operation_id', $crossA['operation_id'] )->firstOrFail(); $crossAPrimary = CloudStorageReplica::whereKey( (int) $crossAOperation->replica_id )->firstOrFail(); $crossAArtifact = CloudStorageArtifact::whereKey( (int) $crossA['artifact_id'] )->firstOrFail(); $crossBOperation = CloudStorageOperation::where( 'operation_id', $crossB['operation_id'] )->firstOrFail(); $crossBPrimary = CloudStorageReplica::whereKey( (int) $crossBOperation->replica_id )->firstOrFail(); $crossBArtifact = CloudStorageArtifact::whereKey( (int) $crossB['artifact_id'] )->firstOrFail(); $assert((string) $crossAOperation->phase === 'dispatching' && $crossAOperation->outcome === null && (string) $crossAPrimary->state === 'publishing' && (int) $crossAPrimary->size_bytes === 0 && (string) $crossAArtifact->state === 'publishing' && (string) $crossBOperation->phase === 'dispatching' && $crossBOperation->outcome === null && (string) $crossBPrimary->state === 'publishing' && (int) $crossBPrimary->size_bytes === 0 && (string) $crossBArtifact->state === 'publishing', 'Cross-artifact primary receipt changed one of the two ledgers.'); // A required-two artifact must cross the initial publication gate with both // replicas. Once the owner has finalized it, later loss of one replica is a // degraded-but-readable repair condition and must not reopen the outbox. $runtimeReadView = CloudStorageArtifactService::beginPublication( 'template', 'cloud-templates/runtime-read/system.qcow2', 4096, str_repeat('6', 64), 'template-runtime-read-policy', hash('sha256', 'template-runtime-read-policy'), ['template_import_intent_id' => str_repeat('6', 32)] ); CloudStorageArtifactService::markPrimaryDispatching( $runtimeReadView['operation_id'] ); CloudStorageArtifactService::completePrimary( $runtimeReadView['operation_id'], (int) $runtimeReadView['artifact_id'], [ 'size_bytes' => 4096, 'sha256' => str_repeat('6', 64), 'etag' => '"runtime-primary"', 'version_id' => '', ] ); $runtimePublication = CloudStoragePublication::where( 'artifact_id', (int) $runtimeReadView['artifact_id'] )->firstOrFail(); $assert((string) $runtimePublication->state === 'waiting_protection', 'One template replica prematurely released the required-two publication.'); $runtimeArtifact = CloudStorageArtifact::whereKey( (int) $runtimeReadView['artifact_id'] )->firstOrFail(); $runtimeReplicas = CloudStorageReplica::where( 'artifact_id', (int) $runtimeArtifact->id )->orderBy('id')->get(); $runtimeSecondary = $runtimeReplicas->firstWhere('role', 'secondary'); $assert($runtimeSecondary instanceof CloudStorageReplica, 'Required-two fixture is missing its secondary replica.'); $runtimeSecondary->state = 'ready'; $runtimeSecondary->size_bytes = 4096; $runtimeSecondary->sha256 = str_repeat('6', 64); $runtimeSecondary->object_etag = '"runtime-secondary"'; $runtimeSecondary->verified_at = time(); $runtimeSecondary->error_code = ''; $runtimeSecondary->save(); $project = (new ReflectionClass(CloudStorageArtifactService::class)) ->getMethod('projectArtifactLocked'); $project->invoke( null, $runtimeArtifact, $runtimeReplicas->all(), time() ); $runtimePublication->refresh(); $assert((string) $runtimePublication->state === 'ready_to_finalize', 'Two verified template replicas did not release publication.'); CloudStoragePublicationFinalizerService::complete( 'template', str_repeat('6', 32), (int) $runtimeArtifact->id, (string) $runtimeReadView['operation_id'] ); $runtimeSecondary->state = 'degraded'; $runtimeSecondary->verified_at = 0; $runtimeSecondary->error_code = 'replica_unavailable'; $runtimeSecondary->save(); $runtimeArtifact->refresh(); $runtimeReplicas = CloudStorageReplica::where( 'artifact_id', (int) $runtimeArtifact->id )->orderBy('id')->get(); $project->invoke( null, $runtimeArtifact, $runtimeReplicas->all(), time() ); $runtimeArtifact->refresh(); $runtimePublication->refresh(); $runtimeProjection = CloudStorageDurabilityPolicy::project( [ 'state' => (string) $runtimeArtifact->state, 'size_bytes' => (int) $runtimeArtifact->size_bytes, 'sha256' => (string) $runtimeArtifact->sha256, 'desired_copies' => (int) $runtimeArtifact->desired_copies, 'required_copies' => (int) $runtimeArtifact->required_copies, ], $runtimeReplicas->map(static fn (CloudStorageReplica $replica): array => [ 'backend_id' => (int) $replica->backend_id, 'state' => (string) $replica->state, 'size_bytes' => (int) $replica->size_bytes, 'sha256' => (string) $replica->sha256, 'verified_at' => (int) $replica->verified_at, ])->values()->all() ); $assert((string) $runtimeArtifact->state === 'degraded' && $runtimeProjection['can_read'] && !$runtimeProjection['meets_publication_requirement'] && (string) $runtimePublication->state === 'completed' && (int) $runtimePublication->finalized_at > 0, 'Replica degradation made a published template unreadable or reopened its outbox.'); CloudStorageBackend::whereKey(2)->update([ 'admin_state' => 'disabled', 'health_state' => 'unreachable', 'tested_revision' => 0, ]); $artifactCountBeforeRejectedPublication = CloudStorageArtifact::count(); $expect(static fn () => CloudStorageArtifactService::beginPublication( 'template', 'cloud-templates/t/g/system.qcow2', 8192, str_repeat('b', 64), 'template-publication-0001', hash('sha256', 't'), ['template_import_intent_id' => str_repeat('d', 32)] ), 503); $assert(CloudStorageArtifact::count() === $artifactCountBeforeRejectedPublication, 'Required-two preflight left a partial template artifact.'); echo "Storage artifact service tests passed.\n";