addConnection(['driver' => 'sqlite', 'database' => ':memory:']); $db->setAsGlobal(); $db->bootEloquent(); $db->schema()->create('cloud_storage_artifacts', static function ( Blueprint $table ): void { $table->increments('id'); $table->string('artifact_key')->unique(); $table->string('kind'); $table->unsignedBigInteger('maximum_size_bytes'); $table->unsignedBigInteger('size_bytes'); $table->string('sha256'); $table->string('durability_policy'); $table->unsignedTinyInteger('desired_copies'); $table->unsignedTinyInteger('required_copies'); $table->string('state'); $table->string('last_operation_id')->nullable(); $table->unsignedBigInteger('verified_at'); $table->unsignedBigInteger('created_at'); $table->unsignedBigInteger('updated_at'); }); $db->schema()->create('cloud_storage_publications', static function ( Blueprint $table ): void { $table->increments('id'); $table->string('publication_key')->unique(); $table->unsignedBigInteger('artifact_id')->unique(); $table->unsignedBigInteger('backup_id')->nullable()->unique(); $table->string('template_import_intent_id')->nullable()->unique(); $table->string('rescue_template_id')->nullable()->unique(); $table->string('state'); $table->unsignedInteger('attempts')->default(0); $table->unsignedBigInteger('available_at')->default(0); $table->string('lease_token')->nullable(); $table->unsignedBigInteger('lease_until')->default(0); $table->string('last_error_code')->default(''); $table->unsignedBigInteger('coverage_ready_at')->nullable(); $table->unsignedBigInteger('finalized_at')->nullable(); $table->unsignedBigInteger('created_at'); $table->unsignedBigInteger('updated_at'); }); $db->schema()->create('cloud_storage_operations', static function ( Blueprint $table ): void { $table->increments('id'); $table->string('operation_id')->unique(); $table->string('request_idempotency_key')->unique(); $table->string('request_fingerprint'); $table->unsignedBigInteger('artifact_id'); $table->unsignedBigInteger('replica_id')->nullable(); $table->unsignedBigInteger('source_backend_id')->nullable(); $table->unsignedBigInteger('target_backend_id')->nullable(); $table->string('kind'); $table->string('phase'); $table->string('outcome')->nullable(); $table->string('effect_resolution'); $table->boolean('effect_may_have_applied')->nullable(); $table->boolean('safe_to_resubmit')->nullable(); $table->unsignedInteger('attempts')->default(0); $table->string('lease_token')->nullable(); $table->unsignedBigInteger('lease_until')->default(0); $table->unsignedBigInteger('available_at')->default(0); $table->unsignedBigInteger('next_check_at')->default(0); $table->string('error_code')->default(''); $table->text('result_json')->nullable(); $table->unsignedBigInteger('terminal_at')->nullable(); $table->unsignedBigInteger('created_at'); $table->unsignedBigInteger('updated_at'); }); $db->schema()->create('cloud_storage_replicas', static function ( Blueprint $table ): void { $table->increments('id'); $table->string('replica_key')->unique(); $table->unsignedBigInteger('artifact_id'); $table->unsignedBigInteger('backend_id'); $table->string('role'); $table->string('object_key'); $table->string('object_version_id')->default(''); $table->string('object_etag')->default(''); $table->string('state'); $table->string('publish_operation_id')->nullable(); $table->string('delete_operation_id')->nullable(); $table->unsignedInteger('delete_generation')->default(0); $table->unsignedInteger('delete_cycle_attempts')->default(0); $table->unsignedInteger('delete_presence_confirmations')->default(0); $table->unsignedBigInteger('delete_last_checked_at')->default(0); $table->unsignedBigInteger('delete_next_check_at')->default(0); $table->unsignedBigInteger('delete_attention_at')->nullable(); $table->string('lease_token')->nullable(); $table->unsignedBigInteger('lease_until')->default(0); $table->unsignedBigInteger('size_bytes')->default(0); $table->string('sha256')->default(''); $table->string('error_code')->default(''); $table->unsignedBigInteger('verified_at')->default(0); $table->unsignedBigInteger('created_at'); $table->unsignedBigInteger('updated_at'); }); $db->schema()->create('cloud_storage_backends', static function ( Blueprint $table ): void { $table->increments('id'); $table->string('name'); }); $clock = 2000000000; $ownerA = str_repeat('a', 32); $ownerB = str_repeat('b', 32); $create = static function ( string $kind, string $owner, string $state = 'ready_to_finalize', int $attempts = 0, int $availableAt = 0 ) use (&$clock): CloudStoragePublication { $artifact = new CloudStorageArtifact(); $artifact->artifact_key = bin2hex(random_bytes(16)); $artifact->kind = $kind; $artifact->maximum_size_bytes = 4096; $artifact->size_bytes = 4096; $artifact->sha256 = str_repeat($kind === 'template' ? 'a' : 'b', 64); $artifact->durability_policy = 'mirror_required_2'; $artifact->desired_copies = 2; $artifact->required_copies = 2; $artifact->state = 'protected'; $artifact->last_operation_id = null; $artifact->verified_at = $clock - 10; $artifact->created_at = $artifact->updated_at = $clock - 20; $artifact->save(); $publication = new CloudStoragePublication(); $publication->publication_key = bin2hex(random_bytes(16)); $publication->artifact_id = (int) $artifact->id; $publication->backup_id = null; $publication->template_import_intent_id = $kind === 'template' ? $owner : null; $publication->rescue_template_id = $kind === 'rescue' ? $owner : null; $publication->state = $state; $publication->attempts = $attempts; $publication->available_at = $availableAt; $publication->lease_token = null; $publication->lease_until = 0; $publication->last_error_code = ''; $publication->coverage_ready_at = $clock - 5; $publication->finalized_at = null; $publication->created_at = $publication->updated_at = $clock - 20; $publication->save(); return $publication; }; $template = $create('template', $ownerA); $calls = []; $worker = new CloudStoragePublicationOutboxService( static function ( string $type, int|string $owner, int $publicationId, int $artifactId ) use (&$calls, &$clock, $assert): void { $assert(Capsule::connection()->transactionLevel() > 0, 'Business finalization escaped the owner transaction.'); $calls[] = [$type, $owner, $publicationId, $artifactId]; $row = CloudStoragePublication::query() ->whereKey($publicationId)->lockForUpdate()->firstOrFail(); $row->state = 'completed'; $row->attempts = (int) $row->attempts + 1; $row->finalized_at = $clock; $row->updated_at = $clock; $row->save(); }, static function () use (&$clock): int { return $clock; } ); $assert($worker->tick(), 'A due template publication was not consumed.'); $template->refresh(); $assert((string) $template->state === 'completed' && (int) $template->finalized_at === $clock && $calls === [[ 'template', $ownerA, (int) $template->id, (int) $template->artifact_id, ]], 'Template outbox finalization lost its frozen owner identity.'); $assert(!$worker->tick(), 'A completed publication was consumed twice.'); $rescue = $create('rescue', $ownerB, 'ready_to_finalize', 0, $clock + 60); $assert(!$worker->tick(), 'A future publication bypassed its retry delay.'); $clock += 60; $assert($worker->tick(), 'A due rescue publication was not consumed.'); $rescue->refresh(); $assert((string) $rescue->state === 'completed' && $calls[1][0] === 'rescue' && $calls[1][1] === $ownerB, 'Rescue outbox dispatch selected the wrong owner.'); $failing = $create('template', str_repeat('c', 32)); $failingWorker = new CloudStoragePublicationOutboxService( static function ( string $type, int|string $owner, int $publicationId ) use (&$clock): void { // Prove that a consumer which mutates the business publication and // then crashes is rolled back to the savepoint before the durable // retry delay is committed. $partial = CloudStoragePublication::query() ->whereKey($publicationId)->lockForUpdate()->firstOrFail(); $partial->state = 'completed'; $partial->finalized_at = $clock; $partial->save(); throw new CloudError('fixture owner failure', 409); }, static function () use (&$clock): int { return $clock; } ); $assert($failingWorker->tick(), 'A failed outbox attempt was not observed.'); $failing->refresh(); $assert((string) $failing->state === 'ready_to_finalize' && (int) $failing->attempts === 1 && $failing->finalized_at === null && (int) $failing->available_at === $clock + 60 && (string) $failing->last_error_code === 'storage_publication_finalization_failed' && $failingWorker->observationErrors() === ['storage_publication_finalization_failed'], 'A failed owner finalization was not durably delayed.'); $assert(!$failingWorker->tick(), 'A delayed failure entered a hot loop.'); $attention = $create( 'rescue', str_repeat('d', 32), 'ready_to_finalize', 5, $clock ); $assert($failingWorker->tick(), 'The bounded final attempt was not handled.'); $attention->refresh(); $assert((string) $attention->state === 'failed' && (int) $attention->attempts === 6 && (string) $attention->last_error_code === 'storage_publication_finalization_attention', 'Exhausted owner finalization did not become explicit attention.'); $attentionView = CloudStorageReplicaAttentionService::listView(); $assert(($attentionView['summary']['finalization_count'] ?? 0) === 1 && count($attentionView['finalization_items'] ?? []) === 1 && ($attentionView['finalization_items'][0]['operation_kind'] ?? '') === 'finalize' && ($attentionView['finalization_items'][0]['operation_id'] ?? '') === (string) $attention->publication_key && ($attentionView['finalization_items'][0]['can_retry'] ?? false), 'Finalization attention is not visible on the administrator recovery page.'); $retry = CloudStorageReplicaAttentionService::retry( (string) $attention->publication_key, 'finalize' ); $attention->refresh(); $assert((string) $attention->state === 'ready_to_finalize' && (int) $attention->attempts === 0 && (int) $attention->available_at > 0 && (string) $attention->last_error_code === '' && ($retry['operation_kind'] ?? '') === 'finalize', 'Administrator recovery did not reopen the same publication intent.'); $attention->state = 'completed'; $attention->finalized_at = $clock; $attention->save(); // Finalization attention has an independent stable paginator. Prove adjacent // pages are disjoint and together expose every row instead of silently // truncating after the first page. $pagedAttention = []; foreach (['1', '2', '3'] as $ownerDigit) { $row = $create('template', str_repeat($ownerDigit, 32)); $row->state = 'failed'; $row->attempts = 6; $row->available_at = 0; $row->last_error_code = 'storage_publication_finalization_attention'; $row->updated_at = $clock + (int) $ownerDigit; $row->save(); $pagedAttention[] = $row; } $finalizationPageOne = CloudStorageReplicaAttentionService::listView([ 'page' => 1, 'finalization_page' => 1, 'per_page' => 2, ]); $finalizationPageTwo = CloudStorageReplicaAttentionService::listView([ 'page' => 1, 'finalization_page' => 2, 'per_page' => 2, ]); $pageOneIds = array_column( $finalizationPageOne['finalization_items'], 'operation_id' ); $pageTwoIds = array_column( $finalizationPageTwo['finalization_items'], 'operation_id' ); $assert(count($pageOneIds) === 2 && count($pageTwoIds) === 1 && array_intersect($pageOneIds, $pageTwoIds) === [] && count(array_unique(array_merge($pageOneIds, $pageTwoIds))) === 3 && ($finalizationPageOne['finalization_pagination']['total'] ?? 0) === 3 && ($finalizationPageOne['finalization_pagination']['total_pages'] ?? 0) === 2, 'Finalization attention pagination repeated or omitted an item.'); foreach ($pagedAttention as $row) { $row->state = 'completed'; $row->finalized_at = $clock; $row->save(); } $databaseFault = $create('rescue', str_repeat('4', 32)); $databaseFaultWorker = new CloudStoragePublicationOutboxService( static function (): void { throw new PDOException('fixture database outage'); }, static function () use (&$clock): int { return $clock; } ); $databaseFaultRaised = false; try { $databaseFaultWorker->tick(); } catch (PDOException) { $databaseFaultRaised = true; } $databaseFault->refresh(); $assert($databaseFaultRaised && (string) $databaseFault->state === 'ready_to_finalize' && (int) $databaseFault->attempts === 0, 'Database failure consumed the bounded owner-finalization budget.'); $databaseFault->state = 'completed'; $databaseFault->finalized_at = $clock; $databaseFault->save(); // Illuminate wraps ordinary SQL failures and deadlocks in QueryException. // In the installed database component it extends PDOException, so the same // infrastructure path must escape without consuming owner failure attempts. $wrappedDatabaseFault = $create('template', str_repeat('5', 32)); $wrappedDatabaseFaultWorker = new CloudStoragePublicationOutboxService( static function (): void { throw new QueryException( 'default', 'UPDATE cloud_storage_publications SET state = ?', ['completed'], new PDOException('fixture wrapped database outage') ); }, static function () use (&$clock): int { return $clock; } ); $wrappedDatabaseFaultRaised = false; try { $wrappedDatabaseFaultWorker->tick(); } catch (QueryException) { $wrappedDatabaseFaultRaised = true; } $wrappedDatabaseFault->refresh(); $assert($wrappedDatabaseFaultRaised && (string) $wrappedDatabaseFault->state === 'ready_to_finalize' && (int) $wrappedDatabaseFault->attempts === 0, 'Wrapped database failure consumed owner-finalization attempts.'); $wrappedDatabaseFault->state = 'completed'; $wrappedDatabaseFault->finalized_at = $clock; $wrappedDatabaseFault->save(); $wrong = $create('template', str_repeat('e', 32)); $noSettlement = new CloudStoragePublicationOutboxService( static function (): void { // A domain consumer returning without completing the outbox is an // error; it must roll back and enter the same bounded retry path. }, static function () use (&$clock): int { return $clock; } ); $assert($noSettlement->tick(), 'Missing business settlement was not handled.'); $wrong->refresh(); $assert((string) $wrong->state === 'ready_to_finalize' && (int) $wrong->attempts === 1, 'A no-op business consumer was treated as successful.'); $wrong->state = 'completed'; $wrong->finalized_at = $clock; $wrong->save(); $workerOwned = $create('rescue', str_repeat('f', 32)); $copyArtifact = new CloudStorageArtifact(); $copyArtifact->artifact_key = bin2hex(random_bytes(16)); $copyArtifact->kind = 'backup'; $copyArtifact->maximum_size_bytes = 4096; $copyArtifact->size_bytes = 4096; $copyArtifact->sha256 = str_repeat('9', 64); $copyArtifact->durability_policy = 'mirror_required_2'; $copyArtifact->desired_copies = 2; $copyArtifact->required_copies = 2; $copyArtifact->state = 'available'; $copyArtifact->last_operation_id = null; $copyArtifact->verified_at = $clock; $copyArtifact->created_at = $copyArtifact->updated_at = $clock; $copyArtifact->save(); $sourceReplica = new CloudStorageReplica(); $sourceReplica->replica_key = bin2hex(random_bytes(16)); $sourceReplica->artifact_id = (int) $copyArtifact->id; $sourceReplica->backend_id = 1; $sourceReplica->role = 'primary'; $sourceReplica->object_key = 'fixture/source.qcow2'; $sourceReplica->object_version_id = ''; $sourceReplica->object_etag = '"source"'; $sourceReplica->size_bytes = 4096; $sourceReplica->sha256 = str_repeat('9', 64); $sourceReplica->state = 'ready'; $sourceReplica->publish_operation_id = null; $sourceReplica->delete_operation_id = null; $sourceReplica->lease_token = null; $sourceReplica->lease_until = 0; $sourceReplica->error_code = ''; $sourceReplica->verified_at = $clock; $sourceReplica->created_at = $sourceReplica->updated_at = $clock; $sourceReplica->save(); $targetReplica = new CloudStorageReplica(); $targetReplica->replica_key = bin2hex(random_bytes(16)); $targetReplica->artifact_id = (int) $copyArtifact->id; $targetReplica->backend_id = 2; $targetReplica->role = 'secondary'; $targetReplica->object_key = 'fixture/target.qcow2'; $targetReplica->object_version_id = ''; $targetReplica->object_etag = ''; $targetReplica->size_bytes = 0; $targetReplica->sha256 = ''; $targetReplica->state = 'reserved'; $targetReplica->publish_operation_id = null; $targetReplica->delete_operation_id = null; $targetReplica->lease_token = null; $targetReplica->lease_until = 0; $targetReplica->error_code = ''; $targetReplica->verified_at = 0; $targetReplica->created_at = $targetReplica->updated_at = $clock; $targetReplica->save(); $copyOperation = new CloudStorageOperation(); $copyOperation->operation_id = bin2hex(random_bytes(16)); $copyOperation->request_idempotency_key = 'publication-worker-copy-fixture'; $copyOperation->request_fingerprint = hash('sha256', 'copy-fixture'); $copyOperation->artifact_id = (int) $copyArtifact->id; $copyOperation->replica_id = (int) $targetReplica->id; $copyOperation->source_backend_id = 1; $copyOperation->target_backend_id = 2; $copyOperation->kind = 'copy'; $copyOperation->phase = 'accepted'; $copyOperation->outcome = null; $copyOperation->effect_resolution = 'not_applicable'; $copyOperation->effect_may_have_applied = null; $copyOperation->safe_to_resubmit = null; $copyOperation->attempts = 0; $copyOperation->lease_token = null; $copyOperation->lease_until = 0; $copyOperation->available_at = $clock; $copyOperation->next_check_at = 0; $copyOperation->error_code = ''; $copyOperation->result_json = null; $copyOperation->terminal_at = null; $copyOperation->created_at = $copyOperation->updated_at = $clock; $copyOperation->save(); $integratedOutbox = new CloudStoragePublicationOutboxService( static function ( string $type, int|string $owner, int $publicationId, int $artifactId ) use (&$clock, $assert, $sourceReplica): void { $assert($type === 'rescue' && $owner === str_repeat('f', 32), 'Replica worker dispatched the wrong publication owner.'); $publication = CloudStoragePublication::query() ->whereKey($publicationId)->lockForUpdate()->firstOrFail(); $publication->state = 'completed'; $publication->finalized_at = $clock; $publication->save(); // Makes the replica source invalid only after publication work ran; // the following replica_source_not_ready assertion therefore proves // the short outbox step was consumed before the replica claim. $sourceReplica->state = 'degraded'; $sourceReplica->error_code = 'fixture_not_ready'; $sourceReplica->verified_at = 0; $sourceReplica->save(); }, static function () use (&$clock): int { return $clock; } ); $replicaWorker = new CloudStorageReplicaWorkerService( null, null, null, static function () use (&$clock): int { return $clock; }, $integratedOutbox ); $assert($replicaWorker->tick() && $replicaWorker->observationErrors() === ['replica_source_not_ready'], 'Replica worker did not process both bounded work queues in one tick.'); $workerOwned->refresh(); $copyOperation->refresh(); $assert((string) $workerOwned->state === 'completed', 'Replica worker did not settle the owner publication.'); $assert((string) $copyOperation->phase === 'accepted' && (int) $copyOperation->attempts === 1 && (int) $copyOperation->available_at === $clock + 60 && (string) $copyOperation->error_code === 'replica_source_not_ready', 'Publication finalization starved the replica queue in the same tick.'); echo "Storage publication outbox behavior passed.\n";