addConnection(['driver' => 'sqlite', 'database' => ':memory:']); $db->setAsGlobal(); $db->bootEloquent(); $db->schema()->create('cloud_storage_artifacts', static function ( Blueprint $table ): void { $table->increments('id'); $table->string('artifact_key')->unique(); $table->string('kind'); $table->unsignedBigInteger('maximum_size_bytes'); $table->unsignedBigInteger('size_bytes'); $table->string('sha256'); $table->string('durability_policy'); $table->unsignedTinyInteger('desired_copies'); $table->unsignedTinyInteger('required_copies'); $table->string('state'); $table->string('last_operation_id')->nullable(); $table->unsignedBigInteger('verified_at'); $table->unsignedBigInteger('created_at'); $table->unsignedBigInteger('updated_at'); }); $db->schema()->create('cloud_storage_publications', static function ( Blueprint $table ): void { $table->increments('id'); $table->string('publication_key')->unique(); $table->unsignedBigInteger('artifact_id')->unique(); $table->unsignedBigInteger('backup_id')->nullable()->unique(); $table->string('template_import_intent_id')->nullable()->unique(); $table->string('rescue_template_id')->nullable()->unique(); $table->string('state'); $table->unsignedInteger('attempts')->default(0); $table->unsignedBigInteger('available_at')->default(0); $table->string('lease_token')->nullable(); $table->unsignedBigInteger('lease_until')->default(0); $table->string('last_error_code')->default(''); $table->unsignedBigInteger('coverage_ready_at')->nullable(); $table->unsignedBigInteger('finalized_at')->nullable(); $table->unsignedBigInteger('created_at'); $table->unsignedBigInteger('updated_at'); }); $db->schema()->create('cloud_storage_operations', static function ( Blueprint $table ): void { $table->increments('id'); $table->string('operation_id')->unique(); $table->unsignedBigInteger('artifact_id'); $table->unsignedBigInteger('replica_id')->nullable(); $table->unsignedBigInteger('source_backend_id')->nullable(); $table->unsignedBigInteger('target_backend_id')->nullable(); $table->string('kind'); $table->string('phase'); $table->string('outcome')->nullable(); $table->string('effect_resolution'); $table->boolean('effect_may_have_applied')->nullable(); $table->boolean('safe_to_resubmit')->nullable(); $table->unsignedInteger('attempts')->default(0); $table->string('lease_token')->nullable(); $table->unsignedBigInteger('lease_until')->default(0); $table->unsignedBigInteger('available_at')->default(0); $table->unsignedBigInteger('next_check_at')->default(0); $table->string('error_code')->default(''); $table->unsignedBigInteger('terminal_at')->nullable(); }); $db->schema()->create('cloud_storage_replicas', static function ( Blueprint $table ): void { $table->increments('id'); $table->unsignedBigInteger('artifact_id'); $table->unsignedBigInteger('backend_id'); $table->string('state'); $table->unsignedBigInteger('size_bytes')->default(0); $table->string('sha256')->default(''); $table->string('error_code')->default(''); }); $db->schema()->create('cloud_storage_backends', static function ( Blueprint $table ): void { $table->increments('id'); $table->string('name'); $table->string('prefix')->default(''); $table->string('failure_domain')->default(''); $table->unsignedBigInteger('config_revision')->default(1); $table->unsignedBigInteger('tested_revision')->default(0); $table->string('admin_state')->default('draft'); $table->string('health_state')->default('unknown'); $table->unsignedInteger('write_priority')->default(0); }); $clock = 2000000000; $ownerA = str_repeat('a', 32); $ownerB = str_repeat('b', 32); $create = static function ( string $kind, string $owner, string $state = 'ready_to_finalize', int $attempts = 0, int $availableAt = 0 ) use (&$clock): CloudStoragePublication { $artifact = new CloudStorageArtifact(); $artifact->artifact_key = bin2hex(random_bytes(16)); $artifact->kind = $kind; $artifact->maximum_size_bytes = 4096; $artifact->size_bytes = 4096; $artifact->sha256 = str_repeat($kind === 'template' ? 'a' : 'b', 64); $artifact->durability_policy = 'mirror_required_2'; $artifact->desired_copies = 2; $artifact->required_copies = 2; $artifact->state = 'protected'; $artifact->last_operation_id = null; $artifact->verified_at = $clock - 10; $artifact->created_at = $artifact->updated_at = $clock - 20; $artifact->save(); $publication = new CloudStoragePublication(); $publication->publication_key = bin2hex(random_bytes(16)); $publication->artifact_id = (int) $artifact->id; $publication->backup_id = null; $publication->template_import_intent_id = $kind === 'template' ? $owner : null; $publication->rescue_template_id = $kind === 'rescue' ? $owner : null; $publication->state = $state; $publication->attempts = $attempts; $publication->available_at = $availableAt; $publication->lease_token = null; $publication->lease_until = 0; $publication->last_error_code = ''; $publication->coverage_ready_at = $clock - 5; $publication->finalized_at = null; $publication->created_at = $publication->updated_at = $clock - 20; $publication->save(); return $publication; }; $template = $create('template', $ownerA); $calls = []; $worker = new CloudStoragePublicationOutboxService( static function ( string $type, int|string $owner, int $publicationId, int $artifactId ) use (&$calls, &$clock, $assert): void { $assert(Capsule::connection()->transactionLevel() > 0, 'Business finalization escaped the owner transaction.'); $calls[] = [$type, $owner, $publicationId, $artifactId]; $row = CloudStoragePublication::query() ->whereKey($publicationId)->lockForUpdate()->firstOrFail(); $row->state = 'completed'; $row->attempts = (int) $row->attempts + 1; $row->finalized_at = $clock; $row->updated_at = $clock; $row->save(); }, static function () use (&$clock): int { return $clock; } ); $assert($worker->tick(), 'A due template publication was not consumed.'); $template->refresh(); $assert((string) $template->state === 'completed' && (int) $template->finalized_at === $clock && $calls === [[ 'template', $ownerA, (int) $template->id, (int) $template->artifact_id, ]], 'Template outbox finalization lost its frozen owner identity.'); $assert(!$worker->tick(), 'A completed publication was consumed twice.'); $rescue = $create('rescue', $ownerB, 'ready_to_finalize', 0, $clock + 60); $assert(!$worker->tick(), 'A future publication bypassed its retry delay.'); $clock += 60; $assert($worker->tick(), 'A due rescue publication was not consumed.'); $rescue->refresh(); $assert((string) $rescue->state === 'completed' && $calls[1][0] === 'rescue' && $calls[1][1] === $ownerB, 'Rescue outbox dispatch selected the wrong owner.'); $failing = $create('template', str_repeat('c', 32)); $failingWorker = new CloudStoragePublicationOutboxService( static function ( string $type, int|string $owner, int $publicationId ) use (&$clock): void { // Prove that a consumer which mutates the business publication and // then crashes is rolled back to the savepoint before the durable // retry delay is committed. $partial = CloudStoragePublication::query() ->whereKey($publicationId)->lockForUpdate()->firstOrFail(); $partial->state = 'completed'; $partial->finalized_at = $clock; $partial->save(); throw new CloudError('fixture owner failure', 409); }, static function () use (&$clock): int { return $clock; } ); $assert($failingWorker->tick(), 'A failed outbox attempt was not observed.'); $failing->refresh(); $assert((string) $failing->state === 'ready_to_finalize' && (int) $failing->attempts === 1 && $failing->finalized_at === null && (int) $failing->available_at === $clock + 60 && (string) $failing->last_error_code === 'storage_publication_finalization_failed' && $failingWorker->observationErrors() === ['storage_publication_finalization_failed'], 'A failed owner finalization was not durably delayed.'); $assert(!$failingWorker->tick(), 'A delayed failure entered a hot loop.'); $attention = $create( 'rescue', str_repeat('d', 32), 'ready_to_finalize', 5, $clock ); $assert($failingWorker->tick(), 'The bounded final attempt was not handled.'); $attention->refresh(); $assert((string) $attention->state === 'failed' && (int) $attention->attempts === 6 && (string) $attention->last_error_code === 'storage_publication_finalization_attention', 'Exhausted owner finalization did not become explicit attention.'); $attentionView = CloudStorageReplicaAttentionService::listView(); $assert(($attentionView['summary']['finalization_count'] ?? 0) === 1 && count($attentionView['finalization_items'] ?? []) === 1 && ($attentionView['finalization_items'][0]['operation_kind'] ?? '') === 'finalize' && ($attentionView['finalization_items'][0]['operation_id'] ?? '') === (string) $attention->publication_key && ($attentionView['finalization_items'][0]['can_retry'] ?? false), 'Finalization attention is not visible on the administrator recovery page.'); $retry = CloudStorageReplicaAttentionService::retry( (string) $attention->publication_key, 'finalize' ); $attention->refresh(); $assert((string) $attention->state === 'ready_to_finalize' && (int) $attention->attempts === 0 && (int) $attention->available_at > 0 && (string) $attention->last_error_code === '' && ($retry['operation_kind'] ?? '') === 'finalize', 'Administrator recovery did not reopen the same publication intent.'); $attention->state = 'completed'; $attention->finalized_at = $clock; $attention->save(); // Finalization attention has an independent stable paginator. Prove adjacent // pages are disjoint and together expose every row instead of silently // truncating after the first page. $pagedAttention = []; foreach (['1', '2', '3'] as $ownerDigit) { $row = $create('template', str_repeat($ownerDigit, 32)); $row->state = 'failed'; $row->attempts = 6; $row->available_at = 0; $row->last_error_code = 'storage_publication_finalization_attention'; $row->updated_at = $clock + (int) $ownerDigit; $row->save(); $pagedAttention[] = $row; } $finalizationPageOne = CloudStorageReplicaAttentionService::listView([ 'page' => 1, 'finalization_page' => 1, 'per_page' => 2, ]); $finalizationPageTwo = CloudStorageReplicaAttentionService::listView([ 'page' => 1, 'finalization_page' => 2, 'per_page' => 2, ]); $pageOneIds = array_column( $finalizationPageOne['finalization_items'], 'operation_id' ); $pageTwoIds = array_column( $finalizationPageTwo['finalization_items'], 'operation_id' ); $assert(count($pageOneIds) === 2 && count($pageTwoIds) === 1 && array_intersect($pageOneIds, $pageTwoIds) === [] && count(array_unique(array_merge($pageOneIds, $pageTwoIds))) === 3 && ($finalizationPageOne['finalization_pagination']['total'] ?? 0) === 3 && ($finalizationPageOne['finalization_pagination']['total_pages'] ?? 0) === 2, 'Finalization attention pagination repeated or omitted an item.'); foreach ($pagedAttention as $row) { $row->state = 'completed'; $row->finalized_at = $clock; $row->save(); } $databaseFault = $create('rescue', str_repeat('4', 32)); $databaseFaultWorker = new CloudStoragePublicationOutboxService( static function (): void { throw new PDOException('fixture database outage'); }, static function () use (&$clock): int { return $clock; } ); $databaseFaultRaised = false; try { $databaseFaultWorker->tick(); } catch (PDOException) { $databaseFaultRaised = true; } $databaseFault->refresh(); $assert($databaseFaultRaised && (string) $databaseFault->state === 'ready_to_finalize' && (int) $databaseFault->attempts === 0, 'Database failure consumed the bounded owner-finalization budget.'); $databaseFault->state = 'completed'; $databaseFault->finalized_at = $clock; $databaseFault->save(); // Illuminate wraps ordinary SQL failures and deadlocks in QueryException. // In the installed database component it extends PDOException, so the same // infrastructure path must escape without consuming owner failure attempts. $wrappedDatabaseFault = $create('template', str_repeat('5', 32)); $wrappedDatabaseFaultWorker = new CloudStoragePublicationOutboxService( static function (): void { throw new QueryException( 'default', 'UPDATE cloud_storage_publications SET state = ?', ['completed'], new PDOException('fixture wrapped database outage') ); }, static function () use (&$clock): int { return $clock; } ); $wrappedDatabaseFaultRaised = false; try { $wrappedDatabaseFaultWorker->tick(); } catch (QueryException) { $wrappedDatabaseFaultRaised = true; } $wrappedDatabaseFault->refresh(); $assert($wrappedDatabaseFaultRaised && (string) $wrappedDatabaseFault->state === 'ready_to_finalize' && (int) $wrappedDatabaseFault->attempts === 0, 'Wrapped database failure consumed owner-finalization attempts.'); $wrappedDatabaseFault->state = 'completed'; $wrappedDatabaseFault->finalized_at = $clock; $wrappedDatabaseFault->save(); $wrong = $create('template', str_repeat('e', 32)); $noSettlement = new CloudStoragePublicationOutboxService( static function (): void { // A domain consumer returning without completing the outbox is an // error; it must roll back and enter the same bounded retry path. }, static function () use (&$clock): int { return $clock; } ); $assert($noSettlement->tick(), 'Missing business settlement was not handled.'); $wrong->refresh(); $assert((string) $wrong->state === 'ready_to_finalize' && (int) $wrong->attempts === 1, 'A no-op business consumer was treated as successful.'); $wrong->state = 'completed'; $wrong->finalized_at = $clock; $wrong->save(); echo "Storage publication outbox behavior passed.\n";