addConnection(['driver' => 'sqlite', 'database' => ':memory:']); $db->setAsGlobal(); $db->bootEloquent(); $schema = $db->schema(); $schema->create('cloud_storage_backends', static function (Blueprint $t): void { $t->increments('id'); $t->string('backend_key')->unique(); $t->string('identity_sha256')->unique(); $t->string('name'); $t->string('driver'); $t->string('endpoint'); $t->string('region'); $t->string('bucket'); $t->string('prefix'); $t->boolean('path_style'); $t->string('failure_domain'); $t->text('sealed_credentials'); $t->unsignedBigInteger('config_revision'); $t->unsignedBigInteger('tested_revision'); $t->string('admin_state'); $t->string('health_state'); $t->unsignedInteger('write_priority'); $t->unsignedInteger('consecutive_failures')->default(0); $t->unsignedInteger('consecutive_successes')->default(0); $t->string('last_error_code')->default(''); $t->unsignedBigInteger('last_tested_at')->default(0); $t->unsignedBigInteger('last_healthy_at')->default(0); $t->unsignedBigInteger('next_probe_at')->default(0); $t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at'); }); $schema->create('cloud_storage_artifacts', static function (Blueprint $t): void { $t->increments('id'); $t->string('artifact_key')->unique(); $t->string('kind'); $t->unsignedBigInteger('maximum_size_bytes'); $t->unsignedBigInteger('size_bytes'); $t->string('sha256'); $t->string('durability_policy'); $t->unsignedTinyInteger('desired_copies'); $t->unsignedTinyInteger('required_copies'); $t->string('state'); $t->string('last_operation_id')->nullable(); $t->unsignedBigInteger('verified_at'); $t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at'); }); $schema->create('cloud_storage_replicas', static function (Blueprint $t): void { $t->increments('id'); $t->string('replica_key')->unique(); $t->unsignedBigInteger('artifact_id'); $t->unsignedBigInteger('backend_id'); $t->string('role'); $t->string('object_key'); $t->string('object_version_id')->default(''); $t->string('object_etag')->default(''); $t->unsignedBigInteger('size_bytes')->default(0); $t->string('sha256')->default(''); $t->string('state'); $t->string('publish_operation_id')->nullable(); $t->string('delete_operation_id')->nullable(); $t->unsignedInteger('delete_generation')->default(0); $t->unsignedInteger('delete_cycle_attempts')->default(0); $t->unsignedInteger('delete_presence_confirmations')->default(0); $t->unsignedBigInteger('delete_last_checked_at')->default(0); $t->unsignedBigInteger('delete_next_check_at')->default(0); $t->unsignedBigInteger('delete_attention_at')->nullable(); $t->string('lease_token')->nullable(); $t->unsignedBigInteger('lease_until')->default(0); $t->string('error_code')->default(''); $t->unsignedBigInteger('verified_at')->default(0); $t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at'); $t->unique(['artifact_id', 'backend_id']); $t->unique(['backend_id', 'object_key']); }); $schema->create('cloud_storage_operations', static function (Blueprint $t): void { $t->increments('id'); $t->string('operation_id')->unique(); $t->string('request_idempotency_key')->unique(); $t->string('request_fingerprint'); $t->unsignedBigInteger('artifact_id'); $t->unsignedBigInteger('replica_id')->nullable(); $t->unsignedBigInteger('source_backend_id')->nullable(); $t->unsignedBigInteger('target_backend_id')->nullable(); $t->string('kind'); $t->string('phase'); $t->string('outcome')->nullable(); $t->string('effect_resolution'); $t->boolean('effect_may_have_applied')->nullable(); $t->boolean('safe_to_resubmit')->nullable(); $t->unsignedInteger('attempts'); $t->string('lease_token')->nullable(); $t->unsignedBigInteger('lease_until'); $t->unsignedBigInteger('available_at'); $t->unsignedBigInteger('next_check_at'); $t->string('error_code'); $t->text('result_json')->nullable(); $t->unsignedBigInteger('terminal_at')->nullable(); $t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at'); }); $schema->create('cloud_storage_publications', static function (Blueprint $t): void { $t->increments('id'); $t->string('publication_key')->unique(); $t->unsignedBigInteger('artifact_id')->unique(); $t->unsignedBigInteger('backup_id')->nullable()->unique(); $t->string('template_import_intent_id')->nullable()->unique(); $t->string('rescue_template_id')->nullable()->unique(); $t->string('state'); $t->unsignedInteger('attempts')->default(0); $t->unsignedBigInteger('available_at')->default(0); $t->string('lease_token')->nullable(); $t->unsignedBigInteger('lease_until')->default(0); $t->string('last_error_code')->default(''); $t->unsignedBigInteger('coverage_ready_at')->nullable(); $t->unsignedBigInteger('finalized_at')->nullable(); $t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at'); }); $now = 1800000000; $insertBackend = static function ( int $id, string $state, string $health, int $tested, string $domain, int $priority, string $prefix ) use ($now): void { Capsule::table('cloud_storage_backends')->insert([ 'id' => $id, 'backend_key' => str_pad((string) $id, 32, 'b'), 'identity_sha256' => str_pad((string) $id, 64, 'c'), 'name' => 'backend-' . $id, 'driver' => 's3', 'endpoint' => 'https://storage-' . $id . '.example.test', 'region' => 'us-east-1', 'bucket' => 'bucket-' . $id, 'prefix' => $prefix, 'path_style' => 1, 'failure_domain' => $domain, 'sealed_credentials' => 'sealed', 'config_revision' => 1, 'tested_revision' => $tested, 'admin_state' => $state, 'health_state' => $health, 'write_priority' => $priority, 'consecutive_failures' => 0, 'consecutive_successes' => $health === 'healthy' ? 1 : 0, 'last_error_code' => '', 'last_tested_at' => $tested > 0 ? $now : 0, 'last_healthy_at' => $health === 'healthy' ? $now : 0, 'next_probe_at' => $now + 300, 'created_at' => $now, 'updated_at' => $now, ]); }; $insertBackend(1, 'active', 'healthy', 1, 'provider-a', 10, 'primary'); $insertBackend(2, 'draft', 'unknown', 0, 'provider-b', 20, 'secondary'); $view = CloudStorageArtifactService::beginPublication( 'backup', 'backups/late/system.qcow2', 4096, str_repeat('a', 64), 'repair-fixture-primary', hash('sha256', 'repair-fixture'), ['backup_id' => 101] ); $assert(CloudStorageReplica::where('artifact_id', $view['artifact_id'])->count() === 1, 'Unavailable second backend was unexpectedly reserved at publication time.'); CloudStorageArtifactService::markPrimaryDispatching($view['operation_id']); CloudStorageArtifactService::completePrimary( $view['operation_id'], (int) $view['artifact_id'], [ 'size_bytes' => 4096, 'sha256' => str_repeat('a', 64), 'etag' => 'primary-etag', 'version_id' => 'primary-version', ] ); CloudStoragePublicationFinalizerService::complete( 'backup', 101, (int) $view['artifact_id'], $view['operation_id'] ); Capsule::table('cloud_storage_backends')->where('id', 2)->update([ 'admin_state' => 'active', 'health_state' => 'healthy', 'tested_revision' => 1, 'last_tested_at' => $now, 'last_healthy_at' => $now, 'updated_at' => $now, ]); $repair = new CloudStorageReplicaRepairService(); $assert($repair->tick($now + 1), 'Recovered second backend did not reserve a missing replica.'); $replicas = CloudStorageReplica::where('artifact_id', $view['artifact_id']) ->orderBy('id')->get(); $copy = CloudStorageOperation::where('artifact_id', $view['artifact_id']) ->where('kind', 'copy')->firstOrFail(); $assert($replicas->count() === 2 && (int) $replicas->last()->backend_id === 2 && (string) $replicas->last()->object_key === 'secondary/backups/late/system.qcow2' && (int) $copy->source_backend_id === 1 && (int) $copy->target_backend_id === 2 && (string) $copy->phase === 'accepted' && (int) $copy->available_at === $now + 1, 'Missing-replica repair did not freeze the expected copy intent.'); $assert(!$repair->tick($now + 2) && CloudStorageReplica::where('artifact_id', $view['artifact_id'])->count() === 2 && CloudStorageOperation::where('artifact_id', $view['artifact_id']) ->where('kind', 'copy')->count() === 1, 'Repeated repair allocated duplicate replica work.'); CloudStorageArtifact::whereKey($view['artifact_id'])->update([ 'desired_copies' => 1, ]); // A healthy replica on a draining backend must not occupy forward durability // coverage. This lets maintenance evacuate to a replacement before the old // provider is taken offline. Required-two still needs a distinct domain. $insertBackend(3, 'active', 'healthy', 1, 'provider-a', 1, 'same-domain'); $insertBackend(4, 'active', 'healthy', 1, 'provider-c', 50, 'replacement'); $artifactId = Capsule::table('cloud_storage_artifacts')->insertGetId([ 'artifact_key' => str_repeat('d', 32), 'kind' => 'template', 'maximum_size_bytes' => 8192, 'size_bytes' => 8192, 'sha256' => str_repeat('e', 64), 'durability_policy' => 'mirror_required_2', 'desired_copies' => 2, 'required_copies' => 2, 'state' => 'available', 'last_operation_id' => str_repeat('1', 32), 'verified_at' => $now, 'created_at' => $now, 'updated_at' => $now, ]); $primaryReplicaId = Capsule::table('cloud_storage_replicas')->insertGetId([ 'replica_key' => str_repeat('4', 32), 'artifact_id' => $artifactId, 'backend_id' => 1, 'role' => 'primary', 'object_key' => 'primary/templates/image.qcow2', 'object_version_id' => 'v1', 'object_etag' => 'e1', 'size_bytes' => 8192, 'sha256' => str_repeat('e', 64), 'state' => 'ready', 'publish_operation_id' => str_repeat('1', 32), 'lease_until' => 0, 'error_code' => '', 'verified_at' => $now, 'created_at' => $now, 'updated_at' => $now, ]); Capsule::table('cloud_storage_replicas')->insert([ 'replica_key' => str_repeat('5', 32), 'artifact_id' => $artifactId, 'backend_id' => 2, 'role' => 'secondary', 'object_key' => 'secondary/templates/image.qcow2', 'object_version_id' => 'v2', 'object_etag' => 'e2', 'size_bytes' => 8192, 'sha256' => str_repeat('e', 64), 'state' => 'ready', 'publish_operation_id' => str_repeat('2', 32), 'lease_until' => 0, 'error_code' => '', 'verified_at' => $now, 'created_at' => $now, 'updated_at' => $now, ]); Capsule::table('cloud_storage_operations')->insert([ 'operation_id' => str_repeat('1', 32), 'request_idempotency_key' => 'template-primary-fixture', 'request_fingerprint' => str_repeat('6', 64), 'artifact_id' => $artifactId, 'replica_id' => $primaryReplicaId, 'target_backend_id' => 1, 'kind' => 'publish', 'phase' => 'terminal', 'outcome' => 'succeeded', 'effect_resolution' => 'applied', 'effect_may_have_applied' => 0, 'safe_to_resubmit' => 0, 'attempts' => 1, 'lease_until' => 0, 'available_at' => 0, 'next_check_at' => 0, 'error_code' => '', 'terminal_at' => $now, 'created_at' => $now, 'updated_at' => $now, ]); Capsule::table('cloud_storage_operations')->insert([ 'operation_id' => str_repeat('2', 32), 'request_idempotency_key' => 'template-old-copy-fixture', 'request_fingerprint' => str_repeat('7', 64), 'artifact_id' => $artifactId, 'replica_id' => CloudStorageReplica::where('artifact_id', $artifactId) ->where('backend_id', 2)->value('id'), 'source_backend_id' => 1, 'target_backend_id' => 2, 'kind' => 'copy', 'phase' => 'terminal', 'outcome' => 'failed', 'effect_resolution' => 'not_applied', 'effect_may_have_applied' => 0, 'safe_to_resubmit' => 0, 'attempts' => 1, 'lease_until' => 0, 'available_at' => 0, 'next_check_at' => 0, 'error_code' => 'provider_lost', 'terminal_at' => $now, 'created_at' => $now, 'updated_at' => $now, ]); Capsule::table('cloud_storage_publications')->insert([ 'publication_key' => str_repeat('8', 32), 'artifact_id' => $artifactId, 'template_import_intent_id' => str_repeat('9', 32), 'state' => 'waiting_protection', 'attempts' => 1, 'available_at' => 0, 'lease_until' => 0, 'last_error_code' => '', 'coverage_ready_at' => null, 'finalized_at' => null, 'created_at' => $now, 'updated_at' => $now, ]); Capsule::table('cloud_storage_backends')->where('id', 2)->update([ 'admin_state' => 'draining', 'health_state' => 'healthy', 'tested_revision' => 1, 'last_error_code' => '', 'updated_at' => $now, ]); $assert($repair->tick($now + 3), 'Replacement backend did not repair required-two coverage while the ' . 'owner publication was still waiting for protection.'); $replacement = CloudStorageReplica::where('artifact_id', $artifactId) ->where('state', 'reserved')->firstOrFail(); $assert((int) $replacement->backend_id === 4 && (string) $replacement->object_key === 'replacement/templates/image.qcow2' && CloudStorageReplica::where('artifact_id', $artifactId)->count() === 3, 'Repair counted draining evidence or selected an existing failure domain.'); $workerSource = file_get_contents(dirname(__DIR__) . '/app/Services/Cloud/CloudStorageReplicaWorkerService.php'); $repairPosition = is_string($workerSource) ? strpos($workerSource, 'CloudStorageReplicaRepairService') : false; $claimPosition = is_string($workerSource) ? strpos($workerSource, 'claimReplicaWork') : false; $assert(is_string($workerSource) && str_contains($workerSource, 'CloudStorageReplicaRepairService') && str_contains($workerSource, 'storage_replica_repair_tick_failed') && is_int($repairPosition) && is_int($claimPosition) && $repairPosition < $claimPosition, 'Replica worker does not consume the bounded repair queue.'); echo "Storage replica repair behavior passed.\n";