diff --git a/web/app/Services/Cloud/CloudStorageReplicaRepairService.php b/web/app/Services/Cloud/CloudStorageReplicaRepairService.php new file mode 100644 index 00000000..472e7b22 --- /dev/null +++ b/web/app/Services/Cloud/CloudStorageReplicaRepairService.php @@ -0,0 +1,368 @@ +whereNotIn('state', [ + 'retiring', 'delete_attention', 'deleted', + ]) + ->whereIn('id', static function ($query): void { + $query->select('artifact_id') + ->from('cloud_storage_publications') + ->where('state', 'completed'); + }) + ->whereRaw( + "(SELECT COUNT(*) FROM cloud_storage_replicas repair_coverage " + . "INNER JOIN cloud_storage_backends repair_backend " + . "ON repair_backend.id = repair_coverage.backend_id " + . "WHERE repair_coverage.artifact_id = cloud_storage_artifacts.id " + . "AND ((repair_coverage.state = 'ready' " + . "AND repair_coverage.size_bytes = cloud_storage_artifacts.size_bytes " + . "AND repair_coverage.sha256 = cloud_storage_artifacts.sha256 " + . "AND repair_coverage.verified_at > 0 " + . "AND repair_backend.admin_state <> 'retired' " + . "AND repair_backend.health_state IN ('healthy','degraded','unknown')) " + . "OR (repair_coverage.state IN ('reserved','publishing','copying') " + . "AND repair_backend.admin_state = 'active' " + . "AND repair_backend.health_state = 'healthy' " + . "AND repair_backend.tested_revision = repair_backend.config_revision))) " + . '< cloud_storage_artifacts.desired_copies' + ) + ->orderBy('id'); + + // Page through every SQL-level coverage gap. A hard first-page limit + // would let an unrecoverable early artifact starve all later repairs. + foreach ($candidates->lazyById(100, 'id') as $artifact) { + if ($this->reserveForArtifact((int) $artifact->id, $now)) { + return true; + } + } + return false; + } + + private function reserveForArtifact(int $artifactId, int $now): bool + { + $connection = (new CloudStorageArtifact())->getConnection(); + return $connection->transaction(static function () use ( + $artifactId, + $now + ): bool { + // Match primary settlement's lock order. + $primaryOperation = CloudStorageOperation::query() + ->where('artifact_id', $artifactId) + ->where('kind', 'publish')->orderBy('id') + ->lockForUpdate()->first(); + if (!$primaryOperation instanceof CloudStorageOperation + || (string) $primaryOperation->phase !== 'terminal' + || (string) $primaryOperation->outcome !== 'succeeded') { + return false; + } + CloudStorageOperation::query() + ->where('artifact_id', $artifactId) + ->where('kind', 'copy')->orderBy('id') + ->lockForUpdate()->get(); + $artifact = CloudStorageArtifact::query()->whereKey($artifactId) + ->lockForUpdate()->first(); + if (!$artifact instanceof CloudStorageArtifact + || in_array((string) $artifact->state, + ['retiring', 'delete_attention', 'deleted'], true)) { + return false; + } + $replicas = CloudStorageReplica::query() + ->where('artifact_id', $artifactId)->orderBy('id') + ->lockForUpdate()->get()->all(); + $publication = CloudStoragePublication::query() + ->where('artifact_id', $artifactId) + ->lockForUpdate()->first(); + if (!$publication instanceof CloudStoragePublication + || (string) $publication->state !== 'completed') { + return false; + } + $backends = CloudStorageBackend::query()->orderBy('id') + ->lockForUpdate()->get()->keyBy('id'); + + $ready = []; + $pending = 0; + $represented = []; + foreach ($replicas as $replica) { + $backendId = (int) $replica->backend_id; + $represented[$backendId] = true; + if (self::isExactReady($artifact, $replica)) { + $backend = $backends->get($backendId); + if ($backend instanceof CloudStorageBackend + && (string) $backend->admin_state !== 'retired' + && in_array((string) $backend->health_state, + ['healthy', 'degraded', 'unknown'], true)) { + $ready[] = [$replica, $backend]; + } + } elseif (in_array((string) $replica->state, + ['reserved', 'publishing', 'copying'], true)) { + $backend = $backends->get($backendId); + if ($backend instanceof CloudStorageBackend + && (string) $backend->admin_state === 'active' + && (string) $backend->health_state === 'healthy' + && (int) $backend->config_revision > 0 + && (int) $backend->tested_revision + === (int) $backend->config_revision) { + $pending++; + } + } + } + $desired = (int) $artifact->desired_copies; + if ($desired < 1 || $desired > 2 + || count($ready) >= $desired + || count($ready) + $pending >= $desired + || $ready === []) { + return false; + } + + usort($ready, static function (array $left, array $right): int { + $leftBackend = $left[1]; + $rightBackend = $right[1]; + return [ + self::healthRank((string) $leftBackend->health_state), + self::adminRank((string) $leftBackend->admin_state), + (int) $leftBackend->write_priority, + (int) $leftBackend->id, + ] <=> [ + self::healthRank((string) $rightBackend->health_state), + self::adminRank((string) $rightBackend->admin_state), + (int) $rightBackend->write_priority, + (int) $rightBackend->id, + ]; + }); + $source = $ready[0][0]; + $sourceBackend = $ready[0][1]; + $suffix = self::objectSuffixFromSource( + (string) $sourceBackend->prefix, + (string) $source->object_key + ); + if ($suffix === null) { + return false; + } + + $readyDomains = []; + foreach ($ready as [, $backend]) { + $readyDomains[self::domain($backend)] = true; + } + $candidates = []; + foreach ($backends as $backend) { + if (!$backend instanceof CloudStorageBackend + || isset($represented[(int) $backend->id]) + || (string) $backend->admin_state !== 'active' + || (string) $backend->health_state !== 'healthy' + || (int) $backend->config_revision < 1 + || (int) $backend->tested_revision + !== (int) $backend->config_revision) { + continue; + } + $isNewDomain = !isset($readyDomains[self::domain($backend)]); + if ((int) $artifact->required_copies > 1 && !$isNewDomain) { + continue; + } + $candidates[] = [$backend, $isNewDomain ? 0 : 1]; + } + usort($candidates, static fn (array $left, array $right): int => [ + $left[1], + (int) $left[0]->write_priority, + (int) $left[0]->id, + ] <=> [ + $right[1], + (int) $right[0]->write_priority, + (int) $right[0]->id, + ]); + if ($candidates === []) { + return false; + } + $target = $candidates[0][0]; + $targetKey = self::objectKey((string) $target->prefix, $suffix); + $collision = CloudStorageReplica::query() + ->where('backend_id', (int) $target->id) + ->where('object_key', $targetKey) + ->lockForUpdate()->first(); + if ($collision instanceof CloudStorageReplica) { + return false; + } + + self::createRepair( + $artifact, + $source, + $sourceBackend, + $target, + $targetKey, + $now + ); + return true; + }); + } + + private static function createRepair( + CloudStorageArtifact $artifact, + CloudStorageReplica $source, + CloudStorageBackend $sourceBackend, + CloudStorageBackend $target, + string $targetKey, + int $now + ): void { + $operationId = bin2hex(random_bytes(16)); + $replica = new CloudStorageReplica(); + $replica->replica_key = bin2hex(random_bytes(16)); + $replica->artifact_id = (int) $artifact->id; + $replica->backend_id = (int) $target->id; + $replica->role = 'secondary'; + $replica->object_key = $targetKey; + $replica->object_version_id = ''; + $replica->object_etag = ''; + $replica->size_bytes = 0; + $replica->sha256 = ''; + $replica->state = 'reserved'; + $replica->publish_operation_id = $operationId; + $replica->delete_operation_id = null; + $replica->delete_generation = 0; + $replica->delete_cycle_attempts = 0; + $replica->delete_presence_confirmations = 0; + $replica->delete_last_checked_at = 0; + $replica->delete_next_check_at = 0; + $replica->delete_attention_at = null; + $replica->lease_token = null; + $replica->lease_until = 0; + $replica->error_code = ''; + $replica->verified_at = 0; + $replica->created_at = $now; + $replica->updated_at = $now; + $replica->save(); + + $operation = new CloudStorageOperation(); + $operation->operation_id = $operationId; + $operation->request_idempotency_key = + 'storage-repair-copy-' . substr(hash('sha256', + (string) $artifact->artifact_key . '|' + . (string) $target->id), 0, 48); + $operation->request_fingerprint = hash('sha256', implode('|', [ + 'repair-copy-v1', + (string) $artifact->artifact_key, + (string) $sourceBackend->id, + (string) $source->object_key, + (string) $source->object_version_id, + (string) $target->id, + $targetKey, + (string) $artifact->size_bytes, + (string) $artifact->sha256, + ])); + $operation->artifact_id = (int) $artifact->id; + $operation->replica_id = (int) $replica->id; + $operation->source_backend_id = (int) $sourceBackend->id; + $operation->target_backend_id = (int) $target->id; + $operation->kind = 'copy'; + $operation->phase = 'accepted'; + $operation->outcome = null; + $operation->effect_resolution = 'not_applicable'; + $operation->effect_may_have_applied = null; + $operation->safe_to_resubmit = null; + $operation->attempts = 0; + $operation->lease_token = null; + $operation->lease_until = 0; + $operation->available_at = $now; + $operation->next_check_at = 0; + $operation->error_code = ''; + $operation->result_json = null; + $operation->terminal_at = null; + $operation->created_at = $now; + $operation->updated_at = $now; + $operation->save(); + } + + private static function isExactReady( + CloudStorageArtifact $artifact, + CloudStorageReplica $replica + ): bool { + return (string) $replica->state === 'ready' + && (int) $replica->size_bytes === (int) $artifact->size_bytes + && hash_equals((string) $artifact->sha256, + (string) $replica->sha256) + && (int) $replica->verified_at > 0; + } + + private static function objectSuffixFromSource( + string $prefix, + string $objectKey + ): ?string { + $prefix = trim($prefix, '/'); + if ($prefix !== '') { + $start = $prefix . '/'; + if (!str_starts_with($objectKey, $start)) { + return null; + } + $objectKey = substr($objectKey, strlen($start)); + } + if ($objectKey === '' || strlen($objectKey) > 512 + || preg_match('/^[A-Za-z0-9][A-Za-z0-9._\/-]*$/D', + $objectKey) !== 1 + || str_contains($objectKey, '//')) { + return null; + } + foreach (explode('/', $objectKey) as $segment) { + if ($segment === '.' || $segment === '..') { + return null; + } + } + return $objectKey; + } + + private static function objectKey(string $prefix, string $suffix): string + { + $prefix = trim($prefix, '/'); + $key = ($prefix === '' ? '' : $prefix . '/') . $suffix; + if (strlen($key) > 512) { + throw new CloudError('对象存储制品路径过长', 422); + } + return $key; + } + + private static function domain(CloudStorageBackend $backend): string + { + $domain = trim((string) $backend->failure_domain); + return $domain !== '' ? $domain : 'backend:' . $backend->id; + } + + private static function healthRank(string $state): int + { + return match ($state) { + 'healthy' => 0, + 'degraded' => 1, + 'unknown' => 2, + default => 3, + }; + } + + private static function adminRank(string $state): int + { + return match ($state) { + 'active' => 0, + 'read_only' => 1, + 'draining' => 2, + default => 3, + }; + } +} diff --git a/web/app/Services/Cloud/CloudStorageReplicaWorkerService.php b/web/app/Services/Cloud/CloudStorageReplicaWorkerService.php index af12d6b2..a7738527 100644 --- a/web/app/Services/Cloud/CloudStorageReplicaWorkerService.php +++ b/web/app/Services/Cloud/CloudStorageReplicaWorkerService.php @@ -32,7 +32,9 @@ final class CloudStorageReplicaWorkerService ?callable $runtimeResolver = null, ?callable $clock = null, private readonly ?CloudStoragePublicationOutboxService - $publicationOutbox = null + $publicationOutbox = null, + private readonly ?CloudStorageReplicaRepairService + $replicaRepair = null ) { $this->runtimeResolver = $runtimeResolver === null ? static fn (int $backendId, bool $healthy): array => @@ -66,6 +68,18 @@ final class CloudStorageReplicaWorkerService $this->errors[] = 'storage_publication_outbox_tick_failed'; } + // Repair only reserves one local, durable copy intent. It performs no + // provider I/O, so a backend that became writable can start healing + // old single-copy artifacts without delaying existing remote work. + $repairWorked = false; + try { + $repair = $this->replicaRepair + ?? new CloudStorageReplicaRepairService(); + $repairWorked = $repair->tick($now); + } catch (\Throwable) { + $this->errors[] = 'storage_replica_repair_tick_failed'; + } + $work = CloudStorageArtifactService::claimReplicaWork($now); $replicaWorked = $work !== null; if ($work !== null) { @@ -92,7 +106,7 @@ final class CloudStorageReplicaWorkerService } } $this->errors = array_values(array_unique($this->errors)); - return $replicaWorked || $publicationWorked; + return $replicaWorked || $publicationWorked || $repairWorked; } /** @return list */ diff --git a/web/tests/CloudStoragePublicationOutboxServiceTest.php b/web/tests/CloudStoragePublicationOutboxServiceTest.php index 12749d06..2a28fc6f 100644 --- a/web/tests/CloudStoragePublicationOutboxServiceTest.php +++ b/web/tests/CloudStoragePublicationOutboxServiceTest.php @@ -133,6 +133,13 @@ $db->schema()->create('cloud_storage_backends', static function ( ): void { $table->increments('id'); $table->string('name'); + $table->string('prefix')->default(''); + $table->string('failure_domain')->default(''); + $table->unsignedBigInteger('config_revision')->default(1); + $table->unsignedBigInteger('tested_revision')->default(0); + $table->string('admin_state')->default('draft'); + $table->string('health_state')->default('unknown'); + $table->unsignedInteger('write_priority')->default(0); }); $clock = 2000000000; diff --git a/web/tests/CloudStorageReplicaRepairServiceTest.php b/web/tests/CloudStorageReplicaRepairServiceTest.php new file mode 100644 index 00000000..3e62cff7 --- /dev/null +++ b/web/tests/CloudStorageReplicaRepairServiceTest.php @@ -0,0 +1,281 @@ +addConnection(['driver' => 'sqlite', 'database' => ':memory:']); +$db->setAsGlobal(); +$db->bootEloquent(); +$schema = $db->schema(); +$schema->create('cloud_storage_backends', static function (Blueprint $t): void { + $t->increments('id'); + $t->string('backend_key')->unique(); $t->string('identity_sha256')->unique(); + $t->string('name'); $t->string('driver'); $t->string('endpoint'); + $t->string('region'); $t->string('bucket'); $t->string('prefix'); + $t->boolean('path_style'); $t->string('failure_domain'); + $t->text('sealed_credentials'); $t->unsignedBigInteger('config_revision'); + $t->unsignedBigInteger('tested_revision'); $t->string('admin_state'); + $t->string('health_state'); $t->unsignedInteger('write_priority'); + $t->unsignedInteger('consecutive_failures')->default(0); + $t->unsignedInteger('consecutive_successes')->default(0); + $t->string('last_error_code')->default(''); + $t->unsignedBigInteger('last_tested_at')->default(0); + $t->unsignedBigInteger('last_healthy_at')->default(0); + $t->unsignedBigInteger('next_probe_at')->default(0); + $t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at'); +}); +$schema->create('cloud_storage_artifacts', static function (Blueprint $t): void { + $t->increments('id'); $t->string('artifact_key')->unique(); + $t->string('kind'); $t->unsignedBigInteger('maximum_size_bytes'); + $t->unsignedBigInteger('size_bytes'); $t->string('sha256'); + $t->string('durability_policy'); $t->unsignedTinyInteger('desired_copies'); + $t->unsignedTinyInteger('required_copies'); $t->string('state'); + $t->string('last_operation_id')->nullable(); + $t->unsignedBigInteger('verified_at'); + $t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at'); +}); +$schema->create('cloud_storage_replicas', static function (Blueprint $t): void { + $t->increments('id'); $t->string('replica_key')->unique(); + $t->unsignedBigInteger('artifact_id'); $t->unsignedBigInteger('backend_id'); + $t->string('role'); $t->string('object_key'); + $t->string('object_version_id')->default(''); + $t->string('object_etag')->default(''); + $t->unsignedBigInteger('size_bytes')->default(0); + $t->string('sha256')->default(''); $t->string('state'); + $t->string('publish_operation_id')->nullable(); + $t->string('delete_operation_id')->nullable(); + $t->unsignedInteger('delete_generation')->default(0); + $t->unsignedInteger('delete_cycle_attempts')->default(0); + $t->unsignedInteger('delete_presence_confirmations')->default(0); + $t->unsignedBigInteger('delete_last_checked_at')->default(0); + $t->unsignedBigInteger('delete_next_check_at')->default(0); + $t->unsignedBigInteger('delete_attention_at')->nullable(); + $t->string('lease_token')->nullable(); + $t->unsignedBigInteger('lease_until')->default(0); + $t->string('error_code')->default(''); + $t->unsignedBigInteger('verified_at')->default(0); + $t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at'); + $t->unique(['artifact_id', 'backend_id']); + $t->unique(['backend_id', 'object_key']); +}); +$schema->create('cloud_storage_operations', static function (Blueprint $t): void { + $t->increments('id'); $t->string('operation_id')->unique(); + $t->string('request_idempotency_key')->unique(); + $t->string('request_fingerprint'); $t->unsignedBigInteger('artifact_id'); + $t->unsignedBigInteger('replica_id')->nullable(); + $t->unsignedBigInteger('source_backend_id')->nullable(); + $t->unsignedBigInteger('target_backend_id')->nullable(); + $t->string('kind'); $t->string('phase'); $t->string('outcome')->nullable(); + $t->string('effect_resolution'); + $t->boolean('effect_may_have_applied')->nullable(); + $t->boolean('safe_to_resubmit')->nullable(); + $t->unsignedInteger('attempts'); $t->string('lease_token')->nullable(); + $t->unsignedBigInteger('lease_until'); $t->unsignedBigInteger('available_at'); + $t->unsignedBigInteger('next_check_at'); $t->string('error_code'); + $t->text('result_json')->nullable(); $t->unsignedBigInteger('terminal_at')->nullable(); + $t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at'); +}); +$schema->create('cloud_storage_publications', static function (Blueprint $t): void { + $t->increments('id'); $t->string('publication_key')->unique(); + $t->unsignedBigInteger('artifact_id')->unique(); + $t->unsignedBigInteger('backup_id')->nullable()->unique(); + $t->string('template_import_intent_id')->nullable()->unique(); + $t->string('rescue_template_id')->nullable()->unique(); + $t->string('state'); $t->unsignedInteger('attempts')->default(0); + $t->unsignedBigInteger('available_at')->default(0); + $t->string('lease_token')->nullable(); + $t->unsignedBigInteger('lease_until')->default(0); + $t->string('last_error_code')->default(''); + $t->unsignedBigInteger('coverage_ready_at')->nullable(); + $t->unsignedBigInteger('finalized_at')->nullable(); + $t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at'); +}); + +$now = 1800000000; +$insertBackend = static function ( + int $id, + string $state, + string $health, + int $tested, + string $domain, + int $priority, + string $prefix +) use ($now): void { + Capsule::table('cloud_storage_backends')->insert([ + 'id' => $id, 'backend_key' => str_pad((string) $id, 32, 'b'), + 'identity_sha256' => str_pad((string) $id, 64, 'c'), + 'name' => 'backend-' . $id, 'driver' => 's3', + 'endpoint' => 'https://storage-' . $id . '.example.test', + 'region' => 'us-east-1', 'bucket' => 'bucket-' . $id, + 'prefix' => $prefix, 'path_style' => 1, + 'failure_domain' => $domain, 'sealed_credentials' => 'sealed', + 'config_revision' => 1, 'tested_revision' => $tested, + 'admin_state' => $state, 'health_state' => $health, + 'write_priority' => $priority, 'consecutive_failures' => 0, + 'consecutive_successes' => $health === 'healthy' ? 1 : 0, + 'last_error_code' => '', 'last_tested_at' => $tested > 0 ? $now : 0, + 'last_healthy_at' => $health === 'healthy' ? $now : 0, + 'next_probe_at' => $now + 300, + 'created_at' => $now, 'updated_at' => $now, + ]); +}; +$insertBackend(1, 'active', 'healthy', 1, 'provider-a', 10, 'primary'); +$insertBackend(2, 'draft', 'unknown', 0, 'provider-b', 20, 'secondary'); + +$view = CloudStorageArtifactService::beginPublication( + 'backup', 'backups/late/system.qcow2', 4096, str_repeat('a', 64), + 'repair-fixture-primary', hash('sha256', 'repair-fixture'), + ['backup_id' => 101] +); +$assert(CloudStorageReplica::where('artifact_id', $view['artifact_id'])->count() === 1, + 'Unavailable second backend was unexpectedly reserved at publication time.'); +CloudStorageArtifactService::markPrimaryDispatching($view['operation_id']); +CloudStorageArtifactService::completePrimary( + $view['operation_id'], (int) $view['artifact_id'], [ + 'size_bytes' => 4096, 'sha256' => str_repeat('a', 64), + 'etag' => 'primary-etag', 'version_id' => 'primary-version', + ] +); +CloudStoragePublicationFinalizerService::complete( + 'backup', 101, (int) $view['artifact_id'], $view['operation_id'] +); +Capsule::table('cloud_storage_backends')->where('id', 2)->update([ + 'admin_state' => 'active', 'health_state' => 'healthy', + 'tested_revision' => 1, 'last_tested_at' => $now, + 'last_healthy_at' => $now, 'updated_at' => $now, +]); + +$repair = new CloudStorageReplicaRepairService(); +$assert($repair->tick($now + 1), + 'Recovered second backend did not reserve a missing replica.'); +$replicas = CloudStorageReplica::where('artifact_id', $view['artifact_id']) + ->orderBy('id')->get(); +$copy = CloudStorageOperation::where('artifact_id', $view['artifact_id']) + ->where('kind', 'copy')->firstOrFail(); +$assert($replicas->count() === 2 + && (int) $replicas->last()->backend_id === 2 + && (string) $replicas->last()->object_key + === 'secondary/backups/late/system.qcow2' + && (int) $copy->source_backend_id === 1 + && (int) $copy->target_backend_id === 2 + && (string) $copy->phase === 'accepted' + && (int) $copy->available_at === $now + 1, + 'Missing-replica repair did not freeze the expected copy intent.'); +$assert(!$repair->tick($now + 2) + && CloudStorageReplica::where('artifact_id', $view['artifact_id'])->count() === 2 + && CloudStorageOperation::where('artifact_id', $view['artifact_id']) + ->where('kind', 'copy')->count() === 1, + 'Repeated repair allocated duplicate replica work.'); +CloudStorageArtifact::whereKey($view['artifact_id'])->update([ + 'desired_copies' => 1, +]); + +// A failed old replica must not occupy durability coverage. For a required-two +// artifact, replacement also must use a failure domain distinct from the only +// exact ready source. +$insertBackend(3, 'active', 'healthy', 1, 'provider-a', 1, 'same-domain'); +$insertBackend(4, 'active', 'healthy', 1, 'provider-c', 50, 'replacement'); +$artifactId = Capsule::table('cloud_storage_artifacts')->insertGetId([ + 'artifact_key' => str_repeat('d', 32), 'kind' => 'template', + 'maximum_size_bytes' => 8192, 'size_bytes' => 8192, + 'sha256' => str_repeat('e', 64), + 'durability_policy' => 'mirror_required_2', + 'desired_copies' => 2, 'required_copies' => 2, + 'state' => 'protected', 'last_operation_id' => str_repeat('1', 32), + 'verified_at' => $now, 'created_at' => $now, 'updated_at' => $now, +]); +$primaryReplicaId = Capsule::table('cloud_storage_replicas')->insertGetId([ + 'replica_key' => str_repeat('4', 32), 'artifact_id' => $artifactId, + 'backend_id' => 1, 'role' => 'primary', + 'object_key' => 'primary/templates/image.qcow2', + 'object_version_id' => 'v1', 'object_etag' => 'e1', + 'size_bytes' => 8192, 'sha256' => str_repeat('e', 64), + 'state' => 'ready', 'publish_operation_id' => str_repeat('1', 32), + 'lease_until' => 0, 'error_code' => '', 'verified_at' => $now, + 'created_at' => $now, 'updated_at' => $now, +]); +Capsule::table('cloud_storage_replicas')->insert([ + 'replica_key' => str_repeat('5', 32), 'artifact_id' => $artifactId, + 'backend_id' => 2, 'role' => 'secondary', + 'object_key' => 'secondary/templates/image.qcow2', + 'object_version_id' => 'v2', 'object_etag' => 'e2', + 'size_bytes' => 8192, 'sha256' => str_repeat('e', 64), + 'state' => 'ready', 'publish_operation_id' => str_repeat('2', 32), + 'lease_until' => 0, 'error_code' => '', 'verified_at' => $now, + 'created_at' => $now, 'updated_at' => $now, +]); +Capsule::table('cloud_storage_operations')->insert([ + 'operation_id' => str_repeat('1', 32), + 'request_idempotency_key' => 'template-primary-fixture', + 'request_fingerprint' => str_repeat('6', 64), + 'artifact_id' => $artifactId, 'replica_id' => $primaryReplicaId, + 'target_backend_id' => 1, 'kind' => 'publish', 'phase' => 'terminal', + 'outcome' => 'succeeded', 'effect_resolution' => 'applied', + 'effect_may_have_applied' => 0, 'safe_to_resubmit' => 0, + 'attempts' => 1, 'lease_until' => 0, 'available_at' => 0, + 'next_check_at' => 0, 'error_code' => '', + 'terminal_at' => $now, 'created_at' => $now, 'updated_at' => $now, +]); +Capsule::table('cloud_storage_operations')->insert([ + 'operation_id' => str_repeat('2', 32), + 'request_idempotency_key' => 'template-old-copy-fixture', + 'request_fingerprint' => str_repeat('7', 64), + 'artifact_id' => $artifactId, + 'replica_id' => CloudStorageReplica::where('artifact_id', $artifactId) + ->where('backend_id', 2)->value('id'), + 'source_backend_id' => 1, 'target_backend_id' => 2, + 'kind' => 'copy', 'phase' => 'terminal', 'outcome' => 'failed', + 'effect_resolution' => 'not_applied', 'effect_may_have_applied' => 0, + 'safe_to_resubmit' => 0, 'attempts' => 1, 'lease_until' => 0, + 'available_at' => 0, 'next_check_at' => 0, + 'error_code' => 'provider_lost', 'terminal_at' => $now, + 'created_at' => $now, 'updated_at' => $now, +]); +Capsule::table('cloud_storage_publications')->insert([ + 'publication_key' => str_repeat('8', 32), 'artifact_id' => $artifactId, + 'template_import_intent_id' => str_repeat('9', 32), + 'state' => 'completed', 'attempts' => 1, 'available_at' => 0, + 'lease_until' => 0, 'last_error_code' => '', + 'coverage_ready_at' => $now, 'finalized_at' => $now, + 'created_at' => $now, 'updated_at' => $now, +]); +Capsule::table('cloud_storage_backends')->where('id', 2)->update([ + 'admin_state' => 'read_only', 'health_state' => 'unreachable', + 'tested_revision' => 0, 'last_error_code' => 'endpoint_unreachable', + 'updated_at' => $now, +]); + +$assert($repair->tick($now + 3), + 'Replacement backend did not repair unreachable required-two coverage.'); +$replacement = CloudStorageReplica::where('artifact_id', $artifactId) + ->where('state', 'reserved')->firstOrFail(); +$assert((int) $replacement->backend_id === 4 + && (string) $replacement->object_key + === 'replacement/templates/image.qcow2' + && CloudStorageReplica::where('artifact_id', $artifactId)->count() === 3, + 'Repair counted degraded evidence or selected the source failure domain.'); + +$workerSource = file_get_contents(dirname(__DIR__) + . '/app/Services/Cloud/CloudStorageReplicaWorkerService.php'); +$assert(is_string($workerSource) + && str_contains($workerSource, 'CloudStorageReplicaRepairService') + && str_contains($workerSource, 'storage_replica_repair_tick_failed'), + 'Replica worker does not consume the bounded repair queue.'); + +echo "Storage replica repair behavior passed.\n";