diff --git a/web/app/Services/Cloud/CloudStorageProtectionSummaryService.php b/web/app/Services/Cloud/CloudStorageProtectionSummaryService.php index cc08df37..41b35244 100644 --- a/web/app/Services/Cloud/CloudStorageProtectionSummaryService.php +++ b/web/app/Services/Cloud/CloudStorageProtectionSummaryService.php @@ -34,6 +34,12 @@ final class CloudStorageProtectionSummaryService ) ->where('replicas.verified_at', '>', 0); }) + ->leftJoin( + 'cloud_storage_backends as backends', + 'backends.id', + '=', + 'replicas.backend_id' + ) ->whereNotIn('artifacts.state', [ 'retiring', 'delete_attention', 'deleted', ]) @@ -49,7 +55,20 @@ final class CloudStorageProtectionSummaryService 'artifacts.required_copies', 'artifacts.created_at', CloudStorageArtifact::query()->getQuery()->raw( - 'COUNT(replicas.id) AS ready_copies' + "COUNT(DISTINCT CASE WHEN " + . "backends.admin_state IN ('active','read_only') " + . "AND backends.health_state IN " + . "('healthy','degraded','unknown') " + . 'THEN replicas.id END) AS ready_copies' + ), + CloudStorageArtifact::query()->getQuery()->raw( + "COUNT(DISTINCT CASE WHEN " + . "backends.admin_state IN ('active','read_only') " + . "AND backends.health_state IN " + . "('healthy','degraded','unknown') " + . "AND TRIM(backends.failure_domain) <> '' " + . 'THEN LOWER(TRIM(backends.failure_domain)) END) ' + . 'AS ready_domains' ), ]); @@ -61,13 +80,16 @@ final class CloudStorageProtectionSummaryService $ready = (int) ($row->ready_copies ?? 0); $desired = (int) $row->desired_copies; $required = (int) $row->required_copies; - if ($ready >= $desired) { + $coverage = $required > 1 + ? min($ready, (int) ($row->ready_domains ?? 0)) + : $ready; + if ($coverage >= $desired) { $protected++; continue; } $backlog++; - if ($ready >= $required) { + if ($coverage >= $required) { $degraded++; } $createdAt = (int) $row->created_at; diff --git a/web/app/Services/Cloud/CloudStorageReplicaRepairService.php b/web/app/Services/Cloud/CloudStorageReplicaRepairService.php index ebfb9dae..c0c648b3 100644 --- a/web/app/Services/Cloud/CloudStorageReplicaRepairService.php +++ b/web/app/Services/Cloud/CloudStorageReplicaRepairService.php @@ -44,7 +44,11 @@ final class CloudStorageReplicaRepairService ); }) ->whereRaw( - "(SELECT COUNT(*) FROM cloud_storage_replicas repair_coverage " + "(SELECT COUNT(DISTINCT CASE " + . "WHEN cloud_storage_artifacts.required_copies > 1 " + . "THEN LOWER(TRIM(repair_backend.failure_domain)) " + . "ELSE repair_coverage.id END) " + . "FROM cloud_storage_replicas repair_coverage " . "INNER JOIN cloud_storage_backends repair_backend " . "ON repair_backend.id = repair_coverage.backend_id " . "WHERE repair_coverage.artifact_id = cloud_storage_artifacts.id " @@ -118,8 +122,11 @@ final class CloudStorageReplicaRepairService ->lockForUpdate()->get()->keyBy('id'); $ready = []; + $required = (int) $artifact->required_copies; $coveredReady = 0; $pending = 0; + $coveredReadyDomains = []; + $pendingDomains = []; $represented = []; foreach ($replicas as $replica) { $backendId = (int) $replica->backend_id; @@ -133,7 +140,13 @@ final class CloudStorageReplicaRepairService $ready[] = [$replica, $backend]; if (in_array((string) $backend->admin_state, ['active', 'read_only'], true)) { - $coveredReady++; + if ($required > 1) { + $coveredReadyDomains[ + self::domain($backend) + ] = true; + } else { + $coveredReady++; + } } } } elseif (in_array((string) $replica->state, @@ -145,10 +158,21 @@ final class CloudStorageReplicaRepairService && (int) $backend->config_revision > 0 && (int) $backend->tested_revision === (int) $backend->config_revision) { - $pending++; + if ($required > 1) { + $pendingDomains[self::domain($backend)] = true; + } else { + $pending++; + } } } } + if ($required > 1) { + $coveredReady = count($coveredReadyDomains); + $pending = count(array_diff_key( + $pendingDomains, + $coveredReadyDomains + )); + } $desired = (int) $artifact->desired_copies; if ($desired < 1 || $desired > 2 || $coveredReady >= $desired diff --git a/web/tests/CloudStorageProtectionSummaryServiceTest.php b/web/tests/CloudStorageProtectionSummaryServiceTest.php index 7f23d319..2cba54b3 100644 --- a/web/tests/CloudStorageProtectionSummaryServiceTest.php +++ b/web/tests/CloudStorageProtectionSummaryServiceTest.php @@ -28,11 +28,18 @@ $db->schema()->create('cloud_storage_artifacts', static function (Blueprint $t): $db->schema()->create('cloud_storage_replicas', static function (Blueprint $t): void { $t->increments('id'); $t->unsignedBigInteger('artifact_id'); + $t->unsignedBigInteger('backend_id'); $t->unsignedBigInteger('size_bytes'); $t->string('sha256'); $t->unsignedBigInteger('verified_at'); $t->string('state'); }); +$db->schema()->create('cloud_storage_backends', static function (Blueprint $t): void { + $t->increments('id'); + $t->string('admin_state'); + $t->string('health_state'); + $t->string('failure_domain'); +}); $now = 2_000; $artifacts = [ @@ -50,23 +57,35 @@ $artifacts = [ 'state' => 'retiring', 'created_at' => 1_000], ]; Capsule::table('cloud_storage_artifacts')->insert($artifacts); +Capsule::table('cloud_storage_backends')->insert([ + ['id' => 1, 'admin_state' => 'active', 'health_state' => 'healthy', + 'failure_domain' => 'provider-a'], + ['id' => 2, 'admin_state' => 'active', 'health_state' => 'healthy', + 'failure_domain' => 'provider-b'], + ['id' => 3, 'admin_state' => 'draining', 'health_state' => 'healthy', + 'failure_domain' => 'provider-c'], + ['id' => 4, 'admin_state' => 'active', 'health_state' => 'unreachable', + 'failure_domain' => 'provider-d'], + ['id' => 5, 'admin_state' => 'read_only', 'health_state' => 'degraded', + 'failure_domain' => 'provider-a'], +]); Capsule::table('cloud_storage_replicas')->insert([ - ['artifact_id' => 1, 'size_bytes' => 100, + ['artifact_id' => 1, 'backend_id' => 1, 'size_bytes' => 100, 'sha256' => str_repeat('a', 64), 'verified_at' => 1, 'state' => 'ready'], - ['artifact_id' => 1, 'size_bytes' => 100, + ['artifact_id' => 1, 'backend_id' => 2, 'size_bytes' => 100, 'sha256' => str_repeat('a', 64), 'verified_at' => 2, 'state' => 'ready'], - ['artifact_id' => 2, 'size_bytes' => 200, + ['artifact_id' => 2, 'backend_id' => 1, 'size_bytes' => 200, 'sha256' => str_repeat('b', 64), 'verified_at' => 3, 'state' => 'ready'], - ['artifact_id' => 2, 'size_bytes' => 0, 'sha256' => '', + ['artifact_id' => 2, 'backend_id' => 2, 'size_bytes' => 0, 'sha256' => '', 'verified_at' => 0, 'state' => 'copying'], // State alone must not count a corrupted or stale identity as coverage. - ['artifact_id' => 3, 'size_bytes' => 301, + ['artifact_id' => 3, 'backend_id' => 1, 'size_bytes' => 301, 'sha256' => str_repeat('c', 64), 'verified_at' => 4, 'state' => 'ready'], - ['artifact_id' => 4, 'size_bytes' => 400, + ['artifact_id' => 4, 'backend_id' => 3, 'size_bytes' => 400, 'sha256' => str_repeat('d', 64), 'verified_at' => 5, 'state' => 'ready'], ]); @@ -79,6 +98,16 @@ $assert($summary === [ 'oldest_replication_age_seconds' => 400, ], 'Replica coverage summary does not match verified ready replicas.'); +Capsule::table('cloud_storage_backends')->where('id', 2) + ->update(['admin_state' => 'draining']); +$draining = CloudStorageProtectionSummaryService::summary($now); +$assert($draining['protected_artifact_count'] === 0 + && $draining['degraded_artifact_count'] === 2 + && $draining['replication_backlog_count'] === 3, + 'A draining backend still counted toward forward protection.'); +Capsule::table('cloud_storage_backends')->where('id', 2) + ->update(['admin_state' => 'active']); + Capsule::table('cloud_storage_replicas')->where('artifact_id', 2) ->where('state', 'copying')->update([ 'state' => 'ready', 'size_bytes' => 200, @@ -96,6 +125,22 @@ $assert($recovered['protected_artifact_count'] === 2 && $recovered['oldest_replication_age_seconds'] === 400, 'Summary did not converge when replicas became ready.'); +Capsule::table('cloud_storage_replicas')->insert([ + 'artifact_id' => 3, 'backend_id' => 5, 'size_bytes' => 300, + 'sha256' => str_repeat('c', 64), 'verified_at' => 8, + 'state' => 'ready', +]); +$sameDomain = CloudStorageProtectionSummaryService::summary($now); +$assert($sameDomain['protected_artifact_count'] === 2 + && $sameDomain['replication_backlog_count'] === 1, + 'Required-two protection accepted two replicas in one failure domain.'); +Capsule::table('cloud_storage_backends')->where('id', 5) + ->update(['failure_domain' => 'provider-e']); +$distinctDomains = CloudStorageProtectionSummaryService::summary($now); +$assert($distinctDomains['protected_artifact_count'] === 3 + && $distinctDomains['replication_backlog_count'] === 0, + 'Required-two protection did not recognize two explicit failure domains.'); + Capsule::table('cloud_storage_artifacts')->whereIn('id', [1, 2, 3]) ->update(['state' => 'deleted']); $empty = CloudStorageProtectionSummaryService::summary($now);