diff --git a/web/app/Services/Cloud/CloudStorageReplicaRepairService.php b/web/app/Services/Cloud/CloudStorageReplicaRepairService.php index 5d3c61e3..bcb9de2c 100644 --- a/web/app/Services/Cloud/CloudStorageReplicaRepairService.php +++ b/web/app/Services/Cloud/CloudStorageReplicaRepairService.php @@ -52,7 +52,7 @@ final class CloudStorageReplicaRepairService . "AND repair_coverage.size_bytes = cloud_storage_artifacts.size_bytes " . "AND repair_coverage.sha256 = cloud_storage_artifacts.sha256 " . "AND repair_coverage.verified_at > 0 " - . "AND repair_backend.admin_state <> 'retired' " + . "AND repair_backend.admin_state IN ('active','read_only') " . "AND repair_backend.health_state IN ('healthy','degraded','unknown')) " . "OR (repair_coverage.state IN ('reserved','publishing','copying') " . "AND repair_backend.admin_state = 'active' " @@ -118,6 +118,7 @@ final class CloudStorageReplicaRepairService ->lockForUpdate()->get()->keyBy('id'); $ready = []; + $coveredReady = 0; $pending = 0; $represented = []; foreach ($replicas as $replica) { @@ -130,6 +131,10 @@ final class CloudStorageReplicaRepairService && in_array((string) $backend->health_state, ['healthy', 'degraded', 'unknown'], true)) { $ready[] = [$replica, $backend]; + if (in_array((string) $backend->admin_state, + ['active', 'read_only'], true)) { + $coveredReady++; + } } } elseif (in_array((string) $replica->state, ['reserved', 'publishing', 'copying'], true)) { @@ -146,8 +151,8 @@ final class CloudStorageReplicaRepairService } $desired = (int) $artifact->desired_copies; if ($desired < 1 || $desired > 2 - || count($ready) >= $desired - || count($ready) + $pending >= $desired + || $coveredReady >= $desired + || $coveredReady + $pending >= $desired || $ready === []) { return false; } diff --git a/web/tests/CloudStorageReplicaRepairServiceTest.php b/web/tests/CloudStorageReplicaRepairServiceTest.php index 5e6bd395..24a6d7f9 100644 --- a/web/tests/CloudStorageReplicaRepairServiceTest.php +++ b/web/tests/CloudStorageReplicaRepairServiceTest.php @@ -186,9 +186,9 @@ CloudStorageArtifact::whereKey($view['artifact_id'])->update([ 'desired_copies' => 1, ]); -// A failed old replica must not occupy durability coverage. For a required-two -// artifact, replacement also must use a failure domain distinct from the only -// exact ready source. +// A healthy replica on a draining backend must not occupy forward durability +// coverage. This lets maintenance evacuate to a replacement before the old +// provider is taken offline. Required-two still needs a distinct domain. $insertBackend(3, 'active', 'healthy', 1, 'provider-a', 1, 'same-domain'); $insertBackend(4, 'active', 'healthy', 1, 'provider-c', 50, 'replacement'); $artifactId = Capsule::table('cloud_storage_artifacts')->insertGetId([ @@ -256,8 +256,8 @@ Capsule::table('cloud_storage_publications')->insert([ 'created_at' => $now, 'updated_at' => $now, ]); Capsule::table('cloud_storage_backends')->where('id', 2)->update([ - 'admin_state' => 'read_only', 'health_state' => 'unreachable', - 'tested_revision' => 0, 'last_error_code' => 'endpoint_unreachable', + 'admin_state' => 'draining', 'health_state' => 'healthy', + 'tested_revision' => 1, 'last_error_code' => '', 'updated_at' => $now, ]); @@ -270,7 +270,7 @@ $assert((int) $replacement->backend_id === 4 && (string) $replacement->object_key === 'replacement/templates/image.qcow2' && CloudStorageReplica::where('artifact_id', $artifactId)->count() === 3, - 'Repair counted degraded evidence or selected the source failure domain.'); + 'Repair counted draining evidence or selected an existing failure domain.'); $workerSource = file_get_contents(dirname(__DIR__) . '/app/Services/Cloud/CloudStorageReplicaWorkerService.php');