feat(storage): evacuate draining replica coverage

go
codex 2 weeks ago
parent e8ae6326bd
commit 0757e98c9d

@ -52,7 +52,7 @@ final class CloudStorageReplicaRepairService
. "AND repair_coverage.size_bytes = cloud_storage_artifacts.size_bytes "
. "AND repair_coverage.sha256 = cloud_storage_artifacts.sha256 "
. "AND repair_coverage.verified_at > 0 "
. "AND repair_backend.admin_state <> 'retired' "
. "AND repair_backend.admin_state IN ('active','read_only') "
. "AND repair_backend.health_state IN ('healthy','degraded','unknown')) "
. "OR (repair_coverage.state IN ('reserved','publishing','copying') "
. "AND repair_backend.admin_state = 'active' "
@ -118,6 +118,7 @@ final class CloudStorageReplicaRepairService
->lockForUpdate()->get()->keyBy('id');
$ready = [];
$coveredReady = 0;
$pending = 0;
$represented = [];
foreach ($replicas as $replica) {
@ -130,6 +131,10 @@ final class CloudStorageReplicaRepairService
&& in_array((string) $backend->health_state,
['healthy', 'degraded', 'unknown'], true)) {
$ready[] = [$replica, $backend];
if (in_array((string) $backend->admin_state,
['active', 'read_only'], true)) {
$coveredReady++;
}
}
} elseif (in_array((string) $replica->state,
['reserved', 'publishing', 'copying'], true)) {
@ -146,8 +151,8 @@ final class CloudStorageReplicaRepairService
}
$desired = (int) $artifact->desired_copies;
if ($desired < 1 || $desired > 2
|| count($ready) >= $desired
|| count($ready) + $pending >= $desired
|| $coveredReady >= $desired
|| $coveredReady + $pending >= $desired
|| $ready === []) {
return false;
}

@ -186,9 +186,9 @@ CloudStorageArtifact::whereKey($view['artifact_id'])->update([
'desired_copies' => 1,
]);
// A failed old replica must not occupy durability coverage. For a required-two
// artifact, replacement also must use a failure domain distinct from the only
// exact ready source.
// A healthy replica on a draining backend must not occupy forward durability
// coverage. This lets maintenance evacuate to a replacement before the old
// provider is taken offline. Required-two still needs a distinct domain.
$insertBackend(3, 'active', 'healthy', 1, 'provider-a', 1, 'same-domain');
$insertBackend(4, 'active', 'healthy', 1, 'provider-c', 50, 'replacement');
$artifactId = Capsule::table('cloud_storage_artifacts')->insertGetId([
@ -256,8 +256,8 @@ Capsule::table('cloud_storage_publications')->insert([
'created_at' => $now, 'updated_at' => $now,
]);
Capsule::table('cloud_storage_backends')->where('id', 2)->update([
'admin_state' => 'read_only', 'health_state' => 'unreachable',
'tested_revision' => 0, 'last_error_code' => 'endpoint_unreachable',
'admin_state' => 'draining', 'health_state' => 'healthy',
'tested_revision' => 1, 'last_error_code' => '',
'updated_at' => $now,
]);
@ -270,7 +270,7 @@ $assert((int) $replacement->backend_id === 4
&& (string) $replacement->object_key
=== 'replacement/templates/image.qcow2'
&& CloudStorageReplica::where('artifact_id', $artifactId)->count() === 3,
'Repair counted degraded evidence or selected the source failure domain.');
'Repair counted draining evidence or selected an existing failure domain.');
$workerSource = file_get_contents(dirname(__DIR__)
. '/app/Services/Cloud/CloudStorageReplicaWorkerService.php');

Loading…
Cancel
Save