From bfe0a5eba2bd8714d503e3dfddccf04d82e3a33b Mon Sep 17 00:00:00 2001 From: codex Date: Fri, 6 Nov 2026 19:59:28 +0800 Subject: [PATCH] fix(storage): reserve deferred backup replica placement --- .../Cloud/CloudStoragePlacementPolicy.php | 74 +++++++++++-------- .../CloudStorageDurabilityPolicyTest.php | 19 +++++ 2 files changed, 63 insertions(+), 30 deletions(-) diff --git a/web/app/Services/Cloud/CloudStoragePlacementPolicy.php b/web/app/Services/Cloud/CloudStoragePlacementPolicy.php index 1039c414..1fc887bb 100644 --- a/web/app/Services/Cloud/CloudStoragePlacementPolicy.php +++ b/web/app/Services/Cloud/CloudStoragePlacementPolicy.php @@ -21,6 +21,7 @@ final class CloudStoragePlacementPolicy ); } $eligible = []; + $healthy = []; $seen = []; foreach ($backends as $backend) { $id = $backend['id'] ?? null; @@ -32,49 +33,62 @@ final class CloudStoragePlacementPolicy || !is_int($tested) || $tested !== $revision || !is_int($priority) || $priority < 0 || ($backend['admin_state'] ?? null) !== 'active' - || ($backend['health_state'] ?? null) !== 'healthy') { + || !in_array(($backend['health_state'] ?? null), [ + 'healthy', 'degraded', 'unknown', 'unreachable', + ], true)) { continue; } $seen[$id] = true; $eligible[] = $backend; + if ($backend['health_state'] === 'healthy') { + $healthy[] = $backend; + } } usort($eligible, static fn (array $left, array $right): int => [$left['write_priority'], $left['id']] <=> [$right['write_priority'], $right['id']]); + usort($healthy, static fn (array $left, array $right): int => + [$left['write_priority'], $left['id']] + <=> [$right['write_priority'], $right['id']]); - if (count($eligible) < $requiredCopies) { + if (count($healthy) < $requiredCopies) { throw new CloudError('健康且已验证的对象存储后端数量不足', 503); } - $selected = []; - $domains = []; - foreach ($eligible as $backend) { - if ($selected === []) { - $selected[] = $backend; - $domains[self::domain($backend)] = true; - continue; - } - if (count($selected) >= $desiredCopies) { - break; - } - $domain = self::domain($backend); - if (!isset($domains[$domain])) { - $selected[] = $backend; - $domains[$domain] = true; - } - } - if (count($selected) < $desiredCopies) { - $chosen = array_fill_keys(array_map( - static fn (array $row): int => $row['id'], - $selected - ), true); - foreach ($eligible as $backend) { - if (count($selected) >= $desiredCopies) { - break; - } - if (!isset($chosen[$backend['id']])) { + // The primary must always be writable now. For an asynchronous + // required-one artifact, however, reserve a second active/tested + // backend even while it is unhealthy. Its copy operation remains + // deferred and converges after the backend recovers instead of + // leaving the artifact permanently single-copy. + $selected = [$healthy[0]]; + $chosen = [(int) $healthy[0]['id'] => true]; + $domains = [self::domain($healthy[0]) => true]; + $healthyRemainder = array_values(array_filter( + $healthy, + static fn (array $backend): bool => + !isset($chosen[(int) $backend['id']]) + )); + $standby = array_values(array_filter( + $eligible, + static fn (array $backend): bool => + ($backend['health_state'] ?? null) !== 'healthy' + )); + + foreach ([$healthyRemainder, $standby] as $tier) { + foreach ([true, false] as $requireNewDomain) { + foreach ($tier as $backend) { + if (count($selected) >= $desiredCopies) { + break 3; + } + $id = (int) $backend['id']; + $domain = self::domain($backend); + if (isset($chosen[$id]) + || ($requireNewDomain && isset($domains[$domain]))) { + continue; + } $selected[] = $backend; - $chosen[$backend['id']] = true; + $chosen[$id] = true; + $domains[$domain] = true; } } } diff --git a/web/tests/CloudStorageDurabilityPolicyTest.php b/web/tests/CloudStorageDurabilityPolicyTest.php index ea6278cf..b5fad73d 100644 --- a/web/tests/CloudStorageDurabilityPolicyTest.php +++ b/web/tests/CloudStorageDurabilityPolicyTest.php @@ -129,6 +129,25 @@ $backends[1]['health_state'] = 'unreachable'; $backupPlacement = CloudStoragePlacementPolicy::select($backends, 2, 1); $assert(array_column($backupPlacement, 'id') === [1, 3], 'Async backup placement may use two healthy backends in one domain.'); +$backends[2]['health_state'] = 'unreachable'; +$backupDeferredPlacement = CloudStoragePlacementPolicy::select( + $backends, + 2, + 1 +); +$assert(array_column($backupDeferredPlacement, 'id') === [1, 2], + 'Async backup placement must reserve a deferred second failure domain.'); +$backends[0]['health_state'] = 'unreachable'; +$backends[1]['health_state'] = 'healthy'; +$backupFailoverPlacement = CloudStoragePlacementPolicy::select( + $backends, + 2, + 1 +); +$assert(array_column($backupFailoverPlacement, 'id') === [2, 1], + 'Async backup placement must choose a healthy primary before a standby.'); +$backends[0]['health_state'] = 'healthy'; +$backends[1]['health_state'] = 'unreachable'; $backends[2]['tested_revision'] = 0; try { CloudStoragePlacementPolicy::select($backends, 2, 2);