fix(storage): align protection with placement coverage

go
codex 2 weeks ago
parent d53dcbaad1
commit 88fb0e5aa8

@ -34,6 +34,12 @@ final class CloudStorageProtectionSummaryService
)
->where('replicas.verified_at', '>', 0);
})
->leftJoin(
'cloud_storage_backends as backends',
'backends.id',
'=',
'replicas.backend_id'
)
->whereNotIn('artifacts.state', [
'retiring', 'delete_attention', 'deleted',
])
@ -49,7 +55,20 @@ final class CloudStorageProtectionSummaryService
'artifacts.required_copies',
'artifacts.created_at',
CloudStorageArtifact::query()->getQuery()->raw(
'COUNT(replicas.id) AS ready_copies'
"COUNT(DISTINCT CASE WHEN "
. "backends.admin_state IN ('active','read_only') "
. "AND backends.health_state IN "
. "('healthy','degraded','unknown') "
. 'THEN replicas.id END) AS ready_copies'
),
CloudStorageArtifact::query()->getQuery()->raw(
"COUNT(DISTINCT CASE WHEN "
. "backends.admin_state IN ('active','read_only') "
. "AND backends.health_state IN "
. "('healthy','degraded','unknown') "
. "AND TRIM(backends.failure_domain) <> '' "
. 'THEN LOWER(TRIM(backends.failure_domain)) END) '
. 'AS ready_domains'
),
]);
@ -61,13 +80,16 @@ final class CloudStorageProtectionSummaryService
$ready = (int) ($row->ready_copies ?? 0);
$desired = (int) $row->desired_copies;
$required = (int) $row->required_copies;
if ($ready >= $desired) {
$coverage = $required > 1
? min($ready, (int) ($row->ready_domains ?? 0))
: $ready;
if ($coverage >= $desired) {
$protected++;
continue;
}
$backlog++;
if ($ready >= $required) {
if ($coverage >= $required) {
$degraded++;
}
$createdAt = (int) $row->created_at;

@ -44,7 +44,11 @@ final class CloudStorageReplicaRepairService
);
})
->whereRaw(
"(SELECT COUNT(*) FROM cloud_storage_replicas repair_coverage "
"(SELECT COUNT(DISTINCT CASE "
. "WHEN cloud_storage_artifacts.required_copies > 1 "
. "THEN LOWER(TRIM(repair_backend.failure_domain)) "
. "ELSE repair_coverage.id END) "
. "FROM cloud_storage_replicas repair_coverage "
. "INNER JOIN cloud_storage_backends repair_backend "
. "ON repair_backend.id = repair_coverage.backend_id "
. "WHERE repair_coverage.artifact_id = cloud_storage_artifacts.id "
@ -118,8 +122,11 @@ final class CloudStorageReplicaRepairService
->lockForUpdate()->get()->keyBy('id');
$ready = [];
$required = (int) $artifact->required_copies;
$coveredReady = 0;
$pending = 0;
$coveredReadyDomains = [];
$pendingDomains = [];
$represented = [];
foreach ($replicas as $replica) {
$backendId = (int) $replica->backend_id;
@ -133,7 +140,13 @@ final class CloudStorageReplicaRepairService
$ready[] = [$replica, $backend];
if (in_array((string) $backend->admin_state,
['active', 'read_only'], true)) {
$coveredReady++;
if ($required > 1) {
$coveredReadyDomains[
self::domain($backend)
] = true;
} else {
$coveredReady++;
}
}
}
} elseif (in_array((string) $replica->state,
@ -145,10 +158,21 @@ final class CloudStorageReplicaRepairService
&& (int) $backend->config_revision > 0
&& (int) $backend->tested_revision
=== (int) $backend->config_revision) {
$pending++;
if ($required > 1) {
$pendingDomains[self::domain($backend)] = true;
} else {
$pending++;
}
}
}
}
if ($required > 1) {
$coveredReady = count($coveredReadyDomains);
$pending = count(array_diff_key(
$pendingDomains,
$coveredReadyDomains
));
}
$desired = (int) $artifact->desired_copies;
if ($desired < 1 || $desired > 2
|| $coveredReady >= $desired

@ -28,11 +28,18 @@ $db->schema()->create('cloud_storage_artifacts', static function (Blueprint $t):
$db->schema()->create('cloud_storage_replicas', static function (Blueprint $t): void {
$t->increments('id');
$t->unsignedBigInteger('artifact_id');
$t->unsignedBigInteger('backend_id');
$t->unsignedBigInteger('size_bytes');
$t->string('sha256');
$t->unsignedBigInteger('verified_at');
$t->string('state');
});
$db->schema()->create('cloud_storage_backends', static function (Blueprint $t): void {
$t->increments('id');
$t->string('admin_state');
$t->string('health_state');
$t->string('failure_domain');
});
$now = 2_000;
$artifacts = [
@ -50,23 +57,35 @@ $artifacts = [
'state' => 'retiring', 'created_at' => 1_000],
];
Capsule::table('cloud_storage_artifacts')->insert($artifacts);
Capsule::table('cloud_storage_backends')->insert([
['id' => 1, 'admin_state' => 'active', 'health_state' => 'healthy',
'failure_domain' => 'provider-a'],
['id' => 2, 'admin_state' => 'active', 'health_state' => 'healthy',
'failure_domain' => 'provider-b'],
['id' => 3, 'admin_state' => 'draining', 'health_state' => 'healthy',
'failure_domain' => 'provider-c'],
['id' => 4, 'admin_state' => 'active', 'health_state' => 'unreachable',
'failure_domain' => 'provider-d'],
['id' => 5, 'admin_state' => 'read_only', 'health_state' => 'degraded',
'failure_domain' => 'provider-a'],
]);
Capsule::table('cloud_storage_replicas')->insert([
['artifact_id' => 1, 'size_bytes' => 100,
['artifact_id' => 1, 'backend_id' => 1, 'size_bytes' => 100,
'sha256' => str_repeat('a', 64), 'verified_at' => 1,
'state' => 'ready'],
['artifact_id' => 1, 'size_bytes' => 100,
['artifact_id' => 1, 'backend_id' => 2, 'size_bytes' => 100,
'sha256' => str_repeat('a', 64), 'verified_at' => 2,
'state' => 'ready'],
['artifact_id' => 2, 'size_bytes' => 200,
['artifact_id' => 2, 'backend_id' => 1, 'size_bytes' => 200,
'sha256' => str_repeat('b', 64), 'verified_at' => 3,
'state' => 'ready'],
['artifact_id' => 2, 'size_bytes' => 0, 'sha256' => '',
['artifact_id' => 2, 'backend_id' => 2, 'size_bytes' => 0, 'sha256' => '',
'verified_at' => 0, 'state' => 'copying'],
// State alone must not count a corrupted or stale identity as coverage.
['artifact_id' => 3, 'size_bytes' => 301,
['artifact_id' => 3, 'backend_id' => 1, 'size_bytes' => 301,
'sha256' => str_repeat('c', 64), 'verified_at' => 4,
'state' => 'ready'],
['artifact_id' => 4, 'size_bytes' => 400,
['artifact_id' => 4, 'backend_id' => 3, 'size_bytes' => 400,
'sha256' => str_repeat('d', 64), 'verified_at' => 5,
'state' => 'ready'],
]);
@ -79,6 +98,16 @@ $assert($summary === [
'oldest_replication_age_seconds' => 400,
], 'Replica coverage summary does not match verified ready replicas.');
Capsule::table('cloud_storage_backends')->where('id', 2)
->update(['admin_state' => 'draining']);
$draining = CloudStorageProtectionSummaryService::summary($now);
$assert($draining['protected_artifact_count'] === 0
&& $draining['degraded_artifact_count'] === 2
&& $draining['replication_backlog_count'] === 3,
'A draining backend still counted toward forward protection.');
Capsule::table('cloud_storage_backends')->where('id', 2)
->update(['admin_state' => 'active']);
Capsule::table('cloud_storage_replicas')->where('artifact_id', 2)
->where('state', 'copying')->update([
'state' => 'ready', 'size_bytes' => 200,
@ -96,6 +125,22 @@ $assert($recovered['protected_artifact_count'] === 2
&& $recovered['oldest_replication_age_seconds'] === 400,
'Summary did not converge when replicas became ready.');
Capsule::table('cloud_storage_replicas')->insert([
'artifact_id' => 3, 'backend_id' => 5, 'size_bytes' => 300,
'sha256' => str_repeat('c', 64), 'verified_at' => 8,
'state' => 'ready',
]);
$sameDomain = CloudStorageProtectionSummaryService::summary($now);
$assert($sameDomain['protected_artifact_count'] === 2
&& $sameDomain['replication_backlog_count'] === 1,
'Required-two protection accepted two replicas in one failure domain.');
Capsule::table('cloud_storage_backends')->where('id', 5)
->update(['failure_domain' => 'provider-e']);
$distinctDomains = CloudStorageProtectionSummaryService::summary($now);
$assert($distinctDomains['protected_artifact_count'] === 3
&& $distinctDomains['replication_backlog_count'] === 0,
'Required-two protection did not recognize two explicit failure domains.');
Capsule::table('cloud_storage_artifacts')->whereIn('id', [1, 2, 3])
->update(['state' => 'deleted']);
$empty = CloudStorageProtectionSummaryService::summary($now);

Loading…
Cancel
Save