fix(storage): separate publication quorum from read availability

go
codex 2 weeks ago
parent 0a50734a40
commit 5a4e59c72a

@ -2588,7 +2588,7 @@ final class CloudStorageArtifactService
if ($publication instanceof CloudStoragePublication
&& !in_array((string) $publication->state,
['finalizing', 'completed', 'failed', 'retired'], true)) {
if ($projection['can_read']) {
if ($projection['meets_publication_requirement']) {
$publication->state = 'ready_to_finalize';
$publication->available_at = $now;
if ($publication->coverage_ready_at === null) {

@ -31,7 +31,8 @@ final class CloudStorageDurabilityPolicy
* @param array<string,mixed> $artifact
* @param list<array<string,mixed>> $replicas
* @return array{state:string,ready_copies:int,pending_copies:int,
* attention_copies:int,can_read:bool,protected:bool}
* attention_copies:int,meets_publication_requirement:bool,
* can_read:bool,protected:bool}
*/
public static function project(array $artifact, array $replicas): array
{
@ -108,8 +109,14 @@ final class CloudStorageDurabilityPolicy
'ready_copies' => $ready,
'pending_copies' => $pending,
'attention_copies' => $attention,
'can_read' => $ready >= $required
&& !in_array($state, ['retiring', 'deleted'], true),
// `required_copies` is a publication gate, not a permanent read
// quorum. Once an owner has finalized a required-two artifact, a
// later single-replica failure must not hide the remaining exact,
// verified copy while repair is pending.
'meets_publication_requirement' => $ready >= $required,
'can_read' => $ready >= 1
&& !in_array($state,
['retiring', 'delete_attention', 'deleted'], true),
'protected' => $ready >= $desired,
];
}

@ -972,6 +972,162 @@ $assert((string) $deletedCopy->phase === 'terminal'
=== 'delete_projection_sentinel',
'Copy fence overwrote an existing deletion projection.');
// A receipt must be bound to the caller's frozen artifact before any row is
// changed, even when two artifacts intentionally share the same bytes.
$crossA = CloudStorageArtifactService::beginPublication(
'backup', 'backups/cross-a/system.qcow2', 4096,
str_repeat('7', 64), 'backup-publication-cross-a',
hash('sha256', 'cross-a'), ['backup_id' => 110]
);
$crossB = CloudStorageArtifactService::beginPublication(
'backup', 'backups/cross-b/system.qcow2', 4096,
str_repeat('7', 64), 'backup-publication-cross-b',
hash('sha256', 'cross-b'), ['backup_id' => 111]
);
CloudStorageArtifactService::markPrimaryDispatching(
$crossA['operation_id']
);
CloudStorageArtifactService::markPrimaryDispatching(
$crossB['operation_id']
);
$expect(static fn () => CloudStorageArtifactService::completePrimary(
$crossA['operation_id'], (int) $crossB['artifact_id'], [
'size_bytes' => 4096, 'sha256' => str_repeat('7', 64),
'etag' => '"cross-a"', 'version_id' => '',
]
), 409);
$crossAOperation = CloudStorageOperation::where(
'operation_id', $crossA['operation_id']
)->firstOrFail();
$crossAPrimary = CloudStorageReplica::whereKey(
(int) $crossAOperation->replica_id
)->firstOrFail();
$crossAArtifact = CloudStorageArtifact::whereKey(
(int) $crossA['artifact_id']
)->firstOrFail();
$crossBOperation = CloudStorageOperation::where(
'operation_id', $crossB['operation_id']
)->firstOrFail();
$crossBPrimary = CloudStorageReplica::whereKey(
(int) $crossBOperation->replica_id
)->firstOrFail();
$crossBArtifact = CloudStorageArtifact::whereKey(
(int) $crossB['artifact_id']
)->firstOrFail();
$assert((string) $crossAOperation->phase === 'dispatching'
&& $crossAOperation->outcome === null
&& (string) $crossAPrimary->state === 'publishing'
&& (int) $crossAPrimary->size_bytes === 0
&& (string) $crossAArtifact->state === 'publishing'
&& (string) $crossBOperation->phase === 'dispatching'
&& $crossBOperation->outcome === null
&& (string) $crossBPrimary->state === 'publishing'
&& (int) $crossBPrimary->size_bytes === 0
&& (string) $crossBArtifact->state === 'publishing',
'Cross-artifact primary receipt changed one of the two ledgers.');
// A required-two artifact must cross the initial publication gate with both
// replicas. Once the owner has finalized it, later loss of one replica is a
// degraded-but-readable repair condition and must not reopen the outbox.
$runtimeReadView = CloudStorageArtifactService::beginPublication(
'template',
'cloud-templates/runtime-read/system.qcow2',
4096,
str_repeat('6', 64),
'template-runtime-read-policy',
hash('sha256', 'template-runtime-read-policy'),
['template_import_intent_id' => str_repeat('6', 32)]
);
CloudStorageArtifactService::markPrimaryDispatching(
$runtimeReadView['operation_id']
);
CloudStorageArtifactService::completePrimary(
$runtimeReadView['operation_id'],
(int) $runtimeReadView['artifact_id'],
[
'size_bytes' => 4096,
'sha256' => str_repeat('6', 64),
'etag' => '"runtime-primary"',
'version_id' => '',
]
);
$runtimePublication = CloudStoragePublication::where(
'artifact_id', (int) $runtimeReadView['artifact_id']
)->firstOrFail();
$assert((string) $runtimePublication->state === 'waiting_protection',
'One template replica prematurely released the required-two publication.');
$runtimeArtifact = CloudStorageArtifact::whereKey(
(int) $runtimeReadView['artifact_id']
)->firstOrFail();
$runtimeReplicas = CloudStorageReplica::where(
'artifact_id', (int) $runtimeArtifact->id
)->orderBy('id')->get();
$runtimeSecondary = $runtimeReplicas->firstWhere('role', 'secondary');
$assert($runtimeSecondary instanceof CloudStorageReplica,
'Required-two fixture is missing its secondary replica.');
$runtimeSecondary->state = 'ready';
$runtimeSecondary->size_bytes = 4096;
$runtimeSecondary->sha256 = str_repeat('6', 64);
$runtimeSecondary->object_etag = '"runtime-secondary"';
$runtimeSecondary->verified_at = time();
$runtimeSecondary->error_code = '';
$runtimeSecondary->save();
$project = (new ReflectionClass(CloudStorageArtifactService::class))
->getMethod('projectArtifactLocked');
$project->invoke(
null,
$runtimeArtifact,
$runtimeReplicas->all(),
time()
);
$runtimePublication->refresh();
$assert((string) $runtimePublication->state === 'ready_to_finalize',
'Two verified template replicas did not release publication.');
CloudStoragePublicationFinalizerService::complete(
'template',
str_repeat('6', 32),
(int) $runtimeArtifact->id,
(string) $runtimeReadView['operation_id']
);
$runtimeSecondary->state = 'degraded';
$runtimeSecondary->verified_at = 0;
$runtimeSecondary->error_code = 'replica_unavailable';
$runtimeSecondary->save();
$runtimeArtifact->refresh();
$runtimeReplicas = CloudStorageReplica::where(
'artifact_id', (int) $runtimeArtifact->id
)->orderBy('id')->get();
$project->invoke(
null,
$runtimeArtifact,
$runtimeReplicas->all(),
time()
);
$runtimeArtifact->refresh();
$runtimePublication->refresh();
$runtimeProjection = CloudStorageDurabilityPolicy::project(
[
'state' => (string) $runtimeArtifact->state,
'size_bytes' => (int) $runtimeArtifact->size_bytes,
'sha256' => (string) $runtimeArtifact->sha256,
'desired_copies' => (int) $runtimeArtifact->desired_copies,
'required_copies' => (int) $runtimeArtifact->required_copies,
],
$runtimeReplicas->map(static fn (CloudStorageReplica $replica): array => [
'backend_id' => (int) $replica->backend_id,
'state' => (string) $replica->state,
'size_bytes' => (int) $replica->size_bytes,
'sha256' => (string) $replica->sha256,
'verified_at' => (int) $replica->verified_at,
])->values()->all()
);
$assert((string) $runtimeArtifact->state === 'degraded'
&& $runtimeProjection['can_read']
&& !$runtimeProjection['meets_publication_requirement']
&& (string) $runtimePublication->state === 'completed'
&& (int) $runtimePublication->finalized_at > 0,
'Replica degradation made a published template unreadable or reopened its outbox.');
CloudStorageBackend::whereKey(2)->update([
'admin_state' => 'disabled', 'health_state' => 'unreachable',
'tested_revision' => 0,

@ -31,6 +31,7 @@ $replica = static fn (int $id, int $backendId, string $state = 'ready'): array =
$backup = $artifact('backup');
$one = CloudStorageDurabilityPolicy::project($backup, [$replica(1, 10)]);
$assert($one['state'] === 'available' && $one['can_read']
&& $one['meets_publication_requirement']
&& !$one['protected'],
'One verified backup copy must be available but not protected.');
$two = CloudStorageDurabilityPolicy::project(
@ -46,16 +47,51 @@ $oneTemplate = CloudStorageDurabilityPolicy::project(
[$replica(1, 10), $replica(2, 20, 'copying')]
);
$assert($oneTemplate['state'] === 'publishing'
&& !$oneTemplate['can_read'],
'A required-two template must not become readable after one copy.');
&& $oneTemplate['can_read']
&& !$oneTemplate['meets_publication_requirement'],
'One exact template replica must remain readable without releasing publication.');
$twoTemplate = CloudStorageDurabilityPolicy::project(
$template,
[$replica(1, 10), $replica(2, 20)]
);
$assert($twoTemplate['state'] === 'protected'
&& $twoTemplate['can_read'],
&& $twoTemplate['can_read']
&& $twoTemplate['meets_publication_requirement'],
'A required-two template must become readable after both copies verify.');
$degradedTemplate = CloudStorageDurabilityPolicy::project(
array_replace($template, ['state' => 'protected']),
[$replica(1, 10), $replica(2, 20, 'degraded')]
);
$assert($degradedTemplate['state'] === 'degraded'
&& $degradedTemplate['can_read']
&& !$degradedTemplate['meets_publication_requirement']
&& !$degradedTemplate['protected'],
'A published template must remain readable from one verified replica while degraded.');
$rescue = $artifact('rescue');
$oneRescue = CloudStorageDurabilityPolicy::project(
$rescue,
[$replica(1, 10), $replica(2, 20, 'copying')]
);
$assert($oneRescue['can_read']
&& !$oneRescue['meets_publication_requirement'],
'One rescue replica must not satisfy its initial publication requirement.');
$noHealthyRescue = CloudStorageDurabilityPolicy::project(
$rescue,
[$replica(1, 10, 'degraded'), $replica(2, 20, 'degraded')]
);
$assert(!$noHealthyRescue['can_read']
&& !$noHealthyRescue['meets_publication_requirement'],
'A rescue artifact without a verified replica must not be readable.');
$deletingTemplate = CloudStorageDurabilityPolicy::project(
array_replace($template, ['state' => 'delete_attention']),
[$replica(1, 10), $replica(2, 20)]
);
$assert($deletingTemplate['state'] === 'delete_attention'
&& !$deletingTemplate['can_read'],
'A delete-attention artifact must not mint new read capabilities.');
$candidates = [
$replica(1, 10) + ['backend' => [
'admin_state' => 'active', 'health_state' => 'unreachable',

Loading…
Cancel
Save