diff --git a/web/app/Services/Cloud/CloudStorageArtifactService.php b/web/app/Services/Cloud/CloudStorageArtifactService.php index e2570f63..e7c4529f 100644 --- a/web/app/Services/Cloud/CloudStorageArtifactService.php +++ b/web/app/Services/Cloud/CloudStorageArtifactService.php @@ -2588,7 +2588,7 @@ final class CloudStorageArtifactService if ($publication instanceof CloudStoragePublication && !in_array((string) $publication->state, ['finalizing', 'completed', 'failed', 'retired'], true)) { - if ($projection['can_read']) { + if ($projection['meets_publication_requirement']) { $publication->state = 'ready_to_finalize'; $publication->available_at = $now; if ($publication->coverage_ready_at === null) { diff --git a/web/app/Services/Cloud/CloudStorageDurabilityPolicy.php b/web/app/Services/Cloud/CloudStorageDurabilityPolicy.php index 246d30be..617f24b0 100644 --- a/web/app/Services/Cloud/CloudStorageDurabilityPolicy.php +++ b/web/app/Services/Cloud/CloudStorageDurabilityPolicy.php @@ -31,7 +31,8 @@ final class CloudStorageDurabilityPolicy * @param array $artifact * @param list> $replicas * @return array{state:string,ready_copies:int,pending_copies:int, - * attention_copies:int,can_read:bool,protected:bool} + * attention_copies:int,meets_publication_requirement:bool, + * can_read:bool,protected:bool} */ public static function project(array $artifact, array $replicas): array { @@ -108,8 +109,14 @@ final class CloudStorageDurabilityPolicy 'ready_copies' => $ready, 'pending_copies' => $pending, 'attention_copies' => $attention, - 'can_read' => $ready >= $required - && !in_array($state, ['retiring', 'deleted'], true), + // `required_copies` is a publication gate, not a permanent read + // quorum. Once an owner has finalized a required-two artifact, a + // later single-replica failure must not hide the remaining exact, + // verified copy while repair is pending. + 'meets_publication_requirement' => $ready >= $required, + 'can_read' => $ready >= 1 + && !in_array($state, + ['retiring', 'delete_attention', 'deleted'], true), 'protected' => $ready >= $desired, ]; } diff --git a/web/tests/CloudStorageArtifactServiceTest.php b/web/tests/CloudStorageArtifactServiceTest.php index f1020d76..0f848745 100644 --- a/web/tests/CloudStorageArtifactServiceTest.php +++ b/web/tests/CloudStorageArtifactServiceTest.php @@ -972,6 +972,162 @@ $assert((string) $deletedCopy->phase === 'terminal' === 'delete_projection_sentinel', 'Copy fence overwrote an existing deletion projection.'); +// A receipt must be bound to the caller's frozen artifact before any row is +// changed, even when two artifacts intentionally share the same bytes. +$crossA = CloudStorageArtifactService::beginPublication( + 'backup', 'backups/cross-a/system.qcow2', 4096, + str_repeat('7', 64), 'backup-publication-cross-a', + hash('sha256', 'cross-a'), ['backup_id' => 110] +); +$crossB = CloudStorageArtifactService::beginPublication( + 'backup', 'backups/cross-b/system.qcow2', 4096, + str_repeat('7', 64), 'backup-publication-cross-b', + hash('sha256', 'cross-b'), ['backup_id' => 111] +); +CloudStorageArtifactService::markPrimaryDispatching( + $crossA['operation_id'] +); +CloudStorageArtifactService::markPrimaryDispatching( + $crossB['operation_id'] +); +$expect(static fn () => CloudStorageArtifactService::completePrimary( + $crossA['operation_id'], (int) $crossB['artifact_id'], [ + 'size_bytes' => 4096, 'sha256' => str_repeat('7', 64), + 'etag' => '"cross-a"', 'version_id' => '', + ] +), 409); +$crossAOperation = CloudStorageOperation::where( + 'operation_id', $crossA['operation_id'] +)->firstOrFail(); +$crossAPrimary = CloudStorageReplica::whereKey( + (int) $crossAOperation->replica_id +)->firstOrFail(); +$crossAArtifact = CloudStorageArtifact::whereKey( + (int) $crossA['artifact_id'] +)->firstOrFail(); +$crossBOperation = CloudStorageOperation::where( + 'operation_id', $crossB['operation_id'] +)->firstOrFail(); +$crossBPrimary = CloudStorageReplica::whereKey( + (int) $crossBOperation->replica_id +)->firstOrFail(); +$crossBArtifact = CloudStorageArtifact::whereKey( + (int) $crossB['artifact_id'] +)->firstOrFail(); +$assert((string) $crossAOperation->phase === 'dispatching' + && $crossAOperation->outcome === null + && (string) $crossAPrimary->state === 'publishing' + && (int) $crossAPrimary->size_bytes === 0 + && (string) $crossAArtifact->state === 'publishing' + && (string) $crossBOperation->phase === 'dispatching' + && $crossBOperation->outcome === null + && (string) $crossBPrimary->state === 'publishing' + && (int) $crossBPrimary->size_bytes === 0 + && (string) $crossBArtifact->state === 'publishing', + 'Cross-artifact primary receipt changed one of the two ledgers.'); + +// A required-two artifact must cross the initial publication gate with both +// replicas. Once the owner has finalized it, later loss of one replica is a +// degraded-but-readable repair condition and must not reopen the outbox. +$runtimeReadView = CloudStorageArtifactService::beginPublication( + 'template', + 'cloud-templates/runtime-read/system.qcow2', + 4096, + str_repeat('6', 64), + 'template-runtime-read-policy', + hash('sha256', 'template-runtime-read-policy'), + ['template_import_intent_id' => str_repeat('6', 32)] +); +CloudStorageArtifactService::markPrimaryDispatching( + $runtimeReadView['operation_id'] +); +CloudStorageArtifactService::completePrimary( + $runtimeReadView['operation_id'], + (int) $runtimeReadView['artifact_id'], + [ + 'size_bytes' => 4096, + 'sha256' => str_repeat('6', 64), + 'etag' => '"runtime-primary"', + 'version_id' => '', + ] +); +$runtimePublication = CloudStoragePublication::where( + 'artifact_id', (int) $runtimeReadView['artifact_id'] +)->firstOrFail(); +$assert((string) $runtimePublication->state === 'waiting_protection', + 'One template replica prematurely released the required-two publication.'); +$runtimeArtifact = CloudStorageArtifact::whereKey( + (int) $runtimeReadView['artifact_id'] +)->firstOrFail(); +$runtimeReplicas = CloudStorageReplica::where( + 'artifact_id', (int) $runtimeArtifact->id +)->orderBy('id')->get(); +$runtimeSecondary = $runtimeReplicas->firstWhere('role', 'secondary'); +$assert($runtimeSecondary instanceof CloudStorageReplica, + 'Required-two fixture is missing its secondary replica.'); +$runtimeSecondary->state = 'ready'; +$runtimeSecondary->size_bytes = 4096; +$runtimeSecondary->sha256 = str_repeat('6', 64); +$runtimeSecondary->object_etag = '"runtime-secondary"'; +$runtimeSecondary->verified_at = time(); +$runtimeSecondary->error_code = ''; +$runtimeSecondary->save(); +$project = (new ReflectionClass(CloudStorageArtifactService::class)) + ->getMethod('projectArtifactLocked'); +$project->invoke( + null, + $runtimeArtifact, + $runtimeReplicas->all(), + time() +); +$runtimePublication->refresh(); +$assert((string) $runtimePublication->state === 'ready_to_finalize', + 'Two verified template replicas did not release publication.'); +CloudStoragePublicationFinalizerService::complete( + 'template', + str_repeat('6', 32), + (int) $runtimeArtifact->id, + (string) $runtimeReadView['operation_id'] +); +$runtimeSecondary->state = 'degraded'; +$runtimeSecondary->verified_at = 0; +$runtimeSecondary->error_code = 'replica_unavailable'; +$runtimeSecondary->save(); +$runtimeArtifact->refresh(); +$runtimeReplicas = CloudStorageReplica::where( + 'artifact_id', (int) $runtimeArtifact->id +)->orderBy('id')->get(); +$project->invoke( + null, + $runtimeArtifact, + $runtimeReplicas->all(), + time() +); +$runtimeArtifact->refresh(); +$runtimePublication->refresh(); +$runtimeProjection = CloudStorageDurabilityPolicy::project( + [ + 'state' => (string) $runtimeArtifact->state, + 'size_bytes' => (int) $runtimeArtifact->size_bytes, + 'sha256' => (string) $runtimeArtifact->sha256, + 'desired_copies' => (int) $runtimeArtifact->desired_copies, + 'required_copies' => (int) $runtimeArtifact->required_copies, + ], + $runtimeReplicas->map(static fn (CloudStorageReplica $replica): array => [ + 'backend_id' => (int) $replica->backend_id, + 'state' => (string) $replica->state, + 'size_bytes' => (int) $replica->size_bytes, + 'sha256' => (string) $replica->sha256, + 'verified_at' => (int) $replica->verified_at, + ])->values()->all() +); +$assert((string) $runtimeArtifact->state === 'degraded' + && $runtimeProjection['can_read'] + && !$runtimeProjection['meets_publication_requirement'] + && (string) $runtimePublication->state === 'completed' + && (int) $runtimePublication->finalized_at > 0, + 'Replica degradation made a published template unreadable or reopened its outbox.'); + CloudStorageBackend::whereKey(2)->update([ 'admin_state' => 'disabled', 'health_state' => 'unreachable', 'tested_revision' => 0, diff --git a/web/tests/CloudStorageDurabilityPolicyTest.php b/web/tests/CloudStorageDurabilityPolicyTest.php index 062d5811..ea6278cf 100644 --- a/web/tests/CloudStorageDurabilityPolicyTest.php +++ b/web/tests/CloudStorageDurabilityPolicyTest.php @@ -31,6 +31,7 @@ $replica = static fn (int $id, int $backendId, string $state = 'ready'): array = $backup = $artifact('backup'); $one = CloudStorageDurabilityPolicy::project($backup, [$replica(1, 10)]); $assert($one['state'] === 'available' && $one['can_read'] + && $one['meets_publication_requirement'] && !$one['protected'], 'One verified backup copy must be available but not protected.'); $two = CloudStorageDurabilityPolicy::project( @@ -46,16 +47,51 @@ $oneTemplate = CloudStorageDurabilityPolicy::project( [$replica(1, 10), $replica(2, 20, 'copying')] ); $assert($oneTemplate['state'] === 'publishing' - && !$oneTemplate['can_read'], - 'A required-two template must not become readable after one copy.'); + && $oneTemplate['can_read'] + && !$oneTemplate['meets_publication_requirement'], + 'One exact template replica must remain readable without releasing publication.'); $twoTemplate = CloudStorageDurabilityPolicy::project( $template, [$replica(1, 10), $replica(2, 20)] ); $assert($twoTemplate['state'] === 'protected' - && $twoTemplate['can_read'], + && $twoTemplate['can_read'] + && $twoTemplate['meets_publication_requirement'], 'A required-two template must become readable after both copies verify.'); +$degradedTemplate = CloudStorageDurabilityPolicy::project( + array_replace($template, ['state' => 'protected']), + [$replica(1, 10), $replica(2, 20, 'degraded')] +); +$assert($degradedTemplate['state'] === 'degraded' + && $degradedTemplate['can_read'] + && !$degradedTemplate['meets_publication_requirement'] + && !$degradedTemplate['protected'], + 'A published template must remain readable from one verified replica while degraded.'); + +$rescue = $artifact('rescue'); +$oneRescue = CloudStorageDurabilityPolicy::project( + $rescue, + [$replica(1, 10), $replica(2, 20, 'copying')] +); +$assert($oneRescue['can_read'] + && !$oneRescue['meets_publication_requirement'], + 'One rescue replica must not satisfy its initial publication requirement.'); +$noHealthyRescue = CloudStorageDurabilityPolicy::project( + $rescue, + [$replica(1, 10, 'degraded'), $replica(2, 20, 'degraded')] +); +$assert(!$noHealthyRescue['can_read'] + && !$noHealthyRescue['meets_publication_requirement'], + 'A rescue artifact without a verified replica must not be readable.'); +$deletingTemplate = CloudStorageDurabilityPolicy::project( + array_replace($template, ['state' => 'delete_attention']), + [$replica(1, 10), $replica(2, 20)] +); +$assert($deletingTemplate['state'] === 'delete_attention' + && !$deletingTemplate['can_read'], + 'A delete-attention artifact must not mint new read capabilities.'); + $candidates = [ $replica(1, 10) + ['backend' => [ 'admin_state' => 'active', 'health_state' => 'unreachable',