From 72a0358903d0b7192dbd173e04e3996fe2e34bb5 Mon Sep 17 00:00:00 2001 From: codex Date: Fri, 6 Nov 2026 19:49:38 +0800 Subject: [PATCH] feat(storage): add durable immutable primary publisher --- ...loudStorageImmutablePrimaryPublication.php | 895 ++++++++++++++++++ ...StorageImmutablePrimaryPublicationTest.php | 544 +++++++++++ 2 files changed, 1439 insertions(+) create mode 100644 web/app/Services/Cloud/CloudStorageImmutablePrimaryPublication.php create mode 100644 web/tests/CloudStorageImmutablePrimaryPublicationTest.php diff --git a/web/app/Services/Cloud/CloudStorageImmutablePrimaryPublication.php b/web/app/Services/Cloud/CloudStorageImmutablePrimaryPublication.php new file mode 100644 index 00000000..76dd773f --- /dev/null +++ b/web/app/Services/Cloud/CloudStorageImmutablePrimaryPublication.php @@ -0,0 +1,895 @@ + */ + private readonly \Closure $runtimeResolver; + + /** @var \Closure():int */ + private readonly \Closure $clock; + + /** + * @param null|callable(int,bool):array $runtimeResolver + * @param null|callable():int $clock + */ + public function __construct( + private readonly ?CloudObjectStorageS3Client $client = null, + ?callable $runtimeResolver = null, + ?callable $clock = null + ) { + $this->runtimeResolver = $runtimeResolver === null + ? static fn (int $backendId, bool $healthy): array => + CloudObjectStorageBackendRegistry::runtime( + $backendId, + $healthy + ) + : \Closure::fromCallable($runtimeResolver); + $this->clock = $clock === null + ? static fn (): int => time() + : \Closure::fromCallable($clock); + } + + /** + * Advance one immutable primary publication by at most one remote action. + * + * @param array{path:string,size_bytes:int,sha256:string} $file + * @return array + */ + public function publish( + string $operationId, + int $expectedArtifactId, + array $file + ): array { + self::assertOperationId($operationId); + if ($expectedArtifactId < 1) { + throw new CloudError('对象存储制品身份无效', 422); + } + $snapshot = $this->snapshot($operationId, $expectedArtifactId); + if (($snapshot['terminal'] ?? false) === true) { + return $snapshot; + } + // Reconciliation is intentionally independent of the temporary local + // file. Once a PUT may have happened, only the frozen ledger identity + // and an exact-key HEAD are authoritative. + if ((string) $snapshot['phase'] === 'accepted') { + self::assertFile($file, $snapshot); + } + $now = ($this->clock)(); + $work = $this->claim( + $operationId, + $expectedArtifactId, + $now + ); + if ($work === null) { + return $this->snapshot($operationId, $expectedArtifactId); + } + if ((string) $work['mode'] === 'dispatch') { + try { + // Recheck after the dispatch lease is owned so a changed + // source can never reach the provider under this intent. + self::assertFile($file, $work); + } catch (\Throwable $error) { + $this->safeFailure( + $work, + 'immutable_primary_source_changed', + $now + ); + throw $error; + } + } + try { + $runtime = ($this->runtimeResolver)( + (int) $work['backend_id'], + (string) $work['mode'] === 'dispatch' + ); + self::assertRuntime($runtime); + } catch (\Throwable) { + if ((string) $work['mode'] === 'dispatch') { + $this->safeFailure( + $work, + 'immutable_primary_runtime_unavailable', + $now + ); + } else { + $this->reconcileFailure( + $work, + 'immutable_primary_runtime_unavailable', + $now + ); + } + return $this->snapshot($operationId, $expectedArtifactId); + } + + if ((string) $work['mode'] === 'dispatch') { + $this->dispatch($work, $runtime, $file); + } else { + $this->reconcile($work, $runtime); + } + return $this->snapshot($operationId, $expectedArtifactId); + } + + /** @return array */ + private function snapshot( + string $operationId, + int $expectedArtifactId + ): array { + $operation = CloudStorageOperation::query() + ->where('operation_id', $operationId)->first(); + if (!$operation instanceof CloudStorageOperation + || (string) $operation->kind !== 'publish' + || (int) $operation->artifact_id !== $expectedArtifactId) { + throw new CloudError('对象存储发布操作与制品身份不一致', 409); + } + $artifact = CloudStorageArtifact::query() + ->whereKey($expectedArtifactId)->first(); + $replica = CloudStorageReplica::query() + ->whereKey((int) $operation->replica_id)->first(); + $publication = CloudStoragePublication::query() + ->where('artifact_id', $expectedArtifactId)->first(); + if (!$artifact instanceof CloudStorageArtifact + || !$replica instanceof CloudStorageReplica + || !$publication instanceof CloudStoragePublication + || (int) $replica->artifact_id !== $expectedArtifactId + || (string) $replica->role !== 'primary') { + throw new CloudError('对象存储主副本账本不一致', 409); + } + $terminal = (string) $operation->phase === 'terminal'; + return [ + 'operation_id' => (string) $operation->operation_id, + 'artifact_id' => (int) $artifact->id, + 'phase' => (string) $operation->phase, + 'outcome' => $operation->outcome === null + ? null : (string) $operation->outcome, + 'effect_resolution' => (string) $operation->effect_resolution, + 'attempts' => (int) $operation->attempts, + 'next_check_at' => (int) $operation->next_check_at, + 'error_code' => (string) $operation->error_code, + 'terminal' => $terminal, + 'primary_completed' => $terminal + && (string) $operation->outcome === 'succeeded', + 'publication_state' => (string) $publication->state, + 'ready_to_finalize' => in_array( + (string) $publication->state, + ['ready_to_finalize', 'finalizing', 'completed'], + true + ), + 'requires_attention' => ($terminal + && (string) $operation->outcome === 'failed') + || (string) $publication->state === 'failed', + 'object_key' => (string) $replica->object_key, + 'size_bytes' => (int) $artifact->size_bytes, + 'sha256' => (string) $artifact->sha256, + ]; + } + + /** @return array|null */ + private function claim( + string $operationId, + int $expectedArtifactId, + int $now + ): ?array { + $connection = (new CloudStorageOperation())->getConnection(); + return $connection->transaction(function () use ( + $operationId, + $expectedArtifactId, + $now + ): ?array { + $operation = CloudStorageOperation::query() + ->where('operation_id', $operationId) + ->lockForUpdate()->first(); + if (!$operation instanceof CloudStorageOperation + || (string) $operation->kind !== 'publish' + || (int) $operation->artifact_id !== $expectedArtifactId) { + throw new CloudError( + '对象存储发布操作与制品身份不一致', + 409 + ); + } + [$artifact, $replica, $publication] = + $this->lockedContext($operation); + if ((string) $operation->phase === 'terminal' + || in_array((string) $publication->state, + ['completed', 'failed', 'retired'], true)) { + return null; + } + if ((string) $artifact->state !== 'publishing' + || !in_array((string) $replica->state, + ['reserved', 'publishing'], true)) { + throw new CloudError('对象存储主副本发布状态不一致', 409); + } + if (is_string($operation->lease_token) + && $operation->lease_token !== '' + && (int) $operation->lease_until > $now) { + return null; + } + + $phase = (string) $operation->phase; + $mode = null; + if ($phase === 'accepted' + && (int) $operation->available_at <= $now) { + $mode = 'dispatch'; + $operation->phase = 'dispatching'; + $operation->effect_resolution = 'not_applicable'; + $operation->attempts = (int) $operation->attempts + 1; + $replica->state = 'publishing'; + } elseif ($phase === 'reconciling' + && (int) $operation->next_check_at <= $now) { + $mode = 'reconcile'; + } elseif (in_array($phase, + ['dispatching', 'awaiting_receipt'], true) + && (int) $operation->lease_until <= $now) { + $mode = 'reconcile'; + $operation->phase = 'reconciling'; + $operation->effect_resolution = 'unknown'; + $operation->error_code = + 'immutable_primary_lease_expired'; + $operation->next_check_at = $now; + } + if ($mode === null) { + return null; + } + $lease = bin2hex(random_bytes(16)); + $operation->lease_token = $lease; + $operation->lease_until = $now + self::LEASE_SECONDS; + $operation->updated_at = $now; + $operation->save(); + $replica->lease_token = $lease; + $replica->lease_until = $now + self::LEASE_SECONDS; + $replica->updated_at = $now; + $replica->save(); + return [ + 'mode' => $mode, + 'operation_id' => (string) $operation->operation_id, + 'artifact_id' => (int) $artifact->id, + 'replica_id' => (int) $replica->id, + 'backend_id' => (int) $replica->backend_id, + 'object_key' => (string) $replica->object_key, + 'size_bytes' => (int) $artifact->size_bytes, + 'sha256' => (string) $artifact->sha256, + 'lease_token' => $lease, + 'attempts' => (int) $operation->attempts, + ]; + }, 3); + } + + /** + * @param array $work + * @param array $runtime + * @param array{path:string,size_bytes:int,sha256:string} $file + */ + private function dispatch(array $work, array $runtime, array $file): void + { + try { + $identity = $this->s3()->putImmutableFile( + $runtime['config'], + $runtime['credentials'], + (string) $work['object_key'], + (string) $file['path'], + (int) $file['size_bytes'], + (string) $file['sha256'] + ); + } catch (CloudObjectStorageTransferError $error) { + if ($error->effectMayHaveApplied + || $error->publicCode === 'object_already_exists') { + $this->markUnknown($work, $error->publicCode); + } else { + $this->safeFailure( + $work, + $error->publicCode, + ($this->clock)() + ); + } + return; + } catch (\Throwable) { + $this->markUnknown($work, 'immutable_primary_put_unknown'); + return; + } + if ((int) ($identity['size_bytes'] ?? 0) + !== (int) $work['size_bytes'] + || !is_string($identity['etag'] ?? null) + || !is_string($identity['version_id'] ?? null)) { + $this->markUnknown( + $work, + 'immutable_primary_put_receipt_invalid' + ); + return; + } + CloudStorageArtifactService::completePrimary( + (string) $work['operation_id'], + (int) $work['artifact_id'], + [ + 'size_bytes' => (int) $work['size_bytes'], + 'sha256' => (string) $work['sha256'], + 'etag' => (string) $identity['etag'], + 'version_id' => (string) $identity['version_id'], + ] + ); + } + + /** @param array $work @param array $runtime */ + private function reconcile(array $work, array $runtime): void + { + try { + $identity = $this->s3()->inspectObject( + $runtime['config'], + $runtime['credentials'], + (string) $work['object_key'] + ); + } catch (\Throwable) { + $this->reconcileFailure( + $work, + 'immutable_primary_head_unavailable', + ($this->clock)() + ); + return; + } + if ($identity === null) { + $this->recordAbsence($work, ($this->clock)()); + return; + } + if ((int) ($identity['size_bytes'] ?? 0) + !== (int) $work['size_bytes'] + || !is_string($identity['sha256'] ?? null) + || !hash_equals( + (string) $work['sha256'], + (string) $identity['sha256'] + )) { + $this->attention( + $work, + 'immutable_primary_identity_conflict', + ($this->clock)() + ); + return; + } + CloudStorageArtifactService::completePrimary( + (string) $work['operation_id'], + (int) $work['artifact_id'], + [ + 'size_bytes' => (int) $work['size_bytes'], + 'sha256' => (string) $work['sha256'], + 'etag' => (string) ($identity['etag'] ?? ''), + 'version_id' => (string) ($identity['version_id'] ?? ''), + ] + ); + } + + /** @param array $work */ + private function markUnknown(array $work, string $code): void + { + $now = ($this->clock)(); + $this->mutateLeased($work, function ( + CloudStorageOperation $operation, + CloudStorageArtifact $artifact, + CloudStorageReplica $replica, + CloudStoragePublication $publication, + array $copyOperations, + array $replicas + ) use ($code, $now): void { + $metadata = self::metadata($operation->result_json); + $metadata['absence_confirmations'] = 0; + $operation->phase = 'reconciling'; + $operation->effect_resolution = 'unknown'; + $operation->lease_token = null; + $operation->lease_until = 0; + $operation->next_check_at = $now + self::RECONCILE_DELAY; + $operation->error_code = self::errorCode($code); + $operation->result_json = self::encodedMetadata($metadata); + $operation->updated_at = $now; + $operation->save(); + $replica->state = 'publishing'; + $replica->lease_token = null; + $replica->lease_until = 0; + $replica->error_code = self::errorCode($code); + $replica->updated_at = $now; + $replica->save(); + $artifact->state = 'publishing'; + $artifact->updated_at = $now; + $artifact->save(); + }); + } + + /** @param array $work */ + private function safeFailure(array $work, string $code, int $now): void + { + $this->mutateLeased($work, function ( + CloudStorageOperation $operation, + CloudStorageArtifact $artifact, + CloudStorageReplica $replica, + CloudStoragePublication $publication, + array $copyOperations, + array $replicas + ) use ($code, $now): void { + $code = self::errorCode($code); + if ((int) $operation->attempts >= self::MAX_DISPATCH_ATTEMPTS) { + $this->settleAttention( + $operation, + $artifact, + $replica, + $publication, + $copyOperations, + $replicas, + false, + 'immutable_primary_publish_retry_exhausted', + $now + ); + return; + } + $operation->phase = 'accepted'; + $operation->effect_resolution = 'not_applicable'; + $operation->lease_token = null; + $operation->lease_until = 0; + $operation->available_at = $now + self::RETRY_DELAY; + $operation->next_check_at = 0; + $operation->error_code = $code; + $operation->result_json = self::encodedMetadata( + self::resetEvidence($operation->result_json) + ); + $operation->updated_at = $now; + $operation->save(); + $replica->state = 'reserved'; + $replica->lease_token = null; + $replica->lease_until = 0; + $replica->error_code = $code; + $replica->updated_at = $now; + $replica->save(); + }); + } + + /** @param array $work */ + private function reconcileFailure( + array $work, + string $code, + int $now + ): void { + $this->mutateLeased($work, function ( + CloudStorageOperation $operation, + CloudStorageArtifact $artifact, + CloudStorageReplica $replica, + CloudStoragePublication $publication, + array $copyOperations, + array $replicas + ) use ($code, $now): void { + $metadata = self::metadata($operation->result_json); + $failures = (int) ($metadata['reconcile_failures'] ?? 0) + 1; + if ($failures >= self::MAX_RECONCILE_FAILURES) { + $this->settleAttention( + $operation, + $artifact, + $replica, + $publication, + $copyOperations, + $replicas, + true, + 'immutable_primary_reconcile_exhausted', + $now + ); + return; + } + $metadata['reconcile_failures'] = $failures; + $operation->phase = 'reconciling'; + $operation->effect_resolution = 'unknown'; + $operation->lease_token = null; + $operation->lease_until = 0; + $operation->next_check_at = $now + self::RETRY_DELAY; + $operation->error_code = self::errorCode($code); + $operation->result_json = self::encodedMetadata($metadata); + $operation->updated_at = $now; + $operation->save(); + $replica->state = 'publishing'; + $replica->lease_token = null; + $replica->lease_until = 0; + $replica->error_code = self::errorCode($code); + $replica->updated_at = $now; + $replica->save(); + }); + } + + /** @param array $work */ + private function recordAbsence(array $work, int $now): void + { + $this->mutateLeased($work, function ( + CloudStorageOperation $operation, + CloudStorageArtifact $artifact, + CloudStorageReplica $replica, + CloudStoragePublication $publication, + array $copyOperations, + array $replicas + ) use ($now): void { + $metadata = self::metadata($operation->result_json); + $confirmations = min( + 2, + (int) ($metadata['absence_confirmations'] ?? 0) + 1 + ); + $metadata['absence_confirmations'] = $confirmations; + $metadata['reconcile_failures'] = 0; + if ($confirmations < 2) { + $operation->phase = 'reconciling'; + $operation->effect_resolution = 'unknown'; + $operation->lease_token = null; + $operation->lease_until = 0; + $operation->next_check_at = $now + self::RECONCILE_DELAY; + $operation->error_code = 'immutable_primary_absent_once'; + $operation->result_json = self::encodedMetadata($metadata); + $operation->updated_at = $now; + $operation->save(); + $replica->state = 'publishing'; + $replica->lease_token = null; + $replica->lease_until = 0; + $replica->error_code = 'immutable_primary_absent_once'; + $replica->updated_at = $now; + $replica->save(); + return; + } + if ((int) $operation->attempts >= self::MAX_DISPATCH_ATTEMPTS) { + $this->settleAttention( + $operation, + $artifact, + $replica, + $publication, + $copyOperations, + $replicas, + false, + 'immutable_primary_absence_retry_exhausted', + $now + ); + return; + } + $operation->phase = 'accepted'; + $operation->effect_resolution = 'not_applicable'; + $operation->lease_token = null; + $operation->lease_until = 0; + $operation->available_at = $now + self::RECONCILE_DELAY; + $operation->next_check_at = 0; + $operation->error_code = ''; + $operation->result_json = self::encodedMetadata( + self::resetEvidence($operation->result_json) + ); + $operation->updated_at = $now; + $operation->save(); + $replica->state = 'reserved'; + $replica->lease_token = null; + $replica->lease_until = 0; + $replica->error_code = ''; + $replica->updated_at = $now; + $replica->save(); + }); + } + + /** @param array $work */ + private function attention(array $work, string $code, int $now): void + { + $this->mutateLeased($work, function ( + CloudStorageOperation $operation, + CloudStorageArtifact $artifact, + CloudStorageReplica $replica, + CloudStoragePublication $publication, + array $copyOperations, + array $replicas + ) use ($code, $now): void { + $this->settleAttention( + $operation, + $artifact, + $replica, + $publication, + $copyOperations, + $replicas, + true, + self::errorCode($code), + $now + ); + }); + } + + /** + * @param array $work + * @param callable(CloudStorageOperation,CloudStorageArtifact, + * CloudStorageReplica,CloudStoragePublication, + * array,array):void $callback + */ + private function mutateLeased(array $work, callable $callback): void + { + $connection = (new CloudStorageOperation())->getConnection(); + $connection->transaction(function () use ($work, $callback): void { + $operation = CloudStorageOperation::query() + ->where('operation_id', (string) $work['operation_id']) + ->lockForUpdate()->firstOrFail(); + if ((int) $operation->artifact_id !== (int) $work['artifact_id']) { + throw new CloudError('对象存储发布制品身份已变化', 409); + } + $copyOperations = CloudStorageOperation::query() + ->where('artifact_id', (int) $operation->artifact_id) + ->where('kind', 'copy')->orderBy('id') + ->lockForUpdate()->get()->all(); + [$artifact, $replica, $publication, $replicas] = + $this->lockedContext($operation, true); + if (!hash_equals( + (string) $operation->lease_token, + (string) $work['lease_token'] + ) + || !hash_equals( + (string) $replica->lease_token, + (string) $work['lease_token'] + )) { + throw new CloudError('对象存储主副本发布租约已失效', 409); + } + $callback( + $operation, + $artifact, + $replica, + $publication, + $copyOperations, + $replicas + ); + }, 3); + } + + /** + * @return array{0:CloudStorageArtifact,1:CloudStorageReplica, + * 2:CloudStoragePublication,3?:array} + */ + private function lockedContext( + CloudStorageOperation $operation, + bool $includeReplicas = false + ): array { + $artifact = CloudStorageArtifact::query() + ->whereKey((int) $operation->artifact_id) + ->lockForUpdate()->firstOrFail(); + $replicas = CloudStorageReplica::query() + ->where('artifact_id', (int) $artifact->id)->orderBy('id') + ->lockForUpdate()->get()->all(); + $replica = null; + foreach ($replicas as $candidate) { + if ((int) $candidate->id === (int) $operation->replica_id) { + $replica = $candidate; + break; + } + } + $publication = CloudStoragePublication::query() + ->where('artifact_id', (int) $artifact->id) + ->lockForUpdate()->firstOrFail(); + if (!$replica instanceof CloudStorageReplica + || (string) $replica->role !== 'primary') { + throw new CloudError('对象存储主副本账本不一致', 409); + } + return $includeReplicas + ? [$artifact, $replica, $publication, $replicas] + : [$artifact, $replica, $publication]; + } + + /** + * @param array $copyOperations + * @param array $replicas + */ + private function settleAttention( + CloudStorageOperation $operation, + CloudStorageArtifact $artifact, + CloudStorageReplica $replica, + CloudStoragePublication $publication, + array $copyOperations, + array $replicas, + bool $effectUnknown, + string $code, + int $now + ): void { + $operation->phase = 'terminal'; + $operation->outcome = 'failed'; + $operation->effect_resolution = $effectUnknown + ? 'unknown' : 'not_applied'; + $operation->effect_may_have_applied = $effectUnknown ? 1 : 0; + $operation->safe_to_resubmit = 0; + $operation->lease_token = null; + $operation->lease_until = 0; + $operation->available_at = 0; + $operation->next_check_at = 0; + $operation->error_code = self::errorCode($code); + $operation->terminal_at = $now; + $operation->updated_at = $now; + $operation->save(); + $replica->state = 'degraded'; + $replica->lease_token = null; + $replica->lease_until = 0; + $replica->error_code = self::errorCode($code); + $replica->updated_at = $now; + $replica->save(); + foreach ($copyOperations as $copy) { + if ((string) $copy->phase === 'terminal') { + continue; + } + // Secondary work is blocked until primary success. Never invent a + // safe non-dispatch result if an inconsistent ledger says a copy + // has already crossed its own dispatch fence. + if ((string) $copy->phase !== 'accepted') { + continue; + } + $copy->phase = 'terminal'; + $copy->outcome = 'failed'; + $copy->effect_resolution = 'not_applied'; + $copy->effect_may_have_applied = 0; + $copy->safe_to_resubmit = 0; + $copy->lease_token = null; + $copy->lease_until = 0; + $copy->available_at = 0; + $copy->next_check_at = 0; + $copy->error_code = 'primary_publish_attention'; + $copy->terminal_at = $now; + $copy->updated_at = $now; + $copy->save(); + foreach ($replicas as $secondary) { + if ((int) $secondary->id !== (int) $copy->replica_id) { + continue; + } + $secondary->state = 'degraded'; + $secondary->lease_token = null; + $secondary->lease_until = 0; + $secondary->error_code = 'primary_publish_attention'; + $secondary->updated_at = $now; + $secondary->save(); + break; + } + } + $artifact->state = 'degraded'; + $artifact->updated_at = $now; + $artifact->save(); + $publication->state = 'failed'; + $publication->available_at = 0; + $publication->lease_token = null; + $publication->lease_until = 0; + $publication->last_error_code = self::errorCode($code); + $publication->updated_at = $now; + $publication->save(); + } + + /** + * @return array{absence_confirmations:int,reconcile_failures:int, + * maximum_size_bytes?:int,object_suffix?:string} + */ + private static function metadata(?string $json): array + { + if ($json === null || trim($json) === '') { + return [ + 'absence_confirmations' => 0, + 'reconcile_failures' => 0, + ]; + } + try { + $value = json_decode($json, true, 16, JSON_THROW_ON_ERROR); + } catch (\Throwable) { + throw new CloudError('对象存储发布核对证据无效', 409); + } + if (!is_array($value) || array_is_list($value) + || array_diff(array_keys($value), [ + 'absence_confirmations', 'reconcile_failures', + 'maximum_size_bytes', 'object_suffix', + ]) !== []) { + throw new CloudError('对象存储发布核对证据无效', 409); + } + $absence = $value['absence_confirmations'] ?? 0; + $failures = $value['reconcile_failures'] ?? 0; + if (!is_int($absence) || $absence < 0 || $absence > 2 + || !is_int($failures) || $failures < 0 + || $failures > self::MAX_RECONCILE_FAILURES) { + throw new CloudError('对象存储发布核对证据无效', 409); + } + $metadata = [ + 'absence_confirmations' => $absence, + 'reconcile_failures' => $failures, + ]; + if (array_key_exists('maximum_size_bytes', $value)) { + $maximum = $value['maximum_size_bytes']; + if (!is_int($maximum) || $maximum < 1) { + throw new CloudError('对象存储发布核对证据无效', 409); + } + $metadata['maximum_size_bytes'] = $maximum; + } + if (array_key_exists('object_suffix', $value)) { + $suffix = $value['object_suffix']; + if (!is_string($suffix) || $suffix === '' + || strlen($suffix) > 512 + || preg_match('/[\x00-\x1f\x7f]/', $suffix) === 1) { + throw new CloudError('对象存储发布核对证据无效', 409); + } + $metadata['object_suffix'] = $suffix; + } + return $metadata; + } + + private static function encodedMetadata(array $metadata): string + { + $normalized = self::metadata(json_encode( + $metadata, + JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES + )); + return json_encode( + $normalized, + JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES + ); + } + + /** + * Preserve beginPublication()'s immutable context while clearing only + * this module's transient reconciliation evidence. + * + * @return array{absence_confirmations:int,reconcile_failures:int, + * maximum_size_bytes?:int,object_suffix?:string} + */ + private static function resetEvidence(?string $json): array + { + $metadata = self::metadata($json); + $metadata['absence_confirmations'] = 0; + $metadata['reconcile_failures'] = 0; + return $metadata; + } + + /** @param array $file @param array $snapshot */ + private static function assertFile(array $file, array $snapshot): void + { + $path = $file['path'] ?? null; + $size = $file['size_bytes'] ?? null; + $sha = $file['sha256'] ?? null; + $actual = is_string($path) && is_file($path) && is_readable($path) + ? hash_file('sha256', $path) : false; + if (!is_string($path) || !is_int($size) || $size < 1 + || !is_string($sha) + || preg_match('/^[a-f0-9]{64}$/D', $sha) !== 1 + || @filesize($path) !== $size + || !is_string($actual) || !hash_equals($sha, $actual) + || $size !== (int) $snapshot['size_bytes'] + || !hash_equals($sha, (string) $snapshot['sha256'])) { + throw new CloudError('本地不可变制品文件身份无效', 422); + } + } + + /** @param array $runtime */ + private static function assertRuntime(array $runtime): void + { + if (!is_array($runtime['config'] ?? null) + || array_is_list($runtime['config']) + || !is_array($runtime['credentials'] ?? null) + || array_is_list($runtime['credentials']) + || !is_int($runtime['revision'] ?? null) + || (int) $runtime['revision'] < 1) { + throw new CloudError('对象存储发布运行时无效', 503); + } + } + + private static function assertOperationId(string $operationId): void + { + if (preg_match('/^[a-f0-9]{32}$/D', $operationId) !== 1) { + throw new CloudError('对象存储发布操作标识无效', 422); + } + } + + private static function errorCode(string $code): string + { + $code = strtolower(trim($code)); + return preg_match('/^[a-z][a-z0-9_]{0,95}$/D', $code) === 1 + ? $code : 'immutable_primary_publication_failed'; + } + + private function s3(): CloudObjectStorageS3Client + { + return $this->client ?? new CloudObjectStorageS3Client(); + } +} diff --git a/web/tests/CloudStorageImmutablePrimaryPublicationTest.php b/web/tests/CloudStorageImmutablePrimaryPublicationTest.php new file mode 100644 index 00000000..7783ac4f --- /dev/null +++ b/web/tests/CloudStorageImmutablePrimaryPublicationTest.php @@ -0,0 +1,544 @@ +httpStatus === $status, + 'Unexpected CloudError status: ' . $error->httpStatus); + return; + } + throw new RuntimeException('Expected CloudError was not raised.'); +}; + +$db = new Capsule(); +$db->addConnection(['driver' => 'sqlite', 'database' => ':memory:']); +$db->setAsGlobal(); +$db->bootEloquent(); +$db->schema()->create('cloud_storage_backends', static function ( + Blueprint $table +): void { + $table->increments('id'); + $table->string('backend_key'); + $table->string('identity_sha256'); + $table->string('name'); + $table->string('driver'); + $table->string('endpoint'); + $table->string('region'); + $table->string('bucket'); + $table->string('prefix'); + $table->boolean('path_style'); + $table->string('failure_domain'); + $table->text('sealed_credentials'); + $table->unsignedBigInteger('config_revision'); + $table->unsignedBigInteger('tested_revision'); + $table->string('admin_state'); + $table->string('health_state'); + $table->unsignedInteger('write_priority'); + $table->unsignedInteger('consecutive_failures')->default(0); + $table->unsignedInteger('consecutive_successes')->default(0); + $table->string('last_error_code')->default(''); + $table->unsignedBigInteger('last_tested_at')->default(0); + $table->unsignedBigInteger('last_healthy_at')->default(0); + $table->unsignedBigInteger('next_probe_at')->default(0); + $table->unsignedBigInteger('created_at'); + $table->unsignedBigInteger('updated_at'); +}); +$db->schema()->create('cloud_storage_artifacts', static function ( + Blueprint $table +): void { + $table->increments('id'); + $table->string('artifact_key')->unique(); + $table->string('kind'); + $table->unsignedBigInteger('maximum_size_bytes'); + $table->unsignedBigInteger('size_bytes'); + $table->string('sha256'); + $table->string('durability_policy'); + $table->unsignedTinyInteger('desired_copies'); + $table->unsignedTinyInteger('required_copies'); + $table->string('state'); + $table->string('last_operation_id')->nullable(); + $table->unsignedBigInteger('verified_at'); + $table->unsignedBigInteger('created_at'); + $table->unsignedBigInteger('updated_at'); +}); +$db->schema()->create('cloud_storage_replicas', static function ( + Blueprint $table +): void { + $table->increments('id'); + $table->string('replica_key')->unique(); + $table->unsignedBigInteger('artifact_id'); + $table->unsignedBigInteger('backend_id'); + $table->string('role'); + $table->string('object_key'); + $table->string('object_version_id')->default(''); + $table->string('object_etag')->default(''); + $table->unsignedBigInteger('size_bytes')->default(0); + $table->string('sha256')->default(''); + $table->string('state'); + $table->string('publish_operation_id')->nullable(); + $table->string('delete_operation_id')->nullable(); + $table->unsignedInteger('delete_generation')->default(0); + $table->unsignedInteger('delete_cycle_attempts')->default(0); + $table->unsignedInteger('delete_presence_confirmations')->default(0); + $table->unsignedBigInteger('delete_last_checked_at')->default(0); + $table->unsignedBigInteger('delete_next_check_at')->default(0); + $table->unsignedBigInteger('delete_attention_at')->nullable(); + $table->string('lease_token')->nullable(); + $table->unsignedBigInteger('lease_until')->default(0); + $table->string('error_code')->default(''); + $table->unsignedBigInteger('verified_at')->default(0); + $table->unsignedBigInteger('created_at'); + $table->unsignedBigInteger('updated_at'); +}); +$db->schema()->create('cloud_storage_operations', static function ( + Blueprint $table +): void { + $table->increments('id'); + $table->string('operation_id')->unique(); + $table->string('request_idempotency_key')->unique(); + $table->string('request_fingerprint'); + $table->unsignedBigInteger('artifact_id'); + $table->unsignedBigInteger('replica_id')->nullable(); + $table->unsignedBigInteger('source_backend_id')->nullable(); + $table->unsignedBigInteger('target_backend_id')->nullable(); + $table->string('kind'); + $table->string('phase'); + $table->string('outcome')->nullable(); + $table->string('effect_resolution'); + $table->boolean('effect_may_have_applied')->nullable(); + $table->boolean('safe_to_resubmit')->nullable(); + $table->unsignedInteger('attempts'); + $table->string('lease_token')->nullable(); + $table->unsignedBigInteger('lease_until'); + $table->unsignedBigInteger('available_at'); + $table->unsignedBigInteger('next_check_at'); + $table->string('error_code'); + $table->text('result_json')->nullable(); + $table->unsignedBigInteger('terminal_at')->nullable(); + $table->unsignedBigInteger('created_at'); + $table->unsignedBigInteger('updated_at'); +}); +$db->schema()->create('cloud_storage_publications', static function ( + Blueprint $table +): void { + $table->increments('id'); + $table->string('publication_key')->unique(); + $table->unsignedBigInteger('artifact_id')->unique(); + $table->unsignedBigInteger('backup_id')->nullable()->unique(); + $table->string('template_import_intent_id')->nullable()->unique(); + $table->string('rescue_template_id')->nullable()->unique(); + $table->string('state'); + $table->unsignedInteger('attempts')->default(0); + $table->unsignedBigInteger('available_at')->default(0); + $table->string('lease_token')->nullable(); + $table->unsignedBigInteger('lease_until')->default(0); + $table->string('last_error_code')->default(''); + $table->unsignedBigInteger('coverage_ready_at')->nullable(); + $table->unsignedBigInteger('finalized_at')->nullable(); + $table->unsignedBigInteger('created_at'); + $table->unsignedBigInteger('updated_at'); +}); + +$databaseNow = time(); +foreach ([ + [1, 'primary', 'provider-a', 10], + [2, 'secondary', 'provider-b', 20], +] as [$id, $name, $domain, $priority]) { + Capsule::table('cloud_storage_backends')->insert([ + 'id' => $id, + 'backend_key' => str_repeat((string) $id, 32), + 'identity_sha256' => str_repeat((string) $id, 64), + 'name' => $name, + 'driver' => 's3', + 'endpoint' => 'https://' . $name . '.example.test', + 'region' => 'us-east-1', + 'bucket' => $name . '-bucket', + 'prefix' => $name, + 'path_style' => 1, + 'failure_domain' => $domain, + 'sealed_credentials' => 'sealed', + 'config_revision' => 1, + 'tested_revision' => 1, + 'admin_state' => 'active', + 'health_state' => 'healthy', + 'write_priority' => $priority, + 'consecutive_failures' => 0, + 'consecutive_successes' => 1, + 'last_error_code' => '', + 'last_tested_at' => $databaseNow, + 'last_healthy_at' => $databaseNow, + 'next_probe_at' => $databaseNow + 300, + 'created_at' => $databaseNow, + 'updated_at' => $databaseNow, + ]); +} + +$temporaryPaths = []; +register_shutdown_function(static function () use (&$temporaryPaths): void { + foreach ($temporaryPaths as $path) { + @unlink($path); + } +}); +$fixtureSequence = 0; +$fixture = static function ( + string $kind = 'template' +) use (&$fixtureSequence, &$temporaryPaths): array { + $fixtureSequence++; + $path = tempnam(sys_get_temp_dir(), 'virt-primary-publication-'); + if ($path === false) { + throw new RuntimeException('Cannot create publication fixture.'); + } + file_put_contents( + $path, + str_repeat('immutable-primary-' . $fixtureSequence . '-', 64) + ); + $temporaryPaths[] = $path; + $size = filesize($path); + $sha256 = hash_file('sha256', $path); + if (!is_int($size) || !is_string($sha256)) { + throw new RuntimeException('Cannot identify publication fixture.'); + } + $ownerId = str_pad(dechex($fixtureSequence), 32, '0', STR_PAD_LEFT); + $owner = $kind === 'rescue' + ? ['rescue_template_id' => $ownerId] + : ['template_import_intent_id' => $ownerId]; + $view = CloudStorageArtifactService::beginPublication( + $kind, + $kind . '/fixture-' . $fixtureSequence . '/system.qcow2', + $size, + $sha256, + 'immutable-primary-' . str_pad( + (string) $fixtureSequence, + 4, + '0', + STR_PAD_LEFT + ), + hash('sha256', $kind . '|' . $fixtureSequence), + $owner + ); + return [ + 'path' => $path, + 'size_bytes' => $size, + 'sha256' => $sha256, + 'operation_id' => (string) $view['operation_id'], + 'artifact_id' => (int) $view['artifact_id'], + 'replica_id' => (int) $view['replica_id'], + 'object_key' => (string) $view['object_key'], + ]; +}; + +$publisher = static function ( + array &$responses, + array &$requests, + array &$transactionLevels, + int &$clock +): CloudStorageImmutablePrimaryPublication { + $handler = static function ( + RequestInterface $request, + array $options + ) use (&$responses, &$requests, &$transactionLevels) { + $transactionLevels[] = Capsule::connection()->transactionLevel(); + $requests[] = $request; + if ($responses === []) { + throw new RuntimeException('Unexpected object-storage request.'); + } + $response = array_shift($responses); + if (!$response instanceof Response) { + throw new RuntimeException('Invalid object-storage response fixture.'); + } + return Create::promiseFor($response); + }; + $client = new CloudObjectStorageS3Client(new Client([ + 'handler' => HandlerStack::create($handler), + ])); + return new CloudStorageImmutablePrimaryPublication( + $client, + static fn (int $backendId, bool $healthy): array => [ + 'backend_id' => $backendId, + 'revision' => 1, + 'config' => [ + 'endpoint' => 'https://objects.example.test', + 'region' => 'us-east-1', + 'bucket' => 'immutable-fixtures', + 'prefix' => 'unused-runtime-prefix', + 'path_style' => true, + ], + 'credentials' => [ + 'access_key_id' => 'IMMUTABLEACCESSKEY', + 'secret_access_key' => 'immutable-secret-must-not-persist', + ], + ], + static function () use (&$clock): int { + return $clock; + } + ); +}; + +// A successful immutable PUT settles only the primary. Template publication +// remains behind the required-two protection gate until its copy completes. +$applied = $fixture('template'); +$clock = time() + 10; +$responses = [new Response(200, [ + 'ETag' => '"applied-etag"', + 'x-amz-version-id' => 'applied-version', +])]; +$requests = []; +$transactionLevels = []; +$module = $publisher($responses, $requests, $transactionLevels, $clock); +$appliedResult = $module->publish( + $applied['operation_id'], + $applied['artifact_id'], + $applied +); +$assert($appliedResult['primary_completed'] === true + && $appliedResult['publication_state'] === 'waiting_protection' + && $appliedResult['ready_to_finalize'] === false, + 'Primary success bypassed the template publication quorum.'); +$assert(count($requests) === 1 && $requests[0]->getMethod() === 'PUT' + && $requests[0]->getHeaderLine('If-None-Match') === '*' + && $requests[0]->getHeaderLine('x-amz-meta-virt-sha256') + === $applied['sha256'] + && str_contains( + $requests[0]->getHeaderLine('Authorization'), + 'x-amz-meta-virt-sha256' + ), 'Immutable primary PUT lost its conditional signed identity.'); +$assert($transactionLevels === [0], + 'Immutable primary PUT ran inside a database transaction.'); + +// A provider rejection proves non-application and schedules a bounded retry; +// it never recursively replays inside the same call. +$rejected = $fixture('rescue'); +$clock += 1000; +$responses = [new Response(403)]; +$requests = []; +$transactionLevels = []; +$module = $publisher($responses, $requests, $transactionLevels, $clock); +$rejectedResult = $module->publish( + $rejected['operation_id'], + $rejected['artifact_id'], + $rejected +); +$rejectedOperation = CloudStorageOperation::where( + 'operation_id', + $rejected['operation_id'] +)->firstOrFail(); +$rejectedReplica = CloudStorageReplica::whereKey( + $rejected['replica_id'] +)->firstOrFail(); +$assert($rejectedResult['phase'] === 'accepted' + && (int) $rejectedOperation->attempts === 1 + && $rejectedOperation->outcome === null + && (int) $rejectedOperation->available_at > $clock + && (string) $rejectedReplica->state === 'reserved' + && count($requests) === 1, + 'Proven non-application was not delayed without replay.'); + +// A lost PUT receipt is settled only by exact-key size plus signed SHA +// metadata. The temporary local file is deliberately removed before HEAD. +$unknownApplied = $fixture('template'); +$clock += 1000; +$responses = [ + new Response(503), + new Response(200, [ + 'Content-Length' => (string) $unknownApplied['size_bytes'], + 'ETag' => '"unknown-applied-etag"', + 'x-amz-version-id' => 'unknown-applied-version', + 'x-amz-meta-virt-sha256' => $unknownApplied['sha256'], + ]), +]; +$requests = []; +$transactionLevels = []; +$module = $publisher($responses, $requests, $transactionLevels, $clock); +$unknownResult = $module->publish( + $unknownApplied['operation_id'], + $unknownApplied['artifact_id'], + $unknownApplied +); +$assert($unknownResult['phase'] === 'reconciling', + 'Unknown PUT did not enter read-only reconciliation.'); +@unlink($unknownApplied['path']); +$clock += 60; +$unknownResult = $module->publish( + $unknownApplied['operation_id'], + $unknownApplied['artifact_id'], + $unknownApplied +); +$assert($unknownResult['primary_completed'] === true + && array_map( + static fn (RequestInterface $request): string => $request->getMethod(), + $requests + ) === ['PUT', 'HEAD'] + && (string) $requests[0]->getUri() === (string) $requests[1]->getUri(), + 'Unknown PUT was replayed or reconciled against a different key.'); +$assert($transactionLevels === [0, 0], + 'Immutable PUT reconciliation ran remote I/O inside a transaction.'); + +// Two spaced exact-key absences are required before the original operation, +// artifact and key may become dispatchable again. +$absent = $fixture('template'); +$clock += 1000; +$responses = [ + new Response(503), + new Response(404), + new Response(404), + new Response(200, [ + 'ETag' => '"retried-etag"', + 'x-amz-version-id' => 'retried-version', + ]), +]; +$requests = []; +$transactionLevels = []; +$module = $publisher($responses, $requests, $transactionLevels, $clock); +$artifactCount = CloudStorageArtifact::count(); +$operationCount = CloudStorageOperation::count(); +$module->publish($absent['operation_id'], $absent['artifact_id'], $absent); +$clock += 60; +$firstAbsent = $module->publish( + $absent['operation_id'], $absent['artifact_id'], $absent +); +$assert($firstAbsent['phase'] === 'reconciling', + 'One absence observation incorrectly authorized a retry.'); +$clock += 60; +$secondAbsent = $module->publish( + $absent['operation_id'], $absent['artifact_id'], $absent +); +$assert($secondAbsent['phase'] === 'accepted', + 'Two stable absence observations did not release the original intent.'); +$clock += 60; +$retried = $module->publish( + $absent['operation_id'], $absent['artifact_id'], $absent +); +$methods = array_map( + static fn (RequestInterface $request): string => $request->getMethod(), + $requests +); +$assert($retried['primary_completed'] === true + && $methods === ['PUT', 'HEAD', 'HEAD', 'PUT'] + && (string) $requests[0]->getUri() === (string) $requests[1]->getUri() + && (string) $requests[0]->getUri() === (string) $requests[2]->getUri() + && (string) $requests[0]->getUri() === (string) $requests[3]->getUri() + && (int) CloudStorageOperation::where( + 'operation_id', $absent['operation_id'] + )->value('attempts') === 2 + && CloudStorageArtifact::count() === $artifactCount + && CloudStorageOperation::count() === $operationCount, + 'Stable absence did not retry the same frozen publication intent.'); + +// Existing content with a different signed identity is an attention state, +// never evidence for success and never permission to overwrite the key. +$conflict = $fixture('rescue'); +$clock += 1000; +$responses = [ + new Response(503), + new Response(200, [ + 'Content-Length' => (string) $conflict['size_bytes'], + 'ETag' => '"conflicting-etag"', + 'x-amz-version-id' => 'conflicting-version', + 'x-amz-meta-virt-sha256' => str_repeat('f', 64), + ]), +]; +$requests = []; +$transactionLevels = []; +$module = $publisher($responses, $requests, $transactionLevels, $clock); +$module->publish($conflict['operation_id'], $conflict['artifact_id'], $conflict); +$clock += 60; +$conflictResult = $module->publish( + $conflict['operation_id'], $conflict['artifact_id'], $conflict +); +$conflictOperation = CloudStorageOperation::where( + 'operation_id', $conflict['operation_id'] +)->firstOrFail(); +$assert($conflictResult['requires_attention'] === true + && (string) $conflictOperation->phase === 'terminal' + && (string) $conflictOperation->outcome === 'failed' + && (string) $conflictOperation->effect_resolution === 'unknown' + && (bool) $conflictOperation->safe_to_resubmit === false + && (string) CloudStorageArtifact::whereKey( + $conflict['artifact_id'] + )->value('state') === 'degraded' + && (string) CloudStoragePublication::where( + 'artifact_id', $conflict['artifact_id'] + )->value('state') === 'failed' + && count($requests) === 2, + 'Conflicting immutable identity did not stop in operator attention.'); + +// The expected artifact fence runs before source validation or provider I/O. +$fencedA = $fixture('template'); +$fencedB = $fixture('rescue'); +$clock += 1000; +$responses = [new Response(200)]; +$requests = []; +$transactionLevels = []; +$module = $publisher($responses, $requests, $transactionLevels, $clock); +$beforeA = CloudStorageOperation::where( + 'operation_id', $fencedA['operation_id'] +)->firstOrFail()->toArray(); +$expectCloudError(static fn () => $module->publish( + $fencedA['operation_id'], $fencedB['artifact_id'], $fencedA +), 409); +$afterA = CloudStorageOperation::where( + 'operation_id', $fencedA['operation_id'] +)->firstOrFail()->toArray(); +$assert($requests === [] && $beforeA === $afterA, + 'Wrong expected artifact crossed the durable or remote mutation fence.'); + +// A settled publication is idempotent even after its local source is gone. +@unlink($applied['path']); +$requestsBefore = count($requests); +$terminalAgain = new CloudStorageImmutablePrimaryPublication( + new CloudObjectStorageS3Client(new Client([ + 'handler' => HandlerStack::create(static function () { + throw new RuntimeException('Terminal publication used HTTP.'); + }), + ])), + static function (): array { + throw new RuntimeException('Terminal publication resolved runtime.'); + }, + static fn (): int => time() +); +$idempotent = $terminalAgain->publish( + $applied['operation_id'], $applied['artifact_id'], $applied +); +$assert($idempotent['primary_completed'] === true + && (int) CloudStorageOperation::where( + 'operation_id', $applied['operation_id'] + )->value('attempts') === 1 + && count($requests) === $requestsBefore, + 'Terminal publication was not idempotent without its local file.'); + +foreach (CloudStorageOperation::query()->get() as $operation) { + $stored = (string) $operation->result_json; + $assert(!str_contains($stored, 'https://') + && !str_contains($stored, 'IMMUTABLEACCESSKEY') + && !str_contains($stored, 'immutable-secret-must-not-persist'), + 'Publication ledger persisted a credential or capability URL.'); +} + +echo "Immutable primary publication behavior passed.\n";