diff --git a/web/app/Services/Cloud/CloudStorageImmutablePrimaryPublication.php b/web/app/Services/Cloud/CloudStorageImmutablePrimaryPublication.php index 76dd773f..d8b963f7 100644 --- a/web/app/Services/Cloud/CloudStorageImmutablePrimaryPublication.php +++ b/web/app/Services/Cloud/CloudStorageImmutablePrimaryPublication.php @@ -18,12 +18,26 @@ use App\Models\CloudStorageReplica; */ final class CloudStorageImmutablePrimaryPublication { + /** Secondary work remains dormant until the primary is verified again. */ + private const COPY_BLOCKED_UNTIL_PRIMARY = 253402300799; private const LEASE_SECONDS = 300; private const RECONCILE_DELAY = 60; private const RETRY_DELAY = 300; private const MAX_DISPATCH_ATTEMPTS = 3; private const MAX_RECONCILE_FAILURES = 6; + /** @var list */ + private const UNKNOWN_ATTENTION_ERRORS = [ + 'immutable_primary_identity_conflict', + 'immutable_primary_reconcile_exhausted', + ]; + + /** @var list */ + private const NOT_APPLIED_ATTENTION_ERRORS = [ + 'immutable_primary_publish_retry_exhausted', + 'immutable_primary_absence_retry_exhausted', + ]; + /** @var \Closure(int,bool):array */ private readonly \Closure $runtimeResolver; @@ -51,6 +65,205 @@ final class CloudStorageImmutablePrimaryPublication : \Closure::fromCallable($clock); } + /** @return list */ + public static function attentionErrors(): array + { + return array_merge( + self::UNKNOWN_ATTENTION_ERRORS, + self::NOT_APPLIED_ATTENTION_ERRORS + ); + } + + /** @return list */ + public static function unknownAttentionErrors(): array + { + return self::UNKNOWN_ATTENTION_ERRORS; + } + + /** @return list */ + public static function notAppliedAttentionErrors(): array + { + return self::NOT_APPLIED_ATTENTION_ERRORS; + } + + /** + * Re-open only the original frozen publish operation. + * + * Unknown effects resume with an exact-key HEAD. A provider write becomes + * dispatchable again only after this class observes two spaced absences. + * Proven non-application may dispatch the same operation again. Secondary + * copies stopped solely by the primary attention state are restored behind + * the primary fence; no new artifact, key or operation is allocated. + */ + public static function reopenAttention( + string $operationId, + int $now + ): void { + self::assertOperationId($operationId); + if ($now < 1) { + throw new \InvalidArgumentException('Invalid attention time'); + } + $connection = (new CloudStorageOperation())->getConnection(); + $connection->transaction(static function () use ( + $operationId, + $now + ): void { + $operation = CloudStorageOperation::query() + ->where('operation_id', $operationId) + ->lockForUpdate()->first(); + if (!$operation instanceof CloudStorageOperation + || (string) $operation->kind !== 'publish') { + throw new CloudError('对象存储发布核对操作不存在', 404); + } + $copyOperations = CloudStorageOperation::query() + ->where('artifact_id', (int) $operation->artifact_id) + ->where('kind', 'copy')->orderBy('id') + ->lockForUpdate()->get()->all(); + $artifact = CloudStorageArtifact::query() + ->whereKey((int) $operation->artifact_id) + ->lockForUpdate()->first(); + $replicas = CloudStorageReplica::query() + ->where('artifact_id', (int) $operation->artifact_id) + ->orderBy('id')->lockForUpdate()->get()->all(); + $publication = CloudStoragePublication::query() + ->where('artifact_id', (int) $operation->artifact_id) + ->lockForUpdate()->first(); + if (!$artifact instanceof CloudStorageArtifact + || !$publication instanceof CloudStoragePublication + || $publication->backup_id !== null + || (($publication->template_import_intent_id === null) + === ($publication->rescue_template_id === null))) { + throw new CloudError('对象存储发布核对归属不一致', 409); + } + $primary = null; + $replicasById = []; + foreach ($replicas as $replica) { + $replicasById[(int) $replica->id] = $replica; + if ((int) $replica->id === (int) $operation->replica_id) { + $primary = $replica; + } + } + $errorCode = (string) $operation->error_code; + $unknown = in_array( + $errorCode, + self::UNKNOWN_ATTENTION_ERRORS, + true + ); + $notApplied = in_array( + $errorCode, + self::NOT_APPLIED_ATTENTION_ERRORS, + true + ); + if (!$primary instanceof CloudStorageReplica + || (string) $primary->role !== 'primary' + || (!$unknown && !$notApplied) + || (string) $operation->phase !== 'terminal' + || (string) $operation->outcome !== 'failed' + || (bool) $operation->safe_to_resubmit + || (bool) $operation->effect_may_have_applied !== $unknown + || (string) $operation->effect_resolution + !== ($unknown ? 'unknown' : 'not_applied') + || (string) $artifact->state !== 'degraded' + || (string) $primary->state !== 'degraded' + || !hash_equals($errorCode, (string) $primary->error_code) + || (string) $publication->state !== 'failed' + || !hash_equals( + $errorCode, + (string) $publication->last_error_code + )) { + throw new CloudError( + '对象存储主副本不在可重新核对的人工处理终态', + 409 + ); + } + foreach ($copyOperations as $copy) { + $secondary = $replicasById[(int) $copy->replica_id] ?? null; + if (!$secondary instanceof CloudStorageReplica + || (string) $secondary->role !== 'secondary' + || (string) $copy->phase !== 'terminal' + || (string) $copy->outcome !== 'failed' + || (string) $copy->effect_resolution !== 'not_applied' + || (bool) $copy->effect_may_have_applied + || (bool) $copy->safe_to_resubmit + || (int) $copy->attempts !== 0 + || (string) $copy->error_code + !== 'primary_publish_attention' + || (string) $secondary->state !== 'degraded' + || (string) $secondary->error_code + !== 'primary_publish_attention') { + throw new CloudError( + '对象存储次副本核对账本不一致', + 409 + ); + } + } + + $metadata = self::resetEvidence($operation->result_json); + $operation->phase = $unknown ? 'reconciling' : 'accepted'; + $operation->outcome = null; + $operation->effect_resolution = $unknown + ? 'unknown' : 'not_applicable'; + $operation->effect_may_have_applied = null; + $operation->safe_to_resubmit = null; + $operation->lease_token = null; + $operation->lease_until = 0; + $operation->available_at = $unknown ? 0 : $now; + $operation->next_check_at = $unknown ? $now : 0; + $operation->error_code = $unknown + ? 'admin_reconcile_requested' + : 'admin_retry_requested'; + $operation->result_json = self::encodedMetadata($metadata); + $operation->terminal_at = null; + $operation->updated_at = $now; + $operation->save(); + + $primary->state = $unknown ? 'publishing' : 'reserved'; + $primary->lease_token = null; + $primary->lease_until = 0; + $primary->error_code = $operation->error_code; + $primary->updated_at = $now; + $primary->save(); + + foreach ($copyOperations as $copy) { + /** @var CloudStorageReplica $secondary */ + $secondary = $replicasById[(int) $copy->replica_id]; + $copy->phase = 'accepted'; + $copy->outcome = null; + $copy->effect_resolution = 'not_applicable'; + $copy->effect_may_have_applied = null; + $copy->safe_to_resubmit = null; + $copy->lease_token = null; + $copy->lease_until = 0; + $copy->available_at = self::COPY_BLOCKED_UNTIL_PRIMARY; + $copy->next_check_at = 0; + $copy->error_code = ''; + $copy->terminal_at = null; + $copy->updated_at = $now; + $copy->save(); + $secondary->state = 'reserved'; + $secondary->lease_token = null; + $secondary->lease_until = 0; + $secondary->error_code = ''; + $secondary->updated_at = $now; + $secondary->save(); + } + + $artifact->state = 'publishing'; + $artifact->updated_at = $now; + $artifact->save(); + $publication->state = 'waiting_primary'; + $publication->attempts = 0; + $publication->available_at = 0; + $publication->lease_token = null; + $publication->lease_until = 0; + $publication->last_error_code = ''; + $publication->coverage_ready_at = null; + $publication->finalized_at = null; + $publication->updated_at = $now; + $publication->save(); + }, 3); + } + /** * Advance one immutable primary publication by at most one remote action. * diff --git a/web/app/Services/Cloud/CloudStorageReplicaAttentionService.php b/web/app/Services/Cloud/CloudStorageReplicaAttentionService.php index 51d34d4e..d9c1a5b6 100644 --- a/web/app/Services/Cloud/CloudStorageReplicaAttentionService.php +++ b/web/app/Services/Cloud/CloudStorageReplicaAttentionService.php @@ -31,24 +31,65 @@ final class CloudStorageReplicaAttentionService 100000 ); $perPage = self::boundedInt($input['per_page'] ?? 20, 1, 50); + $immutableUnknown = CloudStorageImmutablePrimaryPublication:: + unknownAttentionErrors(); + $immutableNotApplied = CloudStorageImmutablePrimaryPublication:: + notAppliedAttentionErrors(); $base = CloudStorageOperation::query() ->where('phase', 'terminal') ->where('outcome', 'failed') - ->where('effect_resolution', 'unknown') - ->where('effect_may_have_applied', 1) ->where('safe_to_resubmit', 0) - ->where(static function ($attention): void { + ->where(static function ($attention) use ( + $immutableUnknown, + $immutableNotApplied + ): void { $attention->where(static function ($copy): void { - $copy->where('kind', 'copy')->whereIn( - 'error_code', - CloudStorageArtifactService::replicaAttentionErrors() - ); + $copy->where('kind', 'copy') + ->where('effect_resolution', 'unknown') + ->where('effect_may_have_applied', 1) + ->whereIn( + 'error_code', + CloudStorageArtifactService:: + replicaAttentionErrors() + ); })->orWhere(static function ($primary): void { - $primary->where('kind', 'publish')->whereIn( - 'error_code', - CloudStorageArtifactService:: - primaryPreparationAttentionErrors() - ); + $primary->where('kind', 'publish') + ->where('effect_resolution', 'unknown') + ->where('effect_may_have_applied', 1) + ->whereIn( + 'error_code', + CloudStorageArtifactService:: + primaryPreparationAttentionErrors() + ); + })->orWhere(static function ($immutable) use ( + $immutableUnknown, + $immutableNotApplied + ): void { + $immutable->where('kind', 'publish') + ->where(static function ($shape) use ( + $immutableUnknown, + $immutableNotApplied + ): void { + $shape->where(static function ($unknown) use ( + $immutableUnknown + ): void { + $unknown->whereIn( + 'error_code', + $immutableUnknown + )->where('effect_resolution', 'unknown') + ->where('effect_may_have_applied', 1); + })->orWhere(static function ($notApplied) use ( + $immutableNotApplied + ): void { + $notApplied->whereIn( + 'error_code', + $immutableNotApplied + )->where( + 'effect_resolution', + 'not_applied' + )->where('effect_may_have_applied', 0); + }); + }); }); }); $total = (clone $base)->count(); @@ -96,6 +137,11 @@ final class CloudStorageReplicaAttentionService $target = $sourceReplicas[(int) $operation->artifact_id . ':' . (int) $operation->target_backend_id] ?? null; $copy = (string) $operation->kind === 'copy'; + $immutable = !$copy && in_array( + (string) $operation->error_code, + CloudStorageImmutablePrimaryPublication::attentionErrors(), + true + ); $publication = $copy ? null : $publications->get((int) $operation->artifact_id); $backup = $publication instanceof CloudStoragePublication @@ -112,16 +158,41 @@ final class CloudStorageReplicaAttentionService (string) $operation->error_code, (string) $target->error_code ); - $canRetry = $baseRetry && ($copy - ? ($source instanceof CloudStorageReplica + $immutableOwner = self::immutableOwner($publication); + $canRetry = false; + $ownerType = 'backup'; + $ownerId = (int) ($publication?->backup_id ?? 0); + $ownerText = '备份 #' . $ownerId; + if ($copy) { + $canRetry = $baseRetry + && $source instanceof CloudStorageReplica && (string) $source->state === 'ready' && (int) $source->size_bytes === (int) $artifact->size_bytes && hash_equals( (string) $artifact->sha256, (string) $source->sha256 - )) - : ($publication instanceof CloudStoragePublication + ); + $ownerType = 'artifact'; + $ownerId = (int) $operation->artifact_id; + $ownerText = '制品 #' . $ownerId; + } elseif ($immutable && $immutableOwner !== null) { + $canRetry = $baseRetry + && $publication instanceof CloudStoragePublication + && (string) $publication->state === 'failed' + && hash_equals( + (string) $operation->error_code, + (string) $publication->last_error_code + ) + && (string) $artifact->kind + === (string) $immutableOwner['type']; + $ownerType = (string) $immutableOwner['type']; + $ownerId = (string) $immutableOwner['id']; + $ownerText = ($ownerType === 'template' + ? '云模板导入 ' : '救援 ISO ') . $ownerId; + } else { + $canRetry = $baseRetry + && $publication instanceof CloudStoragePublication && (string) $publication->state === 'failed' && hash_equals( (string) $operation->error_code, @@ -133,12 +204,15 @@ final class CloudStorageReplicaAttentionService && $backup->active_job_id === null && $backup->last_operation_id === null && (int) $backup->storage_artifact_id - === (int) $artifact->id)); + === (int) $artifact->id; + } $items[] = [ 'operation_id' => (string) $operation->operation_id, 'operation_kind' => (string) $operation->kind, 'operation_type' => $copy - ? '副本复制' : '主副本发布准备', + ? '副本复制' + : ($immutable + ? '不可变主副本发布' : '主副本发布准备'), 'artifact_id' => (int) $operation->artifact_id, 'artifact_kind' => $artifact instanceof CloudStorageArtifact ? (string) $artifact->kind : 'unknown', @@ -147,12 +221,9 @@ final class CloudStorageReplicaAttentionService 'backend_id' => (int) $operation->target_backend_id, 'backend_name' => $backend instanceof CloudStorageBackend ? (string) $backend->name : '未知后端', - 'owner_type' => $copy ? 'artifact' : 'backup', - 'owner_id' => $copy ? (int) $operation->artifact_id - : (int) ($publication?->backup_id ?? 0), - 'owner_text' => $copy - ? '制品 #' . (int) $operation->artifact_id - : '备份 #' . (int) ($publication?->backup_id ?? 0), + 'owner_type' => $ownerType, + 'owner_id' => $ownerId, + 'owner_text' => $ownerText, 'error_code' => (string) $operation->error_code, 'error_text' => self::errorText( (string) $operation->error_code @@ -280,10 +351,21 @@ final class CloudStorageReplicaAttentionService time() ); } elseif ($kind === 'publish') { - CloudStorageArtifactService::reopenPrimaryPreparationAttention( - $operationId, - time() - ); + if (in_array( + (string) $operation->error_code, + CloudStorageImmutablePrimaryPublication::attentionErrors(), + true + )) { + CloudStorageImmutablePrimaryPublication::reopenAttention( + $operationId, + time() + ); + } else { + CloudStorageArtifactService::reopenPrimaryPreparationAttention( + $operationId, + time() + ); + } } else { throw new CloudError('对象存储操作不支持人工核对', 409); } @@ -342,7 +424,40 @@ final class CloudStorageReplicaAttentionService '主副本记录的分段上传会话与远端不一致', 'primary_multipart_identity_untrusted' => '主副本远端会话无法归属到原发布操作', + 'immutable_primary_identity_conflict' => + '远端对象身份与冻结制品不一致', + 'immutable_primary_reconcile_exhausted' => + '主副本只读核对连续失败', + 'immutable_primary_publish_retry_exhausted' => + '主副本已确认未写入,但自动发布重试已用尽', + 'immutable_primary_absence_retry_exhausted' => + '主副本已稳定确认不存在,但自动发布重试已用尽', default => '分段上传身份需要人工核对', }; } + + /** @return array{type:string,id:string}|null */ + private static function immutableOwner( + ?CloudStoragePublication $publication + ): ?array { + if (!$publication instanceof CloudStoragePublication + || $publication->backup_id !== null) { + return null; + } + $template = (string) ( + $publication->template_import_intent_id ?? '' + ); + $rescue = (string) ($publication->rescue_template_id ?? ''); + if (($template === '') === ($rescue === '')) { + return null; + } + $id = $template !== '' ? $template : $rescue; + if (preg_match('/^[a-f0-9]{32}$/D', $id) !== 1) { + return null; + } + return [ + 'type' => $template !== '' ? 'template' : 'rescue', + 'id' => $id, + ]; + } } diff --git a/web/tests/CloudStorageImmutablePrimaryPublicationTest.php b/web/tests/CloudStorageImmutablePrimaryPublicationTest.php index 7783ac4f..c86fbdd6 100644 --- a/web/tests/CloudStorageImmutablePrimaryPublicationTest.php +++ b/web/tests/CloudStorageImmutablePrimaryPublicationTest.php @@ -10,6 +10,7 @@ use App\Services\Cloud\CloudError; use App\Services\Cloud\CloudObjectStorageS3Client; use App\Services\Cloud\CloudStorageArtifactService; use App\Services\Cloud\CloudStorageImmutablePrimaryPublication; +use App\Services\Cloud\CloudStorageReplicaAttentionService; use GuzzleHttp\Client; use GuzzleHttp\HandlerStack; use GuzzleHttp\Promise\Create; @@ -489,6 +490,132 @@ $assert($conflictResult['requires_attention'] === true && count($requests) === 2, 'Conflicting immutable identity did not stop in operator attention.'); +// An administrator may reopen only the same unknown-effect operation for +// read-only HEAD reconciliation. Secondary work stopped by the primary fence +// is restored but remains blocked until that exact primary succeeds. +$artifactCountBeforeReopen = CloudStorageArtifact::count(); +$operationCountBeforeReopen = CloudStorageOperation::count(); +$clock += 1; +$attention = CloudStorageReplicaAttentionService::listView(); +$attentionItem = collect($attention['items'])->firstWhere( + 'operation_id', + $conflict['operation_id'] +); +$assert(is_array($attentionItem) + && ($attentionItem['operation_type'] ?? null) + === '不可变主副本发布' + && ($attentionItem['owner_type'] ?? null) === 'rescue' + && ($attentionItem['can_retry'] ?? false) === true, + 'Immutable primary attention was not exposed to administrators.'); +$attentionResult = CloudStorageReplicaAttentionService::retry( + $conflict['operation_id'], + 'publish' +); +$assert(($attentionResult['phase'] ?? null) === 'reconciling', + 'Immutable primary attention did not reopen through the admin service.'); +$reopenedConflict = CloudStorageOperation::where( + 'operation_id', $conflict['operation_id'] +)->firstOrFail(); +$reopenedCopy = CloudStorageOperation::where( + 'artifact_id', $conflict['artifact_id'] +)->where('kind', 'copy')->firstOrFail(); +$reopenedPrimary = CloudStorageReplica::whereKey( + $conflict['replica_id'] +)->firstOrFail(); +$reopenedSecondary = CloudStorageReplica::whereKey( + (int) $reopenedCopy->replica_id +)->firstOrFail(); +$assert((string) $reopenedConflict->phase === 'reconciling' + && (string) $reopenedConflict->effect_resolution === 'unknown' + && $reopenedConflict->outcome === null + && (string) $reopenedPrimary->state === 'publishing' + && (string) $reopenedCopy->phase === 'accepted' + && (int) $reopenedCopy->available_at === 253402300799 + && (string) $reopenedSecondary->state === 'reserved' + && (string) CloudStorageArtifact::whereKey( + $conflict['artifact_id'] + )->value('state') === 'publishing' + && (string) CloudStoragePublication::where( + 'artifact_id', $conflict['artifact_id'] + )->value('state') === 'waiting_primary' + && CloudStorageArtifact::count() === $artifactCountBeforeReopen + && CloudStorageOperation::count() === $operationCountBeforeReopen, + 'Administrator reconciliation did not reopen the original fenced ledger.'); + +$responses = [new Response(404), new Response(404)]; +$requests = []; +$transactionLevels = []; +$module = $publisher($responses, $requests, $transactionLevels, $clock); +$module->publish( + $conflict['operation_id'], $conflict['artifact_id'], $conflict +); +$clock += 60; +$reconciledAbsence = $module->publish( + $conflict['operation_id'], $conflict['artifact_id'], $conflict +); +$assert($reconciledAbsence['phase'] === 'accepted' + && array_map( + static fn (RequestInterface $request): string => $request->getMethod(), + $requests + ) === ['HEAD', 'HEAD'], + 'Administrator recovery replayed a write before stable absence evidence.'); + +// Repeated proven non-application also stops for operator review. Reopening +// authorizes one new dispatch of the same operation and immutable key only. +$manualRetry = $fixture('template'); +$clock += 1000; +$responses = [new Response(403), new Response(403), new Response(403)]; +$requests = []; +$transactionLevels = []; +$module = $publisher($responses, $requests, $transactionLevels, $clock); +for ($attempt = 0; $attempt < 3; $attempt++) { + $module->publish( + $manualRetry['operation_id'], + $manualRetry['artifact_id'], + $manualRetry + ); + $clock += 300; +} +$manualRetryOperation = CloudStorageOperation::where( + 'operation_id', $manualRetry['operation_id'] +)->firstOrFail(); +$assert((string) $manualRetryOperation->phase === 'terminal' + && (string) $manualRetryOperation->effect_resolution === 'not_applied' + && (string) $manualRetryOperation->error_code + === 'immutable_primary_publish_retry_exhausted', + 'Bounded proven failures did not enter manual retry attention.'); +$manualAttention = CloudStorageReplicaAttentionService::listView(); +$manualAttentionItem = collect($manualAttention['items'])->firstWhere( + 'operation_id', + $manualRetry['operation_id'] +); +$assert(is_array($manualAttentionItem) + && ($manualAttentionItem['can_retry'] ?? false) === true, + 'Proven-not-applied primary attention was not exposed for retry.'); +CloudStorageReplicaAttentionService::retry( + $manualRetry['operation_id'], + 'publish' +); +$responses = [new Response(200, [ + 'ETag' => '"manual-retry-etag"', + 'x-amz-version-id' => 'manual-retry-version', +])]; +$requests = []; +$transactionLevels = []; +$module = $publisher($responses, $requests, $transactionLevels, $clock); +$manualRetryResult = $module->publish( + $manualRetry['operation_id'], + $manualRetry['artifact_id'], + $manualRetry +); +$assert($manualRetryResult['primary_completed'] === true + && count($requests) === 1 + && $requests[0]->getMethod() === 'PUT' + && (int) CloudStorageOperation::where( + 'operation_id', $manualRetry['operation_id'] + )->value('attempts') === 4, + 'Manual retry did not reuse the exact proven-not-applied operation.'); + // The expected artifact fence runs before source validation or provider I/O. $fencedA = $fixture('template'); $fencedB = $fixture('rescue');