diff --git a/web/app/Controllers/Admin/ObjectStorageController.php b/web/app/Controllers/Admin/ObjectStorageController.php index e622b6ed..1aac2cd3 100644 --- a/web/app/Controllers/Admin/ObjectStorageController.php +++ b/web/app/Controllers/Admin/ObjectStorageController.php @@ -65,6 +65,8 @@ final class ObjectStorageController extends AdminController $query = $request->getQueryParams(); $replicaAttention = CloudStorageReplicaAttentionService::listView([ 'page' => $query['attention_page'] ?? 1, + 'finalization_page' => + $query['finalization_attention_page'] ?? 1, 'per_page' => 20, ]); } catch (CloudError $exception) { @@ -255,10 +257,20 @@ final class ObjectStorageController extends AdminController { try { $operationId = (string) ($args['operation_id'] ?? ''); - $result = CloudStorageReplicaAttentionService::retry($operationId); + $input = CloudInput::object($request); + CloudInput::only($input, ['operation_kind']); + $operationKind = isset($input['operation_kind']) + ? (string) $input['operation_kind'] : null; + $result = CloudStorageReplicaAttentionService::retry( + $operationId, + $operationKind + ); + $finalization = ($result['operation_kind'] ?? '') === 'finalize'; return $response->withJson([ 'ret' => 1, - 'msg' => '已使用原操作恢复只读核对;不会新建上传或终止远端会话。', + 'msg' => $finalization + ? '已重新打开本地业务最终化;不会发起对象存储写入。' + : '已使用原操作恢复只读核对;不会新建上传或终止远端会话。', 'replica_attention' => $result, ])->withHeader('Cache-Control', 'private, no-store'); } catch (CloudError $exception) { diff --git a/web/app/Services/Cloud/CloudStoragePublicationOutboxService.php b/web/app/Services/Cloud/CloudStoragePublicationOutboxService.php new file mode 100644 index 00000000..ec1106d5 --- /dev/null +++ b/web/app/Services/Cloud/CloudStoragePublicationOutboxService.php @@ -0,0 +1,278 @@ + */ + private array $errors = []; + + /** + * @param null|callable(string,int|string,int,int):void $dispatcher + * @param null|callable():int $clock + */ + public function __construct( + ?callable $dispatcher = null, + ?callable $clock = null + ) { + $this->dispatcher = $dispatcher === null + ? static function ( + string $ownerType, + int|string $ownerId, + int $publicationId, + int $artifactId + ): void { + if ($ownerType === 'template') { + (new CloudTemplateStoragePublicationConsumer()) + ->finalizeReady((string) $ownerId); + return; + } + if ($ownerType === 'rescue') { + CloudRescueStoragePublicationService::finalizeReadyOnly( + (string) $ownerId + ); + return; + } + throw new \LogicException( + 'Unsupported storage publication owner' + ); + } + : \Closure::fromCallable($dispatcher); + $this->clock = $clock === null + ? static fn (): int => time() + : \Closure::fromCallable($clock); + } + + /** Consume at most one due owner publication. */ + public function tick(): bool + { + $this->errors = []; + $now = ($this->clock)(); + if (!is_int($now) || $now < 1) { + throw new \LogicException('Storage publication clock is invalid'); + } + $candidate = CloudStoragePublication::query() + ->where('state', 'ready_to_finalize') + ->where('available_at', '<=', $now) + ->whereNull('backup_id') + ->where(static function ($owner): void { + $owner->whereNotNull('template_import_intent_id') + ->orWhereNotNull('rescue_template_id'); + }) + ->orderBy('coverage_ready_at')->orderBy('id')->first(); + if (!$candidate instanceof CloudStoragePublication) { + return false; + } + $publicationId = (int) $candidate->id; + $result = (new CloudStoragePublication())->getConnection() + ->transaction(function ($connection) use ( + $publicationId, + $now + ): array { + $publication = CloudStoragePublication::query() + ->whereKey($publicationId) + ->lockForUpdate()->first(); + if (!$publication instanceof CloudStoragePublication + || (string) $publication->state + !== 'ready_to_finalize' + || (int) $publication->available_at > $now) { + return ['worked' => false, 'failed' => false]; + } + try { + // A nested transaction creates a savepoint. Domain + // writes from a failing owner consumer are rolled back + // to it, while the outer publication row lock remains + // held until the retry delay is durably committed. + // A second worker therefore cannot run the same owner + // in the failure/update gap. + $connection->transaction(function () use ( + $publication, + $publicationId + ): void { + [$ownerType, $ownerId] = self::owner($publication); + $artifact = CloudStorageArtifact::query() + ->whereKey((int) $publication->artifact_id) + ->lockForUpdate()->first(); + if (!$artifact instanceof CloudStorageArtifact + || (string) $artifact->kind !== $ownerType + || !in_array((string) $artifact->state, + ['available', 'protected'], true)) { + throw new CloudError( + '对象存储发布最终化账本不一致', + 409 + ); + } + ($this->dispatcher)( + $ownerType, + $ownerId, + (int) $publication->id, + (int) $artifact->id + ); + $settled = CloudStoragePublication::query() + ->whereKey($publicationId) + ->lockForUpdate()->first(); + if (!$settled instanceof CloudStoragePublication + || (string) $settled->state !== 'completed' + || (int) ($settled->finalized_at ?? 0) < 1) { + throw new CloudError( + '对象存储业务最终化没有完成发布账本', + 409 + ); + } + }, 1); + } catch (\Throwable $error) { + if ($error instanceof \PDOException) { + // Let the outer transaction retry deadlocks or report + // database outages. They are not owner failures and + // must not consume the bounded domain-attempt budget. + throw $error; + } + self::recordFailureLocked($publication, $now); + return ['worked' => true, 'failed' => true]; + } + return ['worked' => true, 'failed' => false]; + }, 3); + if (($result['failed'] ?? false) === true) { + $this->errors[] = 'storage_publication_finalization_failed'; + } + return ($result['worked'] ?? false) === true; + } + + /** @return list */ + public function observationErrors(): array + { + return $this->errors; + } + + /** Re-open only an outbox item stopped by this consumer. */ + public static function reopenAttention( + int $publicationId, + ?int $now = null + ): void { + if ($publicationId < 1) { + throw new CloudError('对象存储发布编号无效', 422); + } + $now ??= time(); + (new CloudStoragePublication())->getConnection()->transaction( + static function () use ($publicationId, $now): void { + $publication = CloudStoragePublication::query() + ->whereKey($publicationId)->lockForUpdate()->first(); + if (!$publication instanceof CloudStoragePublication) { + throw new CloudError('对象存储发布记录不存在', 404); + } + if ((string) $publication->state !== 'failed' + || !hash_equals( + self::ATTENTION_ERROR, + (string) $publication->last_error_code + ) + || $publication->backup_id !== null) { + throw new CloudError( + '对象存储发布不在可重开状态', + 409 + ); + } + [$ownerType] = self::owner($publication); + $artifact = CloudStorageArtifact::query() + ->whereKey((int) $publication->artifact_id) + ->lockForUpdate()->first(); + if (!$artifact instanceof CloudStorageArtifact + || (string) $artifact->kind !== $ownerType + || !in_array((string) $artifact->state, + ['available', 'protected'], true)) { + throw new CloudError( + '对象存储发布最终化账本不一致', + 409 + ); + } + $publication->state = 'ready_to_finalize'; + $publication->attempts = 0; + $publication->available_at = $now; + $publication->lease_token = null; + $publication->lease_until = 0; + $publication->last_error_code = ''; + $publication->updated_at = $now; + $publication->save(); + }, + 3 + ); + } + + private static function recordFailureLocked( + CloudStoragePublication $publication, + int $now + ): void { + // The owner savepoint has already rolled back and the caller still + // owns this publication row lock. Record the delay in that same outer + // transaction so failure and retry eligibility cannot be observed + // separately by concurrent workers. + $publication->refresh(); + if ((string) $publication->state !== 'ready_to_finalize' + || $publication->backup_id !== null) { + return; + } + $attempts = (int) $publication->attempts + 1; + $publication->attempts = $attempts; + $publication->lease_token = null; + $publication->lease_until = 0; + if ($attempts >= self::MAX_ATTEMPTS) { + $publication->state = 'failed'; + $publication->available_at = 0; + $publication->last_error_code = self::ATTENTION_ERROR; + } else { + $publication->available_at = $now + + self::backoff($attempts); + $publication->last_error_code = + 'storage_publication_finalization_failed'; + } + $publication->updated_at = $now; + $publication->save(); + } + + private static function backoff(int $attempts): int + { + return min( + self::MAX_BACKOFF_SECONDS, + 60 * (2 ** max(0, min(8, $attempts - 1))) + ); + } + + /** @return array{0:string,1:string} */ + private static function owner( + CloudStoragePublication $publication + ): array { + $templateId = (string) ($publication->template_import_intent_id ?? ''); + $rescueId = (string) ($publication->rescue_template_id ?? ''); + if ($publication->backup_id !== null + || ($templateId === '') === ($rescueId === '')) { + throw new CloudError('对象存储发布归属不一致', 409); + } + $ownerId = $templateId !== '' ? $templateId : $rescueId; + if (preg_match('/^[a-f0-9]{32}$/D', $ownerId) !== 1) { + throw new CloudError('对象存储发布归属无效', 409); + } + return [$templateId !== '' ? 'template' : 'rescue', $ownerId]; + } +} diff --git a/web/app/Services/Cloud/CloudStorageReplicaAttentionService.php b/web/app/Services/Cloud/CloudStorageReplicaAttentionService.php index 8295ee69..51d34d4e 100644 --- a/web/app/Services/Cloud/CloudStorageReplicaAttentionService.php +++ b/web/app/Services/Cloud/CloudStorageReplicaAttentionService.php @@ -10,18 +10,26 @@ use App\Models\CloudStoragePublication; use App\Models\CloudStorageReplica; /** - * Administrator recovery surface for stable multipart identity conflicts. + * Administrator recovery for provider reconciliation and local finalization. * - * It exposes no object key, upload id or provider capability. Retrying only - * re-opens the original durable operation for read-only reconciliation. + * It exposes no object key, upload id or provider capability. Provider items + * re-open only their original read-only reconciliation; finalization items + * re-open only the existing local publication outbox. */ final class CloudStorageReplicaAttentionService { /** @return array */ public static function listView(array $input = []): array { - CloudInput::only($input, ['page', 'per_page']); + CloudInput::only($input, [ + 'page', 'per_page', 'finalization_page', + ]); $page = self::boundedInt($input['page'] ?? 1, 1, 100000); + $finalizationPage = self::boundedInt( + $input['finalization_page'] ?? 1, + 1, + 100000 + ); $perPage = self::boundedInt($input['per_page'] ?? 20, 1, 50); $base = CloudStorageOperation::query() ->where('phase', 'terminal') @@ -153,31 +161,119 @@ final class CloudStorageReplicaAttentionService 'can_retry' => $canRetry, ]; } + $finalizationBase = CloudStoragePublication::query() + ->where('state', 'failed') + ->where('last_error_code', + 'storage_publication_finalization_attention') + ->whereNull('backup_id') + ->where(static function ($owner): void { + $owner->whereNotNull('template_import_intent_id') + ->orWhereNotNull('rescue_template_id'); + }); + $finalizationTotal = (clone $finalizationBase)->count(); + $finalizations = $finalizationBase->orderByDesc('updated_at') + ->orderByDesc('id') + ->offset(($finalizationPage - 1) * $perPage) + ->limit($perPage)->get(); + $finalizationArtifacts = CloudStorageArtifact::query() + ->whereIn('id', $finalizations->pluck('artifact_id')->all()) + ->get()->keyBy('id'); + $finalizationItems = []; + foreach ($finalizations as $publication) { + $artifact = $finalizationArtifacts->get( + (int) $publication->artifact_id + ); + $templateOwner = (string) ( + $publication->template_import_intent_id ?? '' + ); + $rescueOwner = (string) ( + $publication->rescue_template_id ?? '' + ); + $ownerType = $templateOwner !== '' ? 'template' : 'rescue'; + $ownerId = $templateOwner !== '' ? $templateOwner : $rescueOwner; + $canRetry = $artifact instanceof CloudStorageArtifact + && (string) $artifact->kind === $ownerType + && in_array((string) $artifact->state, + ['available', 'protected'], true) + && preg_match('/^[a-f0-9]{32}$/D', $ownerId) === 1; + $finalizationItems[] = [ + 'operation_id' => (string) $publication->publication_key, + 'operation_kind' => 'finalize', + 'operation_type' => '业务目录最终化', + 'artifact_id' => (int) $publication->artifact_id, + 'artifact_kind' => $artifact instanceof CloudStorageArtifact + ? (string) $artifact->kind : 'unknown', + 'artifact_state' => $artifact instanceof CloudStorageArtifact + ? (string) $artifact->state : 'missing', + 'backend_id' => 0, + 'backend_name' => '不涉及远端写入', + 'owner_type' => $ownerType, + 'owner_id' => $ownerId, + 'owner_text' => ($ownerType === 'template' + ? '云模板导入 ' : '救援 ISO ') . $ownerId, + 'error_code' => (string) $publication->last_error_code, + 'error_text' => '目录最终化连续失败,已停止自动重试', + 'attention_at' => (int) $publication->updated_at, + 'can_retry' => $canRetry, + ]; + } $pages = max(1, (int) ceil($total / $perPage)); + $finalizationPages = max( + 1, + (int) ceil($finalizationTotal / $perPage) + ); return [ - 'summary' => ['count' => $total], + 'summary' => [ + 'count' => $total + $finalizationTotal, + 'provider_operation_count' => $total, + 'finalization_count' => $finalizationTotal, + ], 'items' => $items, + 'finalization_items' => $finalizationItems, 'pagination' => [ 'page' => $page, 'per_page' => $perPage, 'total' => $total, 'total_pages' => $pages, ], + 'finalization_pagination' => [ + 'page' => $finalizationPage, + 'per_page' => $perPage, + 'total' => $finalizationTotal, + 'total_pages' => $finalizationPages, + ], ]; } - /** @return array{operation_id:string,phase:string} */ - public static function retry(string $operationId): array + /** @return array{operation_id:string,operation_kind:string,phase:string} */ + public static function retry( + string $operationId, + ?string $expectedKind = null + ): array { if (preg_match('/^[a-f0-9]{32}$/D', $operationId) !== 1) { throw new CloudError('对象存储核对操作标识无效', 422); } + if ($expectedKind !== null + && !in_array($expectedKind, + ['copy', 'publish', 'finalize'], true)) { + throw new CloudError('对象存储人工处理类型无效', 422); + } + if ($expectedKind === 'finalize') { + return self::retryFinalization($operationId); + } $operation = CloudStorageOperation::query() ->where('operation_id', $operationId)->first(); if (!$operation instanceof CloudStorageOperation) { - throw new CloudError('对象存储核对操作不存在', 404); + if ($expectedKind !== null) { + throw new CloudError('对象存储核对操作不存在', 404); + } + return self::retryFinalization($operationId); } $kind = (string) $operation->kind; + if ($expectedKind !== null && $kind !== $expectedKind) { + throw new CloudError('对象存储人工处理类型不一致', 409); + } if ($kind === 'copy') { CloudStorageArtifactService::reopenReplicaAttention( $operationId, @@ -198,6 +294,25 @@ final class CloudStorageReplicaAttentionService ]; } + /** @return array{operation_id:string,operation_kind:string,phase:string} */ + private static function retryFinalization(string $publicationKey): array + { + $publication = CloudStoragePublication::query() + ->where('publication_key', $publicationKey)->first(); + if (!$publication instanceof CloudStoragePublication) { + throw new CloudError('对象存储核对操作不存在', 404); + } + CloudStoragePublicationOutboxService::reopenAttention( + (int) $publication->id, + time() + ); + return [ + 'operation_id' => $publicationKey, + 'operation_kind' => 'finalize', + 'phase' => 'ready_to_finalize', + ]; + } + private static function boundedInt($value, int $minimum, int $maximum): int { $validated = filter_var($value, FILTER_VALIDATE_INT, [ diff --git a/web/app/Services/Cloud/CloudStorageReplicaWorkerService.php b/web/app/Services/Cloud/CloudStorageReplicaWorkerService.php index ac5c1084..af12d6b2 100644 --- a/web/app/Services/Cloud/CloudStorageReplicaWorkerService.php +++ b/web/app/Services/Cloud/CloudStorageReplicaWorkerService.php @@ -30,7 +30,9 @@ final class CloudStorageReplicaWorkerService private readonly ?CloudStorageReplicaHelper $helper = null, private readonly ?CloudObjectStorageS3Client $client = null, ?callable $runtimeResolver = null, - ?callable $clock = null + ?callable $clock = null, + private readonly ?CloudStoragePublicationOutboxService + $publicationOutbox = null ) { $this->runtimeResolver = $runtimeResolver === null ? static fn (int $backendId, bool $healthy): array => @@ -45,31 +47,52 @@ final class CloudStorageReplicaWorkerService { $this->errors = []; $now = ($this->clock)(); - $work = CloudStorageArtifactService::claimReplicaWork($now); - if ($work === null) { - return false; - } + // Publication finalization is a short, local DB operation. Run it + // before a remote copy that may occupy this worker for hours, while + // still bounding it to one item so replication always gets its turn. + $publicationWorked = false; try { - if (($work['mode'] ?? null) === 'dispatch') { - $this->dispatch($work, $now); - } else { - $this->reconcile($work, $now); - } + $outbox = $this->publicationOutbox + ?? new CloudStoragePublicationOutboxService(); + $publicationWorked = $outbox->tick(); + $this->errors = array_merge( + $this->errors, + $outbox->observationErrors() + ); } catch (\Throwable) { - $this->errors[] = 'storage_replica_tick_failed'; + // A local outbox/database failure must not prevent an unrelated + // replica operation from making progress. The unchanged durable + // publication remains eligible for a later worker cycle. + $this->errors[] = 'storage_publication_outbox_tick_failed'; + } + + $work = CloudStorageArtifactService::claimReplicaWork($now); + $replicaWorked = $work !== null; + if ($work !== null) { try { - $this->markUnknown( - $work, - 'storage_replica_tick_failed', - is_array($work['metadata'] ?? null) - ? $work['metadata'] : [] - ); + if (($work['mode'] ?? null) === 'dispatch') { + $this->dispatch($work, $now); + } else { + $this->reconcile($work, $now); + } } catch (\Throwable) { - // The durable lease remains the recovery fence. Never create - // a replacement operation after an uncertain settlement. + $this->errors[] = 'storage_replica_tick_failed'; + try { + $this->markUnknown( + $work, + 'storage_replica_tick_failed', + is_array($work['metadata'] ?? null) + ? $work['metadata'] : [] + ); + } catch (\Throwable) { + // The durable lease remains the recovery fence. Never + // create a replacement operation after an uncertain + // settlement. + } } } - return true; + $this->errors = array_values(array_unique($this->errors)); + return $replicaWorked || $publicationWorked; } /** @return list */ diff --git a/web/public/cloud/admin-object-storage.js b/web/public/cloud/admin-object-storage.js index 5b486e1e..ffb6e50d 100644 --- a/web/public/cloud/admin-object-storage.js +++ b/web/public/cloud/admin-object-storage.js @@ -283,18 +283,26 @@ var operationId = String(button.getAttribute('data-operation-id') || ''); var operationKind = String(button.getAttribute('data-operation-kind') || 'copy'); if (!/^[a-f0-9]{32}$/.test(operationId) || button.disabled) return; + var finalization = operationKind === 'finalize'; var primary = operationKind === 'publish'; confirmAction({ - title: primary ? '重新核对主副本发布' : '重新核对副本复制', - message: '请先在对象存储提供商确认已清理多余的 multipart 会话。本次点击只重开原操作的只读核对,不会终止会话或直接新建上传;只有对象与会话的无效果证据稳定后才会沿用原操作安全重试。', - confirmLabel: '开始核对', + title: finalization ? '重新执行目录最终化' + : (primary ? '重新核对主副本发布' : '重新核对副本复制'), + message: finalization + ? '本次只重开已经完成双副本保护的本地目录最终化,不会上传、覆盖或删除对象存储内容。' + : '请先在对象存储提供商确认已清理多余的 multipart 会话。本次点击只重开原操作的只读核对,不会终止会话或直接新建上传;只有对象与会话的无效果证据稳定后才会沿用原操作安全重试。', + confirmLabel: finalization ? '重新最终化' : '开始核对', tone: 'primary', }).then(function (confirmed) { if (!confirmed) return; button.disabled = true; - showNotice(notice, '正在重开原操作的只读核对…', true, false); + showNotice(notice, finalization + ? '正在重新打开本地目录最终化…' + : '正在重开原操作的只读核对…', true, false); request('/admin/object-storage/replica-attention/' - + encodeURIComponent(operationId) + '/retry', {}).then(function (result) { + + encodeURIComponent(operationId) + '/retry', { + operation_kind: operationKind, + }).then(function (result) { showNotice(notice, result.msg, true, false); window.setTimeout(function () { window.location.hash = 'protection'; diff --git a/web/resources/views/venix/admin/object_storage.tpl b/web/resources/views/venix/admin/object_storage.tpl index 11f9b587..53926c8c 100644 --- a/web/resources/views/venix/admin/object_storage.tpl +++ b/web/resources/views/venix/admin/object_storage.tpl @@ -59,16 +59,18 @@ diff --git a/web/tests/CloudStoragePublicationOutboxServiceTest.php b/web/tests/CloudStoragePublicationOutboxServiceTest.php new file mode 100644 index 00000000..bc948011 --- /dev/null +++ b/web/tests/CloudStoragePublicationOutboxServiceTest.php @@ -0,0 +1,517 @@ +addConnection(['driver' => 'sqlite', 'database' => ':memory:']); +$db->setAsGlobal(); +$db->bootEloquent(); +$db->schema()->create('cloud_storage_artifacts', static function ( + Blueprint $table +): void { + $table->increments('id'); + $table->string('artifact_key')->unique(); + $table->string('kind'); + $table->unsignedBigInteger('maximum_size_bytes'); + $table->unsignedBigInteger('size_bytes'); + $table->string('sha256'); + $table->string('durability_policy'); + $table->unsignedTinyInteger('desired_copies'); + $table->unsignedTinyInteger('required_copies'); + $table->string('state'); + $table->string('last_operation_id')->nullable(); + $table->unsignedBigInteger('verified_at'); + $table->unsignedBigInteger('created_at'); + $table->unsignedBigInteger('updated_at'); +}); +$db->schema()->create('cloud_storage_publications', static function ( + Blueprint $table +): void { + $table->increments('id'); + $table->string('publication_key')->unique(); + $table->unsignedBigInteger('artifact_id')->unique(); + $table->unsignedBigInteger('backup_id')->nullable()->unique(); + $table->string('template_import_intent_id')->nullable()->unique(); + $table->string('rescue_template_id')->nullable()->unique(); + $table->string('state'); + $table->unsignedInteger('attempts')->default(0); + $table->unsignedBigInteger('available_at')->default(0); + $table->string('lease_token')->nullable(); + $table->unsignedBigInteger('lease_until')->default(0); + $table->string('last_error_code')->default(''); + $table->unsignedBigInteger('coverage_ready_at')->nullable(); + $table->unsignedBigInteger('finalized_at')->nullable(); + $table->unsignedBigInteger('created_at'); + $table->unsignedBigInteger('updated_at'); +}); +$db->schema()->create('cloud_storage_operations', static function ( + Blueprint $table +): void { + $table->increments('id'); + $table->string('operation_id')->unique(); + $table->string('request_idempotency_key')->unique(); + $table->string('request_fingerprint'); + $table->unsignedBigInteger('artifact_id'); + $table->unsignedBigInteger('replica_id')->nullable(); + $table->unsignedBigInteger('source_backend_id')->nullable(); + $table->unsignedBigInteger('target_backend_id')->nullable(); + $table->string('kind'); + $table->string('phase'); + $table->string('outcome')->nullable(); + $table->string('effect_resolution'); + $table->boolean('effect_may_have_applied')->nullable(); + $table->boolean('safe_to_resubmit')->nullable(); + $table->unsignedInteger('attempts')->default(0); + $table->string('lease_token')->nullable(); + $table->unsignedBigInteger('lease_until')->default(0); + $table->unsignedBigInteger('available_at')->default(0); + $table->unsignedBigInteger('next_check_at')->default(0); + $table->string('error_code')->default(''); + $table->text('result_json')->nullable(); + $table->unsignedBigInteger('terminal_at')->nullable(); + $table->unsignedBigInteger('created_at'); + $table->unsignedBigInteger('updated_at'); +}); +$db->schema()->create('cloud_storage_replicas', static function ( + Blueprint $table +): void { + $table->increments('id'); + $table->string('replica_key')->unique(); + $table->unsignedBigInteger('artifact_id'); + $table->unsignedBigInteger('backend_id'); + $table->string('role'); + $table->string('object_key'); + $table->string('object_version_id')->default(''); + $table->string('object_etag')->default(''); + $table->string('state'); + $table->string('publish_operation_id')->nullable(); + $table->string('delete_operation_id')->nullable(); + $table->unsignedInteger('delete_generation')->default(0); + $table->unsignedInteger('delete_cycle_attempts')->default(0); + $table->unsignedInteger('delete_presence_confirmations')->default(0); + $table->unsignedBigInteger('delete_last_checked_at')->default(0); + $table->unsignedBigInteger('delete_next_check_at')->default(0); + $table->unsignedBigInteger('delete_attention_at')->nullable(); + $table->string('lease_token')->nullable(); + $table->unsignedBigInteger('lease_until')->default(0); + $table->unsignedBigInteger('size_bytes')->default(0); + $table->string('sha256')->default(''); + $table->string('error_code')->default(''); + $table->unsignedBigInteger('verified_at')->default(0); + $table->unsignedBigInteger('created_at'); + $table->unsignedBigInteger('updated_at'); +}); +$db->schema()->create('cloud_storage_backends', static function ( + Blueprint $table +): void { + $table->increments('id'); + $table->string('name'); +}); + +$clock = 2000000000; +$ownerA = str_repeat('a', 32); +$ownerB = str_repeat('b', 32); +$create = static function ( + string $kind, + string $owner, + string $state = 'ready_to_finalize', + int $attempts = 0, + int $availableAt = 0 +) use (&$clock): CloudStoragePublication { + $artifact = new CloudStorageArtifact(); + $artifact->artifact_key = bin2hex(random_bytes(16)); + $artifact->kind = $kind; + $artifact->maximum_size_bytes = 4096; + $artifact->size_bytes = 4096; + $artifact->sha256 = str_repeat($kind === 'template' ? 'a' : 'b', 64); + $artifact->durability_policy = 'mirror_required_2'; + $artifact->desired_copies = 2; + $artifact->required_copies = 2; + $artifact->state = 'protected'; + $artifact->last_operation_id = null; + $artifact->verified_at = $clock - 10; + $artifact->created_at = $artifact->updated_at = $clock - 20; + $artifact->save(); + + $publication = new CloudStoragePublication(); + $publication->publication_key = bin2hex(random_bytes(16)); + $publication->artifact_id = (int) $artifact->id; + $publication->backup_id = null; + $publication->template_import_intent_id = + $kind === 'template' ? $owner : null; + $publication->rescue_template_id = + $kind === 'rescue' ? $owner : null; + $publication->state = $state; + $publication->attempts = $attempts; + $publication->available_at = $availableAt; + $publication->lease_token = null; + $publication->lease_until = 0; + $publication->last_error_code = ''; + $publication->coverage_ready_at = $clock - 5; + $publication->finalized_at = null; + $publication->created_at = $publication->updated_at = $clock - 20; + $publication->save(); + return $publication; +}; + +$template = $create('template', $ownerA); +$calls = []; +$worker = new CloudStoragePublicationOutboxService( + static function ( + string $type, + int|string $owner, + int $publicationId, + int $artifactId + ) use (&$calls, &$clock, $assert): void { + $assert(Capsule::connection()->transactionLevel() > 0, + 'Business finalization escaped the owner transaction.'); + $calls[] = [$type, $owner, $publicationId, $artifactId]; + $row = CloudStoragePublication::query() + ->whereKey($publicationId)->lockForUpdate()->firstOrFail(); + $row->state = 'completed'; + $row->attempts = (int) $row->attempts + 1; + $row->finalized_at = $clock; + $row->updated_at = $clock; + $row->save(); + }, + static function () use (&$clock): int { + return $clock; + } +); +$assert($worker->tick(), 'A due template publication was not consumed.'); +$template->refresh(); +$assert((string) $template->state === 'completed' + && (int) $template->finalized_at === $clock + && $calls === [[ + 'template', $ownerA, (int) $template->id, + (int) $template->artifact_id, + ]], 'Template outbox finalization lost its frozen owner identity.'); +$assert(!$worker->tick(), 'A completed publication was consumed twice.'); + +$rescue = $create('rescue', $ownerB, 'ready_to_finalize', 0, $clock + 60); +$assert(!$worker->tick(), 'A future publication bypassed its retry delay.'); +$clock += 60; +$assert($worker->tick(), 'A due rescue publication was not consumed.'); +$rescue->refresh(); +$assert((string) $rescue->state === 'completed' + && $calls[1][0] === 'rescue' && $calls[1][1] === $ownerB, + 'Rescue outbox dispatch selected the wrong owner.'); + +$failing = $create('template', str_repeat('c', 32)); +$failingWorker = new CloudStoragePublicationOutboxService( + static function ( + string $type, + int|string $owner, + int $publicationId + ) use (&$clock): void { + // Prove that a consumer which mutates the business publication and + // then crashes is rolled back to the savepoint before the durable + // retry delay is committed. + $partial = CloudStoragePublication::query() + ->whereKey($publicationId)->lockForUpdate()->firstOrFail(); + $partial->state = 'completed'; + $partial->finalized_at = $clock; + $partial->save(); + throw new CloudError('fixture owner failure', 409); + }, + static function () use (&$clock): int { + return $clock; + } +); +$assert($failingWorker->tick(), 'A failed outbox attempt was not observed.'); +$failing->refresh(); +$assert((string) $failing->state === 'ready_to_finalize' + && (int) $failing->attempts === 1 + && $failing->finalized_at === null + && (int) $failing->available_at === $clock + 60 + && (string) $failing->last_error_code + === 'storage_publication_finalization_failed' + && $failingWorker->observationErrors() + === ['storage_publication_finalization_failed'], + 'A failed owner finalization was not durably delayed.'); +$assert(!$failingWorker->tick(), 'A delayed failure entered a hot loop.'); + +$attention = $create( + 'rescue', + str_repeat('d', 32), + 'ready_to_finalize', + 5, + $clock +); +$assert($failingWorker->tick(), 'The bounded final attempt was not handled.'); +$attention->refresh(); +$assert((string) $attention->state === 'failed' + && (int) $attention->attempts === 6 + && (string) $attention->last_error_code + === 'storage_publication_finalization_attention', + 'Exhausted owner finalization did not become explicit attention.'); +$attentionView = CloudStorageReplicaAttentionService::listView(); +$assert(($attentionView['summary']['finalization_count'] ?? 0) === 1 + && count($attentionView['finalization_items'] ?? []) === 1 + && ($attentionView['finalization_items'][0]['operation_kind'] ?? '') + === 'finalize' + && ($attentionView['finalization_items'][0]['operation_id'] ?? '') + === (string) $attention->publication_key + && ($attentionView['finalization_items'][0]['can_retry'] ?? false), + 'Finalization attention is not visible on the administrator recovery page.'); +$retry = CloudStorageReplicaAttentionService::retry( + (string) $attention->publication_key, + 'finalize' +); +$attention->refresh(); +$assert((string) $attention->state === 'ready_to_finalize' + && (int) $attention->attempts === 0 + && (int) $attention->available_at > 0 + && (string) $attention->last_error_code === '' + && ($retry['operation_kind'] ?? '') === 'finalize', + 'Administrator recovery did not reopen the same publication intent.'); +$attention->state = 'completed'; +$attention->finalized_at = $clock; +$attention->save(); + +// Finalization attention has an independent stable paginator. Prove adjacent +// pages are disjoint and together expose every row instead of silently +// truncating after the first page. +$pagedAttention = []; +foreach (['1', '2', '3'] as $ownerDigit) { + $row = $create('template', str_repeat($ownerDigit, 32)); + $row->state = 'failed'; + $row->attempts = 6; + $row->available_at = 0; + $row->last_error_code = + 'storage_publication_finalization_attention'; + $row->updated_at = $clock + (int) $ownerDigit; + $row->save(); + $pagedAttention[] = $row; +} +$finalizationPageOne = CloudStorageReplicaAttentionService::listView([ + 'page' => 1, + 'finalization_page' => 1, + 'per_page' => 2, +]); +$finalizationPageTwo = CloudStorageReplicaAttentionService::listView([ + 'page' => 1, + 'finalization_page' => 2, + 'per_page' => 2, +]); +$pageOneIds = array_column( + $finalizationPageOne['finalization_items'], + 'operation_id' +); +$pageTwoIds = array_column( + $finalizationPageTwo['finalization_items'], + 'operation_id' +); +$assert(count($pageOneIds) === 2 && count($pageTwoIds) === 1 + && array_intersect($pageOneIds, $pageTwoIds) === [] + && count(array_unique(array_merge($pageOneIds, $pageTwoIds))) === 3 + && ($finalizationPageOne['finalization_pagination']['total'] ?? 0) === 3 + && ($finalizationPageOne['finalization_pagination']['total_pages'] ?? 0) + === 2, + 'Finalization attention pagination repeated or omitted an item.'); +foreach ($pagedAttention as $row) { + $row->state = 'completed'; + $row->finalized_at = $clock; + $row->save(); +} + +$databaseFault = $create('rescue', str_repeat('4', 32)); +$databaseFaultWorker = new CloudStoragePublicationOutboxService( + static function (): void { + throw new PDOException('fixture database outage'); + }, + static function () use (&$clock): int { + return $clock; + } +); +$databaseFaultRaised = false; +try { + $databaseFaultWorker->tick(); +} catch (PDOException) { + $databaseFaultRaised = true; +} +$databaseFault->refresh(); +$assert($databaseFaultRaised + && (string) $databaseFault->state === 'ready_to_finalize' + && (int) $databaseFault->attempts === 0, + 'Database failure consumed the bounded owner-finalization budget.'); +$databaseFault->state = 'completed'; +$databaseFault->finalized_at = $clock; +$databaseFault->save(); + +$wrong = $create('template', str_repeat('e', 32)); +$noSettlement = new CloudStoragePublicationOutboxService( + static function (): void { + // A domain consumer returning without completing the outbox is an + // error; it must roll back and enter the same bounded retry path. + }, + static function () use (&$clock): int { + return $clock; + } +); +$assert($noSettlement->tick(), 'Missing business settlement was not handled.'); +$wrong->refresh(); +$assert((string) $wrong->state === 'ready_to_finalize' + && (int) $wrong->attempts === 1, + 'A no-op business consumer was treated as successful.'); +$wrong->state = 'completed'; +$wrong->finalized_at = $clock; +$wrong->save(); + +$workerOwned = $create('rescue', str_repeat('f', 32)); +$copyArtifact = new CloudStorageArtifact(); +$copyArtifact->artifact_key = bin2hex(random_bytes(16)); +$copyArtifact->kind = 'backup'; +$copyArtifact->maximum_size_bytes = 4096; +$copyArtifact->size_bytes = 4096; +$copyArtifact->sha256 = str_repeat('9', 64); +$copyArtifact->durability_policy = 'mirror_required_2'; +$copyArtifact->desired_copies = 2; +$copyArtifact->required_copies = 2; +$copyArtifact->state = 'available'; +$copyArtifact->last_operation_id = null; +$copyArtifact->verified_at = $clock; +$copyArtifact->created_at = $copyArtifact->updated_at = $clock; +$copyArtifact->save(); + +$sourceReplica = new CloudStorageReplica(); +$sourceReplica->replica_key = bin2hex(random_bytes(16)); +$sourceReplica->artifact_id = (int) $copyArtifact->id; +$sourceReplica->backend_id = 1; +$sourceReplica->role = 'primary'; +$sourceReplica->object_key = 'fixture/source.qcow2'; +$sourceReplica->object_version_id = ''; +$sourceReplica->object_etag = '"source"'; +$sourceReplica->size_bytes = 4096; +$sourceReplica->sha256 = str_repeat('9', 64); +$sourceReplica->state = 'ready'; +$sourceReplica->publish_operation_id = null; +$sourceReplica->delete_operation_id = null; +$sourceReplica->lease_token = null; +$sourceReplica->lease_until = 0; +$sourceReplica->error_code = ''; +$sourceReplica->verified_at = $clock; +$sourceReplica->created_at = $sourceReplica->updated_at = $clock; +$sourceReplica->save(); + +$targetReplica = new CloudStorageReplica(); +$targetReplica->replica_key = bin2hex(random_bytes(16)); +$targetReplica->artifact_id = (int) $copyArtifact->id; +$targetReplica->backend_id = 2; +$targetReplica->role = 'secondary'; +$targetReplica->object_key = 'fixture/target.qcow2'; +$targetReplica->object_version_id = ''; +$targetReplica->object_etag = ''; +$targetReplica->size_bytes = 0; +$targetReplica->sha256 = ''; +$targetReplica->state = 'reserved'; +$targetReplica->publish_operation_id = null; +$targetReplica->delete_operation_id = null; +$targetReplica->lease_token = null; +$targetReplica->lease_until = 0; +$targetReplica->error_code = ''; +$targetReplica->verified_at = 0; +$targetReplica->created_at = $targetReplica->updated_at = $clock; +$targetReplica->save(); + +$copyOperation = new CloudStorageOperation(); +$copyOperation->operation_id = bin2hex(random_bytes(16)); +$copyOperation->request_idempotency_key = + 'publication-worker-copy-fixture'; +$copyOperation->request_fingerprint = hash('sha256', 'copy-fixture'); +$copyOperation->artifact_id = (int) $copyArtifact->id; +$copyOperation->replica_id = (int) $targetReplica->id; +$copyOperation->source_backend_id = 1; +$copyOperation->target_backend_id = 2; +$copyOperation->kind = 'copy'; +$copyOperation->phase = 'accepted'; +$copyOperation->outcome = null; +$copyOperation->effect_resolution = 'not_applicable'; +$copyOperation->effect_may_have_applied = null; +$copyOperation->safe_to_resubmit = null; +$copyOperation->attempts = 0; +$copyOperation->lease_token = null; +$copyOperation->lease_until = 0; +$copyOperation->available_at = $clock; +$copyOperation->next_check_at = 0; +$copyOperation->error_code = ''; +$copyOperation->result_json = null; +$copyOperation->terminal_at = null; +$copyOperation->created_at = $copyOperation->updated_at = $clock; +$copyOperation->save(); +$integratedOutbox = new CloudStoragePublicationOutboxService( + static function ( + string $type, + int|string $owner, + int $publicationId, + int $artifactId + ) use (&$clock, $assert, $sourceReplica): void { + $assert($type === 'rescue' && $owner === str_repeat('f', 32), + 'Replica worker dispatched the wrong publication owner.'); + $publication = CloudStoragePublication::query() + ->whereKey($publicationId)->lockForUpdate()->firstOrFail(); + $publication->state = 'completed'; + $publication->finalized_at = $clock; + $publication->save(); + // Makes the replica source invalid only after publication work ran; + // the following replica_source_not_ready assertion therefore proves + // the short outbox step was consumed before the replica claim. + $sourceReplica->state = 'degraded'; + $sourceReplica->error_code = 'fixture_not_ready'; + $sourceReplica->verified_at = 0; + $sourceReplica->save(); + }, + static function () use (&$clock): int { + return $clock; + } +); +$replicaWorker = new CloudStorageReplicaWorkerService( + null, + null, + null, + static function () use (&$clock): int { + return $clock; + }, + $integratedOutbox +); +$assert($replicaWorker->tick() + && $replicaWorker->observationErrors() + === ['replica_source_not_ready'], + 'Replica worker did not process both bounded work queues in one tick.'); +$workerOwned->refresh(); +$copyOperation->refresh(); +$assert((string) $workerOwned->state === 'completed', + 'Replica worker did not settle the owner publication.'); +$assert((string) $copyOperation->phase === 'accepted' + && (int) $copyOperation->attempts === 1 + && (int) $copyOperation->available_at === $clock + 60 + && (string) $copyOperation->error_code === 'replica_source_not_ready', + 'Publication finalization starved the replica queue in the same tick.'); + +echo "Storage publication outbox behavior passed.\n";