feat(storage): consume owner publication outbox

go
codex 2 weeks ago
parent 1cfe6a8866
commit f5cc5afe6c

@ -65,6 +65,8 @@ final class ObjectStorageController extends AdminController
$query = $request->getQueryParams();
$replicaAttention = CloudStorageReplicaAttentionService::listView([
'page' => $query['attention_page'] ?? 1,
'finalization_page' =>
$query['finalization_attention_page'] ?? 1,
'per_page' => 20,
]);
} catch (CloudError $exception) {
@ -255,10 +257,20 @@ final class ObjectStorageController extends AdminController
{
try {
$operationId = (string) ($args['operation_id'] ?? '');
$result = CloudStorageReplicaAttentionService::retry($operationId);
$input = CloudInput::object($request);
CloudInput::only($input, ['operation_kind']);
$operationKind = isset($input['operation_kind'])
? (string) $input['operation_kind'] : null;
$result = CloudStorageReplicaAttentionService::retry(
$operationId,
$operationKind
);
$finalization = ($result['operation_kind'] ?? '') === 'finalize';
return $response->withJson([
'ret' => 1,
'msg' => '已使用原操作恢复只读核对;不会新建上传或终止远端会话。',
'msg' => $finalization
? '已重新打开本地业务最终化;不会发起对象存储写入。'
: '已使用原操作恢复只读核对;不会新建上传或终止远端会话。',
'replica_attention' => $result,
])->withHeader('Cache-Control', 'private, no-store');
} catch (CloudError $exception) {

@ -0,0 +1,278 @@
<?php
namespace App\Services\Cloud;
use App\Models\CloudStorageArtifact;
use App\Models\CloudStoragePublication;
/**
* Atomically consumes business finalization after storage coverage is ready.
*
* Template and rescue finalizers are database-only operations. Keeping the
* owner projection and publication completion under one outer transaction
* means a process crash can only commit both or neither. Repeated failures are
* delayed and eventually become an explicit, manually reopenable attention
* state instead of a hot worker loop.
*/
final class CloudStoragePublicationOutboxService
{
private const MAX_ATTEMPTS = 6;
private const MAX_BACKOFF_SECONDS = 21600;
private const ATTENTION_ERROR =
'storage_publication_finalization_attention';
/** @var \Closure(string,int|string,int,int):void */
private readonly \Closure $dispatcher;
/** @var \Closure():int */
private readonly \Closure $clock;
/** @var list<string> */
private array $errors = [];
/**
* @param null|callable(string,int|string,int,int):void $dispatcher
* @param null|callable():int $clock
*/
public function __construct(
?callable $dispatcher = null,
?callable $clock = null
) {
$this->dispatcher = $dispatcher === null
? static function (
string $ownerType,
int|string $ownerId,
int $publicationId,
int $artifactId
): void {
if ($ownerType === 'template') {
(new CloudTemplateStoragePublicationConsumer())
->finalizeReady((string) $ownerId);
return;
}
if ($ownerType === 'rescue') {
CloudRescueStoragePublicationService::finalizeReadyOnly(
(string) $ownerId
);
return;
}
throw new \LogicException(
'Unsupported storage publication owner'
);
}
: \Closure::fromCallable($dispatcher);
$this->clock = $clock === null
? static fn (): int => time()
: \Closure::fromCallable($clock);
}
/** Consume at most one due owner publication. */
public function tick(): bool
{
$this->errors = [];
$now = ($this->clock)();
if (!is_int($now) || $now < 1) {
throw new \LogicException('Storage publication clock is invalid');
}
$candidate = CloudStoragePublication::query()
->where('state', 'ready_to_finalize')
->where('available_at', '<=', $now)
->whereNull('backup_id')
->where(static function ($owner): void {
$owner->whereNotNull('template_import_intent_id')
->orWhereNotNull('rescue_template_id');
})
->orderBy('coverage_ready_at')->orderBy('id')->first();
if (!$candidate instanceof CloudStoragePublication) {
return false;
}
$publicationId = (int) $candidate->id;
$result = (new CloudStoragePublication())->getConnection()
->transaction(function ($connection) use (
$publicationId,
$now
): array {
$publication = CloudStoragePublication::query()
->whereKey($publicationId)
->lockForUpdate()->first();
if (!$publication instanceof CloudStoragePublication
|| (string) $publication->state
!== 'ready_to_finalize'
|| (int) $publication->available_at > $now) {
return ['worked' => false, 'failed' => false];
}
try {
// A nested transaction creates a savepoint. Domain
// writes from a failing owner consumer are rolled back
// to it, while the outer publication row lock remains
// held until the retry delay is durably committed.
// A second worker therefore cannot run the same owner
// in the failure/update gap.
$connection->transaction(function () use (
$publication,
$publicationId
): void {
[$ownerType, $ownerId] = self::owner($publication);
$artifact = CloudStorageArtifact::query()
->whereKey((int) $publication->artifact_id)
->lockForUpdate()->first();
if (!$artifact instanceof CloudStorageArtifact
|| (string) $artifact->kind !== $ownerType
|| !in_array((string) $artifact->state,
['available', 'protected'], true)) {
throw new CloudError(
'对象存储发布最终化账本不一致',
409
);
}
($this->dispatcher)(
$ownerType,
$ownerId,
(int) $publication->id,
(int) $artifact->id
);
$settled = CloudStoragePublication::query()
->whereKey($publicationId)
->lockForUpdate()->first();
if (!$settled instanceof CloudStoragePublication
|| (string) $settled->state !== 'completed'
|| (int) ($settled->finalized_at ?? 0) < 1) {
throw new CloudError(
'对象存储业务最终化没有完成发布账本',
409
);
}
}, 1);
} catch (\Throwable $error) {
if ($error instanceof \PDOException) {
// Let the outer transaction retry deadlocks or report
// database outages. They are not owner failures and
// must not consume the bounded domain-attempt budget.
throw $error;
}
self::recordFailureLocked($publication, $now);
return ['worked' => true, 'failed' => true];
}
return ['worked' => true, 'failed' => false];
}, 3);
if (($result['failed'] ?? false) === true) {
$this->errors[] = 'storage_publication_finalization_failed';
}
return ($result['worked'] ?? false) === true;
}
/** @return list<string> */
public function observationErrors(): array
{
return $this->errors;
}
/** Re-open only an outbox item stopped by this consumer. */
public static function reopenAttention(
int $publicationId,
?int $now = null
): void {
if ($publicationId < 1) {
throw new CloudError('对象存储发布编号无效', 422);
}
$now ??= time();
(new CloudStoragePublication())->getConnection()->transaction(
static function () use ($publicationId, $now): void {
$publication = CloudStoragePublication::query()
->whereKey($publicationId)->lockForUpdate()->first();
if (!$publication instanceof CloudStoragePublication) {
throw new CloudError('对象存储发布记录不存在', 404);
}
if ((string) $publication->state !== 'failed'
|| !hash_equals(
self::ATTENTION_ERROR,
(string) $publication->last_error_code
)
|| $publication->backup_id !== null) {
throw new CloudError(
'对象存储发布不在可重开状态',
409
);
}
[$ownerType] = self::owner($publication);
$artifact = CloudStorageArtifact::query()
->whereKey((int) $publication->artifact_id)
->lockForUpdate()->first();
if (!$artifact instanceof CloudStorageArtifact
|| (string) $artifact->kind !== $ownerType
|| !in_array((string) $artifact->state,
['available', 'protected'], true)) {
throw new CloudError(
'对象存储发布最终化账本不一致',
409
);
}
$publication->state = 'ready_to_finalize';
$publication->attempts = 0;
$publication->available_at = $now;
$publication->lease_token = null;
$publication->lease_until = 0;
$publication->last_error_code = '';
$publication->updated_at = $now;
$publication->save();
},
3
);
}
private static function recordFailureLocked(
CloudStoragePublication $publication,
int $now
): void {
// The owner savepoint has already rolled back and the caller still
// owns this publication row lock. Record the delay in that same outer
// transaction so failure and retry eligibility cannot be observed
// separately by concurrent workers.
$publication->refresh();
if ((string) $publication->state !== 'ready_to_finalize'
|| $publication->backup_id !== null) {
return;
}
$attempts = (int) $publication->attempts + 1;
$publication->attempts = $attempts;
$publication->lease_token = null;
$publication->lease_until = 0;
if ($attempts >= self::MAX_ATTEMPTS) {
$publication->state = 'failed';
$publication->available_at = 0;
$publication->last_error_code = self::ATTENTION_ERROR;
} else {
$publication->available_at = $now
+ self::backoff($attempts);
$publication->last_error_code =
'storage_publication_finalization_failed';
}
$publication->updated_at = $now;
$publication->save();
}
private static function backoff(int $attempts): int
{
return min(
self::MAX_BACKOFF_SECONDS,
60 * (2 ** max(0, min(8, $attempts - 1)))
);
}
/** @return array{0:string,1:string} */
private static function owner(
CloudStoragePublication $publication
): array {
$templateId = (string) ($publication->template_import_intent_id ?? '');
$rescueId = (string) ($publication->rescue_template_id ?? '');
if ($publication->backup_id !== null
|| ($templateId === '') === ($rescueId === '')) {
throw new CloudError('对象存储发布归属不一致', 409);
}
$ownerId = $templateId !== '' ? $templateId : $rescueId;
if (preg_match('/^[a-f0-9]{32}$/D', $ownerId) !== 1) {
throw new CloudError('对象存储发布归属无效', 409);
}
return [$templateId !== '' ? 'template' : 'rescue', $ownerId];
}
}

@ -10,18 +10,26 @@ use App\Models\CloudStoragePublication;
use App\Models\CloudStorageReplica;
/**
* Administrator recovery surface for stable multipart identity conflicts.
* Administrator recovery for provider reconciliation and local finalization.
*
* It exposes no object key, upload id or provider capability. Retrying only
* re-opens the original durable operation for read-only reconciliation.
* It exposes no object key, upload id or provider capability. Provider items
* re-open only their original read-only reconciliation; finalization items
* re-open only the existing local publication outbox.
*/
final class CloudStorageReplicaAttentionService
{
/** @return array<string,mixed> */
public static function listView(array $input = []): array
{
CloudInput::only($input, ['page', 'per_page']);
CloudInput::only($input, [
'page', 'per_page', 'finalization_page',
]);
$page = self::boundedInt($input['page'] ?? 1, 1, 100000);
$finalizationPage = self::boundedInt(
$input['finalization_page'] ?? 1,
1,
100000
);
$perPage = self::boundedInt($input['per_page'] ?? 20, 1, 50);
$base = CloudStorageOperation::query()
->where('phase', 'terminal')
@ -153,31 +161,119 @@ final class CloudStorageReplicaAttentionService
'can_retry' => $canRetry,
];
}
$finalizationBase = CloudStoragePublication::query()
->where('state', 'failed')
->where('last_error_code',
'storage_publication_finalization_attention')
->whereNull('backup_id')
->where(static function ($owner): void {
$owner->whereNotNull('template_import_intent_id')
->orWhereNotNull('rescue_template_id');
});
$finalizationTotal = (clone $finalizationBase)->count();
$finalizations = $finalizationBase->orderByDesc('updated_at')
->orderByDesc('id')
->offset(($finalizationPage - 1) * $perPage)
->limit($perPage)->get();
$finalizationArtifacts = CloudStorageArtifact::query()
->whereIn('id', $finalizations->pluck('artifact_id')->all())
->get()->keyBy('id');
$finalizationItems = [];
foreach ($finalizations as $publication) {
$artifact = $finalizationArtifacts->get(
(int) $publication->artifact_id
);
$templateOwner = (string) (
$publication->template_import_intent_id ?? ''
);
$rescueOwner = (string) (
$publication->rescue_template_id ?? ''
);
$ownerType = $templateOwner !== '' ? 'template' : 'rescue';
$ownerId = $templateOwner !== '' ? $templateOwner : $rescueOwner;
$canRetry = $artifact instanceof CloudStorageArtifact
&& (string) $artifact->kind === $ownerType
&& in_array((string) $artifact->state,
['available', 'protected'], true)
&& preg_match('/^[a-f0-9]{32}$/D', $ownerId) === 1;
$finalizationItems[] = [
'operation_id' => (string) $publication->publication_key,
'operation_kind' => 'finalize',
'operation_type' => '业务目录最终化',
'artifact_id' => (int) $publication->artifact_id,
'artifact_kind' => $artifact instanceof CloudStorageArtifact
? (string) $artifact->kind : 'unknown',
'artifact_state' => $artifact instanceof CloudStorageArtifact
? (string) $artifact->state : 'missing',
'backend_id' => 0,
'backend_name' => '不涉及远端写入',
'owner_type' => $ownerType,
'owner_id' => $ownerId,
'owner_text' => ($ownerType === 'template'
? '云模板导入 ' : '救援 ISO ') . $ownerId,
'error_code' => (string) $publication->last_error_code,
'error_text' => '目录最终化连续失败,已停止自动重试',
'attention_at' => (int) $publication->updated_at,
'can_retry' => $canRetry,
];
}
$pages = max(1, (int) ceil($total / $perPage));
$finalizationPages = max(
1,
(int) ceil($finalizationTotal / $perPage)
);
return [
'summary' => ['count' => $total],
'summary' => [
'count' => $total + $finalizationTotal,
'provider_operation_count' => $total,
'finalization_count' => $finalizationTotal,
],
'items' => $items,
'finalization_items' => $finalizationItems,
'pagination' => [
'page' => $page,
'per_page' => $perPage,
'total' => $total,
'total_pages' => $pages,
],
'finalization_pagination' => [
'page' => $finalizationPage,
'per_page' => $perPage,
'total' => $finalizationTotal,
'total_pages' => $finalizationPages,
],
];
}
/** @return array{operation_id:string,phase:string} */
public static function retry(string $operationId): array
/** @return array{operation_id:string,operation_kind:string,phase:string} */
public static function retry(
string $operationId,
?string $expectedKind = null
): array
{
if (preg_match('/^[a-f0-9]{32}$/D', $operationId) !== 1) {
throw new CloudError('对象存储核对操作标识无效', 422);
}
if ($expectedKind !== null
&& !in_array($expectedKind,
['copy', 'publish', 'finalize'], true)) {
throw new CloudError('对象存储人工处理类型无效', 422);
}
if ($expectedKind === 'finalize') {
return self::retryFinalization($operationId);
}
$operation = CloudStorageOperation::query()
->where('operation_id', $operationId)->first();
if (!$operation instanceof CloudStorageOperation) {
throw new CloudError('对象存储核对操作不存在', 404);
if ($expectedKind !== null) {
throw new CloudError('对象存储核对操作不存在', 404);
}
return self::retryFinalization($operationId);
}
$kind = (string) $operation->kind;
if ($expectedKind !== null && $kind !== $expectedKind) {
throw new CloudError('对象存储人工处理类型不一致', 409);
}
if ($kind === 'copy') {
CloudStorageArtifactService::reopenReplicaAttention(
$operationId,
@ -198,6 +294,25 @@ final class CloudStorageReplicaAttentionService
];
}
/** @return array{operation_id:string,operation_kind:string,phase:string} */
private static function retryFinalization(string $publicationKey): array
{
$publication = CloudStoragePublication::query()
->where('publication_key', $publicationKey)->first();
if (!$publication instanceof CloudStoragePublication) {
throw new CloudError('对象存储核对操作不存在', 404);
}
CloudStoragePublicationOutboxService::reopenAttention(
(int) $publication->id,
time()
);
return [
'operation_id' => $publicationKey,
'operation_kind' => 'finalize',
'phase' => 'ready_to_finalize',
];
}
private static function boundedInt($value, int $minimum, int $maximum): int
{
$validated = filter_var($value, FILTER_VALIDATE_INT, [

@ -30,7 +30,9 @@ final class CloudStorageReplicaWorkerService
private readonly ?CloudStorageReplicaHelper $helper = null,
private readonly ?CloudObjectStorageS3Client $client = null,
?callable $runtimeResolver = null,
?callable $clock = null
?callable $clock = null,
private readonly ?CloudStoragePublicationOutboxService
$publicationOutbox = null
) {
$this->runtimeResolver = $runtimeResolver === null
? static fn (int $backendId, bool $healthy): array =>
@ -45,31 +47,52 @@ final class CloudStorageReplicaWorkerService
{
$this->errors = [];
$now = ($this->clock)();
$work = CloudStorageArtifactService::claimReplicaWork($now);
if ($work === null) {
return false;
}
// Publication finalization is a short, local DB operation. Run it
// before a remote copy that may occupy this worker for hours, while
// still bounding it to one item so replication always gets its turn.
$publicationWorked = false;
try {
if (($work['mode'] ?? null) === 'dispatch') {
$this->dispatch($work, $now);
} else {
$this->reconcile($work, $now);
}
$outbox = $this->publicationOutbox
?? new CloudStoragePublicationOutboxService();
$publicationWorked = $outbox->tick();
$this->errors = array_merge(
$this->errors,
$outbox->observationErrors()
);
} catch (\Throwable) {
$this->errors[] = 'storage_replica_tick_failed';
// A local outbox/database failure must not prevent an unrelated
// replica operation from making progress. The unchanged durable
// publication remains eligible for a later worker cycle.
$this->errors[] = 'storage_publication_outbox_tick_failed';
}
$work = CloudStorageArtifactService::claimReplicaWork($now);
$replicaWorked = $work !== null;
if ($work !== null) {
try {
$this->markUnknown(
$work,
'storage_replica_tick_failed',
is_array($work['metadata'] ?? null)
? $work['metadata'] : []
);
if (($work['mode'] ?? null) === 'dispatch') {
$this->dispatch($work, $now);
} else {
$this->reconcile($work, $now);
}
} catch (\Throwable) {
// The durable lease remains the recovery fence. Never create
// a replacement operation after an uncertain settlement.
$this->errors[] = 'storage_replica_tick_failed';
try {
$this->markUnknown(
$work,
'storage_replica_tick_failed',
is_array($work['metadata'] ?? null)
? $work['metadata'] : []
);
} catch (\Throwable) {
// The durable lease remains the recovery fence. Never
// create a replacement operation after an uncertain
// settlement.
}
}
}
return true;
$this->errors = array_values(array_unique($this->errors));
return $replicaWorked || $publicationWorked;
}
/** @return list<string> */

@ -283,18 +283,26 @@
var operationId = String(button.getAttribute('data-operation-id') || '');
var operationKind = String(button.getAttribute('data-operation-kind') || 'copy');
if (!/^[a-f0-9]{32}$/.test(operationId) || button.disabled) return;
var finalization = operationKind === 'finalize';
var primary = operationKind === 'publish';
confirmAction({
title: primary ? '重新核对主副本发布' : '重新核对副本复制',
message: '请先在对象存储提供商确认已清理多余的 multipart 会话。本次点击只重开原操作的只读核对,不会终止会话或直接新建上传;只有对象与会话的无效果证据稳定后才会沿用原操作安全重试。',
confirmLabel: '开始核对',
title: finalization ? '重新执行目录最终化'
: (primary ? '重新核对主副本发布' : '重新核对副本复制'),
message: finalization
? '本次只重开已经完成双副本保护的本地目录最终化,不会上传、覆盖或删除对象存储内容。'
: '请先在对象存储提供商确认已清理多余的 multipart 会话。本次点击只重开原操作的只读核对,不会终止会话或直接新建上传;只有对象与会话的无效果证据稳定后才会沿用原操作安全重试。',
confirmLabel: finalization ? '重新最终化' : '开始核对',
tone: 'primary',
}).then(function (confirmed) {
if (!confirmed) return;
button.disabled = true;
showNotice(notice, '正在重开原操作的只读核对…', true, false);
showNotice(notice, finalization
? '正在重新打开本地目录最终化…'
: '正在重开原操作的只读核对…', true, false);
request('/admin/object-storage/replica-attention/'
+ encodeURIComponent(operationId) + '/retry', {}).then(function (result) {
+ encodeURIComponent(operationId) + '/retry', {
operation_kind: operationKind,
}).then(function (result) {
showNotice(notice, result.msg, true, false);
window.setTimeout(function () {
window.location.hash = 'protection';

@ -59,16 +59,18 @@
<div class="os-panel" id="osPanelProtection" role="tabpanel" aria-labelledby="osTabProtection" data-os-panel="protection" hidden>
<section class="os-card"><div class="os-card-head"><div><p class="os-section-kicker">Replica coverage</p><h2>副本保护</h2><p>只有经大小与 SHA-256 核对为就绪的副本才计入覆盖率;连接已验证不等于对象已有双副本。</p></div></div><div class="os-status"><div class="os-status-item"><span>已保护制品</span><strong>{$object_storage_summary.protected_artifact_count|default:0}</strong></div><div class="os-status-item"><span>待复制 / 修复</span><strong>{$object_storage_summary.replication_backlog_count|default:0}</strong></div><div class="os-status-item"><span>降级制品</span><strong>{$object_storage_summary.degraded_artifact_count|default:0}</strong></div></div><div class="os-alert is-info">后台只为新操作选择健康后端;已经开始的 multipart 上传不会因健康状态变化而切换目标。读取仅从经过验证的就绪副本中选择。</div></section>
<section class="os-card" id="objectReplicaAttention">
<div class="os-card-head"><div><p class="os-section-kicker">Multipart reconciliation</p><h2>对象存储人工核对</h2><p>主副本发布或副本复制连续两次发现分段上传身份冲突后会停止自动轮询。请先在存储提供商清理多余会话,再使用原操作重新核对;本次点击不直接新建上传,只有对象与会话的无效果证据稳定后才会沿用原操作安全重试。</p></div><button class="btn btn-outline-secondary btn-sm" id="osAttentionRefresh" type="button"><i class="mdi mdi-refresh me-1"></i>刷新</button></div>
<div class="os-card-head"><div><p class="os-section-kicker">Recovery attention</p><h2>对象存储人工处理</h2><p>远端身份项只会重开原操作的只读核对,不会直接新建上传;目录最终化项只会重开已经完成副本保护的本地业务投影,不会上传、覆盖或删除远端对象。</p></div><button class="btn btn-outline-secondary btn-sm" id="osAttentionRefresh" type="button"><i class="mdi mdi-refresh me-1"></i>刷新</button></div>
{if ($replica_attention_error|default:'') != ''}<div class="os-alert is-error">{$replica_attention_error|escape}</div>
{elseif $replica_attention|default:false}
<div class="os-status os-debt-summary"><div class="os-status-item"><span>需人工核对</span><strong>{$replica_attention.summary.count} 个操作</strong></div><div class="os-status-item"><span>恢复边界</span><strong>先只读核对</strong><small>不会 Abort;无效果证据稳定后才安全重试</small></div></div>
{if $replica_attention.items|@count > 0}
<div class="os-debt-list" role="list"><div class="os-debt-list-head" aria-hidden="true"><span>制品</span><span>目标后端</span><span>冲突原因</span><span>停止时间</span><span>操作</span></div>
<div class="os-status os-debt-summary"><div class="os-status-item"><span>需人工处理</span><strong>{$replica_attention.summary.count} 个操作</strong></div><div class="os-status-item"><span>远端身份核对</span><strong>{$replica_attention.summary.provider_operation_count} 个</strong><small>只读核对原操作,不会 Abort</small></div><div class="os-status-item"><span>本地目录最终化</span><strong>{$replica_attention.summary.finalization_count} 个</strong><small>不涉及远端对象写入</small></div></div>
{if $replica_attention.items|@count > 0 || $replica_attention.finalization_items|@count > 0}
<div class="os-debt-list" role="list"><div class="os-debt-list-head" aria-hidden="true"><span>制品</span><span>处理范围</span><span>异常原因</span><span>停止时间</span><span>操作</span></div>
{foreach $replica_attention.items as $item}<article class="os-debt-row" role="listitem"><div class="os-debt-cell"><span class="os-cell-label">制品</span><strong>#{$item.artifact_id} {$item.artifact_kind|escape}</strong><small>{$item.operation_type|escape} · {$item.owner_text|escape}</small></div><div class="os-debt-cell"><span class="os-cell-label">目标后端</span><strong>#{$item.backend_id} {$item.backend_name|escape}</strong><small class="os-mono">{$item.operation_id|escape}</small></div><div class="os-debt-cell"><span class="os-cell-label">冲突原因</span><strong>{$item.error_text|escape}</strong><small class="os-mono">{$item.error_code|escape}</small></div><div class="os-debt-cell"><span class="os-cell-label">停止时间</span><strong>{if $item.attention_at > 0}{$item.attention_at|date_format:'%Y-%m-%d %H:%M:%S'|escape}{else}未记录{/if}</strong></div><div class="os-debt-cell os-debt-action"><span class="os-cell-label">操作</span>{if $item.can_retry && $admin_can_manage|default:false}<button class="btn btn-sm btn-outline-warning os-attention-retry" type="button" data-operation-id="{$item.operation_id|escape}" data-operation-kind="{$item.operation_kind|escape}">重新核对</button>{else}<span class="os-waiting-action"><i class="mdi mdi-lock-clock"></i>原操作不可恢复</span>{/if}</div></article>{/foreach}
{foreach $replica_attention.finalization_items as $item}<article class="os-debt-row" role="listitem"><div class="os-debt-cell"><span class="os-cell-label">制品</span><strong>#{$item.artifact_id} {$item.artifact_kind|escape}</strong><small>{$item.operation_type|escape} · {$item.owner_text|escape}</small></div><div class="os-debt-cell"><span class="os-cell-label">处理范围</span><strong>{$item.backend_name|escape}</strong><small class="os-mono">{$item.operation_id|escape}</small></div><div class="os-debt-cell"><span class="os-cell-label">失败原因</span><strong>{$item.error_text|escape}</strong><small class="os-mono">{$item.error_code|escape}</small></div><div class="os-debt-cell"><span class="os-cell-label">停止时间</span><strong>{if $item.attention_at > 0}{$item.attention_at|date_format:'%Y-%m-%d %H:%M:%S'|escape}{else}未记录{/if}</strong></div><div class="os-debt-cell os-debt-action"><span class="os-cell-label">操作</span>{if $item.can_retry && $admin_can_manage|default:false}<button class="btn btn-sm btn-outline-warning os-attention-retry" type="button" data-operation-id="{$item.operation_id|escape}" data-operation-kind="finalize">重新最终化</button>{else}<span class="os-waiting-action"><i class="mdi mdi-lock-clock"></i>目录状态需检查</span>{/if}</div></article>{/foreach}
</div>
{if $replica_attention.pagination.total_pages > 1}<div class="os-actions os-pagination"><a class="btn btn-sm btn-outline-secondary {if $replica_attention.pagination.page <= 1}disabled{/if}" href="/admin/object-storage?attention_page={$replica_attention.pagination.page-1}#protection">上一页</a><span class="os-help">第 {$replica_attention.pagination.page} / {$replica_attention.pagination.total_pages} 页</span><a class="btn btn-sm btn-outline-secondary {if $replica_attention.pagination.page >= $replica_attention.pagination.total_pages}disabled{/if}" href="/admin/object-storage?attention_page={$replica_attention.pagination.page+1}#protection">下一页</a></div>{/if}
{else}<div class="os-empty"><i class="mdi mdi-check-circle-outline"></i><strong>当前没有对象存储身份冲突</strong><span>主副本发布与副本复制均未被不确定的分段会话阻断。</span></div>{/if}{/if}
{if $replica_attention.pagination.total_pages > 1}<div class="os-actions os-pagination"><span class="os-help">远端身份核对</span><a class="btn btn-sm btn-outline-secondary {if $replica_attention.pagination.page <= 1}disabled{/if}" href="/admin/object-storage?attention_page={$replica_attention.pagination.page-1}&finalization_attention_page={$replica_attention.finalization_pagination.page}#protection">上一页</a><span class="os-help">第 {$replica_attention.pagination.page} / {$replica_attention.pagination.total_pages} 页</span><a class="btn btn-sm btn-outline-secondary {if $replica_attention.pagination.page >= $replica_attention.pagination.total_pages}disabled{/if}" href="/admin/object-storage?attention_page={$replica_attention.pagination.page+1}&finalization_attention_page={$replica_attention.finalization_pagination.page}#protection">下一页</a></div>{/if}
{if $replica_attention.finalization_pagination.total_pages > 1}<div class="os-actions os-pagination"><span class="os-help">本地目录最终化</span><a class="btn btn-sm btn-outline-secondary {if $replica_attention.finalization_pagination.page <= 1}disabled{/if}" href="/admin/object-storage?attention_page={$replica_attention.pagination.page}&finalization_attention_page={$replica_attention.finalization_pagination.page-1}#protection">上一页</a><span class="os-help">第 {$replica_attention.finalization_pagination.page} / {$replica_attention.finalization_pagination.total_pages} 页</span><a class="btn btn-sm btn-outline-secondary {if $replica_attention.finalization_pagination.page >= $replica_attention.finalization_pagination.total_pages}disabled{/if}" href="/admin/object-storage?attention_page={$replica_attention.pagination.page}&finalization_attention_page={$replica_attention.finalization_pagination.page+1}#protection">下一页</a></div>{/if}
{else}<div class="os-empty"><i class="mdi mdi-check-circle-outline"></i><strong>当前没有对象存储人工处理项</strong><span>远端身份核对与本地目录最终化均未被阻断。</span></div>{/if}{/if}
<div class="os-notice" id="osAttentionNotice" role="status" aria-live="polite"></div>
</section>
</div>

@ -0,0 +1,517 @@
<?php
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudStorageArtifact;
use App\Models\CloudStorageOperation;
use App\Models\CloudStoragePublication;
use App\Models\CloudStorageReplica;
use App\Services\Cloud\CloudError;
use App\Services\Cloud\CloudRescueStoragePublicationService;
use App\Services\Cloud\CloudStoragePublicationOutboxService;
use App\Services\Cloud\CloudStorageReplicaAttentionService;
use App\Services\Cloud\CloudStorageReplicaWorkerService;
use App\Services\Cloud\CloudTemplateStoragePublicationConsumer;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$assert(method_exists(
CloudTemplateStoragePublicationConsumer::class,
'finalizeReady'
) && method_exists(
CloudRescueStoragePublicationService::class,
'finalizeReadyOnly'
), 'Publication outbox lost a local-only business finalization consumer.');
$db = new Capsule();
$db->addConnection(['driver' => 'sqlite', 'database' => ':memory:']);
$db->setAsGlobal();
$db->bootEloquent();
$db->schema()->create('cloud_storage_artifacts', static function (
Blueprint $table
): void {
$table->increments('id');
$table->string('artifact_key')->unique();
$table->string('kind');
$table->unsignedBigInteger('maximum_size_bytes');
$table->unsignedBigInteger('size_bytes');
$table->string('sha256');
$table->string('durability_policy');
$table->unsignedTinyInteger('desired_copies');
$table->unsignedTinyInteger('required_copies');
$table->string('state');
$table->string('last_operation_id')->nullable();
$table->unsignedBigInteger('verified_at');
$table->unsignedBigInteger('created_at');
$table->unsignedBigInteger('updated_at');
});
$db->schema()->create('cloud_storage_publications', static function (
Blueprint $table
): void {
$table->increments('id');
$table->string('publication_key')->unique();
$table->unsignedBigInteger('artifact_id')->unique();
$table->unsignedBigInteger('backup_id')->nullable()->unique();
$table->string('template_import_intent_id')->nullable()->unique();
$table->string('rescue_template_id')->nullable()->unique();
$table->string('state');
$table->unsignedInteger('attempts')->default(0);
$table->unsignedBigInteger('available_at')->default(0);
$table->string('lease_token')->nullable();
$table->unsignedBigInteger('lease_until')->default(0);
$table->string('last_error_code')->default('');
$table->unsignedBigInteger('coverage_ready_at')->nullable();
$table->unsignedBigInteger('finalized_at')->nullable();
$table->unsignedBigInteger('created_at');
$table->unsignedBigInteger('updated_at');
});
$db->schema()->create('cloud_storage_operations', static function (
Blueprint $table
): void {
$table->increments('id');
$table->string('operation_id')->unique();
$table->string('request_idempotency_key')->unique();
$table->string('request_fingerprint');
$table->unsignedBigInteger('artifact_id');
$table->unsignedBigInteger('replica_id')->nullable();
$table->unsignedBigInteger('source_backend_id')->nullable();
$table->unsignedBigInteger('target_backend_id')->nullable();
$table->string('kind');
$table->string('phase');
$table->string('outcome')->nullable();
$table->string('effect_resolution');
$table->boolean('effect_may_have_applied')->nullable();
$table->boolean('safe_to_resubmit')->nullable();
$table->unsignedInteger('attempts')->default(0);
$table->string('lease_token')->nullable();
$table->unsignedBigInteger('lease_until')->default(0);
$table->unsignedBigInteger('available_at')->default(0);
$table->unsignedBigInteger('next_check_at')->default(0);
$table->string('error_code')->default('');
$table->text('result_json')->nullable();
$table->unsignedBigInteger('terminal_at')->nullable();
$table->unsignedBigInteger('created_at');
$table->unsignedBigInteger('updated_at');
});
$db->schema()->create('cloud_storage_replicas', static function (
Blueprint $table
): void {
$table->increments('id');
$table->string('replica_key')->unique();
$table->unsignedBigInteger('artifact_id');
$table->unsignedBigInteger('backend_id');
$table->string('role');
$table->string('object_key');
$table->string('object_version_id')->default('');
$table->string('object_etag')->default('');
$table->string('state');
$table->string('publish_operation_id')->nullable();
$table->string('delete_operation_id')->nullable();
$table->unsignedInteger('delete_generation')->default(0);
$table->unsignedInteger('delete_cycle_attempts')->default(0);
$table->unsignedInteger('delete_presence_confirmations')->default(0);
$table->unsignedBigInteger('delete_last_checked_at')->default(0);
$table->unsignedBigInteger('delete_next_check_at')->default(0);
$table->unsignedBigInteger('delete_attention_at')->nullable();
$table->string('lease_token')->nullable();
$table->unsignedBigInteger('lease_until')->default(0);
$table->unsignedBigInteger('size_bytes')->default(0);
$table->string('sha256')->default('');
$table->string('error_code')->default('');
$table->unsignedBigInteger('verified_at')->default(0);
$table->unsignedBigInteger('created_at');
$table->unsignedBigInteger('updated_at');
});
$db->schema()->create('cloud_storage_backends', static function (
Blueprint $table
): void {
$table->increments('id');
$table->string('name');
});
$clock = 2000000000;
$ownerA = str_repeat('a', 32);
$ownerB = str_repeat('b', 32);
$create = static function (
string $kind,
string $owner,
string $state = 'ready_to_finalize',
int $attempts = 0,
int $availableAt = 0
) use (&$clock): CloudStoragePublication {
$artifact = new CloudStorageArtifact();
$artifact->artifact_key = bin2hex(random_bytes(16));
$artifact->kind = $kind;
$artifact->maximum_size_bytes = 4096;
$artifact->size_bytes = 4096;
$artifact->sha256 = str_repeat($kind === 'template' ? 'a' : 'b', 64);
$artifact->durability_policy = 'mirror_required_2';
$artifact->desired_copies = 2;
$artifact->required_copies = 2;
$artifact->state = 'protected';
$artifact->last_operation_id = null;
$artifact->verified_at = $clock - 10;
$artifact->created_at = $artifact->updated_at = $clock - 20;
$artifact->save();
$publication = new CloudStoragePublication();
$publication->publication_key = bin2hex(random_bytes(16));
$publication->artifact_id = (int) $artifact->id;
$publication->backup_id = null;
$publication->template_import_intent_id =
$kind === 'template' ? $owner : null;
$publication->rescue_template_id =
$kind === 'rescue' ? $owner : null;
$publication->state = $state;
$publication->attempts = $attempts;
$publication->available_at = $availableAt;
$publication->lease_token = null;
$publication->lease_until = 0;
$publication->last_error_code = '';
$publication->coverage_ready_at = $clock - 5;
$publication->finalized_at = null;
$publication->created_at = $publication->updated_at = $clock - 20;
$publication->save();
return $publication;
};
$template = $create('template', $ownerA);
$calls = [];
$worker = new CloudStoragePublicationOutboxService(
static function (
string $type,
int|string $owner,
int $publicationId,
int $artifactId
) use (&$calls, &$clock, $assert): void {
$assert(Capsule::connection()->transactionLevel() > 0,
'Business finalization escaped the owner transaction.');
$calls[] = [$type, $owner, $publicationId, $artifactId];
$row = CloudStoragePublication::query()
->whereKey($publicationId)->lockForUpdate()->firstOrFail();
$row->state = 'completed';
$row->attempts = (int) $row->attempts + 1;
$row->finalized_at = $clock;
$row->updated_at = $clock;
$row->save();
},
static function () use (&$clock): int {
return $clock;
}
);
$assert($worker->tick(), 'A due template publication was not consumed.');
$template->refresh();
$assert((string) $template->state === 'completed'
&& (int) $template->finalized_at === $clock
&& $calls === [[
'template', $ownerA, (int) $template->id,
(int) $template->artifact_id,
]], 'Template outbox finalization lost its frozen owner identity.');
$assert(!$worker->tick(), 'A completed publication was consumed twice.');
$rescue = $create('rescue', $ownerB, 'ready_to_finalize', 0, $clock + 60);
$assert(!$worker->tick(), 'A future publication bypassed its retry delay.');
$clock += 60;
$assert($worker->tick(), 'A due rescue publication was not consumed.');
$rescue->refresh();
$assert((string) $rescue->state === 'completed'
&& $calls[1][0] === 'rescue' && $calls[1][1] === $ownerB,
'Rescue outbox dispatch selected the wrong owner.');
$failing = $create('template', str_repeat('c', 32));
$failingWorker = new CloudStoragePublicationOutboxService(
static function (
string $type,
int|string $owner,
int $publicationId
) use (&$clock): void {
// Prove that a consumer which mutates the business publication and
// then crashes is rolled back to the savepoint before the durable
// retry delay is committed.
$partial = CloudStoragePublication::query()
->whereKey($publicationId)->lockForUpdate()->firstOrFail();
$partial->state = 'completed';
$partial->finalized_at = $clock;
$partial->save();
throw new CloudError('fixture owner failure', 409);
},
static function () use (&$clock): int {
return $clock;
}
);
$assert($failingWorker->tick(), 'A failed outbox attempt was not observed.');
$failing->refresh();
$assert((string) $failing->state === 'ready_to_finalize'
&& (int) $failing->attempts === 1
&& $failing->finalized_at === null
&& (int) $failing->available_at === $clock + 60
&& (string) $failing->last_error_code
=== 'storage_publication_finalization_failed'
&& $failingWorker->observationErrors()
=== ['storage_publication_finalization_failed'],
'A failed owner finalization was not durably delayed.');
$assert(!$failingWorker->tick(), 'A delayed failure entered a hot loop.');
$attention = $create(
'rescue',
str_repeat('d', 32),
'ready_to_finalize',
5,
$clock
);
$assert($failingWorker->tick(), 'The bounded final attempt was not handled.');
$attention->refresh();
$assert((string) $attention->state === 'failed'
&& (int) $attention->attempts === 6
&& (string) $attention->last_error_code
=== 'storage_publication_finalization_attention',
'Exhausted owner finalization did not become explicit attention.');
$attentionView = CloudStorageReplicaAttentionService::listView();
$assert(($attentionView['summary']['finalization_count'] ?? 0) === 1
&& count($attentionView['finalization_items'] ?? []) === 1
&& ($attentionView['finalization_items'][0]['operation_kind'] ?? '')
=== 'finalize'
&& ($attentionView['finalization_items'][0]['operation_id'] ?? '')
=== (string) $attention->publication_key
&& ($attentionView['finalization_items'][0]['can_retry'] ?? false),
'Finalization attention is not visible on the administrator recovery page.');
$retry = CloudStorageReplicaAttentionService::retry(
(string) $attention->publication_key,
'finalize'
);
$attention->refresh();
$assert((string) $attention->state === 'ready_to_finalize'
&& (int) $attention->attempts === 0
&& (int) $attention->available_at > 0
&& (string) $attention->last_error_code === ''
&& ($retry['operation_kind'] ?? '') === 'finalize',
'Administrator recovery did not reopen the same publication intent.');
$attention->state = 'completed';
$attention->finalized_at = $clock;
$attention->save();
// Finalization attention has an independent stable paginator. Prove adjacent
// pages are disjoint and together expose every row instead of silently
// truncating after the first page.
$pagedAttention = [];
foreach (['1', '2', '3'] as $ownerDigit) {
$row = $create('template', str_repeat($ownerDigit, 32));
$row->state = 'failed';
$row->attempts = 6;
$row->available_at = 0;
$row->last_error_code =
'storage_publication_finalization_attention';
$row->updated_at = $clock + (int) $ownerDigit;
$row->save();
$pagedAttention[] = $row;
}
$finalizationPageOne = CloudStorageReplicaAttentionService::listView([
'page' => 1,
'finalization_page' => 1,
'per_page' => 2,
]);
$finalizationPageTwo = CloudStorageReplicaAttentionService::listView([
'page' => 1,
'finalization_page' => 2,
'per_page' => 2,
]);
$pageOneIds = array_column(
$finalizationPageOne['finalization_items'],
'operation_id'
);
$pageTwoIds = array_column(
$finalizationPageTwo['finalization_items'],
'operation_id'
);
$assert(count($pageOneIds) === 2 && count($pageTwoIds) === 1
&& array_intersect($pageOneIds, $pageTwoIds) === []
&& count(array_unique(array_merge($pageOneIds, $pageTwoIds))) === 3
&& ($finalizationPageOne['finalization_pagination']['total'] ?? 0) === 3
&& ($finalizationPageOne['finalization_pagination']['total_pages'] ?? 0)
=== 2,
'Finalization attention pagination repeated or omitted an item.');
foreach ($pagedAttention as $row) {
$row->state = 'completed';
$row->finalized_at = $clock;
$row->save();
}
$databaseFault = $create('rescue', str_repeat('4', 32));
$databaseFaultWorker = new CloudStoragePublicationOutboxService(
static function (): void {
throw new PDOException('fixture database outage');
},
static function () use (&$clock): int {
return $clock;
}
);
$databaseFaultRaised = false;
try {
$databaseFaultWorker->tick();
} catch (PDOException) {
$databaseFaultRaised = true;
}
$databaseFault->refresh();
$assert($databaseFaultRaised
&& (string) $databaseFault->state === 'ready_to_finalize'
&& (int) $databaseFault->attempts === 0,
'Database failure consumed the bounded owner-finalization budget.');
$databaseFault->state = 'completed';
$databaseFault->finalized_at = $clock;
$databaseFault->save();
$wrong = $create('template', str_repeat('e', 32));
$noSettlement = new CloudStoragePublicationOutboxService(
static function (): void {
// A domain consumer returning without completing the outbox is an
// error; it must roll back and enter the same bounded retry path.
},
static function () use (&$clock): int {
return $clock;
}
);
$assert($noSettlement->tick(), 'Missing business settlement was not handled.');
$wrong->refresh();
$assert((string) $wrong->state === 'ready_to_finalize'
&& (int) $wrong->attempts === 1,
'A no-op business consumer was treated as successful.');
$wrong->state = 'completed';
$wrong->finalized_at = $clock;
$wrong->save();
$workerOwned = $create('rescue', str_repeat('f', 32));
$copyArtifact = new CloudStorageArtifact();
$copyArtifact->artifact_key = bin2hex(random_bytes(16));
$copyArtifact->kind = 'backup';
$copyArtifact->maximum_size_bytes = 4096;
$copyArtifact->size_bytes = 4096;
$copyArtifact->sha256 = str_repeat('9', 64);
$copyArtifact->durability_policy = 'mirror_required_2';
$copyArtifact->desired_copies = 2;
$copyArtifact->required_copies = 2;
$copyArtifact->state = 'available';
$copyArtifact->last_operation_id = null;
$copyArtifact->verified_at = $clock;
$copyArtifact->created_at = $copyArtifact->updated_at = $clock;
$copyArtifact->save();
$sourceReplica = new CloudStorageReplica();
$sourceReplica->replica_key = bin2hex(random_bytes(16));
$sourceReplica->artifact_id = (int) $copyArtifact->id;
$sourceReplica->backend_id = 1;
$sourceReplica->role = 'primary';
$sourceReplica->object_key = 'fixture/source.qcow2';
$sourceReplica->object_version_id = '';
$sourceReplica->object_etag = '"source"';
$sourceReplica->size_bytes = 4096;
$sourceReplica->sha256 = str_repeat('9', 64);
$sourceReplica->state = 'ready';
$sourceReplica->publish_operation_id = null;
$sourceReplica->delete_operation_id = null;
$sourceReplica->lease_token = null;
$sourceReplica->lease_until = 0;
$sourceReplica->error_code = '';
$sourceReplica->verified_at = $clock;
$sourceReplica->created_at = $sourceReplica->updated_at = $clock;
$sourceReplica->save();
$targetReplica = new CloudStorageReplica();
$targetReplica->replica_key = bin2hex(random_bytes(16));
$targetReplica->artifact_id = (int) $copyArtifact->id;
$targetReplica->backend_id = 2;
$targetReplica->role = 'secondary';
$targetReplica->object_key = 'fixture/target.qcow2';
$targetReplica->object_version_id = '';
$targetReplica->object_etag = '';
$targetReplica->size_bytes = 0;
$targetReplica->sha256 = '';
$targetReplica->state = 'reserved';
$targetReplica->publish_operation_id = null;
$targetReplica->delete_operation_id = null;
$targetReplica->lease_token = null;
$targetReplica->lease_until = 0;
$targetReplica->error_code = '';
$targetReplica->verified_at = 0;
$targetReplica->created_at = $targetReplica->updated_at = $clock;
$targetReplica->save();
$copyOperation = new CloudStorageOperation();
$copyOperation->operation_id = bin2hex(random_bytes(16));
$copyOperation->request_idempotency_key =
'publication-worker-copy-fixture';
$copyOperation->request_fingerprint = hash('sha256', 'copy-fixture');
$copyOperation->artifact_id = (int) $copyArtifact->id;
$copyOperation->replica_id = (int) $targetReplica->id;
$copyOperation->source_backend_id = 1;
$copyOperation->target_backend_id = 2;
$copyOperation->kind = 'copy';
$copyOperation->phase = 'accepted';
$copyOperation->outcome = null;
$copyOperation->effect_resolution = 'not_applicable';
$copyOperation->effect_may_have_applied = null;
$copyOperation->safe_to_resubmit = null;
$copyOperation->attempts = 0;
$copyOperation->lease_token = null;
$copyOperation->lease_until = 0;
$copyOperation->available_at = $clock;
$copyOperation->next_check_at = 0;
$copyOperation->error_code = '';
$copyOperation->result_json = null;
$copyOperation->terminal_at = null;
$copyOperation->created_at = $copyOperation->updated_at = $clock;
$copyOperation->save();
$integratedOutbox = new CloudStoragePublicationOutboxService(
static function (
string $type,
int|string $owner,
int $publicationId,
int $artifactId
) use (&$clock, $assert, $sourceReplica): void {
$assert($type === 'rescue' && $owner === str_repeat('f', 32),
'Replica worker dispatched the wrong publication owner.');
$publication = CloudStoragePublication::query()
->whereKey($publicationId)->lockForUpdate()->firstOrFail();
$publication->state = 'completed';
$publication->finalized_at = $clock;
$publication->save();
// Makes the replica source invalid only after publication work ran;
// the following replica_source_not_ready assertion therefore proves
// the short outbox step was consumed before the replica claim.
$sourceReplica->state = 'degraded';
$sourceReplica->error_code = 'fixture_not_ready';
$sourceReplica->verified_at = 0;
$sourceReplica->save();
},
static function () use (&$clock): int {
return $clock;
}
);
$replicaWorker = new CloudStorageReplicaWorkerService(
null,
null,
null,
static function () use (&$clock): int {
return $clock;
},
$integratedOutbox
);
$assert($replicaWorker->tick()
&& $replicaWorker->observationErrors()
=== ['replica_source_not_ready'],
'Replica worker did not process both bounded work queues in one tick.');
$workerOwned->refresh();
$copyOperation->refresh();
$assert((string) $workerOwned->state === 'completed',
'Replica worker did not settle the owner publication.');
$assert((string) $copyOperation->phase === 'accepted'
&& (int) $copyOperation->attempts === 1
&& (int) $copyOperation->available_at === $clock + 60
&& (string) $copyOperation->error_code === 'replica_source_not_ready',
'Publication finalization starved the replica queue in the same tick.');
echo "Storage publication outbox behavior passed.\n";
Loading…
Cancel
Save