fix(storage): reopen immutable primary attention

go
codex 2 weeks ago
parent 7dc7e31160
commit cc48eb151a

@ -18,12 +18,26 @@ use App\Models\CloudStorageReplica;
*/
final class CloudStorageImmutablePrimaryPublication
{
/** Secondary work remains dormant until the primary is verified again. */
private const COPY_BLOCKED_UNTIL_PRIMARY = 253402300799;
private const LEASE_SECONDS = 300;
private const RECONCILE_DELAY = 60;
private const RETRY_DELAY = 300;
private const MAX_DISPATCH_ATTEMPTS = 3;
private const MAX_RECONCILE_FAILURES = 6;
/** @var list<string> */
private const UNKNOWN_ATTENTION_ERRORS = [
'immutable_primary_identity_conflict',
'immutable_primary_reconcile_exhausted',
];
/** @var list<string> */
private const NOT_APPLIED_ATTENTION_ERRORS = [
'immutable_primary_publish_retry_exhausted',
'immutable_primary_absence_retry_exhausted',
];
/** @var \Closure(int,bool):array<string,mixed> */
private readonly \Closure $runtimeResolver;
@ -51,6 +65,205 @@ final class CloudStorageImmutablePrimaryPublication
: \Closure::fromCallable($clock);
}
/** @return list<string> */
public static function attentionErrors(): array
{
return array_merge(
self::UNKNOWN_ATTENTION_ERRORS,
self::NOT_APPLIED_ATTENTION_ERRORS
);
}
/** @return list<string> */
public static function unknownAttentionErrors(): array
{
return self::UNKNOWN_ATTENTION_ERRORS;
}
/** @return list<string> */
public static function notAppliedAttentionErrors(): array
{
return self::NOT_APPLIED_ATTENTION_ERRORS;
}
/**
* Re-open only the original frozen publish operation.
*
* Unknown effects resume with an exact-key HEAD. A provider write becomes
* dispatchable again only after this class observes two spaced absences.
* Proven non-application may dispatch the same operation again. Secondary
* copies stopped solely by the primary attention state are restored behind
* the primary fence; no new artifact, key or operation is allocated.
*/
public static function reopenAttention(
string $operationId,
int $now
): void {
self::assertOperationId($operationId);
if ($now < 1) {
throw new \InvalidArgumentException('Invalid attention time');
}
$connection = (new CloudStorageOperation())->getConnection();
$connection->transaction(static function () use (
$operationId,
$now
): void {
$operation = CloudStorageOperation::query()
->where('operation_id', $operationId)
->lockForUpdate()->first();
if (!$operation instanceof CloudStorageOperation
|| (string) $operation->kind !== 'publish') {
throw new CloudError('对象存储发布核对操作不存在', 404);
}
$copyOperations = CloudStorageOperation::query()
->where('artifact_id', (int) $operation->artifact_id)
->where('kind', 'copy')->orderBy('id')
->lockForUpdate()->get()->all();
$artifact = CloudStorageArtifact::query()
->whereKey((int) $operation->artifact_id)
->lockForUpdate()->first();
$replicas = CloudStorageReplica::query()
->where('artifact_id', (int) $operation->artifact_id)
->orderBy('id')->lockForUpdate()->get()->all();
$publication = CloudStoragePublication::query()
->where('artifact_id', (int) $operation->artifact_id)
->lockForUpdate()->first();
if (!$artifact instanceof CloudStorageArtifact
|| !$publication instanceof CloudStoragePublication
|| $publication->backup_id !== null
|| (($publication->template_import_intent_id === null)
=== ($publication->rescue_template_id === null))) {
throw new CloudError('对象存储发布核对归属不一致', 409);
}
$primary = null;
$replicasById = [];
foreach ($replicas as $replica) {
$replicasById[(int) $replica->id] = $replica;
if ((int) $replica->id === (int) $operation->replica_id) {
$primary = $replica;
}
}
$errorCode = (string) $operation->error_code;
$unknown = in_array(
$errorCode,
self::UNKNOWN_ATTENTION_ERRORS,
true
);
$notApplied = in_array(
$errorCode,
self::NOT_APPLIED_ATTENTION_ERRORS,
true
);
if (!$primary instanceof CloudStorageReplica
|| (string) $primary->role !== 'primary'
|| (!$unknown && !$notApplied)
|| (string) $operation->phase !== 'terminal'
|| (string) $operation->outcome !== 'failed'
|| (bool) $operation->safe_to_resubmit
|| (bool) $operation->effect_may_have_applied !== $unknown
|| (string) $operation->effect_resolution
!== ($unknown ? 'unknown' : 'not_applied')
|| (string) $artifact->state !== 'degraded'
|| (string) $primary->state !== 'degraded'
|| !hash_equals($errorCode, (string) $primary->error_code)
|| (string) $publication->state !== 'failed'
|| !hash_equals(
$errorCode,
(string) $publication->last_error_code
)) {
throw new CloudError(
'对象存储主副本不在可重新核对的人工处理终态',
409
);
}
foreach ($copyOperations as $copy) {
$secondary = $replicasById[(int) $copy->replica_id] ?? null;
if (!$secondary instanceof CloudStorageReplica
|| (string) $secondary->role !== 'secondary'
|| (string) $copy->phase !== 'terminal'
|| (string) $copy->outcome !== 'failed'
|| (string) $copy->effect_resolution !== 'not_applied'
|| (bool) $copy->effect_may_have_applied
|| (bool) $copy->safe_to_resubmit
|| (int) $copy->attempts !== 0
|| (string) $copy->error_code
!== 'primary_publish_attention'
|| (string) $secondary->state !== 'degraded'
|| (string) $secondary->error_code
!== 'primary_publish_attention') {
throw new CloudError(
'对象存储次副本核对账本不一致',
409
);
}
}
$metadata = self::resetEvidence($operation->result_json);
$operation->phase = $unknown ? 'reconciling' : 'accepted';
$operation->outcome = null;
$operation->effect_resolution = $unknown
? 'unknown' : 'not_applicable';
$operation->effect_may_have_applied = null;
$operation->safe_to_resubmit = null;
$operation->lease_token = null;
$operation->lease_until = 0;
$operation->available_at = $unknown ? 0 : $now;
$operation->next_check_at = $unknown ? $now : 0;
$operation->error_code = $unknown
? 'admin_reconcile_requested'
: 'admin_retry_requested';
$operation->result_json = self::encodedMetadata($metadata);
$operation->terminal_at = null;
$operation->updated_at = $now;
$operation->save();
$primary->state = $unknown ? 'publishing' : 'reserved';
$primary->lease_token = null;
$primary->lease_until = 0;
$primary->error_code = $operation->error_code;
$primary->updated_at = $now;
$primary->save();
foreach ($copyOperations as $copy) {
/** @var CloudStorageReplica $secondary */
$secondary = $replicasById[(int) $copy->replica_id];
$copy->phase = 'accepted';
$copy->outcome = null;
$copy->effect_resolution = 'not_applicable';
$copy->effect_may_have_applied = null;
$copy->safe_to_resubmit = null;
$copy->lease_token = null;
$copy->lease_until = 0;
$copy->available_at = self::COPY_BLOCKED_UNTIL_PRIMARY;
$copy->next_check_at = 0;
$copy->error_code = '';
$copy->terminal_at = null;
$copy->updated_at = $now;
$copy->save();
$secondary->state = 'reserved';
$secondary->lease_token = null;
$secondary->lease_until = 0;
$secondary->error_code = '';
$secondary->updated_at = $now;
$secondary->save();
}
$artifact->state = 'publishing';
$artifact->updated_at = $now;
$artifact->save();
$publication->state = 'waiting_primary';
$publication->attempts = 0;
$publication->available_at = 0;
$publication->lease_token = null;
$publication->lease_until = 0;
$publication->last_error_code = '';
$publication->coverage_ready_at = null;
$publication->finalized_at = null;
$publication->updated_at = $now;
$publication->save();
}, 3);
}
/**
* Advance one immutable primary publication by at most one remote action.
*

@ -31,24 +31,65 @@ final class CloudStorageReplicaAttentionService
100000
);
$perPage = self::boundedInt($input['per_page'] ?? 20, 1, 50);
$immutableUnknown = CloudStorageImmutablePrimaryPublication::
unknownAttentionErrors();
$immutableNotApplied = CloudStorageImmutablePrimaryPublication::
notAppliedAttentionErrors();
$base = CloudStorageOperation::query()
->where('phase', 'terminal')
->where('outcome', 'failed')
->where('effect_resolution', 'unknown')
->where('effect_may_have_applied', 1)
->where('safe_to_resubmit', 0)
->where(static function ($attention): void {
->where(static function ($attention) use (
$immutableUnknown,
$immutableNotApplied
): void {
$attention->where(static function ($copy): void {
$copy->where('kind', 'copy')->whereIn(
'error_code',
CloudStorageArtifactService::replicaAttentionErrors()
);
$copy->where('kind', 'copy')
->where('effect_resolution', 'unknown')
->where('effect_may_have_applied', 1)
->whereIn(
'error_code',
CloudStorageArtifactService::
replicaAttentionErrors()
);
})->orWhere(static function ($primary): void {
$primary->where('kind', 'publish')->whereIn(
'error_code',
CloudStorageArtifactService::
primaryPreparationAttentionErrors()
);
$primary->where('kind', 'publish')
->where('effect_resolution', 'unknown')
->where('effect_may_have_applied', 1)
->whereIn(
'error_code',
CloudStorageArtifactService::
primaryPreparationAttentionErrors()
);
})->orWhere(static function ($immutable) use (
$immutableUnknown,
$immutableNotApplied
): void {
$immutable->where('kind', 'publish')
->where(static function ($shape) use (
$immutableUnknown,
$immutableNotApplied
): void {
$shape->where(static function ($unknown) use (
$immutableUnknown
): void {
$unknown->whereIn(
'error_code',
$immutableUnknown
)->where('effect_resolution', 'unknown')
->where('effect_may_have_applied', 1);
})->orWhere(static function ($notApplied) use (
$immutableNotApplied
): void {
$notApplied->whereIn(
'error_code',
$immutableNotApplied
)->where(
'effect_resolution',
'not_applied'
)->where('effect_may_have_applied', 0);
});
});
});
});
$total = (clone $base)->count();
@ -96,6 +137,11 @@ final class CloudStorageReplicaAttentionService
$target = $sourceReplicas[(int) $operation->artifact_id
. ':' . (int) $operation->target_backend_id] ?? null;
$copy = (string) $operation->kind === 'copy';
$immutable = !$copy && in_array(
(string) $operation->error_code,
CloudStorageImmutablePrimaryPublication::attentionErrors(),
true
);
$publication = $copy ? null
: $publications->get((int) $operation->artifact_id);
$backup = $publication instanceof CloudStoragePublication
@ -112,16 +158,41 @@ final class CloudStorageReplicaAttentionService
(string) $operation->error_code,
(string) $target->error_code
);
$canRetry = $baseRetry && ($copy
? ($source instanceof CloudStorageReplica
$immutableOwner = self::immutableOwner($publication);
$canRetry = false;
$ownerType = 'backup';
$ownerId = (int) ($publication?->backup_id ?? 0);
$ownerText = '备份 #' . $ownerId;
if ($copy) {
$canRetry = $baseRetry
&& $source instanceof CloudStorageReplica
&& (string) $source->state === 'ready'
&& (int) $source->size_bytes
=== (int) $artifact->size_bytes
&& hash_equals(
(string) $artifact->sha256,
(string) $source->sha256
))
: ($publication instanceof CloudStoragePublication
);
$ownerType = 'artifact';
$ownerId = (int) $operation->artifact_id;
$ownerText = '制品 #' . $ownerId;
} elseif ($immutable && $immutableOwner !== null) {
$canRetry = $baseRetry
&& $publication instanceof CloudStoragePublication
&& (string) $publication->state === 'failed'
&& hash_equals(
(string) $operation->error_code,
(string) $publication->last_error_code
)
&& (string) $artifact->kind
=== (string) $immutableOwner['type'];
$ownerType = (string) $immutableOwner['type'];
$ownerId = (string) $immutableOwner['id'];
$ownerText = ($ownerType === 'template'
? '云模板导入 ' : '救援 ISO ') . $ownerId;
} else {
$canRetry = $baseRetry
&& $publication instanceof CloudStoragePublication
&& (string) $publication->state === 'failed'
&& hash_equals(
(string) $operation->error_code,
@ -133,12 +204,15 @@ final class CloudStorageReplicaAttentionService
&& $backup->active_job_id === null
&& $backup->last_operation_id === null
&& (int) $backup->storage_artifact_id
=== (int) $artifact->id));
=== (int) $artifact->id;
}
$items[] = [
'operation_id' => (string) $operation->operation_id,
'operation_kind' => (string) $operation->kind,
'operation_type' => $copy
? '副本复制' : '主副本发布准备',
? '副本复制'
: ($immutable
? '不可变主副本发布' : '主副本发布准备'),
'artifact_id' => (int) $operation->artifact_id,
'artifact_kind' => $artifact instanceof CloudStorageArtifact
? (string) $artifact->kind : 'unknown',
@ -147,12 +221,9 @@ final class CloudStorageReplicaAttentionService
'backend_id' => (int) $operation->target_backend_id,
'backend_name' => $backend instanceof CloudStorageBackend
? (string) $backend->name : '未知后端',
'owner_type' => $copy ? 'artifact' : 'backup',
'owner_id' => $copy ? (int) $operation->artifact_id
: (int) ($publication?->backup_id ?? 0),
'owner_text' => $copy
? '制品 #' . (int) $operation->artifact_id
: '备份 #' . (int) ($publication?->backup_id ?? 0),
'owner_type' => $ownerType,
'owner_id' => $ownerId,
'owner_text' => $ownerText,
'error_code' => (string) $operation->error_code,
'error_text' => self::errorText(
(string) $operation->error_code
@ -280,10 +351,21 @@ final class CloudStorageReplicaAttentionService
time()
);
} elseif ($kind === 'publish') {
CloudStorageArtifactService::reopenPrimaryPreparationAttention(
$operationId,
time()
);
if (in_array(
(string) $operation->error_code,
CloudStorageImmutablePrimaryPublication::attentionErrors(),
true
)) {
CloudStorageImmutablePrimaryPublication::reopenAttention(
$operationId,
time()
);
} else {
CloudStorageArtifactService::reopenPrimaryPreparationAttention(
$operationId,
time()
);
}
} else {
throw new CloudError('对象存储操作不支持人工核对', 409);
}
@ -342,7 +424,40 @@ final class CloudStorageReplicaAttentionService
'主副本记录的分段上传会话与远端不一致',
'primary_multipart_identity_untrusted' =>
'主副本远端会话无法归属到原发布操作',
'immutable_primary_identity_conflict' =>
'远端对象身份与冻结制品不一致',
'immutable_primary_reconcile_exhausted' =>
'主副本只读核对连续失败',
'immutable_primary_publish_retry_exhausted' =>
'主副本已确认未写入,但自动发布重试已用尽',
'immutable_primary_absence_retry_exhausted' =>
'主副本已稳定确认不存在,但自动发布重试已用尽',
default => '分段上传身份需要人工核对',
};
}
/** @return array{type:string,id:string}|null */
private static function immutableOwner(
?CloudStoragePublication $publication
): ?array {
if (!$publication instanceof CloudStoragePublication
|| $publication->backup_id !== null) {
return null;
}
$template = (string) (
$publication->template_import_intent_id ?? ''
);
$rescue = (string) ($publication->rescue_template_id ?? '');
if (($template === '') === ($rescue === '')) {
return null;
}
$id = $template !== '' ? $template : $rescue;
if (preg_match('/^[a-f0-9]{32}$/D', $id) !== 1) {
return null;
}
return [
'type' => $template !== '' ? 'template' : 'rescue',
'id' => $id,
];
}
}

@ -10,6 +10,7 @@ use App\Services\Cloud\CloudError;
use App\Services\Cloud\CloudObjectStorageS3Client;
use App\Services\Cloud\CloudStorageArtifactService;
use App\Services\Cloud\CloudStorageImmutablePrimaryPublication;
use App\Services\Cloud\CloudStorageReplicaAttentionService;
use GuzzleHttp\Client;
use GuzzleHttp\HandlerStack;
use GuzzleHttp\Promise\Create;
@ -489,6 +490,132 @@ $assert($conflictResult['requires_attention'] === true
&& count($requests) === 2,
'Conflicting immutable identity did not stop in operator attention.');
// An administrator may reopen only the same unknown-effect operation for
// read-only HEAD reconciliation. Secondary work stopped by the primary fence
// is restored but remains blocked until that exact primary succeeds.
$artifactCountBeforeReopen = CloudStorageArtifact::count();
$operationCountBeforeReopen = CloudStorageOperation::count();
$clock += 1;
$attention = CloudStorageReplicaAttentionService::listView();
$attentionItem = collect($attention['items'])->firstWhere(
'operation_id',
$conflict['operation_id']
);
$assert(is_array($attentionItem)
&& ($attentionItem['operation_type'] ?? null)
=== '不可变主副本发布'
&& ($attentionItem['owner_type'] ?? null) === 'rescue'
&& ($attentionItem['can_retry'] ?? false) === true,
'Immutable primary attention was not exposed to administrators.');
$attentionResult = CloudStorageReplicaAttentionService::retry(
$conflict['operation_id'],
'publish'
);
$assert(($attentionResult['phase'] ?? null) === 'reconciling',
'Immutable primary attention did not reopen through the admin service.');
$reopenedConflict = CloudStorageOperation::where(
'operation_id', $conflict['operation_id']
)->firstOrFail();
$reopenedCopy = CloudStorageOperation::where(
'artifact_id', $conflict['artifact_id']
)->where('kind', 'copy')->firstOrFail();
$reopenedPrimary = CloudStorageReplica::whereKey(
$conflict['replica_id']
)->firstOrFail();
$reopenedSecondary = CloudStorageReplica::whereKey(
(int) $reopenedCopy->replica_id
)->firstOrFail();
$assert((string) $reopenedConflict->phase === 'reconciling'
&& (string) $reopenedConflict->effect_resolution === 'unknown'
&& $reopenedConflict->outcome === null
&& (string) $reopenedPrimary->state === 'publishing'
&& (string) $reopenedCopy->phase === 'accepted'
&& (int) $reopenedCopy->available_at === 253402300799
&& (string) $reopenedSecondary->state === 'reserved'
&& (string) CloudStorageArtifact::whereKey(
$conflict['artifact_id']
)->value('state') === 'publishing'
&& (string) CloudStoragePublication::where(
'artifact_id', $conflict['artifact_id']
)->value('state') === 'waiting_primary'
&& CloudStorageArtifact::count() === $artifactCountBeforeReopen
&& CloudStorageOperation::count() === $operationCountBeforeReopen,
'Administrator reconciliation did not reopen the original fenced ledger.');
$responses = [new Response(404), new Response(404)];
$requests = [];
$transactionLevels = [];
$module = $publisher($responses, $requests, $transactionLevels, $clock);
$module->publish(
$conflict['operation_id'], $conflict['artifact_id'], $conflict
);
$clock += 60;
$reconciledAbsence = $module->publish(
$conflict['operation_id'], $conflict['artifact_id'], $conflict
);
$assert($reconciledAbsence['phase'] === 'accepted'
&& array_map(
static fn (RequestInterface $request): string => $request->getMethod(),
$requests
) === ['HEAD', 'HEAD'],
'Administrator recovery replayed a write before stable absence evidence.');
// Repeated proven non-application also stops for operator review. Reopening
// authorizes one new dispatch of the same operation and immutable key only.
$manualRetry = $fixture('template');
$clock += 1000;
$responses = [new Response(403), new Response(403), new Response(403)];
$requests = [];
$transactionLevels = [];
$module = $publisher($responses, $requests, $transactionLevels, $clock);
for ($attempt = 0; $attempt < 3; $attempt++) {
$module->publish(
$manualRetry['operation_id'],
$manualRetry['artifact_id'],
$manualRetry
);
$clock += 300;
}
$manualRetryOperation = CloudStorageOperation::where(
'operation_id', $manualRetry['operation_id']
)->firstOrFail();
$assert((string) $manualRetryOperation->phase === 'terminal'
&& (string) $manualRetryOperation->effect_resolution === 'not_applied'
&& (string) $manualRetryOperation->error_code
=== 'immutable_primary_publish_retry_exhausted',
'Bounded proven failures did not enter manual retry attention.');
$manualAttention = CloudStorageReplicaAttentionService::listView();
$manualAttentionItem = collect($manualAttention['items'])->firstWhere(
'operation_id',
$manualRetry['operation_id']
);
$assert(is_array($manualAttentionItem)
&& ($manualAttentionItem['can_retry'] ?? false) === true,
'Proven-not-applied primary attention was not exposed for retry.');
CloudStorageReplicaAttentionService::retry(
$manualRetry['operation_id'],
'publish'
);
$responses = [new Response(200, [
'ETag' => '"manual-retry-etag"',
'x-amz-version-id' => 'manual-retry-version',
])];
$requests = [];
$transactionLevels = [];
$module = $publisher($responses, $requests, $transactionLevels, $clock);
$manualRetryResult = $module->publish(
$manualRetry['operation_id'],
$manualRetry['artifact_id'],
$manualRetry
);
$assert($manualRetryResult['primary_completed'] === true
&& count($requests) === 1
&& $requests[0]->getMethod() === 'PUT'
&& (int) CloudStorageOperation::where(
'operation_id', $manualRetry['operation_id']
)->value('attempts') === 4,
'Manual retry did not reuse the exact proven-not-applied operation.');
// The expected artifact fence runs before source validation or provider I/O.
$fencedA = $fixture('template');
$fencedB = $fixture('rescue');

Loading…
Cancel
Save