|
|
|
|
@ -18,12 +18,26 @@ use App\Models\CloudStorageReplica;
|
|
|
|
|
*/
|
|
|
|
|
final class CloudStorageImmutablePrimaryPublication
|
|
|
|
|
{
|
|
|
|
|
/** Secondary work remains dormant until the primary is verified again. */
|
|
|
|
|
private const COPY_BLOCKED_UNTIL_PRIMARY = 253402300799;
|
|
|
|
|
private const LEASE_SECONDS = 300;
|
|
|
|
|
private const RECONCILE_DELAY = 60;
|
|
|
|
|
private const RETRY_DELAY = 300;
|
|
|
|
|
private const MAX_DISPATCH_ATTEMPTS = 3;
|
|
|
|
|
private const MAX_RECONCILE_FAILURES = 6;
|
|
|
|
|
|
|
|
|
|
/** @var list<string> */
|
|
|
|
|
private const UNKNOWN_ATTENTION_ERRORS = [
|
|
|
|
|
'immutable_primary_identity_conflict',
|
|
|
|
|
'immutable_primary_reconcile_exhausted',
|
|
|
|
|
];
|
|
|
|
|
|
|
|
|
|
/** @var list<string> */
|
|
|
|
|
private const NOT_APPLIED_ATTENTION_ERRORS = [
|
|
|
|
|
'immutable_primary_publish_retry_exhausted',
|
|
|
|
|
'immutable_primary_absence_retry_exhausted',
|
|
|
|
|
];
|
|
|
|
|
|
|
|
|
|
/** @var \Closure(int,bool):array<string,mixed> */
|
|
|
|
|
private readonly \Closure $runtimeResolver;
|
|
|
|
|
|
|
|
|
|
@ -51,6 +65,205 @@ final class CloudStorageImmutablePrimaryPublication
|
|
|
|
|
: \Closure::fromCallable($clock);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** @return list<string> */
|
|
|
|
|
public static function attentionErrors(): array
|
|
|
|
|
{
|
|
|
|
|
return array_merge(
|
|
|
|
|
self::UNKNOWN_ATTENTION_ERRORS,
|
|
|
|
|
self::NOT_APPLIED_ATTENTION_ERRORS
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** @return list<string> */
|
|
|
|
|
public static function unknownAttentionErrors(): array
|
|
|
|
|
{
|
|
|
|
|
return self::UNKNOWN_ATTENTION_ERRORS;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** @return list<string> */
|
|
|
|
|
public static function notAppliedAttentionErrors(): array
|
|
|
|
|
{
|
|
|
|
|
return self::NOT_APPLIED_ATTENTION_ERRORS;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Re-open only the original frozen publish operation.
|
|
|
|
|
*
|
|
|
|
|
* Unknown effects resume with an exact-key HEAD. A provider write becomes
|
|
|
|
|
* dispatchable again only after this class observes two spaced absences.
|
|
|
|
|
* Proven non-application may dispatch the same operation again. Secondary
|
|
|
|
|
* copies stopped solely by the primary attention state are restored behind
|
|
|
|
|
* the primary fence; no new artifact, key or operation is allocated.
|
|
|
|
|
*/
|
|
|
|
|
public static function reopenAttention(
|
|
|
|
|
string $operationId,
|
|
|
|
|
int $now
|
|
|
|
|
): void {
|
|
|
|
|
self::assertOperationId($operationId);
|
|
|
|
|
if ($now < 1) {
|
|
|
|
|
throw new \InvalidArgumentException('Invalid attention time');
|
|
|
|
|
}
|
|
|
|
|
$connection = (new CloudStorageOperation())->getConnection();
|
|
|
|
|
$connection->transaction(static function () use (
|
|
|
|
|
$operationId,
|
|
|
|
|
$now
|
|
|
|
|
): void {
|
|
|
|
|
$operation = CloudStorageOperation::query()
|
|
|
|
|
->where('operation_id', $operationId)
|
|
|
|
|
->lockForUpdate()->first();
|
|
|
|
|
if (!$operation instanceof CloudStorageOperation
|
|
|
|
|
|| (string) $operation->kind !== 'publish') {
|
|
|
|
|
throw new CloudError('对象存储发布核对操作不存在', 404);
|
|
|
|
|
}
|
|
|
|
|
$copyOperations = CloudStorageOperation::query()
|
|
|
|
|
->where('artifact_id', (int) $operation->artifact_id)
|
|
|
|
|
->where('kind', 'copy')->orderBy('id')
|
|
|
|
|
->lockForUpdate()->get()->all();
|
|
|
|
|
$artifact = CloudStorageArtifact::query()
|
|
|
|
|
->whereKey((int) $operation->artifact_id)
|
|
|
|
|
->lockForUpdate()->first();
|
|
|
|
|
$replicas = CloudStorageReplica::query()
|
|
|
|
|
->where('artifact_id', (int) $operation->artifact_id)
|
|
|
|
|
->orderBy('id')->lockForUpdate()->get()->all();
|
|
|
|
|
$publication = CloudStoragePublication::query()
|
|
|
|
|
->where('artifact_id', (int) $operation->artifact_id)
|
|
|
|
|
->lockForUpdate()->first();
|
|
|
|
|
if (!$artifact instanceof CloudStorageArtifact
|
|
|
|
|
|| !$publication instanceof CloudStoragePublication
|
|
|
|
|
|| $publication->backup_id !== null
|
|
|
|
|
|| (($publication->template_import_intent_id === null)
|
|
|
|
|
=== ($publication->rescue_template_id === null))) {
|
|
|
|
|
throw new CloudError('对象存储发布核对归属不一致', 409);
|
|
|
|
|
}
|
|
|
|
|
$primary = null;
|
|
|
|
|
$replicasById = [];
|
|
|
|
|
foreach ($replicas as $replica) {
|
|
|
|
|
$replicasById[(int) $replica->id] = $replica;
|
|
|
|
|
if ((int) $replica->id === (int) $operation->replica_id) {
|
|
|
|
|
$primary = $replica;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
$errorCode = (string) $operation->error_code;
|
|
|
|
|
$unknown = in_array(
|
|
|
|
|
$errorCode,
|
|
|
|
|
self::UNKNOWN_ATTENTION_ERRORS,
|
|
|
|
|
true
|
|
|
|
|
);
|
|
|
|
|
$notApplied = in_array(
|
|
|
|
|
$errorCode,
|
|
|
|
|
self::NOT_APPLIED_ATTENTION_ERRORS,
|
|
|
|
|
true
|
|
|
|
|
);
|
|
|
|
|
if (!$primary instanceof CloudStorageReplica
|
|
|
|
|
|| (string) $primary->role !== 'primary'
|
|
|
|
|
|| (!$unknown && !$notApplied)
|
|
|
|
|
|| (string) $operation->phase !== 'terminal'
|
|
|
|
|
|| (string) $operation->outcome !== 'failed'
|
|
|
|
|
|| (bool) $operation->safe_to_resubmit
|
|
|
|
|
|| (bool) $operation->effect_may_have_applied !== $unknown
|
|
|
|
|
|| (string) $operation->effect_resolution
|
|
|
|
|
!== ($unknown ? 'unknown' : 'not_applied')
|
|
|
|
|
|| (string) $artifact->state !== 'degraded'
|
|
|
|
|
|| (string) $primary->state !== 'degraded'
|
|
|
|
|
|| !hash_equals($errorCode, (string) $primary->error_code)
|
|
|
|
|
|| (string) $publication->state !== 'failed'
|
|
|
|
|
|| !hash_equals(
|
|
|
|
|
$errorCode,
|
|
|
|
|
(string) $publication->last_error_code
|
|
|
|
|
)) {
|
|
|
|
|
throw new CloudError(
|
|
|
|
|
'对象存储主副本不在可重新核对的人工处理终态',
|
|
|
|
|
409
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
foreach ($copyOperations as $copy) {
|
|
|
|
|
$secondary = $replicasById[(int) $copy->replica_id] ?? null;
|
|
|
|
|
if (!$secondary instanceof CloudStorageReplica
|
|
|
|
|
|| (string) $secondary->role !== 'secondary'
|
|
|
|
|
|| (string) $copy->phase !== 'terminal'
|
|
|
|
|
|| (string) $copy->outcome !== 'failed'
|
|
|
|
|
|| (string) $copy->effect_resolution !== 'not_applied'
|
|
|
|
|
|| (bool) $copy->effect_may_have_applied
|
|
|
|
|
|| (bool) $copy->safe_to_resubmit
|
|
|
|
|
|| (int) $copy->attempts !== 0
|
|
|
|
|
|| (string) $copy->error_code
|
|
|
|
|
!== 'primary_publish_attention'
|
|
|
|
|
|| (string) $secondary->state !== 'degraded'
|
|
|
|
|
|| (string) $secondary->error_code
|
|
|
|
|
!== 'primary_publish_attention') {
|
|
|
|
|
throw new CloudError(
|
|
|
|
|
'对象存储次副本核对账本不一致',
|
|
|
|
|
409
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
$metadata = self::resetEvidence($operation->result_json);
|
|
|
|
|
$operation->phase = $unknown ? 'reconciling' : 'accepted';
|
|
|
|
|
$operation->outcome = null;
|
|
|
|
|
$operation->effect_resolution = $unknown
|
|
|
|
|
? 'unknown' : 'not_applicable';
|
|
|
|
|
$operation->effect_may_have_applied = null;
|
|
|
|
|
$operation->safe_to_resubmit = null;
|
|
|
|
|
$operation->lease_token = null;
|
|
|
|
|
$operation->lease_until = 0;
|
|
|
|
|
$operation->available_at = $unknown ? 0 : $now;
|
|
|
|
|
$operation->next_check_at = $unknown ? $now : 0;
|
|
|
|
|
$operation->error_code = $unknown
|
|
|
|
|
? 'admin_reconcile_requested'
|
|
|
|
|
: 'admin_retry_requested';
|
|
|
|
|
$operation->result_json = self::encodedMetadata($metadata);
|
|
|
|
|
$operation->terminal_at = null;
|
|
|
|
|
$operation->updated_at = $now;
|
|
|
|
|
$operation->save();
|
|
|
|
|
|
|
|
|
|
$primary->state = $unknown ? 'publishing' : 'reserved';
|
|
|
|
|
$primary->lease_token = null;
|
|
|
|
|
$primary->lease_until = 0;
|
|
|
|
|
$primary->error_code = $operation->error_code;
|
|
|
|
|
$primary->updated_at = $now;
|
|
|
|
|
$primary->save();
|
|
|
|
|
|
|
|
|
|
foreach ($copyOperations as $copy) {
|
|
|
|
|
/** @var CloudStorageReplica $secondary */
|
|
|
|
|
$secondary = $replicasById[(int) $copy->replica_id];
|
|
|
|
|
$copy->phase = 'accepted';
|
|
|
|
|
$copy->outcome = null;
|
|
|
|
|
$copy->effect_resolution = 'not_applicable';
|
|
|
|
|
$copy->effect_may_have_applied = null;
|
|
|
|
|
$copy->safe_to_resubmit = null;
|
|
|
|
|
$copy->lease_token = null;
|
|
|
|
|
$copy->lease_until = 0;
|
|
|
|
|
$copy->available_at = self::COPY_BLOCKED_UNTIL_PRIMARY;
|
|
|
|
|
$copy->next_check_at = 0;
|
|
|
|
|
$copy->error_code = '';
|
|
|
|
|
$copy->terminal_at = null;
|
|
|
|
|
$copy->updated_at = $now;
|
|
|
|
|
$copy->save();
|
|
|
|
|
$secondary->state = 'reserved';
|
|
|
|
|
$secondary->lease_token = null;
|
|
|
|
|
$secondary->lease_until = 0;
|
|
|
|
|
$secondary->error_code = '';
|
|
|
|
|
$secondary->updated_at = $now;
|
|
|
|
|
$secondary->save();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
$artifact->state = 'publishing';
|
|
|
|
|
$artifact->updated_at = $now;
|
|
|
|
|
$artifact->save();
|
|
|
|
|
$publication->state = 'waiting_primary';
|
|
|
|
|
$publication->attempts = 0;
|
|
|
|
|
$publication->available_at = 0;
|
|
|
|
|
$publication->lease_token = null;
|
|
|
|
|
$publication->lease_until = 0;
|
|
|
|
|
$publication->last_error_code = '';
|
|
|
|
|
$publication->coverage_ready_at = null;
|
|
|
|
|
$publication->finalized_at = null;
|
|
|
|
|
$publication->updated_at = $now;
|
|
|
|
|
$publication->save();
|
|
|
|
|
}, 3);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Advance one immutable primary publication by at most one remote action.
|
|
|
|
|
*
|
|
|
|
|
|