You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/app/Services/Cloud/CloudStoragePublicationOutb...

279 lines
11 KiB
PHP

<?php
namespace App\Services\Cloud;
use App\Models\CloudStorageArtifact;
use App\Models\CloudStoragePublication;
/**
* Atomically consumes business finalization after storage coverage is ready.
*
* Template and rescue finalizers are database-only operations. Keeping the
* owner projection and publication completion under one outer transaction
* means a process crash can only commit both or neither. Repeated failures are
* delayed and eventually become an explicit, manually reopenable attention
* state instead of a hot worker loop.
*/
final class CloudStoragePublicationOutboxService
{
private const MAX_ATTEMPTS = 6;
private const MAX_BACKOFF_SECONDS = 21600;
private const ATTENTION_ERROR =
'storage_publication_finalization_attention';
/** @var \Closure(string,int|string,int,int):void */
private readonly \Closure $dispatcher;
/** @var \Closure():int */
private readonly \Closure $clock;
/** @var list<string> */
private array $errors = [];
/**
* @param null|callable(string,int|string,int,int):void $dispatcher
* @param null|callable():int $clock
*/
public function __construct(
?callable $dispatcher = null,
?callable $clock = null
) {
$this->dispatcher = $dispatcher === null
? static function (
string $ownerType,
int|string $ownerId,
int $publicationId,
int $artifactId
): void {
if ($ownerType === 'template') {
(new CloudTemplateStoragePublicationConsumer())
->finalizeReady((string) $ownerId);
return;
}
if ($ownerType === 'rescue') {
CloudRescueStoragePublicationService::finalizeReadyOnly(
(string) $ownerId
);
return;
}
throw new \LogicException(
'Unsupported storage publication owner'
);
}
: \Closure::fromCallable($dispatcher);
$this->clock = $clock === null
? static fn (): int => time()
: \Closure::fromCallable($clock);
}
/** Consume at most one due owner publication. */
public function tick(): bool
{
$this->errors = [];
$now = ($this->clock)();
if (!is_int($now) || $now < 1) {
throw new \LogicException('Storage publication clock is invalid');
}
$candidate = CloudStoragePublication::query()
->where('state', 'ready_to_finalize')
->where('available_at', '<=', $now)
->whereNull('backup_id')
->where(static function ($owner): void {
$owner->whereNotNull('template_import_intent_id')
->orWhereNotNull('rescue_template_id');
})
->orderBy('coverage_ready_at')->orderBy('id')->first();
if (!$candidate instanceof CloudStoragePublication) {
return false;
}
$publicationId = (int) $candidate->id;
$result = (new CloudStoragePublication())->getConnection()
->transaction(function ($connection) use (
$publicationId,
$now
): array {
$publication = CloudStoragePublication::query()
->whereKey($publicationId)
->lockForUpdate()->first();
if (!$publication instanceof CloudStoragePublication
|| (string) $publication->state
!== 'ready_to_finalize'
|| (int) $publication->available_at > $now) {
return ['worked' => false, 'failed' => false];
}
try {
// A nested transaction creates a savepoint. Domain
// writes from a failing owner consumer are rolled back
// to it, while the outer publication row lock remains
// held until the retry delay is durably committed.
// A second worker therefore cannot run the same owner
// in the failure/update gap.
$connection->transaction(function () use (
$publication,
$publicationId
): void {
[$ownerType, $ownerId] = self::owner($publication);
$artifact = CloudStorageArtifact::query()
->whereKey((int) $publication->artifact_id)
->lockForUpdate()->first();
if (!$artifact instanceof CloudStorageArtifact
|| (string) $artifact->kind !== $ownerType
|| !in_array((string) $artifact->state,
['available', 'protected'], true)) {
throw new CloudError(
'对象存储发布最终化账本不一致',
409
);
}
($this->dispatcher)(
$ownerType,
$ownerId,
(int) $publication->id,
(int) $artifact->id
);
$settled = CloudStoragePublication::query()
->whereKey($publicationId)
->lockForUpdate()->first();
if (!$settled instanceof CloudStoragePublication
|| (string) $settled->state !== 'completed'
|| (int) ($settled->finalized_at ?? 0) < 1) {
throw new CloudError(
'对象存储业务最终化没有完成发布账本',
409
);
}
}, 1);
} catch (\Throwable $error) {
if ($error instanceof \PDOException) {
// Let the outer transaction retry deadlocks or report
// database outages. They are not owner failures and
// must not consume the bounded domain-attempt budget.
throw $error;
}
self::recordFailureLocked($publication, $now);
return ['worked' => true, 'failed' => true];
}
return ['worked' => true, 'failed' => false];
}, 3);
if (($result['failed'] ?? false) === true) {
$this->errors[] = 'storage_publication_finalization_failed';
}
return ($result['worked'] ?? false) === true;
}
/** @return list<string> */
public function observationErrors(): array
{
return $this->errors;
}
/** Re-open only an outbox item stopped by this consumer. */
public static function reopenAttention(
int $publicationId,
?int $now = null
): void {
if ($publicationId < 1) {
throw new CloudError('对象存储发布编号无效', 422);
}
$now ??= time();
(new CloudStoragePublication())->getConnection()->transaction(
static function () use ($publicationId, $now): void {
$publication = CloudStoragePublication::query()
->whereKey($publicationId)->lockForUpdate()->first();
if (!$publication instanceof CloudStoragePublication) {
throw new CloudError('对象存储发布记录不存在', 404);
}
if ((string) $publication->state !== 'failed'
|| !hash_equals(
self::ATTENTION_ERROR,
(string) $publication->last_error_code
)
|| $publication->backup_id !== null) {
throw new CloudError(
'对象存储发布不在可重开状态',
409
);
}
[$ownerType] = self::owner($publication);
$artifact = CloudStorageArtifact::query()
->whereKey((int) $publication->artifact_id)
->lockForUpdate()->first();
if (!$artifact instanceof CloudStorageArtifact
|| (string) $artifact->kind !== $ownerType
|| !in_array((string) $artifact->state,
['available', 'protected'], true)) {
throw new CloudError(
'对象存储发布最终化账本不一致',
409
);
}
$publication->state = 'ready_to_finalize';
$publication->attempts = 0;
$publication->available_at = $now;
$publication->lease_token = null;
$publication->lease_until = 0;
$publication->last_error_code = '';
$publication->updated_at = $now;
$publication->save();
},
3
);
}
private static function recordFailureLocked(
CloudStoragePublication $publication,
int $now
): void {
// The owner savepoint has already rolled back and the caller still
// owns this publication row lock. Record the delay in that same outer
// transaction so failure and retry eligibility cannot be observed
// separately by concurrent workers.
$publication->refresh();
if ((string) $publication->state !== 'ready_to_finalize'
|| $publication->backup_id !== null) {
return;
}
$attempts = (int) $publication->attempts + 1;
$publication->attempts = $attempts;
$publication->lease_token = null;
$publication->lease_until = 0;
if ($attempts >= self::MAX_ATTEMPTS) {
$publication->state = 'failed';
$publication->available_at = 0;
$publication->last_error_code = self::ATTENTION_ERROR;
} else {
$publication->available_at = $now
+ self::backoff($attempts);
$publication->last_error_code =
'storage_publication_finalization_failed';
}
$publication->updated_at = $now;
$publication->save();
}
private static function backoff(int $attempts): int
{
return min(
self::MAX_BACKOFF_SECONDS,
60 * (2 ** max(0, min(8, $attempts - 1)))
);
}
/** @return array{0:string,1:string} */
private static function owner(
CloudStoragePublication $publication
): array {
$templateId = (string) ($publication->template_import_intent_id ?? '');
$rescueId = (string) ($publication->rescue_template_id ?? '');
if ($publication->backup_id !== null
|| ($templateId === '') === ($rescueId === '')) {
throw new CloudError('对象存储发布归属不一致', 409);
}
$ownerId = $templateId !== '' ? $templateId : $rescueId;
if (preg_match('/^[a-f0-9]{32}$/D', $ownerId) !== 1) {
throw new CloudError('对象存储发布归属无效', 409);
}
return [$templateId !== '' ? 'template' : 'rescue', $ownerId];
}
}