feat(storage): consume owner publication outbox
parent
1cfe6a8866
commit
f5cc5afe6c
@ -0,0 +1,278 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Cloud;
|
||||
|
||||
use App\Models\CloudStorageArtifact;
|
||||
use App\Models\CloudStoragePublication;
|
||||
|
||||
/**
|
||||
* Atomically consumes business finalization after storage coverage is ready.
|
||||
*
|
||||
* Template and rescue finalizers are database-only operations. Keeping the
|
||||
* owner projection and publication completion under one outer transaction
|
||||
* means a process crash can only commit both or neither. Repeated failures are
|
||||
* delayed and eventually become an explicit, manually reopenable attention
|
||||
* state instead of a hot worker loop.
|
||||
*/
|
||||
final class CloudStoragePublicationOutboxService
|
||||
{
|
||||
private const MAX_ATTEMPTS = 6;
|
||||
private const MAX_BACKOFF_SECONDS = 21600;
|
||||
private const ATTENTION_ERROR =
|
||||
'storage_publication_finalization_attention';
|
||||
|
||||
/** @var \Closure(string,int|string,int,int):void */
|
||||
private readonly \Closure $dispatcher;
|
||||
|
||||
/** @var \Closure():int */
|
||||
private readonly \Closure $clock;
|
||||
|
||||
/** @var list<string> */
|
||||
private array $errors = [];
|
||||
|
||||
/**
|
||||
* @param null|callable(string,int|string,int,int):void $dispatcher
|
||||
* @param null|callable():int $clock
|
||||
*/
|
||||
public function __construct(
|
||||
?callable $dispatcher = null,
|
||||
?callable $clock = null
|
||||
) {
|
||||
$this->dispatcher = $dispatcher === null
|
||||
? static function (
|
||||
string $ownerType,
|
||||
int|string $ownerId,
|
||||
int $publicationId,
|
||||
int $artifactId
|
||||
): void {
|
||||
if ($ownerType === 'template') {
|
||||
(new CloudTemplateStoragePublicationConsumer())
|
||||
->finalizeReady((string) $ownerId);
|
||||
return;
|
||||
}
|
||||
if ($ownerType === 'rescue') {
|
||||
CloudRescueStoragePublicationService::finalizeReadyOnly(
|
||||
(string) $ownerId
|
||||
);
|
||||
return;
|
||||
}
|
||||
throw new \LogicException(
|
||||
'Unsupported storage publication owner'
|
||||
);
|
||||
}
|
||||
: \Closure::fromCallable($dispatcher);
|
||||
$this->clock = $clock === null
|
||||
? static fn (): int => time()
|
||||
: \Closure::fromCallable($clock);
|
||||
}
|
||||
|
||||
/** Consume at most one due owner publication. */
|
||||
public function tick(): bool
|
||||
{
|
||||
$this->errors = [];
|
||||
$now = ($this->clock)();
|
||||
if (!is_int($now) || $now < 1) {
|
||||
throw new \LogicException('Storage publication clock is invalid');
|
||||
}
|
||||
$candidate = CloudStoragePublication::query()
|
||||
->where('state', 'ready_to_finalize')
|
||||
->where('available_at', '<=', $now)
|
||||
->whereNull('backup_id')
|
||||
->where(static function ($owner): void {
|
||||
$owner->whereNotNull('template_import_intent_id')
|
||||
->orWhereNotNull('rescue_template_id');
|
||||
})
|
||||
->orderBy('coverage_ready_at')->orderBy('id')->first();
|
||||
if (!$candidate instanceof CloudStoragePublication) {
|
||||
return false;
|
||||
}
|
||||
$publicationId = (int) $candidate->id;
|
||||
$result = (new CloudStoragePublication())->getConnection()
|
||||
->transaction(function ($connection) use (
|
||||
$publicationId,
|
||||
$now
|
||||
): array {
|
||||
$publication = CloudStoragePublication::query()
|
||||
->whereKey($publicationId)
|
||||
->lockForUpdate()->first();
|
||||
if (!$publication instanceof CloudStoragePublication
|
||||
|| (string) $publication->state
|
||||
!== 'ready_to_finalize'
|
||||
|| (int) $publication->available_at > $now) {
|
||||
return ['worked' => false, 'failed' => false];
|
||||
}
|
||||
try {
|
||||
// A nested transaction creates a savepoint. Domain
|
||||
// writes from a failing owner consumer are rolled back
|
||||
// to it, while the outer publication row lock remains
|
||||
// held until the retry delay is durably committed.
|
||||
// A second worker therefore cannot run the same owner
|
||||
// in the failure/update gap.
|
||||
$connection->transaction(function () use (
|
||||
$publication,
|
||||
$publicationId
|
||||
): void {
|
||||
[$ownerType, $ownerId] = self::owner($publication);
|
||||
$artifact = CloudStorageArtifact::query()
|
||||
->whereKey((int) $publication->artifact_id)
|
||||
->lockForUpdate()->first();
|
||||
if (!$artifact instanceof CloudStorageArtifact
|
||||
|| (string) $artifact->kind !== $ownerType
|
||||
|| !in_array((string) $artifact->state,
|
||||
['available', 'protected'], true)) {
|
||||
throw new CloudError(
|
||||
'对象存储发布最终化账本不一致',
|
||||
409
|
||||
);
|
||||
}
|
||||
($this->dispatcher)(
|
||||
$ownerType,
|
||||
$ownerId,
|
||||
(int) $publication->id,
|
||||
(int) $artifact->id
|
||||
);
|
||||
$settled = CloudStoragePublication::query()
|
||||
->whereKey($publicationId)
|
||||
->lockForUpdate()->first();
|
||||
if (!$settled instanceof CloudStoragePublication
|
||||
|| (string) $settled->state !== 'completed'
|
||||
|| (int) ($settled->finalized_at ?? 0) < 1) {
|
||||
throw new CloudError(
|
||||
'对象存储业务最终化没有完成发布账本',
|
||||
409
|
||||
);
|
||||
}
|
||||
}, 1);
|
||||
} catch (\Throwable $error) {
|
||||
if ($error instanceof \PDOException) {
|
||||
// Let the outer transaction retry deadlocks or report
|
||||
// database outages. They are not owner failures and
|
||||
// must not consume the bounded domain-attempt budget.
|
||||
throw $error;
|
||||
}
|
||||
self::recordFailureLocked($publication, $now);
|
||||
return ['worked' => true, 'failed' => true];
|
||||
}
|
||||
return ['worked' => true, 'failed' => false];
|
||||
}, 3);
|
||||
if (($result['failed'] ?? false) === true) {
|
||||
$this->errors[] = 'storage_publication_finalization_failed';
|
||||
}
|
||||
return ($result['worked'] ?? false) === true;
|
||||
}
|
||||
|
||||
/** @return list<string> */
|
||||
public function observationErrors(): array
|
||||
{
|
||||
return $this->errors;
|
||||
}
|
||||
|
||||
/** Re-open only an outbox item stopped by this consumer. */
|
||||
public static function reopenAttention(
|
||||
int $publicationId,
|
||||
?int $now = null
|
||||
): void {
|
||||
if ($publicationId < 1) {
|
||||
throw new CloudError('对象存储发布编号无效', 422);
|
||||
}
|
||||
$now ??= time();
|
||||
(new CloudStoragePublication())->getConnection()->transaction(
|
||||
static function () use ($publicationId, $now): void {
|
||||
$publication = CloudStoragePublication::query()
|
||||
->whereKey($publicationId)->lockForUpdate()->first();
|
||||
if (!$publication instanceof CloudStoragePublication) {
|
||||
throw new CloudError('对象存储发布记录不存在', 404);
|
||||
}
|
||||
if ((string) $publication->state !== 'failed'
|
||||
|| !hash_equals(
|
||||
self::ATTENTION_ERROR,
|
||||
(string) $publication->last_error_code
|
||||
)
|
||||
|| $publication->backup_id !== null) {
|
||||
throw new CloudError(
|
||||
'对象存储发布不在可重开状态',
|
||||
409
|
||||
);
|
||||
}
|
||||
[$ownerType] = self::owner($publication);
|
||||
$artifact = CloudStorageArtifact::query()
|
||||
->whereKey((int) $publication->artifact_id)
|
||||
->lockForUpdate()->first();
|
||||
if (!$artifact instanceof CloudStorageArtifact
|
||||
|| (string) $artifact->kind !== $ownerType
|
||||
|| !in_array((string) $artifact->state,
|
||||
['available', 'protected'], true)) {
|
||||
throw new CloudError(
|
||||
'对象存储发布最终化账本不一致',
|
||||
409
|
||||
);
|
||||
}
|
||||
$publication->state = 'ready_to_finalize';
|
||||
$publication->attempts = 0;
|
||||
$publication->available_at = $now;
|
||||
$publication->lease_token = null;
|
||||
$publication->lease_until = 0;
|
||||
$publication->last_error_code = '';
|
||||
$publication->updated_at = $now;
|
||||
$publication->save();
|
||||
},
|
||||
3
|
||||
);
|
||||
}
|
||||
|
||||
private static function recordFailureLocked(
|
||||
CloudStoragePublication $publication,
|
||||
int $now
|
||||
): void {
|
||||
// The owner savepoint has already rolled back and the caller still
|
||||
// owns this publication row lock. Record the delay in that same outer
|
||||
// transaction so failure and retry eligibility cannot be observed
|
||||
// separately by concurrent workers.
|
||||
$publication->refresh();
|
||||
if ((string) $publication->state !== 'ready_to_finalize'
|
||||
|| $publication->backup_id !== null) {
|
||||
return;
|
||||
}
|
||||
$attempts = (int) $publication->attempts + 1;
|
||||
$publication->attempts = $attempts;
|
||||
$publication->lease_token = null;
|
||||
$publication->lease_until = 0;
|
||||
if ($attempts >= self::MAX_ATTEMPTS) {
|
||||
$publication->state = 'failed';
|
||||
$publication->available_at = 0;
|
||||
$publication->last_error_code = self::ATTENTION_ERROR;
|
||||
} else {
|
||||
$publication->available_at = $now
|
||||
+ self::backoff($attempts);
|
||||
$publication->last_error_code =
|
||||
'storage_publication_finalization_failed';
|
||||
}
|
||||
$publication->updated_at = $now;
|
||||
$publication->save();
|
||||
}
|
||||
|
||||
private static function backoff(int $attempts): int
|
||||
{
|
||||
return min(
|
||||
self::MAX_BACKOFF_SECONDS,
|
||||
60 * (2 ** max(0, min(8, $attempts - 1)))
|
||||
);
|
||||
}
|
||||
|
||||
/** @return array{0:string,1:string} */
|
||||
private static function owner(
|
||||
CloudStoragePublication $publication
|
||||
): array {
|
||||
$templateId = (string) ($publication->template_import_intent_id ?? '');
|
||||
$rescueId = (string) ($publication->rescue_template_id ?? '');
|
||||
if ($publication->backup_id !== null
|
||||
|| ($templateId === '') === ($rescueId === '')) {
|
||||
throw new CloudError('对象存储发布归属不一致', 409);
|
||||
}
|
||||
$ownerId = $templateId !== '' ? $templateId : $rescueId;
|
||||
if (preg_match('/^[a-f0-9]{32}$/D', $ownerId) !== 1) {
|
||||
throw new CloudError('对象存储发布归属无效', 409);
|
||||
}
|
||||
return [$templateId !== '' ? 'template' : 'rescue', $ownerId];
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,517 @@
|
||||
<?php
|
||||
|
||||
require dirname(__DIR__) . '/vendor/autoload.php';
|
||||
|
||||
use App\Models\CloudStorageArtifact;
|
||||
use App\Models\CloudStorageOperation;
|
||||
use App\Models\CloudStoragePublication;
|
||||
use App\Models\CloudStorageReplica;
|
||||
use App\Services\Cloud\CloudError;
|
||||
use App\Services\Cloud\CloudRescueStoragePublicationService;
|
||||
use App\Services\Cloud\CloudStoragePublicationOutboxService;
|
||||
use App\Services\Cloud\CloudStorageReplicaAttentionService;
|
||||
use App\Services\Cloud\CloudStorageReplicaWorkerService;
|
||||
use App\Services\Cloud\CloudTemplateStoragePublicationConsumer;
|
||||
use Illuminate\Database\Capsule\Manager as Capsule;
|
||||
use Illuminate\Database\Schema\Blueprint;
|
||||
|
||||
$assert = static function (bool $condition, string $message): void {
|
||||
if (!$condition) {
|
||||
throw new RuntimeException($message);
|
||||
}
|
||||
};
|
||||
$assert(method_exists(
|
||||
CloudTemplateStoragePublicationConsumer::class,
|
||||
'finalizeReady'
|
||||
) && method_exists(
|
||||
CloudRescueStoragePublicationService::class,
|
||||
'finalizeReadyOnly'
|
||||
), 'Publication outbox lost a local-only business finalization consumer.');
|
||||
|
||||
$db = new Capsule();
|
||||
$db->addConnection(['driver' => 'sqlite', 'database' => ':memory:']);
|
||||
$db->setAsGlobal();
|
||||
$db->bootEloquent();
|
||||
$db->schema()->create('cloud_storage_artifacts', static function (
|
||||
Blueprint $table
|
||||
): void {
|
||||
$table->increments('id');
|
||||
$table->string('artifact_key')->unique();
|
||||
$table->string('kind');
|
||||
$table->unsignedBigInteger('maximum_size_bytes');
|
||||
$table->unsignedBigInteger('size_bytes');
|
||||
$table->string('sha256');
|
||||
$table->string('durability_policy');
|
||||
$table->unsignedTinyInteger('desired_copies');
|
||||
$table->unsignedTinyInteger('required_copies');
|
||||
$table->string('state');
|
||||
$table->string('last_operation_id')->nullable();
|
||||
$table->unsignedBigInteger('verified_at');
|
||||
$table->unsignedBigInteger('created_at');
|
||||
$table->unsignedBigInteger('updated_at');
|
||||
});
|
||||
$db->schema()->create('cloud_storage_publications', static function (
|
||||
Blueprint $table
|
||||
): void {
|
||||
$table->increments('id');
|
||||
$table->string('publication_key')->unique();
|
||||
$table->unsignedBigInteger('artifact_id')->unique();
|
||||
$table->unsignedBigInteger('backup_id')->nullable()->unique();
|
||||
$table->string('template_import_intent_id')->nullable()->unique();
|
||||
$table->string('rescue_template_id')->nullable()->unique();
|
||||
$table->string('state');
|
||||
$table->unsignedInteger('attempts')->default(0);
|
||||
$table->unsignedBigInteger('available_at')->default(0);
|
||||
$table->string('lease_token')->nullable();
|
||||
$table->unsignedBigInteger('lease_until')->default(0);
|
||||
$table->string('last_error_code')->default('');
|
||||
$table->unsignedBigInteger('coverage_ready_at')->nullable();
|
||||
$table->unsignedBigInteger('finalized_at')->nullable();
|
||||
$table->unsignedBigInteger('created_at');
|
||||
$table->unsignedBigInteger('updated_at');
|
||||
});
|
||||
$db->schema()->create('cloud_storage_operations', static function (
|
||||
Blueprint $table
|
||||
): void {
|
||||
$table->increments('id');
|
||||
$table->string('operation_id')->unique();
|
||||
$table->string('request_idempotency_key')->unique();
|
||||
$table->string('request_fingerprint');
|
||||
$table->unsignedBigInteger('artifact_id');
|
||||
$table->unsignedBigInteger('replica_id')->nullable();
|
||||
$table->unsignedBigInteger('source_backend_id')->nullable();
|
||||
$table->unsignedBigInteger('target_backend_id')->nullable();
|
||||
$table->string('kind');
|
||||
$table->string('phase');
|
||||
$table->string('outcome')->nullable();
|
||||
$table->string('effect_resolution');
|
||||
$table->boolean('effect_may_have_applied')->nullable();
|
||||
$table->boolean('safe_to_resubmit')->nullable();
|
||||
$table->unsignedInteger('attempts')->default(0);
|
||||
$table->string('lease_token')->nullable();
|
||||
$table->unsignedBigInteger('lease_until')->default(0);
|
||||
$table->unsignedBigInteger('available_at')->default(0);
|
||||
$table->unsignedBigInteger('next_check_at')->default(0);
|
||||
$table->string('error_code')->default('');
|
||||
$table->text('result_json')->nullable();
|
||||
$table->unsignedBigInteger('terminal_at')->nullable();
|
||||
$table->unsignedBigInteger('created_at');
|
||||
$table->unsignedBigInteger('updated_at');
|
||||
});
|
||||
$db->schema()->create('cloud_storage_replicas', static function (
|
||||
Blueprint $table
|
||||
): void {
|
||||
$table->increments('id');
|
||||
$table->string('replica_key')->unique();
|
||||
$table->unsignedBigInteger('artifact_id');
|
||||
$table->unsignedBigInteger('backend_id');
|
||||
$table->string('role');
|
||||
$table->string('object_key');
|
||||
$table->string('object_version_id')->default('');
|
||||
$table->string('object_etag')->default('');
|
||||
$table->string('state');
|
||||
$table->string('publish_operation_id')->nullable();
|
||||
$table->string('delete_operation_id')->nullable();
|
||||
$table->unsignedInteger('delete_generation')->default(0);
|
||||
$table->unsignedInteger('delete_cycle_attempts')->default(0);
|
||||
$table->unsignedInteger('delete_presence_confirmations')->default(0);
|
||||
$table->unsignedBigInteger('delete_last_checked_at')->default(0);
|
||||
$table->unsignedBigInteger('delete_next_check_at')->default(0);
|
||||
$table->unsignedBigInteger('delete_attention_at')->nullable();
|
||||
$table->string('lease_token')->nullable();
|
||||
$table->unsignedBigInteger('lease_until')->default(0);
|
||||
$table->unsignedBigInteger('size_bytes')->default(0);
|
||||
$table->string('sha256')->default('');
|
||||
$table->string('error_code')->default('');
|
||||
$table->unsignedBigInteger('verified_at')->default(0);
|
||||
$table->unsignedBigInteger('created_at');
|
||||
$table->unsignedBigInteger('updated_at');
|
||||
});
|
||||
$db->schema()->create('cloud_storage_backends', static function (
|
||||
Blueprint $table
|
||||
): void {
|
||||
$table->increments('id');
|
||||
$table->string('name');
|
||||
});
|
||||
|
||||
$clock = 2000000000;
|
||||
$ownerA = str_repeat('a', 32);
|
||||
$ownerB = str_repeat('b', 32);
|
||||
$create = static function (
|
||||
string $kind,
|
||||
string $owner,
|
||||
string $state = 'ready_to_finalize',
|
||||
int $attempts = 0,
|
||||
int $availableAt = 0
|
||||
) use (&$clock): CloudStoragePublication {
|
||||
$artifact = new CloudStorageArtifact();
|
||||
$artifact->artifact_key = bin2hex(random_bytes(16));
|
||||
$artifact->kind = $kind;
|
||||
$artifact->maximum_size_bytes = 4096;
|
||||
$artifact->size_bytes = 4096;
|
||||
$artifact->sha256 = str_repeat($kind === 'template' ? 'a' : 'b', 64);
|
||||
$artifact->durability_policy = 'mirror_required_2';
|
||||
$artifact->desired_copies = 2;
|
||||
$artifact->required_copies = 2;
|
||||
$artifact->state = 'protected';
|
||||
$artifact->last_operation_id = null;
|
||||
$artifact->verified_at = $clock - 10;
|
||||
$artifact->created_at = $artifact->updated_at = $clock - 20;
|
||||
$artifact->save();
|
||||
|
||||
$publication = new CloudStoragePublication();
|
||||
$publication->publication_key = bin2hex(random_bytes(16));
|
||||
$publication->artifact_id = (int) $artifact->id;
|
||||
$publication->backup_id = null;
|
||||
$publication->template_import_intent_id =
|
||||
$kind === 'template' ? $owner : null;
|
||||
$publication->rescue_template_id =
|
||||
$kind === 'rescue' ? $owner : null;
|
||||
$publication->state = $state;
|
||||
$publication->attempts = $attempts;
|
||||
$publication->available_at = $availableAt;
|
||||
$publication->lease_token = null;
|
||||
$publication->lease_until = 0;
|
||||
$publication->last_error_code = '';
|
||||
$publication->coverage_ready_at = $clock - 5;
|
||||
$publication->finalized_at = null;
|
||||
$publication->created_at = $publication->updated_at = $clock - 20;
|
||||
$publication->save();
|
||||
return $publication;
|
||||
};
|
||||
|
||||
$template = $create('template', $ownerA);
|
||||
$calls = [];
|
||||
$worker = new CloudStoragePublicationOutboxService(
|
||||
static function (
|
||||
string $type,
|
||||
int|string $owner,
|
||||
int $publicationId,
|
||||
int $artifactId
|
||||
) use (&$calls, &$clock, $assert): void {
|
||||
$assert(Capsule::connection()->transactionLevel() > 0,
|
||||
'Business finalization escaped the owner transaction.');
|
||||
$calls[] = [$type, $owner, $publicationId, $artifactId];
|
||||
$row = CloudStoragePublication::query()
|
||||
->whereKey($publicationId)->lockForUpdate()->firstOrFail();
|
||||
$row->state = 'completed';
|
||||
$row->attempts = (int) $row->attempts + 1;
|
||||
$row->finalized_at = $clock;
|
||||
$row->updated_at = $clock;
|
||||
$row->save();
|
||||
},
|
||||
static function () use (&$clock): int {
|
||||
return $clock;
|
||||
}
|
||||
);
|
||||
$assert($worker->tick(), 'A due template publication was not consumed.');
|
||||
$template->refresh();
|
||||
$assert((string) $template->state === 'completed'
|
||||
&& (int) $template->finalized_at === $clock
|
||||
&& $calls === [[
|
||||
'template', $ownerA, (int) $template->id,
|
||||
(int) $template->artifact_id,
|
||||
]], 'Template outbox finalization lost its frozen owner identity.');
|
||||
$assert(!$worker->tick(), 'A completed publication was consumed twice.');
|
||||
|
||||
$rescue = $create('rescue', $ownerB, 'ready_to_finalize', 0, $clock + 60);
|
||||
$assert(!$worker->tick(), 'A future publication bypassed its retry delay.');
|
||||
$clock += 60;
|
||||
$assert($worker->tick(), 'A due rescue publication was not consumed.');
|
||||
$rescue->refresh();
|
||||
$assert((string) $rescue->state === 'completed'
|
||||
&& $calls[1][0] === 'rescue' && $calls[1][1] === $ownerB,
|
||||
'Rescue outbox dispatch selected the wrong owner.');
|
||||
|
||||
$failing = $create('template', str_repeat('c', 32));
|
||||
$failingWorker = new CloudStoragePublicationOutboxService(
|
||||
static function (
|
||||
string $type,
|
||||
int|string $owner,
|
||||
int $publicationId
|
||||
) use (&$clock): void {
|
||||
// Prove that a consumer which mutates the business publication and
|
||||
// then crashes is rolled back to the savepoint before the durable
|
||||
// retry delay is committed.
|
||||
$partial = CloudStoragePublication::query()
|
||||
->whereKey($publicationId)->lockForUpdate()->firstOrFail();
|
||||
$partial->state = 'completed';
|
||||
$partial->finalized_at = $clock;
|
||||
$partial->save();
|
||||
throw new CloudError('fixture owner failure', 409);
|
||||
},
|
||||
static function () use (&$clock): int {
|
||||
return $clock;
|
||||
}
|
||||
);
|
||||
$assert($failingWorker->tick(), 'A failed outbox attempt was not observed.');
|
||||
$failing->refresh();
|
||||
$assert((string) $failing->state === 'ready_to_finalize'
|
||||
&& (int) $failing->attempts === 1
|
||||
&& $failing->finalized_at === null
|
||||
&& (int) $failing->available_at === $clock + 60
|
||||
&& (string) $failing->last_error_code
|
||||
=== 'storage_publication_finalization_failed'
|
||||
&& $failingWorker->observationErrors()
|
||||
=== ['storage_publication_finalization_failed'],
|
||||
'A failed owner finalization was not durably delayed.');
|
||||
$assert(!$failingWorker->tick(), 'A delayed failure entered a hot loop.');
|
||||
|
||||
$attention = $create(
|
||||
'rescue',
|
||||
str_repeat('d', 32),
|
||||
'ready_to_finalize',
|
||||
5,
|
||||
$clock
|
||||
);
|
||||
$assert($failingWorker->tick(), 'The bounded final attempt was not handled.');
|
||||
$attention->refresh();
|
||||
$assert((string) $attention->state === 'failed'
|
||||
&& (int) $attention->attempts === 6
|
||||
&& (string) $attention->last_error_code
|
||||
=== 'storage_publication_finalization_attention',
|
||||
'Exhausted owner finalization did not become explicit attention.');
|
||||
$attentionView = CloudStorageReplicaAttentionService::listView();
|
||||
$assert(($attentionView['summary']['finalization_count'] ?? 0) === 1
|
||||
&& count($attentionView['finalization_items'] ?? []) === 1
|
||||
&& ($attentionView['finalization_items'][0]['operation_kind'] ?? '')
|
||||
=== 'finalize'
|
||||
&& ($attentionView['finalization_items'][0]['operation_id'] ?? '')
|
||||
=== (string) $attention->publication_key
|
||||
&& ($attentionView['finalization_items'][0]['can_retry'] ?? false),
|
||||
'Finalization attention is not visible on the administrator recovery page.');
|
||||
$retry = CloudStorageReplicaAttentionService::retry(
|
||||
(string) $attention->publication_key,
|
||||
'finalize'
|
||||
);
|
||||
$attention->refresh();
|
||||
$assert((string) $attention->state === 'ready_to_finalize'
|
||||
&& (int) $attention->attempts === 0
|
||||
&& (int) $attention->available_at > 0
|
||||
&& (string) $attention->last_error_code === ''
|
||||
&& ($retry['operation_kind'] ?? '') === 'finalize',
|
||||
'Administrator recovery did not reopen the same publication intent.');
|
||||
$attention->state = 'completed';
|
||||
$attention->finalized_at = $clock;
|
||||
$attention->save();
|
||||
|
||||
// Finalization attention has an independent stable paginator. Prove adjacent
|
||||
// pages are disjoint and together expose every row instead of silently
|
||||
// truncating after the first page.
|
||||
$pagedAttention = [];
|
||||
foreach (['1', '2', '3'] as $ownerDigit) {
|
||||
$row = $create('template', str_repeat($ownerDigit, 32));
|
||||
$row->state = 'failed';
|
||||
$row->attempts = 6;
|
||||
$row->available_at = 0;
|
||||
$row->last_error_code =
|
||||
'storage_publication_finalization_attention';
|
||||
$row->updated_at = $clock + (int) $ownerDigit;
|
||||
$row->save();
|
||||
$pagedAttention[] = $row;
|
||||
}
|
||||
$finalizationPageOne = CloudStorageReplicaAttentionService::listView([
|
||||
'page' => 1,
|
||||
'finalization_page' => 1,
|
||||
'per_page' => 2,
|
||||
]);
|
||||
$finalizationPageTwo = CloudStorageReplicaAttentionService::listView([
|
||||
'page' => 1,
|
||||
'finalization_page' => 2,
|
||||
'per_page' => 2,
|
||||
]);
|
||||
$pageOneIds = array_column(
|
||||
$finalizationPageOne['finalization_items'],
|
||||
'operation_id'
|
||||
);
|
||||
$pageTwoIds = array_column(
|
||||
$finalizationPageTwo['finalization_items'],
|
||||
'operation_id'
|
||||
);
|
||||
$assert(count($pageOneIds) === 2 && count($pageTwoIds) === 1
|
||||
&& array_intersect($pageOneIds, $pageTwoIds) === []
|
||||
&& count(array_unique(array_merge($pageOneIds, $pageTwoIds))) === 3
|
||||
&& ($finalizationPageOne['finalization_pagination']['total'] ?? 0) === 3
|
||||
&& ($finalizationPageOne['finalization_pagination']['total_pages'] ?? 0)
|
||||
=== 2,
|
||||
'Finalization attention pagination repeated or omitted an item.');
|
||||
foreach ($pagedAttention as $row) {
|
||||
$row->state = 'completed';
|
||||
$row->finalized_at = $clock;
|
||||
$row->save();
|
||||
}
|
||||
|
||||
$databaseFault = $create('rescue', str_repeat('4', 32));
|
||||
$databaseFaultWorker = new CloudStoragePublicationOutboxService(
|
||||
static function (): void {
|
||||
throw new PDOException('fixture database outage');
|
||||
},
|
||||
static function () use (&$clock): int {
|
||||
return $clock;
|
||||
}
|
||||
);
|
||||
$databaseFaultRaised = false;
|
||||
try {
|
||||
$databaseFaultWorker->tick();
|
||||
} catch (PDOException) {
|
||||
$databaseFaultRaised = true;
|
||||
}
|
||||
$databaseFault->refresh();
|
||||
$assert($databaseFaultRaised
|
||||
&& (string) $databaseFault->state === 'ready_to_finalize'
|
||||
&& (int) $databaseFault->attempts === 0,
|
||||
'Database failure consumed the bounded owner-finalization budget.');
|
||||
$databaseFault->state = 'completed';
|
||||
$databaseFault->finalized_at = $clock;
|
||||
$databaseFault->save();
|
||||
|
||||
$wrong = $create('template', str_repeat('e', 32));
|
||||
$noSettlement = new CloudStoragePublicationOutboxService(
|
||||
static function (): void {
|
||||
// A domain consumer returning without completing the outbox is an
|
||||
// error; it must roll back and enter the same bounded retry path.
|
||||
},
|
||||
static function () use (&$clock): int {
|
||||
return $clock;
|
||||
}
|
||||
);
|
||||
$assert($noSettlement->tick(), 'Missing business settlement was not handled.');
|
||||
$wrong->refresh();
|
||||
$assert((string) $wrong->state === 'ready_to_finalize'
|
||||
&& (int) $wrong->attempts === 1,
|
||||
'A no-op business consumer was treated as successful.');
|
||||
$wrong->state = 'completed';
|
||||
$wrong->finalized_at = $clock;
|
||||
$wrong->save();
|
||||
|
||||
$workerOwned = $create('rescue', str_repeat('f', 32));
|
||||
$copyArtifact = new CloudStorageArtifact();
|
||||
$copyArtifact->artifact_key = bin2hex(random_bytes(16));
|
||||
$copyArtifact->kind = 'backup';
|
||||
$copyArtifact->maximum_size_bytes = 4096;
|
||||
$copyArtifact->size_bytes = 4096;
|
||||
$copyArtifact->sha256 = str_repeat('9', 64);
|
||||
$copyArtifact->durability_policy = 'mirror_required_2';
|
||||
$copyArtifact->desired_copies = 2;
|
||||
$copyArtifact->required_copies = 2;
|
||||
$copyArtifact->state = 'available';
|
||||
$copyArtifact->last_operation_id = null;
|
||||
$copyArtifact->verified_at = $clock;
|
||||
$copyArtifact->created_at = $copyArtifact->updated_at = $clock;
|
||||
$copyArtifact->save();
|
||||
|
||||
$sourceReplica = new CloudStorageReplica();
|
||||
$sourceReplica->replica_key = bin2hex(random_bytes(16));
|
||||
$sourceReplica->artifact_id = (int) $copyArtifact->id;
|
||||
$sourceReplica->backend_id = 1;
|
||||
$sourceReplica->role = 'primary';
|
||||
$sourceReplica->object_key = 'fixture/source.qcow2';
|
||||
$sourceReplica->object_version_id = '';
|
||||
$sourceReplica->object_etag = '"source"';
|
||||
$sourceReplica->size_bytes = 4096;
|
||||
$sourceReplica->sha256 = str_repeat('9', 64);
|
||||
$sourceReplica->state = 'ready';
|
||||
$sourceReplica->publish_operation_id = null;
|
||||
$sourceReplica->delete_operation_id = null;
|
||||
$sourceReplica->lease_token = null;
|
||||
$sourceReplica->lease_until = 0;
|
||||
$sourceReplica->error_code = '';
|
||||
$sourceReplica->verified_at = $clock;
|
||||
$sourceReplica->created_at = $sourceReplica->updated_at = $clock;
|
||||
$sourceReplica->save();
|
||||
|
||||
$targetReplica = new CloudStorageReplica();
|
||||
$targetReplica->replica_key = bin2hex(random_bytes(16));
|
||||
$targetReplica->artifact_id = (int) $copyArtifact->id;
|
||||
$targetReplica->backend_id = 2;
|
||||
$targetReplica->role = 'secondary';
|
||||
$targetReplica->object_key = 'fixture/target.qcow2';
|
||||
$targetReplica->object_version_id = '';
|
||||
$targetReplica->object_etag = '';
|
||||
$targetReplica->size_bytes = 0;
|
||||
$targetReplica->sha256 = '';
|
||||
$targetReplica->state = 'reserved';
|
||||
$targetReplica->publish_operation_id = null;
|
||||
$targetReplica->delete_operation_id = null;
|
||||
$targetReplica->lease_token = null;
|
||||
$targetReplica->lease_until = 0;
|
||||
$targetReplica->error_code = '';
|
||||
$targetReplica->verified_at = 0;
|
||||
$targetReplica->created_at = $targetReplica->updated_at = $clock;
|
||||
$targetReplica->save();
|
||||
|
||||
$copyOperation = new CloudStorageOperation();
|
||||
$copyOperation->operation_id = bin2hex(random_bytes(16));
|
||||
$copyOperation->request_idempotency_key =
|
||||
'publication-worker-copy-fixture';
|
||||
$copyOperation->request_fingerprint = hash('sha256', 'copy-fixture');
|
||||
$copyOperation->artifact_id = (int) $copyArtifact->id;
|
||||
$copyOperation->replica_id = (int) $targetReplica->id;
|
||||
$copyOperation->source_backend_id = 1;
|
||||
$copyOperation->target_backend_id = 2;
|
||||
$copyOperation->kind = 'copy';
|
||||
$copyOperation->phase = 'accepted';
|
||||
$copyOperation->outcome = null;
|
||||
$copyOperation->effect_resolution = 'not_applicable';
|
||||
$copyOperation->effect_may_have_applied = null;
|
||||
$copyOperation->safe_to_resubmit = null;
|
||||
$copyOperation->attempts = 0;
|
||||
$copyOperation->lease_token = null;
|
||||
$copyOperation->lease_until = 0;
|
||||
$copyOperation->available_at = $clock;
|
||||
$copyOperation->next_check_at = 0;
|
||||
$copyOperation->error_code = '';
|
||||
$copyOperation->result_json = null;
|
||||
$copyOperation->terminal_at = null;
|
||||
$copyOperation->created_at = $copyOperation->updated_at = $clock;
|
||||
$copyOperation->save();
|
||||
$integratedOutbox = new CloudStoragePublicationOutboxService(
|
||||
static function (
|
||||
string $type,
|
||||
int|string $owner,
|
||||
int $publicationId,
|
||||
int $artifactId
|
||||
) use (&$clock, $assert, $sourceReplica): void {
|
||||
$assert($type === 'rescue' && $owner === str_repeat('f', 32),
|
||||
'Replica worker dispatched the wrong publication owner.');
|
||||
$publication = CloudStoragePublication::query()
|
||||
->whereKey($publicationId)->lockForUpdate()->firstOrFail();
|
||||
$publication->state = 'completed';
|
||||
$publication->finalized_at = $clock;
|
||||
$publication->save();
|
||||
// Makes the replica source invalid only after publication work ran;
|
||||
// the following replica_source_not_ready assertion therefore proves
|
||||
// the short outbox step was consumed before the replica claim.
|
||||
$sourceReplica->state = 'degraded';
|
||||
$sourceReplica->error_code = 'fixture_not_ready';
|
||||
$sourceReplica->verified_at = 0;
|
||||
$sourceReplica->save();
|
||||
},
|
||||
static function () use (&$clock): int {
|
||||
return $clock;
|
||||
}
|
||||
);
|
||||
$replicaWorker = new CloudStorageReplicaWorkerService(
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
static function () use (&$clock): int {
|
||||
return $clock;
|
||||
},
|
||||
$integratedOutbox
|
||||
);
|
||||
$assert($replicaWorker->tick()
|
||||
&& $replicaWorker->observationErrors()
|
||||
=== ['replica_source_not_ready'],
|
||||
'Replica worker did not process both bounded work queues in one tick.');
|
||||
$workerOwned->refresh();
|
||||
$copyOperation->refresh();
|
||||
$assert((string) $workerOwned->state === 'completed',
|
||||
'Replica worker did not settle the owner publication.');
|
||||
$assert((string) $copyOperation->phase === 'accepted'
|
||||
&& (int) $copyOperation->attempts === 1
|
||||
&& (int) $copyOperation->available_at === $clock + 60
|
||||
&& (string) $copyOperation->error_code === 'replica_source_not_ready',
|
||||
'Publication finalization starved the replica queue in the same tick.');
|
||||
|
||||
echo "Storage publication outbox behavior passed.\n";
|
||||
Loading…
Reference in New Issue