You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
551 lines
21 KiB
PHP
551 lines
21 KiB
PHP
<?php
|
|
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
|
|
|
use App\Models\CloudStorageArtifact;
|
|
use App\Models\CloudStorageOperation;
|
|
use App\Models\CloudStoragePublication;
|
|
use App\Models\CloudStorageReplica;
|
|
use App\Services\Cloud\CloudError;
|
|
use App\Services\Cloud\CloudRescueStoragePublicationService;
|
|
use App\Services\Cloud\CloudStoragePublicationOutboxService;
|
|
use App\Services\Cloud\CloudStorageReplicaAttentionService;
|
|
use App\Services\Cloud\CloudStorageReplicaWorkerService;
|
|
use App\Services\Cloud\CloudTemplateStoragePublicationConsumer;
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
use Illuminate\Database\QueryException;
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
|
|
$assert = static function (bool $condition, string $message): void {
|
|
if (!$condition) {
|
|
throw new RuntimeException($message);
|
|
}
|
|
};
|
|
$assert(method_exists(
|
|
CloudTemplateStoragePublicationConsumer::class,
|
|
'finalizeReady'
|
|
) && method_exists(
|
|
CloudRescueStoragePublicationService::class,
|
|
'finalizeReadyOnly'
|
|
), 'Publication outbox lost a local-only business finalization consumer.');
|
|
|
|
$db = new Capsule();
|
|
$db->addConnection(['driver' => 'sqlite', 'database' => ':memory:']);
|
|
$db->setAsGlobal();
|
|
$db->bootEloquent();
|
|
$db->schema()->create('cloud_storage_artifacts', static function (
|
|
Blueprint $table
|
|
): void {
|
|
$table->increments('id');
|
|
$table->string('artifact_key')->unique();
|
|
$table->string('kind');
|
|
$table->unsignedBigInteger('maximum_size_bytes');
|
|
$table->unsignedBigInteger('size_bytes');
|
|
$table->string('sha256');
|
|
$table->string('durability_policy');
|
|
$table->unsignedTinyInteger('desired_copies');
|
|
$table->unsignedTinyInteger('required_copies');
|
|
$table->string('state');
|
|
$table->string('last_operation_id')->nullable();
|
|
$table->unsignedBigInteger('verified_at');
|
|
$table->unsignedBigInteger('created_at');
|
|
$table->unsignedBigInteger('updated_at');
|
|
});
|
|
$db->schema()->create('cloud_storage_publications', static function (
|
|
Blueprint $table
|
|
): void {
|
|
$table->increments('id');
|
|
$table->string('publication_key')->unique();
|
|
$table->unsignedBigInteger('artifact_id')->unique();
|
|
$table->unsignedBigInteger('backup_id')->nullable()->unique();
|
|
$table->string('template_import_intent_id')->nullable()->unique();
|
|
$table->string('rescue_template_id')->nullable()->unique();
|
|
$table->string('state');
|
|
$table->unsignedInteger('attempts')->default(0);
|
|
$table->unsignedBigInteger('available_at')->default(0);
|
|
$table->string('lease_token')->nullable();
|
|
$table->unsignedBigInteger('lease_until')->default(0);
|
|
$table->string('last_error_code')->default('');
|
|
$table->unsignedBigInteger('coverage_ready_at')->nullable();
|
|
$table->unsignedBigInteger('finalized_at')->nullable();
|
|
$table->unsignedBigInteger('created_at');
|
|
$table->unsignedBigInteger('updated_at');
|
|
});
|
|
$db->schema()->create('cloud_storage_operations', static function (
|
|
Blueprint $table
|
|
): void {
|
|
$table->increments('id');
|
|
$table->string('operation_id')->unique();
|
|
$table->string('request_idempotency_key')->unique();
|
|
$table->string('request_fingerprint');
|
|
$table->unsignedBigInteger('artifact_id');
|
|
$table->unsignedBigInteger('replica_id')->nullable();
|
|
$table->unsignedBigInteger('source_backend_id')->nullable();
|
|
$table->unsignedBigInteger('target_backend_id')->nullable();
|
|
$table->string('kind');
|
|
$table->string('phase');
|
|
$table->string('outcome')->nullable();
|
|
$table->string('effect_resolution');
|
|
$table->boolean('effect_may_have_applied')->nullable();
|
|
$table->boolean('safe_to_resubmit')->nullable();
|
|
$table->unsignedInteger('attempts')->default(0);
|
|
$table->string('lease_token')->nullable();
|
|
$table->unsignedBigInteger('lease_until')->default(0);
|
|
$table->unsignedBigInteger('available_at')->default(0);
|
|
$table->unsignedBigInteger('next_check_at')->default(0);
|
|
$table->string('error_code')->default('');
|
|
$table->text('result_json')->nullable();
|
|
$table->unsignedBigInteger('terminal_at')->nullable();
|
|
$table->unsignedBigInteger('created_at');
|
|
$table->unsignedBigInteger('updated_at');
|
|
});
|
|
$db->schema()->create('cloud_storage_replicas', static function (
|
|
Blueprint $table
|
|
): void {
|
|
$table->increments('id');
|
|
$table->string('replica_key')->unique();
|
|
$table->unsignedBigInteger('artifact_id');
|
|
$table->unsignedBigInteger('backend_id');
|
|
$table->string('role');
|
|
$table->string('object_key');
|
|
$table->string('object_version_id')->default('');
|
|
$table->string('object_etag')->default('');
|
|
$table->string('state');
|
|
$table->string('publish_operation_id')->nullable();
|
|
$table->string('delete_operation_id')->nullable();
|
|
$table->unsignedInteger('delete_generation')->default(0);
|
|
$table->unsignedInteger('delete_cycle_attempts')->default(0);
|
|
$table->unsignedInteger('delete_presence_confirmations')->default(0);
|
|
$table->unsignedBigInteger('delete_last_checked_at')->default(0);
|
|
$table->unsignedBigInteger('delete_next_check_at')->default(0);
|
|
$table->unsignedBigInteger('delete_attention_at')->nullable();
|
|
$table->string('lease_token')->nullable();
|
|
$table->unsignedBigInteger('lease_until')->default(0);
|
|
$table->unsignedBigInteger('size_bytes')->default(0);
|
|
$table->string('sha256')->default('');
|
|
$table->string('error_code')->default('');
|
|
$table->unsignedBigInteger('verified_at')->default(0);
|
|
$table->unsignedBigInteger('created_at');
|
|
$table->unsignedBigInteger('updated_at');
|
|
});
|
|
$db->schema()->create('cloud_storage_backends', static function (
|
|
Blueprint $table
|
|
): void {
|
|
$table->increments('id');
|
|
$table->string('name');
|
|
});
|
|
|
|
$clock = 2000000000;
|
|
$ownerA = str_repeat('a', 32);
|
|
$ownerB = str_repeat('b', 32);
|
|
$create = static function (
|
|
string $kind,
|
|
string $owner,
|
|
string $state = 'ready_to_finalize',
|
|
int $attempts = 0,
|
|
int $availableAt = 0
|
|
) use (&$clock): CloudStoragePublication {
|
|
$artifact = new CloudStorageArtifact();
|
|
$artifact->artifact_key = bin2hex(random_bytes(16));
|
|
$artifact->kind = $kind;
|
|
$artifact->maximum_size_bytes = 4096;
|
|
$artifact->size_bytes = 4096;
|
|
$artifact->sha256 = str_repeat($kind === 'template' ? 'a' : 'b', 64);
|
|
$artifact->durability_policy = 'mirror_required_2';
|
|
$artifact->desired_copies = 2;
|
|
$artifact->required_copies = 2;
|
|
$artifact->state = 'protected';
|
|
$artifact->last_operation_id = null;
|
|
$artifact->verified_at = $clock - 10;
|
|
$artifact->created_at = $artifact->updated_at = $clock - 20;
|
|
$artifact->save();
|
|
|
|
$publication = new CloudStoragePublication();
|
|
$publication->publication_key = bin2hex(random_bytes(16));
|
|
$publication->artifact_id = (int) $artifact->id;
|
|
$publication->backup_id = null;
|
|
$publication->template_import_intent_id =
|
|
$kind === 'template' ? $owner : null;
|
|
$publication->rescue_template_id =
|
|
$kind === 'rescue' ? $owner : null;
|
|
$publication->state = $state;
|
|
$publication->attempts = $attempts;
|
|
$publication->available_at = $availableAt;
|
|
$publication->lease_token = null;
|
|
$publication->lease_until = 0;
|
|
$publication->last_error_code = '';
|
|
$publication->coverage_ready_at = $clock - 5;
|
|
$publication->finalized_at = null;
|
|
$publication->created_at = $publication->updated_at = $clock - 20;
|
|
$publication->save();
|
|
return $publication;
|
|
};
|
|
|
|
$template = $create('template', $ownerA);
|
|
$calls = [];
|
|
$worker = new CloudStoragePublicationOutboxService(
|
|
static function (
|
|
string $type,
|
|
int|string $owner,
|
|
int $publicationId,
|
|
int $artifactId
|
|
) use (&$calls, &$clock, $assert): void {
|
|
$assert(Capsule::connection()->transactionLevel() > 0,
|
|
'Business finalization escaped the owner transaction.');
|
|
$calls[] = [$type, $owner, $publicationId, $artifactId];
|
|
$row = CloudStoragePublication::query()
|
|
->whereKey($publicationId)->lockForUpdate()->firstOrFail();
|
|
$row->state = 'completed';
|
|
$row->attempts = (int) $row->attempts + 1;
|
|
$row->finalized_at = $clock;
|
|
$row->updated_at = $clock;
|
|
$row->save();
|
|
},
|
|
static function () use (&$clock): int {
|
|
return $clock;
|
|
}
|
|
);
|
|
$assert($worker->tick(), 'A due template publication was not consumed.');
|
|
$template->refresh();
|
|
$assert((string) $template->state === 'completed'
|
|
&& (int) $template->finalized_at === $clock
|
|
&& $calls === [[
|
|
'template', $ownerA, (int) $template->id,
|
|
(int) $template->artifact_id,
|
|
]], 'Template outbox finalization lost its frozen owner identity.');
|
|
$assert(!$worker->tick(), 'A completed publication was consumed twice.');
|
|
|
|
$rescue = $create('rescue', $ownerB, 'ready_to_finalize', 0, $clock + 60);
|
|
$assert(!$worker->tick(), 'A future publication bypassed its retry delay.');
|
|
$clock += 60;
|
|
$assert($worker->tick(), 'A due rescue publication was not consumed.');
|
|
$rescue->refresh();
|
|
$assert((string) $rescue->state === 'completed'
|
|
&& $calls[1][0] === 'rescue' && $calls[1][1] === $ownerB,
|
|
'Rescue outbox dispatch selected the wrong owner.');
|
|
|
|
$failing = $create('template', str_repeat('c', 32));
|
|
$failingWorker = new CloudStoragePublicationOutboxService(
|
|
static function (
|
|
string $type,
|
|
int|string $owner,
|
|
int $publicationId
|
|
) use (&$clock): void {
|
|
// Prove that a consumer which mutates the business publication and
|
|
// then crashes is rolled back to the savepoint before the durable
|
|
// retry delay is committed.
|
|
$partial = CloudStoragePublication::query()
|
|
->whereKey($publicationId)->lockForUpdate()->firstOrFail();
|
|
$partial->state = 'completed';
|
|
$partial->finalized_at = $clock;
|
|
$partial->save();
|
|
throw new CloudError('fixture owner failure', 409);
|
|
},
|
|
static function () use (&$clock): int {
|
|
return $clock;
|
|
}
|
|
);
|
|
$assert($failingWorker->tick(), 'A failed outbox attempt was not observed.');
|
|
$failing->refresh();
|
|
$assert((string) $failing->state === 'ready_to_finalize'
|
|
&& (int) $failing->attempts === 1
|
|
&& $failing->finalized_at === null
|
|
&& (int) $failing->available_at === $clock + 60
|
|
&& (string) $failing->last_error_code
|
|
=== 'storage_publication_finalization_failed'
|
|
&& $failingWorker->observationErrors()
|
|
=== ['storage_publication_finalization_failed'],
|
|
'A failed owner finalization was not durably delayed.');
|
|
$assert(!$failingWorker->tick(), 'A delayed failure entered a hot loop.');
|
|
|
|
$attention = $create(
|
|
'rescue',
|
|
str_repeat('d', 32),
|
|
'ready_to_finalize',
|
|
5,
|
|
$clock
|
|
);
|
|
$assert($failingWorker->tick(), 'The bounded final attempt was not handled.');
|
|
$attention->refresh();
|
|
$assert((string) $attention->state === 'failed'
|
|
&& (int) $attention->attempts === 6
|
|
&& (string) $attention->last_error_code
|
|
=== 'storage_publication_finalization_attention',
|
|
'Exhausted owner finalization did not become explicit attention.');
|
|
$attentionView = CloudStorageReplicaAttentionService::listView();
|
|
$assert(($attentionView['summary']['finalization_count'] ?? 0) === 1
|
|
&& count($attentionView['finalization_items'] ?? []) === 1
|
|
&& ($attentionView['finalization_items'][0]['operation_kind'] ?? '')
|
|
=== 'finalize'
|
|
&& ($attentionView['finalization_items'][0]['operation_id'] ?? '')
|
|
=== (string) $attention->publication_key
|
|
&& ($attentionView['finalization_items'][0]['can_retry'] ?? false),
|
|
'Finalization attention is not visible on the administrator recovery page.');
|
|
$retry = CloudStorageReplicaAttentionService::retry(
|
|
(string) $attention->publication_key,
|
|
'finalize'
|
|
);
|
|
$attention->refresh();
|
|
$assert((string) $attention->state === 'ready_to_finalize'
|
|
&& (int) $attention->attempts === 0
|
|
&& (int) $attention->available_at > 0
|
|
&& (string) $attention->last_error_code === ''
|
|
&& ($retry['operation_kind'] ?? '') === 'finalize',
|
|
'Administrator recovery did not reopen the same publication intent.');
|
|
$attention->state = 'completed';
|
|
$attention->finalized_at = $clock;
|
|
$attention->save();
|
|
|
|
// Finalization attention has an independent stable paginator. Prove adjacent
|
|
// pages are disjoint and together expose every row instead of silently
|
|
// truncating after the first page.
|
|
$pagedAttention = [];
|
|
foreach (['1', '2', '3'] as $ownerDigit) {
|
|
$row = $create('template', str_repeat($ownerDigit, 32));
|
|
$row->state = 'failed';
|
|
$row->attempts = 6;
|
|
$row->available_at = 0;
|
|
$row->last_error_code =
|
|
'storage_publication_finalization_attention';
|
|
$row->updated_at = $clock + (int) $ownerDigit;
|
|
$row->save();
|
|
$pagedAttention[] = $row;
|
|
}
|
|
$finalizationPageOne = CloudStorageReplicaAttentionService::listView([
|
|
'page' => 1,
|
|
'finalization_page' => 1,
|
|
'per_page' => 2,
|
|
]);
|
|
$finalizationPageTwo = CloudStorageReplicaAttentionService::listView([
|
|
'page' => 1,
|
|
'finalization_page' => 2,
|
|
'per_page' => 2,
|
|
]);
|
|
$pageOneIds = array_column(
|
|
$finalizationPageOne['finalization_items'],
|
|
'operation_id'
|
|
);
|
|
$pageTwoIds = array_column(
|
|
$finalizationPageTwo['finalization_items'],
|
|
'operation_id'
|
|
);
|
|
$assert(count($pageOneIds) === 2 && count($pageTwoIds) === 1
|
|
&& array_intersect($pageOneIds, $pageTwoIds) === []
|
|
&& count(array_unique(array_merge($pageOneIds, $pageTwoIds))) === 3
|
|
&& ($finalizationPageOne['finalization_pagination']['total'] ?? 0) === 3
|
|
&& ($finalizationPageOne['finalization_pagination']['total_pages'] ?? 0)
|
|
=== 2,
|
|
'Finalization attention pagination repeated or omitted an item.');
|
|
foreach ($pagedAttention as $row) {
|
|
$row->state = 'completed';
|
|
$row->finalized_at = $clock;
|
|
$row->save();
|
|
}
|
|
|
|
$databaseFault = $create('rescue', str_repeat('4', 32));
|
|
$databaseFaultWorker = new CloudStoragePublicationOutboxService(
|
|
static function (): void {
|
|
throw new PDOException('fixture database outage');
|
|
},
|
|
static function () use (&$clock): int {
|
|
return $clock;
|
|
}
|
|
);
|
|
$databaseFaultRaised = false;
|
|
try {
|
|
$databaseFaultWorker->tick();
|
|
} catch (PDOException) {
|
|
$databaseFaultRaised = true;
|
|
}
|
|
$databaseFault->refresh();
|
|
$assert($databaseFaultRaised
|
|
&& (string) $databaseFault->state === 'ready_to_finalize'
|
|
&& (int) $databaseFault->attempts === 0,
|
|
'Database failure consumed the bounded owner-finalization budget.');
|
|
$databaseFault->state = 'completed';
|
|
$databaseFault->finalized_at = $clock;
|
|
$databaseFault->save();
|
|
|
|
// Illuminate wraps ordinary SQL failures and deadlocks in QueryException.
|
|
// In the installed database component it extends PDOException, so the same
|
|
// infrastructure path must escape without consuming owner failure attempts.
|
|
$wrappedDatabaseFault = $create('template', str_repeat('5', 32));
|
|
$wrappedDatabaseFaultWorker = new CloudStoragePublicationOutboxService(
|
|
static function (): void {
|
|
throw new QueryException(
|
|
'default',
|
|
'UPDATE cloud_storage_publications SET state = ?',
|
|
['completed'],
|
|
new PDOException('fixture wrapped database outage')
|
|
);
|
|
},
|
|
static function () use (&$clock): int {
|
|
return $clock;
|
|
}
|
|
);
|
|
$wrappedDatabaseFaultRaised = false;
|
|
try {
|
|
$wrappedDatabaseFaultWorker->tick();
|
|
} catch (QueryException) {
|
|
$wrappedDatabaseFaultRaised = true;
|
|
}
|
|
$wrappedDatabaseFault->refresh();
|
|
$assert($wrappedDatabaseFaultRaised
|
|
&& (string) $wrappedDatabaseFault->state === 'ready_to_finalize'
|
|
&& (int) $wrappedDatabaseFault->attempts === 0,
|
|
'Wrapped database failure consumed owner-finalization attempts.');
|
|
$wrappedDatabaseFault->state = 'completed';
|
|
$wrappedDatabaseFault->finalized_at = $clock;
|
|
$wrappedDatabaseFault->save();
|
|
|
|
$wrong = $create('template', str_repeat('e', 32));
|
|
$noSettlement = new CloudStoragePublicationOutboxService(
|
|
static function (): void {
|
|
// A domain consumer returning without completing the outbox is an
|
|
// error; it must roll back and enter the same bounded retry path.
|
|
},
|
|
static function () use (&$clock): int {
|
|
return $clock;
|
|
}
|
|
);
|
|
$assert($noSettlement->tick(), 'Missing business settlement was not handled.');
|
|
$wrong->refresh();
|
|
$assert((string) $wrong->state === 'ready_to_finalize'
|
|
&& (int) $wrong->attempts === 1,
|
|
'A no-op business consumer was treated as successful.');
|
|
$wrong->state = 'completed';
|
|
$wrong->finalized_at = $clock;
|
|
$wrong->save();
|
|
|
|
$workerOwned = $create('rescue', str_repeat('f', 32));
|
|
$copyArtifact = new CloudStorageArtifact();
|
|
$copyArtifact->artifact_key = bin2hex(random_bytes(16));
|
|
$copyArtifact->kind = 'backup';
|
|
$copyArtifact->maximum_size_bytes = 4096;
|
|
$copyArtifact->size_bytes = 4096;
|
|
$copyArtifact->sha256 = str_repeat('9', 64);
|
|
$copyArtifact->durability_policy = 'mirror_required_2';
|
|
$copyArtifact->desired_copies = 2;
|
|
$copyArtifact->required_copies = 2;
|
|
$copyArtifact->state = 'available';
|
|
$copyArtifact->last_operation_id = null;
|
|
$copyArtifact->verified_at = $clock;
|
|
$copyArtifact->created_at = $copyArtifact->updated_at = $clock;
|
|
$copyArtifact->save();
|
|
|
|
$sourceReplica = new CloudStorageReplica();
|
|
$sourceReplica->replica_key = bin2hex(random_bytes(16));
|
|
$sourceReplica->artifact_id = (int) $copyArtifact->id;
|
|
$sourceReplica->backend_id = 1;
|
|
$sourceReplica->role = 'primary';
|
|
$sourceReplica->object_key = 'fixture/source.qcow2';
|
|
$sourceReplica->object_version_id = '';
|
|
$sourceReplica->object_etag = '"source"';
|
|
$sourceReplica->size_bytes = 4096;
|
|
$sourceReplica->sha256 = str_repeat('9', 64);
|
|
$sourceReplica->state = 'ready';
|
|
$sourceReplica->publish_operation_id = null;
|
|
$sourceReplica->delete_operation_id = null;
|
|
$sourceReplica->lease_token = null;
|
|
$sourceReplica->lease_until = 0;
|
|
$sourceReplica->error_code = '';
|
|
$sourceReplica->verified_at = $clock;
|
|
$sourceReplica->created_at = $sourceReplica->updated_at = $clock;
|
|
$sourceReplica->save();
|
|
|
|
$targetReplica = new CloudStorageReplica();
|
|
$targetReplica->replica_key = bin2hex(random_bytes(16));
|
|
$targetReplica->artifact_id = (int) $copyArtifact->id;
|
|
$targetReplica->backend_id = 2;
|
|
$targetReplica->role = 'secondary';
|
|
$targetReplica->object_key = 'fixture/target.qcow2';
|
|
$targetReplica->object_version_id = '';
|
|
$targetReplica->object_etag = '';
|
|
$targetReplica->size_bytes = 0;
|
|
$targetReplica->sha256 = '';
|
|
$targetReplica->state = 'reserved';
|
|
$targetReplica->publish_operation_id = null;
|
|
$targetReplica->delete_operation_id = null;
|
|
$targetReplica->lease_token = null;
|
|
$targetReplica->lease_until = 0;
|
|
$targetReplica->error_code = '';
|
|
$targetReplica->verified_at = 0;
|
|
$targetReplica->created_at = $targetReplica->updated_at = $clock;
|
|
$targetReplica->save();
|
|
|
|
$copyOperation = new CloudStorageOperation();
|
|
$copyOperation->operation_id = bin2hex(random_bytes(16));
|
|
$copyOperation->request_idempotency_key =
|
|
'publication-worker-copy-fixture';
|
|
$copyOperation->request_fingerprint = hash('sha256', 'copy-fixture');
|
|
$copyOperation->artifact_id = (int) $copyArtifact->id;
|
|
$copyOperation->replica_id = (int) $targetReplica->id;
|
|
$copyOperation->source_backend_id = 1;
|
|
$copyOperation->target_backend_id = 2;
|
|
$copyOperation->kind = 'copy';
|
|
$copyOperation->phase = 'accepted';
|
|
$copyOperation->outcome = null;
|
|
$copyOperation->effect_resolution = 'not_applicable';
|
|
$copyOperation->effect_may_have_applied = null;
|
|
$copyOperation->safe_to_resubmit = null;
|
|
$copyOperation->attempts = 0;
|
|
$copyOperation->lease_token = null;
|
|
$copyOperation->lease_until = 0;
|
|
$copyOperation->available_at = $clock;
|
|
$copyOperation->next_check_at = 0;
|
|
$copyOperation->error_code = '';
|
|
$copyOperation->result_json = null;
|
|
$copyOperation->terminal_at = null;
|
|
$copyOperation->created_at = $copyOperation->updated_at = $clock;
|
|
$copyOperation->save();
|
|
$integratedOutbox = new CloudStoragePublicationOutboxService(
|
|
static function (
|
|
string $type,
|
|
int|string $owner,
|
|
int $publicationId,
|
|
int $artifactId
|
|
) use (&$clock, $assert, $sourceReplica): void {
|
|
$assert($type === 'rescue' && $owner === str_repeat('f', 32),
|
|
'Replica worker dispatched the wrong publication owner.');
|
|
$publication = CloudStoragePublication::query()
|
|
->whereKey($publicationId)->lockForUpdate()->firstOrFail();
|
|
$publication->state = 'completed';
|
|
$publication->finalized_at = $clock;
|
|
$publication->save();
|
|
// Makes the replica source invalid only after publication work ran;
|
|
// the following replica_source_not_ready assertion therefore proves
|
|
// the short outbox step was consumed before the replica claim.
|
|
$sourceReplica->state = 'degraded';
|
|
$sourceReplica->error_code = 'fixture_not_ready';
|
|
$sourceReplica->verified_at = 0;
|
|
$sourceReplica->save();
|
|
},
|
|
static function () use (&$clock): int {
|
|
return $clock;
|
|
}
|
|
);
|
|
$replicaWorker = new CloudStorageReplicaWorkerService(
|
|
null,
|
|
null,
|
|
null,
|
|
static function () use (&$clock): int {
|
|
return $clock;
|
|
},
|
|
$integratedOutbox
|
|
);
|
|
$assert($replicaWorker->tick()
|
|
&& $replicaWorker->observationErrors()
|
|
=== ['replica_source_not_ready'],
|
|
'Replica worker did not process both bounded work queues in one tick.');
|
|
$workerOwned->refresh();
|
|
$copyOperation->refresh();
|
|
$assert((string) $workerOwned->state === 'completed',
|
|
'Replica worker did not settle the owner publication.');
|
|
$assert((string) $copyOperation->phase === 'accepted'
|
|
&& (int) $copyOperation->attempts === 1
|
|
&& (int) $copyOperation->available_at === $clock + 60
|
|
&& (string) $copyOperation->error_code === 'replica_source_not_ready',
|
|
'Publication finalization starved the replica queue in the same tick.');
|
|
|
|
echo "Storage publication outbox behavior passed.\n";
|