|
|
|
|
<?php
|
|
|
|
|
|
|
|
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
|
|
|
|
|
|
|
|
|
use App\Models\CloudStorageArtifact;
|
|
|
|
|
use App\Models\CloudStoragePublication;
|
|
|
|
|
use App\Services\Cloud\CloudError;
|
|
|
|
|
use App\Services\Cloud\CloudRescueStoragePublicationService;
|
|
|
|
|
use App\Services\Cloud\CloudStoragePublicationOutboxService;
|
|
|
|
|
use App\Services\Cloud\CloudStorageReplicaAttentionService;
|
|
|
|
|
use App\Services\Cloud\CloudTemplateStoragePublicationConsumer;
|
|
|
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
|
|
|
use Illuminate\Database\QueryException;
|
|
|
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
|
|
|
|
|
|
|
|
$assert = static function (bool $condition, string $message): void {
|
|
|
|
|
if (!$condition) {
|
|
|
|
|
throw new RuntimeException($message);
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
$assert(method_exists(
|
|
|
|
|
CloudTemplateStoragePublicationConsumer::class,
|
|
|
|
|
'finalizeReady'
|
|
|
|
|
) && method_exists(
|
|
|
|
|
CloudRescueStoragePublicationService::class,
|
|
|
|
|
'finalizeReadyOnly'
|
|
|
|
|
), 'Publication outbox lost a local-only business finalization consumer.');
|
|
|
|
|
|
|
|
|
|
$db = new Capsule();
|
|
|
|
|
$db->addConnection(['driver' => 'sqlite', 'database' => ':memory:']);
|
|
|
|
|
$db->setAsGlobal();
|
|
|
|
|
$db->bootEloquent();
|
|
|
|
|
$db->schema()->create('cloud_storage_artifacts', static function (
|
|
|
|
|
Blueprint $table
|
|
|
|
|
): void {
|
|
|
|
|
$table->increments('id');
|
|
|
|
|
$table->string('artifact_key')->unique();
|
|
|
|
|
$table->string('kind');
|
|
|
|
|
$table->unsignedBigInteger('maximum_size_bytes');
|
|
|
|
|
$table->unsignedBigInteger('size_bytes');
|
|
|
|
|
$table->string('sha256');
|
|
|
|
|
$table->string('durability_policy');
|
|
|
|
|
$table->unsignedTinyInteger('desired_copies');
|
|
|
|
|
$table->unsignedTinyInteger('required_copies');
|
|
|
|
|
$table->string('state');
|
|
|
|
|
$table->string('last_operation_id')->nullable();
|
|
|
|
|
$table->unsignedBigInteger('verified_at');
|
|
|
|
|
$table->unsignedBigInteger('created_at');
|
|
|
|
|
$table->unsignedBigInteger('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$db->schema()->create('cloud_storage_publications', static function (
|
|
|
|
|
Blueprint $table
|
|
|
|
|
): void {
|
|
|
|
|
$table->increments('id');
|
|
|
|
|
$table->string('publication_key')->unique();
|
|
|
|
|
$table->unsignedBigInteger('artifact_id')->unique();
|
|
|
|
|
$table->unsignedBigInteger('backup_id')->nullable()->unique();
|
|
|
|
|
$table->string('template_import_intent_id')->nullable()->unique();
|
|
|
|
|
$table->string('rescue_template_id')->nullable()->unique();
|
|
|
|
|
$table->string('state');
|
|
|
|
|
$table->unsignedInteger('attempts')->default(0);
|
|
|
|
|
$table->unsignedBigInteger('available_at')->default(0);
|
|
|
|
|
$table->string('lease_token')->nullable();
|
|
|
|
|
$table->unsignedBigInteger('lease_until')->default(0);
|
|
|
|
|
$table->string('last_error_code')->default('');
|
|
|
|
|
$table->unsignedBigInteger('coverage_ready_at')->nullable();
|
|
|
|
|
$table->unsignedBigInteger('finalized_at')->nullable();
|
|
|
|
|
$table->unsignedBigInteger('created_at');
|
|
|
|
|
$table->unsignedBigInteger('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$db->schema()->create('cloud_storage_operations', static function (
|
|
|
|
|
Blueprint $table
|
|
|
|
|
): void {
|
|
|
|
|
$table->increments('id');
|
|
|
|
|
$table->string('operation_id')->unique();
|
|
|
|
|
$table->unsignedBigInteger('artifact_id');
|
|
|
|
|
$table->unsignedBigInteger('replica_id')->nullable();
|
|
|
|
|
$table->unsignedBigInteger('source_backend_id')->nullable();
|
|
|
|
|
$table->unsignedBigInteger('target_backend_id')->nullable();
|
|
|
|
|
$table->string('kind');
|
|
|
|
|
$table->string('phase');
|
|
|
|
|
$table->string('outcome')->nullable();
|
|
|
|
|
$table->string('effect_resolution');
|
|
|
|
|
$table->boolean('effect_may_have_applied')->nullable();
|
|
|
|
|
$table->boolean('safe_to_resubmit')->nullable();
|
|
|
|
|
$table->unsignedInteger('attempts')->default(0);
|
|
|
|
|
$table->string('lease_token')->nullable();
|
|
|
|
|
$table->unsignedBigInteger('lease_until')->default(0);
|
|
|
|
|
$table->unsignedBigInteger('available_at')->default(0);
|
|
|
|
|
$table->unsignedBigInteger('next_check_at')->default(0);
|
|
|
|
|
$table->string('error_code')->default('');
|
|
|
|
|
$table->unsignedBigInteger('terminal_at')->nullable();
|
|
|
|
|
});
|
|
|
|
|
$db->schema()->create('cloud_storage_replicas', static function (
|
|
|
|
|
Blueprint $table
|
|
|
|
|
): void {
|
|
|
|
|
$table->increments('id');
|
|
|
|
|
$table->unsignedBigInteger('artifact_id');
|
|
|
|
|
$table->unsignedBigInteger('backend_id');
|
|
|
|
|
$table->string('state');
|
|
|
|
|
$table->unsignedBigInteger('size_bytes')->default(0);
|
|
|
|
|
$table->string('sha256')->default('');
|
|
|
|
|
$table->string('error_code')->default('');
|
|
|
|
|
});
|
|
|
|
|
$db->schema()->create('cloud_storage_backends', static function (
|
|
|
|
|
Blueprint $table
|
|
|
|
|
): void {
|
|
|
|
|
$table->increments('id');
|
|
|
|
|
$table->string('name');
|
|
|
|
|
$table->string('prefix')->default('');
|
|
|
|
|
$table->string('failure_domain')->default('');
|
|
|
|
|
$table->unsignedBigInteger('config_revision')->default(1);
|
|
|
|
|
$table->unsignedBigInteger('tested_revision')->default(0);
|
|
|
|
|
$table->string('admin_state')->default('draft');
|
|
|
|
|
$table->string('health_state')->default('unknown');
|
|
|
|
|
$table->unsignedInteger('write_priority')->default(0);
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
$clock = 2000000000;
|
|
|
|
|
$ownerA = str_repeat('a', 32);
|
|
|
|
|
$ownerB = str_repeat('b', 32);
|
|
|
|
|
$create = static function (
|
|
|
|
|
string $kind,
|
|
|
|
|
string $owner,
|
|
|
|
|
string $state = 'ready_to_finalize',
|
|
|
|
|
int $attempts = 0,
|
|
|
|
|
int $availableAt = 0
|
|
|
|
|
) use (&$clock): CloudStoragePublication {
|
|
|
|
|
$artifact = new CloudStorageArtifact();
|
|
|
|
|
$artifact->artifact_key = bin2hex(random_bytes(16));
|
|
|
|
|
$artifact->kind = $kind;
|
|
|
|
|
$artifact->maximum_size_bytes = 4096;
|
|
|
|
|
$artifact->size_bytes = 4096;
|
|
|
|
|
$artifact->sha256 = str_repeat($kind === 'template' ? 'a' : 'b', 64);
|
|
|
|
|
$artifact->durability_policy = 'mirror_required_2';
|
|
|
|
|
$artifact->desired_copies = 2;
|
|
|
|
|
$artifact->required_copies = 2;
|
|
|
|
|
$artifact->state = 'protected';
|
|
|
|
|
$artifact->last_operation_id = null;
|
|
|
|
|
$artifact->verified_at = $clock - 10;
|
|
|
|
|
$artifact->created_at = $artifact->updated_at = $clock - 20;
|
|
|
|
|
$artifact->save();
|
|
|
|
|
|
|
|
|
|
$publication = new CloudStoragePublication();
|
|
|
|
|
$publication->publication_key = bin2hex(random_bytes(16));
|
|
|
|
|
$publication->artifact_id = (int) $artifact->id;
|
|
|
|
|
$publication->backup_id = null;
|
|
|
|
|
$publication->template_import_intent_id =
|
|
|
|
|
$kind === 'template' ? $owner : null;
|
|
|
|
|
$publication->rescue_template_id =
|
|
|
|
|
$kind === 'rescue' ? $owner : null;
|
|
|
|
|
$publication->state = $state;
|
|
|
|
|
$publication->attempts = $attempts;
|
|
|
|
|
$publication->available_at = $availableAt;
|
|
|
|
|
$publication->lease_token = null;
|
|
|
|
|
$publication->lease_until = 0;
|
|
|
|
|
$publication->last_error_code = '';
|
|
|
|
|
$publication->coverage_ready_at = $clock - 5;
|
|
|
|
|
$publication->finalized_at = null;
|
|
|
|
|
$publication->created_at = $publication->updated_at = $clock - 20;
|
|
|
|
|
$publication->save();
|
|
|
|
|
return $publication;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
$template = $create('template', $ownerA);
|
|
|
|
|
$calls = [];
|
|
|
|
|
$worker = new CloudStoragePublicationOutboxService(
|
|
|
|
|
static function (
|
|
|
|
|
string $type,
|
|
|
|
|
int|string $owner,
|
|
|
|
|
int $publicationId,
|
|
|
|
|
int $artifactId
|
|
|
|
|
) use (&$calls, &$clock, $assert): void {
|
|
|
|
|
$assert(Capsule::connection()->transactionLevel() > 0,
|
|
|
|
|
'Business finalization escaped the owner transaction.');
|
|
|
|
|
$calls[] = [$type, $owner, $publicationId, $artifactId];
|
|
|
|
|
$row = CloudStoragePublication::query()
|
|
|
|
|
->whereKey($publicationId)->lockForUpdate()->firstOrFail();
|
|
|
|
|
$row->state = 'completed';
|
|
|
|
|
$row->attempts = (int) $row->attempts + 1;
|
|
|
|
|
$row->finalized_at = $clock;
|
|
|
|
|
$row->updated_at = $clock;
|
|
|
|
|
$row->save();
|
|
|
|
|
},
|
|
|
|
|
static function () use (&$clock): int {
|
|
|
|
|
return $clock;
|
|
|
|
|
}
|
|
|
|
|
);
|
|
|
|
|
$assert($worker->tick(), 'A due template publication was not consumed.');
|
|
|
|
|
$template->refresh();
|
|
|
|
|
$assert((string) $template->state === 'completed'
|
|
|
|
|
&& (int) $template->finalized_at === $clock
|
|
|
|
|
&& $calls === [[
|
|
|
|
|
'template', $ownerA, (int) $template->id,
|
|
|
|
|
(int) $template->artifact_id,
|
|
|
|
|
]], 'Template outbox finalization lost its frozen owner identity.');
|
|
|
|
|
$assert(!$worker->tick(), 'A completed publication was consumed twice.');
|
|
|
|
|
|
|
|
|
|
$rescue = $create('rescue', $ownerB, 'ready_to_finalize', 0, $clock + 60);
|
|
|
|
|
$assert(!$worker->tick(), 'A future publication bypassed its retry delay.');
|
|
|
|
|
$clock += 60;
|
|
|
|
|
$assert($worker->tick(), 'A due rescue publication was not consumed.');
|
|
|
|
|
$rescue->refresh();
|
|
|
|
|
$assert((string) $rescue->state === 'completed'
|
|
|
|
|
&& $calls[1][0] === 'rescue' && $calls[1][1] === $ownerB,
|
|
|
|
|
'Rescue outbox dispatch selected the wrong owner.');
|
|
|
|
|
|
|
|
|
|
$failing = $create('template', str_repeat('c', 32));
|
|
|
|
|
$failingWorker = new CloudStoragePublicationOutboxService(
|
|
|
|
|
static function (
|
|
|
|
|
string $type,
|
|
|
|
|
int|string $owner,
|
|
|
|
|
int $publicationId
|
|
|
|
|
) use (&$clock): void {
|
|
|
|
|
// Prove that a consumer which mutates the business publication and
|
|
|
|
|
// then crashes is rolled back to the savepoint before the durable
|
|
|
|
|
// retry delay is committed.
|
|
|
|
|
$partial = CloudStoragePublication::query()
|
|
|
|
|
->whereKey($publicationId)->lockForUpdate()->firstOrFail();
|
|
|
|
|
$partial->state = 'completed';
|
|
|
|
|
$partial->finalized_at = $clock;
|
|
|
|
|
$partial->save();
|
|
|
|
|
throw new CloudError('fixture owner failure', 409);
|
|
|
|
|
},
|
|
|
|
|
static function () use (&$clock): int {
|
|
|
|
|
return $clock;
|
|
|
|
|
}
|
|
|
|
|
);
|
|
|
|
|
$assert($failingWorker->tick(), 'A failed outbox attempt was not observed.');
|
|
|
|
|
$failing->refresh();
|
|
|
|
|
$assert((string) $failing->state === 'ready_to_finalize'
|
|
|
|
|
&& (int) $failing->attempts === 1
|
|
|
|
|
&& $failing->finalized_at === null
|
|
|
|
|
&& (int) $failing->available_at === $clock + 60
|
|
|
|
|
&& (string) $failing->last_error_code
|
|
|
|
|
=== 'storage_publication_finalization_failed'
|
|
|
|
|
&& $failingWorker->observationErrors()
|
|
|
|
|
=== ['storage_publication_finalization_failed'],
|
|
|
|
|
'A failed owner finalization was not durably delayed.');
|
|
|
|
|
$assert(!$failingWorker->tick(), 'A delayed failure entered a hot loop.');
|
|
|
|
|
|
|
|
|
|
$attention = $create(
|
|
|
|
|
'rescue',
|
|
|
|
|
str_repeat('d', 32),
|
|
|
|
|
'ready_to_finalize',
|
|
|
|
|
5,
|
|
|
|
|
$clock
|
|
|
|
|
);
|
|
|
|
|
$assert($failingWorker->tick(), 'The bounded final attempt was not handled.');
|
|
|
|
|
$attention->refresh();
|
|
|
|
|
$assert((string) $attention->state === 'failed'
|
|
|
|
|
&& (int) $attention->attempts === 6
|
|
|
|
|
&& (string) $attention->last_error_code
|
|
|
|
|
=== 'storage_publication_finalization_attention',
|
|
|
|
|
'Exhausted owner finalization did not become explicit attention.');
|
|
|
|
|
$attentionView = CloudStorageReplicaAttentionService::listView();
|
|
|
|
|
$assert(($attentionView['summary']['finalization_count'] ?? 0) === 1
|
|
|
|
|
&& count($attentionView['finalization_items'] ?? []) === 1
|
|
|
|
|
&& ($attentionView['finalization_items'][0]['operation_kind'] ?? '')
|
|
|
|
|
=== 'finalize'
|
|
|
|
|
&& ($attentionView['finalization_items'][0]['operation_id'] ?? '')
|
|
|
|
|
=== (string) $attention->publication_key
|
|
|
|
|
&& ($attentionView['finalization_items'][0]['can_retry'] ?? false),
|
|
|
|
|
'Finalization attention is not visible on the administrator recovery page.');
|
|
|
|
|
$retry = CloudStorageReplicaAttentionService::retry(
|
|
|
|
|
(string) $attention->publication_key,
|
|
|
|
|
'finalize'
|
|
|
|
|
);
|
|
|
|
|
$attention->refresh();
|
|
|
|
|
$assert((string) $attention->state === 'ready_to_finalize'
|
|
|
|
|
&& (int) $attention->attempts === 0
|
|
|
|
|
&& (int) $attention->available_at > 0
|
|
|
|
|
&& (string) $attention->last_error_code === ''
|
|
|
|
|
&& ($retry['operation_kind'] ?? '') === 'finalize',
|
|
|
|
|
'Administrator recovery did not reopen the same publication intent.');
|
|
|
|
|
$attention->state = 'completed';
|
|
|
|
|
$attention->finalized_at = $clock;
|
|
|
|
|
$attention->save();
|
|
|
|
|
|
|
|
|
|
// Finalization attention has an independent stable paginator. Prove adjacent
|
|
|
|
|
// pages are disjoint and together expose every row instead of silently
|
|
|
|
|
// truncating after the first page.
|
|
|
|
|
$pagedAttention = [];
|
|
|
|
|
foreach (['1', '2', '3'] as $ownerDigit) {
|
|
|
|
|
$row = $create('template', str_repeat($ownerDigit, 32));
|
|
|
|
|
$row->state = 'failed';
|
|
|
|
|
$row->attempts = 6;
|
|
|
|
|
$row->available_at = 0;
|
|
|
|
|
$row->last_error_code =
|
|
|
|
|
'storage_publication_finalization_attention';
|
|
|
|
|
$row->updated_at = $clock + (int) $ownerDigit;
|
|
|
|
|
$row->save();
|
|
|
|
|
$pagedAttention[] = $row;
|
|
|
|
|
}
|
|
|
|
|
$finalizationPageOne = CloudStorageReplicaAttentionService::listView([
|
|
|
|
|
'page' => 1,
|
|
|
|
|
'finalization_page' => 1,
|
|
|
|
|
'per_page' => 2,
|
|
|
|
|
]);
|
|
|
|
|
$finalizationPageTwo = CloudStorageReplicaAttentionService::listView([
|
|
|
|
|
'page' => 1,
|
|
|
|
|
'finalization_page' => 2,
|
|
|
|
|
'per_page' => 2,
|
|
|
|
|
]);
|
|
|
|
|
$pageOneIds = array_column(
|
|
|
|
|
$finalizationPageOne['finalization_items'],
|
|
|
|
|
'operation_id'
|
|
|
|
|
);
|
|
|
|
|
$pageTwoIds = array_column(
|
|
|
|
|
$finalizationPageTwo['finalization_items'],
|
|
|
|
|
'operation_id'
|
|
|
|
|
);
|
|
|
|
|
$assert(count($pageOneIds) === 2 && count($pageTwoIds) === 1
|
|
|
|
|
&& array_intersect($pageOneIds, $pageTwoIds) === []
|
|
|
|
|
&& count(array_unique(array_merge($pageOneIds, $pageTwoIds))) === 3
|
|
|
|
|
&& ($finalizationPageOne['finalization_pagination']['total'] ?? 0) === 3
|
|
|
|
|
&& ($finalizationPageOne['finalization_pagination']['total_pages'] ?? 0)
|
|
|
|
|
=== 2,
|
|
|
|
|
'Finalization attention pagination repeated or omitted an item.');
|
|
|
|
|
foreach ($pagedAttention as $row) {
|
|
|
|
|
$row->state = 'completed';
|
|
|
|
|
$row->finalized_at = $clock;
|
|
|
|
|
$row->save();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
$databaseFault = $create('rescue', str_repeat('4', 32));
|
|
|
|
|
$databaseFaultWorker = new CloudStoragePublicationOutboxService(
|
|
|
|
|
static function (): void {
|
|
|
|
|
throw new PDOException('fixture database outage');
|
|
|
|
|
},
|
|
|
|
|
static function () use (&$clock): int {
|
|
|
|
|
return $clock;
|
|
|
|
|
}
|
|
|
|
|
);
|
|
|
|
|
$databaseFaultRaised = false;
|
|
|
|
|
try {
|
|
|
|
|
$databaseFaultWorker->tick();
|
|
|
|
|
} catch (PDOException) {
|
|
|
|
|
$databaseFaultRaised = true;
|
|
|
|
|
}
|
|
|
|
|
$databaseFault->refresh();
|
|
|
|
|
$assert($databaseFaultRaised
|
|
|
|
|
&& (string) $databaseFault->state === 'ready_to_finalize'
|
|
|
|
|
&& (int) $databaseFault->attempts === 0,
|
|
|
|
|
'Database failure consumed the bounded owner-finalization budget.');
|
|
|
|
|
$databaseFault->state = 'completed';
|
|
|
|
|
$databaseFault->finalized_at = $clock;
|
|
|
|
|
$databaseFault->save();
|
|
|
|
|
|
|
|
|
|
// Illuminate wraps ordinary SQL failures and deadlocks in QueryException.
|
|
|
|
|
// In the installed database component it extends PDOException, so the same
|
|
|
|
|
// infrastructure path must escape without consuming owner failure attempts.
|
|
|
|
|
$wrappedDatabaseFault = $create('template', str_repeat('5', 32));
|
|
|
|
|
$wrappedDatabaseFaultWorker = new CloudStoragePublicationOutboxService(
|
|
|
|
|
static function (): void {
|
|
|
|
|
throw new QueryException(
|
|
|
|
|
'default',
|
|
|
|
|
'UPDATE cloud_storage_publications SET state = ?',
|
|
|
|
|
['completed'],
|
|
|
|
|
new PDOException('fixture wrapped database outage')
|
|
|
|
|
);
|
|
|
|
|
},
|
|
|
|
|
static function () use (&$clock): int {
|
|
|
|
|
return $clock;
|
|
|
|
|
}
|
|
|
|
|
);
|
|
|
|
|
$wrappedDatabaseFaultRaised = false;
|
|
|
|
|
try {
|
|
|
|
|
$wrappedDatabaseFaultWorker->tick();
|
|
|
|
|
} catch (QueryException) {
|
|
|
|
|
$wrappedDatabaseFaultRaised = true;
|
|
|
|
|
}
|
|
|
|
|
$wrappedDatabaseFault->refresh();
|
|
|
|
|
$assert($wrappedDatabaseFaultRaised
|
|
|
|
|
&& (string) $wrappedDatabaseFault->state === 'ready_to_finalize'
|
|
|
|
|
&& (int) $wrappedDatabaseFault->attempts === 0,
|
|
|
|
|
'Wrapped database failure consumed owner-finalization attempts.');
|
|
|
|
|
$wrappedDatabaseFault->state = 'completed';
|
|
|
|
|
$wrappedDatabaseFault->finalized_at = $clock;
|
|
|
|
|
$wrappedDatabaseFault->save();
|
|
|
|
|
|
|
|
|
|
$wrong = $create('template', str_repeat('e', 32));
|
|
|
|
|
$noSettlement = new CloudStoragePublicationOutboxService(
|
|
|
|
|
static function (): void {
|
|
|
|
|
// A domain consumer returning without completing the outbox is an
|
|
|
|
|
// error; it must roll back and enter the same bounded retry path.
|
|
|
|
|
},
|
|
|
|
|
static function () use (&$clock): int {
|
|
|
|
|
return $clock;
|
|
|
|
|
}
|
|
|
|
|
);
|
|
|
|
|
$assert($noSettlement->tick(), 'Missing business settlement was not handled.');
|
|
|
|
|
$wrong->refresh();
|
|
|
|
|
$assert((string) $wrong->state === 'ready_to_finalize'
|
|
|
|
|
&& (int) $wrong->attempts === 1,
|
|
|
|
|
'A no-op business consumer was treated as successful.');
|
|
|
|
|
$wrong->state = 'completed';
|
|
|
|
|
$wrong->finalized_at = $clock;
|
|
|
|
|
$wrong->save();
|
|
|
|
|
|
|
|
|
|
echo "Storage publication outbox behavior passed.\n";
|