You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudStoragePublicationOutb...

402 lines
15 KiB
PHP

<?php
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudStorageArtifact;
use App\Models\CloudStoragePublication;
use App\Services\Cloud\CloudError;
use App\Services\Cloud\CloudRescueStoragePublicationService;
use App\Services\Cloud\CloudStoragePublicationOutboxService;
use App\Services\Cloud\CloudStorageReplicaAttentionService;
use App\Services\Cloud\CloudTemplateStoragePublicationConsumer;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\QueryException;
use Illuminate\Database\Schema\Blueprint;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$assert(method_exists(
CloudTemplateStoragePublicationConsumer::class,
'finalizeReady'
) && method_exists(
CloudRescueStoragePublicationService::class,
'finalizeReadyOnly'
), 'Publication outbox lost a local-only business finalization consumer.');
$db = new Capsule();
$db->addConnection(['driver' => 'sqlite', 'database' => ':memory:']);
$db->setAsGlobal();
$db->bootEloquent();
$db->schema()->create('cloud_storage_artifacts', static function (
Blueprint $table
): void {
$table->increments('id');
$table->string('artifact_key')->unique();
$table->string('kind');
$table->unsignedBigInteger('maximum_size_bytes');
$table->unsignedBigInteger('size_bytes');
$table->string('sha256');
$table->string('durability_policy');
$table->unsignedTinyInteger('desired_copies');
$table->unsignedTinyInteger('required_copies');
$table->string('state');
$table->string('last_operation_id')->nullable();
$table->unsignedBigInteger('verified_at');
$table->unsignedBigInteger('created_at');
$table->unsignedBigInteger('updated_at');
});
$db->schema()->create('cloud_storage_publications', static function (
Blueprint $table
): void {
$table->increments('id');
$table->string('publication_key')->unique();
$table->unsignedBigInteger('artifact_id')->unique();
$table->unsignedBigInteger('backup_id')->nullable()->unique();
$table->string('template_import_intent_id')->nullable()->unique();
$table->string('rescue_template_id')->nullable()->unique();
$table->string('state');
$table->unsignedInteger('attempts')->default(0);
$table->unsignedBigInteger('available_at')->default(0);
$table->string('lease_token')->nullable();
$table->unsignedBigInteger('lease_until')->default(0);
$table->string('last_error_code')->default('');
$table->unsignedBigInteger('coverage_ready_at')->nullable();
$table->unsignedBigInteger('finalized_at')->nullable();
$table->unsignedBigInteger('created_at');
$table->unsignedBigInteger('updated_at');
});
$db->schema()->create('cloud_storage_operations', static function (
Blueprint $table
): void {
$table->increments('id');
$table->string('operation_id')->unique();
$table->unsignedBigInteger('artifact_id');
$table->unsignedBigInteger('replica_id')->nullable();
$table->unsignedBigInteger('source_backend_id')->nullable();
$table->unsignedBigInteger('target_backend_id')->nullable();
$table->string('kind');
$table->string('phase');
$table->string('outcome')->nullable();
$table->string('effect_resolution');
$table->boolean('effect_may_have_applied')->nullable();
$table->boolean('safe_to_resubmit')->nullable();
$table->unsignedInteger('attempts')->default(0);
$table->string('lease_token')->nullable();
$table->unsignedBigInteger('lease_until')->default(0);
$table->unsignedBigInteger('available_at')->default(0);
$table->unsignedBigInteger('next_check_at')->default(0);
$table->string('error_code')->default('');
$table->unsignedBigInteger('terminal_at')->nullable();
});
$db->schema()->create('cloud_storage_replicas', static function (
Blueprint $table
): void {
$table->increments('id');
$table->unsignedBigInteger('artifact_id');
$table->unsignedBigInteger('backend_id');
$table->string('state');
$table->unsignedBigInteger('size_bytes')->default(0);
$table->string('sha256')->default('');
$table->string('error_code')->default('');
});
$db->schema()->create('cloud_storage_backends', static function (
Blueprint $table
): void {
$table->increments('id');
$table->string('name');
$table->string('prefix')->default('');
$table->string('failure_domain')->default('');
$table->unsignedBigInteger('config_revision')->default(1);
$table->unsignedBigInteger('tested_revision')->default(0);
$table->string('admin_state')->default('draft');
$table->string('health_state')->default('unknown');
$table->unsignedInteger('write_priority')->default(0);
});
$clock = 2000000000;
$ownerA = str_repeat('a', 32);
$ownerB = str_repeat('b', 32);
$create = static function (
string $kind,
string $owner,
string $state = 'ready_to_finalize',
int $attempts = 0,
int $availableAt = 0
) use (&$clock): CloudStoragePublication {
$artifact = new CloudStorageArtifact();
$artifact->artifact_key = bin2hex(random_bytes(16));
$artifact->kind = $kind;
$artifact->maximum_size_bytes = 4096;
$artifact->size_bytes = 4096;
$artifact->sha256 = str_repeat($kind === 'template' ? 'a' : 'b', 64);
$artifact->durability_policy = 'mirror_required_2';
$artifact->desired_copies = 2;
$artifact->required_copies = 2;
$artifact->state = 'protected';
$artifact->last_operation_id = null;
$artifact->verified_at = $clock - 10;
$artifact->created_at = $artifact->updated_at = $clock - 20;
$artifact->save();
$publication = new CloudStoragePublication();
$publication->publication_key = bin2hex(random_bytes(16));
$publication->artifact_id = (int) $artifact->id;
$publication->backup_id = null;
$publication->template_import_intent_id =
$kind === 'template' ? $owner : null;
$publication->rescue_template_id =
$kind === 'rescue' ? $owner : null;
$publication->state = $state;
$publication->attempts = $attempts;
$publication->available_at = $availableAt;
$publication->lease_token = null;
$publication->lease_until = 0;
$publication->last_error_code = '';
$publication->coverage_ready_at = $clock - 5;
$publication->finalized_at = null;
$publication->created_at = $publication->updated_at = $clock - 20;
$publication->save();
return $publication;
};
$template = $create('template', $ownerA);
$calls = [];
$worker = new CloudStoragePublicationOutboxService(
static function (
string $type,
int|string $owner,
int $publicationId,
int $artifactId
) use (&$calls, &$clock, $assert): void {
$assert(Capsule::connection()->transactionLevel() > 0,
'Business finalization escaped the owner transaction.');
$calls[] = [$type, $owner, $publicationId, $artifactId];
$row = CloudStoragePublication::query()
->whereKey($publicationId)->lockForUpdate()->firstOrFail();
$row->state = 'completed';
$row->attempts = (int) $row->attempts + 1;
$row->finalized_at = $clock;
$row->updated_at = $clock;
$row->save();
},
static function () use (&$clock): int {
return $clock;
}
);
$assert($worker->tick(), 'A due template publication was not consumed.');
$template->refresh();
$assert((string) $template->state === 'completed'
&& (int) $template->finalized_at === $clock
&& $calls === [[
'template', $ownerA, (int) $template->id,
(int) $template->artifact_id,
]], 'Template outbox finalization lost its frozen owner identity.');
$assert(!$worker->tick(), 'A completed publication was consumed twice.');
$rescue = $create('rescue', $ownerB, 'ready_to_finalize', 0, $clock + 60);
$assert(!$worker->tick(), 'A future publication bypassed its retry delay.');
$clock += 60;
$assert($worker->tick(), 'A due rescue publication was not consumed.');
$rescue->refresh();
$assert((string) $rescue->state === 'completed'
&& $calls[1][0] === 'rescue' && $calls[1][1] === $ownerB,
'Rescue outbox dispatch selected the wrong owner.');
$failing = $create('template', str_repeat('c', 32));
$failingWorker = new CloudStoragePublicationOutboxService(
static function (
string $type,
int|string $owner,
int $publicationId
) use (&$clock): void {
// Prove that a consumer which mutates the business publication and
// then crashes is rolled back to the savepoint before the durable
// retry delay is committed.
$partial = CloudStoragePublication::query()
->whereKey($publicationId)->lockForUpdate()->firstOrFail();
$partial->state = 'completed';
$partial->finalized_at = $clock;
$partial->save();
throw new CloudError('fixture owner failure', 409);
},
static function () use (&$clock): int {
return $clock;
}
);
$assert($failingWorker->tick(), 'A failed outbox attempt was not observed.');
$failing->refresh();
$assert((string) $failing->state === 'ready_to_finalize'
&& (int) $failing->attempts === 1
&& $failing->finalized_at === null
&& (int) $failing->available_at === $clock + 60
&& (string) $failing->last_error_code
=== 'storage_publication_finalization_failed'
&& $failingWorker->observationErrors()
=== ['storage_publication_finalization_failed'],
'A failed owner finalization was not durably delayed.');
$assert(!$failingWorker->tick(), 'A delayed failure entered a hot loop.');
$attention = $create(
'rescue',
str_repeat('d', 32),
'ready_to_finalize',
5,
$clock
);
$assert($failingWorker->tick(), 'The bounded final attempt was not handled.');
$attention->refresh();
$assert((string) $attention->state === 'failed'
&& (int) $attention->attempts === 6
&& (string) $attention->last_error_code
=== 'storage_publication_finalization_attention',
'Exhausted owner finalization did not become explicit attention.');
$attentionView = CloudStorageReplicaAttentionService::listView();
$assert(($attentionView['summary']['finalization_count'] ?? 0) === 1
&& count($attentionView['finalization_items'] ?? []) === 1
&& ($attentionView['finalization_items'][0]['operation_kind'] ?? '')
=== 'finalize'
&& ($attentionView['finalization_items'][0]['operation_id'] ?? '')
=== (string) $attention->publication_key
&& ($attentionView['finalization_items'][0]['can_retry'] ?? false),
'Finalization attention is not visible on the administrator recovery page.');
$retry = CloudStorageReplicaAttentionService::retry(
(string) $attention->publication_key,
'finalize'
);
$attention->refresh();
$assert((string) $attention->state === 'ready_to_finalize'
&& (int) $attention->attempts === 0
&& (int) $attention->available_at > 0
&& (string) $attention->last_error_code === ''
&& ($retry['operation_kind'] ?? '') === 'finalize',
'Administrator recovery did not reopen the same publication intent.');
$attention->state = 'completed';
$attention->finalized_at = $clock;
$attention->save();
// Finalization attention has an independent stable paginator. Prove adjacent
// pages are disjoint and together expose every row instead of silently
// truncating after the first page.
$pagedAttention = [];
foreach (['1', '2', '3'] as $ownerDigit) {
$row = $create('template', str_repeat($ownerDigit, 32));
$row->state = 'failed';
$row->attempts = 6;
$row->available_at = 0;
$row->last_error_code =
'storage_publication_finalization_attention';
$row->updated_at = $clock + (int) $ownerDigit;
$row->save();
$pagedAttention[] = $row;
}
$finalizationPageOne = CloudStorageReplicaAttentionService::listView([
'page' => 1,
'finalization_page' => 1,
'per_page' => 2,
]);
$finalizationPageTwo = CloudStorageReplicaAttentionService::listView([
'page' => 1,
'finalization_page' => 2,
'per_page' => 2,
]);
$pageOneIds = array_column(
$finalizationPageOne['finalization_items'],
'operation_id'
);
$pageTwoIds = array_column(
$finalizationPageTwo['finalization_items'],
'operation_id'
);
$assert(count($pageOneIds) === 2 && count($pageTwoIds) === 1
&& array_intersect($pageOneIds, $pageTwoIds) === []
&& count(array_unique(array_merge($pageOneIds, $pageTwoIds))) === 3
&& ($finalizationPageOne['finalization_pagination']['total'] ?? 0) === 3
&& ($finalizationPageOne['finalization_pagination']['total_pages'] ?? 0)
=== 2,
'Finalization attention pagination repeated or omitted an item.');
foreach ($pagedAttention as $row) {
$row->state = 'completed';
$row->finalized_at = $clock;
$row->save();
}
$databaseFault = $create('rescue', str_repeat('4', 32));
$databaseFaultWorker = new CloudStoragePublicationOutboxService(
static function (): void {
throw new PDOException('fixture database outage');
},
static function () use (&$clock): int {
return $clock;
}
);
$databaseFaultRaised = false;
try {
$databaseFaultWorker->tick();
} catch (PDOException) {
$databaseFaultRaised = true;
}
$databaseFault->refresh();
$assert($databaseFaultRaised
&& (string) $databaseFault->state === 'ready_to_finalize'
&& (int) $databaseFault->attempts === 0,
'Database failure consumed the bounded owner-finalization budget.');
$databaseFault->state = 'completed';
$databaseFault->finalized_at = $clock;
$databaseFault->save();
// Illuminate wraps ordinary SQL failures and deadlocks in QueryException.
// In the installed database component it extends PDOException, so the same
// infrastructure path must escape without consuming owner failure attempts.
$wrappedDatabaseFault = $create('template', str_repeat('5', 32));
$wrappedDatabaseFaultWorker = new CloudStoragePublicationOutboxService(
static function (): void {
throw new QueryException(
'default',
'UPDATE cloud_storage_publications SET state = ?',
['completed'],
new PDOException('fixture wrapped database outage')
);
},
static function () use (&$clock): int {
return $clock;
}
);
$wrappedDatabaseFaultRaised = false;
try {
$wrappedDatabaseFaultWorker->tick();
} catch (QueryException) {
$wrappedDatabaseFaultRaised = true;
}
$wrappedDatabaseFault->refresh();
$assert($wrappedDatabaseFaultRaised
&& (string) $wrappedDatabaseFault->state === 'ready_to_finalize'
&& (int) $wrappedDatabaseFault->attempts === 0,
'Wrapped database failure consumed owner-finalization attempts.');
$wrappedDatabaseFault->state = 'completed';
$wrappedDatabaseFault->finalized_at = $clock;
$wrappedDatabaseFault->save();
$wrong = $create('template', str_repeat('e', 32));
$noSettlement = new CloudStoragePublicationOutboxService(
static function (): void {
// A domain consumer returning without completing the outbox is an
// error; it must roll back and enter the same bounded retry path.
},
static function () use (&$clock): int {
return $clock;
}
);
$assert($noSettlement->tick(), 'Missing business settlement was not handled.');
$wrong->refresh();
$assert((string) $wrong->state === 'ready_to_finalize'
&& (int) $wrong->attempts === 1,
'A no-op business consumer was treated as successful.');
$wrong->state = 'completed';
$wrong->finalized_at = $clock;
$wrong->save();
echo "Storage publication outbox behavior passed.\n";