You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
211 lines
7.7 KiB
PHP
211 lines
7.7 KiB
PHP
<?php
|
|
declare(strict_types=1);
|
|
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
|
|
|
use App\Services\Cloud\CloudServiceCompensationExecutionService;
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
|
|
$assert = static function (bool $condition, string $message): void {
|
|
if (!$condition) {
|
|
throw new RuntimeException($message);
|
|
}
|
|
};
|
|
|
|
$capsule = new Capsule();
|
|
$capsule->addConnection([
|
|
'driver' => 'sqlite',
|
|
'database' => ':memory:',
|
|
], 'default');
|
|
$capsule->setAsGlobal();
|
|
$capsule->bootEloquent();
|
|
$schema = $capsule->schema('default');
|
|
|
|
$schema->create(
|
|
'cloud_service_compensations',
|
|
static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->integer('instance_count');
|
|
$table->boolean('notify_users')->default(false);
|
|
$table->integer('recipient_count')->default(0);
|
|
$table->integer('applied_count')->default(0);
|
|
$table->integer('deferred_count')->default(0);
|
|
$table->integer('rejected_count')->default(0);
|
|
$table->integer('attention_count')->default(0);
|
|
$table->string('state');
|
|
$table->string('last_error_code')->default('');
|
|
$table->integer('terminal_at')->nullable();
|
|
$table->integer('updated_at');
|
|
}
|
|
);
|
|
$schema->create(
|
|
'cloud_service_compensation_items',
|
|
static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('item_key');
|
|
$table->integer('batch_id');
|
|
$table->integer('instance_id');
|
|
$table->string('state');
|
|
$table->integer('attempts')->default(0);
|
|
$table->integer('available_at');
|
|
$table->string('lease_token')->nullable();
|
|
$table->integer('lease_until')->default(0);
|
|
$table->integer('audit_job_id')->nullable();
|
|
$table->integer('before_expires_at')->nullable();
|
|
$table->integer('after_expires_at')->nullable();
|
|
$table->integer('applied_at')->nullable();
|
|
$table->integer('terminal_at')->nullable();
|
|
$table->string('last_error_code')->default('');
|
|
$table->string('last_error')->default('');
|
|
$table->integer('updated_at');
|
|
}
|
|
);
|
|
|
|
$now = 2_100_000_000;
|
|
$service = new CloudServiceCompensationExecutionService(
|
|
static function () use (&$now): int {
|
|
return $now;
|
|
}
|
|
);
|
|
$batch = static function (int $id, int $at): array {
|
|
return [
|
|
'id' => $id,
|
|
'instance_count' => 1,
|
|
'notify_users' => 0,
|
|
'recipient_count' => 0,
|
|
'applied_count' => 0,
|
|
'deferred_count' => 0,
|
|
'rejected_count' => 0,
|
|
'attention_count' => 0,
|
|
'state' => 'pending',
|
|
'last_error_code' => '',
|
|
'terminal_at' => null,
|
|
'updated_at' => $at,
|
|
];
|
|
};
|
|
$item = static function (
|
|
int $id,
|
|
int $batchId,
|
|
int $attempts,
|
|
int $at
|
|
): array {
|
|
return [
|
|
'id' => $id,
|
|
'item_key' => str_pad((string) $id, 32, 'a'),
|
|
'batch_id' => $batchId,
|
|
'instance_id' => 100 + $id,
|
|
'state' => 'pending',
|
|
'attempts' => $attempts,
|
|
'available_at' => $at,
|
|
'lease_token' => null,
|
|
'lease_until' => 0,
|
|
'audit_job_id' => null,
|
|
'before_expires_at' => null,
|
|
'after_expires_at' => null,
|
|
'applied_at' => null,
|
|
'terminal_at' => null,
|
|
'last_error_code' => '',
|
|
'last_error' => '',
|
|
'updated_at' => $at,
|
|
];
|
|
};
|
|
|
|
Capsule::table('cloud_service_compensations')->insert($batch(1, $now));
|
|
Capsule::table('cloud_service_compensation_items')->insert(
|
|
$item(1, 1, 0, $now)
|
|
);
|
|
|
|
$claim = $service->claimOne(1);
|
|
$assert(is_array($claim), 'due item should be claimed');
|
|
$assert((int) $claim['item_id'] === 1, 'claim should identify the item');
|
|
$assert((int) $claim['lease_until'] === $now + 120,
|
|
'claim should create exactly a 120 second lease');
|
|
$assert(preg_match('/^[a-f0-9]{32}$/D', (string) $claim['token']) === 1,
|
|
'claim should return a strong lease token');
|
|
$claimed = Capsule::table('cloud_service_compensation_items')
|
|
->where('id', 1)->first();
|
|
$assert((string) $claimed->state === 'applying'
|
|
&& (int) $claimed->attempts === 1,
|
|
'claim should enter applying and increment attempts once');
|
|
$assert((string) Capsule::table('cloud_service_compensations')
|
|
->where('id', 1)->value('state') === 'applying',
|
|
'claim should project an applying batch');
|
|
|
|
$now += 119;
|
|
$assert($service->recoverExpired(1) === 0,
|
|
'a live lease must not be recovered early');
|
|
$now += 1;
|
|
$assert($service->recoverExpired(1) === 1,
|
|
'an expired lease should be recovered once');
|
|
$recovered = Capsule::table('cloud_service_compensation_items')
|
|
->where('id', 1)->first();
|
|
$assert((string) $recovered->state === 'deferred',
|
|
'first lost lease should defer safely');
|
|
$assert((int) $recovered->available_at === $now + 30,
|
|
'first retry should use a 30 second backoff');
|
|
$assert($recovered->lease_token === null
|
|
&& (int) $recovered->lease_until === 0,
|
|
'recovery should clear the abandoned lease');
|
|
$assert((string) $recovered->last_error_code === 'worker_lease_expired',
|
|
'recovery should preserve a stable diagnostic code');
|
|
$batchOne = Capsule::table('cloud_service_compensations')
|
|
->where('id', 1)->first();
|
|
$assert((string) $batchOne->state === 'applying'
|
|
&& (int) $batchOne->deferred_count === 1
|
|
&& $batchOne->terminal_at === null,
|
|
'a deferred item should keep the batch nonterminal and counted');
|
|
$assert($service->claimOne(1) === null,
|
|
'a deferred item must remain unavailable during backoff');
|
|
|
|
Capsule::table('cloud_service_compensations')->insert($batch(2, $now));
|
|
Capsule::table('cloud_service_compensation_items')->insert(
|
|
$item(2, 2, 7, $now)
|
|
);
|
|
$lastClaim = $service->claimOne(2);
|
|
$assert(is_array($lastClaim), 'seventh-attempt item should receive final claim');
|
|
$attemptEight = Capsule::table('cloud_service_compensation_items')
|
|
->where('id', 2)->first();
|
|
$assert((int) $attemptEight->attempts === 8
|
|
&& (string) $attemptEight->state === 'applying',
|
|
'final claim should be attempt eight');
|
|
$now += 120;
|
|
$assert($service->recoverExpired(2) === 1,
|
|
'the final abandoned lease should converge');
|
|
$terminal = Capsule::table('cloud_service_compensation_items')
|
|
->where('id', 2)->first();
|
|
$assert((string) $terminal->state === 'attention'
|
|
&& (string) $terminal->last_error_code === 'retry_exhausted'
|
|
&& (int) $terminal->terminal_at === $now,
|
|
'attempt eight should become terminal attention');
|
|
$batchTwo = Capsule::table('cloud_service_compensations')
|
|
->where('id', 2)->first();
|
|
$assert((string) $batchTwo->state === 'attention'
|
|
&& (int) $batchTwo->attention_count === 1
|
|
&& (int) $batchTwo->terminal_at === $now,
|
|
'terminal item should project batch attention');
|
|
$assert($service->recoverExpired(2) === 0,
|
|
'terminal attention must not be recovered again');
|
|
|
|
// A corrupt due item under a terminal batch must not starve later healthy
|
|
// batches in the global queue.
|
|
Capsule::table('cloud_service_compensation_items')->delete();
|
|
Capsule::table('cloud_service_compensations')->delete();
|
|
$terminalBatch = $batch(3, $now);
|
|
$terminalBatch['state'] = 'attention';
|
|
$terminalBatch['attention_count'] = 1;
|
|
$terminalBatch['terminal_at'] = $now;
|
|
Capsule::table('cloud_service_compensations')->insert($terminalBatch);
|
|
Capsule::table('cloud_service_compensation_items')->insert(
|
|
$item(3, 3, 0, $now)
|
|
);
|
|
Capsule::table('cloud_service_compensations')->insert($batch(4, $now));
|
|
Capsule::table('cloud_service_compensation_items')->insert(
|
|
$item(4, 4, 0, $now)
|
|
);
|
|
$healthyClaim = $service->claimOne();
|
|
$assert(is_array($healthyClaim) && (int) $healthyClaim['item_id'] === 4,
|
|
'a terminal parent batch starved a later healthy queue item');
|
|
|
|
echo "Service compensation execution lease tests passed.\n";
|