You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudWorkerCatalogFairnessT...

214 lines
7.6 KiB
PHP

<?php
declare(strict_types=1);
namespace App\Services\Cloud {
// Catalog doubles make the worker scheduling contract observable without
// loading any production catalog implementation in this isolated process.
final class CloudTemplateCatalogService
{
public static int $calls = 0;
public static bool $fail = false;
public static function reconcileOnePendingImport(): bool
{
self::$calls++;
if (self::$fail) {
throw new \RuntimeException('template catalog unavailable');
}
return true;
}
}
final class CloudRescueTemplateImportService
{
public static int $calls = 0;
public static function reconcileOnePendingImport(): bool
{
self::$calls++;
return true;
}
}
final class CloudUserIsoService
{
public static int $calls = 0;
public static function reconcileOnePendingImport(): bool
{
self::$calls++;
return true;
}
}
// Test-only in-memory envelope storage; production CloudSecrets is not
// loaded in this process.
final class CloudSecrets
{
private static array $values = [];
public static function encrypt(array $value, string $purpose): string
{
$key = bin2hex(random_bytes(16));
self::$values[$key] = $value;
return 'worker-fairness:' . $key;
}
public static function decrypt(string $value, string $purpose): array
{
$key = substr($value, strlen('worker-fairness:'));
if (!isset(self::$values[$key])) {
throw new \RuntimeException('missing test payload');
}
return self::$values[$key];
}
}
}
namespace {
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudJob;
use App\Services\Cloud\CloudAgentClient;
use App\Services\Cloud\CloudRescueTemplateImportService;
use App\Services\Cloud\CloudSecrets;
use App\Services\Cloud\CloudTemplateCatalogService;
use App\Services\Cloud\CloudUserIsoService;
use App\Services\Cloud\CloudWorkerService;
use GuzzleHttp\Client;
use GuzzleHttp\Handler\MockHandler;
use GuzzleHttp\HandlerStack;
use GuzzleHttp\Psr7\Response;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$capsule = new Capsule();
$capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'],
'default');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$schema = $capsule->schema('default');
$schema->create('user', static function (Blueprint $table): void {
$table->increments('id');
$table->integer('enable')->default(1);
$table->integer('is_admin')->default(0);
});
$schema->create('cloud_jobs', static function (Blueprint $table): void {
$table->increments('id');
$table->string('operation_id')->unique();
$table->integer('order_id')->nullable();
$table->integer('instance_id')->nullable();
$table->integer('snapshot_id')->nullable();
$table->integer('user_id');
$table->integer('host_id');
$table->string('kind');
$table->string('status');
$table->text('sealed_payload');
$table->integer('attempts')->default(0);
$table->integer('dispatched_at')->nullable();
$table->string('lease_token')->nullable();
$table->integer('lease_until')->default(0);
$table->integer('available_at');
$table->string('error_code')->nullable();
$table->text('result_json')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
Capsule::table('user')->insert(['id' => 1, 'enable' => 1, 'is_admin' => 0]);
$responses = [];
$now = time();
for ($number = 1; $number <= 3; $number++) {
$operationId = str_repeat((string) $number, 32);
Capsule::table('cloud_jobs')->insert([
'operation_id' => $operationId,
'order_id' => null,
'instance_id' => null,
'snapshot_id' => null,
'user_id' => 1,
'host_id' => 7,
'kind' => 'action',
'status' => 'queued',
'sealed_payload' => CloudSecrets::encrypt([
'method' => 'POST',
'request_target' => '/v1/vms/test-vm/actions',
'body' => [
'operation_id' => $operationId,
'action' => 'start',
],
'owner_id' => 1,
], 'job-payload'),
'attempts' => 0,
'dispatched_at' => null,
'lease_token' => null,
'lease_until' => 0,
'available_at' => 0,
'error_code' => null,
'result_json' => null,
'created_at' => $now,
'updated_at' => $now,
]);
$responses[] = new Response(200, [], json_encode([
'operation_id' => $operationId,
'status' => 'dispatching',
], JSON_THROW_ON_ERROR));
}
$client = new CloudAgentClient(
'7',
'test-agent-secret-at-least-32-bytes',
new Client(['handler' => HandlerStack::create(new MockHandler($responses))]),
20.0,
'http://127.0.0.1:8001/internal/agent/request',
'test-gateway-secret-at-least-32-bytes'
);
$worker = new CloudWorkerService($client);
for ($tick = 1; $tick <= 3; $tick++) {
$assert($worker->tick(), 'A due cloud job was not processed.');
$assert(CloudJob::query()->where('status', 'running')->count() === $tick,
'The persistent cloud_jobs backlog did not advance one row.');
$assert(CloudTemplateCatalogService::$calls
+ CloudRescueTemplateImportService::$calls
+ CloudUserIsoService::$calls === $tick,
'One worker tick ran more than one catalog action.');
}
$assert(CloudTemplateCatalogService::$calls === 1
&& CloudRescueTemplateImportService::$calls === 1
&& CloudUserIsoService::$calls === 1,
'A continuously non-empty cloud_jobs queue starved a catalog.');
// A catalog exception is observable but is not work. Returning false keeps
// the command loop's one-second idle backoff and prevents a log/DB hot loop.
CloudTemplateCatalogService::$fail = true;
$assert($worker->tick() === false,
'A catalog exception incorrectly reported successful work.');
$assert($worker->observationErrors() === ['worker_tick_failed'],
'A catalog exception was not exposed through worker health.');
$assert(CloudTemplateCatalogService::$calls === 2
&& CloudRescueTemplateImportService::$calls === 1
&& CloudUserIsoService::$calls === 1,
'The worker ran a second catalog after an ambiguous catalog failure.');
$beforePolicy = [
CloudTemplateCatalogService::$calls,
CloudRescueTemplateImportService::$calls,
CloudUserIsoService::$calls,
];
$policyWorker = new CloudWorkerService(null, true);
$assert($policyWorker->tick() === false
&& $beforePolicy === [
CloudTemplateCatalogService::$calls,
CloudRescueTemplateImportService::$calls,
CloudUserIsoService::$calls,
], 'The policy-only worker was delayed by catalog convergence.');
echo "Cloud worker catalog fairness and failure backoff tests passed.\n";
}