You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudAuxCatalogWorkerConver...

485 lines
20 KiB
PHP

<?php
declare(strict_types=1);
namespace App\Services\Cloud {
final class CloudSecrets
{
public static function encrypt(array $payload, string $purpose): string
{
return 'test-sealed:' . $purpose;
}
}
final class CloudDaemonActor
{
public static function resolve(string $sessionId): ?array
{
return [
'user_id' => 1,
'is_admin' => true,
'is_super_admin' => true,
'can_manage' => true,
'session_id' => $sessionId,
];
}
}
final class AuxCatalogWorkerFakeAgent
{
public array $requests = [];
public array $getResponses = [];
public array $reconcileResponses = [];
public array $getErrors = [];
public $onRequest = null;
public function request(string $method, string $target, array $actor,
?array $payload = null): array
{
$this->requests[] = compact('method', 'target', 'actor', 'payload');
if (is_callable($this->onRequest)) {
($this->onRequest)($method, $target, $actor, $payload);
}
if (preg_match('#^/v1/operations/([A-Za-z0-9_-]+)$#D',
$target, $match) === 1 && $method === 'GET') {
$operationId = rawurldecode($match[1]);
if (isset($this->getErrors[$operationId])) {
throw $this->getErrors[$operationId];
}
return ['operation_id' => $operationId]
+ ($this->getResponses[$operationId]
?? ['status' => 'running']);
}
if (preg_match(
'#^/v1/operations/([A-Za-z0-9_-]+)/reconcile$#D',
$target, $match) === 1 && $method === 'POST') {
$operationId = rawurldecode($match[1]);
return ['operation_id' => $operationId]
+ ($this->reconcileResponses[$operationId]
?? ['status' => 'uncertain']);
}
if ($method === 'POST' && $target === '/v1/templates') {
return [
'operation_id' => (string) $payload['operation_id'],
'status' => 'succeeded',
'result' => [
'id' => (string) $payload['id'],
'revision' => (int) $payload['revision'],
'generation_token' =>
(string) $payload['generation_token'],
'size_bytes' => 4096,
'sha256' => str_repeat('a', 64),
],
];
}
if ($method === 'POST' && $target === '/v1/user-isos') {
return [
'operation_id' => (string) $payload['operation_id'],
'status' => 'succeeded',
'result' => [
'id' => (string) $payload['id'],
'owner_id' => (int) $payload['owner_id'],
'size_bytes' => 8192,
'sha256' => str_repeat('b', 64),
],
];
}
throw new \RuntimeException('Background worker replayed an import');
}
}
final class CloudAgentSettingsService
{
public static ?AuxCatalogWorkerFakeAgent $client = null;
public static array $hostIds = [11];
public static function enabledHostIds(): array
{
return self::$hostIds;
}
public static function clientForHost(int $hostId,
float $timeoutSeconds = 20.0): AuxCatalogWorkerFakeAgent
{
if (!in_array($hostId, self::$hostIds, true)
|| self::$client === null) {
throw new \RuntimeException('Missing fake Agent');
}
return self::$client;
}
}
}
namespace {
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudRescueTemplateImport;
use App\Models\CloudTemplate;
use App\Models\CloudTemplateReplica;
use App\Models\CloudUserIso;
use App\Models\CloudUserIsoReplica;
use App\Services\Cloud\AuxCatalogWorkerFakeAgent;
use App\Services\Cloud\CloudAgentError;
use App\Services\Cloud\CloudAgentSettingsService;
use App\Services\Cloud\CloudRescueTemplateImportService;
use App\Services\Cloud\CloudTemplateCatalogService;
use App\Services\Cloud\CloudUserIsoService;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$capsule = new Capsule();
$capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'],
'default');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$schema = $capsule->schema('default');
$schema->create('cloud_templates', static function (Blueprint $table): void {
$table->string('id')->primary();
$table->string('name');
$table->string('os_family');
$table->string('os_version');
$table->string('source_type');
$table->text('sealed_source_url');
$table->integer('image_bytes')->default(0);
$table->string('sha256')->default('');
$table->text('startup_script')->default('');
$table->integer('revision');
$table->string('generation_token');
$table->string('state');
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_template_replicas',
static function (Blueprint $table): void {
$table->integer('host_id');
$table->string('template_id');
$table->integer('revision');
$table->string('generation_token');
$table->string('state');
$table->integer('image_bytes')->default(0);
$table->string('sha256')->default('');
$table->string('operation_id')->default('');
$table->string('error_code')->default('');
$table->integer('created_at');
$table->integer('updated_at');
$table->primary(['host_id', 'template_id']);
});
$schema->create('cloud_rescue_template_imports',
static function (Blueprint $table): void {
$table->string('operation_id')->primary();
$table->integer('host_id');
$table->string('template_id');
$table->string('import_type');
$table->string('state');
$table->string('error_code')->default('');
$table->integer('created_at');
$table->integer('updated_at');
$table->unique(['host_id', 'template_id']);
});
$schema->create('cloud_user_isos', static function (Blueprint $table): void {
$table->string('id')->primary();
$table->integer('user_id');
$table->string('name');
$table->text('sealed_source_url');
$table->integer('size_bytes')->default(0);
$table->string('sha256')->default('');
$table->string('state');
$table->string('import_operation_id');
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_user_iso_replicas',
static function (Blueprint $table): void {
$table->integer('host_id');
$table->string('user_iso_id');
$table->integer('user_id');
$table->string('state');
$table->integer('size_bytes')->default(0);
$table->string('sha256')->default('');
$table->string('operation_id')->default('');
$table->string('error_code')->default('');
$table->integer('created_at');
$table->integer('updated_at');
$table->primary(['host_id', 'user_iso_id']);
});
$agent = new AuxCatalogWorkerFakeAgent();
CloudAgentSettingsService::$client = $agent;
$now = time();
// Both global catalogs must persist their complete target-host snapshot
// before the first host mutation. A crash at any dispatch boundary then
// leaves one durable operation per expected host, never a partial set.
CloudAgentSettingsService::$hostIds = [11, 12];
$agent->onRequest = static function (string $method, string $target,
array $actor, ?array $payload) use ($assert): void {
if ($method !== 'POST') {
return;
}
if ($target === '/v1/templates') {
$hostIds = CloudTemplateReplica::query()
->where('template_id', (string) $payload['id'])
->orderBy('host_id')->pluck('host_id')->map(
static fn ($value): int => (int) $value
)->all();
$assert($hostIds === [11, 12],
'Template dispatch began before every replica was reserved.');
}
if ($target === '/v1/user-isos') {
$hostIds = CloudUserIsoReplica::query()
->where('user_iso_id', (string) $payload['id'])
->orderBy('host_id')->pluck('host_id')->map(
static fn ($value): int => (int) $value
)->all();
$assert($hostIds === [11, 12],
'ISO dispatch began before every replica was reserved.');
}
};
$templateReservation = CloudTemplateCatalogService::importRemote([
'name' => 'Debian 13 atomic reservation',
'url' => 'https://images.example/debian-13.qcow2',
'sha256' => '',
], ['user_id' => 1, 'is_admin' => true, 'can_manage' => true]);
$isoReservation = CloudUserIsoService::import([
'name' => 'Atomic reservation ISO',
'url' => 'https://images.example/atomic-reservation.iso',
], ['user_id' => 90, 'is_admin' => false, 'can_manage' => false]);
$assert($templateReservation['status'] === 'succeeded'
&& $isoReservation['status'] === 'succeeded',
'Atomic catalog reservations did not complete in the test fixture.');
$agent->onRequest = null;
$agent->requests = [];
CloudAgentSettingsService::$hostIds = [11];
$rescueOperation = str_repeat('a', 32);
$freshRescueOperation = str_repeat('b', 32);
$rescueTemplate = str_repeat('1', 32);
$freshRescueTemplate = str_repeat('2', 32);
Capsule::table('cloud_rescue_template_imports')->insert([
[
'operation_id' => $rescueOperation,
'host_id' => 11,
'template_id' => $rescueTemplate,
'import_type' => 'remote',
'state' => 'running',
'error_code' => '',
'created_at' => $now - 30,
'updated_at' => $now - 30,
],
[
'operation_id' => $freshRescueOperation,
'host_id' => 11,
'template_id' => $freshRescueTemplate,
'import_type' => 'remote',
'state' => 'running',
'error_code' => '',
'created_at' => $now,
'updated_at' => $now,
],
]);
$agent->getResponses[$rescueOperation] = [
'status' => 'uncertain',
'error' => ['code' => 'operation_uncertain'],
];
$agent->reconcileResponses[$rescueOperation] = [
'status' => 'succeeded',
'result' => ['id' => $rescueTemplate],
];
$assert(CloudRescueTemplateImportService::reconcileOnePendingImport(),
'Rescue import worker did not claim one due operation.');
$rescue = CloudRescueTemplateImport::find($rescueOperation);
$freshRescue = CloudRescueTemplateImport::find($freshRescueOperation);
$assert((string) $rescue->state === 'succeeded'
&& (string) $freshRescue->state === 'running'
&& count($agent->requests) === 2
&& $agent->requests[0]['method'] === 'GET'
&& $agent->requests[0]['target']
=== '/v1/operations/' . $rescueOperation
&& $agent->requests[1]['method'] === 'POST'
&& $agent->requests[1]['target']
=== '/v1/operations/' . $rescueOperation . '/reconcile'
&& ($agent->requests[0]['actor']['session_id'] ?? null)
=== 'cloud-rescue-template-import-worker'
&& ($agent->requests[0]['actor']['operation_id'] ?? null)
=== $rescueOperation,
'Rescue worker did not preserve the saved operation boundary.');
$assert(!CloudRescueTemplateImportService::reconcileOnePendingImport(),
'Fresh rescue import bypassed the five-second poll backoff.');
Capsule::table('cloud_rescue_template_imports')
->where('operation_id', $freshRescueOperation)
->update(['updated_at' => $now - 30]);
$agent->getErrors[$freshRescueOperation] = new CloudAgentError(
'temporary failure', 503, 'agent_transport_error', true, 503
);
try {
CloudRescueTemplateImportService::reconcileOnePendingImport();
throw new RuntimeException('Rescue transport error was swallowed.');
} catch (CloudAgentError $error) {
$assert($error->agentCode === 'agent_transport_error',
'Rescue worker threw the wrong transport error.');
}
$freshRescue->refresh();
$assert((string) $freshRescue->state === 'running'
&& (string) $freshRescue->error_code === 'agent_unavailable'
&& !CloudRescueTemplateImportService::reconcileOnePendingImport(),
'Rescue transport failure was not persisted with poll backoff.');
$insertIso = static function (string $id, string $jobId, int $userId,
string $state, int $updatedAt): void {
Capsule::table('cloud_user_isos')->insert([
'id' => $id,
'user_id' => $userId,
'name' => $id,
'sealed_source_url' => 'test-sealed-source',
'size_bytes' => 0,
'sha256' => '',
'state' => $state,
'import_operation_id' => $jobId,
'created_at' => $updatedAt,
'updated_at' => $updatedAt,
]);
};
$insertIsoReplica = static function (string $id, int $userId,
string $operationId, string $state, int $updatedAt,
int $bytes = 0, string $sha256 = ''): void {
Capsule::table('cloud_user_iso_replicas')->insert([
'host_id' => 11,
'user_iso_id' => $id,
'user_id' => $userId,
'state' => $state,
'size_bytes' => $bytes,
'sha256' => $sha256,
'operation_id' => $operationId,
'error_code' => '',
'created_at' => $updatedAt,
'updated_at' => $updatedAt,
]);
};
$isoId = str_repeat('3', 32);
$freshIsoId = str_repeat('4', 32);
$healIsoId = str_repeat('5', 32);
$isoOperation = str_repeat('c', 32);
$freshIsoOperation = str_repeat('d', 32);
$digest = str_repeat('e', 64);
$insertIso($isoId, str_repeat('6', 32), 42, 'importing', $now - 30);
$insertIsoReplica($isoId, 42, $isoOperation, 'pending', $now - 30);
$insertIso($freshIsoId, str_repeat('7', 32), 43, 'importing', $now);
$insertIsoReplica($freshIsoId, 43, $freshIsoOperation, 'pending', $now);
$insertIso($healIsoId, str_repeat('8', 32), 44, 'importing', $now - 20);
$insertIsoReplica($healIsoId, 44, str_repeat('f', 32), 'active',
$now - 20, 4096, $digest);
Capsule::table('cloud_user_isos')->where('id', $healIsoId)->update([
'size_bytes' => 4096,
'sha256' => $digest,
]);
$agent->getResponses[$isoOperation] = [
'status' => 'uncertain',
'error' => ['code' => 'operation_uncertain'],
];
$agent->reconcileResponses[$isoOperation] = [
'status' => 'succeeded',
'result' => [
'id' => $isoId,
'owner_id' => 42,
'size_bytes' => 8192,
'sha256' => $digest,
],
];
$requestsBeforeIso = count($agent->requests);
$assert(CloudUserIsoService::reconcileOnePendingImport(),
'ISO worker did not claim one due replica.');
$iso = CloudUserIso::find($isoId);
$isoReplica = CloudUserIsoReplica::query()->where('host_id', 11)
->where('user_iso_id', $isoId)->first();
$freshIsoReplica = CloudUserIsoReplica::query()->where('host_id', 11)
->where('user_iso_id', $freshIsoId)->first();
$isoRequests = array_slice($agent->requests, $requestsBeforeIso);
$assert((string) $iso->state === 'active'
&& (string) $isoReplica->state === 'active'
&& (string) $freshIsoReplica->state === 'pending'
&& count($isoRequests) === 2
&& $isoRequests[0]['method'] === 'GET'
&& $isoRequests[0]['target'] === '/v1/operations/' . $isoOperation
&& $isoRequests[1]['method'] === 'POST'
&& $isoRequests[1]['target']
=== '/v1/operations/' . $isoOperation . '/reconcile'
&& ($isoRequests[0]['actor']['session_id'] ?? null)
=== 'cloud-user-iso-import-worker'
&& ($isoRequests[0]['actor']['operation_id'] ?? null)
=== $isoOperation,
'ISO worker did not preserve one saved operation boundary.');
$requestsBeforeHeal = count($agent->requests);
$assert(CloudUserIsoService::reconcileOnePendingImport()
&& (string) CloudUserIso::find($healIsoId)->state === 'active'
&& count($agent->requests) === $requestsBeforeHeal,
'ISO worker did not repair an all-active parent locally.');
$assert(!CloudUserIsoService::reconcileOnePendingImport(),
'Fresh ISO replica bypassed the five-second poll backoff.');
Capsule::table('cloud_user_iso_replicas')
->where('user_iso_id', $freshIsoId)
->update(['updated_at' => $now - 30]);
$agent->getErrors[$freshIsoOperation] = new CloudAgentError(
'temporary failure', 503, 'agent_transport_error', true, 503
);
try {
CloudUserIsoService::reconcileOnePendingImport();
throw new RuntimeException('ISO transport error was swallowed.');
} catch (CloudAgentError $error) {
$assert($error->agentCode === 'agent_transport_error',
'ISO worker threw the wrong transport error.');
}
$freshIso = CloudUserIso::find($freshIsoId);
$freshIsoReplica->refresh();
$assert((string) $freshIso->state === 'importing'
&& (string) $freshIsoReplica->state === 'pending'
&& (string) $freshIsoReplica->error_code === 'agent_unavailable'
&& !CloudUserIsoService::reconcileOnePendingImport(),
'ISO transport failure was not persisted with poll backoff.');
$orphanTemplateId = 'debian-orphan-parent';
Capsule::table('cloud_templates')->insert([
'id' => $orphanTemplateId,
'name' => 'Orphan template',
'os_family' => 'Debian',
'os_version' => '13',
'source_type' => 'remote',
'sealed_source_url' => 'test-sealed:template-source',
'image_bytes' => 0,
'sha256' => '',
'startup_script' => '',
'revision' => 1,
'generation_token' => str_repeat('0', 32),
'state' => 'importing',
'created_at' => $now - 180,
'updated_at' => $now - 180,
]);
$orphanIsoId = str_repeat('9', 32);
$insertIso($orphanIsoId, str_repeat('0', 32), 45, 'importing',
$now - 180);
$requestsBeforeOrphans = count($agent->requests);
$assert(CloudTemplateCatalogService::reconcileOnePendingImport()
&& (string) CloudTemplate::find($orphanTemplateId)->state === 'error'
&& CloudUserIsoService::reconcileOnePendingImport()
&& (string) CloudUserIso::find($orphanIsoId)->state === 'failed'
&& count($agent->requests) === $requestsBeforeOrphans,
'Replica-free legacy parents were not failed without replay.');
foreach ($agent->requests as $request) {
$assert(!in_array($request['target'],
['/v1/rescue-templates', '/v1/user-isos'], true),
'Background convergence replayed an original import POST.');
}
echo "Rescue and user ISO background import convergence passed.\n";
}