You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
485 lines
20 KiB
PHP
485 lines
20 KiB
PHP
<?php
|
|
declare(strict_types=1);
|
|
|
|
namespace App\Services\Cloud {
|
|
final class CloudSecrets
|
|
{
|
|
public static function encrypt(array $payload, string $purpose): string
|
|
{
|
|
return 'test-sealed:' . $purpose;
|
|
}
|
|
}
|
|
|
|
final class CloudDaemonActor
|
|
{
|
|
public static function resolve(string $sessionId): ?array
|
|
{
|
|
return [
|
|
'user_id' => 1,
|
|
'is_admin' => true,
|
|
'is_super_admin' => true,
|
|
'can_manage' => true,
|
|
'session_id' => $sessionId,
|
|
];
|
|
}
|
|
}
|
|
|
|
final class AuxCatalogWorkerFakeAgent
|
|
{
|
|
public array $requests = [];
|
|
public array $getResponses = [];
|
|
public array $reconcileResponses = [];
|
|
public array $getErrors = [];
|
|
public $onRequest = null;
|
|
|
|
public function request(string $method, string $target, array $actor,
|
|
?array $payload = null): array
|
|
{
|
|
$this->requests[] = compact('method', 'target', 'actor', 'payload');
|
|
if (is_callable($this->onRequest)) {
|
|
($this->onRequest)($method, $target, $actor, $payload);
|
|
}
|
|
if (preg_match('#^/v1/operations/([A-Za-z0-9_-]+)$#D',
|
|
$target, $match) === 1 && $method === 'GET') {
|
|
$operationId = rawurldecode($match[1]);
|
|
if (isset($this->getErrors[$operationId])) {
|
|
throw $this->getErrors[$operationId];
|
|
}
|
|
return ['operation_id' => $operationId]
|
|
+ ($this->getResponses[$operationId]
|
|
?? ['status' => 'running']);
|
|
}
|
|
if (preg_match(
|
|
'#^/v1/operations/([A-Za-z0-9_-]+)/reconcile$#D',
|
|
$target, $match) === 1 && $method === 'POST') {
|
|
$operationId = rawurldecode($match[1]);
|
|
return ['operation_id' => $operationId]
|
|
+ ($this->reconcileResponses[$operationId]
|
|
?? ['status' => 'uncertain']);
|
|
}
|
|
if ($method === 'POST' && $target === '/v1/templates') {
|
|
return [
|
|
'operation_id' => (string) $payload['operation_id'],
|
|
'status' => 'succeeded',
|
|
'result' => [
|
|
'id' => (string) $payload['id'],
|
|
'revision' => (int) $payload['revision'],
|
|
'generation_token' =>
|
|
(string) $payload['generation_token'],
|
|
'size_bytes' => 4096,
|
|
'sha256' => str_repeat('a', 64),
|
|
],
|
|
];
|
|
}
|
|
if ($method === 'POST' && $target === '/v1/user-isos') {
|
|
return [
|
|
'operation_id' => (string) $payload['operation_id'],
|
|
'status' => 'succeeded',
|
|
'result' => [
|
|
'id' => (string) $payload['id'],
|
|
'owner_id' => (int) $payload['owner_id'],
|
|
'size_bytes' => 8192,
|
|
'sha256' => str_repeat('b', 64),
|
|
],
|
|
];
|
|
}
|
|
throw new \RuntimeException('Background worker replayed an import');
|
|
}
|
|
}
|
|
|
|
final class CloudAgentSettingsService
|
|
{
|
|
public static ?AuxCatalogWorkerFakeAgent $client = null;
|
|
public static array $hostIds = [11];
|
|
|
|
public static function enabledHostIds(): array
|
|
{
|
|
return self::$hostIds;
|
|
}
|
|
|
|
public static function clientForHost(int $hostId,
|
|
float $timeoutSeconds = 20.0): AuxCatalogWorkerFakeAgent
|
|
{
|
|
if (!in_array($hostId, self::$hostIds, true)
|
|
|| self::$client === null) {
|
|
throw new \RuntimeException('Missing fake Agent');
|
|
}
|
|
return self::$client;
|
|
}
|
|
}
|
|
}
|
|
|
|
namespace {
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
|
|
|
use App\Models\CloudRescueTemplateImport;
|
|
use App\Models\CloudTemplate;
|
|
use App\Models\CloudTemplateReplica;
|
|
use App\Models\CloudUserIso;
|
|
use App\Models\CloudUserIsoReplica;
|
|
use App\Services\Cloud\AuxCatalogWorkerFakeAgent;
|
|
use App\Services\Cloud\CloudAgentError;
|
|
use App\Services\Cloud\CloudAgentSettingsService;
|
|
use App\Services\Cloud\CloudRescueTemplateImportService;
|
|
use App\Services\Cloud\CloudTemplateCatalogService;
|
|
use App\Services\Cloud\CloudUserIsoService;
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
|
|
$assert = static function (bool $condition, string $message): void {
|
|
if (!$condition) {
|
|
throw new RuntimeException($message);
|
|
}
|
|
};
|
|
|
|
$capsule = new Capsule();
|
|
$capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'],
|
|
'default');
|
|
$capsule->setAsGlobal();
|
|
$capsule->bootEloquent();
|
|
$schema = $capsule->schema('default');
|
|
$schema->create('cloud_templates', static function (Blueprint $table): void {
|
|
$table->string('id')->primary();
|
|
$table->string('name');
|
|
$table->string('os_family');
|
|
$table->string('os_version');
|
|
$table->string('source_type');
|
|
$table->text('sealed_source_url');
|
|
$table->integer('image_bytes')->default(0);
|
|
$table->string('sha256')->default('');
|
|
$table->text('startup_script')->default('');
|
|
$table->integer('revision');
|
|
$table->string('generation_token');
|
|
$table->string('state');
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_template_replicas',
|
|
static function (Blueprint $table): void {
|
|
$table->integer('host_id');
|
|
$table->string('template_id');
|
|
$table->integer('revision');
|
|
$table->string('generation_token');
|
|
$table->string('state');
|
|
$table->integer('image_bytes')->default(0);
|
|
$table->string('sha256')->default('');
|
|
$table->string('operation_id')->default('');
|
|
$table->string('error_code')->default('');
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
$table->primary(['host_id', 'template_id']);
|
|
});
|
|
$schema->create('cloud_rescue_template_imports',
|
|
static function (Blueprint $table): void {
|
|
$table->string('operation_id')->primary();
|
|
$table->integer('host_id');
|
|
$table->string('template_id');
|
|
$table->string('import_type');
|
|
$table->string('state');
|
|
$table->string('error_code')->default('');
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
$table->unique(['host_id', 'template_id']);
|
|
});
|
|
$schema->create('cloud_user_isos', static function (Blueprint $table): void {
|
|
$table->string('id')->primary();
|
|
$table->integer('user_id');
|
|
$table->string('name');
|
|
$table->text('sealed_source_url');
|
|
$table->integer('size_bytes')->default(0);
|
|
$table->string('sha256')->default('');
|
|
$table->string('state');
|
|
$table->string('import_operation_id');
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_user_iso_replicas',
|
|
static function (Blueprint $table): void {
|
|
$table->integer('host_id');
|
|
$table->string('user_iso_id');
|
|
$table->integer('user_id');
|
|
$table->string('state');
|
|
$table->integer('size_bytes')->default(0);
|
|
$table->string('sha256')->default('');
|
|
$table->string('operation_id')->default('');
|
|
$table->string('error_code')->default('');
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
$table->primary(['host_id', 'user_iso_id']);
|
|
});
|
|
|
|
$agent = new AuxCatalogWorkerFakeAgent();
|
|
CloudAgentSettingsService::$client = $agent;
|
|
$now = time();
|
|
|
|
// Both global catalogs must persist their complete target-host snapshot
|
|
// before the first host mutation. A crash at any dispatch boundary then
|
|
// leaves one durable operation per expected host, never a partial set.
|
|
CloudAgentSettingsService::$hostIds = [11, 12];
|
|
$agent->onRequest = static function (string $method, string $target,
|
|
array $actor, ?array $payload) use ($assert): void {
|
|
if ($method !== 'POST') {
|
|
return;
|
|
}
|
|
if ($target === '/v1/templates') {
|
|
$hostIds = CloudTemplateReplica::query()
|
|
->where('template_id', (string) $payload['id'])
|
|
->orderBy('host_id')->pluck('host_id')->map(
|
|
static fn ($value): int => (int) $value
|
|
)->all();
|
|
$assert($hostIds === [11, 12],
|
|
'Template dispatch began before every replica was reserved.');
|
|
}
|
|
if ($target === '/v1/user-isos') {
|
|
$hostIds = CloudUserIsoReplica::query()
|
|
->where('user_iso_id', (string) $payload['id'])
|
|
->orderBy('host_id')->pluck('host_id')->map(
|
|
static fn ($value): int => (int) $value
|
|
)->all();
|
|
$assert($hostIds === [11, 12],
|
|
'ISO dispatch began before every replica was reserved.');
|
|
}
|
|
};
|
|
$templateReservation = CloudTemplateCatalogService::importRemote([
|
|
'name' => 'Debian 13 atomic reservation',
|
|
'url' => 'https://images.example/debian-13.qcow2',
|
|
'sha256' => '',
|
|
], ['user_id' => 1, 'is_admin' => true, 'can_manage' => true]);
|
|
$isoReservation = CloudUserIsoService::import([
|
|
'name' => 'Atomic reservation ISO',
|
|
'url' => 'https://images.example/atomic-reservation.iso',
|
|
], ['user_id' => 90, 'is_admin' => false, 'can_manage' => false]);
|
|
$assert($templateReservation['status'] === 'succeeded'
|
|
&& $isoReservation['status'] === 'succeeded',
|
|
'Atomic catalog reservations did not complete in the test fixture.');
|
|
$agent->onRequest = null;
|
|
$agent->requests = [];
|
|
CloudAgentSettingsService::$hostIds = [11];
|
|
|
|
$rescueOperation = str_repeat('a', 32);
|
|
$freshRescueOperation = str_repeat('b', 32);
|
|
$rescueTemplate = str_repeat('1', 32);
|
|
$freshRescueTemplate = str_repeat('2', 32);
|
|
Capsule::table('cloud_rescue_template_imports')->insert([
|
|
[
|
|
'operation_id' => $rescueOperation,
|
|
'host_id' => 11,
|
|
'template_id' => $rescueTemplate,
|
|
'import_type' => 'remote',
|
|
'state' => 'running',
|
|
'error_code' => '',
|
|
'created_at' => $now - 30,
|
|
'updated_at' => $now - 30,
|
|
],
|
|
[
|
|
'operation_id' => $freshRescueOperation,
|
|
'host_id' => 11,
|
|
'template_id' => $freshRescueTemplate,
|
|
'import_type' => 'remote',
|
|
'state' => 'running',
|
|
'error_code' => '',
|
|
'created_at' => $now,
|
|
'updated_at' => $now,
|
|
],
|
|
]);
|
|
$agent->getResponses[$rescueOperation] = [
|
|
'status' => 'uncertain',
|
|
'error' => ['code' => 'operation_uncertain'],
|
|
];
|
|
$agent->reconcileResponses[$rescueOperation] = [
|
|
'status' => 'succeeded',
|
|
'result' => ['id' => $rescueTemplate],
|
|
];
|
|
|
|
$assert(CloudRescueTemplateImportService::reconcileOnePendingImport(),
|
|
'Rescue import worker did not claim one due operation.');
|
|
$rescue = CloudRescueTemplateImport::find($rescueOperation);
|
|
$freshRescue = CloudRescueTemplateImport::find($freshRescueOperation);
|
|
$assert((string) $rescue->state === 'succeeded'
|
|
&& (string) $freshRescue->state === 'running'
|
|
&& count($agent->requests) === 2
|
|
&& $agent->requests[0]['method'] === 'GET'
|
|
&& $agent->requests[0]['target']
|
|
=== '/v1/operations/' . $rescueOperation
|
|
&& $agent->requests[1]['method'] === 'POST'
|
|
&& $agent->requests[1]['target']
|
|
=== '/v1/operations/' . $rescueOperation . '/reconcile'
|
|
&& ($agent->requests[0]['actor']['session_id'] ?? null)
|
|
=== 'cloud-rescue-template-import-worker'
|
|
&& ($agent->requests[0]['actor']['operation_id'] ?? null)
|
|
=== $rescueOperation,
|
|
'Rescue worker did not preserve the saved operation boundary.');
|
|
$assert(!CloudRescueTemplateImportService::reconcileOnePendingImport(),
|
|
'Fresh rescue import bypassed the five-second poll backoff.');
|
|
|
|
Capsule::table('cloud_rescue_template_imports')
|
|
->where('operation_id', $freshRescueOperation)
|
|
->update(['updated_at' => $now - 30]);
|
|
$agent->getErrors[$freshRescueOperation] = new CloudAgentError(
|
|
'temporary failure', 503, 'agent_transport_error', true, 503
|
|
);
|
|
try {
|
|
CloudRescueTemplateImportService::reconcileOnePendingImport();
|
|
throw new RuntimeException('Rescue transport error was swallowed.');
|
|
} catch (CloudAgentError $error) {
|
|
$assert($error->agentCode === 'agent_transport_error',
|
|
'Rescue worker threw the wrong transport error.');
|
|
}
|
|
$freshRescue->refresh();
|
|
$assert((string) $freshRescue->state === 'running'
|
|
&& (string) $freshRescue->error_code === 'agent_unavailable'
|
|
&& !CloudRescueTemplateImportService::reconcileOnePendingImport(),
|
|
'Rescue transport failure was not persisted with poll backoff.');
|
|
|
|
$insertIso = static function (string $id, string $jobId, int $userId,
|
|
string $state, int $updatedAt): void {
|
|
Capsule::table('cloud_user_isos')->insert([
|
|
'id' => $id,
|
|
'user_id' => $userId,
|
|
'name' => $id,
|
|
'sealed_source_url' => 'test-sealed-source',
|
|
'size_bytes' => 0,
|
|
'sha256' => '',
|
|
'state' => $state,
|
|
'import_operation_id' => $jobId,
|
|
'created_at' => $updatedAt,
|
|
'updated_at' => $updatedAt,
|
|
]);
|
|
};
|
|
$insertIsoReplica = static function (string $id, int $userId,
|
|
string $operationId, string $state, int $updatedAt,
|
|
int $bytes = 0, string $sha256 = ''): void {
|
|
Capsule::table('cloud_user_iso_replicas')->insert([
|
|
'host_id' => 11,
|
|
'user_iso_id' => $id,
|
|
'user_id' => $userId,
|
|
'state' => $state,
|
|
'size_bytes' => $bytes,
|
|
'sha256' => $sha256,
|
|
'operation_id' => $operationId,
|
|
'error_code' => '',
|
|
'created_at' => $updatedAt,
|
|
'updated_at' => $updatedAt,
|
|
]);
|
|
};
|
|
|
|
$isoId = str_repeat('3', 32);
|
|
$freshIsoId = str_repeat('4', 32);
|
|
$healIsoId = str_repeat('5', 32);
|
|
$isoOperation = str_repeat('c', 32);
|
|
$freshIsoOperation = str_repeat('d', 32);
|
|
$digest = str_repeat('e', 64);
|
|
$insertIso($isoId, str_repeat('6', 32), 42, 'importing', $now - 30);
|
|
$insertIsoReplica($isoId, 42, $isoOperation, 'pending', $now - 30);
|
|
$insertIso($freshIsoId, str_repeat('7', 32), 43, 'importing', $now);
|
|
$insertIsoReplica($freshIsoId, 43, $freshIsoOperation, 'pending', $now);
|
|
$insertIso($healIsoId, str_repeat('8', 32), 44, 'importing', $now - 20);
|
|
$insertIsoReplica($healIsoId, 44, str_repeat('f', 32), 'active',
|
|
$now - 20, 4096, $digest);
|
|
Capsule::table('cloud_user_isos')->where('id', $healIsoId)->update([
|
|
'size_bytes' => 4096,
|
|
'sha256' => $digest,
|
|
]);
|
|
$agent->getResponses[$isoOperation] = [
|
|
'status' => 'uncertain',
|
|
'error' => ['code' => 'operation_uncertain'],
|
|
];
|
|
$agent->reconcileResponses[$isoOperation] = [
|
|
'status' => 'succeeded',
|
|
'result' => [
|
|
'id' => $isoId,
|
|
'owner_id' => 42,
|
|
'size_bytes' => 8192,
|
|
'sha256' => $digest,
|
|
],
|
|
];
|
|
$requestsBeforeIso = count($agent->requests);
|
|
$assert(CloudUserIsoService::reconcileOnePendingImport(),
|
|
'ISO worker did not claim one due replica.');
|
|
$iso = CloudUserIso::find($isoId);
|
|
$isoReplica = CloudUserIsoReplica::query()->where('host_id', 11)
|
|
->where('user_iso_id', $isoId)->first();
|
|
$freshIsoReplica = CloudUserIsoReplica::query()->where('host_id', 11)
|
|
->where('user_iso_id', $freshIsoId)->first();
|
|
$isoRequests = array_slice($agent->requests, $requestsBeforeIso);
|
|
$assert((string) $iso->state === 'active'
|
|
&& (string) $isoReplica->state === 'active'
|
|
&& (string) $freshIsoReplica->state === 'pending'
|
|
&& count($isoRequests) === 2
|
|
&& $isoRequests[0]['method'] === 'GET'
|
|
&& $isoRequests[0]['target'] === '/v1/operations/' . $isoOperation
|
|
&& $isoRequests[1]['method'] === 'POST'
|
|
&& $isoRequests[1]['target']
|
|
=== '/v1/operations/' . $isoOperation . '/reconcile'
|
|
&& ($isoRequests[0]['actor']['session_id'] ?? null)
|
|
=== 'cloud-user-iso-import-worker'
|
|
&& ($isoRequests[0]['actor']['operation_id'] ?? null)
|
|
=== $isoOperation,
|
|
'ISO worker did not preserve one saved operation boundary.');
|
|
|
|
$requestsBeforeHeal = count($agent->requests);
|
|
$assert(CloudUserIsoService::reconcileOnePendingImport()
|
|
&& (string) CloudUserIso::find($healIsoId)->state === 'active'
|
|
&& count($agent->requests) === $requestsBeforeHeal,
|
|
'ISO worker did not repair an all-active parent locally.');
|
|
$assert(!CloudUserIsoService::reconcileOnePendingImport(),
|
|
'Fresh ISO replica bypassed the five-second poll backoff.');
|
|
|
|
Capsule::table('cloud_user_iso_replicas')
|
|
->where('user_iso_id', $freshIsoId)
|
|
->update(['updated_at' => $now - 30]);
|
|
$agent->getErrors[$freshIsoOperation] = new CloudAgentError(
|
|
'temporary failure', 503, 'agent_transport_error', true, 503
|
|
);
|
|
try {
|
|
CloudUserIsoService::reconcileOnePendingImport();
|
|
throw new RuntimeException('ISO transport error was swallowed.');
|
|
} catch (CloudAgentError $error) {
|
|
$assert($error->agentCode === 'agent_transport_error',
|
|
'ISO worker threw the wrong transport error.');
|
|
}
|
|
$freshIso = CloudUserIso::find($freshIsoId);
|
|
$freshIsoReplica->refresh();
|
|
$assert((string) $freshIso->state === 'importing'
|
|
&& (string) $freshIsoReplica->state === 'pending'
|
|
&& (string) $freshIsoReplica->error_code === 'agent_unavailable'
|
|
&& !CloudUserIsoService::reconcileOnePendingImport(),
|
|
'ISO transport failure was not persisted with poll backoff.');
|
|
|
|
$orphanTemplateId = 'debian-orphan-parent';
|
|
Capsule::table('cloud_templates')->insert([
|
|
'id' => $orphanTemplateId,
|
|
'name' => 'Orphan template',
|
|
'os_family' => 'Debian',
|
|
'os_version' => '13',
|
|
'source_type' => 'remote',
|
|
'sealed_source_url' => 'test-sealed:template-source',
|
|
'image_bytes' => 0,
|
|
'sha256' => '',
|
|
'startup_script' => '',
|
|
'revision' => 1,
|
|
'generation_token' => str_repeat('0', 32),
|
|
'state' => 'importing',
|
|
'created_at' => $now - 180,
|
|
'updated_at' => $now - 180,
|
|
]);
|
|
$orphanIsoId = str_repeat('9', 32);
|
|
$insertIso($orphanIsoId, str_repeat('0', 32), 45, 'importing',
|
|
$now - 180);
|
|
$requestsBeforeOrphans = count($agent->requests);
|
|
$assert(CloudTemplateCatalogService::reconcileOnePendingImport()
|
|
&& (string) CloudTemplate::find($orphanTemplateId)->state === 'error'
|
|
&& CloudUserIsoService::reconcileOnePendingImport()
|
|
&& (string) CloudUserIso::find($orphanIsoId)->state === 'failed'
|
|
&& count($agent->requests) === $requestsBeforeOrphans,
|
|
'Replica-free legacy parents were not failed without replay.');
|
|
|
|
foreach ($agent->requests as $request) {
|
|
$assert(!in_array($request['target'],
|
|
['/v1/rescue-templates', '/v1/user-isos'], true),
|
|
'Background convergence replayed an original import POST.');
|
|
}
|
|
|
|
echo "Rescue and user ISO background import convergence passed.\n";
|
|
}
|