fix: converge persistent catalog imports
parent
e7fa6256c2
commit
fbe8498deb
@ -0,0 +1,484 @@
|
||||
<?php
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Services\Cloud {
|
||||
final class CloudSecrets
|
||||
{
|
||||
public static function encrypt(array $payload, string $purpose): string
|
||||
{
|
||||
return 'test-sealed:' . $purpose;
|
||||
}
|
||||
}
|
||||
|
||||
final class CloudDaemonActor
|
||||
{
|
||||
public static function resolve(string $sessionId): ?array
|
||||
{
|
||||
return [
|
||||
'user_id' => 1,
|
||||
'is_admin' => true,
|
||||
'is_super_admin' => true,
|
||||
'can_manage' => true,
|
||||
'session_id' => $sessionId,
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
final class AuxCatalogWorkerFakeAgent
|
||||
{
|
||||
public array $requests = [];
|
||||
public array $getResponses = [];
|
||||
public array $reconcileResponses = [];
|
||||
public array $getErrors = [];
|
||||
public $onRequest = null;
|
||||
|
||||
public function request(string $method, string $target, array $actor,
|
||||
?array $payload = null): array
|
||||
{
|
||||
$this->requests[] = compact('method', 'target', 'actor', 'payload');
|
||||
if (is_callable($this->onRequest)) {
|
||||
($this->onRequest)($method, $target, $actor, $payload);
|
||||
}
|
||||
if (preg_match('#^/v1/operations/([A-Za-z0-9_-]+)$#D',
|
||||
$target, $match) === 1 && $method === 'GET') {
|
||||
$operationId = rawurldecode($match[1]);
|
||||
if (isset($this->getErrors[$operationId])) {
|
||||
throw $this->getErrors[$operationId];
|
||||
}
|
||||
return ['operation_id' => $operationId]
|
||||
+ ($this->getResponses[$operationId]
|
||||
?? ['status' => 'running']);
|
||||
}
|
||||
if (preg_match(
|
||||
'#^/v1/operations/([A-Za-z0-9_-]+)/reconcile$#D',
|
||||
$target, $match) === 1 && $method === 'POST') {
|
||||
$operationId = rawurldecode($match[1]);
|
||||
return ['operation_id' => $operationId]
|
||||
+ ($this->reconcileResponses[$operationId]
|
||||
?? ['status' => 'uncertain']);
|
||||
}
|
||||
if ($method === 'POST' && $target === '/v1/templates') {
|
||||
return [
|
||||
'operation_id' => (string) $payload['operation_id'],
|
||||
'status' => 'succeeded',
|
||||
'result' => [
|
||||
'id' => (string) $payload['id'],
|
||||
'revision' => (int) $payload['revision'],
|
||||
'generation_token' =>
|
||||
(string) $payload['generation_token'],
|
||||
'size_bytes' => 4096,
|
||||
'sha256' => str_repeat('a', 64),
|
||||
],
|
||||
];
|
||||
}
|
||||
if ($method === 'POST' && $target === '/v1/user-isos') {
|
||||
return [
|
||||
'operation_id' => (string) $payload['operation_id'],
|
||||
'status' => 'succeeded',
|
||||
'result' => [
|
||||
'id' => (string) $payload['id'],
|
||||
'owner_id' => (int) $payload['owner_id'],
|
||||
'size_bytes' => 8192,
|
||||
'sha256' => str_repeat('b', 64),
|
||||
],
|
||||
];
|
||||
}
|
||||
throw new \RuntimeException('Background worker replayed an import');
|
||||
}
|
||||
}
|
||||
|
||||
final class CloudAgentSettingsService
|
||||
{
|
||||
public static ?AuxCatalogWorkerFakeAgent $client = null;
|
||||
public static array $hostIds = [11];
|
||||
|
||||
public static function enabledHostIds(): array
|
||||
{
|
||||
return self::$hostIds;
|
||||
}
|
||||
|
||||
public static function clientForHost(int $hostId,
|
||||
float $timeoutSeconds = 20.0): AuxCatalogWorkerFakeAgent
|
||||
{
|
||||
if (!in_array($hostId, self::$hostIds, true)
|
||||
|| self::$client === null) {
|
||||
throw new \RuntimeException('Missing fake Agent');
|
||||
}
|
||||
return self::$client;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
namespace {
|
||||
require dirname(__DIR__) . '/vendor/autoload.php';
|
||||
|
||||
use App\Models\CloudRescueTemplateImport;
|
||||
use App\Models\CloudTemplate;
|
||||
use App\Models\CloudTemplateReplica;
|
||||
use App\Models\CloudUserIso;
|
||||
use App\Models\CloudUserIsoReplica;
|
||||
use App\Services\Cloud\AuxCatalogWorkerFakeAgent;
|
||||
use App\Services\Cloud\CloudAgentError;
|
||||
use App\Services\Cloud\CloudAgentSettingsService;
|
||||
use App\Services\Cloud\CloudRescueTemplateImportService;
|
||||
use App\Services\Cloud\CloudTemplateCatalogService;
|
||||
use App\Services\Cloud\CloudUserIsoService;
|
||||
use Illuminate\Database\Capsule\Manager as Capsule;
|
||||
use Illuminate\Database\Schema\Blueprint;
|
||||
|
||||
$assert = static function (bool $condition, string $message): void {
|
||||
if (!$condition) {
|
||||
throw new RuntimeException($message);
|
||||
}
|
||||
};
|
||||
|
||||
$capsule = new Capsule();
|
||||
$capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'],
|
||||
'default');
|
||||
$capsule->setAsGlobal();
|
||||
$capsule->bootEloquent();
|
||||
$schema = $capsule->schema('default');
|
||||
$schema->create('cloud_templates', static function (Blueprint $table): void {
|
||||
$table->string('id')->primary();
|
||||
$table->string('name');
|
||||
$table->string('os_family');
|
||||
$table->string('os_version');
|
||||
$table->string('source_type');
|
||||
$table->text('sealed_source_url');
|
||||
$table->integer('image_bytes')->default(0);
|
||||
$table->string('sha256')->default('');
|
||||
$table->text('startup_script')->default('');
|
||||
$table->integer('revision');
|
||||
$table->string('generation_token');
|
||||
$table->string('state');
|
||||
$table->integer('created_at');
|
||||
$table->integer('updated_at');
|
||||
});
|
||||
$schema->create('cloud_template_replicas',
|
||||
static function (Blueprint $table): void {
|
||||
$table->integer('host_id');
|
||||
$table->string('template_id');
|
||||
$table->integer('revision');
|
||||
$table->string('generation_token');
|
||||
$table->string('state');
|
||||
$table->integer('image_bytes')->default(0);
|
||||
$table->string('sha256')->default('');
|
||||
$table->string('operation_id')->default('');
|
||||
$table->string('error_code')->default('');
|
||||
$table->integer('created_at');
|
||||
$table->integer('updated_at');
|
||||
$table->primary(['host_id', 'template_id']);
|
||||
});
|
||||
$schema->create('cloud_rescue_template_imports',
|
||||
static function (Blueprint $table): void {
|
||||
$table->string('operation_id')->primary();
|
||||
$table->integer('host_id');
|
||||
$table->string('template_id');
|
||||
$table->string('import_type');
|
||||
$table->string('state');
|
||||
$table->string('error_code')->default('');
|
||||
$table->integer('created_at');
|
||||
$table->integer('updated_at');
|
||||
$table->unique(['host_id', 'template_id']);
|
||||
});
|
||||
$schema->create('cloud_user_isos', static function (Blueprint $table): void {
|
||||
$table->string('id')->primary();
|
||||
$table->integer('user_id');
|
||||
$table->string('name');
|
||||
$table->text('sealed_source_url');
|
||||
$table->integer('size_bytes')->default(0);
|
||||
$table->string('sha256')->default('');
|
||||
$table->string('state');
|
||||
$table->string('import_operation_id');
|
||||
$table->integer('created_at');
|
||||
$table->integer('updated_at');
|
||||
});
|
||||
$schema->create('cloud_user_iso_replicas',
|
||||
static function (Blueprint $table): void {
|
||||
$table->integer('host_id');
|
||||
$table->string('user_iso_id');
|
||||
$table->integer('user_id');
|
||||
$table->string('state');
|
||||
$table->integer('size_bytes')->default(0);
|
||||
$table->string('sha256')->default('');
|
||||
$table->string('operation_id')->default('');
|
||||
$table->string('error_code')->default('');
|
||||
$table->integer('created_at');
|
||||
$table->integer('updated_at');
|
||||
$table->primary(['host_id', 'user_iso_id']);
|
||||
});
|
||||
|
||||
$agent = new AuxCatalogWorkerFakeAgent();
|
||||
CloudAgentSettingsService::$client = $agent;
|
||||
$now = time();
|
||||
|
||||
// Both global catalogs must persist their complete target-host snapshot
|
||||
// before the first host mutation. A crash at any dispatch boundary then
|
||||
// leaves one durable operation per expected host, never a partial set.
|
||||
CloudAgentSettingsService::$hostIds = [11, 12];
|
||||
$agent->onRequest = static function (string $method, string $target,
|
||||
array $actor, ?array $payload) use ($assert): void {
|
||||
if ($method !== 'POST') {
|
||||
return;
|
||||
}
|
||||
if ($target === '/v1/templates') {
|
||||
$hostIds = CloudTemplateReplica::query()
|
||||
->where('template_id', (string) $payload['id'])
|
||||
->orderBy('host_id')->pluck('host_id')->map(
|
||||
static fn ($value): int => (int) $value
|
||||
)->all();
|
||||
$assert($hostIds === [11, 12],
|
||||
'Template dispatch began before every replica was reserved.');
|
||||
}
|
||||
if ($target === '/v1/user-isos') {
|
||||
$hostIds = CloudUserIsoReplica::query()
|
||||
->where('user_iso_id', (string) $payload['id'])
|
||||
->orderBy('host_id')->pluck('host_id')->map(
|
||||
static fn ($value): int => (int) $value
|
||||
)->all();
|
||||
$assert($hostIds === [11, 12],
|
||||
'ISO dispatch began before every replica was reserved.');
|
||||
}
|
||||
};
|
||||
$templateReservation = CloudTemplateCatalogService::importRemote([
|
||||
'name' => 'Debian 13 atomic reservation',
|
||||
'url' => 'https://images.example/debian-13.qcow2',
|
||||
'sha256' => '',
|
||||
], ['user_id' => 1, 'is_admin' => true, 'can_manage' => true]);
|
||||
$isoReservation = CloudUserIsoService::import([
|
||||
'name' => 'Atomic reservation ISO',
|
||||
'url' => 'https://images.example/atomic-reservation.iso',
|
||||
], ['user_id' => 90, 'is_admin' => false, 'can_manage' => false]);
|
||||
$assert($templateReservation['status'] === 'succeeded'
|
||||
&& $isoReservation['status'] === 'succeeded',
|
||||
'Atomic catalog reservations did not complete in the test fixture.');
|
||||
$agent->onRequest = null;
|
||||
$agent->requests = [];
|
||||
CloudAgentSettingsService::$hostIds = [11];
|
||||
|
||||
$rescueOperation = str_repeat('a', 32);
|
||||
$freshRescueOperation = str_repeat('b', 32);
|
||||
$rescueTemplate = str_repeat('1', 32);
|
||||
$freshRescueTemplate = str_repeat('2', 32);
|
||||
Capsule::table('cloud_rescue_template_imports')->insert([
|
||||
[
|
||||
'operation_id' => $rescueOperation,
|
||||
'host_id' => 11,
|
||||
'template_id' => $rescueTemplate,
|
||||
'import_type' => 'remote',
|
||||
'state' => 'running',
|
||||
'error_code' => '',
|
||||
'created_at' => $now - 30,
|
||||
'updated_at' => $now - 30,
|
||||
],
|
||||
[
|
||||
'operation_id' => $freshRescueOperation,
|
||||
'host_id' => 11,
|
||||
'template_id' => $freshRescueTemplate,
|
||||
'import_type' => 'remote',
|
||||
'state' => 'running',
|
||||
'error_code' => '',
|
||||
'created_at' => $now,
|
||||
'updated_at' => $now,
|
||||
],
|
||||
]);
|
||||
$agent->getResponses[$rescueOperation] = [
|
||||
'status' => 'uncertain',
|
||||
'error' => ['code' => 'operation_uncertain'],
|
||||
];
|
||||
$agent->reconcileResponses[$rescueOperation] = [
|
||||
'status' => 'succeeded',
|
||||
'result' => ['id' => $rescueTemplate],
|
||||
];
|
||||
|
||||
$assert(CloudRescueTemplateImportService::reconcileOnePendingImport(),
|
||||
'Rescue import worker did not claim one due operation.');
|
||||
$rescue = CloudRescueTemplateImport::find($rescueOperation);
|
||||
$freshRescue = CloudRescueTemplateImport::find($freshRescueOperation);
|
||||
$assert((string) $rescue->state === 'succeeded'
|
||||
&& (string) $freshRescue->state === 'running'
|
||||
&& count($agent->requests) === 2
|
||||
&& $agent->requests[0]['method'] === 'GET'
|
||||
&& $agent->requests[0]['target']
|
||||
=== '/v1/operations/' . $rescueOperation
|
||||
&& $agent->requests[1]['method'] === 'POST'
|
||||
&& $agent->requests[1]['target']
|
||||
=== '/v1/operations/' . $rescueOperation . '/reconcile'
|
||||
&& ($agent->requests[0]['actor']['session_id'] ?? null)
|
||||
=== 'cloud-rescue-template-import-worker'
|
||||
&& ($agent->requests[0]['actor']['operation_id'] ?? null)
|
||||
=== $rescueOperation,
|
||||
'Rescue worker did not preserve the saved operation boundary.');
|
||||
$assert(!CloudRescueTemplateImportService::reconcileOnePendingImport(),
|
||||
'Fresh rescue import bypassed the five-second poll backoff.');
|
||||
|
||||
Capsule::table('cloud_rescue_template_imports')
|
||||
->where('operation_id', $freshRescueOperation)
|
||||
->update(['updated_at' => $now - 30]);
|
||||
$agent->getErrors[$freshRescueOperation] = new CloudAgentError(
|
||||
'temporary failure', 503, 'agent_transport_error', true, 503
|
||||
);
|
||||
try {
|
||||
CloudRescueTemplateImportService::reconcileOnePendingImport();
|
||||
throw new RuntimeException('Rescue transport error was swallowed.');
|
||||
} catch (CloudAgentError $error) {
|
||||
$assert($error->agentCode === 'agent_transport_error',
|
||||
'Rescue worker threw the wrong transport error.');
|
||||
}
|
||||
$freshRescue->refresh();
|
||||
$assert((string) $freshRescue->state === 'running'
|
||||
&& (string) $freshRescue->error_code === 'agent_unavailable'
|
||||
&& !CloudRescueTemplateImportService::reconcileOnePendingImport(),
|
||||
'Rescue transport failure was not persisted with poll backoff.');
|
||||
|
||||
$insertIso = static function (string $id, string $jobId, int $userId,
|
||||
string $state, int $updatedAt): void {
|
||||
Capsule::table('cloud_user_isos')->insert([
|
||||
'id' => $id,
|
||||
'user_id' => $userId,
|
||||
'name' => $id,
|
||||
'sealed_source_url' => 'test-sealed-source',
|
||||
'size_bytes' => 0,
|
||||
'sha256' => '',
|
||||
'state' => $state,
|
||||
'import_operation_id' => $jobId,
|
||||
'created_at' => $updatedAt,
|
||||
'updated_at' => $updatedAt,
|
||||
]);
|
||||
};
|
||||
$insertIsoReplica = static function (string $id, int $userId,
|
||||
string $operationId, string $state, int $updatedAt,
|
||||
int $bytes = 0, string $sha256 = ''): void {
|
||||
Capsule::table('cloud_user_iso_replicas')->insert([
|
||||
'host_id' => 11,
|
||||
'user_iso_id' => $id,
|
||||
'user_id' => $userId,
|
||||
'state' => $state,
|
||||
'size_bytes' => $bytes,
|
||||
'sha256' => $sha256,
|
||||
'operation_id' => $operationId,
|
||||
'error_code' => '',
|
||||
'created_at' => $updatedAt,
|
||||
'updated_at' => $updatedAt,
|
||||
]);
|
||||
};
|
||||
|
||||
$isoId = str_repeat('3', 32);
|
||||
$freshIsoId = str_repeat('4', 32);
|
||||
$healIsoId = str_repeat('5', 32);
|
||||
$isoOperation = str_repeat('c', 32);
|
||||
$freshIsoOperation = str_repeat('d', 32);
|
||||
$digest = str_repeat('e', 64);
|
||||
$insertIso($isoId, str_repeat('6', 32), 42, 'importing', $now - 30);
|
||||
$insertIsoReplica($isoId, 42, $isoOperation, 'pending', $now - 30);
|
||||
$insertIso($freshIsoId, str_repeat('7', 32), 43, 'importing', $now);
|
||||
$insertIsoReplica($freshIsoId, 43, $freshIsoOperation, 'pending', $now);
|
||||
$insertIso($healIsoId, str_repeat('8', 32), 44, 'importing', $now - 20);
|
||||
$insertIsoReplica($healIsoId, 44, str_repeat('f', 32), 'active',
|
||||
$now - 20, 4096, $digest);
|
||||
Capsule::table('cloud_user_isos')->where('id', $healIsoId)->update([
|
||||
'size_bytes' => 4096,
|
||||
'sha256' => $digest,
|
||||
]);
|
||||
$agent->getResponses[$isoOperation] = [
|
||||
'status' => 'uncertain',
|
||||
'error' => ['code' => 'operation_uncertain'],
|
||||
];
|
||||
$agent->reconcileResponses[$isoOperation] = [
|
||||
'status' => 'succeeded',
|
||||
'result' => [
|
||||
'id' => $isoId,
|
||||
'owner_id' => 42,
|
||||
'size_bytes' => 8192,
|
||||
'sha256' => $digest,
|
||||
],
|
||||
];
|
||||
$requestsBeforeIso = count($agent->requests);
|
||||
$assert(CloudUserIsoService::reconcileOnePendingImport(),
|
||||
'ISO worker did not claim one due replica.');
|
||||
$iso = CloudUserIso::find($isoId);
|
||||
$isoReplica = CloudUserIsoReplica::query()->where('host_id', 11)
|
||||
->where('user_iso_id', $isoId)->first();
|
||||
$freshIsoReplica = CloudUserIsoReplica::query()->where('host_id', 11)
|
||||
->where('user_iso_id', $freshIsoId)->first();
|
||||
$isoRequests = array_slice($agent->requests, $requestsBeforeIso);
|
||||
$assert((string) $iso->state === 'active'
|
||||
&& (string) $isoReplica->state === 'active'
|
||||
&& (string) $freshIsoReplica->state === 'pending'
|
||||
&& count($isoRequests) === 2
|
||||
&& $isoRequests[0]['method'] === 'GET'
|
||||
&& $isoRequests[0]['target'] === '/v1/operations/' . $isoOperation
|
||||
&& $isoRequests[1]['method'] === 'POST'
|
||||
&& $isoRequests[1]['target']
|
||||
=== '/v1/operations/' . $isoOperation . '/reconcile'
|
||||
&& ($isoRequests[0]['actor']['session_id'] ?? null)
|
||||
=== 'cloud-user-iso-import-worker'
|
||||
&& ($isoRequests[0]['actor']['operation_id'] ?? null)
|
||||
=== $isoOperation,
|
||||
'ISO worker did not preserve one saved operation boundary.');
|
||||
|
||||
$requestsBeforeHeal = count($agent->requests);
|
||||
$assert(CloudUserIsoService::reconcileOnePendingImport()
|
||||
&& (string) CloudUserIso::find($healIsoId)->state === 'active'
|
||||
&& count($agent->requests) === $requestsBeforeHeal,
|
||||
'ISO worker did not repair an all-active parent locally.');
|
||||
$assert(!CloudUserIsoService::reconcileOnePendingImport(),
|
||||
'Fresh ISO replica bypassed the five-second poll backoff.');
|
||||
|
||||
Capsule::table('cloud_user_iso_replicas')
|
||||
->where('user_iso_id', $freshIsoId)
|
||||
->update(['updated_at' => $now - 30]);
|
||||
$agent->getErrors[$freshIsoOperation] = new CloudAgentError(
|
||||
'temporary failure', 503, 'agent_transport_error', true, 503
|
||||
);
|
||||
try {
|
||||
CloudUserIsoService::reconcileOnePendingImport();
|
||||
throw new RuntimeException('ISO transport error was swallowed.');
|
||||
} catch (CloudAgentError $error) {
|
||||
$assert($error->agentCode === 'agent_transport_error',
|
||||
'ISO worker threw the wrong transport error.');
|
||||
}
|
||||
$freshIso = CloudUserIso::find($freshIsoId);
|
||||
$freshIsoReplica->refresh();
|
||||
$assert((string) $freshIso->state === 'importing'
|
||||
&& (string) $freshIsoReplica->state === 'pending'
|
||||
&& (string) $freshIsoReplica->error_code === 'agent_unavailable'
|
||||
&& !CloudUserIsoService::reconcileOnePendingImport(),
|
||||
'ISO transport failure was not persisted with poll backoff.');
|
||||
|
||||
$orphanTemplateId = 'debian-orphan-parent';
|
||||
Capsule::table('cloud_templates')->insert([
|
||||
'id' => $orphanTemplateId,
|
||||
'name' => 'Orphan template',
|
||||
'os_family' => 'Debian',
|
||||
'os_version' => '13',
|
||||
'source_type' => 'remote',
|
||||
'sealed_source_url' => 'test-sealed:template-source',
|
||||
'image_bytes' => 0,
|
||||
'sha256' => '',
|
||||
'startup_script' => '',
|
||||
'revision' => 1,
|
||||
'generation_token' => str_repeat('0', 32),
|
||||
'state' => 'importing',
|
||||
'created_at' => $now - 180,
|
||||
'updated_at' => $now - 180,
|
||||
]);
|
||||
$orphanIsoId = str_repeat('9', 32);
|
||||
$insertIso($orphanIsoId, str_repeat('0', 32), 45, 'importing',
|
||||
$now - 180);
|
||||
$requestsBeforeOrphans = count($agent->requests);
|
||||
$assert(CloudTemplateCatalogService::reconcileOnePendingImport()
|
||||
&& (string) CloudTemplate::find($orphanTemplateId)->state === 'error'
|
||||
&& CloudUserIsoService::reconcileOnePendingImport()
|
||||
&& (string) CloudUserIso::find($orphanIsoId)->state === 'failed'
|
||||
&& count($agent->requests) === $requestsBeforeOrphans,
|
||||
'Replica-free legacy parents were not failed without replay.');
|
||||
|
||||
foreach ($agent->requests as $request) {
|
||||
$assert(!in_array($request['target'],
|
||||
['/v1/rescue-templates', '/v1/user-isos'], true),
|
||||
'Background convergence replayed an original import POST.');
|
||||
}
|
||||
|
||||
echo "Rescue and user ISO background import convergence passed.\n";
|
||||
}
|
||||
@ -0,0 +1,240 @@
|
||||
<?php
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Services\Cloud {
|
||||
final class CloudDaemonActor
|
||||
{
|
||||
public static function resolve(string $sessionId): ?array
|
||||
{
|
||||
return [
|
||||
'user_id' => 1,
|
||||
'is_admin' => true,
|
||||
'is_super_admin' => true,
|
||||
'can_manage' => true,
|
||||
'session_id' => $sessionId,
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
final class TemplateWorkerFakeAgent
|
||||
{
|
||||
public array $requests = [];
|
||||
public array $results = [];
|
||||
|
||||
public function request(string $method, string $target, array $actor,
|
||||
?array $payload = null): array
|
||||
{
|
||||
$this->requests[] = compact('method', 'target', 'actor', 'payload');
|
||||
if (preg_match('#^/v1/operations/([A-Za-z0-9_-]+)$#D',
|
||||
$target, $match) === 1 && $method === 'GET') {
|
||||
$operationId = rawurldecode($match[1]);
|
||||
return [
|
||||
'operation_id' => $operationId,
|
||||
'status' => 'uncertain',
|
||||
'error' => ['code' => 'operation_uncertain'],
|
||||
];
|
||||
}
|
||||
if (preg_match('#^/v1/operations/([A-Za-z0-9_-]+)/reconcile$#D',
|
||||
$target, $match) === 1 && $method === 'POST') {
|
||||
$operationId = rawurldecode($match[1]);
|
||||
return [
|
||||
'operation_id' => $operationId,
|
||||
'status' => 'succeeded',
|
||||
'result' => $this->results[$operationId],
|
||||
];
|
||||
}
|
||||
throw new \RuntimeException('Template worker replayed an import');
|
||||
}
|
||||
}
|
||||
|
||||
final class CloudAgentSettingsService
|
||||
{
|
||||
public static ?TemplateWorkerFakeAgent $client = null;
|
||||
|
||||
public static function clientForHost(int $hostId,
|
||||
float $timeoutSeconds = 20.0): TemplateWorkerFakeAgent
|
||||
{
|
||||
if ($hostId !== 11 || self::$client === null) {
|
||||
throw new \RuntimeException('Missing fake Agent');
|
||||
}
|
||||
return self::$client;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
namespace {
|
||||
require dirname(__DIR__) . '/vendor/autoload.php';
|
||||
|
||||
use App\Models\CloudTemplate;
|
||||
use App\Models\CloudTemplateReplica;
|
||||
use App\Services\Cloud\CloudAgentSettingsService;
|
||||
use App\Services\Cloud\CloudWorkerService;
|
||||
use App\Services\Cloud\TemplateWorkerFakeAgent;
|
||||
use Illuminate\Database\Capsule\Manager as Capsule;
|
||||
use Illuminate\Database\Schema\Blueprint;
|
||||
|
||||
$assert = static function (bool $condition, string $message): void {
|
||||
if (!$condition) {
|
||||
throw new RuntimeException($message);
|
||||
}
|
||||
};
|
||||
|
||||
$capsule = new Capsule();
|
||||
$capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'],
|
||||
'default');
|
||||
$capsule->setAsGlobal();
|
||||
$capsule->bootEloquent();
|
||||
$schema = $capsule->schema('default');
|
||||
$schema->create('cloud_jobs', static function (Blueprint $table): void {
|
||||
$table->increments('id');
|
||||
$table->string('kind');
|
||||
$table->string('status');
|
||||
$table->integer('available_at');
|
||||
$table->integer('lease_until');
|
||||
});
|
||||
$schema->create('cloud_templates', static function (Blueprint $table): void {
|
||||
$table->string('id')->primary();
|
||||
$table->string('name');
|
||||
$table->string('os_family');
|
||||
$table->string('os_version');
|
||||
$table->string('source_type');
|
||||
$table->text('sealed_source_url');
|
||||
$table->integer('image_bytes')->default(0);
|
||||
$table->string('sha256')->default('');
|
||||
$table->text('startup_script')->default('');
|
||||
$table->integer('revision');
|
||||
$table->string('generation_token');
|
||||
$table->string('state');
|
||||
$table->integer('created_at');
|
||||
$table->integer('updated_at');
|
||||
});
|
||||
$schema->create('cloud_template_replicas',
|
||||
static function (Blueprint $table): void {
|
||||
$table->integer('host_id');
|
||||
$table->string('template_id');
|
||||
$table->integer('revision');
|
||||
$table->string('generation_token');
|
||||
$table->string('state');
|
||||
$table->integer('image_bytes')->default(0);
|
||||
$table->string('sha256')->default('');
|
||||
$table->string('operation_id')->default('');
|
||||
$table->string('error_code')->default('');
|
||||
$table->integer('created_at');
|
||||
$table->integer('updated_at');
|
||||
$table->primary(['host_id', 'template_id']);
|
||||
});
|
||||
|
||||
$now = time();
|
||||
$digest = str_repeat('c', 64);
|
||||
$operationA = str_repeat('a', 32);
|
||||
$operationB = str_repeat('b', 32);
|
||||
$generationA = str_repeat('1', 32);
|
||||
$generationB = str_repeat('2', 32);
|
||||
$insertTemplate = static function (string $id, string $generation,
|
||||
string $digest, string $state, int $updatedAt) use ($now): void {
|
||||
Capsule::table('cloud_templates')->insert([
|
||||
'id' => $id,
|
||||
'name' => $id,
|
||||
'os_family' => 'Debian',
|
||||
'os_version' => '13',
|
||||
'source_type' => 'remote',
|
||||
'sealed_source_url' => 'test-sealed-source',
|
||||
'image_bytes' => 0,
|
||||
'sha256' => $digest,
|
||||
'startup_script' => '',
|
||||
'revision' => 1,
|
||||
'generation_token' => $generation,
|
||||
'state' => $state,
|
||||
'created_at' => $now,
|
||||
'updated_at' => $updatedAt,
|
||||
]);
|
||||
};
|
||||
$insertReplica = static function (string $id, string $generation,
|
||||
string $digest, string $operationId, string $state,
|
||||
int $updatedAt): void {
|
||||
Capsule::table('cloud_template_replicas')->insert([
|
||||
'host_id' => 11,
|
||||
'template_id' => $id,
|
||||
'revision' => 1,
|
||||
'generation_token' => $generation,
|
||||
'state' => $state,
|
||||
'image_bytes' => $state === 'active' ? 8192 : 0,
|
||||
'sha256' => $digest,
|
||||
'operation_id' => $operationId,
|
||||
'error_code' => '',
|
||||
'created_at' => $updatedAt,
|
||||
'updated_at' => $updatedAt,
|
||||
]);
|
||||
};
|
||||
|
||||
$insertTemplate('debian-a', $generationA, $digest, 'importing', $now - 30);
|
||||
$insertReplica('debian-a', $generationA, $digest, $operationA,
|
||||
'pending', $now - 30);
|
||||
$insertTemplate('debian-b', $generationB, $digest, 'importing', $now - 20);
|
||||
$insertReplica('debian-b', $generationB, $digest, $operationB,
|
||||
'pending', $now - 20);
|
||||
|
||||
$agent = new TemplateWorkerFakeAgent();
|
||||
$agent->results[$operationA] = [
|
||||
'id' => 'debian-a',
|
||||
'revision' => 1,
|
||||
'generation_token' => $generationA,
|
||||
'size_bytes' => 8192,
|
||||
'sha256' => $digest,
|
||||
];
|
||||
CloudAgentSettingsService::$client = $agent;
|
||||
|
||||
$policy = new CloudWorkerService(null, true);
|
||||
$assert($policy->tick() === false && $agent->requests === [],
|
||||
'The policy-only worker consumed a template import.');
|
||||
|
||||
$worker = new CloudWorkerService();
|
||||
$assert($worker->tick() === true && $worker->observationErrors() === [],
|
||||
'The ordinary idle worker did not converge one template import.');
|
||||
$templateA = CloudTemplate::find('debian-a');
|
||||
$replicaA = CloudTemplateReplica::query()->where('host_id', 11)
|
||||
->where('template_id', 'debian-a')->first();
|
||||
$templateB = CloudTemplate::find('debian-b');
|
||||
$replicaB = CloudTemplateReplica::query()->where('host_id', 11)
|
||||
->where('template_id', 'debian-b')->first();
|
||||
$assert((string) $templateA->state === 'active'
|
||||
&& (int) $templateA->image_bytes === 8192
|
||||
&& (string) $templateA->sha256 === $digest
|
||||
&& (string) $replicaA->state === 'active'
|
||||
&& (string) $templateB->state === 'importing'
|
||||
&& (string) $replicaB->state === 'pending',
|
||||
'One bounded worker pass changed the wrong import or lost image size.');
|
||||
$assert(count($agent->requests) === 2
|
||||
&& $agent->requests[0]['method'] === 'GET'
|
||||
&& $agent->requests[0]['target'] === '/v1/operations/' . $operationA
|
||||
&& $agent->requests[1]['method'] === 'POST'
|
||||
&& $agent->requests[1]['target']
|
||||
=== '/v1/operations/' . $operationA . '/reconcile'
|
||||
&& ($agent->requests[0]['actor']['session_id'] ?? null)
|
||||
=== 'cloud-template-import-worker'
|
||||
&& ($agent->requests[0]['actor']['operation_id'] ?? null)
|
||||
=== $operationA,
|
||||
'Worker lookup/reconciliation did not preserve the saved operation ID.');
|
||||
foreach ($agent->requests as $request) {
|
||||
$assert($request['target'] !== '/v1/templates',
|
||||
'The worker replayed the original template import POST.');
|
||||
}
|
||||
|
||||
Capsule::table('cloud_template_replicas')->where('template_id', 'debian-b')
|
||||
->update(['state' => 'failed', 'error_code' => 'test_terminal']);
|
||||
Capsule::table('cloud_templates')->where('id', 'debian-b')
|
||||
->update(['state' => 'error']);
|
||||
$generationC = str_repeat('3', 32);
|
||||
$insertTemplate('debian-c', $generationC, $digest, 'importing', $now - 10);
|
||||
$insertReplica('debian-c', $generationC, $digest, str_repeat('d', 32),
|
||||
'active', $now - 10);
|
||||
$requestsBeforeAggregateRepair = count($agent->requests);
|
||||
$assert($worker->tick() === true
|
||||
&& (string) CloudTemplate::find('debian-c')->state === 'active'
|
||||
&& count($agent->requests) === $requestsBeforeAggregateRepair,
|
||||
'Worker did not repair an all-active template aggregate locally.');
|
||||
$assert($worker->tick() === false,
|
||||
'A terminal failed template kept the idle worker in a busy loop.');
|
||||
|
||||
echo "Idle worker template import convergence and no-replay tests passed.\n";
|
||||
}
|
||||
@ -0,0 +1,213 @@
|
||||
<?php
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Services\Cloud {
|
||||
// Catalog doubles make the worker scheduling contract observable without
|
||||
// loading any production catalog implementation in this isolated process.
|
||||
final class CloudTemplateCatalogService
|
||||
{
|
||||
public static int $calls = 0;
|
||||
public static bool $fail = false;
|
||||
|
||||
public static function reconcileOnePendingImport(): bool
|
||||
{
|
||||
self::$calls++;
|
||||
if (self::$fail) {
|
||||
throw new \RuntimeException('template catalog unavailable');
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
final class CloudRescueTemplateImportService
|
||||
{
|
||||
public static int $calls = 0;
|
||||
|
||||
public static function reconcileOnePendingImport(): bool
|
||||
{
|
||||
self::$calls++;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
final class CloudUserIsoService
|
||||
{
|
||||
public static int $calls = 0;
|
||||
|
||||
public static function reconcileOnePendingImport(): bool
|
||||
{
|
||||
self::$calls++;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
// Test-only in-memory envelope storage; production CloudSecrets is not
|
||||
// loaded in this process.
|
||||
final class CloudSecrets
|
||||
{
|
||||
private static array $values = [];
|
||||
|
||||
public static function encrypt(array $value, string $purpose): string
|
||||
{
|
||||
$key = bin2hex(random_bytes(16));
|
||||
self::$values[$key] = $value;
|
||||
return 'worker-fairness:' . $key;
|
||||
}
|
||||
|
||||
public static function decrypt(string $value, string $purpose): array
|
||||
{
|
||||
$key = substr($value, strlen('worker-fairness:'));
|
||||
if (!isset(self::$values[$key])) {
|
||||
throw new \RuntimeException('missing test payload');
|
||||
}
|
||||
return self::$values[$key];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
namespace {
|
||||
require dirname(__DIR__) . '/vendor/autoload.php';
|
||||
|
||||
use App\Models\CloudJob;
|
||||
use App\Services\Cloud\CloudAgentClient;
|
||||
use App\Services\Cloud\CloudRescueTemplateImportService;
|
||||
use App\Services\Cloud\CloudSecrets;
|
||||
use App\Services\Cloud\CloudTemplateCatalogService;
|
||||
use App\Services\Cloud\CloudUserIsoService;
|
||||
use App\Services\Cloud\CloudWorkerService;
|
||||
use GuzzleHttp\Client;
|
||||
use GuzzleHttp\Handler\MockHandler;
|
||||
use GuzzleHttp\HandlerStack;
|
||||
use GuzzleHttp\Psr7\Response;
|
||||
use Illuminate\Database\Capsule\Manager as Capsule;
|
||||
use Illuminate\Database\Schema\Blueprint;
|
||||
|
||||
$assert = static function (bool $condition, string $message): void {
|
||||
if (!$condition) {
|
||||
throw new RuntimeException($message);
|
||||
}
|
||||
};
|
||||
|
||||
$capsule = new Capsule();
|
||||
$capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'],
|
||||
'default');
|
||||
$capsule->setAsGlobal();
|
||||
$capsule->bootEloquent();
|
||||
$schema = $capsule->schema('default');
|
||||
$schema->create('user', static function (Blueprint $table): void {
|
||||
$table->increments('id');
|
||||
$table->integer('enable')->default(1);
|
||||
$table->integer('is_admin')->default(0);
|
||||
});
|
||||
$schema->create('cloud_jobs', static function (Blueprint $table): void {
|
||||
$table->increments('id');
|
||||
$table->string('operation_id')->unique();
|
||||
$table->integer('order_id')->nullable();
|
||||
$table->integer('instance_id')->nullable();
|
||||
$table->integer('snapshot_id')->nullable();
|
||||
$table->integer('user_id');
|
||||
$table->integer('host_id');
|
||||
$table->string('kind');
|
||||
$table->string('status');
|
||||
$table->text('sealed_payload');
|
||||
$table->integer('attempts')->default(0);
|
||||
$table->integer('dispatched_at')->nullable();
|
||||
$table->string('lease_token')->nullable();
|
||||
$table->integer('lease_until')->default(0);
|
||||
$table->integer('available_at');
|
||||
$table->string('error_code')->nullable();
|
||||
$table->text('result_json')->nullable();
|
||||
$table->integer('created_at');
|
||||
$table->integer('updated_at');
|
||||
});
|
||||
Capsule::table('user')->insert(['id' => 1, 'enable' => 1, 'is_admin' => 0]);
|
||||
|
||||
$responses = [];
|
||||
$now = time();
|
||||
for ($number = 1; $number <= 3; $number++) {
|
||||
$operationId = str_repeat((string) $number, 32);
|
||||
Capsule::table('cloud_jobs')->insert([
|
||||
'operation_id' => $operationId,
|
||||
'order_id' => null,
|
||||
'instance_id' => null,
|
||||
'snapshot_id' => null,
|
||||
'user_id' => 1,
|
||||
'host_id' => 7,
|
||||
'kind' => 'action',
|
||||
'status' => 'queued',
|
||||
'sealed_payload' => CloudSecrets::encrypt([
|
||||
'method' => 'POST',
|
||||
'request_target' => '/v1/vms/test-vm/actions',
|
||||
'body' => [
|
||||
'operation_id' => $operationId,
|
||||
'action' => 'start',
|
||||
],
|
||||
'owner_id' => 1,
|
||||
], 'job-payload'),
|
||||
'attempts' => 0,
|
||||
'dispatched_at' => null,
|
||||
'lease_token' => null,
|
||||
'lease_until' => 0,
|
||||
'available_at' => 0,
|
||||
'error_code' => null,
|
||||
'result_json' => null,
|
||||
'created_at' => $now,
|
||||
'updated_at' => $now,
|
||||
]);
|
||||
$responses[] = new Response(200, [], json_encode([
|
||||
'operation_id' => $operationId,
|
||||
'status' => 'dispatching',
|
||||
], JSON_THROW_ON_ERROR));
|
||||
}
|
||||
|
||||
$client = new CloudAgentClient(
|
||||
'7',
|
||||
'test-agent-secret-at-least-32-bytes',
|
||||
new Client(['handler' => HandlerStack::create(new MockHandler($responses))]),
|
||||
20.0,
|
||||
'http://127.0.0.1:8001/internal/agent/request',
|
||||
'test-gateway-secret-at-least-32-bytes'
|
||||
);
|
||||
$worker = new CloudWorkerService($client);
|
||||
|
||||
for ($tick = 1; $tick <= 3; $tick++) {
|
||||
$assert($worker->tick(), 'A due cloud job was not processed.');
|
||||
$assert(CloudJob::query()->where('status', 'running')->count() === $tick,
|
||||
'The persistent cloud_jobs backlog did not advance one row.');
|
||||
$assert(CloudTemplateCatalogService::$calls
|
||||
+ CloudRescueTemplateImportService::$calls
|
||||
+ CloudUserIsoService::$calls === $tick,
|
||||
'One worker tick ran more than one catalog action.');
|
||||
}
|
||||
$assert(CloudTemplateCatalogService::$calls === 1
|
||||
&& CloudRescueTemplateImportService::$calls === 1
|
||||
&& CloudUserIsoService::$calls === 1,
|
||||
'A continuously non-empty cloud_jobs queue starved a catalog.');
|
||||
|
||||
// A catalog exception is observable but is not work. Returning false keeps
|
||||
// the command loop's one-second idle backoff and prevents a log/DB hot loop.
|
||||
CloudTemplateCatalogService::$fail = true;
|
||||
$assert($worker->tick() === false,
|
||||
'A catalog exception incorrectly reported successful work.');
|
||||
$assert($worker->observationErrors() === ['worker_tick_failed'],
|
||||
'A catalog exception was not exposed through worker health.');
|
||||
$assert(CloudTemplateCatalogService::$calls === 2
|
||||
&& CloudRescueTemplateImportService::$calls === 1
|
||||
&& CloudUserIsoService::$calls === 1,
|
||||
'The worker ran a second catalog after an ambiguous catalog failure.');
|
||||
|
||||
$beforePolicy = [
|
||||
CloudTemplateCatalogService::$calls,
|
||||
CloudRescueTemplateImportService::$calls,
|
||||
CloudUserIsoService::$calls,
|
||||
];
|
||||
$policyWorker = new CloudWorkerService(null, true);
|
||||
$assert($policyWorker->tick() === false
|
||||
&& $beforePolicy === [
|
||||
CloudTemplateCatalogService::$calls,
|
||||
CloudRescueTemplateImportService::$calls,
|
||||
CloudUserIsoService::$calls,
|
||||
], 'The policy-only worker was delayed by catalog convergence.');
|
||||
|
||||
echo "Cloud worker catalog fairness and failure backoff tests passed.\n";
|
||||
}
|
||||
Loading…
Reference in New Issue