fix: converge persistent catalog imports

go
codex 1 month ago
parent e7fa6256c2
commit fbe8498deb

@ -208,3 +208,21 @@ authorized host mutation test: 待部署后在用户指定的测试环境按 ope
browser/payment/SMTP test: 自动化 PHP/Node 契约测试通过;真实创建付款需在最终提交动作前取得用户即时确认
known skips or risks: libvirt 调用已执行但结果不可证明时会安全保留 uncertain,必须复用原 operation_id 核对,禁止自动重放;macOS libvirt 链接仅有重复库警告
```
### 2026-10-16 持久化目录任务收敛验收
```text
commit: 本记录所在 go 分支提交(基线 e7fa625)
Go version / libvirt version / PHP version / MariaDB version: Go 1.27.1 / 本机动态链接测试沿用已安装 libvirt / PHP 8.5.11 / MariaDB 未在本机运行
go vet: Agent 默认及 libvirt tag、Gateway 均通过
go test: Agent 默认及 libvirt tag、Gateway 均通过;两个 Go 模块的 mod verify 均通过
go test -race: Agent libvirt tag 与 Gateway 均通过;Gateway race 测试获准绑定本机临时回环端口
Linux AMD64 build: 本提交未修改 Go 源码,继续使用 e7fa625 对应的已验证 Agent 与 Gateway 二进制
PHP tests: 392 个 PHP 文件逐文件 lint 通过,119 个测试文件通过;SourceSyntax 独立检查 396 个文件通过;仅输出既有测试 warning 与第三方依赖 deprecation,退出码为 0
Node tests: 89 个 JavaScript 文件语法检查通过,4 个行为测试通过;部署 JSON 示例有效
systemd-analyze: 本提交未修改 service 文件,未重复执行
read-only host capability check: HKG-A 出站 WSS 已稳定连接;Agent 报告 14 项必要能力、UEFI 可用且 Secure Boot 关闭
authorized host mutation test: HTTPS Debian 13 云镜像导入在 Agent 端确定成功,436207616 字节,SHA-256 为 1010eb3215be632548cfb6793b53bd8243064d8820b5106f2becfcd179099b2f;本提交新增后台同 operation_id 收敛,禁止重放原始导入 POST
browser/payment/SMTP test: 管理端网络选择器已由错误的 bridge:br0 修正为 Agent 声明的 public;旧创建任务已按原 operation_id 核对并明确失败退款;新的真实 10 USD 创建提交仍等待用户即时确认
known skips or risks: worker 每 tick 最多处理一个普通任务和一个目录核对;真实创建、重装、快照恢复和删除属于外部或破坏性操作,必须在动作前取得确认后再完成联调
```

@ -10,6 +10,8 @@ use App\Models\CloudRescueTemplateImport;
*/
final class CloudRescueTemplateImportService
{
private const IMPORT_POLL_INTERVAL_SECONDS = 5;
public static function start(
int $hostId,
string $importType,
@ -86,6 +88,101 @@ final class CloudRescueTemplateImportService
return self::pollIfAllowed($record, $actor);
}
/**
* Converge one already-dispatched rescue import without replaying it.
* The persisted operation ID is the only capability available here.
*/
public static function reconcileOnePendingImport(): bool
{
$connection = (new CloudRescueTemplateImport())->getConnection();
if (!$connection->getSchemaBuilder()->hasTable(
'cloud_rescue_template_imports'
)) {
return false;
}
$now = time();
$record = CloudRescueTemplateImport::query()
->where(function ($query) use ($now): void {
$query->where(function ($running) use ($now): void {
$running->where('state', 'running')
->where('updated_at', '<=', $now
- self::IMPORT_POLL_INTERVAL_SECONDS);
})->orWhere(function ($crossing) use ($now): void {
$crossing->whereIn('state', ['dispatching', 'sending'])
->where('updated_at', '<=', $now - 90);
});
})->orderBy('updated_at')->orderBy('operation_id')->first();
if ($record === null) {
return false;
}
$state = (string) $record->state;
$operationId = (string) $record->operation_id;
$claimed = CloudRescueTemplateImport::query()
->where('operation_id', $operationId)
->where('host_id', (int) $record->host_id)
->where('template_id', (string) $record->template_id)
->where('state', $state)
->where('updated_at', (int) $record->updated_at)
->update(['updated_at' => $now]);
if ($claimed !== 1) {
return false;
}
if ($state === 'dispatching') {
self::mark($record, 'uncertain', 'import_dispatch_unconfirmed');
return true;
}
if ($state === 'sending') {
self::mark($record, 'running', 'import_dispatch_unconfirmed');
$record = self::reload($record);
}
if (preg_match('/^[A-Za-z0-9_-]{8,80}$/D', $operationId) !== 1) {
self::mark($record, 'uncertain', 'operation_id_invalid');
throw new CloudError(
'救援镜像导入操作标识无效,必须人工核查', 503
);
}
$actor = CloudDaemonActor::resolve(
'cloud-rescue-template-import-worker'
);
if ($actor === null) {
self::mark($record, 'running', 'daemon_actor_unavailable');
throw new CloudError('救援镜像导入后台核对未配置管理员身份', 503);
}
$actor['operation_id'] = $operationId;
try {
$client = CloudAgentSettingsService::clientForHost(
(int) $record->host_id
);
$operation = $client->request('GET', '/v1/operations/'
. rawurlencode($operationId), $actor);
$operation = CloudAgentOperationReconciler::ifUncertain(
$client, $operationId, $actor, $operation
);
} catch (CloudAgentError $error) {
$missing = $error->httpStatus === 404
|| $error->agentHttpStatus === 404;
self::mark($record, $missing ? 'uncertain' : 'running',
$missing ? 'operation_not_found' : 'agent_unavailable');
throw $error;
} catch (\Throwable $error) {
self::mark($record, 'running', 'agent_unavailable');
throw $error;
}
try {
self::applyOperation($record, $operation);
} catch (\Throwable $error) {
self::mark($record, 'uncertain', 'panel_persist_failed');
throw $error;
}
return true;
}
private static function reserve(string $operationId, int $hostId,
string $templateId, string $importType): CloudRescueTemplateImport
{

@ -11,6 +11,7 @@ use App\Models\CloudOrder;
final class CloudTemplateCatalogService
{
public const STARTUP_SCRIPT_MAX_BYTES = 32768;
private const IMPORT_POLL_INTERVAL_SECONDS = 5;
public static function adminCatalog(): array
{
@ -49,8 +50,9 @@ final class CloudTemplateCatalogService
if ($hostIds === []) {
throw new CloudError('没有已启用且已配置 Agent 的宿主机', 409);
}
$hostIds = array_values(array_unique(array_map('intval', $hostIds)));
$template = self::newImport($name, $family, $version, 'remote',
$sourceUrl, 0, $sha256);
$sourceUrl, 0, $sha256, $hostIds);
return self::dispatchImport($template, $hostIds, $actor,
static fn (int $hostId): array => [
'source_url' => $sourceUrl,
@ -72,6 +74,7 @@ final class CloudTemplateCatalogService
if ($hostIds === []) {
throw new CloudError('没有已启用且已配置 Agent 的宿主机', 409);
}
$hostIds = array_values(array_unique(array_map('intval', $hostIds)));
$digest = strtolower((string) ($file['sha256'] ?? ''));
$bytes = filter_var($file['size_bytes'] ?? null,
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]);
@ -80,7 +83,7 @@ final class CloudTemplateCatalogService
throw new CloudError('本地云镜像校验结果无效', 503);
}
$template = self::newImport($name, $family, $version, 'upload',
'', (int) $bytes, $digest);
'', (int) $bytes, $digest, $hostIds);
return self::dispatchImport($template, $hostIds, $actor,
static function (int $hostId) use ($file): array {
$staging = CloudUploadStagingService::issue(
@ -147,6 +150,144 @@ final class CloudTemplateCatalogService
return self::importJobDto($template);
}
/**
* Converge one durable template import while the cloud_jobs queue is idle.
* The original source request is intentionally unavailable here: only the
* already-persisted Agent operation ID may be read or reconciled.
*/
public static function reconcileOnePendingImport(): bool
{
$connection = (new CloudTemplateReplica())->getConnection();
$schema = $connection->getSchemaBuilder();
if (!$schema->hasTable('cloud_templates')
|| !$schema->hasTable('cloud_template_replicas')) {
return false;
}
$now = time();
$replica = CloudTemplateReplica::query()
->select('cloud_template_replicas.*')
->join('cloud_templates', 'cloud_templates.id', '=',
'cloud_template_replicas.template_id')
->where('cloud_template_replicas.state', 'pending')
->whereIn('cloud_templates.state', ['importing', 'error'])
->where('cloud_template_replicas.updated_at', '<=',
$now - self::IMPORT_POLL_INTERVAL_SECONDS)
->orderBy('cloud_template_replicas.updated_at')
->orderBy('cloud_template_replicas.host_id')
->orderBy('cloud_template_replicas.template_id')->first();
if ($replica === null) {
// Versions predating atomic parent/replica reservation could
// expose a parent row before the first host operation existed.
// There is no saved mutation to replay, so age it into an error.
$orphan = CloudTemplate::query()->where('state', 'importing')
->where('updated_at', '<=', $now - 90)
->whereNotExists(static function ($query): void {
$query->selectRaw('1')->from('cloud_template_replicas')
->whereColumn('cloud_template_replicas.template_id',
'cloud_templates.id');
})->orderBy('updated_at')->orderBy('id')->first();
if ($orphan !== null) {
return CloudTemplate::query()
->where('id', (string) $orphan->id)
->where('state', 'importing')
->where('updated_at', (int) $orphan->updated_at)
->update([
'state' => 'error',
'updated_at' => $now,
]) === 1;
}
// Repair the narrow crash window where every replica committed but
// the parent aggregate did not. Failed or uncertain replicas are
// excluded so an idle worker cannot spin on a terminal import.
$template = CloudTemplate::query()
->whereIn('state', ['importing', 'error'])
->whereExists(static function ($query): void {
$query->selectRaw('1')->from('cloud_template_replicas')
->whereColumn('cloud_template_replicas.template_id',
'cloud_templates.id');
})->whereNotExists(static function ($query): void {
$query->selectRaw('1')->from('cloud_template_replicas')
->whereColumn('cloud_template_replicas.template_id',
'cloud_templates.id')
->where('cloud_template_replicas.state', '<>', 'active');
})->orderBy('updated_at')->orderBy('id')->first();
if ($template === null) {
return false;
}
self::refreshImportState($template);
return true;
}
$hostId = (int) $replica->host_id;
$templateId = (string) $replica->template_id;
$operationId = (string) $replica->operation_id;
// updated_at is a schema-free claim and backoff. Concurrent workers
// can only duplicate a read, never the original import mutation.
$claimed = CloudTemplateReplica::query()->where('host_id', $hostId)
->where('template_id', $templateId)
->where('state', 'pending')->where('operation_id', $operationId)
->where('updated_at', (int) $replica->updated_at)
->update(['updated_at' => $now]);
if ($claimed !== 1) {
return false;
}
$template = CloudTemplate::query()->where('id', $templateId)
->whereIn('state', ['importing', 'error'])->first();
if ($template === null) {
return true;
}
if (preg_match('/^[A-Za-z0-9_-]{8,80}$/D', $operationId) !== 1) {
self::markReplica($hostId, $templateId, $operationId,
['pending'], 'uncertain', 'operation_id_invalid');
self::refreshImportState($template);
throw new CloudError('云镜像导入操作标识无效,必须人工核查', 503);
}
$actor = CloudDaemonActor::resolve('cloud-template-import-worker');
if ($actor === null) {
self::markReplica($hostId, $templateId, $operationId,
['pending'], 'pending', 'daemon_actor_unavailable');
throw new CloudError('云镜像导入后台核对未配置管理员身份', 503);
}
$actor['operation_id'] = $operationId;
try {
$client = CloudAgentSettingsService::clientForHost($hostId);
$operation = $client->request('GET', '/v1/operations/'
. rawurlencode($operationId), $actor);
$operation = CloudAgentOperationReconciler::ifUncertain(
$client, $operationId, $actor, $operation
);
} catch (CloudAgentError $error) {
$missing = $error->httpStatus === 404
|| $error->agentHttpStatus === 404;
self::markReplica($hostId, $templateId, $operationId,
['pending'], $missing ? 'uncertain' : 'pending',
$missing ? 'operation_not_found' : 'agent_unavailable');
self::refreshImportState($template);
throw $error;
} catch (\Throwable $error) {
self::markReplica($hostId, $templateId, $operationId,
['pending'], 'pending', 'agent_unavailable');
throw $error;
}
try {
self::applyImportOperation($template, $hostId, $operationId,
$operation);
self::refreshImportState($template);
} catch (\Throwable $error) {
self::markReplica($hostId, $templateId, $operationId,
['pending'], 'uncertain', 'panel_persist_failed');
self::refreshImportState($template);
throw $error;
}
return true;
}
/** Push the current PHP name/revision to every existing host replica. */
public static function synchronizeMetadata(array $template,
array $actor): array
@ -160,8 +301,18 @@ final class CloudTemplateCatalogService
|| preg_match('/^[a-f0-9]{32}$/D', $generation) !== 1) {
throw new CloudError('云镜像版本无效', 409);
}
$authority = CloudTemplate::query()->where('id', $id)->first([
'id', 'name', 'revision', 'generation_token', 'state',
]);
if ($authority === null || (string) $authority->state !== 'active'
|| (int) $authority->revision !== (int) $revision
|| !hash_equals((string) $authority->generation_token, $generation)
|| (string) $authority->name !== $name) {
throw new CloudError('仅已完成导入的当前云镜像版本可以同步元数据', 409);
}
$pending = [];
foreach (CloudTemplateReplica::query()->where('template_id', $id)
->whereIn('state', ['active', 'stale'])
->orderBy('host_id')->get() as $replica) {
$hostId = (int) $replica->host_id;
$operationId = bin2hex(random_bytes(16));
@ -191,6 +342,7 @@ final class CloudTemplateCatalogService
CloudTemplateReplica::query()->where('host_id', $hostId)
->where('template_id', $id)
->where('generation_token', $generation)
->whereIn('state', ['active', 'stale'])
// A slower response for revision N must never overwrite a
// concurrent revision N+1 response.
->where('revision', '<=', (int) $revision)->update([
@ -205,6 +357,7 @@ final class CloudTemplateCatalogService
CloudTemplateReplica::query()->where('host_id', $hostId)
->where('template_id', $id)
->where('generation_token', $generation)
->whereIn('state', ['active', 'stale'])
// Do not let a late failure regress a newer successful
// metadata synchronization.
->where('revision', '<=', (int) $revision)->update([
@ -516,6 +669,9 @@ final class CloudTemplateCatalogService
if ($template === null || (string) $template->state === 'retired') {
throw new CloudError('云镜像模板不存在', 404);
}
if ((string) $template->state !== 'active') {
throw new CloudError('云镜像尚未完成导入,暂不能修改启动脚本', 409);
}
if (!hash_equals((string) $template->generation_token, $generationToken)
|| (int) $template->revision !== $expectedRevision) {
throw new CloudError('初始化脚本已被修改,请重新打开对话框', 409);
@ -760,6 +916,9 @@ final class CloudTemplateCatalogService
if ($template === null || (string) $template->state === 'retired') {
throw new CloudError('云镜像模板不存在', 404);
}
if ((string) $template->state !== 'active') {
throw new CloudError('云镜像尚未完成导入,暂不能修改元数据', 409);
}
if ((int) $template->revision !== $expectedRevision
|| !hash_equals((string) $template->generation_token,
$generationToken)) {
@ -784,6 +943,21 @@ final class CloudTemplateCatalogService
private static function adminDto(CloudTemplate $template): array
{
$replicas = [];
$connection = (new CloudTemplateReplica())->getConnection();
if ($connection->getSchemaBuilder()->hasTable(
'cloud_template_replicas')) {
$replicas = CloudTemplateReplica::query()
->where('template_id', (string) $template->id)
->orderBy('host_id')->get([
'host_id', 'state', 'error_code', 'operation_id',
])->map(static fn (CloudTemplateReplica $replica): array => [
'host_id' => (int) $replica->host_id,
'state' => (string) $replica->state,
'error_code' => (string) $replica->error_code,
'operation_id' => (string) $replica->operation_id,
])->all();
}
return [
'id' => (string) $template->id,
'name' => (string) $template->name,
@ -798,62 +972,57 @@ final class CloudTemplateCatalogService
'generation_token' => (string) $template->generation_token,
'created_at' => (int) $template->created_at,
'updated_at' => (int) $template->updated_at,
'replicas' => $replicas,
];
}
private static function writeReplica(int $hostId, CloudTemplate $template,
string $state, string $operationId, string $errorCode, int $bytes,
string $sha256): void
{
$values = [
'revision' => (int) $template->revision,
'generation_token' => (string) $template->generation_token,
'state' => $state,
'image_bytes' => $bytes,
'sha256' => $sha256,
'operation_id' => $operationId,
'error_code' => $errorCode,
'updated_at' => time(),
];
$query = CloudTemplateReplica::query()->where('host_id', $hostId)
->where('template_id', (string) $template->id);
if ($query->exists()) {
$query->update($values);
return;
}
$replica = new CloudTemplateReplica();
foreach ($values + [
'host_id' => $hostId,
'template_id' => (string) $template->id,
'created_at' => time(),
] as $field => $value) {
$replica->{$field} = $value;
}
$replica->save();
}
private static function newImport(string $name, string $family,
string $version, string $sourceType, string $sourceUrl, int $bytes,
string $sha256): CloudTemplate
string $sha256, array $hostIds): CloudTemplate
{
$template = new CloudTemplate();
$template->id = self::newTemplateId($family, $version);
$template->name = $name;
$template->os_family = $family;
$template->os_version = $version;
$template->source_type = $sourceType;
$template->sealed_source_url = CloudSecrets::encrypt(
['url' => $sourceUrl], 'template-source'
);
$template->image_bytes = $bytes;
$template->sha256 = $sha256;
$template->startup_script = '';
$template->revision = 1;
$template->generation_token = bin2hex(random_bytes(16));
$template->state = 'importing';
$template->created_at = $template->updated_at = time();
$template->save();
return $template;
$connection = (new CloudTemplate())->getConnection();
return $connection->transaction(static function () use (
$name, $family, $version, $sourceType, $sourceUrl, $bytes,
$sha256, $hostIds
): CloudTemplate {
$template = new CloudTemplate();
$template->id = self::newTemplateId($family, $version);
$template->name = $name;
$template->os_family = $family;
$template->os_version = $version;
$template->source_type = $sourceType;
$template->sealed_source_url = CloudSecrets::encrypt(
['url' => $sourceUrl], 'template-source'
);
$template->image_bytes = $bytes;
$template->sha256 = $sha256;
$template->startup_script = '';
$template->revision = 1;
$template->generation_token = bin2hex(random_bytes(16));
$template->state = 'importing';
$template->created_at = $template->updated_at = time();
$template->save();
$replicas = [];
foreach ($hostIds as $hostId) {
$replicas[] = [
'host_id' => $hostId,
'template_id' => (string) $template->id,
'revision' => (int) $template->revision,
'generation_token' =>
(string) $template->generation_token,
'state' => 'pending',
'image_bytes' => 0,
'sha256' => (string) $template->sha256,
'operation_id' => bin2hex(random_bytes(16)),
'error_code' => '',
'created_at' => (int) $template->created_at,
'updated_at' => (int) $template->updated_at,
];
}
CloudTemplateReplica::query()->insert($replicas);
return $template;
}, 3);
}
private static function dispatchImport(CloudTemplate $template,
@ -861,11 +1030,20 @@ final class CloudTemplateCatalogService
{
$operations = [];
foreach ($hostIds as $hostId) {
$hostId = (int) $hostId;
$operationId = bin2hex(random_bytes(16));
$operations[$hostId] = $operationId;
self::writeReplica($hostId, $template, 'pending', $operationId,
'', 0, (string) $template->sha256);
$replica = CloudTemplateReplica::query()
->where('host_id', $hostId)
->where('template_id', (string) $template->id)
->where('state', 'pending')->first();
if ($replica === null) {
CloudTemplate::query()->where('id', (string) $template->id)
->where('state', 'importing')->update([
'state' => 'error',
'updated_at' => time(),
]);
$template->refresh();
return self::importJobDto($template);
}
$operations[$hostId] = (string) $replica->operation_id;
}
foreach ($operations as $hostId => $operationId) {
try {
@ -1027,9 +1205,16 @@ final class CloudTemplateCatalogService
'agent_result_mismatch');
return;
}
$metadataChanged = false;
if ((string) $locked->sha256 === '') {
$locked->sha256 = $digest;
$metadataChanged = true;
}
if ((int) $locked->image_bytes === 0) {
$locked->image_bytes = (int) $bytes;
$metadataChanged = true;
}
if ($metadataChanged) {
$locked->updated_at = time();
$locked->save();
}

@ -10,6 +10,7 @@ final class CloudUserIsoService
{
private const MAX_ISOS = 5;
private const MAX_TOTAL_BYTES = 10 * 1024 * 1024 * 1024;
private const IMPORT_POLL_INTERVAL_SECONDS = 5;
public static function listing(int $userId): array
{
@ -37,9 +38,10 @@ final class CloudUserIsoService
if ($hosts === []) {
throw new CloudError('没有可用的宿主机 Agent', 503);
}
$hosts = array_values(array_unique(array_map('intval', $hosts)));
$connection = (new CloudUserIso())->getConnection();
$iso = $connection->transaction(static function () use (
$userId, $name, $url
$userId, $name, $url, $hosts
): CloudUserIso {
$rows = CloudUserIso::query()->where('user_id', $userId)
->where('state', '<>', 'retired')
@ -63,14 +65,48 @@ final class CloudUserIsoService
$iso->import_operation_id = bin2hex(random_bytes(16));
$iso->created_at = $iso->updated_at = time();
$iso->save();
// Persist the complete target-host snapshot before the first
// Agent mutation. A process crash can therefore leave only a
// fully enumerable set of pending operations, never a partial
// replica set that could be mistaken for a completed import.
$replicas = [];
foreach ($hosts as $hostId) {
$replicas[] = [
'host_id' => $hostId,
'user_iso_id' => (string) $iso->id,
'user_id' => $userId,
'state' => 'pending',
'size_bytes' => 0,
'sha256' => '',
'operation_id' => bin2hex(random_bytes(16)),
'error_code' => '',
'created_at' => (int) $iso->created_at,
'updated_at' => (int) $iso->updated_at,
];
}
CloudUserIsoReplica::query()->insert($replicas);
return $iso;
}, 3);
$operations = [];
foreach ($hosts as $hostId) {
$hostId = (int) $hostId;
$operationId = bin2hex(random_bytes(16));
self::writeReplica($hostId, $iso, 'pending', $operationId,
'', 0, '');
$replica = CloudUserIsoReplica::query()
->where('host_id', $hostId)
->where('user_iso_id', (string) $iso->id)
->where('state', 'pending')->first();
if ($replica === null) {
CloudUserIso::query()->where('id', (string) $iso->id)
->where('state', 'importing')->update([
'state' => 'failed',
'updated_at' => time(),
]);
$iso->refresh();
return self::jobDto($iso, self::jobStatus($iso));
}
$operations[$hostId] = (string) $replica->operation_id;
}
foreach ($operations as $hostId => $operationId) {
try {
$requestActor = $actor;
$requestActor['can_manage'] = true;
@ -190,6 +226,149 @@ final class CloudUserIsoService
return self::jobDto($iso, self::jobStatus($iso));
}
/**
* Converge one durable ISO replica using only its saved Agent operation.
* Source URLs are intentionally unavailable, so the import POST cannot be
* replayed by the background worker.
*/
public static function reconcileOnePendingImport(): bool
{
$connection = (new CloudUserIsoReplica())->getConnection();
$schema = $connection->getSchemaBuilder();
if (!$schema->hasTable('cloud_user_isos')
|| !$schema->hasTable('cloud_user_iso_replicas')) {
return false;
}
$now = time();
$replica = CloudUserIsoReplica::query()
->select('cloud_user_iso_replicas.*')
->join('cloud_user_isos', 'cloud_user_isos.id', '=',
'cloud_user_iso_replicas.user_iso_id')
->where('cloud_user_iso_replicas.state', 'pending')
->whereIn('cloud_user_isos.state', ['importing', 'failed'])
->where('cloud_user_iso_replicas.updated_at', '<=',
$now - self::IMPORT_POLL_INTERVAL_SECONDS)
->orderBy('cloud_user_iso_replicas.updated_at')
->orderBy('cloud_user_iso_replicas.host_id')
->orderBy('cloud_user_iso_replicas.user_iso_id')->first();
if ($replica === null) {
// Rows written by versions predating atomic replica reservation
// may have crashed after the parent commit but before the first
// replica. They contain no operation that could be safely
// replayed, so age them into a visible terminal failure.
$orphan = CloudUserIso::query()->where('state', 'importing')
->where('updated_at', '<=', $now - 90)
->whereNotExists(static function ($query): void {
$query->selectRaw('1')->from('cloud_user_iso_replicas')
->whereColumn('cloud_user_iso_replicas.user_iso_id',
'cloud_user_isos.id');
})->orderBy('updated_at')->orderBy('id')->first();
if ($orphan !== null) {
return CloudUserIso::query()
->where('id', (string) $orphan->id)
->where('state', 'importing')
->where('updated_at', (int) $orphan->updated_at)
->update([
'state' => 'failed',
'updated_at' => $now,
]) === 1;
}
// Repair the crash window after every replica committed but before
// the aggregate catalog row was refreshed. Terminal replica sets
// are excluded so an idle worker cannot spin indefinitely.
$iso = CloudUserIso::query()
// A failed all-active aggregate may be a durable quota
// rejection. Only importing is an unambiguous crash window;
// otherwise the idle worker would select the same row forever.
->where('state', 'importing')
->whereExists(static function ($query): void {
$query->selectRaw('1')->from('cloud_user_iso_replicas')
->whereColumn('cloud_user_iso_replicas.user_iso_id',
'cloud_user_isos.id');
})->whereNotExists(static function ($query): void {
$query->selectRaw('1')->from('cloud_user_iso_replicas')
->whereColumn('cloud_user_iso_replicas.user_iso_id',
'cloud_user_isos.id')
->where('cloud_user_iso_replicas.state', '<>',
'active');
})->orderBy('updated_at')->orderBy('id')->first();
if ($iso === null) {
return false;
}
self::refreshCatalogState($iso);
return true;
}
$hostId = (int) $replica->host_id;
$isoId = (string) $replica->user_iso_id;
$operationId = (string) $replica->operation_id;
$claimed = CloudUserIsoReplica::query()->where('host_id', $hostId)
->where('user_iso_id', $isoId)->where('state', 'pending')
->where('operation_id', $operationId)
->where('updated_at', (int) $replica->updated_at)
->update(['updated_at' => $now]);
if ($claimed !== 1) {
return false;
}
$iso = CloudUserIso::query()->where('id', $isoId)
->whereIn('state', ['importing', 'failed'])->first();
if ($iso === null) {
return true;
}
if (preg_match('/^[A-Za-z0-9_-]{8,80}$/D', $operationId) !== 1) {
self::markReplica($hostId, $isoId, $operationId, ['pending'],
'uncertain', 'operation_id_invalid');
self::refreshCatalogState($iso);
throw new CloudError('ISO 导入操作标识无效,必须人工核查', 503);
}
$actor = CloudDaemonActor::resolve('cloud-user-iso-import-worker');
if ($actor === null) {
self::markReplica($hostId, $isoId, $operationId, ['pending'],
'pending', 'daemon_actor_unavailable');
throw new CloudError('ISO 导入后台核对未配置管理员身份', 503);
}
$actor['operation_id'] = $operationId;
try {
$client = CloudAgentSettingsService::clientForHost($hostId);
$operation = $client->request('GET', '/v1/operations/'
. rawurlencode($operationId), $actor);
$operation = CloudAgentOperationReconciler::ifUncertain(
$client, $operationId, $actor, $operation
);
} catch (CloudAgentError $error) {
$missing = $error->httpStatus === 404
|| $error->agentHttpStatus === 404;
self::markReplica($hostId, $isoId, $operationId, ['pending'],
$missing ? 'uncertain' : 'pending',
$missing ? 'operation_not_found' : 'agent_unavailable');
self::refreshCatalogState($iso);
throw $error;
} catch (\Throwable $error) {
self::markReplica($hostId, $isoId, $operationId, ['pending'],
'pending', 'agent_unavailable');
self::refreshCatalogState($iso);
throw $error;
}
try {
self::applyImportOperation($iso, $hostId, $operationId,
$operation);
self::refreshCatalogState($iso);
} catch (\Throwable $error) {
self::markReplica($hostId, $isoId, $operationId, ['pending'],
'uncertain', 'panel_persist_failed');
self::refreshCatalogState($iso);
throw $error;
}
return true;
}
public static function delete(string $id, array $actor): array
{
if (preg_match('/^[a-f0-9]{32}$/D', $id) !== 1) {
@ -656,36 +835,6 @@ final class CloudUserIsoService
]);
}
private static function writeReplica(int $hostId, CloudUserIso $iso,
string $state, string $operationId, string $errorCode, int $bytes,
string $sha256): void
{
$values = [
'user_id' => (int) $iso->user_id,
'state' => $state,
'size_bytes' => $bytes,
'sha256' => $sha256,
'operation_id' => $operationId,
'error_code' => $errorCode,
'updated_at' => time(),
];
$query = CloudUserIsoReplica::query()->where('host_id', $hostId)
->where('user_iso_id', (string) $iso->id);
if ($query->exists()) {
$query->update($values);
return;
}
$replica = new CloudUserIsoReplica();
foreach ($values + [
'host_id' => $hostId,
'user_iso_id' => (string) $iso->id,
'created_at' => time(),
] as $field => $value) {
$replica->{$field} = $value;
}
$replica->save();
}
private static function markReplica(int $hostId, string $isoId,
string $operationId, array $allowedStates, string $state,
string $errorCode): bool

@ -21,6 +21,7 @@ final class CloudWorkerService
private bool $policyOnly;
private array $observedErrors = [];
private int $catalogCursor = 0;
public function observationErrors(): array
{
@ -127,6 +128,17 @@ final class CloudWorkerService
public function tick(): bool
{
$this->observedErrors = [];
$worked = $this->tickJob();
if ($this->policyOnly) {
return $worked;
}
$catalogWorked = $this->convergeOneCatalogImport();
return $worked || $catalogWorked;
}
/** Claim and advance at most one durable cloud job. */
private function tickJob(): bool
{
$job = (new CloudJob())->getConnection()->transaction(function () {
$query = CloudJob::whereIn('status', ['queued', 'running', 'uncertain'])
->where('available_at', '<=', time())->where('lease_until', '<=', time());
@ -1059,6 +1071,49 @@ final class CloudWorkerService
return $operation;
}
/**
* Poll at most one due operation across the PHP-owned catalogs. Each
* service can only read or reconcile the operation ID persisted before
* dispatch; none has the source payload needed to replay an import.
*/
private function convergeOneCatalogImport(): bool
{
$catalogs = [
['template', [CloudTemplateCatalogService::class,
'reconcileOnePendingImport']],
['rescue_template', [CloudRescueTemplateImportService::class,
'reconcileOnePendingImport']],
['user_iso', [CloudUserIsoService::class,
'reconcileOnePendingImport']],
];
$count = count($catalogs);
for ($offset = 0; $offset < $count; $offset++) {
$index = ($this->catalogCursor + $offset) % $count;
[$label, $converge] = $catalogs[$index];
try {
if (!$converge()) {
continue;
}
$this->catalogCursor = ($index + 1) % $count;
return true;
} catch (\Throwable $error) {
// The callback may have claimed or polled an operation before
// failing, so do not run a second catalog action in this tick.
// An error alone is not "work": an idle worker must retain its
// normal sleep instead of spinning against a broken catalog.
$this->catalogCursor = ($index + 1) % $count;
$this->observedErrors[] = 'worker_tick_failed';
error_log('cloud worker ' . $label
. ' import reconciliation failed: ' . get_class($error));
return false;
}
}
// Shift the first probe even when every catalog is idle, so a catalog
// that becomes ready while cloud_jobs stays busy gets the next chance.
$this->catalogCursor = ($this->catalogCursor + 1) % $count;
return false;
}
private static function normalizeAgentOperation(array $operation): array
{
$error = $operation['error'] ?? null;

@ -463,12 +463,20 @@ async function loadCatalog() {
const url = rescue ? `${catalogPath()}?host_id=${encodeURIComponent(hostId)}` : catalogPath();
try { const entries = collection(await api(url), 'templates'); if (epoch !== state.epoch) return;
if (!entries.length) list.append(element('p', 'text-muted', '暂无模版。'));
for (const entry of entries) { const row = element('form', 'cloud-admin-row'); const name = element('input', 'form-control'); name.value = entry.name || ''; name.maxLength = 64; name.required = true; name.setAttribute('aria-label', '模版名称'); name.disabled = !canManage; const save = element('button', 'btn btn-outline-primary', '保存名称'); save.type = 'submit'; save.disabled = !canManage; const remove = element('button', 'btn btn-outline-danger', '删除'); remove.type = 'button'; remove.disabled = !canManage; const imageBytes = Number(entry.image_bytes || entry.size_bytes || 0);
for (const entry of entries) { const row = element('form', 'cloud-admin-row'); const catalogState = String(entry.state || '').toLowerCase(); const active = rescue || catalogState === 'active'; const name = element('input', 'form-control'); name.value = entry.name || ''; name.maxLength = 64; name.required = true; name.setAttribute('aria-label', '模版名称'); name.disabled = !canManage || !active; const save = element('button', 'btn btn-outline-primary', '保存名称'); save.type = 'submit'; save.disabled = !canManage || !active; const remove = element('button', 'btn btn-outline-danger', '删除'); remove.type = 'button'; remove.disabled = !canManage; const imageBytes = Number(entry.image_bytes || entry.size_bytes || 0);
row.append(name, save, remove);
if (!rescue) { const startup = element('button', 'btn btn-outline-secondary cloud-admin-startup-button', canManage ? '设置启动脚本' : '查看启动脚本'); startup.type = 'button'; startup.addEventListener('click', () => openStartupScript(entry, startup)); row.append(startup); }
row.append(element('small', 'cloud-admin-meta', `${entry.os_family || '镜像'} ${entry.os_version || ''} · ${imageBytes > 0 ? (imageBytes / 1024 ** 3).toFixed(2) + ' GiB' : entry.id}`));
if (!rescue) { const startup = element('button', 'btn btn-outline-secondary cloud-admin-startup-button', canManage ? '设置启动脚本' : '查看启动脚本'); startup.type = 'button'; startup.disabled = !active; startup.addEventListener('click', () => openStartupScript(entry, startup)); row.append(startup); }
const stateLabel = rescue ? '' : ({ active: '已就绪', importing: '导入中', error: '导入失败' }[catalogState] || '状态未知') + ' · ';
row.dataset.catalogState = catalogState || 'unknown';
row.append(element('small', 'cloud-admin-meta', `${stateLabel}${entry.os_family || '镜像'} ${entry.os_version || ''} · ${imageBytes > 0 ? (imageBytes / 1024 ** 3).toFixed(2) + ' GiB' : entry.id}`));
if (!rescue && Array.isArray(entry.replicas)) for (const replica of entry.replicas) {
const replicaState = String(replica?.state || 'unknown').toLowerCase();
const replicaLabel = ({ active: '已就绪', pending: '核对中', failed: '失败', uncertain: '待人工核对', stale: '等待同步' }[replicaState] || '状态未知');
const details = [replica?.error_code, replica?.operation_id].filter(value => String(value || '').trim() !== '').map(String);
row.append(element('small', 'cloud-admin-meta', `宿主机 #${Number(replica?.host_id || 0)} · ${replicaLabel}${details.length ? ' · ' + details.join(' · ') : ''}`));
}
if (rescue) { name.readOnly = true; save.hidden = true; save.disabled = true; }
row.addEventListener('submit', async event => { event.preventDefault(); if (rescue) return; if (state.busy) return notice('请等待当前导入任务完成。', true); save.disabled = true; try { await api(`${catalogPath()}/${encodeURIComponent(entry.id)}`, { method: 'PATCH', body: { name: name.value.trim(), expected_revision: Number(entry.revision), generation_token: String(entry.generation_token || '') } }); await loadCatalog(); notice('名称已保存。'); } catch (error) { notice(error.message, true); } finally { save.disabled = !canManage; } });
row.addEventListener('submit', async event => { event.preventDefault(); if (rescue) return; if (!active) return notice('镜像尚未就绪,不能修改名称。', true); if (state.busy) return notice('请等待当前导入任务完成。', true); save.disabled = true; try { await api(`${catalogPath()}/${encodeURIComponent(entry.id)}`, { method: 'PATCH', body: { name: name.value.trim(), expected_revision: Number(entry.revision), generation_token: String(entry.generation_token || '') } }); await loadCatalog(); notice('名称已保存。'); } catch (error) { notice(error.message, true); } finally { save.disabled = !canManage || !active; } });
remove.addEventListener('click', async () => {
if (state.busy) return notice('请等待当前导入任务完成。', true);
const message = rescue
@ -502,7 +510,7 @@ async function importProgress(result) {
const suffix = isRescueCatalog() ? `?host_id=${encodeURIComponent(state.selectedHost)}` : '';
const fallback = `${catalogPath()}/import-jobs/${encodeURIComponent(result.job_id)}${suffix}`;
const url = String(result.status_url || '').startsWith('/api/v1/') ? result.status_url : fallback;
for (let i = 0; i < 1800; i++) { const job = await api(url); notice(job.phase || `导入状态:${job.status}`); if (['succeeded', 'completed', 'done'].includes(job.status)) return; if (['failed', 'uncertain', 'cancelled'].includes(job.status)) throw new Error(job.error || '导入未完成,请核查宿主机状态后再操作。'); await new Promise(resolve => setTimeout(resolve, 2000)); } throw new Error('导入仍在进行,请稍后刷新查看。不要重复提交。');
for (let i = 0; i < 3600; i++) { const job = await api(url); notice(job.phase || `导入状态:${job.status}`); if (['succeeded', 'completed', 'done'].includes(job.status)) return; if (['failed', 'uncertain', 'cancelled'].includes(job.status)) throw new Error(job.error || '导入未完成,请核查宿主机状态后再操作。'); await new Promise(resolve => setTimeout(resolve, 2000)); } throw new Error('导入仍在进行,请稍后刷新查看。不要重复提交。');
}
async function importTemplate(event) {
event.preventDefault(); if (state.busy) return;

@ -23,7 +23,7 @@ class Element {
const flush = async () => { for (let turn = 0; turn < 20; turn += 1) await Promise.resolve(); };
const generation = '0123456789abcdef0123456789abcdef';
const entry = { id: 'debian-12-template', name: '<img src=x>', revision: 1, generation_token: generation };
const entry = { id: 'debian-12-template', name: '<img src=x>', state: 'active', image_bytes: 1073741824, revision: 1, generation_token: generation };
const payload = { id: entry.id, startup_script: '#!/bin/bash\necho ready\n', revision: 2, generation_token: generation };
const submit = { preventDefault() {} };
@ -57,6 +57,9 @@ function fixture(canManage = true) {
const button = test.get('catalog-list').children[0].children
.find(item => item.textContent === '设置启动脚本');
assert.ok(button, 'Cloud template row has no startup-script control.');
const metadata = test.get('catalog-list').children[0].children
.find(item => item.className === 'cloud-admin-meta');
assert.match(metadata.textContent, /^已就绪 · .*1\.00 GiB$/);
const opening = button.listeners.get('click')();
assert.equal(test.get('template-startup-title').textContent, `启动脚本 · ${entry.name}`);
assert.equal(test.get('template-startup-save').disabled, true);
@ -83,6 +86,29 @@ function fixture(canManage = true) {
assert.equal(button.focused, true);
}
{
const test = fixture();
const loading = test.loadCatalog();
await test.respond(test.requests[0], { templates: [{
...entry,
state: 'importing',
image_bytes: 0,
replicas: [{ host_id: 7, state: 'pending', error_code: 'agent_unavailable', operation_id: 'operation-1234' }],
}] });
await loading;
const row = test.get('catalog-list').children[0];
const save = row.children.find(item => item.textContent === '保存名称');
const remove = row.children.find(item => item.textContent === '删除');
const startup = row.children.find(item => item.textContent === '设置启动脚本');
const metadata = row.children.find(item => item.className === 'cloud-admin-meta');
assert.equal(row.dataset.catalogState, 'importing');
assert.equal(save.disabled, true);
assert.equal(startup.disabled, true);
assert.equal(remove.disabled, false, 'An operator must retain access to the catalog deletion workflow.');
assert.match(metadata.textContent, /^导入中 · .*debian-12-template$/);
assert.match(row.children.at(-1).textContent, /^宿主机 #7 · 核对中 · agent_unavailable · operation-1234$/);
}
{
const test = fixture(false);
const opening = test.openStartupScript(entry, new Element());

@ -0,0 +1,484 @@
<?php
declare(strict_types=1);
namespace App\Services\Cloud {
final class CloudSecrets
{
public static function encrypt(array $payload, string $purpose): string
{
return 'test-sealed:' . $purpose;
}
}
final class CloudDaemonActor
{
public static function resolve(string $sessionId): ?array
{
return [
'user_id' => 1,
'is_admin' => true,
'is_super_admin' => true,
'can_manage' => true,
'session_id' => $sessionId,
];
}
}
final class AuxCatalogWorkerFakeAgent
{
public array $requests = [];
public array $getResponses = [];
public array $reconcileResponses = [];
public array $getErrors = [];
public $onRequest = null;
public function request(string $method, string $target, array $actor,
?array $payload = null): array
{
$this->requests[] = compact('method', 'target', 'actor', 'payload');
if (is_callable($this->onRequest)) {
($this->onRequest)($method, $target, $actor, $payload);
}
if (preg_match('#^/v1/operations/([A-Za-z0-9_-]+)$#D',
$target, $match) === 1 && $method === 'GET') {
$operationId = rawurldecode($match[1]);
if (isset($this->getErrors[$operationId])) {
throw $this->getErrors[$operationId];
}
return ['operation_id' => $operationId]
+ ($this->getResponses[$operationId]
?? ['status' => 'running']);
}
if (preg_match(
'#^/v1/operations/([A-Za-z0-9_-]+)/reconcile$#D',
$target, $match) === 1 && $method === 'POST') {
$operationId = rawurldecode($match[1]);
return ['operation_id' => $operationId]
+ ($this->reconcileResponses[$operationId]
?? ['status' => 'uncertain']);
}
if ($method === 'POST' && $target === '/v1/templates') {
return [
'operation_id' => (string) $payload['operation_id'],
'status' => 'succeeded',
'result' => [
'id' => (string) $payload['id'],
'revision' => (int) $payload['revision'],
'generation_token' =>
(string) $payload['generation_token'],
'size_bytes' => 4096,
'sha256' => str_repeat('a', 64),
],
];
}
if ($method === 'POST' && $target === '/v1/user-isos') {
return [
'operation_id' => (string) $payload['operation_id'],
'status' => 'succeeded',
'result' => [
'id' => (string) $payload['id'],
'owner_id' => (int) $payload['owner_id'],
'size_bytes' => 8192,
'sha256' => str_repeat('b', 64),
],
];
}
throw new \RuntimeException('Background worker replayed an import');
}
}
final class CloudAgentSettingsService
{
public static ?AuxCatalogWorkerFakeAgent $client = null;
public static array $hostIds = [11];
public static function enabledHostIds(): array
{
return self::$hostIds;
}
public static function clientForHost(int $hostId,
float $timeoutSeconds = 20.0): AuxCatalogWorkerFakeAgent
{
if (!in_array($hostId, self::$hostIds, true)
|| self::$client === null) {
throw new \RuntimeException('Missing fake Agent');
}
return self::$client;
}
}
}
namespace {
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudRescueTemplateImport;
use App\Models\CloudTemplate;
use App\Models\CloudTemplateReplica;
use App\Models\CloudUserIso;
use App\Models\CloudUserIsoReplica;
use App\Services\Cloud\AuxCatalogWorkerFakeAgent;
use App\Services\Cloud\CloudAgentError;
use App\Services\Cloud\CloudAgentSettingsService;
use App\Services\Cloud\CloudRescueTemplateImportService;
use App\Services\Cloud\CloudTemplateCatalogService;
use App\Services\Cloud\CloudUserIsoService;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$capsule = new Capsule();
$capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'],
'default');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$schema = $capsule->schema('default');
$schema->create('cloud_templates', static function (Blueprint $table): void {
$table->string('id')->primary();
$table->string('name');
$table->string('os_family');
$table->string('os_version');
$table->string('source_type');
$table->text('sealed_source_url');
$table->integer('image_bytes')->default(0);
$table->string('sha256')->default('');
$table->text('startup_script')->default('');
$table->integer('revision');
$table->string('generation_token');
$table->string('state');
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_template_replicas',
static function (Blueprint $table): void {
$table->integer('host_id');
$table->string('template_id');
$table->integer('revision');
$table->string('generation_token');
$table->string('state');
$table->integer('image_bytes')->default(0);
$table->string('sha256')->default('');
$table->string('operation_id')->default('');
$table->string('error_code')->default('');
$table->integer('created_at');
$table->integer('updated_at');
$table->primary(['host_id', 'template_id']);
});
$schema->create('cloud_rescue_template_imports',
static function (Blueprint $table): void {
$table->string('operation_id')->primary();
$table->integer('host_id');
$table->string('template_id');
$table->string('import_type');
$table->string('state');
$table->string('error_code')->default('');
$table->integer('created_at');
$table->integer('updated_at');
$table->unique(['host_id', 'template_id']);
});
$schema->create('cloud_user_isos', static function (Blueprint $table): void {
$table->string('id')->primary();
$table->integer('user_id');
$table->string('name');
$table->text('sealed_source_url');
$table->integer('size_bytes')->default(0);
$table->string('sha256')->default('');
$table->string('state');
$table->string('import_operation_id');
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_user_iso_replicas',
static function (Blueprint $table): void {
$table->integer('host_id');
$table->string('user_iso_id');
$table->integer('user_id');
$table->string('state');
$table->integer('size_bytes')->default(0);
$table->string('sha256')->default('');
$table->string('operation_id')->default('');
$table->string('error_code')->default('');
$table->integer('created_at');
$table->integer('updated_at');
$table->primary(['host_id', 'user_iso_id']);
});
$agent = new AuxCatalogWorkerFakeAgent();
CloudAgentSettingsService::$client = $agent;
$now = time();
// Both global catalogs must persist their complete target-host snapshot
// before the first host mutation. A crash at any dispatch boundary then
// leaves one durable operation per expected host, never a partial set.
CloudAgentSettingsService::$hostIds = [11, 12];
$agent->onRequest = static function (string $method, string $target,
array $actor, ?array $payload) use ($assert): void {
if ($method !== 'POST') {
return;
}
if ($target === '/v1/templates') {
$hostIds = CloudTemplateReplica::query()
->where('template_id', (string) $payload['id'])
->orderBy('host_id')->pluck('host_id')->map(
static fn ($value): int => (int) $value
)->all();
$assert($hostIds === [11, 12],
'Template dispatch began before every replica was reserved.');
}
if ($target === '/v1/user-isos') {
$hostIds = CloudUserIsoReplica::query()
->where('user_iso_id', (string) $payload['id'])
->orderBy('host_id')->pluck('host_id')->map(
static fn ($value): int => (int) $value
)->all();
$assert($hostIds === [11, 12],
'ISO dispatch began before every replica was reserved.');
}
};
$templateReservation = CloudTemplateCatalogService::importRemote([
'name' => 'Debian 13 atomic reservation',
'url' => 'https://images.example/debian-13.qcow2',
'sha256' => '',
], ['user_id' => 1, 'is_admin' => true, 'can_manage' => true]);
$isoReservation = CloudUserIsoService::import([
'name' => 'Atomic reservation ISO',
'url' => 'https://images.example/atomic-reservation.iso',
], ['user_id' => 90, 'is_admin' => false, 'can_manage' => false]);
$assert($templateReservation['status'] === 'succeeded'
&& $isoReservation['status'] === 'succeeded',
'Atomic catalog reservations did not complete in the test fixture.');
$agent->onRequest = null;
$agent->requests = [];
CloudAgentSettingsService::$hostIds = [11];
$rescueOperation = str_repeat('a', 32);
$freshRescueOperation = str_repeat('b', 32);
$rescueTemplate = str_repeat('1', 32);
$freshRescueTemplate = str_repeat('2', 32);
Capsule::table('cloud_rescue_template_imports')->insert([
[
'operation_id' => $rescueOperation,
'host_id' => 11,
'template_id' => $rescueTemplate,
'import_type' => 'remote',
'state' => 'running',
'error_code' => '',
'created_at' => $now - 30,
'updated_at' => $now - 30,
],
[
'operation_id' => $freshRescueOperation,
'host_id' => 11,
'template_id' => $freshRescueTemplate,
'import_type' => 'remote',
'state' => 'running',
'error_code' => '',
'created_at' => $now,
'updated_at' => $now,
],
]);
$agent->getResponses[$rescueOperation] = [
'status' => 'uncertain',
'error' => ['code' => 'operation_uncertain'],
];
$agent->reconcileResponses[$rescueOperation] = [
'status' => 'succeeded',
'result' => ['id' => $rescueTemplate],
];
$assert(CloudRescueTemplateImportService::reconcileOnePendingImport(),
'Rescue import worker did not claim one due operation.');
$rescue = CloudRescueTemplateImport::find($rescueOperation);
$freshRescue = CloudRescueTemplateImport::find($freshRescueOperation);
$assert((string) $rescue->state === 'succeeded'
&& (string) $freshRescue->state === 'running'
&& count($agent->requests) === 2
&& $agent->requests[0]['method'] === 'GET'
&& $agent->requests[0]['target']
=== '/v1/operations/' . $rescueOperation
&& $agent->requests[1]['method'] === 'POST'
&& $agent->requests[1]['target']
=== '/v1/operations/' . $rescueOperation . '/reconcile'
&& ($agent->requests[0]['actor']['session_id'] ?? null)
=== 'cloud-rescue-template-import-worker'
&& ($agent->requests[0]['actor']['operation_id'] ?? null)
=== $rescueOperation,
'Rescue worker did not preserve the saved operation boundary.');
$assert(!CloudRescueTemplateImportService::reconcileOnePendingImport(),
'Fresh rescue import bypassed the five-second poll backoff.');
Capsule::table('cloud_rescue_template_imports')
->where('operation_id', $freshRescueOperation)
->update(['updated_at' => $now - 30]);
$agent->getErrors[$freshRescueOperation] = new CloudAgentError(
'temporary failure', 503, 'agent_transport_error', true, 503
);
try {
CloudRescueTemplateImportService::reconcileOnePendingImport();
throw new RuntimeException('Rescue transport error was swallowed.');
} catch (CloudAgentError $error) {
$assert($error->agentCode === 'agent_transport_error',
'Rescue worker threw the wrong transport error.');
}
$freshRescue->refresh();
$assert((string) $freshRescue->state === 'running'
&& (string) $freshRescue->error_code === 'agent_unavailable'
&& !CloudRescueTemplateImportService::reconcileOnePendingImport(),
'Rescue transport failure was not persisted with poll backoff.');
$insertIso = static function (string $id, string $jobId, int $userId,
string $state, int $updatedAt): void {
Capsule::table('cloud_user_isos')->insert([
'id' => $id,
'user_id' => $userId,
'name' => $id,
'sealed_source_url' => 'test-sealed-source',
'size_bytes' => 0,
'sha256' => '',
'state' => $state,
'import_operation_id' => $jobId,
'created_at' => $updatedAt,
'updated_at' => $updatedAt,
]);
};
$insertIsoReplica = static function (string $id, int $userId,
string $operationId, string $state, int $updatedAt,
int $bytes = 0, string $sha256 = ''): void {
Capsule::table('cloud_user_iso_replicas')->insert([
'host_id' => 11,
'user_iso_id' => $id,
'user_id' => $userId,
'state' => $state,
'size_bytes' => $bytes,
'sha256' => $sha256,
'operation_id' => $operationId,
'error_code' => '',
'created_at' => $updatedAt,
'updated_at' => $updatedAt,
]);
};
$isoId = str_repeat('3', 32);
$freshIsoId = str_repeat('4', 32);
$healIsoId = str_repeat('5', 32);
$isoOperation = str_repeat('c', 32);
$freshIsoOperation = str_repeat('d', 32);
$digest = str_repeat('e', 64);
$insertIso($isoId, str_repeat('6', 32), 42, 'importing', $now - 30);
$insertIsoReplica($isoId, 42, $isoOperation, 'pending', $now - 30);
$insertIso($freshIsoId, str_repeat('7', 32), 43, 'importing', $now);
$insertIsoReplica($freshIsoId, 43, $freshIsoOperation, 'pending', $now);
$insertIso($healIsoId, str_repeat('8', 32), 44, 'importing', $now - 20);
$insertIsoReplica($healIsoId, 44, str_repeat('f', 32), 'active',
$now - 20, 4096, $digest);
Capsule::table('cloud_user_isos')->where('id', $healIsoId)->update([
'size_bytes' => 4096,
'sha256' => $digest,
]);
$agent->getResponses[$isoOperation] = [
'status' => 'uncertain',
'error' => ['code' => 'operation_uncertain'],
];
$agent->reconcileResponses[$isoOperation] = [
'status' => 'succeeded',
'result' => [
'id' => $isoId,
'owner_id' => 42,
'size_bytes' => 8192,
'sha256' => $digest,
],
];
$requestsBeforeIso = count($agent->requests);
$assert(CloudUserIsoService::reconcileOnePendingImport(),
'ISO worker did not claim one due replica.');
$iso = CloudUserIso::find($isoId);
$isoReplica = CloudUserIsoReplica::query()->where('host_id', 11)
->where('user_iso_id', $isoId)->first();
$freshIsoReplica = CloudUserIsoReplica::query()->where('host_id', 11)
->where('user_iso_id', $freshIsoId)->first();
$isoRequests = array_slice($agent->requests, $requestsBeforeIso);
$assert((string) $iso->state === 'active'
&& (string) $isoReplica->state === 'active'
&& (string) $freshIsoReplica->state === 'pending'
&& count($isoRequests) === 2
&& $isoRequests[0]['method'] === 'GET'
&& $isoRequests[0]['target'] === '/v1/operations/' . $isoOperation
&& $isoRequests[1]['method'] === 'POST'
&& $isoRequests[1]['target']
=== '/v1/operations/' . $isoOperation . '/reconcile'
&& ($isoRequests[0]['actor']['session_id'] ?? null)
=== 'cloud-user-iso-import-worker'
&& ($isoRequests[0]['actor']['operation_id'] ?? null)
=== $isoOperation,
'ISO worker did not preserve one saved operation boundary.');
$requestsBeforeHeal = count($agent->requests);
$assert(CloudUserIsoService::reconcileOnePendingImport()
&& (string) CloudUserIso::find($healIsoId)->state === 'active'
&& count($agent->requests) === $requestsBeforeHeal,
'ISO worker did not repair an all-active parent locally.');
$assert(!CloudUserIsoService::reconcileOnePendingImport(),
'Fresh ISO replica bypassed the five-second poll backoff.');
Capsule::table('cloud_user_iso_replicas')
->where('user_iso_id', $freshIsoId)
->update(['updated_at' => $now - 30]);
$agent->getErrors[$freshIsoOperation] = new CloudAgentError(
'temporary failure', 503, 'agent_transport_error', true, 503
);
try {
CloudUserIsoService::reconcileOnePendingImport();
throw new RuntimeException('ISO transport error was swallowed.');
} catch (CloudAgentError $error) {
$assert($error->agentCode === 'agent_transport_error',
'ISO worker threw the wrong transport error.');
}
$freshIso = CloudUserIso::find($freshIsoId);
$freshIsoReplica->refresh();
$assert((string) $freshIso->state === 'importing'
&& (string) $freshIsoReplica->state === 'pending'
&& (string) $freshIsoReplica->error_code === 'agent_unavailable'
&& !CloudUserIsoService::reconcileOnePendingImport(),
'ISO transport failure was not persisted with poll backoff.');
$orphanTemplateId = 'debian-orphan-parent';
Capsule::table('cloud_templates')->insert([
'id' => $orphanTemplateId,
'name' => 'Orphan template',
'os_family' => 'Debian',
'os_version' => '13',
'source_type' => 'remote',
'sealed_source_url' => 'test-sealed:template-source',
'image_bytes' => 0,
'sha256' => '',
'startup_script' => '',
'revision' => 1,
'generation_token' => str_repeat('0', 32),
'state' => 'importing',
'created_at' => $now - 180,
'updated_at' => $now - 180,
]);
$orphanIsoId = str_repeat('9', 32);
$insertIso($orphanIsoId, str_repeat('0', 32), 45, 'importing',
$now - 180);
$requestsBeforeOrphans = count($agent->requests);
$assert(CloudTemplateCatalogService::reconcileOnePendingImport()
&& (string) CloudTemplate::find($orphanTemplateId)->state === 'error'
&& CloudUserIsoService::reconcileOnePendingImport()
&& (string) CloudUserIso::find($orphanIsoId)->state === 'failed'
&& count($agent->requests) === $requestsBeforeOrphans,
'Replica-free legacy parents were not failed without replay.');
foreach ($agent->requests as $request) {
$assert(!in_array($request['target'],
['/v1/rescue-templates', '/v1/user-isos'], true),
'Background convergence replayed an original import POST.');
}
echo "Rescue and user ISO background import convergence passed.\n";
}

@ -146,16 +146,46 @@ $cleared = CloudTemplateCatalogService::saveStartupScript(
$assert($cleared['startup_script'] === '' && $cleared['revision'] === 3,
'An empty script did not disable first-boot execution.');
$operationBeforeBlockedMutation = (string) CloudTemplateReplica::query()
->where('host_id', 1)->where('template_id', 'debian-12-template')
->value('operation_id');
foreach (['importing', 'error'] as $blockedState) {
Capsule::table('cloud_templates')->where('id', 'debian-12-template')
->update(['state' => $blockedState]);
$expect(static fn () => CloudTemplateCatalogService::rename(
'debian-12-template', 'Blocked rename', 3, $generation
), 409, 'A non-active template was renamed.');
$expect(static fn () => CloudTemplateCatalogService::saveStartupScript(
'debian-12-template', 'blocked script', 3, $generation
), 409, 'A non-active template startup script was changed.');
$blocked = CloudTemplate::find('debian-12-template');
$assert((int) $blocked->revision === 3
&& (string) $blocked->name === 'Debian 12'
&& (string) $blocked->startup_script === ''
&& (string) CloudTemplateReplica::query()->where('host_id', 1)
->where('template_id', 'debian-12-template')
->value('operation_id') === $operationBeforeBlockedMutation,
'A blocked metadata mutation changed template or replica identity.');
}
Capsule::table('cloud_templates')->where('id', 'debian-12-template')
->update(['state' => 'active']);
$public = CloudTemplateCatalogService::publicCatalog()['templates'][0];
$admin = CloudTemplateCatalogService::adminCatalog()['templates'][0];
foreach (['startup_script', 'startup_script_configured', 'revision',
'generation_token', 'source_url', 'sealed_source_url', 'sha256']
'generation_token', 'source_url', 'sealed_source_url', 'sha256', 'replicas']
as $privateField) {
$assert(!array_key_exists($privateField, $public),
'Tenant catalog exposed private template field: ' . $privateField);
}
$assert($admin['startup_script_configured'] === false
&& !array_key_exists('startup_script', $admin),
&& !array_key_exists('startup_script', $admin)
&& $admin['replicas'] === [[
'host_id' => 1,
'state' => 'active',
'error_code' => '',
'operation_id' => $operationBeforeBlockedMutation,
]],
'Admin catalog must show script state without returning script content.');
$row = CloudTemplate::find('debian-12-template');
$replica = CloudTemplateReplica::query()->where('host_id', 1)

@ -56,6 +56,7 @@ namespace {
use App\Models\CloudTemplateReplica;
use App\Services\Cloud\CloudAgentSettingsService;
use App\Services\Cloud\CloudError;
use App\Services\Cloud\CloudTemplateCatalogService;
use App\Services\Cloud\TemplateMetadataSyncFakeAgent;
use Illuminate\Database\Capsule\Manager as Capsule;
@ -66,12 +67,31 @@ namespace {
throw new RuntimeException($message);
}
};
$expect = static function (callable $callable, int $status,
string $message) use ($assert): void {
try {
$callable();
} catch (CloudError $error) {
$assert($error->httpStatus === $status,
$message . ' Unexpected status: ' . $error->httpStatus);
return;
}
throw new RuntimeException($message);
};
$capsule = new Capsule();
$capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'],
'default');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$capsule->schema('default')->create('cloud_templates',
static function (Blueprint $table): void {
$table->string('id')->primary();
$table->string('name');
$table->integer('revision');
$table->string('generation_token');
$table->string('state');
});
$capsule->schema('default')->create('cloud_template_replicas',
static function (Blueprint $table): void {
$table->integer('host_id');
@ -91,6 +111,13 @@ namespace {
$generation = str_repeat('a', 32);
$id = 'debian-13-metadata-sync';
$now = time();
Capsule::table('cloud_templates')->insert([
'id' => $id,
'name' => 'Debian 13 renamed',
'revision' => 2,
'generation_token' => $generation,
'state' => 'active',
]);
Capsule::table('cloud_template_replicas')->insert([
'host_id' => 7,
'template_id' => $id,
@ -118,6 +145,14 @@ namespace {
'generation_token' => $generation,
];
$actor = ['user_id' => 1, 'can_manage' => true];
$setAuthority = static function (int $revision, string $name,
string $state = 'active') use ($id): void {
Capsule::table('cloud_templates')->where('id', $id)->update([
'name' => $name,
'revision' => $revision,
'state' => $state,
]);
};
$agent = new TemplateMetadataSyncFakeAgent();
CloudAgentSettingsService::$client = $agent;
@ -139,6 +174,7 @@ namespace {
$agent->mode = 'wrong_operation';
$agent->result = $result(3, 'Debian 13 wrong operation');
$setAuthority(3, 'Debian 13 wrong operation');
$mismatch = CloudTemplateCatalogService::synchronizeMetadata(
$snapshot(3, 'Debian 13 wrong operation'), $actor
);
@ -163,6 +199,7 @@ namespace {
]);
};
$agent->result = $result(4, 'Debian 13 older response');
$setAuthority(4, 'Debian 13 older response');
CloudTemplateCatalogService::synchronizeMetadata(
$snapshot(4, 'Debian 13 older response'), $actor
);
@ -175,6 +212,7 @@ namespace {
$agent->mode = 'uncertain';
$agent->beforeResponse = null;
$agent->result = $result(6, 'Debian 13 reconciled');
$setAuthority(6, 'Debian 13 reconciled');
$before = count($agent->requests);
$reconciled = CloudTemplateCatalogService::synchronizeMetadata(
$snapshot(6, 'Debian 13 reconciled'), $actor
@ -189,5 +227,16 @@ namespace {
&& str_ends_with($requests[1]['target'], '/reconcile'),
'Uncertain metadata synchronization was replayed or not reconciled.');
$setAuthority(7, 'Debian 13 blocked', 'importing');
$requestsBeforeBlocked = count($agent->requests);
$operationBeforeBlocked = (string) $row->operation_id;
$expect(static fn () => CloudTemplateCatalogService::synchronizeMetadata(
$snapshot(7, 'Debian 13 blocked'), $actor
), 409, 'A non-active template dispatched metadata synchronization.');
$row->refresh();
$assert(count($agent->requests) === $requestsBeforeBlocked
&& (string) $row->operation_id === $operationBeforeBlocked,
'Blocked metadata synchronization changed the replica operation ID.');
echo "Template metadata operation binding and CAS tests passed.\n";
}

@ -0,0 +1,240 @@
<?php
declare(strict_types=1);
namespace App\Services\Cloud {
final class CloudDaemonActor
{
public static function resolve(string $sessionId): ?array
{
return [
'user_id' => 1,
'is_admin' => true,
'is_super_admin' => true,
'can_manage' => true,
'session_id' => $sessionId,
];
}
}
final class TemplateWorkerFakeAgent
{
public array $requests = [];
public array $results = [];
public function request(string $method, string $target, array $actor,
?array $payload = null): array
{
$this->requests[] = compact('method', 'target', 'actor', 'payload');
if (preg_match('#^/v1/operations/([A-Za-z0-9_-]+)$#D',
$target, $match) === 1 && $method === 'GET') {
$operationId = rawurldecode($match[1]);
return [
'operation_id' => $operationId,
'status' => 'uncertain',
'error' => ['code' => 'operation_uncertain'],
];
}
if (preg_match('#^/v1/operations/([A-Za-z0-9_-]+)/reconcile$#D',
$target, $match) === 1 && $method === 'POST') {
$operationId = rawurldecode($match[1]);
return [
'operation_id' => $operationId,
'status' => 'succeeded',
'result' => $this->results[$operationId],
];
}
throw new \RuntimeException('Template worker replayed an import');
}
}
final class CloudAgentSettingsService
{
public static ?TemplateWorkerFakeAgent $client = null;
public static function clientForHost(int $hostId,
float $timeoutSeconds = 20.0): TemplateWorkerFakeAgent
{
if ($hostId !== 11 || self::$client === null) {
throw new \RuntimeException('Missing fake Agent');
}
return self::$client;
}
}
}
namespace {
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudTemplate;
use App\Models\CloudTemplateReplica;
use App\Services\Cloud\CloudAgentSettingsService;
use App\Services\Cloud\CloudWorkerService;
use App\Services\Cloud\TemplateWorkerFakeAgent;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$capsule = new Capsule();
$capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'],
'default');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$schema = $capsule->schema('default');
$schema->create('cloud_jobs', static function (Blueprint $table): void {
$table->increments('id');
$table->string('kind');
$table->string('status');
$table->integer('available_at');
$table->integer('lease_until');
});
$schema->create('cloud_templates', static function (Blueprint $table): void {
$table->string('id')->primary();
$table->string('name');
$table->string('os_family');
$table->string('os_version');
$table->string('source_type');
$table->text('sealed_source_url');
$table->integer('image_bytes')->default(0);
$table->string('sha256')->default('');
$table->text('startup_script')->default('');
$table->integer('revision');
$table->string('generation_token');
$table->string('state');
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_template_replicas',
static function (Blueprint $table): void {
$table->integer('host_id');
$table->string('template_id');
$table->integer('revision');
$table->string('generation_token');
$table->string('state');
$table->integer('image_bytes')->default(0);
$table->string('sha256')->default('');
$table->string('operation_id')->default('');
$table->string('error_code')->default('');
$table->integer('created_at');
$table->integer('updated_at');
$table->primary(['host_id', 'template_id']);
});
$now = time();
$digest = str_repeat('c', 64);
$operationA = str_repeat('a', 32);
$operationB = str_repeat('b', 32);
$generationA = str_repeat('1', 32);
$generationB = str_repeat('2', 32);
$insertTemplate = static function (string $id, string $generation,
string $digest, string $state, int $updatedAt) use ($now): void {
Capsule::table('cloud_templates')->insert([
'id' => $id,
'name' => $id,
'os_family' => 'Debian',
'os_version' => '13',
'source_type' => 'remote',
'sealed_source_url' => 'test-sealed-source',
'image_bytes' => 0,
'sha256' => $digest,
'startup_script' => '',
'revision' => 1,
'generation_token' => $generation,
'state' => $state,
'created_at' => $now,
'updated_at' => $updatedAt,
]);
};
$insertReplica = static function (string $id, string $generation,
string $digest, string $operationId, string $state,
int $updatedAt): void {
Capsule::table('cloud_template_replicas')->insert([
'host_id' => 11,
'template_id' => $id,
'revision' => 1,
'generation_token' => $generation,
'state' => $state,
'image_bytes' => $state === 'active' ? 8192 : 0,
'sha256' => $digest,
'operation_id' => $operationId,
'error_code' => '',
'created_at' => $updatedAt,
'updated_at' => $updatedAt,
]);
};
$insertTemplate('debian-a', $generationA, $digest, 'importing', $now - 30);
$insertReplica('debian-a', $generationA, $digest, $operationA,
'pending', $now - 30);
$insertTemplate('debian-b', $generationB, $digest, 'importing', $now - 20);
$insertReplica('debian-b', $generationB, $digest, $operationB,
'pending', $now - 20);
$agent = new TemplateWorkerFakeAgent();
$agent->results[$operationA] = [
'id' => 'debian-a',
'revision' => 1,
'generation_token' => $generationA,
'size_bytes' => 8192,
'sha256' => $digest,
];
CloudAgentSettingsService::$client = $agent;
$policy = new CloudWorkerService(null, true);
$assert($policy->tick() === false && $agent->requests === [],
'The policy-only worker consumed a template import.');
$worker = new CloudWorkerService();
$assert($worker->tick() === true && $worker->observationErrors() === [],
'The ordinary idle worker did not converge one template import.');
$templateA = CloudTemplate::find('debian-a');
$replicaA = CloudTemplateReplica::query()->where('host_id', 11)
->where('template_id', 'debian-a')->first();
$templateB = CloudTemplate::find('debian-b');
$replicaB = CloudTemplateReplica::query()->where('host_id', 11)
->where('template_id', 'debian-b')->first();
$assert((string) $templateA->state === 'active'
&& (int) $templateA->image_bytes === 8192
&& (string) $templateA->sha256 === $digest
&& (string) $replicaA->state === 'active'
&& (string) $templateB->state === 'importing'
&& (string) $replicaB->state === 'pending',
'One bounded worker pass changed the wrong import or lost image size.');
$assert(count($agent->requests) === 2
&& $agent->requests[0]['method'] === 'GET'
&& $agent->requests[0]['target'] === '/v1/operations/' . $operationA
&& $agent->requests[1]['method'] === 'POST'
&& $agent->requests[1]['target']
=== '/v1/operations/' . $operationA . '/reconcile'
&& ($agent->requests[0]['actor']['session_id'] ?? null)
=== 'cloud-template-import-worker'
&& ($agent->requests[0]['actor']['operation_id'] ?? null)
=== $operationA,
'Worker lookup/reconciliation did not preserve the saved operation ID.');
foreach ($agent->requests as $request) {
$assert($request['target'] !== '/v1/templates',
'The worker replayed the original template import POST.');
}
Capsule::table('cloud_template_replicas')->where('template_id', 'debian-b')
->update(['state' => 'failed', 'error_code' => 'test_terminal']);
Capsule::table('cloud_templates')->where('id', 'debian-b')
->update(['state' => 'error']);
$generationC = str_repeat('3', 32);
$insertTemplate('debian-c', $generationC, $digest, 'importing', $now - 10);
$insertReplica('debian-c', $generationC, $digest, str_repeat('d', 32),
'active', $now - 10);
$requestsBeforeAggregateRepair = count($agent->requests);
$assert($worker->tick() === true
&& (string) CloudTemplate::find('debian-c')->state === 'active'
&& count($agent->requests) === $requestsBeforeAggregateRepair,
'Worker did not repair an all-active template aggregate locally.');
$assert($worker->tick() === false,
'A terminal failed template kept the idle worker in a busy loop.');
echo "Idle worker template import convergence and no-replay tests passed.\n";
}

@ -0,0 +1,213 @@
<?php
declare(strict_types=1);
namespace App\Services\Cloud {
// Catalog doubles make the worker scheduling contract observable without
// loading any production catalog implementation in this isolated process.
final class CloudTemplateCatalogService
{
public static int $calls = 0;
public static bool $fail = false;
public static function reconcileOnePendingImport(): bool
{
self::$calls++;
if (self::$fail) {
throw new \RuntimeException('template catalog unavailable');
}
return true;
}
}
final class CloudRescueTemplateImportService
{
public static int $calls = 0;
public static function reconcileOnePendingImport(): bool
{
self::$calls++;
return true;
}
}
final class CloudUserIsoService
{
public static int $calls = 0;
public static function reconcileOnePendingImport(): bool
{
self::$calls++;
return true;
}
}
// Test-only in-memory envelope storage; production CloudSecrets is not
// loaded in this process.
final class CloudSecrets
{
private static array $values = [];
public static function encrypt(array $value, string $purpose): string
{
$key = bin2hex(random_bytes(16));
self::$values[$key] = $value;
return 'worker-fairness:' . $key;
}
public static function decrypt(string $value, string $purpose): array
{
$key = substr($value, strlen('worker-fairness:'));
if (!isset(self::$values[$key])) {
throw new \RuntimeException('missing test payload');
}
return self::$values[$key];
}
}
}
namespace {
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudJob;
use App\Services\Cloud\CloudAgentClient;
use App\Services\Cloud\CloudRescueTemplateImportService;
use App\Services\Cloud\CloudSecrets;
use App\Services\Cloud\CloudTemplateCatalogService;
use App\Services\Cloud\CloudUserIsoService;
use App\Services\Cloud\CloudWorkerService;
use GuzzleHttp\Client;
use GuzzleHttp\Handler\MockHandler;
use GuzzleHttp\HandlerStack;
use GuzzleHttp\Psr7\Response;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$capsule = new Capsule();
$capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'],
'default');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$schema = $capsule->schema('default');
$schema->create('user', static function (Blueprint $table): void {
$table->increments('id');
$table->integer('enable')->default(1);
$table->integer('is_admin')->default(0);
});
$schema->create('cloud_jobs', static function (Blueprint $table): void {
$table->increments('id');
$table->string('operation_id')->unique();
$table->integer('order_id')->nullable();
$table->integer('instance_id')->nullable();
$table->integer('snapshot_id')->nullable();
$table->integer('user_id');
$table->integer('host_id');
$table->string('kind');
$table->string('status');
$table->text('sealed_payload');
$table->integer('attempts')->default(0);
$table->integer('dispatched_at')->nullable();
$table->string('lease_token')->nullable();
$table->integer('lease_until')->default(0);
$table->integer('available_at');
$table->string('error_code')->nullable();
$table->text('result_json')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
Capsule::table('user')->insert(['id' => 1, 'enable' => 1, 'is_admin' => 0]);
$responses = [];
$now = time();
for ($number = 1; $number <= 3; $number++) {
$operationId = str_repeat((string) $number, 32);
Capsule::table('cloud_jobs')->insert([
'operation_id' => $operationId,
'order_id' => null,
'instance_id' => null,
'snapshot_id' => null,
'user_id' => 1,
'host_id' => 7,
'kind' => 'action',
'status' => 'queued',
'sealed_payload' => CloudSecrets::encrypt([
'method' => 'POST',
'request_target' => '/v1/vms/test-vm/actions',
'body' => [
'operation_id' => $operationId,
'action' => 'start',
],
'owner_id' => 1,
], 'job-payload'),
'attempts' => 0,
'dispatched_at' => null,
'lease_token' => null,
'lease_until' => 0,
'available_at' => 0,
'error_code' => null,
'result_json' => null,
'created_at' => $now,
'updated_at' => $now,
]);
$responses[] = new Response(200, [], json_encode([
'operation_id' => $operationId,
'status' => 'dispatching',
], JSON_THROW_ON_ERROR));
}
$client = new CloudAgentClient(
'7',
'test-agent-secret-at-least-32-bytes',
new Client(['handler' => HandlerStack::create(new MockHandler($responses))]),
20.0,
'http://127.0.0.1:8001/internal/agent/request',
'test-gateway-secret-at-least-32-bytes'
);
$worker = new CloudWorkerService($client);
for ($tick = 1; $tick <= 3; $tick++) {
$assert($worker->tick(), 'A due cloud job was not processed.');
$assert(CloudJob::query()->where('status', 'running')->count() === $tick,
'The persistent cloud_jobs backlog did not advance one row.');
$assert(CloudTemplateCatalogService::$calls
+ CloudRescueTemplateImportService::$calls
+ CloudUserIsoService::$calls === $tick,
'One worker tick ran more than one catalog action.');
}
$assert(CloudTemplateCatalogService::$calls === 1
&& CloudRescueTemplateImportService::$calls === 1
&& CloudUserIsoService::$calls === 1,
'A continuously non-empty cloud_jobs queue starved a catalog.');
// A catalog exception is observable but is not work. Returning false keeps
// the command loop's one-second idle backoff and prevents a log/DB hot loop.
CloudTemplateCatalogService::$fail = true;
$assert($worker->tick() === false,
'A catalog exception incorrectly reported successful work.');
$assert($worker->observationErrors() === ['worker_tick_failed'],
'A catalog exception was not exposed through worker health.');
$assert(CloudTemplateCatalogService::$calls === 2
&& CloudRescueTemplateImportService::$calls === 1
&& CloudUserIsoService::$calls === 1,
'The worker ran a second catalog after an ambiguous catalog failure.');
$beforePolicy = [
CloudTemplateCatalogService::$calls,
CloudRescueTemplateImportService::$calls,
CloudUserIsoService::$calls,
];
$policyWorker = new CloudWorkerService(null, true);
$assert($policyWorker->tick() === false
&& $beforePolicy === [
CloudTemplateCatalogService::$calls,
CloudRescueTemplateImportService::$calls,
CloudUserIsoService::$calls,
], 'The policy-only worker was delayed by catalog convergence.');
echo "Cloud worker catalog fairness and failure backoff tests passed.\n";
}
Loading…
Cancel
Save