diff --git a/docs/VALIDATION.md b/docs/VALIDATION.md
index 54aee6cc..2b499704 100644
--- a/docs/VALIDATION.md
+++ b/docs/VALIDATION.md
@@ -208,3 +208,21 @@ authorized host mutation test: 待部署后在用户指定的测试环境按 ope
browser/payment/SMTP test: 自动化 PHP/Node 契约测试通过;真实创建付款需在最终提交动作前取得用户即时确认
known skips or risks: libvirt 调用已执行但结果不可证明时会安全保留 uncertain,必须复用原 operation_id 核对,禁止自动重放;macOS libvirt 链接仅有重复库警告
```
+
+### 2026-10-16 持久化目录任务收敛验收
+
+```text
+commit: 本记录所在 go 分支提交(基线 e7fa625)
+Go version / libvirt version / PHP version / MariaDB version: Go 1.27.1 / 本机动态链接测试沿用已安装 libvirt / PHP 8.5.11 / MariaDB 未在本机运行
+go vet: Agent 默认及 libvirt tag、Gateway 均通过
+go test: Agent 默认及 libvirt tag、Gateway 均通过;两个 Go 模块的 mod verify 均通过
+go test -race: Agent libvirt tag 与 Gateway 均通过;Gateway race 测试获准绑定本机临时回环端口
+Linux AMD64 build: 本提交未修改 Go 源码,继续使用 e7fa625 对应的已验证 Agent 与 Gateway 二进制
+PHP tests: 392 个 PHP 文件逐文件 lint 通过,119 个测试文件通过;SourceSyntax 独立检查 396 个文件通过;仅输出既有测试 warning 与第三方依赖 deprecation,退出码为 0
+Node tests: 89 个 JavaScript 文件语法检查通过,4 个行为测试通过;部署 JSON 示例有效
+systemd-analyze: 本提交未修改 service 文件,未重复执行
+read-only host capability check: HKG-A 出站 WSS 已稳定连接;Agent 报告 14 项必要能力、UEFI 可用且 Secure Boot 关闭
+authorized host mutation test: HTTPS Debian 13 云镜像导入在 Agent 端确定成功,436207616 字节,SHA-256 为 1010eb3215be632548cfb6793b53bd8243064d8820b5106f2becfcd179099b2f;本提交新增后台同 operation_id 收敛,禁止重放原始导入 POST
+browser/payment/SMTP test: 管理端网络选择器已由错误的 bridge:br0 修正为 Agent 声明的 public;旧创建任务已按原 operation_id 核对并明确失败退款;新的真实 10 USD 创建提交仍等待用户即时确认
+known skips or risks: worker 每 tick 最多处理一个普通任务和一个目录核对;真实创建、重装、快照恢复和删除属于外部或破坏性操作,必须在动作前取得确认后再完成联调
+```
diff --git a/web/app/Services/Cloud/CloudRescueTemplateImportService.php b/web/app/Services/Cloud/CloudRescueTemplateImportService.php
index 7d12ec37..3e28cb3b 100644
--- a/web/app/Services/Cloud/CloudRescueTemplateImportService.php
+++ b/web/app/Services/Cloud/CloudRescueTemplateImportService.php
@@ -10,6 +10,8 @@ use App\Models\CloudRescueTemplateImport;
*/
final class CloudRescueTemplateImportService
{
+ private const IMPORT_POLL_INTERVAL_SECONDS = 5;
+
public static function start(
int $hostId,
string $importType,
@@ -86,6 +88,101 @@ final class CloudRescueTemplateImportService
return self::pollIfAllowed($record, $actor);
}
+ /**
+ * Converge one already-dispatched rescue import without replaying it.
+ * The persisted operation ID is the only capability available here.
+ */
+ public static function reconcileOnePendingImport(): bool
+ {
+ $connection = (new CloudRescueTemplateImport())->getConnection();
+ if (!$connection->getSchemaBuilder()->hasTable(
+ 'cloud_rescue_template_imports'
+ )) {
+ return false;
+ }
+
+ $now = time();
+ $record = CloudRescueTemplateImport::query()
+ ->where(function ($query) use ($now): void {
+ $query->where(function ($running) use ($now): void {
+ $running->where('state', 'running')
+ ->where('updated_at', '<=', $now
+ - self::IMPORT_POLL_INTERVAL_SECONDS);
+ })->orWhere(function ($crossing) use ($now): void {
+ $crossing->whereIn('state', ['dispatching', 'sending'])
+ ->where('updated_at', '<=', $now - 90);
+ });
+ })->orderBy('updated_at')->orderBy('operation_id')->first();
+ if ($record === null) {
+ return false;
+ }
+
+ $state = (string) $record->state;
+ $operationId = (string) $record->operation_id;
+ $claimed = CloudRescueTemplateImport::query()
+ ->where('operation_id', $operationId)
+ ->where('host_id', (int) $record->host_id)
+ ->where('template_id', (string) $record->template_id)
+ ->where('state', $state)
+ ->where('updated_at', (int) $record->updated_at)
+ ->update(['updated_at' => $now]);
+ if ($claimed !== 1) {
+ return false;
+ }
+
+ if ($state === 'dispatching') {
+ self::mark($record, 'uncertain', 'import_dispatch_unconfirmed');
+ return true;
+ }
+ if ($state === 'sending') {
+ self::mark($record, 'running', 'import_dispatch_unconfirmed');
+ $record = self::reload($record);
+ }
+ if (preg_match('/^[A-Za-z0-9_-]{8,80}$/D', $operationId) !== 1) {
+ self::mark($record, 'uncertain', 'operation_id_invalid');
+ throw new CloudError(
+ '救援镜像导入操作标识无效,必须人工核查', 503
+ );
+ }
+
+ $actor = CloudDaemonActor::resolve(
+ 'cloud-rescue-template-import-worker'
+ );
+ if ($actor === null) {
+ self::mark($record, 'running', 'daemon_actor_unavailable');
+ throw new CloudError('救援镜像导入后台核对未配置管理员身份', 503);
+ }
+ $actor['operation_id'] = $operationId;
+
+ try {
+ $client = CloudAgentSettingsService::clientForHost(
+ (int) $record->host_id
+ );
+ $operation = $client->request('GET', '/v1/operations/'
+ . rawurlencode($operationId), $actor);
+ $operation = CloudAgentOperationReconciler::ifUncertain(
+ $client, $operationId, $actor, $operation
+ );
+ } catch (CloudAgentError $error) {
+ $missing = $error->httpStatus === 404
+ || $error->agentHttpStatus === 404;
+ self::mark($record, $missing ? 'uncertain' : 'running',
+ $missing ? 'operation_not_found' : 'agent_unavailable');
+ throw $error;
+ } catch (\Throwable $error) {
+ self::mark($record, 'running', 'agent_unavailable');
+ throw $error;
+ }
+
+ try {
+ self::applyOperation($record, $operation);
+ } catch (\Throwable $error) {
+ self::mark($record, 'uncertain', 'panel_persist_failed');
+ throw $error;
+ }
+ return true;
+ }
+
private static function reserve(string $operationId, int $hostId,
string $templateId, string $importType): CloudRescueTemplateImport
{
diff --git a/web/app/Services/Cloud/CloudTemplateCatalogService.php b/web/app/Services/Cloud/CloudTemplateCatalogService.php
index a7583629..2d7c2220 100644
--- a/web/app/Services/Cloud/CloudTemplateCatalogService.php
+++ b/web/app/Services/Cloud/CloudTemplateCatalogService.php
@@ -11,6 +11,7 @@ use App\Models\CloudOrder;
final class CloudTemplateCatalogService
{
public const STARTUP_SCRIPT_MAX_BYTES = 32768;
+ private const IMPORT_POLL_INTERVAL_SECONDS = 5;
public static function adminCatalog(): array
{
@@ -49,8 +50,9 @@ final class CloudTemplateCatalogService
if ($hostIds === []) {
throw new CloudError('没有已启用且已配置 Agent 的宿主机', 409);
}
+ $hostIds = array_values(array_unique(array_map('intval', $hostIds)));
$template = self::newImport($name, $family, $version, 'remote',
- $sourceUrl, 0, $sha256);
+ $sourceUrl, 0, $sha256, $hostIds);
return self::dispatchImport($template, $hostIds, $actor,
static fn (int $hostId): array => [
'source_url' => $sourceUrl,
@@ -72,6 +74,7 @@ final class CloudTemplateCatalogService
if ($hostIds === []) {
throw new CloudError('没有已启用且已配置 Agent 的宿主机', 409);
}
+ $hostIds = array_values(array_unique(array_map('intval', $hostIds)));
$digest = strtolower((string) ($file['sha256'] ?? ''));
$bytes = filter_var($file['size_bytes'] ?? null,
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]);
@@ -80,7 +83,7 @@ final class CloudTemplateCatalogService
throw new CloudError('本地云镜像校验结果无效', 503);
}
$template = self::newImport($name, $family, $version, 'upload',
- '', (int) $bytes, $digest);
+ '', (int) $bytes, $digest, $hostIds);
return self::dispatchImport($template, $hostIds, $actor,
static function (int $hostId) use ($file): array {
$staging = CloudUploadStagingService::issue(
@@ -147,6 +150,144 @@ final class CloudTemplateCatalogService
return self::importJobDto($template);
}
+ /**
+ * Converge one durable template import while the cloud_jobs queue is idle.
+ * The original source request is intentionally unavailable here: only the
+ * already-persisted Agent operation ID may be read or reconciled.
+ */
+ public static function reconcileOnePendingImport(): bool
+ {
+ $connection = (new CloudTemplateReplica())->getConnection();
+ $schema = $connection->getSchemaBuilder();
+ if (!$schema->hasTable('cloud_templates')
+ || !$schema->hasTable('cloud_template_replicas')) {
+ return false;
+ }
+
+ $now = time();
+ $replica = CloudTemplateReplica::query()
+ ->select('cloud_template_replicas.*')
+ ->join('cloud_templates', 'cloud_templates.id', '=',
+ 'cloud_template_replicas.template_id')
+ ->where('cloud_template_replicas.state', 'pending')
+ ->whereIn('cloud_templates.state', ['importing', 'error'])
+ ->where('cloud_template_replicas.updated_at', '<=',
+ $now - self::IMPORT_POLL_INTERVAL_SECONDS)
+ ->orderBy('cloud_template_replicas.updated_at')
+ ->orderBy('cloud_template_replicas.host_id')
+ ->orderBy('cloud_template_replicas.template_id')->first();
+
+ if ($replica === null) {
+ // Versions predating atomic parent/replica reservation could
+ // expose a parent row before the first host operation existed.
+ // There is no saved mutation to replay, so age it into an error.
+ $orphan = CloudTemplate::query()->where('state', 'importing')
+ ->where('updated_at', '<=', $now - 90)
+ ->whereNotExists(static function ($query): void {
+ $query->selectRaw('1')->from('cloud_template_replicas')
+ ->whereColumn('cloud_template_replicas.template_id',
+ 'cloud_templates.id');
+ })->orderBy('updated_at')->orderBy('id')->first();
+ if ($orphan !== null) {
+ return CloudTemplate::query()
+ ->where('id', (string) $orphan->id)
+ ->where('state', 'importing')
+ ->where('updated_at', (int) $orphan->updated_at)
+ ->update([
+ 'state' => 'error',
+ 'updated_at' => $now,
+ ]) === 1;
+ }
+
+ // Repair the narrow crash window where every replica committed but
+ // the parent aggregate did not. Failed or uncertain replicas are
+ // excluded so an idle worker cannot spin on a terminal import.
+ $template = CloudTemplate::query()
+ ->whereIn('state', ['importing', 'error'])
+ ->whereExists(static function ($query): void {
+ $query->selectRaw('1')->from('cloud_template_replicas')
+ ->whereColumn('cloud_template_replicas.template_id',
+ 'cloud_templates.id');
+ })->whereNotExists(static function ($query): void {
+ $query->selectRaw('1')->from('cloud_template_replicas')
+ ->whereColumn('cloud_template_replicas.template_id',
+ 'cloud_templates.id')
+ ->where('cloud_template_replicas.state', '<>', 'active');
+ })->orderBy('updated_at')->orderBy('id')->first();
+ if ($template === null) {
+ return false;
+ }
+ self::refreshImportState($template);
+ return true;
+ }
+
+ $hostId = (int) $replica->host_id;
+ $templateId = (string) $replica->template_id;
+ $operationId = (string) $replica->operation_id;
+ // updated_at is a schema-free claim and backoff. Concurrent workers
+ // can only duplicate a read, never the original import mutation.
+ $claimed = CloudTemplateReplica::query()->where('host_id', $hostId)
+ ->where('template_id', $templateId)
+ ->where('state', 'pending')->where('operation_id', $operationId)
+ ->where('updated_at', (int) $replica->updated_at)
+ ->update(['updated_at' => $now]);
+ if ($claimed !== 1) {
+ return false;
+ }
+
+ $template = CloudTemplate::query()->where('id', $templateId)
+ ->whereIn('state', ['importing', 'error'])->first();
+ if ($template === null) {
+ return true;
+ }
+ if (preg_match('/^[A-Za-z0-9_-]{8,80}$/D', $operationId) !== 1) {
+ self::markReplica($hostId, $templateId, $operationId,
+ ['pending'], 'uncertain', 'operation_id_invalid');
+ self::refreshImportState($template);
+ throw new CloudError('云镜像导入操作标识无效,必须人工核查', 503);
+ }
+ $actor = CloudDaemonActor::resolve('cloud-template-import-worker');
+ if ($actor === null) {
+ self::markReplica($hostId, $templateId, $operationId,
+ ['pending'], 'pending', 'daemon_actor_unavailable');
+ throw new CloudError('云镜像导入后台核对未配置管理员身份', 503);
+ }
+ $actor['operation_id'] = $operationId;
+
+ try {
+ $client = CloudAgentSettingsService::clientForHost($hostId);
+ $operation = $client->request('GET', '/v1/operations/'
+ . rawurlencode($operationId), $actor);
+ $operation = CloudAgentOperationReconciler::ifUncertain(
+ $client, $operationId, $actor, $operation
+ );
+ } catch (CloudAgentError $error) {
+ $missing = $error->httpStatus === 404
+ || $error->agentHttpStatus === 404;
+ self::markReplica($hostId, $templateId, $operationId,
+ ['pending'], $missing ? 'uncertain' : 'pending',
+ $missing ? 'operation_not_found' : 'agent_unavailable');
+ self::refreshImportState($template);
+ throw $error;
+ } catch (\Throwable $error) {
+ self::markReplica($hostId, $templateId, $operationId,
+ ['pending'], 'pending', 'agent_unavailable');
+ throw $error;
+ }
+
+ try {
+ self::applyImportOperation($template, $hostId, $operationId,
+ $operation);
+ self::refreshImportState($template);
+ } catch (\Throwable $error) {
+ self::markReplica($hostId, $templateId, $operationId,
+ ['pending'], 'uncertain', 'panel_persist_failed');
+ self::refreshImportState($template);
+ throw $error;
+ }
+ return true;
+ }
+
/** Push the current PHP name/revision to every existing host replica. */
public static function synchronizeMetadata(array $template,
array $actor): array
@@ -160,8 +301,18 @@ final class CloudTemplateCatalogService
|| preg_match('/^[a-f0-9]{32}$/D', $generation) !== 1) {
throw new CloudError('云镜像版本无效', 409);
}
+ $authority = CloudTemplate::query()->where('id', $id)->first([
+ 'id', 'name', 'revision', 'generation_token', 'state',
+ ]);
+ if ($authority === null || (string) $authority->state !== 'active'
+ || (int) $authority->revision !== (int) $revision
+ || !hash_equals((string) $authority->generation_token, $generation)
+ || (string) $authority->name !== $name) {
+ throw new CloudError('仅已完成导入的当前云镜像版本可以同步元数据', 409);
+ }
$pending = [];
foreach (CloudTemplateReplica::query()->where('template_id', $id)
+ ->whereIn('state', ['active', 'stale'])
->orderBy('host_id')->get() as $replica) {
$hostId = (int) $replica->host_id;
$operationId = bin2hex(random_bytes(16));
@@ -191,6 +342,7 @@ final class CloudTemplateCatalogService
CloudTemplateReplica::query()->where('host_id', $hostId)
->where('template_id', $id)
->where('generation_token', $generation)
+ ->whereIn('state', ['active', 'stale'])
// A slower response for revision N must never overwrite a
// concurrent revision N+1 response.
->where('revision', '<=', (int) $revision)->update([
@@ -205,6 +357,7 @@ final class CloudTemplateCatalogService
CloudTemplateReplica::query()->where('host_id', $hostId)
->where('template_id', $id)
->where('generation_token', $generation)
+ ->whereIn('state', ['active', 'stale'])
// Do not let a late failure regress a newer successful
// metadata synchronization.
->where('revision', '<=', (int) $revision)->update([
@@ -516,6 +669,9 @@ final class CloudTemplateCatalogService
if ($template === null || (string) $template->state === 'retired') {
throw new CloudError('云镜像模板不存在', 404);
}
+ if ((string) $template->state !== 'active') {
+ throw new CloudError('云镜像尚未完成导入,暂不能修改启动脚本', 409);
+ }
if (!hash_equals((string) $template->generation_token, $generationToken)
|| (int) $template->revision !== $expectedRevision) {
throw new CloudError('初始化脚本已被修改,请重新打开对话框', 409);
@@ -760,6 +916,9 @@ final class CloudTemplateCatalogService
if ($template === null || (string) $template->state === 'retired') {
throw new CloudError('云镜像模板不存在', 404);
}
+ if ((string) $template->state !== 'active') {
+ throw new CloudError('云镜像尚未完成导入,暂不能修改元数据', 409);
+ }
if ((int) $template->revision !== $expectedRevision
|| !hash_equals((string) $template->generation_token,
$generationToken)) {
@@ -784,6 +943,21 @@ final class CloudTemplateCatalogService
private static function adminDto(CloudTemplate $template): array
{
+ $replicas = [];
+ $connection = (new CloudTemplateReplica())->getConnection();
+ if ($connection->getSchemaBuilder()->hasTable(
+ 'cloud_template_replicas')) {
+ $replicas = CloudTemplateReplica::query()
+ ->where('template_id', (string) $template->id)
+ ->orderBy('host_id')->get([
+ 'host_id', 'state', 'error_code', 'operation_id',
+ ])->map(static fn (CloudTemplateReplica $replica): array => [
+ 'host_id' => (int) $replica->host_id,
+ 'state' => (string) $replica->state,
+ 'error_code' => (string) $replica->error_code,
+ 'operation_id' => (string) $replica->operation_id,
+ ])->all();
+ }
return [
'id' => (string) $template->id,
'name' => (string) $template->name,
@@ -798,62 +972,57 @@ final class CloudTemplateCatalogService
'generation_token' => (string) $template->generation_token,
'created_at' => (int) $template->created_at,
'updated_at' => (int) $template->updated_at,
+ 'replicas' => $replicas,
];
}
- private static function writeReplica(int $hostId, CloudTemplate $template,
- string $state, string $operationId, string $errorCode, int $bytes,
- string $sha256): void
- {
- $values = [
- 'revision' => (int) $template->revision,
- 'generation_token' => (string) $template->generation_token,
- 'state' => $state,
- 'image_bytes' => $bytes,
- 'sha256' => $sha256,
- 'operation_id' => $operationId,
- 'error_code' => $errorCode,
- 'updated_at' => time(),
- ];
- $query = CloudTemplateReplica::query()->where('host_id', $hostId)
- ->where('template_id', (string) $template->id);
- if ($query->exists()) {
- $query->update($values);
- return;
- }
- $replica = new CloudTemplateReplica();
- foreach ($values + [
- 'host_id' => $hostId,
- 'template_id' => (string) $template->id,
- 'created_at' => time(),
- ] as $field => $value) {
- $replica->{$field} = $value;
- }
- $replica->save();
- }
-
private static function newImport(string $name, string $family,
string $version, string $sourceType, string $sourceUrl, int $bytes,
- string $sha256): CloudTemplate
+ string $sha256, array $hostIds): CloudTemplate
{
- $template = new CloudTemplate();
- $template->id = self::newTemplateId($family, $version);
- $template->name = $name;
- $template->os_family = $family;
- $template->os_version = $version;
- $template->source_type = $sourceType;
- $template->sealed_source_url = CloudSecrets::encrypt(
- ['url' => $sourceUrl], 'template-source'
- );
- $template->image_bytes = $bytes;
- $template->sha256 = $sha256;
- $template->startup_script = '';
- $template->revision = 1;
- $template->generation_token = bin2hex(random_bytes(16));
- $template->state = 'importing';
- $template->created_at = $template->updated_at = time();
- $template->save();
- return $template;
+ $connection = (new CloudTemplate())->getConnection();
+ return $connection->transaction(static function () use (
+ $name, $family, $version, $sourceType, $sourceUrl, $bytes,
+ $sha256, $hostIds
+ ): CloudTemplate {
+ $template = new CloudTemplate();
+ $template->id = self::newTemplateId($family, $version);
+ $template->name = $name;
+ $template->os_family = $family;
+ $template->os_version = $version;
+ $template->source_type = $sourceType;
+ $template->sealed_source_url = CloudSecrets::encrypt(
+ ['url' => $sourceUrl], 'template-source'
+ );
+ $template->image_bytes = $bytes;
+ $template->sha256 = $sha256;
+ $template->startup_script = '';
+ $template->revision = 1;
+ $template->generation_token = bin2hex(random_bytes(16));
+ $template->state = 'importing';
+ $template->created_at = $template->updated_at = time();
+ $template->save();
+
+ $replicas = [];
+ foreach ($hostIds as $hostId) {
+ $replicas[] = [
+ 'host_id' => $hostId,
+ 'template_id' => (string) $template->id,
+ 'revision' => (int) $template->revision,
+ 'generation_token' =>
+ (string) $template->generation_token,
+ 'state' => 'pending',
+ 'image_bytes' => 0,
+ 'sha256' => (string) $template->sha256,
+ 'operation_id' => bin2hex(random_bytes(16)),
+ 'error_code' => '',
+ 'created_at' => (int) $template->created_at,
+ 'updated_at' => (int) $template->updated_at,
+ ];
+ }
+ CloudTemplateReplica::query()->insert($replicas);
+ return $template;
+ }, 3);
}
private static function dispatchImport(CloudTemplate $template,
@@ -861,11 +1030,20 @@ final class CloudTemplateCatalogService
{
$operations = [];
foreach ($hostIds as $hostId) {
- $hostId = (int) $hostId;
- $operationId = bin2hex(random_bytes(16));
- $operations[$hostId] = $operationId;
- self::writeReplica($hostId, $template, 'pending', $operationId,
- '', 0, (string) $template->sha256);
+ $replica = CloudTemplateReplica::query()
+ ->where('host_id', $hostId)
+ ->where('template_id', (string) $template->id)
+ ->where('state', 'pending')->first();
+ if ($replica === null) {
+ CloudTemplate::query()->where('id', (string) $template->id)
+ ->where('state', 'importing')->update([
+ 'state' => 'error',
+ 'updated_at' => time(),
+ ]);
+ $template->refresh();
+ return self::importJobDto($template);
+ }
+ $operations[$hostId] = (string) $replica->operation_id;
}
foreach ($operations as $hostId => $operationId) {
try {
@@ -1027,9 +1205,16 @@ final class CloudTemplateCatalogService
'agent_result_mismatch');
return;
}
+ $metadataChanged = false;
if ((string) $locked->sha256 === '') {
$locked->sha256 = $digest;
+ $metadataChanged = true;
+ }
+ if ((int) $locked->image_bytes === 0) {
$locked->image_bytes = (int) $bytes;
+ $metadataChanged = true;
+ }
+ if ($metadataChanged) {
$locked->updated_at = time();
$locked->save();
}
diff --git a/web/app/Services/Cloud/CloudUserIsoService.php b/web/app/Services/Cloud/CloudUserIsoService.php
index ebbb588e..180bba4a 100644
--- a/web/app/Services/Cloud/CloudUserIsoService.php
+++ b/web/app/Services/Cloud/CloudUserIsoService.php
@@ -10,6 +10,7 @@ final class CloudUserIsoService
{
private const MAX_ISOS = 5;
private const MAX_TOTAL_BYTES = 10 * 1024 * 1024 * 1024;
+ private const IMPORT_POLL_INTERVAL_SECONDS = 5;
public static function listing(int $userId): array
{
@@ -37,9 +38,10 @@ final class CloudUserIsoService
if ($hosts === []) {
throw new CloudError('没有可用的宿主机 Agent', 503);
}
+ $hosts = array_values(array_unique(array_map('intval', $hosts)));
$connection = (new CloudUserIso())->getConnection();
$iso = $connection->transaction(static function () use (
- $userId, $name, $url
+ $userId, $name, $url, $hosts
): CloudUserIso {
$rows = CloudUserIso::query()->where('user_id', $userId)
->where('state', '<>', 'retired')
@@ -63,14 +65,48 @@ final class CloudUserIsoService
$iso->import_operation_id = bin2hex(random_bytes(16));
$iso->created_at = $iso->updated_at = time();
$iso->save();
+
+ // Persist the complete target-host snapshot before the first
+ // Agent mutation. A process crash can therefore leave only a
+ // fully enumerable set of pending operations, never a partial
+ // replica set that could be mistaken for a completed import.
+ $replicas = [];
+ foreach ($hosts as $hostId) {
+ $replicas[] = [
+ 'host_id' => $hostId,
+ 'user_iso_id' => (string) $iso->id,
+ 'user_id' => $userId,
+ 'state' => 'pending',
+ 'size_bytes' => 0,
+ 'sha256' => '',
+ 'operation_id' => bin2hex(random_bytes(16)),
+ 'error_code' => '',
+ 'created_at' => (int) $iso->created_at,
+ 'updated_at' => (int) $iso->updated_at,
+ ];
+ }
+ CloudUserIsoReplica::query()->insert($replicas);
return $iso;
}, 3);
+ $operations = [];
foreach ($hosts as $hostId) {
- $hostId = (int) $hostId;
- $operationId = bin2hex(random_bytes(16));
- self::writeReplica($hostId, $iso, 'pending', $operationId,
- '', 0, '');
+ $replica = CloudUserIsoReplica::query()
+ ->where('host_id', $hostId)
+ ->where('user_iso_id', (string) $iso->id)
+ ->where('state', 'pending')->first();
+ if ($replica === null) {
+ CloudUserIso::query()->where('id', (string) $iso->id)
+ ->where('state', 'importing')->update([
+ 'state' => 'failed',
+ 'updated_at' => time(),
+ ]);
+ $iso->refresh();
+ return self::jobDto($iso, self::jobStatus($iso));
+ }
+ $operations[$hostId] = (string) $replica->operation_id;
+ }
+ foreach ($operations as $hostId => $operationId) {
try {
$requestActor = $actor;
$requestActor['can_manage'] = true;
@@ -190,6 +226,149 @@ final class CloudUserIsoService
return self::jobDto($iso, self::jobStatus($iso));
}
+ /**
+ * Converge one durable ISO replica using only its saved Agent operation.
+ * Source URLs are intentionally unavailable, so the import POST cannot be
+ * replayed by the background worker.
+ */
+ public static function reconcileOnePendingImport(): bool
+ {
+ $connection = (new CloudUserIsoReplica())->getConnection();
+ $schema = $connection->getSchemaBuilder();
+ if (!$schema->hasTable('cloud_user_isos')
+ || !$schema->hasTable('cloud_user_iso_replicas')) {
+ return false;
+ }
+
+ $now = time();
+ $replica = CloudUserIsoReplica::query()
+ ->select('cloud_user_iso_replicas.*')
+ ->join('cloud_user_isos', 'cloud_user_isos.id', '=',
+ 'cloud_user_iso_replicas.user_iso_id')
+ ->where('cloud_user_iso_replicas.state', 'pending')
+ ->whereIn('cloud_user_isos.state', ['importing', 'failed'])
+ ->where('cloud_user_iso_replicas.updated_at', '<=',
+ $now - self::IMPORT_POLL_INTERVAL_SECONDS)
+ ->orderBy('cloud_user_iso_replicas.updated_at')
+ ->orderBy('cloud_user_iso_replicas.host_id')
+ ->orderBy('cloud_user_iso_replicas.user_iso_id')->first();
+
+ if ($replica === null) {
+ // Rows written by versions predating atomic replica reservation
+ // may have crashed after the parent commit but before the first
+ // replica. They contain no operation that could be safely
+ // replayed, so age them into a visible terminal failure.
+ $orphan = CloudUserIso::query()->where('state', 'importing')
+ ->where('updated_at', '<=', $now - 90)
+ ->whereNotExists(static function ($query): void {
+ $query->selectRaw('1')->from('cloud_user_iso_replicas')
+ ->whereColumn('cloud_user_iso_replicas.user_iso_id',
+ 'cloud_user_isos.id');
+ })->orderBy('updated_at')->orderBy('id')->first();
+ if ($orphan !== null) {
+ return CloudUserIso::query()
+ ->where('id', (string) $orphan->id)
+ ->where('state', 'importing')
+ ->where('updated_at', (int) $orphan->updated_at)
+ ->update([
+ 'state' => 'failed',
+ 'updated_at' => $now,
+ ]) === 1;
+ }
+
+ // Repair the crash window after every replica committed but before
+ // the aggregate catalog row was refreshed. Terminal replica sets
+ // are excluded so an idle worker cannot spin indefinitely.
+ $iso = CloudUserIso::query()
+ // A failed all-active aggregate may be a durable quota
+ // rejection. Only importing is an unambiguous crash window;
+ // otherwise the idle worker would select the same row forever.
+ ->where('state', 'importing')
+ ->whereExists(static function ($query): void {
+ $query->selectRaw('1')->from('cloud_user_iso_replicas')
+ ->whereColumn('cloud_user_iso_replicas.user_iso_id',
+ 'cloud_user_isos.id');
+ })->whereNotExists(static function ($query): void {
+ $query->selectRaw('1')->from('cloud_user_iso_replicas')
+ ->whereColumn('cloud_user_iso_replicas.user_iso_id',
+ 'cloud_user_isos.id')
+ ->where('cloud_user_iso_replicas.state', '<>',
+ 'active');
+ })->orderBy('updated_at')->orderBy('id')->first();
+ if ($iso === null) {
+ return false;
+ }
+ self::refreshCatalogState($iso);
+ return true;
+ }
+
+ $hostId = (int) $replica->host_id;
+ $isoId = (string) $replica->user_iso_id;
+ $operationId = (string) $replica->operation_id;
+ $claimed = CloudUserIsoReplica::query()->where('host_id', $hostId)
+ ->where('user_iso_id', $isoId)->where('state', 'pending')
+ ->where('operation_id', $operationId)
+ ->where('updated_at', (int) $replica->updated_at)
+ ->update(['updated_at' => $now]);
+ if ($claimed !== 1) {
+ return false;
+ }
+
+ $iso = CloudUserIso::query()->where('id', $isoId)
+ ->whereIn('state', ['importing', 'failed'])->first();
+ if ($iso === null) {
+ return true;
+ }
+ if (preg_match('/^[A-Za-z0-9_-]{8,80}$/D', $operationId) !== 1) {
+ self::markReplica($hostId, $isoId, $operationId, ['pending'],
+ 'uncertain', 'operation_id_invalid');
+ self::refreshCatalogState($iso);
+ throw new CloudError('ISO 导入操作标识无效,必须人工核查', 503);
+ }
+
+ $actor = CloudDaemonActor::resolve('cloud-user-iso-import-worker');
+ if ($actor === null) {
+ self::markReplica($hostId, $isoId, $operationId, ['pending'],
+ 'pending', 'daemon_actor_unavailable');
+ throw new CloudError('ISO 导入后台核对未配置管理员身份', 503);
+ }
+ $actor['operation_id'] = $operationId;
+
+ try {
+ $client = CloudAgentSettingsService::clientForHost($hostId);
+ $operation = $client->request('GET', '/v1/operations/'
+ . rawurlencode($operationId), $actor);
+ $operation = CloudAgentOperationReconciler::ifUncertain(
+ $client, $operationId, $actor, $operation
+ );
+ } catch (CloudAgentError $error) {
+ $missing = $error->httpStatus === 404
+ || $error->agentHttpStatus === 404;
+ self::markReplica($hostId, $isoId, $operationId, ['pending'],
+ $missing ? 'uncertain' : 'pending',
+ $missing ? 'operation_not_found' : 'agent_unavailable');
+ self::refreshCatalogState($iso);
+ throw $error;
+ } catch (\Throwable $error) {
+ self::markReplica($hostId, $isoId, $operationId, ['pending'],
+ 'pending', 'agent_unavailable');
+ self::refreshCatalogState($iso);
+ throw $error;
+ }
+
+ try {
+ self::applyImportOperation($iso, $hostId, $operationId,
+ $operation);
+ self::refreshCatalogState($iso);
+ } catch (\Throwable $error) {
+ self::markReplica($hostId, $isoId, $operationId, ['pending'],
+ 'uncertain', 'panel_persist_failed');
+ self::refreshCatalogState($iso);
+ throw $error;
+ }
+ return true;
+ }
+
public static function delete(string $id, array $actor): array
{
if (preg_match('/^[a-f0-9]{32}$/D', $id) !== 1) {
@@ -656,36 +835,6 @@ final class CloudUserIsoService
]);
}
- private static function writeReplica(int $hostId, CloudUserIso $iso,
- string $state, string $operationId, string $errorCode, int $bytes,
- string $sha256): void
- {
- $values = [
- 'user_id' => (int) $iso->user_id,
- 'state' => $state,
- 'size_bytes' => $bytes,
- 'sha256' => $sha256,
- 'operation_id' => $operationId,
- 'error_code' => $errorCode,
- 'updated_at' => time(),
- ];
- $query = CloudUserIsoReplica::query()->where('host_id', $hostId)
- ->where('user_iso_id', (string) $iso->id);
- if ($query->exists()) {
- $query->update($values);
- return;
- }
- $replica = new CloudUserIsoReplica();
- foreach ($values + [
- 'host_id' => $hostId,
- 'user_iso_id' => (string) $iso->id,
- 'created_at' => time(),
- ] as $field => $value) {
- $replica->{$field} = $value;
- }
- $replica->save();
- }
-
private static function markReplica(int $hostId, string $isoId,
string $operationId, array $allowedStates, string $state,
string $errorCode): bool
diff --git a/web/app/Services/Cloud/CloudWorkerService.php b/web/app/Services/Cloud/CloudWorkerService.php
index 64841960..01c4fd90 100644
--- a/web/app/Services/Cloud/CloudWorkerService.php
+++ b/web/app/Services/Cloud/CloudWorkerService.php
@@ -21,6 +21,7 @@ final class CloudWorkerService
private bool $policyOnly;
private array $observedErrors = [];
+ private int $catalogCursor = 0;
public function observationErrors(): array
{
@@ -127,6 +128,17 @@ final class CloudWorkerService
public function tick(): bool
{
$this->observedErrors = [];
+ $worked = $this->tickJob();
+ if ($this->policyOnly) {
+ return $worked;
+ }
+ $catalogWorked = $this->convergeOneCatalogImport();
+ return $worked || $catalogWorked;
+ }
+
+ /** Claim and advance at most one durable cloud job. */
+ private function tickJob(): bool
+ {
$job = (new CloudJob())->getConnection()->transaction(function () {
$query = CloudJob::whereIn('status', ['queued', 'running', 'uncertain'])
->where('available_at', '<=', time())->where('lease_until', '<=', time());
@@ -1059,6 +1071,49 @@ final class CloudWorkerService
return $operation;
}
+ /**
+ * Poll at most one due operation across the PHP-owned catalogs. Each
+ * service can only read or reconcile the operation ID persisted before
+ * dispatch; none has the source payload needed to replay an import.
+ */
+ private function convergeOneCatalogImport(): bool
+ {
+ $catalogs = [
+ ['template', [CloudTemplateCatalogService::class,
+ 'reconcileOnePendingImport']],
+ ['rescue_template', [CloudRescueTemplateImportService::class,
+ 'reconcileOnePendingImport']],
+ ['user_iso', [CloudUserIsoService::class,
+ 'reconcileOnePendingImport']],
+ ];
+ $count = count($catalogs);
+ for ($offset = 0; $offset < $count; $offset++) {
+ $index = ($this->catalogCursor + $offset) % $count;
+ [$label, $converge] = $catalogs[$index];
+ try {
+ if (!$converge()) {
+ continue;
+ }
+ $this->catalogCursor = ($index + 1) % $count;
+ return true;
+ } catch (\Throwable $error) {
+ // The callback may have claimed or polled an operation before
+ // failing, so do not run a second catalog action in this tick.
+ // An error alone is not "work": an idle worker must retain its
+ // normal sleep instead of spinning against a broken catalog.
+ $this->catalogCursor = ($index + 1) % $count;
+ $this->observedErrors[] = 'worker_tick_failed';
+ error_log('cloud worker ' . $label
+ . ' import reconciliation failed: ' . get_class($error));
+ return false;
+ }
+ }
+ // Shift the first probe even when every catalog is idle, so a catalog
+ // that becomes ready while cloud_jobs stays busy gets the next chance.
+ $this->catalogCursor = ($this->catalogCursor + 1) % $count;
+ return false;
+ }
+
private static function normalizeAgentOperation(array $operation): array
{
$error = $operation['error'] ?? null;
diff --git a/web/public/cloud/admin.js b/web/public/cloud/admin.js
index 63ced39e..05446a8b 100644
--- a/web/public/cloud/admin.js
+++ b/web/public/cloud/admin.js
@@ -463,12 +463,20 @@ async function loadCatalog() {
const url = rescue ? `${catalogPath()}?host_id=${encodeURIComponent(hostId)}` : catalogPath();
try { const entries = collection(await api(url), 'templates'); if (epoch !== state.epoch) return;
if (!entries.length) list.append(element('p', 'text-muted', '暂无模版。'));
- for (const entry of entries) { const row = element('form', 'cloud-admin-row'); const name = element('input', 'form-control'); name.value = entry.name || ''; name.maxLength = 64; name.required = true; name.setAttribute('aria-label', '模版名称'); name.disabled = !canManage; const save = element('button', 'btn btn-outline-primary', '保存名称'); save.type = 'submit'; save.disabled = !canManage; const remove = element('button', 'btn btn-outline-danger', '删除'); remove.type = 'button'; remove.disabled = !canManage; const imageBytes = Number(entry.image_bytes || entry.size_bytes || 0);
+ for (const entry of entries) { const row = element('form', 'cloud-admin-row'); const catalogState = String(entry.state || '').toLowerCase(); const active = rescue || catalogState === 'active'; const name = element('input', 'form-control'); name.value = entry.name || ''; name.maxLength = 64; name.required = true; name.setAttribute('aria-label', '模版名称'); name.disabled = !canManage || !active; const save = element('button', 'btn btn-outline-primary', '保存名称'); save.type = 'submit'; save.disabled = !canManage || !active; const remove = element('button', 'btn btn-outline-danger', '删除'); remove.type = 'button'; remove.disabled = !canManage; const imageBytes = Number(entry.image_bytes || entry.size_bytes || 0);
row.append(name, save, remove);
- if (!rescue) { const startup = element('button', 'btn btn-outline-secondary cloud-admin-startup-button', canManage ? '设置启动脚本' : '查看启动脚本'); startup.type = 'button'; startup.addEventListener('click', () => openStartupScript(entry, startup)); row.append(startup); }
- row.append(element('small', 'cloud-admin-meta', `${entry.os_family || '镜像'} ${entry.os_version || ''} · ${imageBytes > 0 ? (imageBytes / 1024 ** 3).toFixed(2) + ' GiB' : entry.id}`));
+ if (!rescue) { const startup = element('button', 'btn btn-outline-secondary cloud-admin-startup-button', canManage ? '设置启动脚本' : '查看启动脚本'); startup.type = 'button'; startup.disabled = !active; startup.addEventListener('click', () => openStartupScript(entry, startup)); row.append(startup); }
+ const stateLabel = rescue ? '' : ({ active: '已就绪', importing: '导入中', error: '导入失败' }[catalogState] || '状态未知') + ' · ';
+ row.dataset.catalogState = catalogState || 'unknown';
+ row.append(element('small', 'cloud-admin-meta', `${stateLabel}${entry.os_family || '镜像'} ${entry.os_version || ''} · ${imageBytes > 0 ? (imageBytes / 1024 ** 3).toFixed(2) + ' GiB' : entry.id}`));
+ if (!rescue && Array.isArray(entry.replicas)) for (const replica of entry.replicas) {
+ const replicaState = String(replica?.state || 'unknown').toLowerCase();
+ const replicaLabel = ({ active: '已就绪', pending: '核对中', failed: '失败', uncertain: '待人工核对', stale: '等待同步' }[replicaState] || '状态未知');
+ const details = [replica?.error_code, replica?.operation_id].filter(value => String(value || '').trim() !== '').map(String);
+ row.append(element('small', 'cloud-admin-meta', `宿主机 #${Number(replica?.host_id || 0)} · ${replicaLabel}${details.length ? ' · ' + details.join(' · ') : ''}`));
+ }
if (rescue) { name.readOnly = true; save.hidden = true; save.disabled = true; }
- row.addEventListener('submit', async event => { event.preventDefault(); if (rescue) return; if (state.busy) return notice('请等待当前导入任务完成。', true); save.disabled = true; try { await api(`${catalogPath()}/${encodeURIComponent(entry.id)}`, { method: 'PATCH', body: { name: name.value.trim(), expected_revision: Number(entry.revision), generation_token: String(entry.generation_token || '') } }); await loadCatalog(); notice('名称已保存。'); } catch (error) { notice(error.message, true); } finally { save.disabled = !canManage; } });
+ row.addEventListener('submit', async event => { event.preventDefault(); if (rescue) return; if (!active) return notice('镜像尚未就绪,不能修改名称。', true); if (state.busy) return notice('请等待当前导入任务完成。', true); save.disabled = true; try { await api(`${catalogPath()}/${encodeURIComponent(entry.id)}`, { method: 'PATCH', body: { name: name.value.trim(), expected_revision: Number(entry.revision), generation_token: String(entry.generation_token || '') } }); await loadCatalog(); notice('名称已保存。'); } catch (error) { notice(error.message, true); } finally { save.disabled = !canManage || !active; } });
remove.addEventListener('click', async () => {
if (state.busy) return notice('请等待当前导入任务完成。', true);
const message = rescue
@@ -502,7 +510,7 @@ async function importProgress(result) {
const suffix = isRescueCatalog() ? `?host_id=${encodeURIComponent(state.selectedHost)}` : '';
const fallback = `${catalogPath()}/import-jobs/${encodeURIComponent(result.job_id)}${suffix}`;
const url = String(result.status_url || '').startsWith('/api/v1/') ? result.status_url : fallback;
- for (let i = 0; i < 1800; i++) { const job = await api(url); notice(job.phase || `导入状态:${job.status}`); if (['succeeded', 'completed', 'done'].includes(job.status)) return; if (['failed', 'uncertain', 'cancelled'].includes(job.status)) throw new Error(job.error || '导入未完成,请核查宿主机状态后再操作。'); await new Promise(resolve => setTimeout(resolve, 2000)); } throw new Error('导入仍在进行,请稍后刷新查看。不要重复提交。');
+ for (let i = 0; i < 3600; i++) { const job = await api(url); notice(job.phase || `导入状态:${job.status}`); if (['succeeded', 'completed', 'done'].includes(job.status)) return; if (['failed', 'uncertain', 'cancelled'].includes(job.status)) throw new Error(job.error || '导入未完成,请核查宿主机状态后再操作。'); await new Promise(resolve => setTimeout(resolve, 2000)); } throw new Error('导入仍在进行,请稍后刷新查看。不要重复提交。');
}
async function importTemplate(event) {
event.preventDefault(); if (state.busy) return;
diff --git a/web/tests/AdminTemplateStartupScriptBehaviorTest.mjs b/web/tests/AdminTemplateStartupScriptBehaviorTest.mjs
index c6f18846..ffa136cf 100644
--- a/web/tests/AdminTemplateStartupScriptBehaviorTest.mjs
+++ b/web/tests/AdminTemplateStartupScriptBehaviorTest.mjs
@@ -23,7 +23,7 @@ class Element {
const flush = async () => { for (let turn = 0; turn < 20; turn += 1) await Promise.resolve(); };
const generation = '0123456789abcdef0123456789abcdef';
-const entry = { id: 'debian-12-template', name: '
', revision: 1, generation_token: generation };
+const entry = { id: 'debian-12-template', name: '
', state: 'active', image_bytes: 1073741824, revision: 1, generation_token: generation };
const payload = { id: entry.id, startup_script: '#!/bin/bash\necho ready\n', revision: 2, generation_token: generation };
const submit = { preventDefault() {} };
@@ -57,6 +57,9 @@ function fixture(canManage = true) {
const button = test.get('catalog-list').children[0].children
.find(item => item.textContent === '设置启动脚本');
assert.ok(button, 'Cloud template row has no startup-script control.');
+ const metadata = test.get('catalog-list').children[0].children
+ .find(item => item.className === 'cloud-admin-meta');
+ assert.match(metadata.textContent, /^已就绪 · .*1\.00 GiB$/);
const opening = button.listeners.get('click')();
assert.equal(test.get('template-startup-title').textContent, `启动脚本 · ${entry.name}`);
assert.equal(test.get('template-startup-save').disabled, true);
@@ -83,6 +86,29 @@ function fixture(canManage = true) {
assert.equal(button.focused, true);
}
+{
+ const test = fixture();
+ const loading = test.loadCatalog();
+ await test.respond(test.requests[0], { templates: [{
+ ...entry,
+ state: 'importing',
+ image_bytes: 0,
+ replicas: [{ host_id: 7, state: 'pending', error_code: 'agent_unavailable', operation_id: 'operation-1234' }],
+ }] });
+ await loading;
+ const row = test.get('catalog-list').children[0];
+ const save = row.children.find(item => item.textContent === '保存名称');
+ const remove = row.children.find(item => item.textContent === '删除');
+ const startup = row.children.find(item => item.textContent === '设置启动脚本');
+ const metadata = row.children.find(item => item.className === 'cloud-admin-meta');
+ assert.equal(row.dataset.catalogState, 'importing');
+ assert.equal(save.disabled, true);
+ assert.equal(startup.disabled, true);
+ assert.equal(remove.disabled, false, 'An operator must retain access to the catalog deletion workflow.');
+ assert.match(metadata.textContent, /^导入中 · .*debian-12-template$/);
+ assert.match(row.children.at(-1).textContent, /^宿主机 #7 · 核对中 · agent_unavailable · operation-1234$/);
+}
+
{
const test = fixture(false);
const opening = test.openStartupScript(entry, new Element());
diff --git a/web/tests/CloudAuxCatalogWorkerConvergenceTest.php b/web/tests/CloudAuxCatalogWorkerConvergenceTest.php
new file mode 100644
index 00000000..ebd7e864
--- /dev/null
+++ b/web/tests/CloudAuxCatalogWorkerConvergenceTest.php
@@ -0,0 +1,484 @@
+ 1,
+ 'is_admin' => true,
+ 'is_super_admin' => true,
+ 'can_manage' => true,
+ 'session_id' => $sessionId,
+ ];
+ }
+ }
+
+ final class AuxCatalogWorkerFakeAgent
+ {
+ public array $requests = [];
+ public array $getResponses = [];
+ public array $reconcileResponses = [];
+ public array $getErrors = [];
+ public $onRequest = null;
+
+ public function request(string $method, string $target, array $actor,
+ ?array $payload = null): array
+ {
+ $this->requests[] = compact('method', 'target', 'actor', 'payload');
+ if (is_callable($this->onRequest)) {
+ ($this->onRequest)($method, $target, $actor, $payload);
+ }
+ if (preg_match('#^/v1/operations/([A-Za-z0-9_-]+)$#D',
+ $target, $match) === 1 && $method === 'GET') {
+ $operationId = rawurldecode($match[1]);
+ if (isset($this->getErrors[$operationId])) {
+ throw $this->getErrors[$operationId];
+ }
+ return ['operation_id' => $operationId]
+ + ($this->getResponses[$operationId]
+ ?? ['status' => 'running']);
+ }
+ if (preg_match(
+ '#^/v1/operations/([A-Za-z0-9_-]+)/reconcile$#D',
+ $target, $match) === 1 && $method === 'POST') {
+ $operationId = rawurldecode($match[1]);
+ return ['operation_id' => $operationId]
+ + ($this->reconcileResponses[$operationId]
+ ?? ['status' => 'uncertain']);
+ }
+ if ($method === 'POST' && $target === '/v1/templates') {
+ return [
+ 'operation_id' => (string) $payload['operation_id'],
+ 'status' => 'succeeded',
+ 'result' => [
+ 'id' => (string) $payload['id'],
+ 'revision' => (int) $payload['revision'],
+ 'generation_token' =>
+ (string) $payload['generation_token'],
+ 'size_bytes' => 4096,
+ 'sha256' => str_repeat('a', 64),
+ ],
+ ];
+ }
+ if ($method === 'POST' && $target === '/v1/user-isos') {
+ return [
+ 'operation_id' => (string) $payload['operation_id'],
+ 'status' => 'succeeded',
+ 'result' => [
+ 'id' => (string) $payload['id'],
+ 'owner_id' => (int) $payload['owner_id'],
+ 'size_bytes' => 8192,
+ 'sha256' => str_repeat('b', 64),
+ ],
+ ];
+ }
+ throw new \RuntimeException('Background worker replayed an import');
+ }
+ }
+
+ final class CloudAgentSettingsService
+ {
+ public static ?AuxCatalogWorkerFakeAgent $client = null;
+ public static array $hostIds = [11];
+
+ public static function enabledHostIds(): array
+ {
+ return self::$hostIds;
+ }
+
+ public static function clientForHost(int $hostId,
+ float $timeoutSeconds = 20.0): AuxCatalogWorkerFakeAgent
+ {
+ if (!in_array($hostId, self::$hostIds, true)
+ || self::$client === null) {
+ throw new \RuntimeException('Missing fake Agent');
+ }
+ return self::$client;
+ }
+ }
+}
+
+namespace {
+ require dirname(__DIR__) . '/vendor/autoload.php';
+
+ use App\Models\CloudRescueTemplateImport;
+ use App\Models\CloudTemplate;
+ use App\Models\CloudTemplateReplica;
+ use App\Models\CloudUserIso;
+ use App\Models\CloudUserIsoReplica;
+ use App\Services\Cloud\AuxCatalogWorkerFakeAgent;
+ use App\Services\Cloud\CloudAgentError;
+ use App\Services\Cloud\CloudAgentSettingsService;
+ use App\Services\Cloud\CloudRescueTemplateImportService;
+ use App\Services\Cloud\CloudTemplateCatalogService;
+ use App\Services\Cloud\CloudUserIsoService;
+ use Illuminate\Database\Capsule\Manager as Capsule;
+ use Illuminate\Database\Schema\Blueprint;
+
+ $assert = static function (bool $condition, string $message): void {
+ if (!$condition) {
+ throw new RuntimeException($message);
+ }
+ };
+
+ $capsule = new Capsule();
+ $capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'],
+ 'default');
+ $capsule->setAsGlobal();
+ $capsule->bootEloquent();
+ $schema = $capsule->schema('default');
+ $schema->create('cloud_templates', static function (Blueprint $table): void {
+ $table->string('id')->primary();
+ $table->string('name');
+ $table->string('os_family');
+ $table->string('os_version');
+ $table->string('source_type');
+ $table->text('sealed_source_url');
+ $table->integer('image_bytes')->default(0);
+ $table->string('sha256')->default('');
+ $table->text('startup_script')->default('');
+ $table->integer('revision');
+ $table->string('generation_token');
+ $table->string('state');
+ $table->integer('created_at');
+ $table->integer('updated_at');
+ });
+ $schema->create('cloud_template_replicas',
+ static function (Blueprint $table): void {
+ $table->integer('host_id');
+ $table->string('template_id');
+ $table->integer('revision');
+ $table->string('generation_token');
+ $table->string('state');
+ $table->integer('image_bytes')->default(0);
+ $table->string('sha256')->default('');
+ $table->string('operation_id')->default('');
+ $table->string('error_code')->default('');
+ $table->integer('created_at');
+ $table->integer('updated_at');
+ $table->primary(['host_id', 'template_id']);
+ });
+ $schema->create('cloud_rescue_template_imports',
+ static function (Blueprint $table): void {
+ $table->string('operation_id')->primary();
+ $table->integer('host_id');
+ $table->string('template_id');
+ $table->string('import_type');
+ $table->string('state');
+ $table->string('error_code')->default('');
+ $table->integer('created_at');
+ $table->integer('updated_at');
+ $table->unique(['host_id', 'template_id']);
+ });
+ $schema->create('cloud_user_isos', static function (Blueprint $table): void {
+ $table->string('id')->primary();
+ $table->integer('user_id');
+ $table->string('name');
+ $table->text('sealed_source_url');
+ $table->integer('size_bytes')->default(0);
+ $table->string('sha256')->default('');
+ $table->string('state');
+ $table->string('import_operation_id');
+ $table->integer('created_at');
+ $table->integer('updated_at');
+ });
+ $schema->create('cloud_user_iso_replicas',
+ static function (Blueprint $table): void {
+ $table->integer('host_id');
+ $table->string('user_iso_id');
+ $table->integer('user_id');
+ $table->string('state');
+ $table->integer('size_bytes')->default(0);
+ $table->string('sha256')->default('');
+ $table->string('operation_id')->default('');
+ $table->string('error_code')->default('');
+ $table->integer('created_at');
+ $table->integer('updated_at');
+ $table->primary(['host_id', 'user_iso_id']);
+ });
+
+ $agent = new AuxCatalogWorkerFakeAgent();
+ CloudAgentSettingsService::$client = $agent;
+ $now = time();
+
+ // Both global catalogs must persist their complete target-host snapshot
+ // before the first host mutation. A crash at any dispatch boundary then
+ // leaves one durable operation per expected host, never a partial set.
+ CloudAgentSettingsService::$hostIds = [11, 12];
+ $agent->onRequest = static function (string $method, string $target,
+ array $actor, ?array $payload) use ($assert): void {
+ if ($method !== 'POST') {
+ return;
+ }
+ if ($target === '/v1/templates') {
+ $hostIds = CloudTemplateReplica::query()
+ ->where('template_id', (string) $payload['id'])
+ ->orderBy('host_id')->pluck('host_id')->map(
+ static fn ($value): int => (int) $value
+ )->all();
+ $assert($hostIds === [11, 12],
+ 'Template dispatch began before every replica was reserved.');
+ }
+ if ($target === '/v1/user-isos') {
+ $hostIds = CloudUserIsoReplica::query()
+ ->where('user_iso_id', (string) $payload['id'])
+ ->orderBy('host_id')->pluck('host_id')->map(
+ static fn ($value): int => (int) $value
+ )->all();
+ $assert($hostIds === [11, 12],
+ 'ISO dispatch began before every replica was reserved.');
+ }
+ };
+ $templateReservation = CloudTemplateCatalogService::importRemote([
+ 'name' => 'Debian 13 atomic reservation',
+ 'url' => 'https://images.example/debian-13.qcow2',
+ 'sha256' => '',
+ ], ['user_id' => 1, 'is_admin' => true, 'can_manage' => true]);
+ $isoReservation = CloudUserIsoService::import([
+ 'name' => 'Atomic reservation ISO',
+ 'url' => 'https://images.example/atomic-reservation.iso',
+ ], ['user_id' => 90, 'is_admin' => false, 'can_manage' => false]);
+ $assert($templateReservation['status'] === 'succeeded'
+ && $isoReservation['status'] === 'succeeded',
+ 'Atomic catalog reservations did not complete in the test fixture.');
+ $agent->onRequest = null;
+ $agent->requests = [];
+ CloudAgentSettingsService::$hostIds = [11];
+
+ $rescueOperation = str_repeat('a', 32);
+ $freshRescueOperation = str_repeat('b', 32);
+ $rescueTemplate = str_repeat('1', 32);
+ $freshRescueTemplate = str_repeat('2', 32);
+ Capsule::table('cloud_rescue_template_imports')->insert([
+ [
+ 'operation_id' => $rescueOperation,
+ 'host_id' => 11,
+ 'template_id' => $rescueTemplate,
+ 'import_type' => 'remote',
+ 'state' => 'running',
+ 'error_code' => '',
+ 'created_at' => $now - 30,
+ 'updated_at' => $now - 30,
+ ],
+ [
+ 'operation_id' => $freshRescueOperation,
+ 'host_id' => 11,
+ 'template_id' => $freshRescueTemplate,
+ 'import_type' => 'remote',
+ 'state' => 'running',
+ 'error_code' => '',
+ 'created_at' => $now,
+ 'updated_at' => $now,
+ ],
+ ]);
+ $agent->getResponses[$rescueOperation] = [
+ 'status' => 'uncertain',
+ 'error' => ['code' => 'operation_uncertain'],
+ ];
+ $agent->reconcileResponses[$rescueOperation] = [
+ 'status' => 'succeeded',
+ 'result' => ['id' => $rescueTemplate],
+ ];
+
+ $assert(CloudRescueTemplateImportService::reconcileOnePendingImport(),
+ 'Rescue import worker did not claim one due operation.');
+ $rescue = CloudRescueTemplateImport::find($rescueOperation);
+ $freshRescue = CloudRescueTemplateImport::find($freshRescueOperation);
+ $assert((string) $rescue->state === 'succeeded'
+ && (string) $freshRescue->state === 'running'
+ && count($agent->requests) === 2
+ && $agent->requests[0]['method'] === 'GET'
+ && $agent->requests[0]['target']
+ === '/v1/operations/' . $rescueOperation
+ && $agent->requests[1]['method'] === 'POST'
+ && $agent->requests[1]['target']
+ === '/v1/operations/' . $rescueOperation . '/reconcile'
+ && ($agent->requests[0]['actor']['session_id'] ?? null)
+ === 'cloud-rescue-template-import-worker'
+ && ($agent->requests[0]['actor']['operation_id'] ?? null)
+ === $rescueOperation,
+ 'Rescue worker did not preserve the saved operation boundary.');
+ $assert(!CloudRescueTemplateImportService::reconcileOnePendingImport(),
+ 'Fresh rescue import bypassed the five-second poll backoff.');
+
+ Capsule::table('cloud_rescue_template_imports')
+ ->where('operation_id', $freshRescueOperation)
+ ->update(['updated_at' => $now - 30]);
+ $agent->getErrors[$freshRescueOperation] = new CloudAgentError(
+ 'temporary failure', 503, 'agent_transport_error', true, 503
+ );
+ try {
+ CloudRescueTemplateImportService::reconcileOnePendingImport();
+ throw new RuntimeException('Rescue transport error was swallowed.');
+ } catch (CloudAgentError $error) {
+ $assert($error->agentCode === 'agent_transport_error',
+ 'Rescue worker threw the wrong transport error.');
+ }
+ $freshRescue->refresh();
+ $assert((string) $freshRescue->state === 'running'
+ && (string) $freshRescue->error_code === 'agent_unavailable'
+ && !CloudRescueTemplateImportService::reconcileOnePendingImport(),
+ 'Rescue transport failure was not persisted with poll backoff.');
+
+ $insertIso = static function (string $id, string $jobId, int $userId,
+ string $state, int $updatedAt): void {
+ Capsule::table('cloud_user_isos')->insert([
+ 'id' => $id,
+ 'user_id' => $userId,
+ 'name' => $id,
+ 'sealed_source_url' => 'test-sealed-source',
+ 'size_bytes' => 0,
+ 'sha256' => '',
+ 'state' => $state,
+ 'import_operation_id' => $jobId,
+ 'created_at' => $updatedAt,
+ 'updated_at' => $updatedAt,
+ ]);
+ };
+ $insertIsoReplica = static function (string $id, int $userId,
+ string $operationId, string $state, int $updatedAt,
+ int $bytes = 0, string $sha256 = ''): void {
+ Capsule::table('cloud_user_iso_replicas')->insert([
+ 'host_id' => 11,
+ 'user_iso_id' => $id,
+ 'user_id' => $userId,
+ 'state' => $state,
+ 'size_bytes' => $bytes,
+ 'sha256' => $sha256,
+ 'operation_id' => $operationId,
+ 'error_code' => '',
+ 'created_at' => $updatedAt,
+ 'updated_at' => $updatedAt,
+ ]);
+ };
+
+ $isoId = str_repeat('3', 32);
+ $freshIsoId = str_repeat('4', 32);
+ $healIsoId = str_repeat('5', 32);
+ $isoOperation = str_repeat('c', 32);
+ $freshIsoOperation = str_repeat('d', 32);
+ $digest = str_repeat('e', 64);
+ $insertIso($isoId, str_repeat('6', 32), 42, 'importing', $now - 30);
+ $insertIsoReplica($isoId, 42, $isoOperation, 'pending', $now - 30);
+ $insertIso($freshIsoId, str_repeat('7', 32), 43, 'importing', $now);
+ $insertIsoReplica($freshIsoId, 43, $freshIsoOperation, 'pending', $now);
+ $insertIso($healIsoId, str_repeat('8', 32), 44, 'importing', $now - 20);
+ $insertIsoReplica($healIsoId, 44, str_repeat('f', 32), 'active',
+ $now - 20, 4096, $digest);
+ Capsule::table('cloud_user_isos')->where('id', $healIsoId)->update([
+ 'size_bytes' => 4096,
+ 'sha256' => $digest,
+ ]);
+ $agent->getResponses[$isoOperation] = [
+ 'status' => 'uncertain',
+ 'error' => ['code' => 'operation_uncertain'],
+ ];
+ $agent->reconcileResponses[$isoOperation] = [
+ 'status' => 'succeeded',
+ 'result' => [
+ 'id' => $isoId,
+ 'owner_id' => 42,
+ 'size_bytes' => 8192,
+ 'sha256' => $digest,
+ ],
+ ];
+ $requestsBeforeIso = count($agent->requests);
+ $assert(CloudUserIsoService::reconcileOnePendingImport(),
+ 'ISO worker did not claim one due replica.');
+ $iso = CloudUserIso::find($isoId);
+ $isoReplica = CloudUserIsoReplica::query()->where('host_id', 11)
+ ->where('user_iso_id', $isoId)->first();
+ $freshIsoReplica = CloudUserIsoReplica::query()->where('host_id', 11)
+ ->where('user_iso_id', $freshIsoId)->first();
+ $isoRequests = array_slice($agent->requests, $requestsBeforeIso);
+ $assert((string) $iso->state === 'active'
+ && (string) $isoReplica->state === 'active'
+ && (string) $freshIsoReplica->state === 'pending'
+ && count($isoRequests) === 2
+ && $isoRequests[0]['method'] === 'GET'
+ && $isoRequests[0]['target'] === '/v1/operations/' . $isoOperation
+ && $isoRequests[1]['method'] === 'POST'
+ && $isoRequests[1]['target']
+ === '/v1/operations/' . $isoOperation . '/reconcile'
+ && ($isoRequests[0]['actor']['session_id'] ?? null)
+ === 'cloud-user-iso-import-worker'
+ && ($isoRequests[0]['actor']['operation_id'] ?? null)
+ === $isoOperation,
+ 'ISO worker did not preserve one saved operation boundary.');
+
+ $requestsBeforeHeal = count($agent->requests);
+ $assert(CloudUserIsoService::reconcileOnePendingImport()
+ && (string) CloudUserIso::find($healIsoId)->state === 'active'
+ && count($agent->requests) === $requestsBeforeHeal,
+ 'ISO worker did not repair an all-active parent locally.');
+ $assert(!CloudUserIsoService::reconcileOnePendingImport(),
+ 'Fresh ISO replica bypassed the five-second poll backoff.');
+
+ Capsule::table('cloud_user_iso_replicas')
+ ->where('user_iso_id', $freshIsoId)
+ ->update(['updated_at' => $now - 30]);
+ $agent->getErrors[$freshIsoOperation] = new CloudAgentError(
+ 'temporary failure', 503, 'agent_transport_error', true, 503
+ );
+ try {
+ CloudUserIsoService::reconcileOnePendingImport();
+ throw new RuntimeException('ISO transport error was swallowed.');
+ } catch (CloudAgentError $error) {
+ $assert($error->agentCode === 'agent_transport_error',
+ 'ISO worker threw the wrong transport error.');
+ }
+ $freshIso = CloudUserIso::find($freshIsoId);
+ $freshIsoReplica->refresh();
+ $assert((string) $freshIso->state === 'importing'
+ && (string) $freshIsoReplica->state === 'pending'
+ && (string) $freshIsoReplica->error_code === 'agent_unavailable'
+ && !CloudUserIsoService::reconcileOnePendingImport(),
+ 'ISO transport failure was not persisted with poll backoff.');
+
+ $orphanTemplateId = 'debian-orphan-parent';
+ Capsule::table('cloud_templates')->insert([
+ 'id' => $orphanTemplateId,
+ 'name' => 'Orphan template',
+ 'os_family' => 'Debian',
+ 'os_version' => '13',
+ 'source_type' => 'remote',
+ 'sealed_source_url' => 'test-sealed:template-source',
+ 'image_bytes' => 0,
+ 'sha256' => '',
+ 'startup_script' => '',
+ 'revision' => 1,
+ 'generation_token' => str_repeat('0', 32),
+ 'state' => 'importing',
+ 'created_at' => $now - 180,
+ 'updated_at' => $now - 180,
+ ]);
+ $orphanIsoId = str_repeat('9', 32);
+ $insertIso($orphanIsoId, str_repeat('0', 32), 45, 'importing',
+ $now - 180);
+ $requestsBeforeOrphans = count($agent->requests);
+ $assert(CloudTemplateCatalogService::reconcileOnePendingImport()
+ && (string) CloudTemplate::find($orphanTemplateId)->state === 'error'
+ && CloudUserIsoService::reconcileOnePendingImport()
+ && (string) CloudUserIso::find($orphanIsoId)->state === 'failed'
+ && count($agent->requests) === $requestsBeforeOrphans,
+ 'Replica-free legacy parents were not failed without replay.');
+
+ foreach ($agent->requests as $request) {
+ $assert(!in_array($request['target'],
+ ['/v1/rescue-templates', '/v1/user-isos'], true),
+ 'Background convergence replayed an original import POST.');
+ }
+
+ echo "Rescue and user ISO background import convergence passed.\n";
+}
diff --git a/web/tests/CloudTemplateCatalogTest.php b/web/tests/CloudTemplateCatalogTest.php
index 5c6b10e0..fd4f81f4 100644
--- a/web/tests/CloudTemplateCatalogTest.php
+++ b/web/tests/CloudTemplateCatalogTest.php
@@ -146,16 +146,46 @@ $cleared = CloudTemplateCatalogService::saveStartupScript(
$assert($cleared['startup_script'] === '' && $cleared['revision'] === 3,
'An empty script did not disable first-boot execution.');
+$operationBeforeBlockedMutation = (string) CloudTemplateReplica::query()
+ ->where('host_id', 1)->where('template_id', 'debian-12-template')
+ ->value('operation_id');
+foreach (['importing', 'error'] as $blockedState) {
+ Capsule::table('cloud_templates')->where('id', 'debian-12-template')
+ ->update(['state' => $blockedState]);
+ $expect(static fn () => CloudTemplateCatalogService::rename(
+ 'debian-12-template', 'Blocked rename', 3, $generation
+ ), 409, 'A non-active template was renamed.');
+ $expect(static fn () => CloudTemplateCatalogService::saveStartupScript(
+ 'debian-12-template', 'blocked script', 3, $generation
+ ), 409, 'A non-active template startup script was changed.');
+ $blocked = CloudTemplate::find('debian-12-template');
+ $assert((int) $blocked->revision === 3
+ && (string) $blocked->name === 'Debian 12'
+ && (string) $blocked->startup_script === ''
+ && (string) CloudTemplateReplica::query()->where('host_id', 1)
+ ->where('template_id', 'debian-12-template')
+ ->value('operation_id') === $operationBeforeBlockedMutation,
+ 'A blocked metadata mutation changed template or replica identity.');
+}
+Capsule::table('cloud_templates')->where('id', 'debian-12-template')
+ ->update(['state' => 'active']);
+
$public = CloudTemplateCatalogService::publicCatalog()['templates'][0];
$admin = CloudTemplateCatalogService::adminCatalog()['templates'][0];
foreach (['startup_script', 'startup_script_configured', 'revision',
- 'generation_token', 'source_url', 'sealed_source_url', 'sha256']
+ 'generation_token', 'source_url', 'sealed_source_url', 'sha256', 'replicas']
as $privateField) {
$assert(!array_key_exists($privateField, $public),
'Tenant catalog exposed private template field: ' . $privateField);
}
$assert($admin['startup_script_configured'] === false
- && !array_key_exists('startup_script', $admin),
+ && !array_key_exists('startup_script', $admin)
+ && $admin['replicas'] === [[
+ 'host_id' => 1,
+ 'state' => 'active',
+ 'error_code' => '',
+ 'operation_id' => $operationBeforeBlockedMutation,
+ ]],
'Admin catalog must show script state without returning script content.');
$row = CloudTemplate::find('debian-12-template');
$replica = CloudTemplateReplica::query()->where('host_id', 1)
diff --git a/web/tests/CloudTemplateMetadataSyncStateMachineTest.php b/web/tests/CloudTemplateMetadataSyncStateMachineTest.php
index 7fa7a89a..b21b7c02 100644
--- a/web/tests/CloudTemplateMetadataSyncStateMachineTest.php
+++ b/web/tests/CloudTemplateMetadataSyncStateMachineTest.php
@@ -56,6 +56,7 @@ namespace {
use App\Models\CloudTemplateReplica;
use App\Services\Cloud\CloudAgentSettingsService;
+ use App\Services\Cloud\CloudError;
use App\Services\Cloud\CloudTemplateCatalogService;
use App\Services\Cloud\TemplateMetadataSyncFakeAgent;
use Illuminate\Database\Capsule\Manager as Capsule;
@@ -66,12 +67,31 @@ namespace {
throw new RuntimeException($message);
}
};
+ $expect = static function (callable $callable, int $status,
+ string $message) use ($assert): void {
+ try {
+ $callable();
+ } catch (CloudError $error) {
+ $assert($error->httpStatus === $status,
+ $message . ' Unexpected status: ' . $error->httpStatus);
+ return;
+ }
+ throw new RuntimeException($message);
+ };
$capsule = new Capsule();
$capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'],
'default');
$capsule->setAsGlobal();
$capsule->bootEloquent();
+ $capsule->schema('default')->create('cloud_templates',
+ static function (Blueprint $table): void {
+ $table->string('id')->primary();
+ $table->string('name');
+ $table->integer('revision');
+ $table->string('generation_token');
+ $table->string('state');
+ });
$capsule->schema('default')->create('cloud_template_replicas',
static function (Blueprint $table): void {
$table->integer('host_id');
@@ -91,6 +111,13 @@ namespace {
$generation = str_repeat('a', 32);
$id = 'debian-13-metadata-sync';
$now = time();
+ Capsule::table('cloud_templates')->insert([
+ 'id' => $id,
+ 'name' => 'Debian 13 renamed',
+ 'revision' => 2,
+ 'generation_token' => $generation,
+ 'state' => 'active',
+ ]);
Capsule::table('cloud_template_replicas')->insert([
'host_id' => 7,
'template_id' => $id,
@@ -118,6 +145,14 @@ namespace {
'generation_token' => $generation,
];
$actor = ['user_id' => 1, 'can_manage' => true];
+ $setAuthority = static function (int $revision, string $name,
+ string $state = 'active') use ($id): void {
+ Capsule::table('cloud_templates')->where('id', $id)->update([
+ 'name' => $name,
+ 'revision' => $revision,
+ 'state' => $state,
+ ]);
+ };
$agent = new TemplateMetadataSyncFakeAgent();
CloudAgentSettingsService::$client = $agent;
@@ -139,6 +174,7 @@ namespace {
$agent->mode = 'wrong_operation';
$agent->result = $result(3, 'Debian 13 wrong operation');
+ $setAuthority(3, 'Debian 13 wrong operation');
$mismatch = CloudTemplateCatalogService::synchronizeMetadata(
$snapshot(3, 'Debian 13 wrong operation'), $actor
);
@@ -163,6 +199,7 @@ namespace {
]);
};
$agent->result = $result(4, 'Debian 13 older response');
+ $setAuthority(4, 'Debian 13 older response');
CloudTemplateCatalogService::synchronizeMetadata(
$snapshot(4, 'Debian 13 older response'), $actor
);
@@ -175,6 +212,7 @@ namespace {
$agent->mode = 'uncertain';
$agent->beforeResponse = null;
$agent->result = $result(6, 'Debian 13 reconciled');
+ $setAuthority(6, 'Debian 13 reconciled');
$before = count($agent->requests);
$reconciled = CloudTemplateCatalogService::synchronizeMetadata(
$snapshot(6, 'Debian 13 reconciled'), $actor
@@ -189,5 +227,16 @@ namespace {
&& str_ends_with($requests[1]['target'], '/reconcile'),
'Uncertain metadata synchronization was replayed or not reconciled.');
+ $setAuthority(7, 'Debian 13 blocked', 'importing');
+ $requestsBeforeBlocked = count($agent->requests);
+ $operationBeforeBlocked = (string) $row->operation_id;
+ $expect(static fn () => CloudTemplateCatalogService::synchronizeMetadata(
+ $snapshot(7, 'Debian 13 blocked'), $actor
+ ), 409, 'A non-active template dispatched metadata synchronization.');
+ $row->refresh();
+ $assert(count($agent->requests) === $requestsBeforeBlocked
+ && (string) $row->operation_id === $operationBeforeBlocked,
+ 'Blocked metadata synchronization changed the replica operation ID.');
+
echo "Template metadata operation binding and CAS tests passed.\n";
}
diff --git a/web/tests/CloudTemplateWorkerConvergenceTest.php b/web/tests/CloudTemplateWorkerConvergenceTest.php
new file mode 100644
index 00000000..c8bdfb62
--- /dev/null
+++ b/web/tests/CloudTemplateWorkerConvergenceTest.php
@@ -0,0 +1,240 @@
+ 1,
+ 'is_admin' => true,
+ 'is_super_admin' => true,
+ 'can_manage' => true,
+ 'session_id' => $sessionId,
+ ];
+ }
+ }
+
+ final class TemplateWorkerFakeAgent
+ {
+ public array $requests = [];
+ public array $results = [];
+
+ public function request(string $method, string $target, array $actor,
+ ?array $payload = null): array
+ {
+ $this->requests[] = compact('method', 'target', 'actor', 'payload');
+ if (preg_match('#^/v1/operations/([A-Za-z0-9_-]+)$#D',
+ $target, $match) === 1 && $method === 'GET') {
+ $operationId = rawurldecode($match[1]);
+ return [
+ 'operation_id' => $operationId,
+ 'status' => 'uncertain',
+ 'error' => ['code' => 'operation_uncertain'],
+ ];
+ }
+ if (preg_match('#^/v1/operations/([A-Za-z0-9_-]+)/reconcile$#D',
+ $target, $match) === 1 && $method === 'POST') {
+ $operationId = rawurldecode($match[1]);
+ return [
+ 'operation_id' => $operationId,
+ 'status' => 'succeeded',
+ 'result' => $this->results[$operationId],
+ ];
+ }
+ throw new \RuntimeException('Template worker replayed an import');
+ }
+ }
+
+ final class CloudAgentSettingsService
+ {
+ public static ?TemplateWorkerFakeAgent $client = null;
+
+ public static function clientForHost(int $hostId,
+ float $timeoutSeconds = 20.0): TemplateWorkerFakeAgent
+ {
+ if ($hostId !== 11 || self::$client === null) {
+ throw new \RuntimeException('Missing fake Agent');
+ }
+ return self::$client;
+ }
+ }
+}
+
+namespace {
+ require dirname(__DIR__) . '/vendor/autoload.php';
+
+ use App\Models\CloudTemplate;
+ use App\Models\CloudTemplateReplica;
+ use App\Services\Cloud\CloudAgentSettingsService;
+ use App\Services\Cloud\CloudWorkerService;
+ use App\Services\Cloud\TemplateWorkerFakeAgent;
+ use Illuminate\Database\Capsule\Manager as Capsule;
+ use Illuminate\Database\Schema\Blueprint;
+
+ $assert = static function (bool $condition, string $message): void {
+ if (!$condition) {
+ throw new RuntimeException($message);
+ }
+ };
+
+ $capsule = new Capsule();
+ $capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'],
+ 'default');
+ $capsule->setAsGlobal();
+ $capsule->bootEloquent();
+ $schema = $capsule->schema('default');
+ $schema->create('cloud_jobs', static function (Blueprint $table): void {
+ $table->increments('id');
+ $table->string('kind');
+ $table->string('status');
+ $table->integer('available_at');
+ $table->integer('lease_until');
+ });
+ $schema->create('cloud_templates', static function (Blueprint $table): void {
+ $table->string('id')->primary();
+ $table->string('name');
+ $table->string('os_family');
+ $table->string('os_version');
+ $table->string('source_type');
+ $table->text('sealed_source_url');
+ $table->integer('image_bytes')->default(0);
+ $table->string('sha256')->default('');
+ $table->text('startup_script')->default('');
+ $table->integer('revision');
+ $table->string('generation_token');
+ $table->string('state');
+ $table->integer('created_at');
+ $table->integer('updated_at');
+ });
+ $schema->create('cloud_template_replicas',
+ static function (Blueprint $table): void {
+ $table->integer('host_id');
+ $table->string('template_id');
+ $table->integer('revision');
+ $table->string('generation_token');
+ $table->string('state');
+ $table->integer('image_bytes')->default(0);
+ $table->string('sha256')->default('');
+ $table->string('operation_id')->default('');
+ $table->string('error_code')->default('');
+ $table->integer('created_at');
+ $table->integer('updated_at');
+ $table->primary(['host_id', 'template_id']);
+ });
+
+ $now = time();
+ $digest = str_repeat('c', 64);
+ $operationA = str_repeat('a', 32);
+ $operationB = str_repeat('b', 32);
+ $generationA = str_repeat('1', 32);
+ $generationB = str_repeat('2', 32);
+ $insertTemplate = static function (string $id, string $generation,
+ string $digest, string $state, int $updatedAt) use ($now): void {
+ Capsule::table('cloud_templates')->insert([
+ 'id' => $id,
+ 'name' => $id,
+ 'os_family' => 'Debian',
+ 'os_version' => '13',
+ 'source_type' => 'remote',
+ 'sealed_source_url' => 'test-sealed-source',
+ 'image_bytes' => 0,
+ 'sha256' => $digest,
+ 'startup_script' => '',
+ 'revision' => 1,
+ 'generation_token' => $generation,
+ 'state' => $state,
+ 'created_at' => $now,
+ 'updated_at' => $updatedAt,
+ ]);
+ };
+ $insertReplica = static function (string $id, string $generation,
+ string $digest, string $operationId, string $state,
+ int $updatedAt): void {
+ Capsule::table('cloud_template_replicas')->insert([
+ 'host_id' => 11,
+ 'template_id' => $id,
+ 'revision' => 1,
+ 'generation_token' => $generation,
+ 'state' => $state,
+ 'image_bytes' => $state === 'active' ? 8192 : 0,
+ 'sha256' => $digest,
+ 'operation_id' => $operationId,
+ 'error_code' => '',
+ 'created_at' => $updatedAt,
+ 'updated_at' => $updatedAt,
+ ]);
+ };
+
+ $insertTemplate('debian-a', $generationA, $digest, 'importing', $now - 30);
+ $insertReplica('debian-a', $generationA, $digest, $operationA,
+ 'pending', $now - 30);
+ $insertTemplate('debian-b', $generationB, $digest, 'importing', $now - 20);
+ $insertReplica('debian-b', $generationB, $digest, $operationB,
+ 'pending', $now - 20);
+
+ $agent = new TemplateWorkerFakeAgent();
+ $agent->results[$operationA] = [
+ 'id' => 'debian-a',
+ 'revision' => 1,
+ 'generation_token' => $generationA,
+ 'size_bytes' => 8192,
+ 'sha256' => $digest,
+ ];
+ CloudAgentSettingsService::$client = $agent;
+
+ $policy = new CloudWorkerService(null, true);
+ $assert($policy->tick() === false && $agent->requests === [],
+ 'The policy-only worker consumed a template import.');
+
+ $worker = new CloudWorkerService();
+ $assert($worker->tick() === true && $worker->observationErrors() === [],
+ 'The ordinary idle worker did not converge one template import.');
+ $templateA = CloudTemplate::find('debian-a');
+ $replicaA = CloudTemplateReplica::query()->where('host_id', 11)
+ ->where('template_id', 'debian-a')->first();
+ $templateB = CloudTemplate::find('debian-b');
+ $replicaB = CloudTemplateReplica::query()->where('host_id', 11)
+ ->where('template_id', 'debian-b')->first();
+ $assert((string) $templateA->state === 'active'
+ && (int) $templateA->image_bytes === 8192
+ && (string) $templateA->sha256 === $digest
+ && (string) $replicaA->state === 'active'
+ && (string) $templateB->state === 'importing'
+ && (string) $replicaB->state === 'pending',
+ 'One bounded worker pass changed the wrong import or lost image size.');
+ $assert(count($agent->requests) === 2
+ && $agent->requests[0]['method'] === 'GET'
+ && $agent->requests[0]['target'] === '/v1/operations/' . $operationA
+ && $agent->requests[1]['method'] === 'POST'
+ && $agent->requests[1]['target']
+ === '/v1/operations/' . $operationA . '/reconcile'
+ && ($agent->requests[0]['actor']['session_id'] ?? null)
+ === 'cloud-template-import-worker'
+ && ($agent->requests[0]['actor']['operation_id'] ?? null)
+ === $operationA,
+ 'Worker lookup/reconciliation did not preserve the saved operation ID.');
+ foreach ($agent->requests as $request) {
+ $assert($request['target'] !== '/v1/templates',
+ 'The worker replayed the original template import POST.');
+ }
+
+ Capsule::table('cloud_template_replicas')->where('template_id', 'debian-b')
+ ->update(['state' => 'failed', 'error_code' => 'test_terminal']);
+ Capsule::table('cloud_templates')->where('id', 'debian-b')
+ ->update(['state' => 'error']);
+ $generationC = str_repeat('3', 32);
+ $insertTemplate('debian-c', $generationC, $digest, 'importing', $now - 10);
+ $insertReplica('debian-c', $generationC, $digest, str_repeat('d', 32),
+ 'active', $now - 10);
+ $requestsBeforeAggregateRepair = count($agent->requests);
+ $assert($worker->tick() === true
+ && (string) CloudTemplate::find('debian-c')->state === 'active'
+ && count($agent->requests) === $requestsBeforeAggregateRepair,
+ 'Worker did not repair an all-active template aggregate locally.');
+ $assert($worker->tick() === false,
+ 'A terminal failed template kept the idle worker in a busy loop.');
+
+ echo "Idle worker template import convergence and no-replay tests passed.\n";
+}
diff --git a/web/tests/CloudWorkerCatalogFairnessTest.php b/web/tests/CloudWorkerCatalogFairnessTest.php
new file mode 100644
index 00000000..af759833
--- /dev/null
+++ b/web/tests/CloudWorkerCatalogFairnessTest.php
@@ -0,0 +1,213 @@
+addConnection(['driver' => 'sqlite', 'database' => ':memory:'],
+ 'default');
+ $capsule->setAsGlobal();
+ $capsule->bootEloquent();
+ $schema = $capsule->schema('default');
+ $schema->create('user', static function (Blueprint $table): void {
+ $table->increments('id');
+ $table->integer('enable')->default(1);
+ $table->integer('is_admin')->default(0);
+ });
+ $schema->create('cloud_jobs', static function (Blueprint $table): void {
+ $table->increments('id');
+ $table->string('operation_id')->unique();
+ $table->integer('order_id')->nullable();
+ $table->integer('instance_id')->nullable();
+ $table->integer('snapshot_id')->nullable();
+ $table->integer('user_id');
+ $table->integer('host_id');
+ $table->string('kind');
+ $table->string('status');
+ $table->text('sealed_payload');
+ $table->integer('attempts')->default(0);
+ $table->integer('dispatched_at')->nullable();
+ $table->string('lease_token')->nullable();
+ $table->integer('lease_until')->default(0);
+ $table->integer('available_at');
+ $table->string('error_code')->nullable();
+ $table->text('result_json')->nullable();
+ $table->integer('created_at');
+ $table->integer('updated_at');
+ });
+ Capsule::table('user')->insert(['id' => 1, 'enable' => 1, 'is_admin' => 0]);
+
+ $responses = [];
+ $now = time();
+ for ($number = 1; $number <= 3; $number++) {
+ $operationId = str_repeat((string) $number, 32);
+ Capsule::table('cloud_jobs')->insert([
+ 'operation_id' => $operationId,
+ 'order_id' => null,
+ 'instance_id' => null,
+ 'snapshot_id' => null,
+ 'user_id' => 1,
+ 'host_id' => 7,
+ 'kind' => 'action',
+ 'status' => 'queued',
+ 'sealed_payload' => CloudSecrets::encrypt([
+ 'method' => 'POST',
+ 'request_target' => '/v1/vms/test-vm/actions',
+ 'body' => [
+ 'operation_id' => $operationId,
+ 'action' => 'start',
+ ],
+ 'owner_id' => 1,
+ ], 'job-payload'),
+ 'attempts' => 0,
+ 'dispatched_at' => null,
+ 'lease_token' => null,
+ 'lease_until' => 0,
+ 'available_at' => 0,
+ 'error_code' => null,
+ 'result_json' => null,
+ 'created_at' => $now,
+ 'updated_at' => $now,
+ ]);
+ $responses[] = new Response(200, [], json_encode([
+ 'operation_id' => $operationId,
+ 'status' => 'dispatching',
+ ], JSON_THROW_ON_ERROR));
+ }
+
+ $client = new CloudAgentClient(
+ '7',
+ 'test-agent-secret-at-least-32-bytes',
+ new Client(['handler' => HandlerStack::create(new MockHandler($responses))]),
+ 20.0,
+ 'http://127.0.0.1:8001/internal/agent/request',
+ 'test-gateway-secret-at-least-32-bytes'
+ );
+ $worker = new CloudWorkerService($client);
+
+ for ($tick = 1; $tick <= 3; $tick++) {
+ $assert($worker->tick(), 'A due cloud job was not processed.');
+ $assert(CloudJob::query()->where('status', 'running')->count() === $tick,
+ 'The persistent cloud_jobs backlog did not advance one row.');
+ $assert(CloudTemplateCatalogService::$calls
+ + CloudRescueTemplateImportService::$calls
+ + CloudUserIsoService::$calls === $tick,
+ 'One worker tick ran more than one catalog action.');
+ }
+ $assert(CloudTemplateCatalogService::$calls === 1
+ && CloudRescueTemplateImportService::$calls === 1
+ && CloudUserIsoService::$calls === 1,
+ 'A continuously non-empty cloud_jobs queue starved a catalog.');
+
+ // A catalog exception is observable but is not work. Returning false keeps
+ // the command loop's one-second idle backoff and prevents a log/DB hot loop.
+ CloudTemplateCatalogService::$fail = true;
+ $assert($worker->tick() === false,
+ 'A catalog exception incorrectly reported successful work.');
+ $assert($worker->observationErrors() === ['worker_tick_failed'],
+ 'A catalog exception was not exposed through worker health.');
+ $assert(CloudTemplateCatalogService::$calls === 2
+ && CloudRescueTemplateImportService::$calls === 1
+ && CloudUserIsoService::$calls === 1,
+ 'The worker ran a second catalog after an ambiguous catalog failure.');
+
+ $beforePolicy = [
+ CloudTemplateCatalogService::$calls,
+ CloudRescueTemplateImportService::$calls,
+ CloudUserIsoService::$calls,
+ ];
+ $policyWorker = new CloudWorkerService(null, true);
+ $assert($policyWorker->tick() === false
+ && $beforePolicy === [
+ CloudTemplateCatalogService::$calls,
+ CloudRescueTemplateImportService::$calls,
+ CloudUserIsoService::$calls,
+ ], 'The policy-only worker was delayed by catalog convergence.');
+
+ echo "Cloud worker catalog fairness and failure backoff tests passed.\n";
+}