fix: allow storage configuration after terminal failure

go
codex 4 weeks ago
parent 732b5bce4e
commit 01e3683cb2

@ -117,7 +117,7 @@ Agent 配置字段:
先在宿主机创建五个互相独立的真实目录,并为 libvirt/QEMU 配置正确的属主、权限、挂载和剩余空间;Agent 不会创建目录、挂载文件系统或删除其中的数据。保存并启用 Agent 连接后,在后台“虚拟机管理 -> 宿主机 -> 存储目录”填写系统盘与快照、云系统模板、cloud-init Seed、用户 ISO、救援 ISO 五类目录。Agent 校验目录是 `storage_path_roots` 的真实子目录后,分别注册固定名称的 persistent `dir` pool:`virt-cloud-managed`、`virt-cloud-templates`、`virt-cloud-seeds`、`virt-cloud-user-isos`、`virt-cloud-rescue-isos`,并启动、刷新和启用 autostart。若固定名称已被其他目录占用,Agent 会拒绝注册;不会扫描、复用或迁移其他 pool。
libvirt 的持久 pool 定义就是 Agent 后续使用的映射,不另写存储状态文件。系统盘与快照、模板、Seed、用户 ISO 和救援 ISO 后续只使用各自固定 pool。当前实现不兼容旧存储布局,也不提供自动迁移或回退读取;已有数据必须在启用本功能前另行完成受控迁移。首次成功应用后网页不允许更换路径。注册成功后回到 `/admin/agents` 重新执行连接测试;`storage_configuration` capability 可用前,宿主机不会进入调度。
libvirt 的持久 pool 定义就是 Agent 后续使用的映射,不另写存储状态文件。系统盘与快照、模板、Seed、用户 ISO 和救援 ISO 后续只使用各自固定 pool。当前实现不兼容旧存储布局,也不提供自动迁移或回退读取;已有数据必须在启用本功能前另行完成受控迁移。首次成功应用后网页不允许更换路径。若任务以失败终态结束,旧 CloudJob 和 operation ID 保留为审计记录且绝不重放;管理员可修正目录后显式提交一个新 CloudJob 和新 operation ID。这个新任务只会幂等地核验并补全固定 pool;既不更换已有 pool 的路径,也不迁移或删除数据。注册成功后回到 `/admin/agents` 重新执行连接测试;`storage_configuration` capability 可用前,宿主机不会进入调度。
网络接入统一在宿主机页面填写“接入类型”和“网桥 / 网络名称”,新建默认 Linux 网桥 `br0`。面板保存为 `bridge:br0` 或 `network:default` 这样的 `default_network`;名称以字母或数字开头,后续只能包含字母、数字、点、下划线和短横线,bridge 名称为 1~15 字符,libvirt network 名称为 1~64 字符。宿主机上的网桥或 libvirt network 必须由运维预先准备。Agent 只选择、验证和使用现有网络,不创建网桥、不启动 libvirt network,也不修改宿主机系统网卡、路由或挂载。

@ -7,7 +7,7 @@ use App\Models\CloudHostAgent;
use App\Models\CloudHostStorageConfiguration;
use App\Models\CloudJob;
/** One persisted, single-dispatch registration of a host's storage layout. */
/** Current desired host storage layout plus its latest single-dispatch task. */
final class CloudHostStorageConfigurationService
{
private const DIRECTORY_FIELDS = [
@ -45,20 +45,39 @@ final class CloudHostStorageConfigurationService
$row = CloudHostStorageConfiguration::query()
->where('host_id', $hostId)->lockForUpdate()->first();
$actualRevision = $row === null ? 0 : (int) $row->revision;
$samePaths = $row !== null && self::samePaths($row, $paths);
if ($actualRevision !== $expectedRevision) {
throw new CloudError('存储配置已被修改,请刷新页面后重试', 409);
}
if ($row !== null) {
if (self::samePaths($row, $paths)
&& (int) ($row->job_id ?? 0) > 0) {
$prior = CloudJob::find((int) $row->job_id);
if ($prior !== null) {
$prior = self::linkedJob($row);
$state = (string) $row->state;
if ($state === 'applied') {
if ($samePaths && $prior !== null
&& (string) $prior->phase === 'terminal'
&& (string) $prior->outcome === 'succeeded') {
return $prior;
}
throw new CloudError(
'宿主机存储配置已成功应用,不允许更换路径', 409
);
}
if ($state === 'applying'
&& $prior !== null
&& (string) $prior->phase !== 'terminal') {
if ($samePaths) {
return $prior;
}
throw new CloudError(
'宿主机存储配置任务正在执行,请等待终态回执', 409
);
}
if (!in_array($state, ['applying', 'failed'], true)
|| !self::isTerminalFailure($prior)) {
throw new CloudError(
'宿主机存储配置状态与任务账本不一致,请联系管理员核查', 409
);
}
throw new CloudError(
'宿主机存储配置已有任务或终态记录,禁止重复下发', 409
);
}
$setting = CloudHostAgent::query()->where('host_id', $hostId)
->lockForUpdate()->first();
@ -66,11 +85,16 @@ final class CloudHostStorageConfigurationService
|| trim((string) $setting->sealed_api_secret) === '') {
throw new CloudError('请先启用并配置宿主机 Agent API', 409);
}
// The revision describes the desired paths, not the number of
// attempts. Re-applying the same failed layout gets a new task and
// operation ID without pretending that the configuration changed.
$nextRevision = $row === null || !$samePaths
? $actualRevision + 1 : $actualRevision;
$operationId = bin2hex(random_bytes(16));
[$job, $created] = CloudJobService::reserveStandaloneAgentMutation(
$actor, $hostId, 'host_storage_configure', 'POST',
'/v1/storage/configuration', $paths,
['can_manage' => true, 'storage_revision' => 1],
['can_manage' => true, 'storage_revision' => $nextRevision],
[
'host_id' => $hostId,
'expected_revision' => $expectedRevision,
@ -81,14 +105,17 @@ final class CloudHostStorageConfigurationService
if (!$created) {
return $job;
}
$row = new CloudHostStorageConfiguration();
$row->host_id = $hostId;
$row->revision = 1;
if ($row === null) {
$row = new CloudHostStorageConfiguration();
$row->host_id = $hostId;
$row->created_at = time();
}
$row->revision = $nextRevision;
self::assignPaths($row, $paths);
$row->state = 'applying';
$row->operation_id = $operationId;
$row->job_id = (int) $job->id;
$row->created_at = $row->updated_at = time();
$row->updated_at = time();
$row->save();
CloudAgentSettingsService::invalidateReadiness($setting);
return $job;
@ -107,14 +134,26 @@ final class CloudHostStorageConfigurationService
}
$row = CloudHostStorageConfiguration::query()
->where('host_id', (int) $job->host_id)
->where('operation_id', (string) $job->operation_id)
->lockForUpdate()->first();
if ($row === null) {
throw new CloudError('存储配置任务账本不存在', 409);
}
if (!hash_equals((string) $row->operation_id,
(string) $job->operation_id)) {
// A terminally failed generation may be superseded by an explicit
// new request. A late receipt from that old generation still
// closes its CloudJob, but must never overwrite the current row.
return;
}
if ((int) ($row->job_id ?? 0) !== (int) $job->id) {
throw new CloudError('存储配置任务账本关联不一致', 409);
}
if (in_array((string) $row->state, ['applied', 'failed'], true)) {
return;
}
if ((string) $row->state !== 'applying') {
throw new CloudError('存储配置状态无效', 409);
}
if ($status === 'succeeded') {
if (!is_array($result) || array_is_list($result)
|| ($result['configured'] ?? null) !== true) {
@ -180,6 +219,30 @@ final class CloudHostStorageConfigurationService
return true;
}
private static function linkedJob($row): ?CloudJob
{
$jobId = (int) ($row->job_id ?? 0);
if ($jobId < 1 || trim((string) $row->operation_id) === '') {
return null;
}
$job = CloudJob::find($jobId);
if ($job === null
|| (int) $job->host_id !== (int) $row->host_id
|| (string) $job->kind !== 'host_storage_configure'
|| !hash_equals((string) $row->operation_id,
(string) $job->operation_id)) {
return null;
}
return $job;
}
private static function isTerminalFailure(?CloudJob $job): bool
{
return $job !== null
&& (string) $job->phase === 'terminal'
&& (string) $job->outcome === 'failed';
}
private static function assignPaths($row, array $paths): void
{
foreach (self::DIRECTORY_FIELDS as $field) {
@ -199,12 +262,24 @@ final class CloudHostStorageConfigurationService
{
$setting = CloudHostAgent::find((int) $host->id);
$state = $row === null ? 'unconfigured' : (string) $row->state;
$terminalFailure = $row !== null
&& self::isTerminalFailure(self::linkedJob($row));
// A worker may terminalize its CloudJob after a local finalization
// incident left this projection at applying. The immutable task ledger
// is authoritative, so present that condition as failed and recoverable.
if ($state === 'applying' && $terminalFailure) {
$state = 'failed';
}
$canSubmit = $row === null || ($state === 'failed' && $terminalFailure);
$result = [
'host_id' => (int) $host->id,
'host_name' => (string) $host->name,
'revision' => $row === null ? 0 : (int) $row->revision,
'state' => $state,
'configured' => $state === 'applied',
'can_submit' => $canSubmit,
'operation_id' => $row === null
? null : (string) $row->operation_id,
'requires_agent_retest' => $state === 'applied'
&& ($setting === null || !$setting->isReady()
|| !CloudNetworkSelector::isValid($host->default_network)),

@ -171,6 +171,9 @@ function shownFilters() {
function failureText(job, view) {
if (view.protocolError) return '任务数据不符合公开协议;资源按可能已生效处理并保持锁定。';
if (view.status !== 'failed') return '';
if (job.kind === 'host_storage_configure') {
return '旧存储任务已终止且不会重放;部分固定存储池可能已应用。可到宿主机存储目录页修正后提交新的独立任务。';
}
if (job.effect_may_have_applied === true && job.effect_resolution === 'unknown') {
return '操作效果可能已发生;资源已安全锁定,需管理员人工处理。';
}
@ -376,10 +379,14 @@ function renderDetail(payload) {
['创建时间', date(job.created_at)], ['更新时间', date(job.updated_at)],
]));
const failure = failureText(job, view);
const storageFailure = job.kind === 'host_storage_configure'
&& view.status === 'failed';
if (failure) body.append(el('h3', 'h6 mt-4', '失败详情'), el('p', 'text-danger text-break', failure));
if (job.finalization_incident === true) {
body.append(el('h3', 'h6 mt-4', '业务账本收尾事件'),
el('p', 'text-danger text-break', `任务已终态失败,但业务账本未能安全收敛;资金和资源已保持锁定,不会自动重试。${job.finalization_error_code ? ` 事件代码:${job.finalization_error_code}` : ''}`));
el('p', 'text-danger text-break', storageFailure
? `旧存储任务已终态失败,收尾事件仅保留审计且不会重放;存储目录页仍可以终态任务账本为准提交新的独立任务。${job.finalization_error_code ? ` 事件代码:${job.finalization_error_code}` : ''}`
: `任务已终态失败,但业务账本未能安全收敛;资金和资源已保持锁定,不会自动重试。${job.finalization_error_code ? ` 事件代码:${job.finalization_error_code}` : ''}`));
}
if (state.inspection) {
const inspectedAt = state.inspection.checked_at || state.inspection.inspected_at;
@ -387,7 +394,9 @@ function renderDetail(payload) {
body.append(el('h3', 'h6 mt-4', paymentEvidence ? '支付账本审计' : '宿主机只读检查'),
el('p', 'text-muted fs-13', paymentEvidence
? `已记录外部支付平台核验结论${inspectedAt ? ` · ${date(inspectedAt)}` : ''}。原任务仍保持失败,且没有重新下单。`
: `已完成只读检查${inspectedAt ? ` · ${date(inspectedAt)}` : ''}。检查不会执行 Agent 操作;任务仍以失败详情和资源安全栅栏为准。`));
: storageFailure
? `已完成只读检查${inspectedAt ? ` · ${date(inspectedAt)}` : ''}。检查不会执行 Agent 操作;旧任务仍保持失败,也不会影响存储目录页提交新的独立任务。`
: `已完成只读检查${inspectedAt ? ` · ${date(inspectedAt)}` : ''}。检查不会执行 Agent 操作;任务仍以失败详情和资源安全栅栏为准。`));
}
for (const note of payload.notes || []) body.append(el('p', 'text-muted fs-13 mt-3 mb-0', note));
const links = byId('operation-detail-links'); links.replaceChildren();

@ -183,20 +183,20 @@ function renderHostStorage(payload) {
byId('host-storage-host-id').value = payload.host_id;
byId('host-storage-revision').value = Number(payload.revision || 0);
const validState = ['unconfigured', 'applying', 'failed', 'applied'].includes(payload.state);
const locked = payload.configured === true || payload.state === 'applying'
|| !validState || (payload.state === 'failed' && payload.effect_may_have_applied === true);
const canSubmit = validState && payload.can_submit === true;
const locked = !canSubmit;
for (const [field, id] of Object.entries(storageFields)) {
const input = byId(id); input.value = payload[field] || ''; input.readOnly = locked; input.disabled = !canManage;
}
const labels = { unconfigured: '尚未配置', applying: '正在等待 Agent 完成', failed: '应用失败', applied: '已应用' };
const labels = { unconfigured: '尚未配置', applying: '正在等待 Agent 完成', failed: '上一次任务已失败且不会自动重试;可修正目录后提交新的独立任务', applied: '已应用' };
let message = labels[payload.state] || '状态协议无效,资源已安全锁定,需管理员人工处理';
if (payload.state === 'failed' && payload.effect_may_have_applied === true) {
message = '应用失败,但操作效果可能已经发生。资源已安全锁定,需管理员人工处理';
if (payload.state === 'failed' && !canSubmit) {
message = '应用失败,但任务账本状态不完整,请联系管理员核查';
}
if (payload.operation_id && payload.state !== 'applied') message += ` · operation_id ${payload.operation_id}`;
if (payload.requires_agent_retest) message += ' · 请前往 Agent API 重新测试后再启用调度';
hostStorageNotice(message, payload.state === 'failed' || !validState);
const save = byId('host-storage-save'); if (save) { save.dataset.applied = payload.configured === true ? '1' : '0'; save.dataset.taskPending = '0'; save.disabled = !canManage || payload.configured === true; }
const save = byId('host-storage-save'); if (save) { save.dataset.applied = payload.configured === true ? '1' : '0'; save.dataset.taskPending = '0'; save.disabled = !canManage || !canSubmit; }
}
function lockHostStorageForTask(locked) {
for (const id of Object.values(storageFields)) byId(id).readOnly = locked;
@ -240,8 +240,19 @@ async function saveHostStorage(event) {
.catch(error => hostStorageNotice(`任务 #${task.id} 已成功,但刷新存储配置失败:${error.message}`, true));
return;
}
lockHostStorageForTask(task.safe_to_resubmit !== true);
hostStorageNotice(taskFailureMessage(task, '宿主机存储配置失败。'), true);
const failure = task?.error || '宿主机存储配置失败。';
const partial = task?.effect_may_have_applied === true
? ' 部分固定存储池可能已经应用;新任务只会核验并补全,不会迁移或删除数据。' : '';
void api(`/api/v1/admin/hosts/${encodeURIComponent(hostId)}/storage-configuration`)
.then(payload => {
if (!currentHost()) return;
renderHostStorage(payload);
const next = payload.state === 'failed' && payload.can_submit === true
? ' 旧任务已终止且不会重放;修正目录后可提交新的独立任务。'
: ' 当前任务账本尚未进入可提交状态,请联系管理员核查。';
hostStorageNotice(`${failure}${partial}${next}`, true);
})
.catch(error => hostStorageNotice(`${failure} 刷新存储状态失败:${error.message}`, true));
},
onStop: reason => {
if (currentHost() && reason === 'deadline') hostStorageNotice('自动观察已到达截止时间。任务可能仍在后台执行;当前配置保持锁定,请从任务记录核对结果。', true);

@ -189,7 +189,9 @@ function renderItems(emptyMessage = '暂无任务记录。') {
if (meta.children.length) article.append(meta);
if (task.status === 'failed') {
article.append(element('p', 'panel-task-error', task.error || '任务执行失败。'));
if (task.effect_may_have_applied) {
if (raw?.kind === 'host_storage_configure') {
article.append(element('p', 'panel-task-warning', '旧存储任务已终止且不会重放;可到宿主机存储目录页修正后提交新的独立任务。'));
} else if (task.effect_may_have_applied) {
article.append(element('p', 'panel-task-warning', '操作效果可能已经发生。为避免重复执行,请勿再次提交;请将任务编号提供给管理员核查。'));
} else if (task.safe_to_resubmit === false) {
article.append(element('p', 'panel-task-warning', '此任务不可再次提交,请联系管理员核查。'));

@ -81,8 +81,9 @@ CREATE TABLE cloud_host_agents (
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
-- The panel stores only the requested host-local directories; the Agent owns
-- validation and registration of the five fixed libvirt pools. A dispatched
-- operation_id is retained as an immutable audit link to its one CloudJob.
-- validation and registration of the five fixed libvirt pools. This row
-- points at the latest requested generation; superseded failed generations
-- remain immutable in cloud_jobs under their original operation IDs.
CREATE TABLE cloud_host_storage_configurations (
host_id BIGINT UNSIGNED NOT NULL PRIMARY KEY,
revision BIGINT UNSIGNED NOT NULL DEFAULT 0,

@ -25,8 +25,10 @@ assert.match(storage, /observeTask\(receipt/);
assert.match(storage, /task\.status === 'succeeded'/);
assert.match(storage, /api\(`\/api\/v1\/admin\/hosts\/\$\{encodeURIComponent\(hostId\)\}\/storage-configuration`\)/);
assert.match(storage, /renderHostStorage\(payload\)/);
assert.match(storage, /taskFailureMessage\(task/);
assert.match(storage, /task\.safe_to_resubmit !== true/);
assert.match(storage, /task\?\.effect_may_have_applied === true/);
assert.match(storage, /payload\.state === 'failed' && payload\.can_submit === true/);
assert.match(storage, /旧任务已终止且不会重放/);
assert.doesNotMatch(storage, /task\.safe_to_resubmit/);
assert.doesNotMatch(storage, /setInterval\(/);
const catalog = source.slice(source.indexOf('async function loadCatalog'), source.indexOf('function taskFailureMessage'));

@ -22,6 +22,11 @@ assert.match(source, /label: '协议错误'/);
assert.match(source, /effect_may_have_applied === true/);
assert.match(source, /unresolved_effect/);
assert.match(source, /资源已安全锁定,需管理员人工处理/);
assert.match(source, /job\.kind === 'host_storage_configure'/);
assert.match(source, /可到宿主机存储目录页修正后提交新的独立任务/);
assert.match(source, /storageFailure/);
assert.match(source, /收尾事件仅保留审计且不会重放/);
assert.match(source, /不会影响存储目录页提交新的独立任务/);
// The one retained high-risk action creates a distinct task once. The UI
// validates the fresh receipt, immediately acknowledges it, and observes the

@ -46,8 +46,11 @@ $assert(str_contains($service,
"'host_storage_configure', 'POST'")
&& str_contains($service, "'/v1/storage/configuration', \$paths")
&& str_contains($service, 'CloudJobService::acknowledgement($job)')
&& str_contains($service, 'isTerminalFailure($prior)')
&& str_contains($service, "'can_submit' => \$canSubmit")
&& str_contains($service, '!hash_equals((string) $row->operation_id')
&& !str_contains($service, "'GET', '/v1/storage/configuration'"),
'Storage registration lost its one-shot durable task contract.');
'Storage registration lost its single-dispatch or failure-supersession contract.');
$assert(str_contains($agentSettings,
'cloud_host_storage_configurations as host_storage')
&& str_contains($placement,

@ -1,5 +1,315 @@
<?php
declare(strict_types=1);
// Host storage registration is a persisted CloudJob with exactly one Agent
// dispatch and terminal receipt/deadline finalization.
require __DIR__ . '/CloudSpecialTaskSingleDispatchTest.php';
namespace App\Services\Cloud {
/** In-memory envelope used only by this isolated state-machine test. */
final class CloudSecrets
{
private static array $values = [];
public static function encrypt(array $value, string $purpose): string
{
$key = bin2hex(random_bytes(16));
self::$values[$key] = $value;
return 'storage-test:' . $key;
}
public static function decrypt(string $value, string $purpose): array
{
return self::$values[substr($value, strlen('storage-test:'))];
}
}
}
namespace {
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudHostStorageConfiguration;
use App\Models\CloudJob;
use App\Services\Cloud\CloudError;
use App\Services\Cloud\CloudHostStorageConfigurationService;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$expect = static function (callable $callable, int $status,
string $message) use ($assert): void {
try {
$callable();
} catch (CloudError $error) {
$assert($error->httpStatus === $status,
$message . ' Unexpected status: ' . $error->httpStatus);
return;
}
throw new RuntimeException($message);
};
$terminalize = static function (CloudJob $job, string $outcome,
bool $effectMayHaveApplied = false): void {
$now = time();
$job->phase = 'terminal';
$job->outcome = $outcome;
$job->status = $outcome;
$job->attempts = 1;
$job->dispatch_attempts = 1;
$job->dispatch_started_at = $now - 2;
$job->dispatched_at = $now - 2;
$job->agent_accepted_at = $now - 1;
$job->receipt_deadline_at = $now + 900;
$job->terminal_at = $now;
$job->effect_may_have_applied = $effectMayHaveApplied ? 1 : 0;
$job->safe_to_resubmit = 0;
$job->effect_resolution = $outcome === 'succeeded'
? 'applied' : ($effectMayHaveApplied ? 'unknown' : 'not_applied');
$job->lease_token = null;
$job->lease_until = 0;
$job->available_at = 0;
$job->updated_at = $now;
$job->save();
};
$capsule = new Capsule();
$capsule->addConnection([
'driver' => 'sqlite', 'database' => ':memory:',
], 'default');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$schema = $capsule->schema('default');
$schema->create('cloud_hosts', static function (Blueprint $table): void {
$table->increments('id');
$table->string('name');
$table->string('region');
$table->string('country_code', 2);
$table->boolean('enabled');
$table->string('default_network');
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_host_agents', static function (Blueprint $table): void {
$table->integer('host_id')->primary();
$table->text('sealed_api_secret');
$table->boolean('enabled')->default(false);
$table->integer('revision')->default(1);
$table->integer('tested_revision')->default(0);
$table->string('last_test_status')->default('never');
$table->string('last_error_code')->default('');
$table->string('last_agent_version')->default('');
$table->integer('last_tested_at')->default(0);
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_host_storage_configurations',
static function (Blueprint $table): void {
$table->integer('host_id')->primary();
$table->integer('revision')->default(0);
foreach (['managed', 'template', 'seed', 'user_iso', 'rescue'] as $role) {
$table->text($role . '_directory');
}
$table->string('state')->default('applying');
$table->string('operation_id')->nullable()->unique();
$table->integer('job_id')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_jobs', static function (Blueprint $table): void {
$table->increments('id');
$table->string('operation_id')->unique();
$table->string('request_idempotency_key');
$table->string('request_fingerprint');
$table->integer('parent_job_id')->nullable();
$table->integer('order_id')->nullable();
$table->integer('invoice_id')->nullable();
$table->integer('instance_id')->nullable();
$table->integer('snapshot_id')->nullable();
$table->integer('user_id');
$table->integer('host_id')->nullable();
$table->string('kind');
$table->string('execution_mode');
$table->string('phase');
$table->string('outcome')->nullable();
$table->string('status');
$table->text('sealed_payload');
$table->integer('attempts');
$table->integer('dispatch_attempts');
$table->integer('dispatch_deadline_at');
$table->integer('dispatch_started_at')->nullable();
$table->integer('agent_accepted_at')->nullable();
$table->integer('receipt_deadline_at')->nullable();
$table->integer('terminal_at')->nullable();
$table->boolean('effect_may_have_applied')->nullable();
$table->boolean('safe_to_resubmit')->nullable();
$table->string('effect_resolution');
$table->integer('effect_resolved_at')->nullable();
$table->integer('effect_resolved_by')->nullable();
$table->text('effect_resolution_reason')->nullable();
$table->string('completion_event_id')->nullable();
$table->integer('read_at')->nullable();
$table->integer('dispatched_at')->nullable();
$table->string('lease_token')->nullable();
$table->integer('lease_until')->default(0);
$table->integer('available_at');
$table->string('error_code')->nullable();
$table->boolean('finalization_incident')->default(0);
$table->string('finalization_error_code')->nullable();
$table->text('result_json')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
foreach ([1, 2] as $hostId) {
Capsule::table('cloud_hosts')->insert([
'id' => $hostId, 'name' => 'Host ' . $hostId,
'region' => 'HongKong', 'country_code' => 'HK', 'enabled' => 1,
'default_network' => 'bridge:br0',
'created_at' => 1, 'updated_at' => 1,
]);
Capsule::table('cloud_host_agents')->insert([
'host_id' => $hostId, 'sealed_api_secret' => 'test-secret',
'enabled' => 1, 'revision' => 1, 'tested_revision' => 1,
'last_test_status' => 'healthy', 'last_error_code' => '',
'last_agent_version' => 'test', 'last_tested_at' => 1,
'created_at' => 1, 'updated_at' => 1,
]);
}
$actor = [
'user_id' => 1, 'is_admin' => true, 'is_super_admin' => true,
'can_manage' => true, 'session_id' => 'storage-test',
];
$first = [
'expected_revision' => 0,
'managed_directory' => '/home/libvirt/kvm',
'template_directory' => '/home/libvirt/images',
'seed_directory' => '/home/libvirt/seed',
'user_iso_directory' => '/home/libvirt/iso',
'rescue_directory' => '/home/libvirt/rescue-iso',
];
$empty = CloudHostStorageConfigurationService::read(1);
$assert($empty['state'] === 'unconfigured'
&& $empty['revision'] === 0 && $empty['can_submit'] === true,
'An unconfigured host was not writable.');
$receipt1 = CloudHostStorageConfigurationService::save(1, $first, $actor);
$job1 = CloudJob::find((int) $receipt1['task_id']);
$row = CloudHostStorageConfiguration::find(1);
$assert($job1 !== null && $row !== null
&& (string) $row->operation_id === (string) $job1->operation_id
&& (int) $row->job_id === (int) $job1->id
&& (int) $row->revision === 1 && (string) $row->state === 'applying',
'Initial storage submission did not create one linked task.');
$sameActive = $first;
$sameActive['expected_revision'] = 1;
$activeReceipt = CloudHostStorageConfigurationService::save(
1, $sameActive, $actor
);
$assert($activeReceipt['task_id'] === $receipt1['task_id']
&& CloudJob::count() === 1,
'An active same-path request created another task.');
$changedActive = $sameActive;
$changedActive['rescue_directory'] = '/home/libvirt/rescue-corrected';
$expect(static fn () => CloudHostStorageConfigurationService::save(
1, $changedActive, $actor
), 409, 'An active task allowed concurrent path replacement.');
CloudHostStorageConfigurationService::finishJobLocked(
$job1, 'failed', null, ['effect_may_have_applied' => true]
);
$terminalize($job1, 'failed', true);
$failed = CloudHostStorageConfigurationService::read(1);
$assert($failed['state'] === 'failed'
&& $failed['can_submit'] === true
&& $failed['operation_id'] === (string) $job1->operation_id,
'A terminal failure remained blocked.');
$receipt2 = CloudHostStorageConfigurationService::save(
1, $sameActive, $actor
);
$job2 = CloudJob::find((int) $receipt2['task_id']);
$row = CloudHostStorageConfiguration::find(1);
$assert($job2 !== null
&& (int) $job2->id !== (int) $job1->id
&& (string) $job2->operation_id !== (string) $job1->operation_id
&& (int) $row->revision === 1
&& (string) $row->operation_id === (string) $job2->operation_id
&& (string) $row->state === 'applying'
&& (string) $job1->phase === 'terminal'
&& (string) $job1->outcome === 'failed',
'A same-path correction did not create a new operation while preserving the failed audit.');
CloudHostStorageConfigurationService::finishJobLocked(
$job1, 'succeeded', ['configured' => true], []
);
$row->refresh();
$assert((string) $row->operation_id === (string) $job2->operation_id
&& (string) $row->state === 'applying',
'A late superseded receipt overwrote the current generation.');
CloudHostStorageConfigurationService::finishJobLocked(
$job2, 'failed', null, []
);
$terminalize($job2, 'failed');
$changedAfterFailure = $changedActive;
$receipt3 = CloudHostStorageConfigurationService::save(
1, $changedAfterFailure, $actor
);
$job3 = CloudJob::find((int) $receipt3['task_id']);
$row = CloudHostStorageConfiguration::find(1);
$assert($job3 !== null
&& (string) $job3->operation_id !== (string) $job2->operation_id
&& (int) $row->revision === 2
&& (string) $row->rescue_directory
=== '/home/libvirt/rescue-corrected',
'A corrected path after terminal failure did not create the next revision.');
CloudHostStorageConfigurationService::finishJobLocked(
$job3, 'succeeded', ['configured' => true], []
);
$terminalize($job3, 'succeeded');
$applied = CloudHostStorageConfigurationService::read(1);
$assert($applied['state'] === 'applied'
&& $applied['configured'] === true
&& $applied['can_submit'] === false,
'A confirmed storage layout was not locked as applied.');
$changedAfterFailure['expected_revision'] = 2;
$sameApplied = CloudHostStorageConfigurationService::save(
1, $changedAfterFailure, $actor
);
$assert($sameApplied['task_id'] === $receipt3['task_id']
&& CloudJob::count() === 3,
'An exact applied request was not idempotent.');
$differentApplied = $changedAfterFailure;
$differentApplied['managed_directory'] = '/home/libvirt/other-kvm';
$expect(static fn () => CloudHostStorageConfigurationService::save(
1, $differentApplied, $actor
), 409, 'An applied storage layout was mutable.');
// A worker-finalization incident must not turn a known terminal failure
// into a permanent "applying" fence.
$receipt4 = CloudHostStorageConfigurationService::save(2, $first, $actor);
$job4 = CloudJob::find((int) $receipt4['task_id']);
$terminalize($job4, 'failed', true);
$staleProjection = CloudHostStorageConfigurationService::read(2);
$assert($staleProjection['state'] === 'failed'
&& $staleProjection['can_submit'] === true,
'A terminally failed job remained visually locked behind stale applying state.');
$staleApplying = $first;
$staleApplying['expected_revision'] = 1;
$receipt5 = CloudHostStorageConfigurationService::save(
2, $staleApplying, $actor
);
$job5 = CloudJob::find((int) $receipt5['task_id']);
$row2 = CloudHostStorageConfiguration::find(2);
$assert($job5 !== null
&& (int) $job5->id !== (int) $job4->id
&& (string) $row2->operation_id === (string) $job5->operation_id
&& (int) $row2->revision === 1,
'A linked terminal failure with a stale applying row remained blocked.');
echo "Host storage failure supersession state machine passed.\n";
}

@ -179,6 +179,8 @@ const protocolFailure = task => {
assert.match(source, /\{ title: '进行中'/);
assert.match(source, /\{ title: '已完成'/);
assert.match(source, /if \(task\.terminal && task\.id/);
assert.match(source, /raw\?\.kind === 'host_storage_configure'/);
assert.match(source, /旧存储任务已终止且不会重放/);
assert.doesNotMatch(source, /重试任务|重新提交任务|data-task-retry/);
}

Loading…
Cancel
Save