Allow cancelling stuck power reconciliation tasks

venix
admin 1 month ago
parent 1100792586
commit 61726f87e4

@ -11,7 +11,7 @@
- 可按任务状态、类型、宿主机、任务编号、操作标识、实例名称或 UUID 筛选;每页 20/50/100 条。摘要统计跟随当前筛选。
- 详情区分操作人和实例所属用户,展示调度时间、尝试次数、脱敏错误和关联入口。实例已经删除或关联丢失时仍保留任务记录,不猜测所属用户。
列表、详情和健康检查保持数据库快照读取;超级管理员可以对快照创建/恢复/删除、实例删除及到期清理任务使用“中止任务”按钮。中止只在数据库事务中将未结束任务标记为 `cancelled`、清除执行租约并阻止后续 worker 领取,不删除任务、不退款,也不向引擎发送补偿请求。已经派发到引擎的操作不会被假定已回滚,快照/删除关联状态会保守保留为待核对,`expiry_delete` 会进入 `purge_failed` 等待人工核查。租约过期不等于操作失败,不应据此手动重放破坏性操作。
列表、详情和健康检查保持数据库快照读取;超级管理员可以对快照创建/恢复/删除、实例删除及到期清理任务使用“中止任务”按钮;已经派发并陷入 `uncertain` 核对循环的电源任务也可中止。排队中或 `running` 的电源任务不允许中止,避免第一个宿主机动作尚在执行时就放开第二个动作。中止只在数据库事务中将未结束任务标记为 `cancelled`、清除执行租约并阻止后续 worker 领取,不删除任务、不退款,也不向引擎发送补偿请求。已经派发到引擎的操作不会被假定已回滚,快照/删除关联状态会保守保留为待核对,`expiry_delete` 会进入 `purge_failed` 等待人工核查。租约过期不等于操作失败,不应据此手动重放破坏性操作。
```text
GET /api/v1/admin/operations
@ -20,7 +20,7 @@ GET /api/v1/admin/operations/health
POST /api/v1/admin/operations/jobs/{id}/cancel
```
需要管理员会话,响应禁止缓存,POST 还需要管理端 CSRF 令牌。读取任务不创建引擎会话、不读取任务加密载荷或租约令牌、不返回原始结果 JSON。健康检查独立发送签名的引擎健康请求。重复中止同一任务是幂等的;已完成任务、非允许类型或不存在任务返回受控错误。
需要管理员会话,响应禁止缓存,POST 还需要管理端 CSRF 令牌。读取任务不创建引擎会话、不读取租约令牌、不返回原始结果 JSON 或任务密文。仅当 `kind=action` 时,服务端才条件读取加密载荷,并在服务端解密、校验 `start/shutdown/reboot/poweroff/reset` 白名单,用于安全标签和中止门禁;密文和其他载荷字段始终不进入浏览器响应。健康检查独立发送签名的引擎健康请求。重复中止同一任务是幂等的;已完成任务、非允许类型、不可验证电源载荷或不存在任务返回受控错误。
## 运行状态的含义

@ -17,6 +17,7 @@ final class CloudAdminOperationsService
* engine; it only prevents a worker from claiming the row again.
*/
private const CANCELLABLE_KINDS = [
'action',
'snapshot_create', 'snapshot_restore', 'snapshot_delete',
'delete', 'expiry_delete',
];
@ -115,11 +116,12 @@ final class CloudAdminOperationsService
if ($status === 'cancelled') {
return;
}
if (in_array($status, ['succeeded', 'failed'], true)) {
throw new CloudError('任务已经结束,不能中止', 409);
}
if (!in_array($status, ['queued', 'running', 'uncertain'], true)) {
throw new CloudError('当前任务状态不支持中止', 409);
$action = $kind === 'action' ? CloudJobService::action($job) : null;
[$canCancel, $cancelReason] = self::cancellation(
$job, $now, true, $action
);
if (!$canCancel) {
throw new CloudError($cancelReason ?? '当前任务状态不支持中止', 409);
}
$previousStatus = $status;
@ -296,7 +298,9 @@ final class CloudAdminOperationsService
'operation_actor.user_name as actor_name', 'operation_actor.email as actor_email',
'operation_owner.user_name as owner_name', 'operation_owner.email as owner_email',
'operation_order.id as linked_order_id',
]))->selectRaw('COALESCE(operation_instance.user_id, fallback_instance.user_id, '
]))->selectRaw("CASE WHEN cloud_jobs.kind = 'action' "
. 'THEN cloud_jobs.payload_cipher ELSE NULL END AS payload_cipher, '
. 'COALESCE(operation_instance.user_id, fallback_instance.user_id, '
. 'operation_order.user_id) AS owner_id, '
. 'COALESCE(operation_instance.id, fallback_instance.id) AS linked_instance_id, '
. 'COALESCE(operation_instance.name, fallback_instance.name) AS instance_name, '
@ -378,7 +382,9 @@ final class CloudAdminOperationsService
'queued_delayed' => '可调度后已等待超过 5 分钟', 'lease_expired' => '执行租约已到期,需核对'];
$action = CloudJobService::action($job);
$labels = CloudOperationCatalog::labels((string) $job->kind, $action);
[$canCancel, $cancelReason] = self::cancellation($job, $now, $canManage);
[$canCancel, $cancelReason] = self::cancellation(
$job, $now, $canManage, $action
);
return [
'id' => (string) $job->id,
'operation_id' => preg_match('/^[a-f0-9]{32}$/iD', (string) $job->operation_id) === 1
@ -423,7 +429,9 @@ final class CloudAdminOperationsService
}
/** @return array{0:bool,1:?string} */
private static function cancellation($job, int $now, bool $canManage): array
private static function cancellation(
$job, int $now, bool $canManage, ?string $action = null
): array
{
if (!$canManage) {
return [false, '只有超级管理员可以中止任务'];
@ -441,6 +449,21 @@ final class CloudAdminOperationsService
if (!in_array($status, ['queued', 'running', 'uncertain'], true)) {
return [false, '当前任务状态不支持中止'];
}
if ((string) $job->kind === 'action') {
if ($action === null || !CloudOperationCatalog::isPowerAction($action)) {
return [false, '电源任务内容无法验证,不能安全中止'];
}
// A running power request may still cross the engine boundary.
// Cancelling it would immediately release the instance for a
// second action while the first one can still change libvirt
// state. The requested escape hatch is therefore deliberately
// limited to the post-dispatch reconciliation loop.
return $status === 'uncertain'
? [true, '将停止该电源任务的后续核对;已执行的宿主机动作不会回滚']
: [false, $status === 'running'
? '电源任务正在执行,不能中止;请等待其完成或进入待核对'
: '电源任务尚未进入待核对,当前不允许中止'];
}
return [true, $status === 'running'
? ((int) $job->lease_until > $now
? '任务正在执行;中止后会忽略迟到结果并停止后续重试'

@ -177,6 +177,10 @@ function canCancel(job) {
if (root?.dataset.canManage !== '1') return false;
if (job.can_cancel !== undefined) return job.can_cancel === true;
const status = String(job.status || '');
if (job.kind === 'action') {
return ['start', 'shutdown', 'reboot', 'poweroff', 'reset'].includes(job.action)
&& status === 'uncertain';
}
return ['snapshot_create', 'snapshot_restore', 'snapshot_delete', 'delete', 'expiry_delete'].includes(job.kind)
&& ['queued', 'running', 'uncertain'].includes(status);
}

@ -5,7 +5,7 @@
{assign var=operations_statuses value=['attention'=>'需要关注','all'=>'全部状态','queued'=>'已排队','running'=>'执行中','uncertain'=>'待核对','failed'=>'失败','succeeded'=>'已完成','cancelled'=>'已中止']}
<main data-admin-operations data-can-manage="{if $admin_can_manage|default:false}1{else}0{/if}">
<div class="py-3 d-flex align-items-sm-center flex-sm-row flex-column gap-3">
<div class="flex-grow-1"><h1 class="fs-20 fw-semibold mb-1">系统运行与异常任务</h1><p class="text-muted mb-0">查看后台运行状态与全局任务记录;可中止尚未结束的快照或删除任务,停止后续重试。</p></div>
<div class="flex-grow-1"><h1 class="fs-20 fw-semibold mb-1">系统运行与异常任务</h1><p class="text-muted mb-0">查看后台运行状态与全局任务记录;可中止待核对的电源任务,或尚未结束的快照与删除任务,只停止面板后续重试。</p></div>
<div class="admin-checkout-actions"><span class="admin-config-status {if $operations_error|default:''}is-error{else}is-success{/if}" id="operations-notice" role="status" aria-live="polite">{if $operations_error|default:''}{$operations_error|escape}{else}已读取 {$operations_page.pagination.total|default:0|escape} 条任务{/if}</span><button class="btn btn-light" id="operations-refresh" type="button"><i class="mdi mdi-refresh me-1" aria-hidden="true"></i>刷新任务</button></div>
</div>
<section class="card mb-3" aria-labelledby="operations-health-title">

@ -1,2 +1,2 @@
{* Shared Venix CSS/JS cache key. Increment whenever a shared browser asset changes. *}
{assign var=venix_asset_version value='2026101002' scope='global'}
{assign var=venix_asset_version value='2026101003' scope='global'}

@ -177,6 +177,30 @@ function findButton(test, attribute, value) {
assert.equal(findButton(test, 'operationCancel', 41), null);
}
// A rolling-upgrade response without an explicit capability field only falls
// back to reviewed power actions that are already in reconciliation.
{
const test = fixture(true);
const display = { kind: '关机', status: '待核对', attention: '等待结果核对', lease: '无执行租约' };
const power = job(44, {
kind: 'action', action: 'shutdown', can_cancel: undefined, display,
});
const running = job(45, {
kind: 'action', action: 'shutdown', status: 'running', can_cancel: undefined, display,
});
const forged = job(46, {
kind: 'action', action: 'destroy', can_cancel: undefined, display,
});
await test.click(test.nodes['operations-refresh']);
await test.respond(page([power, running, forged]));
assert.ok(findButton(test, 'operationCancel', 44),
'A supported uncertain power task lost its fallback cancellation control.');
assert.equal(findButton(test, 'operationCancel', 45), null,
'A running power task exposed the fallback cancellation control.');
assert.equal(findButton(test, 'operationCancel', 46), null,
'An unreviewed action exposed the fallback cancellation control.');
}
// Opening and rendering a detail keeps the cancellation button usable after
// links are rebuilt; the button is not detached and looked up as null.
{

@ -18,6 +18,24 @@ namespace App\Services {
}
namespace App\Services\Cloud {
/** Test-only in-memory keyring; production still uses authenticated sodium ciphertext. */
final class CloudSecrets
{
private static array $payloads = [];
public static function encrypt(array $payload): string
{
$id = 'test-only:' . bin2hex(random_bytes(16));
self::$payloads[$id] = $payload;
return $id;
}
public static function decrypt(string $ciphertext): array
{
if (!array_key_exists($ciphertext, self::$payloads)) {
throw new CloudError('任务数据无法解密', 503);
}
return self::$payloads[$ciphertext];
}
}
final class CloudSessionService
{
public static int $calls = 0;
@ -252,9 +270,15 @@ namespace {
$log = $connection->getQueryLog(); $counts[] = count($log);
foreach ($log as $query) {
$sql = strtolower($query['query']);
$hasPayload = str_contains($sql, 'payload_cipher');
$scopedPowerPayload = str_contains($sql,
"case when cloud_jobs.kind = 'action' then cloud_jobs.payload_cipher "
. 'else null end as payload_cipher')
&& substr_count($sql, 'payload_cipher') === 2;
$assert(str_starts_with($sql, 'select ') && !str_contains($sql, 'cloud_sessions')
&& !str_contains($sql, 'payload_cipher') && !str_contains($sql, 'lease_token'),
'Read path touched a sensitive field/session or wrote state.');
&& (!$hasPayload || $scopedPowerPayload)
&& !str_contains($sql, 'lease_token'),
'Read path selected an unscoped secret/session field or wrote state.');
}
}
$assert($counts === [3, 3], 'Per-row query amplification detected.');
@ -303,6 +327,55 @@ namespace {
$assert(Operations::detail($readonlyAdmin, '211')['job']['can_cancel'] === false,
'Read-only administrators received the task cancellation control.');
$reject(static fn () => Operations::cancel($readonlyAdmin, '211'), 403);
$powerPayload = \App\Services\Cloud\CloudSecrets::encrypt([
'method' => 'POST',
'resource' => 'vms/11111111-1111-4111-8111-111111111110/actions',
'body' => ['action' => 'shutdown'],
]);
$insertJob(223, [
'kind' => 'action', 'status' => 'uncertain', 'dispatched_at' => $now - 5,
'payload_cipher' => $powerPayload,
]);
$powerDetail = Operations::detail($admin, '223')['job'];
$assert($powerDetail['can_cancel'] === true
&& $powerDetail['action'] === 'shutdown'
&& $powerDetail['kind_label'] === '关机',
'A reviewed power action did not expose its safe label and retry-stop control.');
$cancelledPower = Operations::cancel($admin, '223');
$cancelledPowerRow = Capsule::table('cloud_jobs')->where('id', 223)->first();
$assert(($cancelledPower['job']['status'] ?? null) === 'cancelled'
&& ($cancelledPower['job']['error_code'] ?? null) === 'admin_cancelled'
&& $cancelledPowerRow !== null && $cancelledPowerRow->lease_token === null
&& (int) $cancelledPowerRow->lease_until === 0
&& (int) $cancelledPowerRow->available_at === 0
&& (int) $cancelledPowerRow->dispatched_at === $now - 5,
'Administrator cancellation did not stop future power-task retries locally.');
$insertJob(224, [
'kind' => 'action', 'status' => 'running', 'dispatched_at' => $now - 4,
'lease_until' => $now + 60, 'payload_cipher' => $powerPayload,
]);
$runningPower = Operations::detail($admin, '224')['job'];
$assert($runningPower['can_cancel'] === false
&& str_contains((string) $runningPower['cancel_reason'], '正在执行'),
'A running power action incorrectly exposed the cancellation control.');
$reject(static fn () => Operations::cancel($admin, '224'), 409);
$assert(Capsule::table('cloud_jobs')->where('id', 224)->value('status') === 'running',
'Rejected running power cancellation mutated the task.');
$insertJob(225, [
'kind' => 'action', 'status' => 'uncertain', 'dispatched_at' => $now - 3,
'payload_cipher' => \App\Services\Cloud\CloudSecrets::encrypt([
'method' => 'POST',
'resource' => 'vms/11111111-1111-4111-8111-111111111110/actions',
'body' => ['action' => 'destroy'],
]),
]);
$invalidPower = Operations::detail($admin, '225')['job'];
$assert($invalidPower['can_cancel'] === false
&& str_contains((string) $invalidPower['cancel_reason'], '无法验证'),
'An unverifiable action payload incorrectly exposed the cancellation control.');
$reject(static fn () => Operations::cancel($admin, '225'), 409);
$assert(Capsule::table('cloud_jobs')->where('id', 225)->value('status') === 'uncertain',
'Rejected forged power cancellation mutated the task.');
$cancelled = Operations::cancel($admin, '211');
$cancelledRow = Capsule::table('cloud_jobs')->where('id', 211)->first();
$assert(($cancelled['job']['status'] ?? null) === 'cancelled'

@ -18,7 +18,7 @@ $assert(str_contains($template, 'data-instance-setting-tab="firewall"')
&& str_contains($template, 'id="vm-firewall-rules"'),
'Instance settings must expose the integrated firewall tab and whole-policy editor.');
$assert(str_contains($assets, "venix_asset_version value='2026101002'"),
$assert(str_contains($assets, "venix_asset_version value='2026101003'"),
'Shared Venix asset cache key must publish the firewall JavaScript and CSS.');
$assert(str_contains($template, '反欺骗保护始终开启')

@ -128,7 +128,7 @@ $assert(str_contains($fabricClient, 'function endpointSource(value)')
&& str_contains($fabricClient, 'firewall_competing_service_unknown')
&& str_contains($fabricClient, 'firewall_manager_conflict'),
'Fabric readiness UI does not distinguish explicit and SSH-IPv4 fallback endpoints.');
$assert(str_contains($assets, "venix_asset_version value='2026101002'"),
$assert(str_contains($assets, "venix_asset_version value='2026101003'"),
'Changed host/Fabric browser assets did not advance the shared cache key.');
echo "Cloud host Fabric endpoint contracts passed.\n";

Loading…
Cancel
Save