'排队中', 'running' => '执行中', 'uncertain' => '待核对', 'failed' => '失败', 'succeeded' => '已完成', 'cancelled' => '已中止', ]; public static function page(array $actor, array $input = []): array { self::authorize($actor); $filters = self::filters($input); $now = time(); $query = self::query(); self::applyFilters($query, $filters, $now); $queued = self::queuedSql(); $expired = self::expiredLeaseSql(); $summaryRow = (clone $query)->selectRaw('COUNT(*) AS total, ' . 'COALESCE(SUM(CASE WHEN ' . self::attentionSql() . ' THEN 1 ELSE 0 END), 0) AS attention, ' . "COALESCE(SUM(CASE WHEN cloud_jobs.status = 'failed' THEN 1 ELSE 0 END), 0) AS failed, " . "COALESCE(SUM(CASE WHEN cloud_jobs.status = 'uncertain' THEN 1 ELSE 0 END), 0) AS uncertain, " . 'COALESCE(SUM(CASE WHEN ' . $queued . ' THEN 1 ELSE 0 END), 0) AS queued_delayed, ' . 'COALESCE(SUM(CASE WHEN ' . $expired . ' THEN 1 ELSE 0 END), 0) AS lease_expired', [$now - self::QUEUED_DELAY_SECONDS, $now, $now, $now - self::QUEUED_DELAY_SECONDS, $now, $now])->first(); $summary = []; foreach (['total', 'attention', 'failed', 'uncertain', 'queued_delayed', 'lease_expired'] as $key) { $summary[$key] = max(0, (int) ($summaryRow?->$key ?? 0)); } $pages = max(1, (int) ceil($summary['total'] / $filters['per_page'])); $filters['page'] = min($filters['page'], $pages); $records = self::select($query)->orderByDesc('cloud_jobs.id') ->offset(($filters['page'] - 1) * $filters['per_page']) ->limit($filters['per_page'])->get(); $items = []; $canManage = self::canManage($actor); foreach ($records as $job) { $items[] = self::row($job, $now, $canManage); } $hosts = []; foreach (CloudHost::orderBy('name')->orderBy('id')->get(['id', 'name', 'region']) as $host) { $hosts[] = ['id' => (string) $host->id, 'name' => (string) $host->name, 'region' => (string) $host->region]; } $from = $summary['total'] === 0 ? 0 : ($filters['page'] - 1) * $filters['per_page'] + 1; return ['items' => $items, 'pagination' => [ 'page' => $filters['page'], 'per_page' => $filters['per_page'], 'total' => $summary['total'], 'total_pages' => $pages, 'from' => $from, 'to' => $from === 0 ? 0 : $from + count($items) - 1, ], 'summary' => $summary, 'filters' => $filters, 'hosts' => $hosts, 'kinds' => CloudOperationCatalog::kindOptions(), 'generated_at' => $now]; } public static function detail(array $actor, string $id): array { self::authorize($actor); $job = self::select(self::query()->where('cloud_jobs.id', self::identifier($id)))->first(); if ($job === null) { throw new CloudError('任务不存在', 404); } $now = time(); return ['job' => self::row($job, $now, self::canManage($actor)), 'generated_at' => $now, 'notes' => [ '此处为数据库记录快照,不代表引擎或虚拟机的实时状态。', '可调度时间表示后台最早可领取或核对任务的时间,不保证该时刻执行。', '租约到期只提示需要核对,不能据此认定操作失败或安全重试。', '操作人与实例所属用户可能不同;关联记录缺失时不会猜测所属用户。', '错误与阶段仅提供脱敏摘要;中止只停止本地后续重试,不删除记录、不触发宿主机操作或退款。', '管理员中止只停止面板后续重试;已经派发到引擎的操作仍须按不确定结果核查。', ]]; } /** * Stop future claims of one durable destructive/reconciliation task. * * This mutation is intentionally local to cloud_jobs. It does not send * a compensating request, delete any record, refund an order, or pretend * that an already-dispatched host operation was undone. */ public static function cancel(array $actor, string $id): array { self::authorize($actor); CloudAccess::manage($actor); $jobId = self::identifier($id); $now = time(); $connection = (new CloudJob())->getConnection(); $connection->transaction(function () use ($jobId, $actor, $now): void { $job = CloudJob::where('id', $jobId)->lockForUpdate()->first(); if ($job === null) { throw new CloudError('任务不存在', 404); } $kind = (string) $job->kind; if (!in_array($kind, self::CANCELLABLE_KINDS, true)) { throw new CloudError('当前任务类型不支持中止,请人工核对', 409); } $status = (string) $job->status; if ($status === 'cancelled') { return; } if (in_array($status, ['succeeded', 'failed'], true)) { throw new CloudError('任务已经结束,不能中止', 409); } if (!in_array($status, ['queued', 'running', 'uncertain'], true)) { throw new CloudError('当前任务状态不支持中止', 409); } $previousStatus = $status; $previousDispatched = $job->dispatched_at !== null; $result = json_decode((string) $job->result_json, true); if (!is_array($result) || array_is_list($result)) { $result = []; } // Keep the prior sanitized engine evidence in result_json; the // cancellation marker is appended rather than replacing it. $result['phase'] = '管理员已中止后续重试'; $result['cancelled_at'] = $now; $result['dispatched'] = $previousDispatched; $job->status = 'cancelled'; $job->error_code = 'admin_cancelled'; $job->result_json = json_encode($result, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR); // Clearing the lease is the durable claim barrier. A worker that // already holds the old token will fail the token/status guard in // finish() and cannot commit a late result. $job->lease_token = null; $job->lease_until = 0; $job->available_at = 0; $job->updated_at = $now; $job->save(); // A snapshot may already have crossed the engine boundary. Keep // its catalog uncertain rather than claiming that cancellation // reversed a libvirt mutation. This also covers a queued row // conservatively if another worker won the dispatch race earlier. if (in_array($kind, [ 'snapshot_create', 'snapshot_restore', 'snapshot_delete', 'delete', 'expiry_delete', ], true)) { CloudSnapshotService::markJobUncertain($job); CloudFirewallService::markJobUncertain($job); } // Lifecycle would otherwise enqueue the same expiry cleanup again // while the instance remains in purge_deleting. Move it to the // existing manual-reconciliation terminal state; no host action // is performed here. if ($kind === 'expiry_delete' && $job->instance_id !== null) { $instance = CloudInstance::where('id', (int) $job->instance_id) ->lockForUpdate()->first(); if ($instance !== null && (string) $instance->status !== 'deleted') { $instance->status = 'delete_failed'; $instance->service_status = 'purge_failed'; $instance->updated_at = $now; $instance->save(); } } elseif ($kind === 'delete' && $job->instance_id !== null) { // User-triggered deletes do not normally enter deleting until // the engine confirms them. If they already did, hold the // instance for manual reconciliation instead of retrying. $instance = CloudInstance::where('id', (int) $job->instance_id) ->lockForUpdate()->first(); if ($instance !== null && (string) $instance->status === 'deleting') { $instance->status = 'delete_failed'; $instance->updated_at = $now; $instance->save(); } } $operationId = preg_replace('/[^A-Za-z0-9_-]/', '?', (string) $job->operation_id) ?? '?'; error_log(sprintf( 'cloud admin cancelled job id=%d operation_id=%s kind=%s previous_status=%s dispatched=%s actor_id=%d', (int) $job->id, substr($operationId, 0, 128), $kind, $previousStatus, $previousDispatched ? '1' : '0', (int) ($actor['user_id'] ?? 0) )); }, 3); return self::detail($actor, (string) $jobId); } public static function filters(array $input): array { foreach ($input as $key => $value) { if (!in_array($key, ['status', 'kind', 'host_id', 'q', 'page', 'per_page'], true) || (!is_string($value) && !is_int($value))) { throw new CloudError('筛选参数无效', 400); } } $status = (string) ($input['status'] ?? 'attention'); $kind = (string) ($input['kind'] ?? ''); if (!in_array($status, array_merge(['all', 'attention'], array_keys(self::STATUSES)), true) || ($kind !== '' && !array_key_exists($kind, CloudOperationCatalog::kinds()))) { throw new CloudError('任务状态或类型无效', 400); } $q = trim((string) ($input['q'] ?? '')); if (mb_strlen($q, 'UTF-8') > 80 || preg_match('/[\x00-\x1f\x7f]/u', $q) !== 0) { throw new CloudError('搜索关键词最多 80 个字符,且不能包含控制字符', 400); } $host = (string) ($input['host_id'] ?? ''); $page = self::identifier((string) ($input['page'] ?? '1')); $perPage = self::identifier((string) ($input['per_page'] ?? '20')); if (!in_array($perPage, [20, 50, 100], true)) { throw new CloudError('每页条数只能为 20、50 或 100', 400); } return ['status' => $status, 'kind' => $kind, 'host_id' => $host === '' ? null : (string) self::identifier($host), 'q' => $q, 'page' => $page, 'per_page' => $perPage]; } private static function authorize(array $actor): void { if (!($actor['is_admin'] ?? false)) { throw new CloudError('需要管理员权限', 403); } } private static function canManage(array $actor): bool { return ($actor['is_admin'] ?? false) === true && ($actor['is_super_admin'] ?? false) === true && ($actor['can_manage'] ?? false) === true; } private static function identifier(string $value): int { if (preg_match('/^[1-9][0-9]{0,17}$/D', $value) !== 1) { throw new CloudError('编号或页码无效', 400); } return (int) $value; } private static function query() { return CloudJob::query() ->leftJoin('cloud_hosts as operation_host', 'operation_host.id', '=', 'cloud_jobs.host_id') ->leftJoin('user as operation_actor', 'operation_actor.id', '=', 'cloud_jobs.user_id') ->leftJoin('cloud_instances as operation_instance', static function ($join): void { $join->on('operation_instance.id', '=', 'cloud_jobs.instance_id') ->on('operation_instance.host_id', '=', 'cloud_jobs.host_id'); }) ->leftJoin('cloud_orders as operation_order', static function ($join): void { $join->on('operation_order.id', '=', 'cloud_jobs.order_id') ->on('operation_order.host_id', '=', 'cloud_jobs.host_id') ->whereRaw('(cloud_jobs.instance_id IS NULL OR ' . '(operation_instance.id IS NOT NULL ' . 'AND operation_order.user_id = operation_instance.user_id ' . 'AND (operation_order.instance_id IS NULL ' . 'OR operation_order.instance_id = operation_instance.id)))'); }) ->leftJoin('cloud_instances as fallback_instance', static function ($join): void { $join->on('fallback_instance.id', '=', 'operation_order.instance_id') ->on('fallback_instance.user_id', '=', 'operation_order.user_id') ->on('fallback_instance.host_id', '=', 'cloud_jobs.host_id') ->whereNull('cloud_jobs.instance_id'); }) ->leftJoin('user as operation_owner', static function ($join): void { $join->whereRaw('operation_owner.id = COALESCE(operation_instance.user_id, ' . 'fallback_instance.user_id, operation_order.user_id)'); }); } private static function select($query) { $columns = []; foreach (['id', 'operation_id', 'user_id', 'host_id', 'kind', 'status', 'attempts', 'error_code', 'dispatched_at', 'lease_until', 'available_at', 'created_at', 'updated_at'] as $column) { $columns[] = 'cloud_jobs.' . $column; } return $query->select(array_merge($columns, [ 'operation_host.name as host_name', 'operation_host.region as host_region', 'operation_actor.user_name as actor_name', 'operation_actor.email as actor_email', 'operation_owner.user_name as owner_name', 'operation_owner.email as owner_email', 'operation_order.id as linked_order_id', ]))->selectRaw('COALESCE(operation_instance.user_id, fallback_instance.user_id, ' . 'operation_order.user_id) AS owner_id, ' . 'COALESCE(operation_instance.id, fallback_instance.id) AS linked_instance_id, ' . 'COALESCE(operation_instance.name, fallback_instance.name) AS instance_name, ' . 'COALESCE(operation_instance.uuid, fallback_instance.uuid) AS instance_uuid, ' . 'COALESCE(operation_instance.status, fallback_instance.status) AS instance_status, ' . "CASE WHEN JSON_VALID(cloud_jobs.result_json) THEN " . "JSON_EXTRACT(cloud_jobs.result_json, '$.phase') ELSE NULL END AS phase_json"); } private static function queuedSql(): string { return "(cloud_jobs.status = 'queued' AND cloud_jobs.available_at <= ? AND cloud_jobs.lease_until <= ?)"; } private static function expiredLeaseSql(): string { return "(cloud_jobs.status = 'running' AND cloud_jobs.lease_until > 0 AND cloud_jobs.lease_until <= ?)"; } private static function attentionSql(): string { return "(cloud_jobs.status IN ('failed', 'uncertain') OR " . self::queuedSql() . ' OR ' . self::expiredLeaseSql() . ')'; } private static function applyFilters($query, array $filters, int $now): void { if ($filters['status'] === 'attention') { $query->whereRaw(self::attentionSql(), [$now - self::QUEUED_DELAY_SECONDS, $now, $now]); } elseif ($filters['status'] !== 'all') { $query->where('cloud_jobs.status', $filters['status']); } if ($filters['kind'] !== '') { $query->where('cloud_jobs.kind', $filters['kind']); } if ($filters['host_id'] !== null) { $query->where('cloud_jobs.host_id', $filters['host_id']); } if ($filters['q'] !== '') { $like = '%' . str_replace(['!', '%', '_'], ['!!', '!%', '!_'], $filters['q']) . '%'; $query->where(static function ($search) use ($filters, $like): void { $search->whereRaw("cloud_jobs.operation_id LIKE ? ESCAPE '!'", [$like]); foreach (['operation_instance', 'fallback_instance'] as $table) { foreach (['uuid', 'name'] as $column) { $search->orWhereRaw($table . '.' . $column . " LIKE ? ESCAPE '!'", [$like]); } } if (preg_match('/^[1-9][0-9]{0,17}$/D', $filters['q']) === 1) { $search->orWhere('cloud_jobs.id', (int) $filters['q']); } }); } } private static function row($job, int $now, bool $canManage = false): array { $status = (string) $job->status; $lease = self::timestamp($job->lease_until); $available = self::timestamp($job->available_at); $reason = in_array($status, ['failed', 'uncertain'], true) ? $status : ''; if ($status === 'queued' && (int) $job->available_at <= $now - self::QUEUED_DELAY_SECONDS && (int) $job->lease_until <= $now) { $reason = 'queued_delayed'; } elseif ($status === 'running' && $lease !== null && $lease <= $now) { $reason = 'lease_expired'; } $phase = $job->phase_json; // MySQL returns a JSON string; SQLite JSON_EXTRACT returns plain text. if (is_string($phase) && str_starts_with($phase, '"')) { $phase = json_decode($phase, true); } if (is_string($phase) && (str_starts_with($phase, '{') || str_starts_with($phase, '[') || in_array($phase, ['null', 'true', 'false'], true))) { $phase = null; } $uuid = (string) ($job->instance_uuid ?? ''); $validUuid = preg_match('/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/iD', $uuid) === 1; $attentionLabels = ['failed' => '任务已失败', 'uncertain' => '等待核对结果', 'queued_delayed' => '可调度后已等待超过 5 分钟', 'lease_expired' => '执行租约已到期,需核对']; $action = CloudJobService::action($job); $labels = CloudOperationCatalog::labels((string) $job->kind, $action); [$canCancel, $cancelReason] = self::cancellation($job, $now, $canManage); return [ 'id' => (string) $job->id, 'operation_id' => preg_match('/^[a-f0-9]{32}$/iD', (string) $job->operation_id) === 1 ? (string) $job->operation_id : null, 'kind' => (string) $job->kind, 'action' => $action, 'kind_label' => $labels['kind_label'], 'action_label' => $labels['action_label'], 'status' => $status, 'can_cancel' => $canCancel, 'cancel_reason' => $cancelReason, 'attempts' => max(0, (int) $job->attempts), 'error_code' => CloudOperationErrorCatalog::publicCode($job->error_code), 'error' => CloudOperationErrorCatalog::adminSummary( (string) $job->kind, $status, $job->error_code, $job->linked_order_id !== null, $action ), 'phase' => self::jobText($phase), 'host_id' => (string) $job->host_id, 'host_name' => (string) ($job->host_name ?? ''), 'region' => (string) ($job->host_region ?? ''), 'actor_id' => (string) $job->user_id, 'actor_name' => AdminDataMasker::account($job->actor_name ?? ''), 'actor_email' => AdminDataMasker::email($job->actor_email ?? ''), 'owner_id' => $job->owner_id === null ? null : (string) $job->owner_id, 'owner_name' => $job->owner_id === null ? null : AdminDataMasker::account($job->owner_name ?? ''), 'owner_email' => $job->owner_id === null ? null : AdminDataMasker::email($job->owner_email ?? ''), 'instance_id' => $job->linked_instance_id === null ? null : (string) $job->linked_instance_id, 'instance_name' => $job->instance_name, 'instance_uuid' => $job->instance_uuid, 'instance_status' => $job->instance_status, 'order_id' => $job->linked_order_id === null ? null : (string) $job->linked_order_id, 'created_at' => max(0, (int) $job->created_at), 'updated_at' => max(0, (int) $job->updated_at), 'dispatched_at' => self::timestamp($job->dispatched_at), 'available_at' => $available, 'lease_until' => $lease, 'attention' => $reason !== '', 'attention_reason' => $reason, 'links' => [ 'instance' => $validUuid && $job->instance_status !== 'deleted' ? '/admin/cloud/instances?q=' . rawurlencode($uuid) : null, 'billing' => $job->linked_order_id === null ? null : '/admin/billing?view=orders&q=' . $job->linked_order_id, ], 'display' => ['kind' => $labels['kind_label'], 'action' => $labels['action_label'], 'status' => self::STATUSES[$status] ?? '其他状态', 'attention' => $attentionLabels[$reason] ?? '无异常标记', 'lease' => $lease === null ? '无执行租约' : ($lease > $now ? '租约有效' : '租约已到期')], ]; } /** @return array{0:bool,1:?string} */ private static function cancellation($job, int $now, bool $canManage): array { if (!$canManage) { return [false, '只有超级管理员可以中止任务']; } if (!in_array((string) $job->kind, self::CANCELLABLE_KINDS, true)) { return [false, '当前任务类型不支持中止']; } $status = (string) $job->status; if ($status === 'cancelled') { return [false, '任务已中止']; } if (in_array($status, ['succeeded', 'failed'], true)) { return [false, '任务已经结束']; } if (!in_array($status, ['queued', 'running', 'uncertain'], true)) { return [false, '当前任务状态不支持中止']; } return [true, $status === 'running' ? ((int) $job->lease_until > $now ? '任务正在执行;中止后会忽略迟到结果并停止后续重试' : '执行租约已到期,将停止后续重试') : '将停止后续重试,已派发操作仍需核查']; } private static function timestamp($value): ?int { return $value !== null && (int) $value > 0 ? (int) $value : null; } private static function jobText($value): ?string { // Apply the same unconditional secret/path redaction to every // administrator role, including super administrators. return CloudOperationErrorCatalog::adminSummary(null, null, $value); } }