You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/app/Services/Cloud/CloudAdminOperationsService...

456 lines
23 KiB
PHP

<?php
namespace App\Services\Cloud;
use App\Models\CloudHost;
use App\Models\CloudInstance;
use App\Models\CloudJob;
use App\Services\AdminDataMasker;
/** Database-backed operations inventory and a local retry-stop control. */
final class CloudAdminOperationsService
{
private const QUEUED_DELAY_SECONDS = 300;
/**
* Cancellation is deliberately limited to durable operations whose next
* retry is unsafe or no longer useful. The endpoint never calls the
* engine; it only prevents a worker from claiming the row again.
*/
private const CANCELLABLE_KINDS = [
'snapshot_create', 'snapshot_restore', 'snapshot_delete',
'delete', 'expiry_delete',
];
private const STATUSES = [
'queued' => '排队中', 'running' => '执行中', 'uncertain' => '待核对',
'failed' => '失败', 'succeeded' => '已完成', 'cancelled' => '已中止',
];
public static function page(array $actor, array $input = []): array
{
self::authorize($actor);
$filters = self::filters($input);
$now = time();
$query = self::query();
self::applyFilters($query, $filters, $now);
$queued = self::queuedSql();
$expired = self::expiredLeaseSql();
$summaryRow = (clone $query)->selectRaw('COUNT(*) AS total, '
. 'COALESCE(SUM(CASE WHEN ' . self::attentionSql() . ' THEN 1 ELSE 0 END), 0) AS attention, '
. "COALESCE(SUM(CASE WHEN cloud_jobs.status = 'failed' THEN 1 ELSE 0 END), 0) AS failed, "
. "COALESCE(SUM(CASE WHEN cloud_jobs.status = 'uncertain' THEN 1 ELSE 0 END), 0) AS uncertain, "
. 'COALESCE(SUM(CASE WHEN ' . $queued . ' THEN 1 ELSE 0 END), 0) AS queued_delayed, '
. 'COALESCE(SUM(CASE WHEN ' . $expired . ' THEN 1 ELSE 0 END), 0) AS lease_expired',
[$now - self::QUEUED_DELAY_SECONDS, $now, $now,
$now - self::QUEUED_DELAY_SECONDS, $now, $now])->first();
$summary = [];
foreach (['total', 'attention', 'failed', 'uncertain', 'queued_delayed', 'lease_expired'] as $key) {
$summary[$key] = max(0, (int) ($summaryRow?->$key ?? 0));
}
$pages = max(1, (int) ceil($summary['total'] / $filters['per_page']));
$filters['page'] = min($filters['page'], $pages);
$records = self::select($query)->orderByDesc('cloud_jobs.id')
->offset(($filters['page'] - 1) * $filters['per_page'])
->limit($filters['per_page'])->get();
$items = [];
$canManage = self::canManage($actor);
foreach ($records as $job) {
$items[] = self::row($job, $now, $canManage);
}
$hosts = [];
foreach (CloudHost::orderBy('name')->orderBy('id')->get(['id', 'name', 'region']) as $host) {
$hosts[] = ['id' => (string) $host->id, 'name' => (string) $host->name,
'region' => (string) $host->region];
}
$from = $summary['total'] === 0 ? 0 : ($filters['page'] - 1) * $filters['per_page'] + 1;
return ['items' => $items, 'pagination' => [
'page' => $filters['page'], 'per_page' => $filters['per_page'],
'total' => $summary['total'], 'total_pages' => $pages,
'from' => $from, 'to' => $from === 0 ? 0 : $from + count($items) - 1,
], 'summary' => $summary, 'filters' => $filters, 'hosts' => $hosts,
'kinds' => CloudOperationCatalog::kindOptions(),
'generated_at' => $now];
}
public static function detail(array $actor, string $id): array
{
self::authorize($actor);
$job = self::select(self::query()->where('cloud_jobs.id', self::identifier($id)))->first();
if ($job === null) {
throw new CloudError('任务不存在', 404);
}
$now = time();
return ['job' => self::row($job, $now, self::canManage($actor)), 'generated_at' => $now, 'notes' => [
'此处为数据库记录快照,不代表引擎或虚拟机的实时状态。',
'可调度时间表示后台最早可领取或核对任务的时间,不保证该时刻执行。',
'租约到期只提示需要核对,不能据此认定操作失败或安全重试。',
'操作人与实例所属用户可能不同;关联记录缺失时不会猜测所属用户。',
'错误与阶段仅提供脱敏摘要;中止只停止本地后续重试,不删除记录、不触发宿主机操作或退款。',
'管理员中止只停止面板后续重试;已经派发到引擎的操作仍须按不确定结果核查。',
]];
}
/**
* Stop future claims of one durable destructive/reconciliation task.
*
* This mutation is intentionally local to cloud_jobs. It does not send
* a compensating request, delete any record, refund an order, or pretend
* that an already-dispatched host operation was undone.
*/
public static function cancel(array $actor, string $id): array
{
self::authorize($actor);
CloudAccess::manage($actor);
$jobId = self::identifier($id);
$now = time();
$connection = (new CloudJob())->getConnection();
$connection->transaction(function () use ($jobId, $actor, $now): void {
$job = CloudJob::where('id', $jobId)->lockForUpdate()->first();
if ($job === null) {
throw new CloudError('任务不存在', 404);
}
$kind = (string) $job->kind;
if (!in_array($kind, self::CANCELLABLE_KINDS, true)) {
throw new CloudError('当前任务类型不支持中止,请人工核对', 409);
}
$status = (string) $job->status;
if ($status === 'cancelled') {
return;
}
if (in_array($status, ['succeeded', 'failed'], true)) {
throw new CloudError('任务已经结束,不能中止', 409);
}
if (!in_array($status, ['queued', 'running', 'uncertain'], true)) {
throw new CloudError('当前任务状态不支持中止', 409);
}
$previousStatus = $status;
$previousDispatched = $job->dispatched_at !== null;
$result = json_decode((string) $job->result_json, true);
if (!is_array($result) || array_is_list($result)) {
$result = [];
}
// Keep the prior sanitized engine evidence in result_json; the
// cancellation marker is appended rather than replacing it.
$result['phase'] = '管理员已中止后续重试';
$result['cancelled_at'] = $now;
$result['dispatched'] = $previousDispatched;
$job->status = 'cancelled';
$job->error_code = 'admin_cancelled';
$job->result_json = json_encode($result,
JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES
| JSON_THROW_ON_ERROR);
// Clearing the lease is the durable claim barrier. A worker that
// already holds the old token will fail the token/status guard in
// finish() and cannot commit a late result.
$job->lease_token = null;
$job->lease_until = 0;
$job->available_at = 0;
$job->updated_at = $now;
$job->save();
// A snapshot may already have crossed the engine boundary. Keep
// its catalog uncertain rather than claiming that cancellation
// reversed a libvirt mutation. This also covers a queued row
// conservatively if another worker won the dispatch race earlier.
if (in_array($kind, [
'snapshot_create', 'snapshot_restore', 'snapshot_delete',
'delete', 'expiry_delete',
], true)) {
CloudSnapshotService::markJobUncertain($job);
CloudFirewallService::markJobUncertain($job);
}
// Lifecycle would otherwise enqueue the same expiry cleanup again
// while the instance remains in purge_deleting. Move it to the
// existing manual-reconciliation terminal state; no host action
// is performed here.
if ($kind === 'expiry_delete' && $job->instance_id !== null) {
$instance = CloudInstance::where('id', (int) $job->instance_id)
->lockForUpdate()->first();
if ($instance !== null && (string) $instance->status !== 'deleted') {
$instance->status = 'delete_failed';
$instance->service_status = 'purge_failed';
$instance->updated_at = $now;
$instance->save();
}
} elseif ($kind === 'delete' && $job->instance_id !== null) {
// User-triggered deletes do not normally enter deleting until
// the engine confirms them. If they already did, hold the
// instance for manual reconciliation instead of retrying.
$instance = CloudInstance::where('id', (int) $job->instance_id)
->lockForUpdate()->first();
if ($instance !== null && (string) $instance->status === 'deleting') {
$instance->status = 'delete_failed';
$instance->updated_at = $now;
$instance->save();
}
}
$operationId = preg_replace('/[^A-Za-z0-9_-]/', '?', (string) $job->operation_id) ?? '?';
error_log(sprintf(
'cloud admin cancelled job id=%d operation_id=%s kind=%s previous_status=%s dispatched=%s actor_id=%d',
(int) $job->id, substr($operationId, 0, 128), $kind,
$previousStatus, $previousDispatched ? '1' : '0',
(int) ($actor['user_id'] ?? 0)
));
}, 3);
return self::detail($actor, (string) $jobId);
}
public static function filters(array $input): array
{
foreach ($input as $key => $value) {
if (!in_array($key, ['status', 'kind', 'host_id', 'q', 'page', 'per_page'], true)
|| (!is_string($value) && !is_int($value))) {
throw new CloudError('筛选参数无效', 400);
}
}
$status = (string) ($input['status'] ?? 'attention');
$kind = (string) ($input['kind'] ?? '');
if (!in_array($status, array_merge(['all', 'attention'], array_keys(self::STATUSES)), true)
|| ($kind !== '' && !array_key_exists($kind, CloudOperationCatalog::kinds()))) {
throw new CloudError('任务状态或类型无效', 400);
}
$q = trim((string) ($input['q'] ?? ''));
if (mb_strlen($q, 'UTF-8') > 80 || preg_match('/[\x00-\x1f\x7f]/u', $q) !== 0) {
throw new CloudError('搜索关键词最多 80 个字符,且不能包含控制字符', 400);
}
$host = (string) ($input['host_id'] ?? '');
$page = self::identifier((string) ($input['page'] ?? '1'));
$perPage = self::identifier((string) ($input['per_page'] ?? '20'));
if (!in_array($perPage, [20, 50, 100], true)) {
throw new CloudError('每页条数只能为 20、50 或 100', 400);
}
return ['status' => $status, 'kind' => $kind,
'host_id' => $host === '' ? null : (string) self::identifier($host),
'q' => $q, 'page' => $page, 'per_page' => $perPage];
}
private static function authorize(array $actor): void
{
if (!($actor['is_admin'] ?? false)) {
throw new CloudError('需要管理员权限', 403);
}
}
private static function canManage(array $actor): bool
{
return ($actor['is_admin'] ?? false) === true
&& ($actor['is_super_admin'] ?? false) === true
&& ($actor['can_manage'] ?? false) === true;
}
private static function identifier(string $value): int
{
if (preg_match('/^[1-9][0-9]{0,17}$/D', $value) !== 1) {
throw new CloudError('编号或页码无效', 400);
}
return (int) $value;
}
private static function query()
{
return CloudJob::query()
->leftJoin('cloud_hosts as operation_host', 'operation_host.id', '=', 'cloud_jobs.host_id')
->leftJoin('user as operation_actor', 'operation_actor.id', '=', 'cloud_jobs.user_id')
->leftJoin('cloud_instances as operation_instance', static function ($join): void {
$join->on('operation_instance.id', '=', 'cloud_jobs.instance_id')
->on('operation_instance.host_id', '=', 'cloud_jobs.host_id');
})
->leftJoin('cloud_orders as operation_order', static function ($join): void {
$join->on('operation_order.id', '=', 'cloud_jobs.order_id')
->on('operation_order.host_id', '=', 'cloud_jobs.host_id')
->whereRaw('(cloud_jobs.instance_id IS NULL OR '
. '(operation_instance.id IS NOT NULL '
. 'AND operation_order.user_id = operation_instance.user_id '
. 'AND (operation_order.instance_id IS NULL '
. 'OR operation_order.instance_id = operation_instance.id)))');
})
->leftJoin('cloud_instances as fallback_instance', static function ($join): void {
$join->on('fallback_instance.id', '=', 'operation_order.instance_id')
->on('fallback_instance.user_id', '=', 'operation_order.user_id')
->on('fallback_instance.host_id', '=', 'cloud_jobs.host_id')
->whereNull('cloud_jobs.instance_id');
})
->leftJoin('user as operation_owner', static function ($join): void {
$join->whereRaw('operation_owner.id = COALESCE(operation_instance.user_id, '
. 'fallback_instance.user_id, operation_order.user_id)');
});
}
private static function select($query)
{
$columns = [];
foreach (['id', 'operation_id', 'user_id', 'host_id', 'kind', 'status', 'attempts',
'error_code', 'dispatched_at', 'lease_until', 'available_at', 'created_at', 'updated_at'] as $column) {
$columns[] = 'cloud_jobs.' . $column;
}
return $query->select(array_merge($columns, [
'operation_host.name as host_name', 'operation_host.region as host_region',
'operation_actor.user_name as actor_name', 'operation_actor.email as actor_email',
'operation_owner.user_name as owner_name', 'operation_owner.email as owner_email',
'operation_order.id as linked_order_id',
]))->selectRaw('COALESCE(operation_instance.user_id, fallback_instance.user_id, '
. 'operation_order.user_id) AS owner_id, '
. 'COALESCE(operation_instance.id, fallback_instance.id) AS linked_instance_id, '
. 'COALESCE(operation_instance.name, fallback_instance.name) AS instance_name, '
. 'COALESCE(operation_instance.uuid, fallback_instance.uuid) AS instance_uuid, '
. 'COALESCE(operation_instance.status, fallback_instance.status) AS instance_status, '
. "CASE WHEN JSON_VALID(cloud_jobs.result_json) THEN "
. "JSON_EXTRACT(cloud_jobs.result_json, '$.phase') ELSE NULL END AS phase_json");
}
private static function queuedSql(): string
{
return "(cloud_jobs.status = 'queued' AND cloud_jobs.available_at <= ? AND cloud_jobs.lease_until <= ?)";
}
private static function expiredLeaseSql(): string
{
return "(cloud_jobs.status = 'running' AND cloud_jobs.lease_until > 0 AND cloud_jobs.lease_until <= ?)";
}
private static function attentionSql(): string
{
return "(cloud_jobs.status IN ('failed', 'uncertain') OR "
. self::queuedSql() . ' OR ' . self::expiredLeaseSql() . ')';
}
private static function applyFilters($query, array $filters, int $now): void
{
if ($filters['status'] === 'attention') {
$query->whereRaw(self::attentionSql(), [$now - self::QUEUED_DELAY_SECONDS, $now, $now]);
} elseif ($filters['status'] !== 'all') {
$query->where('cloud_jobs.status', $filters['status']);
}
if ($filters['kind'] !== '') {
$query->where('cloud_jobs.kind', $filters['kind']);
}
if ($filters['host_id'] !== null) {
$query->where('cloud_jobs.host_id', $filters['host_id']);
}
if ($filters['q'] !== '') {
$like = '%' . str_replace(['!', '%', '_'], ['!!', '!%', '!_'], $filters['q']) . '%';
$query->where(static function ($search) use ($filters, $like): void {
$search->whereRaw("cloud_jobs.operation_id LIKE ? ESCAPE '!'", [$like]);
foreach (['operation_instance', 'fallback_instance'] as $table) {
foreach (['uuid', 'name'] as $column) {
$search->orWhereRaw($table . '.' . $column . " LIKE ? ESCAPE '!'", [$like]);
}
}
if (preg_match('/^[1-9][0-9]{0,17}$/D', $filters['q']) === 1) {
$search->orWhere('cloud_jobs.id', (int) $filters['q']);
}
});
}
}
private static function row($job, int $now, bool $canManage = false): array
{
$status = (string) $job->status;
$lease = self::timestamp($job->lease_until);
$available = self::timestamp($job->available_at);
$reason = in_array($status, ['failed', 'uncertain'], true) ? $status : '';
if ($status === 'queued' && (int) $job->available_at <= $now - self::QUEUED_DELAY_SECONDS
&& (int) $job->lease_until <= $now) {
$reason = 'queued_delayed';
} elseif ($status === 'running' && $lease !== null && $lease <= $now) {
$reason = 'lease_expired';
}
$phase = $job->phase_json;
// MySQL returns a JSON string; SQLite JSON_EXTRACT returns plain text.
if (is_string($phase) && str_starts_with($phase, '"')) {
$phase = json_decode($phase, true);
}
if (is_string($phase) && (str_starts_with($phase, '{') || str_starts_with($phase, '[')
|| in_array($phase, ['null', 'true', 'false'], true))) {
$phase = null;
}
$uuid = (string) ($job->instance_uuid ?? '');
$validUuid = preg_match('/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/iD', $uuid) === 1;
$attentionLabels = ['failed' => '任务已失败', 'uncertain' => '等待核对结果',
'queued_delayed' => '可调度后已等待超过 5 分钟', 'lease_expired' => '执行租约已到期,需核对'];
$action = CloudJobService::action($job);
$labels = CloudOperationCatalog::labels((string) $job->kind, $action);
[$canCancel, $cancelReason] = self::cancellation($job, $now, $canManage);
return [
'id' => (string) $job->id,
'operation_id' => preg_match('/^[a-f0-9]{32}$/iD', (string) $job->operation_id) === 1
? (string) $job->operation_id : null,
'kind' => (string) $job->kind, 'action' => $action,
'kind_label' => $labels['kind_label'], 'action_label' => $labels['action_label'],
'status' => $status,
'can_cancel' => $canCancel,
'cancel_reason' => $cancelReason,
'attempts' => max(0, (int) $job->attempts),
'error_code' => CloudOperationErrorCatalog::publicCode($job->error_code),
'error' => CloudOperationErrorCatalog::adminSummary(
(string) $job->kind, $status, $job->error_code,
$job->linked_order_id !== null, $action
),
'phase' => self::jobText($phase),
'host_id' => (string) $job->host_id, 'host_name' => (string) ($job->host_name ?? ''),
'region' => (string) ($job->host_region ?? ''),
'actor_id' => (string) $job->user_id, 'actor_name' => AdminDataMasker::account($job->actor_name ?? ''),
'actor_email' => AdminDataMasker::email($job->actor_email ?? ''),
'owner_id' => $job->owner_id === null ? null : (string) $job->owner_id,
'owner_name' => $job->owner_id === null ? null : AdminDataMasker::account($job->owner_name ?? ''),
'owner_email' => $job->owner_id === null ? null : AdminDataMasker::email($job->owner_email ?? ''),
'instance_id' => $job->linked_instance_id === null ? null : (string) $job->linked_instance_id,
'instance_name' => $job->instance_name, 'instance_uuid' => $job->instance_uuid,
'instance_status' => $job->instance_status,
'order_id' => $job->linked_order_id === null ? null : (string) $job->linked_order_id,
'created_at' => max(0, (int) $job->created_at), 'updated_at' => max(0, (int) $job->updated_at),
'dispatched_at' => self::timestamp($job->dispatched_at), 'available_at' => $available,
'lease_until' => $lease, 'attention' => $reason !== '', 'attention_reason' => $reason,
'links' => [
'instance' => $validUuid && $job->instance_status !== 'deleted'
? '/admin/cloud/instances?q=' . rawurlencode($uuid) : null,
'billing' => $job->linked_order_id === null ? null
: '/admin/billing?view=orders&q=' . $job->linked_order_id,
],
'display' => ['kind' => $labels['kind_label'], 'action' => $labels['action_label'],
'status' => self::STATUSES[$status] ?? '其他状态',
'attention' => $attentionLabels[$reason] ?? '无异常标记',
'lease' => $lease === null ? '无执行租约' : ($lease > $now ? '租约有效' : '租约已到期')],
];
}
/** @return array{0:bool,1:?string} */
private static function cancellation($job, int $now, bool $canManage): array
{
if (!$canManage) {
return [false, '只有超级管理员可以中止任务'];
}
if (!in_array((string) $job->kind, self::CANCELLABLE_KINDS, true)) {
return [false, '当前任务类型不支持中止'];
}
$status = (string) $job->status;
if ($status === 'cancelled') {
return [false, '任务已中止'];
}
if (in_array($status, ['succeeded', 'failed'], true)) {
return [false, '任务已经结束'];
}
if (!in_array($status, ['queued', 'running', 'uncertain'], true)) {
return [false, '当前任务状态不支持中止'];
}
return [true, $status === 'running'
? ((int) $job->lease_until > $now
? '任务正在执行;中止后会忽略迟到结果并停止后续重试'
: '执行租约已到期,将停止后续重试')
: '将停止后续重试,已派发操作仍需核查'];
}
private static function timestamp($value): ?int
{
return $value !== null && (int) $value > 0 ? (int) $value : null;
}
private static function jobText($value): ?string
{
// Apply the same unconditional secret/path redaction to every
// administrator role, including super administrators.
return CloudOperationErrorCatalog::adminSummary(null, null, $value);
}
}