|
|
<?php
|
|
|
|
|
|
namespace App\Services\Cloud;
|
|
|
|
|
|
use App\Models\CloudHost;
|
|
|
use App\Models\CloudJob;
|
|
|
use App\Services\AdminDataMasker;
|
|
|
|
|
|
/** Database-backed operations inventory with no mutation retry controls. */
|
|
|
final class CloudAdminOperationsService
|
|
|
{
|
|
|
private const STATUSES = [
|
|
|
'active' => '执行中',
|
|
|
'succeeded' => '已完成',
|
|
|
'failed' => '失败',
|
|
|
];
|
|
|
|
|
|
public static function page(array $actor, array $input = []): array
|
|
|
{
|
|
|
self::authorize($actor);
|
|
|
// Keep the administrator inventory convergent even if the worker
|
|
|
// daemon is absent. This is a local deadline write only: no Agent
|
|
|
// request and no mutation replay can occur here.
|
|
|
CloudWorkerService::expireDueJobs(100);
|
|
|
$filters = self::filters($input);
|
|
|
$scope = self::query();
|
|
|
self::applyScopeFilters($scope, $filters);
|
|
|
$lockedSql = self::resourceLockedSql('cloud_jobs');
|
|
|
$attentionSql = self::attentionSql('cloud_jobs');
|
|
|
$summaryRow = (clone $scope)->selectRaw(
|
|
|
"COUNT(*) AS total, "
|
|
|
. "COALESCE(SUM(CASE WHEN cloud_jobs.phase <> 'terminal' THEN 1 ELSE 0 END), 0) AS active, "
|
|
|
. "COALESCE(SUM(CASE WHEN cloud_jobs.phase = 'terminal' AND cloud_jobs.outcome = 'succeeded' THEN 1 ELSE 0 END), 0) AS succeeded, "
|
|
|
. "COALESCE(SUM(CASE WHEN cloud_jobs.phase = 'terminal' AND cloud_jobs.outcome = 'failed' THEN 1 ELSE 0 END), 0) AS failed, "
|
|
|
. 'COALESCE(SUM(CASE WHEN ' . $attentionSql
|
|
|
. ' THEN 1 ELSE 0 END), 0) AS attention, '
|
|
|
. 'COALESCE(SUM(CASE WHEN ' . $lockedSql
|
|
|
. ' THEN 1 ELSE 0 END), 0) AS locked'
|
|
|
)->first();
|
|
|
$summary = [];
|
|
|
foreach (['total', 'active', 'succeeded', 'failed', 'attention',
|
|
|
'locked'] as $key) {
|
|
|
$summary[$key] = max(0, (int) ($summaryRow?->$key ?? 0));
|
|
|
}
|
|
|
$selectedTotal = $summary[$filters['status'] === 'all'
|
|
|
? 'total' : $filters['status']];
|
|
|
$pages = max(1, (int) ceil($selectedTotal
|
|
|
/ $filters['per_page']));
|
|
|
$filters['page'] = min($filters['page'], $pages);
|
|
|
$query = clone $scope;
|
|
|
self::applyStatusFilter($query, $filters['status']);
|
|
|
$records = self::select($query)->orderByDesc('cloud_jobs.id')
|
|
|
->offset(($filters['page'] - 1) * $filters['per_page'])
|
|
|
->limit($filters['per_page'])->get();
|
|
|
$items = [];
|
|
|
$canManage = self::canManage($actor);
|
|
|
foreach ($records as $job) {
|
|
|
$items[] = self::row($job, $canManage);
|
|
|
}
|
|
|
$hosts = [];
|
|
|
foreach (CloudHost::orderBy('name')->orderBy('id')
|
|
|
->get(['id', 'name', 'region']) as $host) {
|
|
|
$hosts[] = [
|
|
|
'id' => (string) $host->id,
|
|
|
'name' => (string) $host->name,
|
|
|
'region' => (string) $host->region,
|
|
|
];
|
|
|
}
|
|
|
$from = $selectedTotal === 0 ? 0
|
|
|
: ($filters['page'] - 1) * $filters['per_page'] + 1;
|
|
|
return [
|
|
|
'items' => $items,
|
|
|
'pagination' => [
|
|
|
'page' => $filters['page'],
|
|
|
'per_page' => $filters['per_page'],
|
|
|
'total' => $selectedTotal,
|
|
|
'total_pages' => $pages,
|
|
|
'from' => $from,
|
|
|
'to' => $from === 0 ? 0 : $from + count($items) - 1,
|
|
|
],
|
|
|
'summary' => $summary,
|
|
|
'summary_links' => self::summaryLinks($filters),
|
|
|
'filters' => $filters,
|
|
|
'hosts' => $hosts,
|
|
|
'kinds' => CloudOperationCatalog::kindOptions(),
|
|
|
'generated_at' => time(),
|
|
|
];
|
|
|
}
|
|
|
|
|
|
public static function detail(array $actor, string $id): array
|
|
|
{
|
|
|
self::authorize($actor);
|
|
|
$jobId = self::identifier($id);
|
|
|
CloudWorkerService::expireDueTaskForAdmin($jobId);
|
|
|
$job = self::select(self::query()->where(
|
|
|
'cloud_jobs.id', $jobId
|
|
|
))->first();
|
|
|
if ($job === null) {
|
|
|
throw new CloudError('任务不存在', 404);
|
|
|
}
|
|
|
$row = self::row($job, self::canManage($actor));
|
|
|
$recovering = ($row['recovery']['pending'] ?? false) === true;
|
|
|
$storageConfiguration = (string) $job->kind
|
|
|
=== 'host_storage_configure';
|
|
|
$effectNote = $storageConfiguration
|
|
|
? '存储配置失败不会保留跨任务资源栅栏;旧 operation ID 仍永久终止且绝不重放,管理员可按已有固定池的同一路径用新 operation ID 显式提交独立任务。'
|
|
|
: ($recovering
|
|
|
? '恢复异常尚未安全收敛时,资源安全栅栏保持生效;系统校验原任务证据并安全收敛面板账本后会自动解除栅栏。'
|
|
|
: '副作用可能已发生时,资源安全栅栏保持生效;只有超级管理员完成新鲜只读检查并记录审计结论后,才会显式解除面板与 Agent 栅栏。');
|
|
|
$ledgerNote = $storageConfiguration
|
|
|
? '新的存储配置任务只会核验或补全路径一致的固定池;不会更换已有池路径,也不会迁移或删除数据。'
|
|
|
: ($recovering
|
|
|
? (string) $row['recovery']['message']
|
|
|
: '其他类型若还没有专用业务账本终结器,系统会拒绝假解锁;任务已终态失败,资源栅栏保持安全锁定。');
|
|
|
return [
|
|
|
'job' => $row,
|
|
|
'generated_at' => time(),
|
|
|
'notes' => [
|
|
|
'任务状态来自数据库,不会因打开页面而联系 Agent。',
|
|
|
'任务原操作最多派发一次;恢复流程只读取原回执和宿主机证据,绝不会重放原操作。',
|
|
|
'宿主机只读检查不改变任务终态,也不执行清理或续作。',
|
|
|
$effectNote,
|
|
|
'开通任务只有在宿主机明确确认虚拟机和管理资源均不存在时,才能按未应用原子退款并释放库存、公网 IP 与资源预留;只读检查不能把开通任务确认成已应用。',
|
|
|
'支付创建结果不明时,签名回调可自动确认已应用;人工结论只收敛本地支付账本,绝不重新向渠道下单。',
|
|
|
$ledgerNote,
|
|
|
'正常关机明确返回 force_required 时,强制删除会创建具有新操作标识的独立任务。',
|
|
|
],
|
|
|
];
|
|
|
}
|
|
|
|
|
|
/** Resolve only the resource-effect lock after a fresh read-only inspection. */
|
|
|
public static function resolveEffect(
|
|
|
array $actor,
|
|
|
string $id,
|
|
|
array $input,
|
|
|
array $inspection,
|
|
|
array $fenceResolution
|
|
|
): array {
|
|
|
self::authorize($actor);
|
|
|
CloudAccess::manage($actor);
|
|
|
if (!(bool) ($actor['is_super_admin'] ?? false)) {
|
|
|
throw new CloudError('只有超级管理员可以确认任务影响', 403);
|
|
|
}
|
|
|
$resolution = $input['resolution'] ?? null;
|
|
|
$reason = is_string($input['reason'] ?? null)
|
|
|
? trim($input['reason']) : '';
|
|
|
$checkedAt = $inspection['checked_at'] ?? null;
|
|
|
$operationId = $inspection['operation_id'] ?? null;
|
|
|
$fenceCheckedAt = $fenceResolution['inspection_checked_at'] ?? null;
|
|
|
$fenceResolvedAt = $fenceResolution['resolved_at'] ?? null;
|
|
|
$fenceResolvedBy = $fenceResolution['resolved_by'] ?? null;
|
|
|
$now = time();
|
|
|
if (!in_array($resolution, ['applied', 'not_applied'], true)
|
|
|
|| mb_strlen($reason) < 20 || mb_strlen($reason) > 500
|
|
|
|| !is_int($checkedAt) || $checkedAt < $now - 120
|
|
|
|| $checkedAt > $now + 5
|
|
|
|| !is_string($operationId)
|
|
|
|| preg_match('/^[a-f0-9]{32}$/D', $operationId) !== 1
|
|
|
|| ($fenceResolution['operation_id'] ?? null) !== $operationId
|
|
|
|| ($fenceResolution['resolution'] ?? null) !== $resolution
|
|
|
|| !is_string($fenceResolution['resolution_id'] ?? null)
|
|
|
|| preg_match('/^[a-f0-9]{32}$/D',
|
|
|
$fenceResolution['resolution_id']) !== 1
|
|
|
|| !is_int($fenceCheckedAt) || $fenceCheckedAt < 1
|
|
|
|| $fenceCheckedAt > $now + 5
|
|
|
|| !is_int($fenceResolvedAt)
|
|
|
|| $fenceResolvedAt < $fenceCheckedAt
|
|
|
|| $fenceResolvedAt > $now + 5
|
|
|
|| ((!is_int($fenceResolvedBy)
|
|
|
|| $fenceResolvedBy < 1)
|
|
|
&& (!is_string($fenceResolvedBy)
|
|
|
|| preg_match('/^[1-9][0-9]{0,18}$/D',
|
|
|
$fenceResolvedBy) !== 1))
|
|
|
|| !is_string($fenceResolution['reason_sha256'] ?? null)
|
|
|
|| preg_match('/^[a-f0-9]{64}$/D',
|
|
|
$fenceResolution['reason_sha256']) !== 1
|
|
|
|| !hash_equals(hash('sha256', $reason),
|
|
|
$fenceResolution['reason_sha256'])) {
|
|
|
throw new CloudError('影响结论、审计说明或只读检查无效', 400);
|
|
|
}
|
|
|
$jobId = self::identifier($id);
|
|
|
$target = CloudJob::where('id', $jobId)
|
|
|
->where('operation_id', $operationId)->first();
|
|
|
if ($target === null) {
|
|
|
throw new CloudError('任务不处于可确认的影响未知状态', 409);
|
|
|
}
|
|
|
if ((string) $target->kind !== 'create') {
|
|
|
throw new CloudError(
|
|
|
(string) $target->kind === 'payment_create'
|
|
|
? '支付任务必须通过本地支付账本核验流程确认'
|
|
|
: '该任务类型尚无可原子收敛业务账本的解除流程',
|
|
|
409
|
|
|
);
|
|
|
}
|
|
|
CloudCreateFailureFinalizer::assertResolutionSupported(
|
|
|
$target,
|
|
|
(string) $resolution,
|
|
|
$inspection
|
|
|
);
|
|
|
CloudCreateFailureFinalizer::resolveNotApplied(
|
|
|
$jobId,
|
|
|
(string) $operationId,
|
|
|
(int) ($actor['user_id'] ?? 0),
|
|
|
$reason,
|
|
|
$inspection,
|
|
|
$now
|
|
|
);
|
|
|
error_log(sprintf(
|
|
|
'cloud admin resolved job effect job_id=%d resolution=%s actor_id=%d reason_sha256=%s inspection_checked_at=%d fence_resolution_id=%s',
|
|
|
$jobId,
|
|
|
$resolution,
|
|
|
(int) ($actor['user_id'] ?? 0),
|
|
|
hash('sha256', $reason),
|
|
|
$checkedAt,
|
|
|
(string) ($fenceResolution['resolution_id'] ?? '')
|
|
|
));
|
|
|
return self::detail($actor, (string) $jobId)
|
|
|
+ ['inspection' => $inspection];
|
|
|
}
|
|
|
|
|
|
public static function filters(array $input): array
|
|
|
{
|
|
|
foreach ($input as $key => $value) {
|
|
|
if (!in_array($key,
|
|
|
['status', 'kind', 'host_id', 'q', 'page', 'per_page'],
|
|
|
true)
|
|
|
|| (!is_string($value) && !is_int($value))) {
|
|
|
throw new CloudError('筛选参数无效', 400);
|
|
|
}
|
|
|
}
|
|
|
$status = (string) ($input['status'] ?? 'all');
|
|
|
$kind = (string) ($input['kind'] ?? '');
|
|
|
if (!in_array($status,
|
|
|
array_merge(
|
|
|
['all', 'attention', 'locked'],
|
|
|
array_keys(self::STATUSES)
|
|
|
),
|
|
|
true)
|
|
|
|| ($kind !== ''
|
|
|
&& !array_key_exists($kind, CloudOperationCatalog::kinds()))) {
|
|
|
throw new CloudError('任务状态或类型无效', 400);
|
|
|
}
|
|
|
$q = trim((string) ($input['q'] ?? ''));
|
|
|
if (mb_strlen($q, 'UTF-8') > 80
|
|
|
|| preg_match('/[\x00-\x1f\x7f]/u', $q) !== 0) {
|
|
|
throw new CloudError('搜索关键词最多 80 个字符,且不能包含控制字符', 400);
|
|
|
}
|
|
|
$host = (string) ($input['host_id'] ?? '');
|
|
|
$page = self::identifier((string) ($input['page'] ?? '1'));
|
|
|
$perPage = self::identifier((string) ($input['per_page'] ?? '20'));
|
|
|
if (!in_array($perPage, [20, 50, 100], true)) {
|
|
|
throw new CloudError('每页条数只能为 20、50 或 100', 400);
|
|
|
}
|
|
|
return [
|
|
|
'status' => $status,
|
|
|
'kind' => $kind,
|
|
|
'host_id' => $host === '' ? null
|
|
|
: (string) self::identifier($host),
|
|
|
'q' => $q,
|
|
|
'page' => $page,
|
|
|
'per_page' => $perPage,
|
|
|
];
|
|
|
}
|
|
|
|
|
|
private static function authorize(array $actor): void
|
|
|
{
|
|
|
if (!($actor['is_admin'] ?? false)) {
|
|
|
throw new CloudError('需要管理员权限', 403);
|
|
|
}
|
|
|
}
|
|
|
|
|
|
private static function canManage(array $actor): bool
|
|
|
{
|
|
|
return ($actor['is_admin'] ?? false) === true
|
|
|
&& ($actor['is_super_admin'] ?? false) === true
|
|
|
&& ($actor['can_manage'] ?? false) === true;
|
|
|
}
|
|
|
|
|
|
private static function identifier(string $value): int
|
|
|
{
|
|
|
if (preg_match('/^[1-9][0-9]{0,17}$/D', $value) !== 1) {
|
|
|
throw new CloudError('编号或页码无效', 400);
|
|
|
}
|
|
|
return (int) $value;
|
|
|
}
|
|
|
|
|
|
private static function query()
|
|
|
{
|
|
|
return CloudJob::query()
|
|
|
->leftJoin('cloud_hosts as operation_host',
|
|
|
'operation_host.id', '=', 'cloud_jobs.host_id')
|
|
|
->leftJoin('user as operation_actor',
|
|
|
'operation_actor.id', '=', 'cloud_jobs.user_id')
|
|
|
->leftJoin('cloud_instances as operation_instance',
|
|
|
static function ($join): void {
|
|
|
$join->on('operation_instance.id', '=',
|
|
|
'cloud_jobs.instance_id')
|
|
|
->on('operation_instance.host_id', '=',
|
|
|
'cloud_jobs.host_id');
|
|
|
})
|
|
|
->leftJoin('cloud_orders as operation_order',
|
|
|
static function ($join): void {
|
|
|
$join->on('operation_order.id', '=', 'cloud_jobs.order_id')
|
|
|
->on('operation_order.host_id', '=',
|
|
|
'cloud_jobs.host_id')
|
|
|
->whereRaw('(cloud_jobs.instance_id IS NULL OR '
|
|
|
. '(operation_instance.id IS NOT NULL '
|
|
|
. 'AND operation_order.user_id = operation_instance.user_id '
|
|
|
. 'AND (operation_order.instance_id IS NULL '
|
|
|
. 'OR operation_order.instance_id = operation_instance.id)))');
|
|
|
})
|
|
|
->leftJoin('cloud_instances as fallback_instance',
|
|
|
static function ($join): void {
|
|
|
$join->on('fallback_instance.id', '=',
|
|
|
'operation_order.instance_id')
|
|
|
->on('fallback_instance.user_id', '=',
|
|
|
'operation_order.user_id')
|
|
|
->on('fallback_instance.host_id', '=',
|
|
|
'cloud_jobs.host_id')
|
|
|
->whereNull('cloud_jobs.instance_id');
|
|
|
})
|
|
|
->leftJoin('user as operation_owner',
|
|
|
static function ($join): void {
|
|
|
$join->whereRaw('operation_owner.id = '
|
|
|
. 'COALESCE(operation_instance.user_id, '
|
|
|
. 'fallback_instance.user_id, operation_order.user_id)');
|
|
|
});
|
|
|
}
|
|
|
|
|
|
private static function select($query)
|
|
|
{
|
|
|
$columns = [];
|
|
|
foreach ([
|
|
|
'id', 'operation_id', 'request_idempotency_key',
|
|
|
'request_fingerprint', 'user_id', 'host_id', 'kind',
|
|
|
'parent_job_id', 'supersedes_job_id',
|
|
|
'vpc_repair_root_job_id', 'execution_mode', 'invoice_id', 'phase',
|
|
|
'order_id', 'instance_id', 'snapshot_id',
|
|
|
'outcome', 'status', 'attempts', 'dispatch_attempts', 'error_code',
|
|
|
'dispatched_at', 'dispatch_deadline_at', 'dispatch_started_at',
|
|
|
'agent_accepted_at', 'receipt_deadline_at', 'completion_event_id',
|
|
|
'terminal_at', 'effect_may_have_applied', 'safe_to_resubmit',
|
|
|
'effect_resolution', 'effect_resolved_at', 'effect_resolved_by',
|
|
|
'effect_resolution_reason',
|
|
|
'finalization_incident', 'finalization_error_code', 'result_json',
|
|
|
'lease_until', 'available_at', 'created_at', 'updated_at',
|
|
|
] as $column) {
|
|
|
$columns[] = 'cloud_jobs.' . $column;
|
|
|
}
|
|
|
return $query->select(array_merge($columns, [
|
|
|
'operation_host.name as host_name',
|
|
|
'operation_host.region as host_region',
|
|
|
'operation_actor.user_name as actor_name',
|
|
|
'operation_actor.email as actor_email',
|
|
|
'operation_owner.user_name as owner_name',
|
|
|
'operation_owner.email as owner_email',
|
|
|
'operation_order.id as linked_order_id',
|
|
|
]))->selectRaw(
|
|
|
'COALESCE(operation_instance.user_id, fallback_instance.user_id, '
|
|
|
. 'operation_order.user_id) AS owner_id, '
|
|
|
. 'COALESCE(operation_instance.id, fallback_instance.id) '
|
|
|
. 'AS linked_instance_id, '
|
|
|
. 'COALESCE(operation_instance.name, fallback_instance.name) '
|
|
|
. 'AS instance_name, '
|
|
|
. 'COALESCE(operation_instance.uuid, fallback_instance.uuid) '
|
|
|
. 'AS instance_uuid, '
|
|
|
. 'COALESCE(operation_instance.status, fallback_instance.status) '
|
|
|
. 'AS instance_status, '
|
|
|
. "CASE WHEN JSON_VALID(cloud_jobs.result_json) THEN "
|
|
|
. "JSON_EXTRACT(cloud_jobs.result_json, '$.cause_code') "
|
|
|
. 'ELSE NULL END AS agent_cause_code_json'
|
|
|
);
|
|
|
}
|
|
|
|
|
|
private static function applyStatusFilter($query, string $status): void
|
|
|
{
|
|
|
if ($status === 'locked') {
|
|
|
$query->whereRaw(self::resourceLockedSql('cloud_jobs'));
|
|
|
} elseif ($status === 'attention') {
|
|
|
$query->whereRaw(self::attentionSql('cloud_jobs'));
|
|
|
} elseif ($status === 'active') {
|
|
|
$query->where('cloud_jobs.phase', '<>', 'terminal');
|
|
|
} elseif ($status !== 'all') {
|
|
|
$query->where('cloud_jobs.phase', 'terminal')
|
|
|
->where('cloud_jobs.outcome', $status);
|
|
|
}
|
|
|
}
|
|
|
|
|
|
private static function applyScopeFilters($query, array $filters): void
|
|
|
{
|
|
|
if ($filters['kind'] !== '') {
|
|
|
$query->where('cloud_jobs.kind', $filters['kind']);
|
|
|
}
|
|
|
if ($filters['host_id'] !== null) {
|
|
|
$query->where('cloud_jobs.host_id', $filters['host_id']);
|
|
|
}
|
|
|
if ($filters['q'] !== '') {
|
|
|
$like = '%' . str_replace(
|
|
|
['!', '%', '_'], ['!!', '!%', '!_'], $filters['q']
|
|
|
) . '%';
|
|
|
$query->where(static function ($search) use ($filters, $like): void {
|
|
|
$search->whereRaw(
|
|
|
"cloud_jobs.operation_id LIKE ? ESCAPE '!'", [$like]
|
|
|
);
|
|
|
foreach (['operation_instance', 'fallback_instance'] as $table) {
|
|
|
foreach (['uuid', 'name'] as $column) {
|
|
|
$search->orWhereRaw(
|
|
|
$table . '.' . $column . " LIKE ? ESCAPE '!'",
|
|
|
[$like]
|
|
|
);
|
|
|
}
|
|
|
}
|
|
|
if (preg_match('/^[1-9][0-9]{0,17}$/D',
|
|
|
$filters['q']) === 1) {
|
|
|
$search->orWhere('cloud_jobs.id', (int) $filters['q']);
|
|
|
}
|
|
|
});
|
|
|
}
|
|
|
}
|
|
|
|
|
|
private static function attentionSql(string $table): string
|
|
|
{
|
|
|
if (preg_match('/^[a-z_][a-z0-9_]*$/D', $table) !== 1) {
|
|
|
throw new \InvalidArgumentException('Invalid task table alias');
|
|
|
}
|
|
|
return '((' . $table . ".phase = 'terminal' AND "
|
|
|
. $table . ".outcome = 'failed') OR "
|
|
|
. CloudJobRecoverySql::pendingSql($table) . ')';
|
|
|
}
|
|
|
|
|
|
/** @return array<string,string> */
|
|
|
private static function summaryLinks(array $filters): array
|
|
|
{
|
|
|
$base = [];
|
|
|
foreach (['q', 'kind', 'host_id'] as $key) {
|
|
|
if ($filters[$key] !== null && $filters[$key] !== '') {
|
|
|
$base[$key] = (string) $filters[$key];
|
|
|
}
|
|
|
}
|
|
|
if ($filters['per_page'] !== 20) {
|
|
|
$base['per_page'] = (string) $filters['per_page'];
|
|
|
}
|
|
|
$links = [];
|
|
|
foreach (['all', 'active', 'attention', 'failed', 'locked'] as $status) {
|
|
|
$query = $base;
|
|
|
if ($status !== 'all') {
|
|
|
$query = ['status' => $status] + $query;
|
|
|
}
|
|
|
$encoded = http_build_query($query, '', '&', PHP_QUERY_RFC3986);
|
|
|
$links[$status] = '/admin/operations'
|
|
|
. ($encoded === '' ? '' : '?' . $encoded);
|
|
|
}
|
|
|
return $links;
|
|
|
}
|
|
|
|
|
|
private static function row($job, bool $canManage = false): array
|
|
|
{
|
|
|
$completed = (string) $job->phase === 'terminal';
|
|
|
$status = $completed && in_array((string) $job->outcome,
|
|
|
['succeeded', 'failed'], true)
|
|
|
? (string) $job->outcome : 'active';
|
|
|
$agentCauseCode = $job->agent_cause_code_json;
|
|
|
if (is_string($agentCauseCode)
|
|
|
&& str_starts_with($agentCauseCode, '"')) {
|
|
|
$agentCauseCode = json_decode($agentCauseCode, true);
|
|
|
}
|
|
|
$agentCauseCode = CloudOperationErrorCatalog::agentCauseCode(
|
|
|
$agentCauseCode
|
|
|
);
|
|
|
$uuid = (string) ($job->instance_uuid ?? '');
|
|
|
$validUuid = preg_match(
|
|
|
'/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/iD',
|
|
|
$uuid
|
|
|
) === 1;
|
|
|
$action = CloudJobService::action($job);
|
|
|
$labels = CloudOperationCatalog::labels((string) $job->kind, $action);
|
|
|
[$canInspect, $inspectReason] = self::inspectable($job, $canManage);
|
|
|
[$canResolve, $resolveReason] = self::effectResolvable(
|
|
|
$job, $canManage
|
|
|
);
|
|
|
[$canForceDelete, $forceDeleteReason] = self::forceDeleteable(
|
|
|
$job, $canManage
|
|
|
);
|
|
|
[$canRepairVpc, $repairVpcReason] = self::vpcRepairable(
|
|
|
$job, $canManage
|
|
|
);
|
|
|
[$canRetireVpcHost, $retireVpcHostReason] =
|
|
|
self::vpcHostRetireable($job, $canManage);
|
|
|
$recovery = CloudJobService::recoveryPresentation($job, true);
|
|
|
$effectScope = $recovery === null
|
|
|
? CloudJobService::effectScope($job) : null;
|
|
|
$localFinalizerRecovery = self::isLocalFinalizerRecovery($job);
|
|
|
$repairIncident = CloudDispatchLedger::hasRepairIncident($job);
|
|
|
$finalizationErrorCode = (bool) $job->finalization_incident
|
|
|
? (string) $job->finalization_error_code : null;
|
|
|
if (is_string($finalizationErrorCode)
|
|
|
&& str_starts_with($finalizationErrorCode, 'recovery_')) {
|
|
|
$finalizationErrorCode = $recovery['code']
|
|
|
?? 'recovery_pending';
|
|
|
} elseif (is_string($finalizationErrorCode)) {
|
|
|
$finalizationErrorCode =
|
|
|
CloudOperationErrorCatalog::agentCauseCode(
|
|
|
$finalizationErrorCode
|
|
|
) ?? 'finalization_failed';
|
|
|
}
|
|
|
return [
|
|
|
'id' => (string) $job->id,
|
|
|
'operation_id' => preg_match('/^[a-f0-9]{32}$/D',
|
|
|
(string) $job->operation_id) === 1
|
|
|
? (string) $job->operation_id : null,
|
|
|
'kind' => (string) $job->kind,
|
|
|
'execution_mode' => (string) $job->execution_mode,
|
|
|
'invoice_id' => $job->invoice_id === null
|
|
|
? null : (string) $job->invoice_id,
|
|
|
'action' => $action,
|
|
|
'kind_label' => $labels['kind_label'],
|
|
|
'action_label' => $labels['action_label'],
|
|
|
'completed' => $completed,
|
|
|
'status' => $status,
|
|
|
'can_inspect' => $canInspect,
|
|
|
'inspect_reason' => $inspectReason,
|
|
|
'can_resolve_effect' => $canResolve,
|
|
|
'resolve_effect_reason' => $resolveReason,
|
|
|
'can_force_delete' => $canForceDelete,
|
|
|
'force_delete_reason' => $forceDeleteReason,
|
|
|
'can_repair_vpc' => $canRepairVpc,
|
|
|
'repair_vpc_reason' => $repairVpcReason,
|
|
|
'can_retire_vpc_host' => $canRetireVpcHost,
|
|
|
'retire_vpc_host_reason' => $retireVpcHostReason,
|
|
|
'supersedes_job_id' => $job->supersedes_job_id === null
|
|
|
? null : (string) $job->supersedes_job_id,
|
|
|
'dispatch_attempts' => $job->dispatch_attempts === null
|
|
|
? null : CloudDispatchLedger::attempts($job),
|
|
|
'dispatch_state' => CloudDispatchLedger::isInvalid($job)
|
|
|
? 'invalid'
|
|
|
: (CloudDispatchLedger::isDispatched($job)
|
|
|
? 'dispatched' : 'not_dispatched'),
|
|
|
'effect_may_have_applied' => $job->effect_may_have_applied === null
|
|
|
? null : (bool) $job->effect_may_have_applied,
|
|
|
'safe_to_resubmit' => $job->safe_to_resubmit === null
|
|
|
? null : (bool) $job->safe_to_resubmit,
|
|
|
'effect_resolution' => (string) $job->effect_resolution,
|
|
|
'effect_scope' => $effectScope,
|
|
|
'effect_resolved_at' => self::timestamp($job->effect_resolved_at),
|
|
|
'effect_resolved_by' => $job->effect_resolved_by === null
|
|
|
? null : (string) $job->effect_resolved_by,
|
|
|
'finalization_incident' => (bool) $job->finalization_incident,
|
|
|
'finalization_error_code' => $finalizationErrorCode,
|
|
|
'recovery' => $recovery,
|
|
|
'error_code' => CloudOperationErrorCatalog::publicCode(
|
|
|
$job->error_code
|
|
|
),
|
|
|
'agent_cause_code' => $agentCauseCode,
|
|
|
'agent_cause_label' =>
|
|
|
CloudOperationErrorCatalog::agentCauseLabel($agentCauseCode),
|
|
|
'error' => $completed && $status === 'failed'
|
|
|
? CloudOperationErrorCatalog::adminSummary(
|
|
|
(string) $job->kind,
|
|
|
$status,
|
|
|
$job->error_code,
|
|
|
$job->linked_order_id !== null,
|
|
|
$action
|
|
|
) : null,
|
|
|
'host_id' => (string) $job->host_id,
|
|
|
'host_name' => (string) ($job->host_name ?? ''),
|
|
|
'region' => (string) ($job->host_region ?? ''),
|
|
|
'actor_id' => (string) $job->user_id,
|
|
|
'actor_name' => AdminDataMasker::account($job->actor_name ?? ''),
|
|
|
'actor_email' => AdminDataMasker::email($job->actor_email ?? ''),
|
|
|
'owner_id' => $job->owner_id === null
|
|
|
? null : (string) $job->owner_id,
|
|
|
'owner_name' => $job->owner_id === null
|
|
|
? null : AdminDataMasker::account($job->owner_name ?? ''),
|
|
|
'owner_email' => $job->owner_id === null
|
|
|
? null : AdminDataMasker::email($job->owner_email ?? ''),
|
|
|
'instance_id' => $job->linked_instance_id === null
|
|
|
? null : (string) $job->linked_instance_id,
|
|
|
'instance_name' => $job->instance_name,
|
|
|
'instance_uuid' => $job->instance_uuid,
|
|
|
'instance_status' => $job->instance_status,
|
|
|
'order_id' => $job->linked_order_id === null
|
|
|
? null : (string) $job->linked_order_id,
|
|
|
'created_at' => max(0, (int) $job->created_at),
|
|
|
'updated_at' => max(0, (int) $job->updated_at),
|
|
|
'dispatched_at' => self::timestamp($job->dispatched_at),
|
|
|
'dispatch_deadline_at' =>
|
|
|
self::timestamp($job->dispatch_deadline_at),
|
|
|
'receipt_deadline_at' =>
|
|
|
self::timestamp($job->receipt_deadline_at),
|
|
|
'terminal_at' => self::timestamp($job->terminal_at),
|
|
|
'links' => [
|
|
|
'instance' => $validUuid && $job->instance_status !== 'deleted'
|
|
|
? '/admin/cloud/instances?q=' . rawurlencode($uuid) : null,
|
|
|
'billing' => $job->linked_order_id === null ? null
|
|
|
: '/admin/billing?view=orders&q=' . $job->linked_order_id,
|
|
|
],
|
|
|
'display' => [
|
|
|
'kind' => $labels['kind_label'],
|
|
|
'action' => $labels['action_label'],
|
|
|
'status' => $recovery !== null
|
|
|
? '恢复处理中' : self::STATUSES[$status],
|
|
|
'effect' => self::effectDisplay(
|
|
|
$job,
|
|
|
$status,
|
|
|
$recovery,
|
|
|
$localFinalizerRecovery,
|
|
|
$repairIncident,
|
|
|
$effectScope
|
|
|
),
|
|
|
],
|
|
|
];
|
|
|
}
|
|
|
|
|
|
/** @return array{0:bool,1:string} */
|
|
|
private static function inspectable($job, bool $canManage): array
|
|
|
{
|
|
|
if (!$canManage) {
|
|
|
return [false, '只有超级管理员可以读取宿主机实时状态'];
|
|
|
}
|
|
|
if ((string) $job->kind === 'payment_create') {
|
|
|
return [false, '支付任务不使用宿主机 Agent 检查'];
|
|
|
}
|
|
|
if (CloudJobService::isAggregateKind((string) ($job->kind ?? ''))
|
|
|
|| (string) ($job->execution_mode ?? '') === 'aggregate') {
|
|
|
return [false, '汇总任务只核对 PHP 账本,不联系宿主机 Agent'];
|
|
|
}
|
|
|
if (CloudOperationRecoveryPolicy::supportsLedgerFinalizer(
|
|
|
(string) ($job->kind ?? '')
|
|
|
) && (string) ($job->execution_mode ?? '') !== 'agent') {
|
|
|
return [false,
|
|
|
'任务执行模式账本异常;仅由 PHP 本地恢复核对,不能联系 Agent'];
|
|
|
}
|
|
|
if (CloudJobService::hasTerminalRecoveryStateConflict($job)) {
|
|
|
return [false,
|
|
|
'任务终态或派发信封矛盾;仅由 PHP 本地恢复核对,不能联系 Agent'];
|
|
|
}
|
|
|
if ((string) ($job->phase ?? '') === 'terminal'
|
|
|
&& (bool) ($job->finalization_incident ?? false)
|
|
|
&& in_array(
|
|
|
(string) ($job->effect_resolution ?? ''),
|
|
|
['applied', 'not_applied'],
|
|
|
true
|
|
|
)) {
|
|
|
return [false,
|
|
|
'任务终态后的恢复账本仍在收敛;不会再次联系 Agent'];
|
|
|
}
|
|
|
if (CloudDispatchLedger::isInvalid($job)) {
|
|
|
return [false,
|
|
|
'任务单次派发计数账本异常;修复为合法值后才能读取宿主机状态'];
|
|
|
}
|
|
|
if (CloudDispatchLedger::hasRepairIncident($job)) {
|
|
|
return [false,
|
|
|
'任务恢复账本异常尚未收敛;保持原终态和资源锁,完成后才能读取宿主机状态'];
|
|
|
}
|
|
|
if (!CloudDispatchLedger::isDispatched($job)) {
|
|
|
return [false, '任务没有跨越 Agent 派发边界'];
|
|
|
}
|
|
|
if ((int) $job->host_id < 1
|
|
|
|| preg_match('/^[a-f0-9]{32}$/D',
|
|
|
(string) $job->operation_id) !== 1) {
|
|
|
return [false, '任务缺少可检查的 Agent 操作标识'];
|
|
|
}
|
|
|
if ($job->dispatched_at === null || (int) $job->dispatched_at < 1) {
|
|
|
return [false, '任务没有跨越 Agent 派发边界'];
|
|
|
}
|
|
|
return [true, '只读检查不会改变任务或宿主机'];
|
|
|
}
|
|
|
|
|
|
/** @return array{0:bool,1:string} */
|
|
|
private static function effectResolvable($job, bool $canManage): array
|
|
|
{
|
|
|
if (!$canManage) {
|
|
|
return [false, '只有超级管理员可以记录影响结论'];
|
|
|
}
|
|
|
if (CloudDispatchLedger::isInvalid($job)
|
|
|
|| CloudDispatchLedger::hasRepairIncident($job)) {
|
|
|
return [false,
|
|
|
'任务恢复账本异常尚未收敛;资源保持锁定,不能人工覆盖原任务结论'];
|
|
|
}
|
|
|
if ((string) $job->phase !== 'terminal'
|
|
|
|| (string) $job->outcome !== 'failed'
|
|
|
|| (string) $job->effect_resolution !== 'unknown'
|
|
|
|| !(bool) $job->effect_may_have_applied) {
|
|
|
return [false, '任务不处于副作用未知的终态失败'];
|
|
|
}
|
|
|
if ((string) $job->execution_mode === 'agent'
|
|
|
&& !CloudDispatchLedger::isDispatched($job)) {
|
|
|
return [false, '任务没有合法的 Agent 派发记录'];
|
|
|
}
|
|
|
if ((string) $job->kind === 'create') {
|
|
|
return [true,
|
|
|
'仅可在宿主机确认虚拟机及管理资源均不存在后选择未应用并退款;不能用只读检查确认已应用'];
|
|
|
}
|
|
|
if ((string) $job->kind === 'payment_create') {
|
|
|
return [true,
|
|
|
'需要先在外部支付平台核实原订单;本操作不会重新下单'];
|
|
|
}
|
|
|
if ((string) $job->kind === 'host_storage_configure') {
|
|
|
return [false,
|
|
|
'存储配置终态失败不保留跨任务资源栅栏;请按已有固定池的同一路径,或只修正尚未定义的角色后提交新任务'];
|
|
|
}
|
|
|
return [false,
|
|
|
'该任务类型尚无可原子收敛业务账本的解除流程;任务已失败终止,资源保持安全锁定'];
|
|
|
}
|
|
|
|
|
|
/** @return array{0:bool,1:string} */
|
|
|
private static function forceDeleteable($job, bool $canManage): array
|
|
|
{
|
|
|
if (!$canManage) {
|
|
|
return [false, '只有超级管理员可以确认强制删除'];
|
|
|
}
|
|
|
if (!in_array((string) $job->kind,
|
|
|
['delete', 'expiry_delete', 'service_suspend'], true)) {
|
|
|
return [false, '当前任务不是可升级的删除任务'];
|
|
|
}
|
|
|
if (CloudDispatchLedger::isInvalid($job)
|
|
|
|| (bool) ($job->finalization_incident ?? false)
|
|
|
|| CloudJobService::hasTerminalRecoveryStateConflict($job)) {
|
|
|
return [false,
|
|
|
'原删除任务的恢复账本异常尚未收敛;不能创建强制删除任务'];
|
|
|
}
|
|
|
if ((string) $job->phase !== 'terminal'
|
|
|
|| (string) $job->outcome !== 'failed'
|
|
|
|| (string) $job->error_code !== 'force_required') {
|
|
|
return [false, '原删除任务没有明确返回 force_required'];
|
|
|
}
|
|
|
if (!CloudDispatchLedger::isDispatched($job)) {
|
|
|
return [false, '原删除任务没有合法的 Agent 派发记录'];
|
|
|
}
|
|
|
if ($job->dispatched_at === null || (int) $job->dispatched_at < 1) {
|
|
|
return [false, '原删除任务没有跨越 Agent 派发边界'];
|
|
|
}
|
|
|
return [true, '将创建新的任务并再次确认数据丢失风险'];
|
|
|
}
|
|
|
|
|
|
/** @return array{0:bool,1:string} */
|
|
|
private static function vpcRepairable($job, bool $canManage): array
|
|
|
{
|
|
|
if (!$canManage) {
|
|
|
return [false, '只有超级管理员可以创建 VPC 修复任务'];
|
|
|
}
|
|
|
if (!in_array((string) $job->kind, [
|
|
|
'vpc_host_apply', 'vpc_host_remove',
|
|
|
'vpc_port_attach', 'vpc_port_detach',
|
|
|
], true)) {
|
|
|
return [false, '当前任务不是 VPC 操作'];
|
|
|
}
|
|
|
if (CloudDispatchLedger::isInvalid($job)
|
|
|
|| CloudDispatchLedger::hasRepairIncident($job)
|
|
|
|| (bool) ($job->finalization_incident ?? false)
|
|
|
|| (string) $job->execution_mode !== 'agent'
|
|
|
|| !CloudDispatchLedger::isDispatched($job)
|
|
|
|| (string) $job->phase !== 'terminal'
|
|
|
|| (string) $job->outcome !== 'failed'
|
|
|
|| (string) $job->status !== 'failed'
|
|
|
|| !(bool) $job->effect_may_have_applied
|
|
|
|| (bool) $job->safe_to_resubmit
|
|
|
|| (string) $job->effect_resolution !== 'unknown'
|
|
|
|| !CloudDispatchLedger::isDispatched($job)
|
|
|
|| (int) ($job->dispatched_at ?? 0) < 1
|
|
|
|| CloudJobService::hasTerminalRecoveryStateConflict($job)) {
|
|
|
return [false, 'VPC 任务不处于可修复的影响未知状态'];
|
|
|
}
|
|
|
return [true,
|
|
|
'服务端会先读取宿主机新鲜证据,再创建具有新操作标识的精确修复任务'];
|
|
|
}
|
|
|
|
|
|
/** @return array{0:bool,1:string} */
|
|
|
private static function vpcHostRetireable($job, bool $canManage): array
|
|
|
{
|
|
|
if (!$canManage) {
|
|
|
return [false, '只有超级管理员可以确认宿主机永久退役'];
|
|
|
}
|
|
|
if ((string) $job->kind !== 'vpc_host_remove') {
|
|
|
return [false, '当前任务不是 VPC 宿主机清理'];
|
|
|
}
|
|
|
if (CloudDispatchLedger::isInvalid($job)
|
|
|
|| CloudDispatchLedger::hasRepairIncident($job)
|
|
|
|| (bool) ($job->finalization_incident ?? false)
|
|
|
|| (string) $job->execution_mode !== 'agent'
|
|
|
|| (string) $job->phase !== 'terminal'
|
|
|
|| (string) $job->outcome !== 'failed'
|
|
|
|| (string) $job->status !== 'failed'
|
|
|
|| !(bool) $job->effect_may_have_applied
|
|
|
|| (bool) $job->safe_to_resubmit
|
|
|
|| (string) $job->effect_resolution !== 'unknown'
|
|
|
|| !CloudDispatchLedger::isDispatched($job)
|
|
|
|| (int) ($job->dispatched_at ?? 0) < 1
|
|
|
|| CloudJobService::hasTerminalRecoveryStateConflict($job)) {
|
|
|
return [false, 'VPC 宿主机清理不处于可退役状态'];
|
|
|
}
|
|
|
return [true,
|
|
|
'仅当宿主机、Agent 和调度均已停用,且实例、资源、迁移和 VPC 网卡全部清空时才会解除该清理栅栏'];
|
|
|
}
|
|
|
|
|
|
private static function timestamp($value): ?int
|
|
|
{
|
|
|
return $value !== null && (int) $value > 0 ? (int) $value : null;
|
|
|
}
|
|
|
|
|
|
/** Shared source of truth for every resource-fence counter and filter. */
|
|
|
public static function resourceLockedSql(string $alias): string
|
|
|
{
|
|
|
return CloudJobRecoverySql::resourceLockedSql($alias);
|
|
|
}
|
|
|
|
|
|
private static function isLocalFinalizerRecovery(object $job): bool
|
|
|
{
|
|
|
return (string) ($job->execution_mode ?? '') === 'agent'
|
|
|
&& (string) ($job->phase ?? '') === 'terminal'
|
|
|
&& (string) ($job->outcome ?? '') === 'failed'
|
|
|
&& (string) ($job->status ?? '') === 'failed'
|
|
|
&& CloudDispatchLedger::isNeverDispatched($job)
|
|
|
&& (bool) ($job->finalization_incident ?? false)
|
|
|
&& !CloudDispatchLedger::hasRepairIncident($job)
|
|
|
&& CloudOperationRecoveryPolicy::supportsLedgerFinalizer(
|
|
|
(string) ($job->kind ?? '')
|
|
|
);
|
|
|
}
|
|
|
|
|
|
private static function effectDisplay(
|
|
|
object $job,
|
|
|
string $status,
|
|
|
?array $recovery,
|
|
|
bool $localFinalizerRecovery,
|
|
|
bool $repairIncident,
|
|
|
?string $effectScope
|
|
|
): string {
|
|
|
if (CloudDispatchLedger::isInvalid($job)) {
|
|
|
return '单次派发计数账本异常;任务终态保持不变,资源安全锁定,等待修复并校验';
|
|
|
}
|
|
|
if ($repairIncident) {
|
|
|
return '原任务终态保持不变;恢复账本尚未安全收敛,资源保持锁定';
|
|
|
}
|
|
|
if ($localFinalizerRecovery) {
|
|
|
return '原操作确认未派发;PHP 业务账本收尾异常,资源保持锁定并在本地持续恢复,不会联系 Agent';
|
|
|
}
|
|
|
if ($recovery !== null
|
|
|
&& (string) ($job->effect_resolution ?? '') !== 'unknown') {
|
|
|
return '任务终态或派发账本证据矛盾;资源保持安全锁定,仅在 PHP 本地持续核对,不会联系 Agent';
|
|
|
}
|
|
|
if ($status === 'failed'
|
|
|
&& (bool) $job->effect_may_have_applied
|
|
|
&& in_array((string) $job->effect_resolution,
|
|
|
['applied', 'not_applied'], true)) {
|
|
|
if ($effectScope === 'partial') {
|
|
|
return '系统已确认部分宿主机已应用、部分未应用;任务仍保留失败审计,未确认状态已收敛,资源锁已解除';
|
|
|
}
|
|
|
return (string) $job->effect_resolution === 'applied'
|
|
|
? '系统已确认操作已应用;任务仍保留失败审计,资源锁已解除'
|
|
|
: '系统已确认操作未应用;任务仍保留失败审计,资源锁已解除';
|
|
|
}
|
|
|
if ((string) $job->effect_resolution === 'unknown') {
|
|
|
if ((string) $job->kind === 'host_storage_configure') {
|
|
|
return '部分固定存储池可能已应用;旧任务已终止且不阻断新的独立任务';
|
|
|
}
|
|
|
return $recovery !== null
|
|
|
? '副作用状态未知;资源保持安全锁定,系统仍会继续自动核对'
|
|
|
: '副作用状态未知;任务已失败,资源已安全锁定';
|
|
|
}
|
|
|
return '无任务级安全栅栏';
|
|
|
}
|
|
|
}
|