'template_import_replica', 'template_metadata_sync' => 'template_metadata_replica', 'template_delete' => 'template_delete_replica', 'user_iso_import' => 'user_iso_import_replica', 'user_iso_delete' => 'user_iso_delete_replica', ]; private const RECOVERY_RETRY_SECONDS = 300; private const RECOVERY_MESSAGES = [ 'recovery_pending' => '系统正在核对原任务结果;不会重复执行原操作,仍会继续自动核对。', 'recovery_limit_reached' => '旧任务已重新进入持续自动核对;不会重复执行原操作,系统仍会继续自动核对。', 'recovery_receipt_limit_reached' => '旧任务回执已重新进入持续自动核对;不会重复执行原操作,系统仍会继续自动核对。', 'recovery_state_invalid' => '任务状态与恢复记录暂不一致;不会重复执行原操作,修复记录后系统仍会继续自动核对。', 'recovery_evidence_conflict' => 'Agent 回执与宿主机证据暂不一致;资源保持安全锁定,系统仍会继续自动核对。', 'recovery_evidence_partial' => '宿主机目前只能确认部分操作结果;资源保持安全锁定,系统仍会继续自动核对。', 'recovery_target_cleanup_pending' => '系统正在自动清理维护迁移目标机残留;原迁移不会重放,清理后会继续核对并修复迁移账本。', 'recovery_evidence_inconclusive' => '宿主机证据暂不足以确认操作结果;资源保持安全锁定,系统仍会继续自动核对。', 'recovery_evidence_invalid' => 'Agent 返回的核对或栅栏证据格式不匹配;不会采信异常证据,系统仍会继续自动核对。', 'recovery_receipt_invalid' => 'Agent 回执未通过完整性核验;不会采信异常回执,系统仍会继续自动核对。', 'recovery_completion_outbox_invalid' => '宿主机的完成回执队列状态损坏;不会采信当前结果,修复后系统仍会继续自动核对。', 'recovery_completion_receipt_missing' => '宿主机的完成回执内容缺失;不会采信当前结果,修复后系统仍会继续自动核对。', 'recovery_completion_receipt_invalid' => '宿主机的完成回执内容无效;不会采信当前结果,修复后系统仍会继续自动核对。', 'recovery_completion_operation_missing' => '宿主机的完成回执缺少对应原操作记录;不会采信当前结果,修复后系统仍会继续自动核对。', 'recovery_completion_operation_invalid' => '宿主机的完成回执对应原操作记录无效;不会采信当前结果,修复后系统仍会继续自动核对。', 'recovery_completion_identity_mismatch' => '宿主机的完成回执身份与原操作记录不匹配;不会采信当前结果,修复后系统仍会继续自动核对。', 'recovery_agent_unavailable' => '宿主机 Agent 或网络暂不可用;通信恢复后系统仍会继续自动核对。', 'recovery_authentication_failed' => '面板与宿主机 Agent 的认证暂未通过;修复认证后系统仍会继续自动核对。', 'recovery_waiting_connection' => '宿主机 Agent 或网络暂不可用;通信恢复后系统仍会继续自动核对。', 'recovery_operation_missing' => '宿主机暂未找到原任务记录;不会重复执行原操作,系统仍会继续自动核对。', 'recovery_operation_conflict' => '任务编号与宿主机记录暂不一致;不会重复执行原操作,系统仍会继续自动核对。', 'recovery_operation_identity_conflict' => 'Agent 证据中的操作、资源或请求指纹与原任务不匹配;不会重复执行或解锁,系统仍会继续自动核对。', 'recovery_payload_invalid' => '任务恢复数据暂时无法安全读取;修复记录后系统仍会继续自动核对。', 'recovery_payload_unavailable' => '任务恢复数据暂时无法安全读取;修复记录后系统仍会继续自动核对。', 'recovery_instance_state_conflict' => '实例状态与任务证据暂不一致;资源保持安全锁定,系统仍会继续自动核对。', 'recovery_aggregate_state_conflict' => '汇总任务与子任务账本关系异常;资源保持安全锁定,修复账本后系统仍会继续自动核对。', 'recovery_dispatch_ledger_conflict' => '任务单次派发计数账本异常;资源保持安全锁定,修复账本后系统仍会继续自动核对。', 'recovery_dispatch_repair_completion_receipt_missing' => '派发计数已修复,但原任务绑定的完成回执缺失;终态保持不变,资源保持锁定并持续恢复。', 'recovery_dispatch_repair_receipt_invalid' => '派发计数已修复,但原任务回执未通过完整性校验;不会采信异常回执或解锁。', 'recovery_dispatch_repair_aggregate_state_conflict' => '派发计数已修复,但汇总任务与子任务账本关系仍不一致;资源保持锁定并持续恢复。', 'recovery_dispatch_repair_payload_unavailable' => '派发计数已修复,但原任务恢复数据暂时无法安全读取;不会重复执行或解锁。', 'recovery_dispatch_repair_operation_identity_conflict' => '派发计数已修复,但回执中的操作、资源或请求指纹与原任务不匹配;不会重复执行或解锁。', 'recovery_dispatch_repair_evidence_conflict' => '派发计数已修复,但原任务回执与宿主机证据仍不一致;资源保持锁定并持续恢复。', 'recovery_dispatch_repair_instance_state_conflict' => '派发计数已修复,但实例状态与原任务证据仍不一致;资源保持锁定并持续恢复。', 'recovery_dispatch_repair_ledger_conflict' => '派发计数已修复,但面板业务账本仍未安全收敛;终态保持不变,资源保持锁定。', 'recovery_dispatch_repair_internal_error' => '派发计数修复后的安全校验遇到内部异常;不会重复执行或解锁,系统会持续恢复。', 'recovery_child_convergence' => '子任务仍在持续自动核对;汇总任务会在子任务收敛后更新,原操作不会重复执行。', 'recovery_ledger_conflict' => '任务结果已确定,但面板账本暂未安全收敛;资源保持安全锁定,系统仍会继续自动核对并恢复。', 'recovery_internal_error' => '自动核对暂时遇到内部异常;不会重复执行原操作,系统仍会继续自动核对。', ]; /** Absolute dispatch windows are selected once and are never extended. */ public static function dispatchDeadlineSeconds(string $kind): int { return $kind === 'host_storage_configure' ? self::STORAGE_DISPATCH_DEADLINE_SECONDS : CloudJob::DISPATCH_DEADLINE_SECONDS; } public static function effectiveDispatchDeadline(CloudJob $job): int { $stored = (int) ($job->dispatch_deadline_at ?? 0); if ($stored < 1 || (string) $job->kind !== 'host_storage_configure') { return $stored; } $createdAt = (int) ($job->created_at ?? 0); return $createdAt < 1 ? $stored : min( $stored, $createdAt + self::STORAGE_DISPATCH_DEADLINE_SECONDS ); } /** Absolute receipt windows are selected once and are never extended. */ public static function receiptDeadlineSeconds(string $kind): int { if (in_array($kind, self::PORTABLE_BACKUP_KINDS, true)) { return self::BACKUP_RECEIPT_DEADLINE_SECONDS; } if (in_array($kind, self::LONG_IMPORT_KINDS, true)) { return self::IMPORT_RECEIPT_DEADLINE_SECONDS; } return $kind === 'host_storage_configure' ? self::STORAGE_RECEIPT_DEADLINE_SECONDS : self::RECEIPT_DEADLINE_SECONDS; } /** Cloud-snapshot creates download a backup and need the restore window. */ public static function receiptDeadlineSecondsForJob(CloudJob $job): int { if ((string) $job->kind === 'create' && $job->backup_id !== null) { return self::BACKUP_RECEIPT_DEADLINE_SECONDS; } return self::receiptDeadlineSeconds((string) $job->kind); } /** * Apply the current bounded window to persisted work as well as new work. * This lets a deployment shorten an old storage task's 960-second value * without rewriting history or extending any previously stored deadline. */ public static function effectiveReceiptDeadline(CloudJob $job): int { $stored = (int) ($job->receipt_deadline_at ?? 0); if ($stored < 1 || (string) $job->kind !== 'host_storage_configure') { return $stored; } $dispatchedAt = (int) ($job->dispatched_at ?? 0); return $dispatchedAt < 1 ? $stored : min( $stored, $dispatchedAt + self::STORAGE_RECEIPT_DEADLINE_SECONDS ); } public static function receiptDeadlineReached(CloudJob $job, ?int $now = null): bool { $deadline = self::effectiveReceiptDeadline($job); return $deadline > 0 && $deadline <= ($now ?? time()); } public static function activeDeadlineReached(CloudJob $job, ?int $now = null): bool { $now ??= time(); if ((string) $job->phase === 'accepted') { $deadline = self::effectiveDispatchDeadline($job); return $deadline > 0 && $deadline <= $now; } return in_array((string) $job->phase, ['dispatching', 'awaiting_receipt'], true) && self::receiptDeadlineReached($job, $now); } public static function aggregateDeadlineSeconds(string $kind): int { return in_array($kind, self::LONG_IMPORT_AGGREGATE_KINDS, true) ? self::IMPORT_AGGREGATE_DEADLINE_SECONDS : self::AGGREGATE_DEADLINE_SECONDS; } /** Aggregate parents are local coordination rows and are never dispatched. */ public static function isAggregateKind(string $kind): bool { return in_array($kind, self::AGGREGATE_KINDS, true); } /** @return list */ public static function aggregateKinds(): array { return self::AGGREGATE_KINDS; } /** True only for a damaged terminal recovery or aggregate-local tuple. */ public static function hasTerminalRecoveryStateConflict(object $job): bool { if ((string) ($job->phase ?? '') !== 'terminal') { return false; } $mode = (string) ($job->execution_mode ?? ''); $kind = (string) ($job->kind ?? ''); if (CloudOperationRecoveryPolicy::supportsLedgerFinalizer($kind)) { if ($mode !== 'agent' || self::hasAgentRecoveryEnvelopeConflict($job)) { return true; } if (self::hasResolvedTerminalStateConflict($job)) { return true; } } if (self::isAggregateKind($kind)) { if (self::hasAggregateLocalStateConflict($job)) { return true; } if ((string) ($job->effect_resolution ?? '') !== 'unknown') { return false; } } elseif ((string) ($job->effect_resolution ?? '') !== 'unknown') { return false; } $supported = ($mode === 'agent' && !CloudDispatchLedger::isInvalid($job) && CloudOperationRecoveryPolicy::supportsLedgerFinalizer($kind)) || ($mode === 'aggregate' && ($job->parent_job_id ?? null) === null && self::isAggregateKind($kind)); if (!$supported) { return false; } // Generation zero proves the mutation never crossed the Agent // boundary. A terminal/unknown row may remain there only while a // durable PHP-only finalizer incident is retrying; without that // incident the otherwise-canonical dispatched safety tuple is itself // contradictory and must stay in the local invalid-state queue. if ($mode === 'agent' && (int) ($job->dispatch_attempts ?? -1) === 0 && !(bool) ($job->finalization_incident ?? false)) { return true; } return (string) ($job->outcome ?? '') !== 'failed' || (string) ($job->status ?? '') !== 'failed' || !(bool) ($job->effect_may_have_applied ?? false) || (bool) ($job->safe_to_resubmit ?? false); } /** Aggregate parents are PHP-local ledgers and must never resemble Agent work. */ public static function hasAggregateLocalStateConflict(object $job): bool { if ((string) ($job->execution_mode ?? '') !== 'aggregate' || !self::isAggregateKind((string) ($job->kind ?? '')) || ($job->parent_job_id ?? null) !== null || ($job->host_id ?? null) !== null || ($job->instance_id ?? null) !== null || ($job->order_id ?? null) !== null || ($job->invoice_id ?? null) !== null || ($job->snapshot_id ?? null) !== null || (int) ($job->user_id ?? 0) < 1 || (string) ($job->attempts ?? '') !== '0' || CloudDispatchLedger::attempts($job) !== 0 || (int) ($job->dispatch_deadline_at ?? 0) < 1 || (int) ($job->receipt_deadline_at ?? 0) < 1 || ($job->dispatch_started_at ?? null) !== null || ($job->agent_accepted_at ?? null) !== null || ($job->dispatched_at ?? null) !== null || ($job->completion_event_id ?? null) !== null || preg_match( '/^[a-f0-9]{32}$/D', (string) ($job->operation_id ?? '') ) !== 1 || preg_match( '/^[a-f0-9]{64}$/D', (string) ($job->request_fingerprint ?? '') ) !== 1 || preg_match( '/^[A-Za-z0-9_.-]{16,128}$/D', (string) ($job->request_idempotency_key ?? '') ) !== 1) { return true; } $phase = (string) ($job->phase ?? ''); if ($phase === 'awaiting_receipt') { return ($job->outcome ?? null) !== null || (string) ($job->status ?? '') !== 'running' || ($job->terminal_at ?? null) !== null || ($job->effect_may_have_applied ?? null) !== null || ($job->safe_to_resubmit ?? null) !== null || (string) ($job->effect_resolution ?? '') !== 'not_applicable' || (bool) ($job->finalization_incident ?? false); } if ($phase !== 'terminal' || (int) ($job->terminal_at ?? 0) < 1 || !in_array( (string) ($job->outcome ?? ''), ['succeeded', 'failed'], true ) || (string) ($job->status ?? '') !== (string) ($job->outcome ?? '')) { return true; } $outcome = (string) $job->outcome; $resolution = (string) ($job->effect_resolution ?? ''); $effectMayHaveApplied = $job->effect_may_have_applied ?? null; $safeToResubmit = $job->safe_to_resubmit ?? null; if ($outcome === 'succeeded') { return !in_array( $effectMayHaveApplied, [0, '0', false], true ) || !in_array($safeToResubmit, [0, '0', false], true) || $resolution !== 'applied' || ($job->error_code ?? null) !== null; } if (!in_array($resolution, [ 'unknown', 'applied', 'not_applied', ], true)) { return true; } if (in_array($effectMayHaveApplied, [1, '1', true], true)) { return !in_array($safeToResubmit, [0, '0', false], true); } return !in_array($effectMayHaveApplied, [0, '0', false], true) || !in_array( $safeToResubmit, [0, '0', false, 1, '1', true], true ) || $resolution !== 'not_applied'; } /** A published terminal effect must retain one canonical immutable tuple. */ public static function hasResolvedTerminalStateConflict(object $job): bool { if ((string) ($job->phase ?? '') !== 'terminal' || !in_array( (string) ($job->effect_resolution ?? ''), ['applied', 'not_applied'], true )) { return false; } if ((int) ($job->terminal_at ?? 0) < 1) { return true; } $outcome = (string) ($job->outcome ?? ''); $status = (string) ($job->status ?? ''); $resolution = (string) $job->effect_resolution; $effect = $job->effect_may_have_applied ?? null; $safe = $job->safe_to_resubmit ?? null; if ($outcome === 'succeeded') { $tupleConflict = $status !== 'succeeded' || $resolution !== 'applied' || !in_array($effect, [0, '0', false], true) || !in_array($safe, [0, '0', false], true) || ($job->error_code ?? null) !== null; if ($tupleConflict) { return true; } } elseif ($outcome !== 'failed' || $status !== 'failed') { return true; } elseif (in_array($effect, [1, '1', true], true)) { if (!in_array($safe, [0, '0', false], true)) { return true; } } elseif (!in_array($effect, [0, '0', false], true) || !in_array($safe, [0, '0', false, 1, '1', true], true) || $resolution !== 'not_applied') { return true; } $kind = (string) ($job->kind ?? ''); if (!CloudOperationRecoveryPolicy::supportsLedgerFinalizer($kind)) { return false; } return self::hasAgentRecoveryEnvelopeConflict($job); } /** True when one supported Agent generation's immutable envelope is bad. */ public static function hasAgentRecoveryEnvelopeConflict(object $job): bool { $kind = (string) ($job->kind ?? ''); $outcome = (string) ($job->outcome ?? ''); if (!CloudOperationRecoveryPolicy::supportsLedgerFinalizer($kind)) { return false; } if ((string) ($job->execution_mode ?? '') !== 'agent' || self::hasAgentRecoveryIdentityConflict($job) || (int) ($job->host_id ?? 0) < 1 || (int) ($job->dispatch_deadline_at ?? 0) < 1) { return true; } $dispatchAttempts = CloudDispatchLedger::attempts($job); if ($dispatchAttempts === 1) { return (string) ($job->attempts ?? '') !== '1' || (int) ($job->dispatch_started_at ?? 0) < 1 || (($job->agent_accepted_at ?? null) !== null && (int) $job->agent_accepted_at < 1) || (int) ($job->dispatched_at ?? 0) < 1 || (int) ($job->receipt_deadline_at ?? 0) < 1; } if ($dispatchAttempts !== 0 || (string) ($job->attempts ?? '') !== '0' || ($job->dispatch_started_at ?? null) !== null || ($job->agent_accepted_at ?? null) !== null || ($job->dispatched_at ?? null) !== null || ($job->receipt_deadline_at ?? null) !== null || ($job->completion_event_id ?? null) !== null) { return true; } if ($outcome !== 'succeeded') { return false; } return !self::isValidUndispatchedFirewallNoop($job); } /** True when one supported Agent generation lost immutable request identity. */ public static function hasAgentRecoveryIdentityConflict(object $job): bool { if (!CloudOperationRecoveryPolicy::supportsLedgerFinalizer( (string) ($job->kind ?? '') )) { return false; } return preg_match( '/^[a-f0-9]{32}$/D', (string) ($job->operation_id ?? '') ) !== 1 || preg_match( '/^[a-f0-9]{64}$/D', (string) ($job->request_fingerprint ?? '') ) !== 1 || preg_match( '/^[A-Za-z0-9_.-]{16,128}$/D', (string) ($job->request_idempotency_key ?? '') ) !== 1; } /** Generation zero success exists only for an unchanged firewall policy. */ private static function isValidUndispatchedFirewallNoop(object $job): bool { if ((string) ($job->kind ?? '') !== 'firewall_apply' || (int) ($job->instance_id ?? 0) < 1 || !is_string($job->result_json ?? null)) { return false; } $result = json_decode((string) $job->result_json, true); return is_array($result) && !array_is_list($result) && ($result['unchanged'] ?? null) === true && is_string($result['policy_digest'] ?? null) && preg_match( '/^[a-f0-9]{64}$/D', $result['policy_digest'] ) === 1 && is_int($result['revision'] ?? null) && $result['revision'] > 0; } /** * Operations that still own an instance resource. Active work blocks new * mutations; a terminal unknown-effect failure remains fenced until a * kind-specific, audited ledger finalizer has safely converged it. */ public static function blockingQueryForInstance(int $instanceId) { return self::blockingQueryForInstances([$instanceId]); } /** @param list $instanceIds */ public static function blockingQueryForInstances(array $instanceIds) { $query = CloudJob::whereIn('instance_id', $instanceIds); self::applyBlockingInstanceState($query); return $query; } /** @param list $orderIds */ public static function blockingCreateQueryForOrders(array $orderIds) { $query = CloudJob::whereIn('order_id', $orderIds) ->where('kind', 'create'); self::applyBlockingInstanceState($query); return $query; } /** * Serialize host-global VPC route mutations before Agent admission. * Independent VM-port jobs remain concurrent. A terminal route with an * unresolved effect or finalization incident keeps the host lane fenced. */ public static function applyVpcHostDispatchFence($query) { $routeKinds = ['vpc_host_apply', 'vpc_host_remove']; return $query->where(static function ($eligible) use ( $routeKinds ): void { $eligible->whereNotIn('cloud_jobs.kind', $routeKinds) ->orWhereNotExists(static function ($predecessors) use ( $routeKinds ): void { $predecessors->selectRaw('1') ->from('cloud_jobs as vpc_route_predecessors') ->whereColumn( 'vpc_route_predecessors.host_id', 'cloud_jobs.host_id' ) ->whereColumn( 'vpc_route_predecessors.id', '<', 'cloud_jobs.id' ) ->whereIn( 'vpc_route_predecessors.kind', $routeKinds ) ->where(static function ($outsideRepair): void { $outsideRepair->whereNull( 'cloud_jobs.vpc_repair_root_job_id' )->orWhereRaw( 'COALESCE(' . 'vpc_route_predecessors.vpc_repair_root_job_id, ' . 'vpc_route_predecessors.id) <> ' . 'cloud_jobs.vpc_repair_root_job_id' ); }) ->where(static function ($blocking): void { $blocking->where( 'vpc_route_predecessors.phase', '<>', 'terminal' )->orWhere(static function ($terminal): void { $terminal->where( 'vpc_route_predecessors.phase', 'terminal' )->where(static function ($unresolved): void { $unresolved->where( 'vpc_route_predecessors.effect_resolution', 'unknown' )->orWhere( 'vpc_route_predecessors.finalization_incident', 1 ); }); }); }); }); }); } /** * A stock-mode switch cannot cross an upgrade that still owns either the * source offer or the target offer. The offer row is locked by the caller, * so enqueue either commits its job first and is observed here, or reads * the administrator's new mode as its immutable source snapshot. */ public static function blockingUpgradeQueryForOffer(int $offerId) { $query = CloudJob::where('kind', 'instance_upgrade') ->where(static function ($linked) use ($offerId): void { $linked->whereExists(static function ($instances) use ($offerId): void { $instances->selectRaw('1')->from('cloud_instances') ->whereColumn('cloud_instances.id', 'cloud_jobs.instance_id') ->where('cloud_instances.offer_id', $offerId); })->orWhereExists(static function ($invoices) use ($offerId): void { $invoices->selectRaw('1')->from('cloud_invoices') ->whereColumn('cloud_invoices.id', 'cloud_jobs.invoice_id') ->where('cloud_invoices.purpose', 'upgrade') ->where('cloud_invoices.offer_id', $offerId); }); }); self::applyBlockingInstanceState($query); return $query; } private static function applyBlockingInstanceState($query): void { $query->where(static function ($query): void { $query->where('phase', '<>', 'terminal') ->orWhere(static function ($terminal): void { $terminal->where('phase', 'terminal') ->where('outcome', 'failed') ->where(static function ($blocked): void { $blocked->where('error_code', 'force_required') ->orWhere(static function ($unknown): void { $unknown->where('effect_resolution', 'unknown') ->where(static function ($effect): void { $effect->where('effect_may_have_applied', 1) ->orWhere('safe_to_resubmit', 0); }); }); }); })->orWhere(static function ($mode): void { $mode->whereIn( 'kind', CloudOperationRecoveryPolicy::automaticKinds() )->where('phase', 'terminal') ->where(static function ($invalid): void { $invalid->whereNull('execution_mode') ->orWhere( 'execution_mode', '<>', 'agent' ); }); })->orWhere(static function ($invalid): void { // Historical rows written by a fail-open release must // become blocking again immediately after upgrade, // even before a recovery worker has touched them. $invalid->where('execution_mode', 'agent') ->where(static function ($counter): void { $counter->whereNull('dispatch_attempts') ->orWhereNotIn( 'dispatch_attempts', [0, 1] ); }); })->orWhere(static function ($incident): void { $incident->where('execution_mode', 'agent') ->where('phase', 'terminal') ->where('finalization_incident', 1) ->where(static function ($recoverable): void { $recoverable->whereIn( 'finalization_error_code', CloudDispatchLedger::repairIncidentCodes() )->orWhere(static function ($local): void { $local->where('outcome', 'failed') ->where('status', 'failed') ->where('dispatch_attempts', 0) ->whereIn( 'kind', CloudOperationRecoveryPolicy::automaticKinds() ); })->orWhere(static function ($resolved): void { $resolved->whereIn( 'dispatch_attempts', [0, 1] ) ->whereIn( 'effect_resolution', ['applied', 'not_applied'] )->whereIn( 'kind', CloudOperationRecoveryPolicy::automaticKinds() ); }); }); })->orWhere(static function ($state): void { $state->where('execution_mode', 'agent') ->whereIn( 'kind', CloudOperationRecoveryPolicy::automaticKinds() )->where('phase', 'terminal') ->whereIn('dispatch_attempts', [0, 1]) ->where('effect_resolution', 'unknown') ->where(static function ($invalid): void { self::applyInvalidTerminalTuple($invalid); }); })->orWhere(static function ($resolvedState): void { $resolvedState->whereIn( 'kind', CloudOperationRecoveryPolicy::automaticKinds() )->where('phase', 'terminal') ->where(static function ($invalid): void { self::applyInvalidResolvedTerminalTuple( $invalid ); }); }); }); } public static function blockingQueryForInvoice(int $invoiceId) { return CloudJob::where('invoice_id', $invoiceId) ->where('kind', 'payment_create') ->where(static function ($query): void { $query->where('phase', '<>', 'terminal') ->orWhere(static function ($terminal): void { $terminal->where('phase', 'terminal') ->where('outcome', 'failed') ->where('effect_resolution', 'unknown') ->where('safe_to_resubmit', 0); }); }); } /** * Reserve the one public task that represents a multi-host mutation. * * Callers must invoke this while holding their catalog authority row (or * before creating that row) inside the same database transaction. The * aggregate is never claimed by a worker and can only be completed by the * terminal transition of its persisted child jobs. * * @return array{0: CloudJob, 1: bool} job and whether it was newly created */ public static function reserveAggregate( array $actor, string $kind, array $canonicalRequest ): array { $userId = (int) ($actor['user_id'] ?? 0); if ($userId < 1 || !self::isAggregateKind($kind)) { throw new CloudError('任务请求无效', 400); } $fingerprint = hash('sha256', json_encode( self::canonicalRequestValue([ 'kind' => $kind, 'request' => $canonicalRequest, ]), JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE )); $requestKey = self::requestKey($actor); if ($requestKey !== null) { $prior = CloudJob::where('user_id', $userId) ->where('request_idempotency_key', $requestKey) ->lockForUpdate()->first(); if ($prior !== null) { if (!hash_equals((string) $prior->request_fingerprint, $fingerprint)) { throw new CloudError( '同一请求标识不能用于不同的任务', 409 ); } return [$prior, false]; } } $job = new CloudJob(); $job->operation_id = bin2hex(random_bytes(16)); $job->request_idempotency_key = $requestKey ?? (string) $job->operation_id; $job->request_fingerprint = $fingerprint; $job->user_id = $userId; $job->host_id = null; $job->kind = $kind; $job->execution_mode = 'aggregate'; $job->sealed_payload = CloudSecrets::encrypt([ 'aggregate' => true, ], 'job-payload'); $now = time(); $job->receipt_deadline_at = $now + self::aggregateDeadlineSeconds($kind); $job->available_at = $job->created_at = $job->updated_at = $now; $job->save(); return [$job, true]; } /** * Reserve one public, host-targeted mutation with a plaintext canonical * fingerprint. Replaying the same HTTP idempotency key returns the first * CloudJob; it never creates another Agent operation. * * @return array{0: CloudJob, 1: bool} */ public static function reserveStandaloneAgentMutation( array $actor, int $hostId, string $kind, string $method, string $target, array $body, array $metadata, array $canonicalRequest, ?string $operationId = null, ?string $idempotencyTarget = null ): array { $userId = (int) ($actor['user_id'] ?? 0); $method = strtoupper($method); if ($userId < 1 || $hostId < 1 || !array_key_exists($kind, CloudOperationCatalog::kinds()) || !in_array($method, ['POST', 'PUT', 'PATCH', 'DELETE'], true) || !str_starts_with($target, '/v1/')) { throw new CloudError('Agent 任务请求无效', 400); } $fingerprint = self::standaloneRequestFingerprint( $hostId, $kind, $method, $idempotencyTarget ?? $target, $canonicalRequest ); $requestKey = self::requestKey($actor); if ($requestKey !== null) { $prior = CloudJob::where('user_id', $userId) ->where('request_idempotency_key', $requestKey) ->lockForUpdate()->first(); if ($prior !== null) { if (!hash_equals((string) $prior->request_fingerprint, $fingerprint)) { throw new CloudError( '同一请求标识不能用于不同的任务', 409 ); } return [$prior, false]; } } $operationId ??= bin2hex(random_bytes(16)); if (preg_match('/^[a-f0-9]{32}$/D', $operationId) !== 1) { throw new CloudError('Agent 任务操作标识无效', 400); } $body['operation_id'] = $operationId; $job = new CloudJob(); $job->operation_id = $operationId; $job->request_idempotency_key = $requestKey ?? $operationId; $job->request_fingerprint = $fingerprint; $job->user_id = $userId; $job->host_id = $hostId; $job->kind = $kind; $job->execution_mode = 'agent'; $job->sealed_payload = CloudSecrets::encrypt(array_merge($metadata, [ 'method' => $method, 'request_target' => $target, 'body' => $body, 'owner_id' => (int) ($metadata['owner_id'] ?? $userId), 'can_manage' => (bool) ($metadata['can_manage'] ?? false), ]), 'job-payload'); $now = time(); $job->dispatch_deadline_at = $now + self::dispatchDeadlineSeconds($kind); $job->available_at = $job->created_at = $job->updated_at = $now; $job->save(); return [$job, true]; } /** Exact public idempotency fingerprint used before external preparation. */ public static function standaloneRequestFingerprint( int $hostId, string $kind, string $method, string $target, array $canonicalRequest ): string { $method = strtoupper($method); if ($hostId < 1 || !array_key_exists($kind, CloudOperationCatalog::kinds()) || !in_array($method, ['POST', 'PUT', 'PATCH', 'DELETE'], true) || !str_starts_with($target, '/v1/')) { throw new CloudError('Agent 任务请求无效', 400); } return hash('sha256', json_encode( self::canonicalRequestValue([ 'host_id' => $hostId, 'kind' => $kind, 'method' => $method, 'target' => $target, 'request' => $canonicalRequest, ]), JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE )); } /** Persist one exact Agent mutation. It can be claimed only once. */ public static function reserveAgentMutation( array $actor, int $hostId, string $kind, string $method, string $target, array $body, array $metadata = [], ?int $parentJobId = null, ?string $operationId = null ): CloudJob { $userId = (int) ($actor['user_id'] ?? 0); $method = strtoupper($method); if ($userId < 1 || $hostId < 1 || !array_key_exists($kind, CloudOperationCatalog::kinds()) || !in_array($method, ['POST', 'PUT', 'PATCH', 'DELETE'], true) || !str_starts_with($target, '/v1/')) { throw new CloudError('Agent 任务请求无效', 400); } $operationId ??= bin2hex(random_bytes(16)); if (preg_match('/^[a-f0-9]{32}$/D', $operationId) !== 1) { throw new CloudError('Agent 任务操作标识无效', 400); } $body['operation_id'] = $operationId; $fingerprint = hash('sha256', json_encode( self::canonicalRequestValue([ 'host_id' => $hostId, 'kind' => $kind, 'method' => $method, 'target' => $target, 'body' => $body, 'metadata' => $metadata, ]), JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE )); $job = new CloudJob(); $job->operation_id = $operationId; $job->request_idempotency_key = $operationId; $job->request_fingerprint = $fingerprint; $job->parent_job_id = $parentJobId; $job->user_id = $userId; $job->host_id = $hostId; $job->kind = $kind; $job->execution_mode = 'agent'; $job->sealed_payload = CloudSecrets::encrypt(array_merge($metadata, [ 'method' => $method, 'request_target' => $target, 'body' => $body, 'owner_id' => (int) ($metadata['owner_id'] ?? $userId), 'can_manage' => (bool) ($metadata['can_manage'] ?? false), ]), 'job-payload'); $now = time(); $job->dispatch_deadline_at = $now + self::dispatchDeadlineSeconds($kind); $job->available_at = $job->created_at = $job->updated_at = $now; $job->save(); return $job; } /** Strict HTTP acknowledgement used by every asynchronous mutation. */ public static function acknowledgement(CloudJob $job): array { return [ 'task_id' => (string) $job->id, 'status_url' => '/api/v1/tasks/' . (int) $job->id, ]; } /** * Record a proven no-op as a successful public task without ever making it * claimable by a worker. The row is first persisted with the normal task * invariants, then terminalized in the caller's database transaction. */ public static function completeWithoutDispatchLocked( CloudJob $job, array $result ): void { if (!$job->exists || (string) $job->phase !== 'accepted' || (int) $job->dispatch_attempts !== 0 || (int) $job->attempts !== 0 || $job->dispatch_started_at !== null || $job->dispatched_at !== null || $job->agent_accepted_at !== null || $job->receipt_deadline_at !== null) { throw new \LogicException( 'Only a fresh, undispatched task can complete as a no-op' ); } self::completeAggregateLocked( $job, 'succeeded', false, false, '', $result ); } /** Complete an aggregate without dispatching another Agent mutation. */ public static function completeAggregateLocked( CloudJob $job, string $outcome, bool $effectMayHaveApplied, bool $safeToResubmit, string $errorCode = '', ?array $result = null ): void { if ((string) $job->phase === 'terminal') { return; } if (!in_array($outcome, ['succeeded', 'failed'], true)) { throw new \InvalidArgumentException('Invalid aggregate outcome'); } if ($outcome !== 'failed') { $effectMayHaveApplied = false; $safeToResubmit = false; $errorCode = ''; } elseif ($effectMayHaveApplied) { $safeToResubmit = false; } $now = time(); $job->phase = 'terminal'; $job->outcome = $outcome; $job->status = $outcome; $job->terminal_at = $now; $job->effect_may_have_applied = $effectMayHaveApplied ? 1 : 0; $job->safe_to_resubmit = $safeToResubmit ? 1 : 0; $job->effect_resolution = $outcome === 'succeeded' ? 'applied' : ($effectMayHaveApplied ? 'unknown' : 'not_applied'); $job->error_code = $errorCode === '' ? null : substr($errorCode, 0, 128); $job->result_json = $result === null ? null : json_encode( $result, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE ); $job->sealed_payload = CloudSecrets::encrypt([], 'job-payload'); $job->lease_token = null; $job->lease_until = 0; $job->available_at = 0; $job->updated_at = $now; $job->save(); } /** * Release a terminal aggregate fence after every Agent child has reached * an evidence-backed resolution. The parent's failed audit outcome remains * immutable; only its effect resolution is published. */ public static function reconcileAggregateParentEffectLocked( CloudJob $child, ?int $resolvedBy ): void { if (!in_array( (string) $child->kind, array_values(self::AGGREGATE_CHILD_KIND_MAP), true )) { return; } $parentId = (int) ($child->parent_job_id ?? 0); if ($parentId < 1) { throw new CloudRecoveryAggregateStateConflict( '汇总子任务缺少父任务账本', 409 ); } $parent = CloudJob::where('id', $parentId) ->lockForUpdate()->first(); if ($parent === null || (string) $parent->execution_mode !== 'aggregate') { throw new CloudRecoveryAggregateStateConflict( '汇总子任务父任务账本不存在或模式不匹配', 409 ); } // Reject a malformed parent/child topology before catalog discovery. // This is a read-only preflight while the parent is held; valid paths // still acquire every write lock in parent -> catalog -> child order // and revalidate the relationship after the child locks are taken. $childSnapshot = CloudJob::where('parent_job_id', $parentId) ->orderBy('id')->get(); if (self::aggregateRelationshipConflict( $parent, $childSnapshot, (int) $child->id )) { self::deferAggregateRecoveryLocked( $parent, 'recovery_aggregate_state_conflict', time() ); return; } CloudTemplateCatalogService::lockAggregateCatalogLocked($parent); CloudUserIsoService::lockAggregateCatalogLocked($parent); $children = CloudJob::where('parent_job_id', $parentId) ->orderBy('id')->lockForUpdate()->get(); if (self::aggregateRelationshipConflict( $parent, $children, (int) $child->id )) { self::deferAggregateRecoveryLocked( $parent, 'recovery_aggregate_state_conflict', time() ); return; } $publishedParent = self::aggregateParentHasPublishedEffect($parent); if ($children->contains(static fn (CloudJob $row): bool => self::aggregateChildRecoveryPending($row))) { if ($publishedParent) { self::deferAggregateRecoveryLocked( $parent, 'recovery_child_convergence', time() ); } return; } $resolution = self::aggregateEffectResolution($children); $metadataMixed = $resolution === 'mixed' && (string) $parent->kind === 'template_metadata_sync'; if ($resolution === null) { self::deferAggregateRecoveryLocked( $parent, 'recovery_aggregate_state_conflict', time() ); return; } if ($publishedParent) { self::recheckPublishedAggregateProjectionLocked( $parent, $resolution, $metadataMixed ); return; } $partialMixed = $resolution === 'mixed' && !$metadataMixed; if ($partialMixed) { // Validate every child-to-replica identity before publishing a // concrete failed-but-applied partial result. The aggregate stays // failed; "applied" means at least one proven host effect exists, // never that every replica succeeded. CloudTemplateCatalogService::settleRecoveredParentLocked( $parent, 'mixed' ); CloudUserIsoService::settleRecoveredParentLocked( $parent, 'mixed' ); if ((string) $parent->phase !== 'terminal') { $parent->refresh(); } } $projectionSettled = $partialMixed; if (!$partialMixed && (string) $parent->phase !== 'terminal') { // Recovered leaf projections are prepared before the matching // Agent fence is released. Only this post-commit hook may now // publish the user-visible aggregate result. CloudTemplateCatalogService::settleRecoveredParentLocked( $parent, $resolution ); CloudUserIsoService::settleRecoveredParentLocked( $parent, $resolution ); $parent->refresh(); if ((string) $parent->outcome !== 'failed' || (string) $parent->effect_resolution !== 'unknown') { return; } $projectionSettled = true; } if ((string) $parent->outcome !== 'failed' || (string) $parent->effect_resolution !== 'unknown') { return; } // A deadline may have failed the immutable public task and marked the // catalog record failed before late Agent evidence converged every // replica. Recompute only that resource projection now. The service // finalizers call completeAggregateLocked(), whose terminal guard keeps // the original parent phase/outcome/status/timestamps/error/result // immutable. if (!$projectionSettled) { CloudTemplateCatalogService::settleRecoveredParentLocked( $parent, $resolution ); CloudUserIsoService::settleRecoveredParentLocked( $parent, $resolution ); } if (!CloudTemplateCatalogService::aggregateEffectProjectionReadyLocked( $parent, $resolution )) { self::deferAggregateRecoveryLocked( $parent, 'recovery_child_convergence', time() ); return; } $publishedResolution = $metadataMixed || $partialMixed ? 'applied' : $resolution; $parent->effect_resolution = $publishedResolution; $parent->effect_resolved_at = time(); $parent->effect_resolved_by = $resolvedBy; $parent->effect_resolution_reason = sprintf( 'child_effects_resolved:%d:%s', (int) $child->id, $partialMixed ? 'mixed' : $publishedResolution ); $parent->finalization_incident = 0; $parent->finalization_error_code = null; $parent->lease_token = null; $parent->lease_until = 0; $parent->available_at = 0; $parent->read_at = null; $parent->updated_at = time(); $parent->save(); } /** * Converge one aggregate fence left behind before the realtime child hook * existed. This is a PHP-ledger-only sweep: it never dispatches or inspects * an Agent operation. Parent then children is the only explicit lock order, * and children are locked by id to avoid sibling-order inversions. It is * also the durable fallback if a realtime sibling hook loses a deadlock * retry or PHP exits after committing the final child. */ private static function wherePublishedAggregateNeedsRecheck( $published ): void { $published->whereIn( 'effect_resolution', ['applied', 'not_applied'] )->where(static function ($needs): void { $needs->whereNull('dispatch_attempts') ->orWhere('dispatch_attempts', '<>', 0) ->orWhere(static function ($incident): void { $incident->where('finalization_incident', 1); })->orWhereExists(static function ($templates): void { $templates->selectRaw('1') ->from('cloud_templates as current_templates') ->whereColumn( 'current_templates.active_job_id', 'cloud_jobs.id' ); })->orWhereExists(static function ($isos): void { $isos->selectRaw('1') ->from('cloud_user_isos as current_isos') ->whereColumn( 'current_isos.active_job_id', 'cloud_jobs.id' ); }); }); } public static function reconcileOneLegacyAggregateEffect(): bool { $connection = (new CloudJob())->getConnection(); $claimedParentId = 0; try { return $connection->transaction( static function () use (&$claimedParentId): bool { $now = time(); $parent = CloudJob::where('phase', 'terminal') ->where(static function ($candidate): void { $candidate->where( static function ($aggregate): void { $aggregate->whereIn( 'kind', array_keys( self::AGGREGATE_CHILD_KIND_MAP ) )->where( static function ($recoverable): void { $recoverable->where( static function ($unknown): void { $unknown->where( 'outcome', 'failed' )->where( 'status', 'failed' )->where( 'effect_may_have_applied', 1 )->where( 'safe_to_resubmit', 0 )->where( 'effect_resolution', 'unknown' ); } )->orWhere( static function ($ledger): void { $ledger->whereNull( 'dispatch_attempts' )->orWhere( 'dispatch_attempts', '<>', 0 ); } )->orWhere( static function ($published): void { self::wherePublishedAggregateNeedsRecheck( $published ); } )->orWhere( static function ($damaged): void { self::whereAggregateLocalStateConflict( $damaged ); } ); } ); } )->orWhereRaw( self::aggregateLocalStateConflictSql( 'cloud_jobs' ) )->orWhere( static function ($bound): void { self::whereCurrentAggregateCatalogBinding( $bound ); } )->orWhere(static function ($incident): void { $incident->whereNull('parent_job_id') ->where('finalization_incident', 1) ->where( 'finalization_error_code', 'recovery_aggregate_state_conflict' ); }); }) ->where('available_at', '<=', $now) ->where('lease_until', '<=', $now) ->orderByRaw( "CASE WHEN effect_resolution = 'unknown' " . 'THEN 0 ELSE 1 END' ) ->orderBy('available_at') ->orderBy('id') ->lockForUpdate() ->first(); if ($parent === null) { return false; } $claimedParentId = (int) $parent->id; // Classify a corrupt aggregate locally before catalog // discovery. The parent lock serializes legitimate child // writers; valid paths still re-read every child under // lock after the catalog row is locked. $childSnapshot = CloudJob::where( 'parent_job_id', (int) $parent->id )->orderBy('id')->get(); if (self::aggregateRelationshipConflict( $parent, $childSnapshot )) { self::deferAggregateRecoveryLocked( $parent, 'recovery_aggregate_state_conflict', $now ); return true; } CloudTemplateCatalogService::lockAggregateCatalogLocked( $parent ); CloudUserIsoService::lockAggregateCatalogLocked($parent); $children = CloudJob::where( 'parent_job_id', (int) $parent->id )->orderBy('id')->lockForUpdate()->get(); if (self::aggregateRelationshipConflict( $parent, $children )) { self::deferAggregateRecoveryLocked( $parent, 'recovery_aggregate_state_conflict', $now ); return true; } $waiting = $children->contains( static fn (CloudJob $child): bool => self::aggregateChildRecoveryPending($child) ); if ($waiting) { self::deferAggregateRecoveryLocked( $parent, 'recovery_child_convergence', $now ); return true; } $resolution = self::aggregateEffectResolution($children); $metadataMixed = $resolution === 'mixed' && (string) $parent->kind === 'template_metadata_sync'; if ($resolution === null) { self::deferAggregateRecoveryLocked( $parent, 'recovery_aggregate_state_conflict', $now ); return true; } if (self::aggregateParentHasPublishedEffect($parent)) { self::recheckPublishedAggregateProjectionLocked( $parent, $resolution, $metadataMixed ); return true; } $partialMixed = $resolution === 'mixed' && !$metadataMixed; if ($partialMixed) { CloudTemplateCatalogService::settleRecoveredParentLocked( $parent, 'mixed' ); CloudUserIsoService::settleRecoveredParentLocked( $parent, 'mixed' ); } // See reconcileAggregateParentEffectLocked(): only repair // the catalog projection; never rewrite the failed audit. if (!$partialMixed) { CloudTemplateCatalogService::settleRecoveredParentLocked( $parent, $resolution ); CloudUserIsoService::settleRecoveredParentLocked( $parent, $resolution ); } if (!CloudTemplateCatalogService::aggregateEffectProjectionReadyLocked( $parent, $resolution )) { self::deferAggregateRecoveryLocked( $parent, 'recovery_child_convergence', $now ); return true; } $publishedResolution = $metadataMixed || $partialMixed ? 'applied' : $resolution; $lastChild = $children->last(); $parent->effect_resolution = $publishedResolution; $parent->effect_resolved_at = $now; $parent->effect_resolved_by = null; $parent->effect_resolution_reason = sprintf( 'legacy_child_effects_resolved:%d:%s', (int) $lastChild->id, $partialMixed ? 'mixed' : $publishedResolution ); $parent->finalization_incident = 0; $parent->finalization_error_code = null; $parent->lease_token = null; $parent->lease_until = 0; $parent->available_at = 0; $parent->read_at = null; $parent->updated_at = $now; $parent->save(); return true; }, 3 ); } catch (\Throwable $error) { if ($claimedParentId < 1) { throw $error; } // The first transaction rolled back both the resource projection // and parent effect resolution. Persist only a delayed incident in // a fresh transaction so one damaged legacy parent cannot abort // every recovery tick or starve later jobs. $connection->transaction( static function () use ($claimedParentId, $error): void { $parent = CloudJob::where('id', $claimedParentId) ->lockForUpdate()->first(); if ($parent === null || (string) $parent->execution_mode !== 'aggregate' || (string) $parent->phase !== 'terminal') { return; } $unknownFence = (string) $parent->outcome === 'failed' && (string) $parent->status === 'failed' && (string) $parent->effect_resolution === 'unknown'; $publishedParent = self::aggregateParentHasPublishedEffect( $parent ); if (!$unknownFence && !$publishedParent) { return; } self::deferAggregateRecoveryLocked( $parent, self::aggregateProjectionErrorCode($error), time() ); }, 3 ); error_log(sprintf( 'cloud aggregate projection recovery pending job_id=%d error=%s', $claimedParentId, get_class($error) )); return true; } } /** Reject aggregate topology damage before any external fence mutation. */ public static function assertAggregateChildRelationshipLocked( CloudJob $parent, CloudJob $child, $children ): void { if (self::aggregateRelationshipConflict( $parent, $children, (int) $child->id )) { throw new CloudRecoveryAggregateStateConflict( '汇总任务与子任务账本关系异常', 409 ); } } private static function aggregateRelationshipConflict( CloudJob $parent, $children, ?int $currentChildId = null ): bool { $parentId = (int) $parent->id; $expectedKind = self::AGGREGATE_CHILD_KIND_MAP[ (string) $parent->kind ] ?? null; return self::hasAggregateLocalStateConflict($parent) || $expectedKind === null || $children->isEmpty() || ($currentChildId !== null && !$children->contains( static fn (CloudJob $child): bool => (int) $child->id === $currentChildId )) || $children->contains( static fn (CloudJob $child): bool => (int) ($child->parent_job_id ?? 0) !== $parentId || (int) $child->user_id !== (int) $parent->user_id || (string) $child->execution_mode !== 'agent' || (string) $child->kind !== $expectedKind || self::aggregateChildHasPhaseResolutionConflict($child) || self::hasResolvedTerminalStateConflict($child) ); } /** * SQL counterpart of hasAggregateLocalStateConflict() for terminal rows. * This keeps an unbound, already-published parent reachable by the local * ledger sweep without polling every healthy historical aggregate. */ private static function whereAggregateLocalStateConflict($damaged): void { $operationInvalid = self::invalidIdentifierCharactersSql( 'operation_id', '0-9a-f' ); $fingerprintInvalid = self::invalidIdentifierCharactersSql( 'request_fingerprint', '0-9a-f' ); $idempotencyInvalid = self::invalidIdentifierCharactersSql( 'request_idempotency_key', 'A-Za-z0-9_.-' ); $damaged->where(static function ($invalid) use ( $operationInvalid, $fingerprintInvalid, $idempotencyInvalid ): void { $invalid->whereNotNull('parent_job_id') ->orWhereNotNull('host_id') ->orWhereNotNull('instance_id') ->orWhereNotNull('order_id') ->orWhereNotNull('invoice_id') ->orWhereNotNull('snapshot_id') ->orWhereNull('user_id') ->orWhere('user_id', '<', 1) ->orWhereNull('attempts') ->orWhere('attempts', '<>', 0) ->orWhereNull('dispatch_attempts') ->orWhere('dispatch_attempts', '<>', 0) ->orWhereNull('dispatch_deadline_at') ->orWhere('dispatch_deadline_at', '<', 1) ->orWhereNull('receipt_deadline_at') ->orWhere('receipt_deadline_at', '<', 1) ->orWhereNotNull('dispatch_started_at') ->orWhereNotNull('agent_accepted_at') ->orWhereNotNull('dispatched_at') ->orWhereNotNull('completion_event_id') ->orWhereNull('operation_id') ->orWhereRaw('LENGTH(operation_id) <> 32') ->orWhereRaw('LOWER(operation_id) <> operation_id') ->orWhereRaw($operationInvalid) ->orWhereNull('request_fingerprint') ->orWhereRaw('LENGTH(request_fingerprint) <> 64') ->orWhereRaw( 'LOWER(request_fingerprint) <> request_fingerprint' )->orWhereRaw($fingerprintInvalid) ->orWhereNull('request_idempotency_key') ->orWhereRaw( 'LENGTH(request_idempotency_key) NOT BETWEEN 16 AND 128' )->orWhereRaw($idempotencyInvalid) ->orWhereNull('terminal_at') ->orWhere('terminal_at', '<', 1) ->orWhereNull('outcome') ->orWhereNotIn('outcome', ['succeeded', 'failed']) ->orWhereNull('status') ->orWhereColumn('status', '<>', 'outcome') ->orWhere(static function ($succeeded): void { $succeeded->where('outcome', 'succeeded') ->where(static function ($mismatch): void { $mismatch->whereNull('effect_resolution') ->orWhere( 'effect_resolution', '<>', 'applied' )->orWhereNull('effect_may_have_applied') ->orWhere('effect_may_have_applied', '<>', 0) ->orWhereNull('safe_to_resubmit') ->orWhere('safe_to_resubmit', '<>', 0) ->orWhereNotNull('error_code'); }); })->orWhere(static function ($failed): void { $failed->where('outcome', 'failed') ->where(static function ($mismatch): void { $mismatch->whereNull('effect_resolution') ->orWhereNotIn( 'effect_resolution', ['unknown', 'applied', 'not_applied'] )->orWhereNull('effect_may_have_applied') ->orWhereNotIn( 'effect_may_have_applied', [0, 1] )->orWhereNull('safe_to_resubmit') ->orWhereNotIn('safe_to_resubmit', [0, 1]) ->orWhere(static function ($effectful): void { $effectful->where( 'effect_may_have_applied', 1 )->where('safe_to_resubmit', '<>', 0); })->orWhere(static function ($notEffectful): void { $notEffectful->where( 'effect_may_have_applied', 0 )->where( 'effect_resolution', '<>', 'not_applied' ); }); }); }); }); } /** Add resolved Agent rows whose immutable dispatch envelope is damaged. */ private static function applyInvalidResolvedAgentEnvelope( $invalid, string $prefix ): void { $operation = $prefix . 'operation_id'; $fingerprint = $prefix . 'request_fingerprint'; $idempotency = $prefix . 'request_idempotency_key'; $invalid->whereNull($prefix . 'execution_mode') ->orWhere($prefix . 'execution_mode', '<>', 'agent') ->orWhereNull($prefix . 'dispatch_attempts') ->orWhereNotIn($prefix . 'dispatch_attempts', [0, 1]) ->orWhereNull($prefix . 'host_id') ->orWhere($prefix . 'host_id', '<', 1) ->orWhereNull($prefix . 'dispatch_deadline_at') ->orWhere($prefix . 'dispatch_deadline_at', '<', 1) ->orWhereNull($operation) ->orWhereRaw('LENGTH(' . $operation . ') <> 32') ->orWhereRaw('LOWER(' . $operation . ') <> ' . $operation) ->orWhereRaw(self::invalidIdentifierCharactersSql( $operation, '0-9a-f' ))->orWhereNull($fingerprint) ->orWhereRaw('LENGTH(' . $fingerprint . ') <> 64') ->orWhereRaw('LOWER(' . $fingerprint . ') <> ' . $fingerprint) ->orWhereRaw(self::invalidIdentifierCharactersSql( $fingerprint, '0-9a-f' ))->orWhereNull($idempotency) ->orWhereRaw( 'LENGTH(' . $idempotency . ') NOT BETWEEN 16 AND 128' )->orWhereRaw(self::invalidIdentifierCharactersSql( $idempotency, 'A-Za-z0-9_.-' ))->orWhere(static function ($dispatched) use ($prefix): void { $dispatched->where($prefix . 'dispatch_attempts', 1) ->where(static function ($broken) use ($prefix): void { $broken->whereNull($prefix . 'attempts') ->orWhere($prefix . 'attempts', '<>', 1) ->orWhereNull($prefix . 'dispatch_started_at') ->orWhere($prefix . 'dispatch_started_at', '<', 1) ->orWhere(static function ($accepted) use ( $prefix ): void { $accepted->whereNotNull( $prefix . 'agent_accepted_at' )->where( $prefix . 'agent_accepted_at', '<', 1 ); }) ->orWhereNull($prefix . 'dispatched_at') ->orWhere($prefix . 'dispatched_at', '<', 1) ->orWhereNull($prefix . 'receipt_deadline_at') ->orWhere($prefix . 'receipt_deadline_at', '<', 1); }); })->orWhere(static function ($undispatched) use ($prefix): void { $undispatched->where($prefix . 'dispatch_attempts', 0) ->where(static function ($broken) use ($prefix): void { $broken->whereNull($prefix . 'attempts') ->orWhere($prefix . 'attempts', '<>', 0) ->orWhereNotNull($prefix . 'dispatch_started_at') ->orWhereNotNull($prefix . 'agent_accepted_at') ->orWhereNotNull($prefix . 'dispatched_at') ->orWhereNotNull($prefix . 'receipt_deadline_at') ->orWhereNotNull($prefix . 'completion_event_id') ->orWhere(static function ($success) use ( $prefix ): void { $success->where( $prefix . 'outcome', 'succeeded' )->where(static function ($notNoop) use ( $prefix ): void { $notNoop->where( $prefix . 'kind', '<>', 'firewall_apply' )->orWhereNull($prefix . 'instance_id') ->orWhere( $prefix . 'instance_id', '<', 1 )->orWhereRaw( self::invalidFirewallNoopResultSql( $prefix . 'result_json' ) ); }); }); }); }); } /** Driver-specific JSON validation for the sole generation-zero success. */ private static function invalidFirewallNoopResultSql(string $column): string { $driver = (new CloudJob())->getConnection()->getDriverName(); if ($driver === 'sqlite') { $digest = "json_extract(" . $column . ", '$.policy_digest')"; return '(CASE WHEN COALESCE(json_valid(' . $column . '), 0) = 1 ' . 'THEN CASE WHEN ' . "json_type(" . $column . ", '$.unchanged') = 'true' AND " . "json_type(" . $column . ", '$.policy_digest') = 'text' AND " . 'LENGTH(' . $digest . ') = 64 AND NOT (' . self::invalidIdentifierCharactersSql($digest, '0-9a-f') . ') AND ' . "json_type(" . $column . ", '$.revision') = 'integer' AND " . "json_extract(" . $column . ", '$.revision') > 0 " . 'THEN 0 ELSE 1 END ELSE 1 END) = 1'; } $digest = "JSON_UNQUOTE(JSON_EXTRACT(" . $column . ", '$.policy_digest'))"; $revision = "JSON_UNQUOTE(JSON_EXTRACT(" . $column . ", '$.revision'))"; return '(CASE WHEN COALESCE(JSON_VALID(' . $column . '), 0) = 1 ' . 'THEN CASE WHEN JSON_UNQUOTE(JSON_EXTRACT(' . $column . ", '$.unchanged')) = 'true' AND LENGTH(" . $digest . ') = 64 AND NOT (' . self::invalidIdentifierCharactersSql($digest, '0-9a-f') . ') AND ' . $revision . " REGEXP '^[1-9][0-9]*$' " . 'THEN 0 ELSE 1 END ELSE 1 END) = 1'; } /** Build a portable invalid-character predicate for MySQL and SQLite. */ private static function invalidIdentifierCharactersSql( string $column, string $allowedClass ): string { $driver = (new CloudJob())->getConnection()->getDriverName(); if ($driver === 'sqlite') { return $column . " GLOB '*[^" . $allowedClass . "]*'"; } return $column . " REGEXP '[^" . $allowedClass . "]'"; } /** * Raw-SQL counterpart of hasTerminalRecoveryStateConflict(). Inventory * filters and counters cannot hydrate every row before filtering, so keep * their fail-closed envelope test in the same service as the DTO check. */ public static function terminalRecoveryStateConflictSql( string $alias ): string { $supported = self::quotedSqlValues( CloudOperationRecoveryPolicy::ledgerFinalizerKinds() ); $operation = $alias . '.operation_id'; $fingerprint = $alias . '.request_fingerprint'; $idempotency = $alias . '.request_idempotency_key'; $invalidEnvelope = '(' . $alias . '.execution_mode IS NULL OR ' . $alias . ".execution_mode <> 'agent' OR " . $alias . '.dispatch_attempts IS NULL OR ' . $alias . '.dispatch_attempts NOT IN (0, 1) OR ' . $alias . '.host_id IS NULL OR ' . $alias . '.host_id < 1 OR ' . $alias . '.dispatch_deadline_at IS NULL OR ' . $alias . '.dispatch_deadline_at < 1 OR ' . $operation . ' IS NULL OR LENGTH(' . $operation . ') <> 32 OR ' . 'LOWER(' . $operation . ') <> ' . $operation . ' OR ' . self::invalidIdentifierCharactersSql($operation, '0-9a-f') . ' OR ' . $fingerprint . ' IS NULL OR LENGTH(' . $fingerprint . ') <> 64 OR LOWER(' . $fingerprint . ') <> ' . $fingerprint . ' OR ' . self::invalidIdentifierCharactersSql($fingerprint, '0-9a-f') . ' OR ' . $idempotency . ' IS NULL OR LENGTH(' . $idempotency . ') NOT BETWEEN 16 AND 128 OR ' . self::invalidIdentifierCharactersSql( $idempotency, 'A-Za-z0-9_.-' ) . ' OR (' . $alias . '.dispatch_attempts = 1 AND (' . $alias . '.attempts IS NULL OR ' . $alias . '.attempts <> 1 OR ' . $alias . '.dispatch_started_at IS NULL OR ' . $alias . '.dispatch_started_at < 1 OR (' . $alias . '.agent_accepted_at IS NOT NULL AND ' . $alias . '.agent_accepted_at < 1) OR ' . $alias . '.dispatched_at IS NULL OR ' . $alias . '.dispatched_at < 1 OR ' . $alias . '.receipt_deadline_at IS NULL OR ' . $alias . '.receipt_deadline_at < 1)) OR (' . $alias . '.dispatch_attempts = 0 AND (' . $alias . '.attempts IS NULL OR ' . $alias . '.attempts <> 0 OR ' . $alias . '.dispatch_started_at IS NOT NULL OR ' . $alias . '.agent_accepted_at IS NOT NULL OR ' . $alias . '.dispatched_at IS NOT NULL OR ' . $alias . '.receipt_deadline_at IS NOT NULL OR ' . $alias . '.completion_event_id IS NOT NULL OR (' . $alias . ".outcome = 'succeeded' AND (" . $alias . ".kind <> 'firewall_apply' OR " . $alias . '.instance_id IS NULL OR ' . $alias . '.instance_id < 1 OR ' . self::invalidFirewallNoopResultSql( $alias . '.result_json' ) . ')))))'; $invalidResolved = '(' . $alias . ".effect_resolution IN ('applied', 'not_applied') AND (" . $alias . '.terminal_at IS NULL OR ' . $alias . '.terminal_at < 1 OR ' . $alias . '.outcome IS NULL OR ' . $alias . ".outcome NOT IN ('succeeded', 'failed') OR (" . $alias . ".outcome = 'succeeded' AND (" . $alias . ".status IS NULL OR " . $alias . ".status <> 'succeeded' OR " . $alias . ".effect_resolution <> 'applied' OR " . $alias . '.effect_may_have_applied IS NULL OR ' . $alias . '.effect_may_have_applied <> 0 OR ' . $alias . '.safe_to_resubmit IS NULL OR ' . $alias . '.safe_to_resubmit <> 0 OR ' . $alias . '.error_code IS NOT NULL)) OR (' . $alias . ".outcome = 'failed' AND (" . $alias . ".status IS NULL OR " . $alias . ".status <> 'failed' OR " . $alias . '.effect_may_have_applied IS NULL OR ' . $alias . '.effect_may_have_applied NOT IN (0, 1) OR (' . $alias . '.effect_may_have_applied = 1 AND (' . $alias . '.safe_to_resubmit IS NULL OR ' . $alias . '.safe_to_resubmit <> 0)) OR (' . $alias . '.effect_may_have_applied = 0 AND (' . $alias . ".effect_resolution <> 'not_applied' OR " . $alias . '.safe_to_resubmit IS NULL OR ' . $alias . '.safe_to_resubmit NOT IN (0, 1))))) OR ' . $invalidEnvelope . '))'; $invalidUnknown = '(' . $alias . ".effect_resolution = 'unknown' AND ((" . $alias . '.dispatch_attempts = 0 AND COALESCE(' . $alias . '.finalization_incident, 0) = 0) OR NOT (' . "COALESCE(" . $alias . ".outcome, '') = 'failed' AND " . "COALESCE(" . $alias . ".status, '') = 'failed' AND " . 'COALESCE(' . $alias . '.effect_may_have_applied, 0) = 1 AND ' . 'COALESCE(' . $alias . '.safe_to_resubmit, 1) = 0)))'; return '(' . $alias . ".phase = 'terminal' AND " . $alias . '.kind IN (' . $supported . ') AND (' . $alias . '.execution_mode IS NULL OR ' . $alias . ".execution_mode <> 'agent' OR " . $alias . '.dispatch_attempts IS NULL OR ' . $alias . '.dispatch_attempts NOT IN (0, 1) OR ' . $invalidEnvelope . ' OR ' . $invalidUnknown . ' OR ' . $invalidResolved . '))'; } /** SQL counterpart of hasAggregateLocalStateConflict() for terminal rows. */ public static function aggregateLocalStateConflictSql( string $alias ): string { $kinds = self::quotedSqlValues(self::AGGREGATE_KINDS); $operation = $alias . '.operation_id'; $fingerprint = $alias . '.request_fingerprint'; $idempotency = $alias . '.request_idempotency_key'; return '(' . $alias . ".phase = 'terminal' AND (" . $alias . ".execution_mode = 'aggregate' OR " . $alias . '.kind IN (' . $kinds . ')) AND (' . $alias . '.execution_mode IS NULL OR ' . $alias . ".execution_mode <> 'aggregate' OR " . $alias . '.kind IS NULL OR ' . $alias . '.kind NOT IN (' . $kinds . ') OR ' . $alias . '.parent_job_id IS NOT NULL OR ' . $alias . '.host_id IS NOT NULL OR ' . $alias . '.instance_id IS NOT NULL OR ' . $alias . '.order_id IS NOT NULL OR ' . $alias . '.invoice_id IS NOT NULL OR ' . $alias . '.snapshot_id IS NOT NULL OR ' . $alias . '.user_id IS NULL OR ' . $alias . '.user_id < 1 OR ' . $alias . '.attempts IS NULL OR ' . $alias . '.attempts <> 0 OR ' . $alias . '.dispatch_attempts IS NULL OR ' . $alias . '.dispatch_attempts <> 0 OR ' . $alias . '.dispatch_deadline_at IS NULL OR ' . $alias . '.dispatch_deadline_at < 1 OR ' . $alias . '.receipt_deadline_at IS NULL OR ' . $alias . '.receipt_deadline_at < 1 OR ' . $alias . '.dispatch_started_at IS NOT NULL OR ' . $alias . '.agent_accepted_at IS NOT NULL OR ' . $alias . '.dispatched_at IS NOT NULL OR ' . $alias . '.completion_event_id IS NOT NULL OR ' . $operation . ' IS NULL OR LENGTH(' . $operation . ') <> 32 OR ' . 'LOWER(' . $operation . ') <> ' . $operation . ' OR ' . self::invalidIdentifierCharactersSql($operation, '0-9a-f') . ' OR ' . $fingerprint . ' IS NULL OR LENGTH(' . $fingerprint . ') <> 64 OR LOWER(' . $fingerprint . ') <> ' . $fingerprint . ' OR ' . self::invalidIdentifierCharactersSql($fingerprint, '0-9a-f') . ' OR ' . $idempotency . ' IS NULL OR LENGTH(' . $idempotency . ') NOT BETWEEN 16 AND 128 OR ' . self::invalidIdentifierCharactersSql( $idempotency, 'A-Za-z0-9_.-' ) . ' OR ' . $alias . '.terminal_at IS NULL OR ' . $alias . '.terminal_at < 1 OR ' . $alias . '.outcome IS NULL OR ' . $alias . ".outcome NOT IN ('succeeded', 'failed') OR " . $alias . '.status IS NULL OR ' . $alias . '.status <> ' . $alias . '.outcome OR (' . $alias . ".outcome = 'succeeded' AND (" . $alias . ".effect_resolution IS NULL OR " . $alias . ".effect_resolution <> 'applied' OR " . $alias . '.effect_may_have_applied IS NULL OR ' . $alias . '.effect_may_have_applied <> 0 OR ' . $alias . '.safe_to_resubmit IS NULL OR ' . $alias . '.safe_to_resubmit <> 0 OR ' . $alias . '.error_code IS NOT NULL)) OR (' . $alias . ".outcome = 'failed' AND (" . $alias . '.effect_resolution IS NULL OR ' . $alias . ".effect_resolution NOT IN ('unknown', 'applied', 'not_applied') OR " . $alias . '.effect_may_have_applied IS NULL OR ' . $alias . '.effect_may_have_applied NOT IN (0, 1) OR ' . $alias . '.safe_to_resubmit IS NULL OR ' . $alias . '.safe_to_resubmit NOT IN (0, 1) OR (' . $alias . '.effect_may_have_applied = 1 AND ' . $alias . '.safe_to_resubmit <> 0) OR (' . $alias . '.effect_may_have_applied = 0 AND ' . $alias . ".effect_resolution <> 'not_applied')))))"; } /** @param list $values */ private static function quotedSqlValues(array $values): string { return implode(', ', array_map( static fn (string $value): string => "'" . str_replace("'", "''", $value) . "'", $values )); } /** Add the catalog-driven candidate predicate without assuming job shape. */ private static function whereCurrentAggregateCatalogBinding($bound): void { $bound->whereExists(static function ($templates): void { $templates->selectRaw('1') ->from('cloud_templates as current_templates') ->whereColumn( 'current_templates.active_job_id', 'cloud_jobs.id' ); })->orWhereExists(static function ($isos): void { $isos->selectRaw('1') ->from('cloud_user_isos as current_isos') ->whereColumn( 'current_isos.active_job_id', 'cloud_jobs.id' ); })->orWhere(static function ($repairable): void { self::whereRepairableMissingAggregateCatalogPointer( $repairable ); }); } /** * Keep a terminal parent discoverable when only active_job_id was cleared. * This predicate is intentionally broad enough to surface ambiguity; the * service-level locked repair performs the complete one-to-one proof. */ private static function whereRepairableMissingAggregateCatalogPointer( $candidate ): void { $settledPartial = self::settledReleasedPartialAggregateSql( 'cloud_jobs' ); $candidate->whereExists(static function ($templates) use ( $settledPartial ): void { $templates->selectRaw('1') ->from('cloud_template_replicas as pointer_template_replicas') ->join( 'cloud_templates as pointer_templates', 'pointer_templates.id', '=', 'pointer_template_replicas.template_id' )->join( 'cloud_jobs as pointer_template_children', 'pointer_template_children.id', '=', 'pointer_template_replicas.job_id' )->whereColumn( 'pointer_template_children.parent_job_id', 'cloud_jobs.id' )->where(static function ($binding) use ( $settledPartial ): void { $binding->where(static function ($missing) use ( $settledPartial ): void { $missing->where(static function ($released): void { $released->whereNull( 'pointer_templates.active_job_id' )->orWhere( 'pointer_templates.active_job_id', '<=', 0 ); })->whereRaw('NOT ' . $settledPartial); })->orWhere(static function ($conflict): void { $conflict->where( 'pointer_templates.active_job_id', '>', 0 )->whereColumn( 'pointer_templates.active_job_id', '<>', 'cloud_jobs.id' ); }); })->where(static function ($identity): void { $identity->where(static function ($import): void { $import->where('cloud_jobs.kind', 'template_import') ->where( 'pointer_template_children.kind', 'template_import_replica' )->whereIn( 'pointer_templates.state', ['importing', 'error', 'active'] ); })->orWhere(static function ($metadata): void { $metadata->where( 'cloud_jobs.kind', 'template_metadata_sync' )->where( 'pointer_template_children.kind', 'template_metadata_replica' )->where( 'pointer_templates.state', 'active' )->where(static function ($evidence): void { $evidence->whereIn( 'pointer_template_replicas.state', ['syncing', 'stale'] )->orWhere(static function ($unknown): void { $unknown->where( 'cloud_jobs.outcome', 'failed' )->where( 'cloud_jobs.status', 'failed' )->where( 'cloud_jobs.effect_resolution', 'unknown' ); })->orWhere( 'cloud_jobs.finalization_incident', 1 )->orWhere(static function ($damaged): void { $damaged->where( 'pointer_templates.active_job_id', '>', 0 )->whereColumn( 'pointer_templates.active_job_id', '<>', 'cloud_jobs.id' ); }); }); })->orWhere(static function ($delete): void { $delete->where('cloud_jobs.kind', 'template_delete') ->where( 'pointer_template_children.kind', 'template_delete_replica' )->whereIn( 'pointer_templates.state', ['deleting', 'delete_failed', 'active'] ); }); }); })->orWhereExists(static function ($isos) use ( $settledPartial ): void { $isos->selectRaw('1') ->from('cloud_user_iso_replicas as pointer_iso_replicas') ->join( 'cloud_user_isos as pointer_isos', 'pointer_isos.id', '=', 'pointer_iso_replicas.user_iso_id' )->join( 'cloud_jobs as pointer_iso_children', 'pointer_iso_children.id', '=', 'pointer_iso_replicas.job_id' )->whereColumn( 'pointer_iso_children.parent_job_id', 'cloud_jobs.id' )->where(static function ($binding) use ( $settledPartial ): void { $binding->where(static function ($missing) use ( $settledPartial ): void { $missing->where(static function ($released): void { $released->whereNull( 'pointer_isos.active_job_id' )->orWhere( 'pointer_isos.active_job_id', '<=', 0 ); })->whereRaw('NOT ' . $settledPartial); })->orWhere(static function ($conflict): void { $conflict->where( 'pointer_isos.active_job_id', '>', 0 )->whereColumn( 'pointer_isos.active_job_id', '<>', 'cloud_jobs.id' ); }); })->where(static function ($identity): void { $identity->where(static function ($import): void { $import->where('cloud_jobs.kind', 'user_iso_import') ->where( 'pointer_iso_children.kind', 'user_iso_import_replica' )->whereIn( 'pointer_isos.state', ['importing', 'import_failed', 'active'] ); })->orWhere(static function ($delete): void { $delete->where('cloud_jobs.kind', 'user_iso_delete') ->where( 'pointer_iso_children.kind', 'user_iso_delete_replica' )->whereIn( 'pointer_isos.state', ['deleting', 'delete_failed', 'active'] ); }); }); }); } /** Exact durable marker for a safely released non-metadata partial. */ private static function settledReleasedPartialAggregateSql( string $alias ): string { $kinds = self::quotedSqlValues([ 'template_import', 'template_delete', 'user_iso_import', 'user_iso_delete', ]); return '(' . $alias . ".execution_mode = 'aggregate' AND " . $alias . '.parent_job_id IS NULL AND ' . $alias . '.kind IN (' . $kinds . ') AND ' . $alias . ".phase = 'terminal' AND " . $alias . ".outcome = 'failed' AND " . $alias . ".status = 'failed' AND " . $alias . '.effect_may_have_applied = 1 AND ' . $alias . '.safe_to_resubmit = 0 AND ' . $alias . ".effect_resolution = 'applied' AND " . $alias . '.effect_resolved_at > 0 AND ' . 'COALESCE(' . $alias . '.finalization_incident, 0) = 0 AND ' . 'NOT ' . self::aggregateLocalStateConflictSql($alias) . ' AND ' . self::settledPartialReasonSql( $alias . '.effect_resolution_reason' ) . ')'; } /** Match only our immutable numeric-child mixed publication reasons. */ private static function settledPartialReasonSql(string $column): string { $driver = (new CloudJob())->getConnection()->getDriverName(); if ($driver !== 'sqlite') { return $column . " REGEXP '^(legacy_)?child_effects_resolved:" . "[1-9][0-9]*:mixed$'"; } $suffix = ':mixed'; $parts = []; foreach ([ 'child_effects_resolved:', 'legacy_child_effects_resolved:', ] as $prefix) { $offset = strlen($prefix) + 1; $middleLength = 'LENGTH(' . $column . ') - ' . (strlen($prefix) + strlen($suffix)); $middle = 'SUBSTR(' . $column . ', ' . $offset . ', ' . $middleLength . ')'; $parts[] = '(SUBSTR(' . $column . ', 1, ' . strlen($prefix) . ") = '" . $prefix . "' AND SUBSTR(" . $column . ", -" . strlen($suffix) . ") = '" . $suffix . "' AND " . $middleLength . ' > 0 AND SUBSTR(' . $middle . ", 1, 1) BETWEEN '1' AND '9' AND NOT (" . $middle . " GLOB '*[^0-9]*'))"; } return '(' . implode(' OR ', $parts) . ')'; } /** A final effect cannot be published while its leaf is still nonterminal. */ private static function aggregateChildHasPhaseResolutionConflict( CloudJob $child ): bool { return (string) $child->phase !== 'terminal' && in_array( (string) $child->effect_resolution, ['applied', 'not_applied'], true ); } /** Return the uniform effect, mixed, or null for an invalid child state. */ private static function aggregateEffectResolution($children): ?string { $resolutions = $children->map( static fn (CloudJob $child): string => (string) $child->effect_resolution )->unique()->values(); if ($resolutions->contains( static fn ($resolution): bool => !in_array( (string) $resolution, ['applied', 'not_applied'], true ) )) { return null; } return $resolutions->count() === 1 ? (string) $resolutions->first() : 'mixed'; } private static function aggregateParentHasPublishedEffect( CloudJob $parent ): bool { return (string) $parent->phase === 'terminal' && in_array( (string) $parent->effect_resolution, ['applied', 'not_applied'], true ); } /** Revalidate a published aggregate without rewriting its terminal audit. */ private static function recheckPublishedAggregateProjectionLocked( CloudJob $parent, string $resolution, bool $metadataMixed ): void { try { $bindingState = self::currentAggregateBindingStateLocked($parent); } catch (\Throwable $error) { self::deferAggregateRecoveryLocked( $parent, self::aggregateProjectionErrorCode($error), time() ); return; } if ($bindingState === 'conflict') { self::deferAggregateRecoveryLocked( $parent, 'recovery_aggregate_state_conflict', time() ); return; } if ($bindingState === 'absent') { if ((bool) $parent->finalization_incident) { $errorCode = strtolower(trim((string) ( $parent->finalization_error_code ?? '' ))); self::deferAggregateRecoveryLocked( $parent, preg_match( '/^recovery_[a-z0-9_]{1,55}$/D', $errorCode ) === 1 ? $errorCode : 'recovery_ledger_conflict', time() ); return; } $parent->lease_token = null; $parent->lease_until = 0; $parent->available_at = time() + self::RECOVERY_RETRY_SECONDS; $parent->updated_at = time(); $parent->save(); return; } $partialMixed = $resolution === 'mixed' && !$metadataMixed; if ($partialMixed && !self::hasPublishedPartialAggregateMarker($parent)) { self::deferAggregateRecoveryLocked( $parent, 'recovery_aggregate_state_conflict', time() ); return; } $publishedResolution = $metadataMixed || $partialMixed ? 'applied' : $resolution; if (!hash_equals( (string) $parent->effect_resolution, $publishedResolution )) { self::deferAggregateRecoveryLocked( $parent, 'recovery_aggregate_state_conflict', time() ); return; } try { if (str_starts_with((string) $parent->kind, 'template_')) { CloudTemplateCatalogService:: assertSettledParentProjectionLocked( $parent, $resolution ); } else { CloudUserIsoService::assertSettledParentProjectionLocked( $parent, $resolution ); } } catch (\Throwable $error) { self::deferAggregateRecoveryLocked( $parent, self::aggregateProjectionErrorCode($error), time() ); return; } $now = time(); $clearedIncident = (bool) $parent->finalization_incident; if ($clearedIncident) { $parent->finalization_incident = 0; $parent->finalization_error_code = null; $parent->read_at = null; } $parent->lease_token = null; $parent->lease_until = 0; $parent->available_at = $now + self::RECOVERY_RETRY_SECONDS; $parent->updated_at = $now; $parent->save(); } /** * Lock both catalog authorities in one order and classify their binding. * * @return 'valid'|'absent'|'conflict' */ private static function currentAggregateBindingStateLocked( CloudJob $parent ): string { $templates = CloudTemplate::query() ->where('active_job_id', (int) $parent->id) ->orderBy('id')->lockForUpdate()->get(['id']); $isos = CloudUserIso::query() ->where('active_job_id', (int) $parent->id) ->orderBy('id')->lockForUpdate()->get(['id']); if ($templates->isEmpty() && $isos->isEmpty()) { $kind = (string) $parent->kind; if (in_array($kind, [ 'template_import', 'template_metadata_sync', 'template_delete', ], true)) { CloudTemplateCatalogService::repairMissingParentBindingLocked( $parent ); } elseif (in_array($kind, [ 'user_iso_import', 'user_iso_delete', ], true)) { CloudUserIsoService::repairMissingParentBindingLocked($parent); } else { return 'absent'; } $templates = CloudTemplate::query() ->where('active_job_id', (int) $parent->id) ->orderBy('id')->lockForUpdate()->get(['id']); $isos = CloudUserIso::query() ->where('active_job_id', (int) $parent->id) ->orderBy('id')->lockForUpdate()->get(['id']); } if (str_starts_with((string) $parent->kind, 'template_')) { return $templates->count() === 1 && $isos->isEmpty() ? 'valid' : 'conflict'; } if (str_starts_with((string) $parent->kind, 'user_iso_')) { return $isos->count() === 1 && $templates->isEmpty() ? 'valid' : 'conflict'; } return 'conflict'; } /** @return list */ public static function aggregateProjectionIncidentCodes(): array { return [ CloudDispatchLedger::CONFLICT_CODE, 'recovery_child_convergence', 'recovery_aggregate_state_conflict', 'recovery_payload_unavailable', 'recovery_operation_identity_conflict', 'recovery_instance_state_conflict', 'recovery_ledger_conflict', 'recovery_internal_error', ]; } private static function deferAggregateRecoveryLocked( CloudJob $parent, string $errorCode, int $now ): void { $parent->read_at = null; $parent->finalization_incident = 1; $parent->finalization_error_code = $errorCode; $parent->lease_token = null; $parent->lease_until = 0; $parent->available_at = $now + self::RECOVERY_RETRY_SECONDS; $parent->updated_at = $now; $parent->save(); } private static function aggregateProjectionErrorCode( \Throwable $error ): string { return match (true) { $error instanceof CloudRecoveryDispatchLedgerConflict => CloudDispatchLedger::CONFLICT_CODE, $error instanceof CloudRecoveryAggregateStateConflict => 'recovery_aggregate_state_conflict', $error instanceof CloudRecoveryPayloadUnavailable => 'recovery_payload_unavailable', $error instanceof CloudRecoveryOperationIdentityConflict => 'recovery_operation_identity_conflict', $error instanceof CloudRecoveryInstanceStateConflict => 'recovery_instance_state_conflict', $error instanceof CloudError => 'recovery_ledger_conflict', default => 'recovery_internal_error', }; } public static function enqueue(CloudInstance $instance, array $actor, string $kind, string $method, string $suffix, array $body = [], ?array $credential = null): CloudJob { CloudAccess::vmActor($instance, $actor); if ($kind === 'network_change') { throw new CloudError('旧公网网络写入口已停用', 409); } if (in_array($kind, ['delete', 'expiry_delete'], true)) { CloudInstanceResourceRelease::assertCanDispatchDelete( $instance ); } $requestKey = self::requestKey($actor); return $instance->getConnection()->transaction(function () use ( $instance, $actor, $kind, $method, $suffix, $body, $credential, $requestKey ) { $templateMetadata = null; $settingsFields = []; if ($kind === 'reinstall') { $body['template_id'] = CloudInput::template( $body['template_id'] ?? '' ); // Defense in depth for internal callers: this UI confirmation // is not part of the strict Agent ReinstallRequest schema. unset($body['confirmation_name']); } if ($kind === 'instance_settings') { CloudInstanceSettingsGuard::assertSupported($instance); CloudInput::only($body, ['hostname', 'root_password', 'ssh_public_key', 'confirm_restart']); if (($body['confirm_restart'] ?? false) !== true) { throw new CloudError('修改实例设置前必须确认虚拟机会短暂重启'); } $changed = false; if (array_key_exists('hostname', $body)) { $body['hostname'] = CloudInput::hostname($body['hostname']); $settingsFields[] = 'hostname'; $changed = true; } if (array_key_exists('root_password', $body)) { if (!is_string($body['root_password']) || $body['root_password'] === '') { throw new CloudError('实例密码不能为空'); } $body['root_password'] = CloudInput::password($body['root_password']); $settingsFields[] = 'password'; $changed = true; } if (array_key_exists('ssh_public_key', $body)) { $parsed = CloudSSHKeys::parse((string) $body['ssh_public_key']); $body['ssh_public_key'] = $parsed['public_key']; $settingsFields[] = 'ssh_key'; $changed = true; } if (!$changed) { throw new CloudError('请至少填写一项需要修改的实例设置'); } } $requestValue = [ 'instance_id' => (int) $instance->id, 'kind' => $kind, 'method' => strtoupper($method), 'suffix' => $suffix, 'body' => $body, ]; $requestFingerprint = hash('sha256', json_encode( self::canonicalRequestValue($requestValue), JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES )); $locked = CloudInstance::where('id', $instance->id)->lockForUpdate()->first(); if ($locked === null || (string) $locked->status === 'deleted') { throw new CloudError('虚拟机不存在', 404); } if (in_array($kind, ['delete', 'expiry_delete'], true)) { CloudInstanceResourceRelease::assertCanDispatchDelete( $locked ); } if ($requestKey !== null) { $prior = CloudJob::where('user_id', (int) $actor['user_id']) ->where('request_idempotency_key', $requestKey) ->lockForUpdate()->first(); if ($prior !== null) { if (!hash_equals((string) $prior->request_fingerprint, $requestFingerprint)) { throw new CloudError( '同一请求标识不能用于不同的虚拟机操作', 409 ); } return $prior; } } if ((string) $locked->status !== 'active') { throw new CloudError('实例正在执行到期销毁流程,不能提交其他操作', 409); } CloudInstanceMigrationService::assertInstanceMutationAvailable( $locked, $kind ); if ($kind === 'reinstall') { CloudSnapshotService::assertReinstallAllowedLocked($locked); $body = CloudTemplateCatalogService:: applyProvisioningSnapshotForHost( $body, (int) $locked->host_id, false ); $templateMetadata = CloudTemplateCatalogService::templateMetadataSnapshot( $body ); } $policyService = ['suspending_expired', 'resume_pending', 'resuming', 'suspend_failed', 'resume_failed', 'expired_grace', 'purge_stopping', 'purge_stop_failed', 'purge_force_required', 'purge_deleting', 'purge_failed']; $policyNetwork = ['blocking', 'blocking_expired', 'blocked_expired', 'restore_pending', 'restoring', 'block_failed', 'restore_failed']; $superAdmin = ($actor['is_super_admin'] ?? false) && ($actor['can_manage'] ?? false); if ($kind !== 'delete' && !$superAdmin && (in_array((string) $locked->service_status, $policyService, true) || in_array((string) $locked->network_status, $policyNetwork, true) || ((string) $locked->service_status !== 'exempt' && ($locked->expires_at === null || (int) $locked->expires_at <= time())))) { throw new CloudError('实例正在执行到期或流量策略,请稍后重试', 409); } $activeJobs = self::blockingQueryForInstance((int) $locked->id); if ($activeJobs->exists()) { throw new CloudError('虚拟机已有任务,请等待完成', 409); } $job = new CloudJob(); $job->operation_id = bin2hex(random_bytes(16)); $job->instance_id = $locked->id; $job->user_id = $actor['user_id']; $job->host_id = $locked->host_id; $job->kind = $kind; $job->execution_mode = 'agent'; $job->status = 'queued'; if ($requestKey !== null) { $job->request_idempotency_key = $requestKey; } $job->request_fingerprint = $requestFingerprint; $jobPayload = ['method' => $method, 'request_target' => '/v1/vms/' . rawurlencode($locked->uuid) . $suffix, 'body' => $body, 'credential' => $credential, 'owner_id' => (int) $locked->user_id, 'can_manage' => (bool) ($actor['can_manage'] ?? false)]; if ($settingsFields !== []) { // This allow-listed metadata remains outside the strict Agent // body and survives terminal secret redaction. Never persist a // hostname, password or public-key value as presentation data. $jobPayload['settings_fields'] = $settingsFields; } if ($templateMetadata !== null) { // Encrypted panel metadata is deliberately outside body: // body is the exact strict Go ReinstallRequest DTO. $jobPayload['template_metadata'] = $templateMetadata; } $job->sealed_payload = CloudSecrets::encrypt( $jobPayload, 'job-payload' ); $job->available_at = $job->created_at = $job->updated_at = time(); $job->save(); return $job; }, 3); } /** * Queue the irreversible second phase of expiry cleanup. * * Only the lifecycle state machine may reach this method. Unlike the * four target-state policy jobs, expiry deletion is intentionally never * replayed after dispatch because an interrupted volume deletion may be * only partly complete. */ public static function enqueueExpiryDelete(CloudInstance $instance): CloudJob { return $instance->getConnection()->transaction(function () use ($instance) { $locked = CloudInstance::where('id', $instance->id)->lockForUpdate()->first(); if ($locked === null || (string) $locked->status === 'deleted') { throw new CloudError('虚拟机不存在', 404); } if ((string) $locked->status !== 'deleting' || (string) $locked->service_status !== 'purge_deleting') { throw new CloudError('实例尚未完成销毁前停机,不能删除磁盘', 409); } CloudInstanceResourceRelease::assertCanDispatchDelete( $locked ); CloudInstanceMigrationService::assertInstanceMutationAvailable( $locked, 'expiry_delete' ); $existing = self::blockingQueryForInstance((int) $locked->id) ->orderByDesc('id')->first(); if ($existing !== null) { if ((string) $existing->kind === 'expiry_delete') { return $existing; } throw new CloudError('虚拟机已有任务,请等待完成', 409); } $job = new CloudJob(); $job->operation_id = bin2hex(random_bytes(16)); $job->instance_id = $locked->id; $job->user_id = $locked->user_id; $job->host_id = $locked->host_id; $job->kind = 'expiry_delete'; $job->execution_mode = 'agent'; $job->status = 'queued'; $job->sealed_payload = CloudSecrets::encrypt([ 'method' => 'DELETE', 'request_target' => '/v1/vms/' . rawurlencode($locked->uuid), 'body' => ['remove_managed_disks' => true], 'owner_id' => (int) $locked->user_id, 'system_cleanup' => true, ], 'job-payload'); $job->available_at = $job->created_at = $job->updated_at = time(); $job->save(); return $job; }, 3); } /** * Queue the separately audited force phase of a delete operation. * * The original graceful operation remains the authorization anchor. It is * never replayed or changed into a force request; the Agent validates that * exact prior operation before accepting this new operation ID. */ public static function enqueueForceDelete( CloudJob $prior, array $actor ): CloudJob { CloudAccess::manage($actor); $priorId = filter_var($prior->id, FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]); $instanceId = filter_var($prior->instance_id, FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]); if ($priorId === false || $instanceId === false) { throw new CloudError('当前任务没有关联实例', 409); } return $prior->getConnection()->transaction(function () use ( $priorId, $instanceId, $actor ) { // Keep the global instance-mutation lock order: instance first, // then jobs. The prior row is fully revalidated after both locks. $instance = CloudInstance::where('id', (int) $instanceId) ->lockForUpdate()->first(); $lockedPrior = CloudJob::where('id', (int) $priorId) ->lockForUpdate()->first(); if ($lockedPrior === null) { throw new CloudError('任务不存在', 404); } if ((bool) ($lockedPrior->finalization_incident ?? false) || self::hasTerminalRecoveryStateConflict($lockedPrior)) { throw new CloudError( '原删除任务的恢复账本异常尚未收敛,不能创建新的强制删除任务', 409 ); } if (!in_array((string) $lockedPrior->kind, ['delete', 'expiry_delete', 'service_suspend'], true) || (string) $lockedPrior->phase !== 'terminal' || (string) $lockedPrior->outcome !== 'failed' || (string) $lockedPrior->error_code !== 'force_required' || !CloudDispatchLedger::isDispatched($lockedPrior) || $lockedPrior->dispatched_at === null || preg_match('/^[a-f0-9]{32}$/D', (string) $lockedPrior->operation_id) !== 1 || (int) $lockedPrior->instance_id !== (int) $instanceId) { throw new CloudError('当前任务没有经过确认的强制删除前置状态', 409); } if ($instance === null || (string) $instance->status === 'deleted' || (int) $instance->host_id !== (int) $lockedPrior->host_id || preg_match('/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/iD', (string) $instance->uuid) !== 1) { throw new CloudError('关联实例已不存在或与原删除任务不一致', 409); } CloudInstanceResourceRelease::assertCanDispatchDelete( $instance ); $priorPayload = CloudSecrets::decrypt( (string) $lockedPrior->sealed_payload, 'job-payload' ); $target = '/v1/vms/' . rawurlencode((string) $instance->uuid); $gracefulDelete = ($priorPayload['method'] ?? null) === 'DELETE' && ($priorPayload['request_target'] ?? null) === $target && ($priorPayload['body'] ?? null) === ['remove_managed_disks' => true]; $expiryGracefulStop = (string) $lockedPrior->kind === 'service_suspend' && ($priorPayload['method'] ?? null) === 'POST' && ($priorPayload['request_target'] ?? null) === $target . '/service/suspend' && ($priorPayload['body'] ?? null) === ['reason' => 'purge'] && ($priorPayload['system_policy'] ?? false) === true && (string) $instance->status === 'deleting' && (string) $instance->service_status === 'purge_force_required'; if (!$gracefulDelete && !$expiryGracefulStop) { throw new CloudError('原删除任务不允许强制删除面板管理磁盘', 409); } $otherBlocker = self::blockingQueryForInstance((int) $instance->id) ->where('id', '<>', (int) $lockedPrior->id) ->lockForUpdate()->first(); if ($otherBlocker !== null) { throw new CloudError('虚拟机还有其他未完成任务,不能发起强制删除', 409); } do { $operationId = bin2hex(random_bytes(16)); } while (hash_equals((string) $lockedPrior->operation_id, $operationId)); $job = new CloudJob(); $job->operation_id = $operationId; $job->instance_id = $instance->id; $job->user_id = (int) ($actor['user_id'] ?? 0); $job->host_id = $instance->host_id; $job->kind = 'delete_force'; $job->execution_mode = 'agent'; $job->status = 'queued'; $job->sealed_payload = CloudSecrets::encrypt([ 'method' => 'POST', 'request_target' => '/v1/vms/' . rawurlencode((string) $instance->uuid) . '/force-delete', 'body' => [ 'operation_id' => $operationId, 'prior_operation_id' => (string) $lockedPrior->operation_id, 'ack_data_loss' => true, 'remove_managed_disks' => true, ], 'owner_id' => (int) $instance->user_id, 'can_manage' => true, 'prior_job_id' => (int) $lockedPrior->id, 'prior_operation_id' => (string) $lockedPrior->operation_id, ], 'job-payload'); $now = time(); $job->available_at = $job->created_at = $job->updated_at = $now; $job->save(); error_log(sprintf( 'cloud admin queued force delete prior_job_id=%d force_job_id=%d prior_operation_id=%s force_operation_id=%s actor_id=%d', (int) $lockedPrior->id, (int) $job->id, (string) $lockedPrior->operation_id, $operationId, (int) ($actor['user_id'] ?? 0) )); return $job; }, 3); } public static function display(CloudJob $job, bool $administrator = false): array { $completed = (string) $job->phase === 'terminal' && in_array((string) $job->outcome, ['succeeded', 'failed'], true); $action = self::action($job); $settingsFields = self::settingsFields($job); $labels = CloudOperationCatalog::labels((string) $job->kind, $action, $settingsFields); $base = [ 'id' => (string) $job->id, 'operation_id' => (string) $job->operation_id, 'kind' => (string) $job->kind, 'completed' => $completed, 'created_at' => max(0, (int) $job->created_at), 'updated_at' => max(0, (int) $job->updated_at), 'status_url' => '/api/v1/tasks/' . (int) $job->id, 'action' => $action, 'kind_label' => $labels['kind_label'], 'action_label' => $labels['action_label'], 'operation_copy_keys' => $labels['copy_keys'], ]; if (!$completed) { $recovery = self::recoveryPresentation($job, $administrator); if ($recovery !== null) { $base['recovery'] = $recovery; } return $base; } $result = $job->result_json === null ? null : json_decode($job->result_json, true); $status = (string) $job->outcome; $safeResult = (string) $job->kind === 'payment_create' && $status === 'succeeded' && is_array($result) ? self::displayPaymentResult($job, $result) : self::displayResult(is_array($result) ? $result : null, $administrator, (string) $job->kind, $status, $job->order_id !== null, $action); $error = $administrator ? CloudOperationErrorCatalog::adminSummary((string) $job->kind, $status, $job->error_code, $job->order_id !== null, $action) : CloudOperationErrorCatalog::userMessage((string) $job->kind, $status, $job->error_code, $job->order_id !== null, $action); $phase = $safeResult['phase'] ?? null; $phase = $administrator ? CloudOperationErrorCatalog::adminSummary((string) $job->kind, $status, $phase, $job->order_id !== null, $action) : CloudOperationErrorCatalog::userPhase($phase, $status, (string) $job->kind); $terminal = [ 'status' => $status, 'error' => $error, 'error_code' => CloudOperationErrorCatalog::publicCode($job->error_code), 'phase' => is_string($phase) ? mb_substr($phase, 0, 120) : null, 'result' => $safeResult, 'effect_may_have_applied' => $job->effect_may_have_applied === null ? null : (bool) $job->effect_may_have_applied, 'safe_to_resubmit' => $job->safe_to_resubmit === null ? null : (bool) $job->safe_to_resubmit, 'unread' => $job->read_at === null, 'terminal_at' => max(0, (int) $job->terminal_at), ]; $effectResolution = (string) $job->effect_resolution; if ($status === 'failed' && (bool) $job->effect_may_have_applied && in_array($effectResolution, ['applied', 'not_applied'], true)) { // This is deliberately not a generic recovery/audit field. The // public contract exposes only the two conclusions needed to stop // presenting an already-released resource fence as active. $terminal['effect_resolution'] = $effectResolution; } $recovery = self::recoveryPresentation($job, $administrator); if ($recovery !== null) { $terminal['recovery'] = $recovery; } else { $effectScope = self::effectScope($job); if ($effectScope !== null) { $terminal['effect_scope'] = $effectScope; } } return array_merge($base, $terminal); } /** * Distinguish a fully proven mixed aggregate from an all-host applied * result without exposing the internal convergence reason to clients. */ public static function effectScope(object $job): ?string { return self::hasPublishedPartialAggregateMarker($job) && !(bool) ($job->finalization_incident ?? false) ? 'partial' : null; } /** Canonical mixed publication tuple, including rows under recheck. */ public static function hasPublishedPartialAggregateMarker( object $job ): bool { $reason = (string) ($job->effect_resolution_reason ?? ''); return (string) ($job->execution_mode ?? '') === 'aggregate' && ($job->parent_job_id ?? null) === null && in_array((string) ($job->kind ?? ''), [ 'template_import', 'template_delete', 'user_iso_import', 'user_iso_delete', ], true) && !self::hasAggregateLocalStateConflict($job) && (string) ($job->phase ?? '') === 'terminal' && (string) ($job->outcome ?? '') === 'failed' && (string) ($job->status ?? '') === 'failed' && (bool) ($job->effect_may_have_applied ?? false) && !(bool) ($job->safe_to_resubmit ?? true) && (string) ($job->effect_resolution ?? '') === 'applied' && (int) ($job->effect_resolved_at ?? 0) > 0 && preg_match( '/^(?:legacy_)?child_effects_resolved:[1-9][0-9]*:mixed$/D', $reason ) === 1; } /** * Return a bounded recovery status for panels. Arbitrary server/Agent * exception text never crosses this DTO; unknown recovery codes collapse * to one reviewed message. */ public static function recoveryPresentation( CloudJob $job, bool $administrator = false ): ?array { $mode = self::recoveryMode($job); if ($mode === null) { return null; } if ($mode === 'aggregate') { $details = self::aggregateRecoveryDetails($job); $code = $details['code']; $message = $details['message']; } else { $candidate = self::agentRecoveryPresentationCandidate($job); if ($mode === 'local_finalizer' && in_array( $candidate, CloudDispatchLedger::repairIncidentCodes(), true )) { $candidate = 'recovery_ledger_conflict'; } $code = array_key_exists($candidate, self::RECOVERY_MESSAGES) ? $candidate : ($mode === 'local_finalizer' ? 'recovery_ledger_conflict' : 'recovery_pending'); $message = self::RECOVERY_MESSAGES[$code]; } $nextCheckAt = max(0, (int) ($job->available_at ?? 0)); $lastCheckedAt = max(0, (int) ($job->updated_at ?? 0)); if ($mode === 'aggregate') { $nextCheckAt = max($nextCheckAt, $details['next_check_at']); $lastCheckedAt = max( $lastCheckedAt, $details['last_checked_at'] ); } $result = [ 'pending' => true, 'message' => $message, 'last_checked_at' => $lastCheckedAt > 0 ? $lastCheckedAt : null, 'next_check_at' => $nextCheckAt > time() ? $nextCheckAt : null, ]; if ($administrator) { $result['code'] = $code; if ($mode === 'aggregate') { $result['affected_child_count'] = $details['affected_child_count']; $result['additional_error_count'] = $details['additional_error_count']; } } return $result; } /** Select the most actionable reviewed cause without exposing raw text. */ private static function agentRecoveryPresentationCandidate( CloudJob $job ): string { if (CloudDispatchLedger::isInvalid($job)) { return CloudDispatchLedger::CONFLICT_CODE; } $stored = strtolower(trim( (string) ($job->finalization_error_code ?? '') )); if (CloudDispatchLedger::hasRepairIncident($job) && array_key_exists($stored, self::RECOVERY_MESSAGES)) { return $stored; } if (self::hasAgentRecoveryIdentityConflict($job)) { return 'recovery_operation_identity_conflict'; } if (in_array($stored, [ 'recovery_operation_identity_conflict', 'recovery_receipt_invalid', 'recovery_completion_outbox_invalid', 'recovery_completion_receipt_missing', 'recovery_completion_receipt_invalid', 'recovery_completion_operation_missing', 'recovery_completion_operation_invalid', 'recovery_completion_identity_mismatch', 'recovery_payload_invalid', 'recovery_payload_unavailable', ], true)) { return $stored; } if (self::hasTerminalRecoveryStateConflict($job)) { return 'recovery_instance_state_conflict'; } return $stored; } private static function recoveryMode(CloudJob $job): ?string { $kind = (string) $job->kind; $aggregateIdentity = (string) $job->execution_mode === 'aggregate' || in_array($kind, self::AGGREGATE_KINDS, true); if ($aggregateIdentity) { $ownFence = (string) $job->phase === 'terminal' && (string) $job->outcome === 'failed' && (string) $job->status === 'failed' && (bool) $job->effect_may_have_applied && !(bool) $job->safe_to_resubmit && (string) $job->effect_resolution === 'unknown'; $stickyIncident = (string) $job->phase === 'terminal' && (bool) $job->finalization_incident; if ($ownFence || $stickyIncident || self::hasTerminalRecoveryStateConflict($job) || self::hasUnresolvedRecoverableChildren($job)) { return 'aggregate'; } return null; } if ((string) $job->execution_mode === 'agent' && CloudDispatchLedger::isInvalid($job)) { return 'agent'; } if (CloudDispatchLedger::hasRepairIncident($job)) { return 'agent'; } if (self::isLocalFinalizerRecovery($job)) { return 'local_finalizer'; } if (self::isResolvedStickyAgentIncident($job)) { return 'agent'; } if (self::hasTerminalRecoveryStateConflict($job)) { return 'agent'; } if ((string) $job->phase !== 'terminal' || (string) $job->outcome !== 'failed' || (string) $job->status !== 'failed' || !(bool) $job->effect_may_have_applied || (bool) $job->safe_to_resubmit || (string) $job->effect_resolution !== 'unknown') { return null; } if ((string) $job->execution_mode === 'agent' && (int) $job->dispatch_attempts === 1 && CloudOperationRecoveryPolicy::supportsLedgerFinalizer($kind)) { return 'agent'; } return null; } private static function hasUnresolvedRecoverableChildren( CloudJob $parent ): bool { if ((int) $parent->id < 1) { return false; } return CloudJob::where('parent_job_id', (int) $parent->id) ->whereIn('kind', CloudOperationRecoveryPolicy::kinds()) ->where(static function ($pending): void { $pending->where(static function ($mode): void { $mode->whereNull('execution_mode') ->orWhere('execution_mode', '<>', 'agent'); })->orWhere(static function ($invalid): void { $invalid->whereNull('dispatch_attempts') ->orWhereNotIn('dispatch_attempts', [0, 1]); })->orWhere(static function ($repair): void { $repair->where('phase', 'terminal') ->where('finalization_incident', 1) ->whereIn( 'finalization_error_code', CloudDispatchLedger::repairIncidentCodes() ); })->orWhere(static function ($local): void { $local->where('phase', 'terminal') ->where('outcome', 'failed') ->where('status', 'failed') ->where('dispatch_attempts', 0) ->where('finalization_incident', 1); })->orWhere(static function ($sticky): void { $sticky->where('phase', 'terminal') ->where('finalization_incident', 1); }) ->orWhere(static function ($normal): void { $normal->where('phase', 'terminal') ->where('outcome', 'failed') ->where('status', 'failed') ->where('dispatch_attempts', 1) ->where('effect_may_have_applied', 1) ->where('safe_to_resubmit', 0) ->where('effect_resolution', 'unknown'); })->orWhere(static function ($state): void { $state->where('phase', 'terminal') ->whereIn('dispatch_attempts', [0, 1]) ->where('effect_resolution', 'unknown') ->where(static function ($invalid): void { self::applyInvalidTerminalTuple($invalid); }); })->orWhere(static function ($phase): void { $phase->where('phase', '<>', 'terminal') ->whereIn( 'effect_resolution', ['applied', 'not_applied'] ); })->orWhere(static function ($resolved): void { $resolved->where('phase', 'terminal') ->whereIn('dispatch_attempts', [0, 1]) ->where(static function ($invalid): void { self::applyInvalidResolvedTerminalTuple( $invalid ); }); }); }) ->exists(); } private static function unresolvedRecoverableChildExists($children): void { $children->selectRaw('1') ->from('cloud_jobs as recovery_children') ->whereColumn( 'recovery_children.parent_job_id', 'cloud_jobs.id' ) ->whereIn( 'recovery_children.kind', CloudOperationRecoveryPolicy::kinds() )->where(static function ($pending): void { $pending->where(static function ($mode): void { $mode->whereNull('recovery_children.execution_mode') ->orWhere( 'recovery_children.execution_mode', '<>', 'agent' ); })->orWhere(static function ($invalid): void { $invalid->whereNull( 'recovery_children.dispatch_attempts' )->orWhereNotIn( 'recovery_children.dispatch_attempts', [0, 1] ); })->orWhere(static function ($repair): void { $repair->where( 'recovery_children.phase', 'terminal' )->where( 'recovery_children.finalization_incident', 1 )->whereIn( 'recovery_children.finalization_error_code', CloudDispatchLedger::repairIncidentCodes() ); })->orWhere(static function ($local): void { $local->where( 'recovery_children.phase', 'terminal' )->where( 'recovery_children.outcome', 'failed' )->where( 'recovery_children.status', 'failed' )->where( 'recovery_children.dispatch_attempts', 0 )->where( 'recovery_children.finalization_incident', 1 ); })->orWhere(static function ($sticky): void { $sticky->where( 'recovery_children.phase', 'terminal' )->where( 'recovery_children.finalization_incident', 1 ); })->orWhere(static function ($normal): void { $normal->where( 'recovery_children.phase', 'terminal' )->where( 'recovery_children.outcome', 'failed' )->where( 'recovery_children.status', 'failed' )->where( 'recovery_children.dispatch_attempts', 1 )->where( 'recovery_children.effect_may_have_applied', 1 )->where( 'recovery_children.safe_to_resubmit', 0 )->where( 'recovery_children.effect_resolution', 'unknown' ); })->orWhere(static function ($state): void { $state->where( 'recovery_children.phase', 'terminal' )->whereIn( 'recovery_children.dispatch_attempts', [0, 1] )->where( 'recovery_children.effect_resolution', 'unknown' )->where(static function ($invalid): void { self::applyInvalidTerminalTuple( $invalid, 'recovery_children.' ); }); })->orWhere(static function ($phase): void { $phase->where( 'recovery_children.phase', '<>', 'terminal' )->whereIn( 'recovery_children.effect_resolution', ['applied', 'not_applied'] ); })->orWhere(static function ($resolved): void { $resolved->where( 'recovery_children.phase', 'terminal' )->whereIn( 'recovery_children.dispatch_attempts', [0, 1] )->where(static function ($invalid): void { self::applyInvalidResolvedTerminalTuple( $invalid, 'recovery_children.' ); }); }); }); } /** * Select one deterministic, reviewed child incident for the public batch. * Raw Agent/exception text and child result payloads never enter this DTO. * * @return array{code:string,message:string,affected_child_count:int,additional_error_count:int,last_checked_at:int,next_check_at:int} */ private static function aggregateRecoveryDetails(CloudJob $parent): array { if (!$parent->exists) { // display() is also used with detached, read-only task snapshots // (for example notification/rendering contracts). Only persisted // aggregate jobs have child rows that can be safely summarized. return [ 'code' => 'recovery_child_convergence', 'message' => self::RECOVERY_MESSAGES[ 'recovery_child_convergence' ], 'affected_child_count' => 0, 'additional_error_count' => 0, 'last_checked_at' => max(0, (int) $parent->updated_at), 'next_check_at' => max(0, (int) $parent->available_at), ]; } $children = CloudJob::where('parent_job_id', (int) $parent->id) ->orderBy('id') ->get([ 'id', 'operation_id', 'request_idempotency_key', 'request_fingerprint', 'parent_job_id', 'instance_id', 'user_id', 'host_id', 'kind', 'execution_mode', 'phase', 'outcome', 'status', 'attempts', 'dispatch_attempts', 'dispatch_deadline_at', 'dispatch_started_at', 'agent_accepted_at', 'dispatched_at', 'receipt_deadline_at', 'terminal_at', 'completion_event_id', 'error_code', 'result_json', 'effect_may_have_applied', 'safe_to_resubmit', 'effect_resolution', 'finalization_incident', 'finalization_error_code', 'available_at', 'updated_at', ]); $expectedKind = self::AGGREGATE_CHILD_KIND_MAP[ (string) $parent->kind ] ?? null; $parentStateConflict = self::hasAggregateLocalStateConflict($parent) || (string) ($parent->finalization_error_code ?? '') === 'recovery_aggregate_state_conflict'; $parentCandidate = $parentStateConflict ? 'recovery_aggregate_state_conflict' : strtolower(trim( (string) ($parent->finalization_error_code ?? '') )); $parentCode = array_key_exists( $parentCandidate, self::RECOVERY_MESSAGES ) ? $parentCandidate : null; if ($parentCode === null && (bool) $parent->finalization_incident) { $parentCode = 'recovery_ledger_conflict'; } $relationshipConflict = $parentCandidate === 'recovery_aggregate_state_conflict' || $expectedKind === null || $children->isEmpty() || $children->contains( static fn (CloudJob $child): bool => (int) $child->user_id !== (int) $parent->user_id || (string) $child->execution_mode !== 'agent' || (string) $child->kind !== $expectedKind || self::aggregateChildHasPhaseResolutionConflict($child) ); $unresolved = $children->filter( static fn (CloudJob $child): bool => (string) $child->execution_mode === 'agent' && CloudOperationRecoveryPolicy::supports( (string) $child->kind ) && (CloudDispatchLedger::isInvalid($child) || (bool) $child->finalization_incident || CloudDispatchLedger::hasRepairIncident($child) || self::isLocalFinalizerRecovery($child) || self::hasTerminalRecoveryStateConflict($child) || self::aggregateChildHasPhaseResolutionConflict($child) || self::hasResolvedTerminalStateConflict($child) || ((string) $child->phase === 'terminal' && (string) $child->outcome === 'failed' && (string) $child->status === 'failed' && CloudDispatchLedger::isDispatched($child) && (bool) $child->effect_may_have_applied && !(bool) $child->safe_to_resubmit && (string) $child->effect_resolution === 'unknown')) ); $lastCheckedAt = (int) ($unresolved->max('updated_at') ?? 0); $nextCheckAt = (int) ($unresolved->max('available_at') ?? 0); $priority = [ 'recovery_dispatch_ledger_conflict', 'recovery_dispatch_repair_completion_receipt_missing', 'recovery_dispatch_repair_receipt_invalid', 'recovery_dispatch_repair_aggregate_state_conflict', 'recovery_dispatch_repair_payload_unavailable', 'recovery_dispatch_repair_operation_identity_conflict', 'recovery_dispatch_repair_evidence_conflict', 'recovery_dispatch_repair_instance_state_conflict', 'recovery_dispatch_repair_ledger_conflict', 'recovery_dispatch_repair_internal_error', 'recovery_operation_identity_conflict', 'recovery_receipt_invalid', 'recovery_completion_identity_mismatch', 'recovery_completion_receipt_invalid', 'recovery_completion_receipt_missing', 'recovery_completion_operation_invalid', 'recovery_completion_operation_missing', 'recovery_completion_outbox_invalid', 'recovery_payload_unavailable', 'recovery_payload_invalid', 'recovery_instance_state_conflict', 'recovery_evidence_conflict', 'recovery_evidence_invalid', 'recovery_state_invalid', 'recovery_authentication_failed', 'recovery_agent_unavailable', 'recovery_waiting_connection', 'recovery_operation_conflict', 'recovery_operation_missing', 'recovery_evidence_partial', 'recovery_evidence_inconclusive', 'recovery_ledger_conflict', 'recovery_internal_error', 'recovery_pending', ]; $codes = $parentCode === null || $parentCode === 'recovery_child_convergence' ? [] : [$parentCode]; if ($relationshipConflict) { $codes[] = 'recovery_aggregate_state_conflict'; } foreach ($unresolved as $child) { $candidate = self::aggregateChildHasPhaseResolutionConflict($child) ? 'recovery_aggregate_state_conflict' : self::agentRecoveryPresentationCandidate($child); if (self::isLocalFinalizerRecovery($child) && in_array( $candidate, CloudDispatchLedger::repairIncidentCodes(), true )) { $candidate = 'recovery_ledger_conflict'; } $codes[] = array_key_exists($candidate, self::RECOVERY_MESSAGES) ? $candidate : 'recovery_pending'; } $codes = array_values(array_unique($codes)); usort($codes, static function (string $left, string $right) use ( $priority ): int { $leftPriority = array_search($left, $priority, true); $rightPriority = array_search($right, $priority, true); $leftPriority = $leftPriority === false ? PHP_INT_MAX : $leftPriority; $rightPriority = $rightPriority === false ? PHP_INT_MAX : $rightPriority; return $leftPriority <=> $rightPriority ?: strcmp($left, $right); }); $concreteParentCode = $parentCode !== null && $parentCode !== 'recovery_child_convergence' ? $parentCode : null; $code = $concreteParentCode ?? ($codes[0] ?? ($parentCode ?? 'recovery_child_convergence')); $parentLevelCode = $concreteParentCode !== null || ($relationshipConflict && $code === 'recovery_aggregate_state_conflict'); $message = $parentLevelCode || $code === 'recovery_child_convergence' ? self::RECOVERY_MESSAGES[$code] : '某个宿主机子任务:' . self::RECOVERY_MESSAGES[$code]; $additional = max(0, count($codes) - 1); if ($additional > 0) { $message .= ';另有 ' . $additional . ' 类子任务异常仍在持续自动核对。'; } return [ 'code' => $code, 'message' => $message, 'affected_child_count' => $relationshipConflict ? max(1, $unresolved->count()) : $unresolved->count(), 'additional_error_count' => $additional, 'last_checked_at' => $lastCheckedAt, 'next_check_at' => $nextCheckAt, ]; } private static function applyRecoveringFilter($query): void { $query->where(static function ($modes): void { $modes->where(static function ($leaf): void { $leaf->whereIn('kind', CloudOperationRecoveryPolicy::ledgerFinalizerKinds()) ->where(static function ($pending): void { $pending->where(static function ($mode): void { $mode->whereNull('execution_mode') ->orWhere('execution_mode', '<>', 'agent'); })->orWhere(static function ($invalid): void { $invalid->whereNull('dispatch_attempts') ->orWhereNotIn( 'dispatch_attempts', [0, 1] ); })->orWhere(static function ($repair): void { $repair->where('phase', 'terminal') ->where('finalization_incident', 1) ->whereIn( 'finalization_error_code', CloudDispatchLedger::repairIncidentCodes() ); })->orWhere(static function ($local): void { $local->where('phase', 'terminal') ->where('outcome', 'failed') ->where('status', 'failed') ->where('dispatch_attempts', 0) ->where('finalization_incident', 1); })->orWhere(static function ($resolved): void { $resolved->where('phase', 'terminal') ->whereIn('dispatch_attempts', [0, 1]) ->whereIn( 'effect_resolution', ['applied', 'not_applied'] )->where('finalization_incident', 1); })->orWhere(static function ($normal): void { $normal->where('phase', 'terminal') ->where('outcome', 'failed') ->where('status', 'failed') ->where('effect_may_have_applied', 1) ->where('safe_to_resubmit', 0) ->where('effect_resolution', 'unknown') ->where('dispatch_attempts', 1); })->orWhere(static function ($state): void { $state->where('phase', 'terminal') ->whereIn('dispatch_attempts', [0, 1]) ->where('effect_resolution', 'unknown') ->where(static function ($invalid): void { self::applyInvalidTerminalTuple($invalid); }); })->orWhere(static function ($resolvedState): void { $resolvedState->where('phase', 'terminal') ->whereIn('dispatch_attempts', [0, 1]) ->where(static function ($invalid): void { self::applyInvalidResolvedTerminalTuple( $invalid ); }); }); }); })->orWhere(static function ($aggregate): void { $aggregate->where('execution_mode', 'aggregate') ->whereIn('kind', self::AGGREGATE_KINDS) ->where(static function ($pending): void { $pending->where(static function ($ledger): void { $ledger->whereNull('dispatch_attempts') ->orWhere('dispatch_attempts', '<>', 0); })->orWhere(static function ($own): void { $own->where('phase', 'terminal') ->where('outcome', 'failed') ->where('status', 'failed') ->where('effect_may_have_applied', 1) ->where('safe_to_resubmit', 0) ->where('effect_resolution', 'unknown'); })->orWhere(static function ($state): void { $state->where('phase', 'terminal') ->where('effect_resolution', 'unknown') ->where(static function ($invalid): void { self::applyInvalidTerminalTuple($invalid); }); })->orWhere(static function ($published): void { $published->where('phase', 'terminal') ->where('finalization_incident', 1); })->orWhereExists( static function ($children): void { self::unresolvedRecoverableChildExists( $children ); } ); }); })->orWhereRaw( self::aggregateLocalStateConflictSql('cloud_jobs') )->orWhere(static function ($damagedCatalogParent): void { $damagedCatalogParent->where('phase', 'terminal') ->where('finalization_incident', 1) ->where(static function ($identity): void { $identity->where('execution_mode', 'aggregate') ->orWhereIn('kind', self::AGGREGATE_KINDS) ->orWhere( 'finalization_error_code', 'recovery_aggregate_state_conflict' ); }); }); }); } private static function isLocalFinalizerRecovery(object $job): bool { return (string) ($job->execution_mode ?? '') === 'agent' && (string) ($job->phase ?? '') === 'terminal' && (string) ($job->outcome ?? '') === 'failed' && (string) ($job->status ?? '') === 'failed' && CloudDispatchLedger:: isLocallyFinalizableNotDispatched($job) && (bool) ($job->finalization_incident ?? false) && !CloudDispatchLedger::hasRepairIncident($job) && CloudOperationRecoveryPolicy::supportsLedgerFinalizer( (string) ($job->kind ?? '') ); } /** A resolved Agent row with a sticky PHP incident remains locally queued. */ private static function isResolvedStickyAgentIncident(object $job): bool { return (string) ($job->execution_mode ?? '') === 'agent' && (string) ($job->phase ?? '') === 'terminal' && !CloudDispatchLedger::isInvalid($job) && in_array( (string) ($job->effect_resolution ?? ''), ['applied', 'not_applied'], true ) && (bool) ($job->finalization_incident ?? false) && !CloudDispatchLedger::hasRepairIncident($job) && CloudOperationRecoveryPolicy::supportsLedgerFinalizer( (string) ($job->kind ?? '') ); } /** Add the negation of the canonical failed/effectful/non-retryable tuple. */ private static function applyInvalidTerminalTuple( $invalid, string $prefix = '' ): void { $invalid->where(static function ($tuple) use ($prefix): void { $tuple->whereNull($prefix . 'outcome') ->orWhere($prefix . 'outcome', '<>', 'failed') ->orWhereNull($prefix . 'status') ->orWhere($prefix . 'status', '<>', 'failed') ->orWhereNull($prefix . 'effect_may_have_applied') ->orWhere($prefix . 'effect_may_have_applied', '<>', 1) ->orWhereNull($prefix . 'safe_to_resubmit') ->orWhere($prefix . 'safe_to_resubmit', '<>', 0); })->orWhere(static function ($neverDispatched) use ($prefix): void { $neverDispatched->where($prefix . 'dispatch_attempts', 0) ->where($prefix . 'finalization_incident', 0); }); } /** Add terminal applied/not-applied rows that contradict their outcome. */ public static function applyInvalidResolvedTerminalTuple( $invalid, string $prefix = '' ): void { $invalid->whereIn( $prefix . 'effect_resolution', ['applied', 'not_applied'] )->where(static function ($shape) use ($prefix): void { $shape->whereNull($prefix . 'terminal_at') ->orWhere($prefix . 'terminal_at', '<=', 0) ->orWhereNull($prefix . 'outcome') ->orWhereNotIn( $prefix . 'outcome', ['succeeded', 'failed'] )->orWhere(static function ($succeeded) use ($prefix): void { $succeeded->where($prefix . 'outcome', 'succeeded') ->where(static function ($mismatch) use ($prefix): void { $mismatch->whereNull($prefix . 'status') ->orWhere( $prefix . 'status', '<>', 'succeeded' )->orWhere( $prefix . 'effect_resolution', '<>', 'applied' )->orWhereNull( $prefix . 'effect_may_have_applied' )->orWhere( $prefix . 'effect_may_have_applied', '<>', 0 )->orWhereNull( $prefix . 'safe_to_resubmit' )->orWhere( $prefix . 'safe_to_resubmit', '<>', 0 )->orWhereNotNull($prefix . 'error_code'); }); })->orWhere(static function ($failed) use ($prefix): void { $failed->where($prefix . 'outcome', 'failed') ->where(static function ($mismatch) use ($prefix): void { $mismatch->whereNull($prefix . 'status') ->orWhere($prefix . 'status', '<>', 'failed') ->orWhereNull( $prefix . 'effect_may_have_applied' )->orWhereNotIn( $prefix . 'effect_may_have_applied', [0, 1] )->orWhere(static function ($effectful) use ( $prefix ): void { $effectful->where( $prefix . 'effect_may_have_applied', 1 )->where(static function ($safe) use ( $prefix ): void { $safe->whereNull( $prefix . 'safe_to_resubmit' )->orWhere( $prefix . 'safe_to_resubmit', '<>', 0 ); }); })->orWhere(static function ($noEffect) use ( $prefix ): void { $noEffect->where( $prefix . 'effect_may_have_applied', 0 )->where(static function ($conclusion) use ( $prefix ): void { $conclusion->where( $prefix . 'effect_resolution', '<>', 'not_applied' )->orWhereNull( $prefix . 'safe_to_resubmit' )->orWhereNotIn( $prefix . 'safe_to_resubmit', [0, 1] ); }); }); }); })->orWhere(static function ($envelope) use ($prefix): void { self::applyInvalidResolvedAgentEnvelope( $envelope, $prefix ); }); }); } /** * Keep an aggregate/catalog boundary fenced until every Agent child has a * trustworthy terminal effect. During ordinary receipt finalization the * current child is intentionally still nonterminal, so only that expected * phase/effect transition is ignored; a corrupt dispatch ledger or sticky * recovery incident on the current child still blocks publication. */ public static function aggregateParentRecoveryPending( CloudJob $parent, ?int $completingChildId = null, bool $lockForUpdate = false ): bool { $query = CloudJob::where('parent_job_id', (int) $parent->id) ->orderBy('id'); if ($lockForUpdate) { $query->lockForUpdate(); } $children = $query->get(); if (self::aggregateRelationshipConflict( $parent, $children, $completingChildId )) { return true; } return $children->contains( static fn (CloudJob $child): bool => self::aggregateChildRecoveryPending( $child, $completingChildId !== null && (int) $child->id === $completingChildId ) ); } /** A child cannot publish its parent while any safety incident is sticky. */ private static function aggregateChildRecoveryPending( CloudJob $child, bool $isCompletingChild = false ): bool { return (!$isCompletingChild && ((string) $child->phase !== 'terminal' || !in_array( (string) $child->effect_resolution, ['applied', 'not_applied'], true ))) || CloudDispatchLedger::isInvalid($child) || (bool) $child->finalization_incident || CloudDispatchLedger::hasRepairIncident($child) || self::isLocalFinalizerRecovery($child) || self::hasResolvedTerminalStateConflict($child); } public static function tasks(array $actor, string $state, ?int $cursor = null, int $limit = 20): array { if (!in_array($state, ['active', 'completed'], true)) { throw new CloudError('任务状态筛选无效', 400); } // The task center is also the owner-scoped daemon-loss fail-safe. It // persists only already-due local outcomes and never contacts Agents. CloudWorkerService::expireDueTasksForOwner( (int) ($actor['user_id'] ?? 0), null, 100 ); $limit = max(1, min(50, $limit)); $owner = static function ($query) use ($actor): void { $query->where('user_id', (int) ($actor['user_id'] ?? 0)); }; // Per-host child jobs are implementation details of one public batch. // Only their parent appears in the task center. $base = CloudJob::query()->whereNull('vpc_batch_id')->where( static function ($public): void { $public->whereNull('parent_job_id') ->orWhere(static function ($currentCatalog): void { self::whereCurrentAggregateCatalogBinding( $currentCatalog ); }); } ); $owner($base); $activeCount = (clone $base)->where('phase', '<>', 'terminal')->count(); $recovering = clone $base; self::applyRecoveringFilter($recovering); $recoveringCount = $recovering->count(); $unreadCount = (clone $base)->where('phase', 'terminal') ->whereNull('read_at')->count(); $query = clone $base; $query->where('phase', $state === 'active' ? '<>' : '=', 'terminal'); if ($cursor !== null) { $query->where('id', '<', $cursor); } $rows = $query->orderByDesc('id')->limit($limit + 1)->get(); $hasMore = $rows->count() > $limit; if ($hasMore) { $rows = $rows->take($limit); } $items = $rows->map(static fn (CloudJob $job): array => self::display($job, (bool) ($actor['is_admin'] ?? false)))->all(); $last = $rows->last(); return [ 'items' => $items, 'pagination' => [ 'next_cursor' => $hasMore && $last !== null ? (string) $last->id : null, 'has_more' => $hasMore, ], 'active_count' => (int) $activeCount, 'recovering_count' => (int) $recoveringCount, 'unread_count' => (int) $unreadCount, ]; } public static function markRead(CloudJob $job): void { if ((string) $job->phase !== 'terminal' || $job->read_at !== null) { return; } CloudJob::where('id', (int) $job->id)->where('phase', 'terminal') ->whereNull('read_at')->update(['read_at' => time()]); } public static function unreadCount(array $actor): int { $query = CloudJob::where('phase', 'terminal')->whereNull('read_at') ->whereNull('vpc_batch_id') ->where('user_id', (int) ($actor['user_id'] ?? 0)) ->where(static function ($public): void { $public->whereNull('parent_job_id') ->orWhere(static function ($currentCatalog): void { self::whereCurrentAggregateCatalogBinding( $currentCatalog ); }); }); return (int) $query->count(); } private static function requestKey(array $actor): ?string { $key = $actor['request_idempotency_key'] ?? null; if ($key === null || $key === '') { return null; } if (!is_string($key) || preg_match('/^[A-Za-z0-9_.-]{16,128}$/D', $key) !== 1) { throw new CloudError('请求幂等标识无效', 400); } return $key; } private static function canonicalRequestValue($value) { if (!is_array($value)) { return $value; } if (!array_is_list($value)) { ksort($value, SORT_STRING); } foreach ($value as $key => $item) { $value[$key] = self::canonicalRequestValue($item); } return $value; } /** * Keep only the catalogued result DTO. Historical result JSON is not a * trusted presentation format and must not bypass the error boundary. */ private static function displayResult(?array $result, bool $administrator, ?string $kind, ?string $status, bool $hasOrder, ?string $action): ?array { if ($result === null) { return null; } return CloudOperationErrorCatalog::result($result, $administrator, $kind, $status, $hasOrder, $action); } /** Resolve a trusted payment DTO from its owner-scoped durable record. */ private static function displayPaymentResult(CloudJob $job, array $result): ?array { $id = $result['payment_id'] ?? null; if (!is_string($id) || preg_match('/^[1-9][0-9]{0,18}$/D', $id) !== 1 || $job->invoice_id === null) { return null; } $payment = \App\Models\CloudInvoicePayment::where('id', (int) $id) ->where('invoice_id', (int) $job->invoice_id) ->where('user_id', (int) $job->user_id)->first(); if ($payment === null) { return null; } try { return ['payment' => CloudInvoicePaymentService::display($payment)]; } catch (\Throwable $error) { return null; } } /** * Return the non-sensitive power action from the encrypted task payload. * * The durable job payload is encrypted at rest and is never returned to * the browser. Terminal jobs retain only CloudInput::safeResult($payload), * so this lookup continues to work for historical rows while removing * credentials and all other payload fields from the public response. */ public static function action(CloudJob $job): ?string { if ((string) $job->kind !== 'action' || trim((string) $job->sealed_payload) === '') { return null; } try { $payload = CloudSecrets::decrypt( (string) $job->sealed_payload, 'job-payload' ); } catch (\Throwable $error) { return null; } $body = $payload['body'] ?? null; $action = is_array($body) && is_string($body['action'] ?? null) ? $body['action'] : ''; return CloudOperationCatalog::isPowerAction($action) ? $action : null; } /** Return the same safe labels used by every administrator DTO. */ public static function labels(CloudJob $job): array { return CloudOperationCatalog::labels((string) $job->kind, self::action($job), self::settingsFields($job)); } /** * Read only safe setting-category codes from an encrypted job payload. * * New jobs persist the codes outside the strict Agent body so terminal * redaction retains them. Jobs already active during deployment can still * be described from body key presence; completed legacy password/key jobs * have intentionally lost those keys and safely fall back to the generic * instance-settings label. */ private static function settingsFields(CloudJob $job): array { if ((string) $job->kind !== 'instance_settings' || trim((string) $job->sealed_payload) === '') { return []; } try { $payload = CloudSecrets::decrypt( (string) $job->sealed_payload, 'job-payload' ); } catch (\Throwable $error) { return []; } if (array_key_exists('settings_fields', $payload)) { $fields = $payload['settings_fields']; return is_array($fields) && array_is_list($fields) ? $fields : []; } $body = $payload['body'] ?? null; if (!is_array($body) || array_is_list($body)) { return []; } $fields = []; foreach ([ 'hostname' => 'hostname', 'root_password' => 'password', 'ssh_public_key' => 'ssh_key', ] as $bodyField => $detailCode) { if (array_key_exists($bodyField, $body)) { $fields[] = $detailCode; } } return $fields; } /** Read the user's complete instance operation history from cloud_jobs. */ public static function operationLogs(CloudInstance $instance, int $page = 1, int $perPage = 10): array { $perPage = max(1, min(50, $perPage)); $query = CloudJob::where('instance_id', $instance->id) ->whereIn('kind', CloudOperationCatalog::instanceLogKinds()) ->orderByDesc('id'); $total = max(0, (int) (clone $query)->count('id')); $pages = max(1, (int) ceil($total / $perPage)); $page = max(1, min($pages, $page)); $jobs = $query->offset(($page - 1) * $perPage)->limit($perPage) ->get(['id', 'operation_id', 'kind', 'order_id', 'status', 'parent_job_id', 'execution_mode', 'dispatch_attempts', 'phase', 'outcome', 'attempts', 'error_code', 'result_json', 'effect_may_have_applied', 'safe_to_resubmit', 'effect_resolution', 'finalization_incident', 'finalization_error_code', 'available_at', 'terminal_at', 'read_at', 'sealed_payload', 'created_at', 'updated_at']); $items = []; foreach ($jobs as $job) { $entry = self::display($job); unset($entry['operation_id'], $entry['result']); $items[] = $entry; } $from = $total === 0 ? 0 : ($page - 1) * $perPage + 1; return ['items' => $items, 'pagination' => [ 'page' => $page, 'per_page' => $perPage, 'total' => $total, 'total_pages' => $pages, 'from' => $from, 'to' => $from === 0 ? 0 : $from + count($items) - 1, ]]; } }