observedErrors)); } public function __construct(private ?CloudAgentClient $agent = null, bool $policyOnly = false, bool $vpcBatchOnly = false, ?CloudServiceCompensationExecutionService $compensations = null) { if ($policyOnly && $vpcBatchOnly) { throw new \InvalidArgumentException( 'Policy-only and VPC-batch-only modes are mutually exclusive' ); } $this->policyOnly = $policyOnly; $this->vpcBatchOnly = $vpcBatchOnly; $this->compensations = $compensations ?? new CloudServiceCompensationExecutionService(); // A cloud-image clone can legitimately take minutes on remote storage. // The layered deadlines are Agent API 840s, Agent WSS control 855s, // Gateway 870s, PHP 900s and worker lease 930s. Each outer layer has // time to persist the inner layer's structured result before expiry. } private function agentForJob(CloudJob $job): CloudAgentClient { if ($this->agent !== null) { return $this->agent; } if ((string) $job->kind === 'host_storage_configure') { return CloudAgentSettingsService::clientForConfiguration( (int) $job->host_id, self::POLICY_AGENT_TIMEOUT_SECONDS ); } return CloudAgentSettingsService::clientForHost( (int) $job->host_id, $this->policyOnly ? self::POLICY_AGENT_TIMEOUT_SECONDS : self::AGENT_TIMEOUT_SECONDS ); } public static function disposition(array $operation, bool $creating, ?string $kind = null): string { $status = is_string($operation['status'] ?? null) ? strtolower(trim($operation['status'])) : 'failed'; return $status === 'succeeded' ? 'succeeded' : ($status === 'failed' ? 'failed' : 'running'); } /** A create may refund only after both the VM and all managed artifacts are absent. */ private static function createFailureConfirmedAbsent(array $operation): bool { $result = $operation['result'] ?? null; return is_array($result) && ($result['creation_absent'] ?? false) === true && ($result['managed_artifacts_absent'] ?? false) === true; } /** Exact Agent evidence required before PHP releases a deleted VM's ledger. */ private static function deleteSuccessConfirmed(array $operation, string $kind): bool { $result = $operation['result'] ?? null; return is_array($result) && !array_is_list($result) && ($result['deleted'] ?? null) === true && ($result['managed_artifacts_absent'] ?? null) === true && ($kind === 'delete_force' ? ($result['forced'] ?? null) === true : ($result['forced'] ?? null) === false); } /** Exact evidence for a create request proven not to have crossed dispatch. */ private static function createNotDispatchedFailure(): array { return [ 'status' => 'failed', 'result' => [ 'creation_absent' => true, 'managed_artifacts_absent' => true, ], ]; } /** Treat unconfirmed reinstall cleanup as a failed receipt, never as active work. */ private static function reinstallNeedsReview(array $operation): bool { $code = is_string($operation['error_code'] ?? null) ? strtolower(trim($operation['error_code'])) : ''; if ($code === '' && is_array($operation['error'] ?? null) && is_string($operation['error']['code'] ?? null)) { $code = strtolower(trim($operation['error']['code'])); } if (in_array($code, self::REINSTALL_REVIEW_CODES, true)) { return true; } $result = $operation['result'] ?? null; return is_array($result) && ($result['cleanup_pending'] ?? false) === true; } public function tick(): bool { $this->observedErrors = []; $quarantined = self::quarantineInvalidExecutionModeJob(); $expired = self::expireDueJobs(); // Deadline convergence is always first. Recovery inspection runs in // the dedicated --recovery-only worker, so this worker never waits on // an offline recovery target before dispatching ordinary work. $jobWorked = !$quarantined && $this->tickJob(); $compensationWorked = false; $backupPreparationWorked = false; $templateCleanup = false; $templateTemporaryCleanup = false; if (!$this->policyOnly && !$this->vpcBatchOnly) { try { // Backup intent commits before any provider I/O. Resume at // most one missing multipart-plan/job binding per tick, after // already accepted Agent work had the first dispatch chance. $backupPreparationWorked = (new CloudBackupPublicationPreparation())->resumeOne(); } catch (\Throwable $error) { $this->observedErrors[] = 'backup_publication_preparation_failed'; } try { // This is a PHP-only ledger mutation. The execution service // owns its bounded lease and idempotency evidence; the worker // advances at most one item and never replays an Agent call. $compensationWorked = $this->compensations->tick(); } catch (\Throwable $error) { // Ordinary Agent work already advanced above and remains // independent from a temporarily unavailable local ledger. $this->observedErrors[] = 'service_compensation_failed'; } try { $templateCleanup = (new CloudTemplateImportCancellation()) ->cleanupOneExpired(); } catch (\Throwable $error) { // The intent has already moved to source_unknown when an S3 // abort cannot be proven. Surface one worker observation and // never loop a destructive provider request blindly. $this->observedErrors[] = 'template_import_cleanup_unresolved'; } try { $templateTemporaryCleanup = CloudTemplateRemoteFetcher::cleanupOneStaleFile(); } catch (\Throwable $error) { $this->observedErrors[] = 'template_import_temporary_cleanup_failed'; } } return $quarantined || $jobWorked || $compensationWorked || $backupPreparationWorked || $templateCleanup || $templateTemporaryCleanup || $expired > 0; } /** * Fail closed when a durable job's execution-domain discriminator is * damaged. Reinterpreting a local row as an Agent mutation (or an * aggregate parent as a leaf) could replay the original operation. Keep * the immutable dispatch evidence and encrypted payload untouched, publish * one sticky PHP-only incident, and let the recovery worker recheck it * with bounded leases forever without contacting the Agent. */ private static function quarantineInvalidExecutionModeJob(): bool { return (new CloudJob())->getConnection()->transaction( static function (): bool { $job = CloudJob::where('phase', '<>', 'terminal') ->where(static function ($invalid): void { $invalid->where(static function ($leaf): void { $leaf->whereIn( 'kind', CloudOperationRecoveryPolicy::ledgerFinalizerKinds() )->where(static function ($mode): void { $mode->whereNull('execution_mode') ->orWhere('execution_mode', '<>', 'agent'); }); })->orWhere(static function ($aggregate): void { $aggregate->whereIn( 'kind', CloudJobService::aggregateKinds() )->where(static function ($mode): void { $mode->whereNull('execution_mode') ->orWhere( 'execution_mode', '<>', 'aggregate' ); }); }); }) ->orderBy('id') ->lockForUpdate() ->first(); if ($job === null) { return false; } $now = time(); $aggregate = CloudJobService::isAggregateKind( (string) $job->kind ); $job->phase = 'terminal'; $job->outcome = 'failed'; $job->status = 'failed'; $job->terminal_at = $now; $job->effect_may_have_applied = 1; $job->safe_to_resubmit = 0; $job->effect_resolution = 'unknown'; $job->error_code = $aggregate ? 'recovery_aggregate_state_conflict' : 'recovery_instance_state_conflict'; $job->finalization_incident = 1; $job->finalization_error_code = $job->error_code; $job->read_at = null; $job->lease_token = null; $job->lease_until = 0; $job->available_at = $now + 300; $job->updated_at = $now; $job->save(); return true; }, 3 ); } /** Claim and advance at most one durable cloud job. */ private function tickJob(): bool { $job = (new CloudJob())->getConnection()->transaction(function () { $now = time(); $query = self::dispatchableBeforeDeadline( CloudJob::where('phase', 'accepted') ->where(static function ($mode): void { $mode->where('execution_mode', 'agent') ->orWhere(static function ($local): void { $local->where('execution_mode', 'local') ->where('kind', 'payment_create'); }); }) ->where('dispatch_attempts', 0), $now )->where('available_at', '<=', $now) ->where('lease_until', '<=', $now); CloudJobService::applyVpcHostDispatchFence($query); if ($this->policyOnly) { $query->whereIn('kind', self::POLICY_KINDS); } elseif ($this->vpcBatchOnly) { $query->whereNotNull('vpc_batch_id') ->whereIn('kind', self::VPC_BATCH_KINDS); } else { // The regular worker remains a fail-safe policy consumer when // the dedicated workers are not installed, but always claims // target-state policy ahead of slow image/provisioning jobs. $quoted = "'" . implode("','", self::POLICY_KINDS) . "'"; $query->orderByRaw("CASE WHEN kind IN (" . $quoted . ") THEN 0 WHEN kind = 'expiry_delete' THEN 1 ELSE 2 END"); } $job = $query->orderBy('id')->lockForUpdate()->first(); if ($job === null) { return null; } $job->lease_token = bin2hex(random_bytes(16)); $job->lease_until = $now + (in_array($job->kind, self::POLICY_KINDS, true) ? 150 : self::LEASE_SECONDS); $job->save(); return $job; }, 3); if ($job === null) { return false; } $payload = []; $dispatched = false; try { $payload = CloudSecrets::decrypt($job->sealed_payload, 'job-payload'); if ((string) $job->kind === 'payment_create') { try { $this->runPaymentCreate($job, $payload, $dispatched); } catch (\Throwable $error) { $this->observedErrors[] = 'payment_task_failed'; $crossed = $dispatched; if (!$crossed) { $fresh = CloudJob::find((int) $job->id); $crossed = $fresh !== null && CloudDispatchLedger::mayHaveDispatched($fresh); } self::forceTerminalFailure( (int) $job->id, (string) $job->lease_token, $crossed, !$crossed, 'payment_finalization_failed' ); error_log('cloud payment task ' . (int) $job->id . ' failed: ' . get_class($error)); } return true; } if (!is_string($payload['method'] ?? null) || !in_array(strtoupper($payload['method']), ['POST', 'PUT', 'PATCH', 'DELETE'], true) || !is_string($payload['request_target'] ?? null) || !str_starts_with($payload['request_target'], '/v1/') || !is_array($payload['body'] ?? null)) { throw new CloudError('Agent 任务请求格式无效', 409); } $policyKind = in_array($job->kind, self::POLICY_KINDS, true); $systemPolicy = ($payload['system_policy'] ?? false) === true && $policyKind && self::validSystemPolicy($job, $payload); $systemCleanup = $job->kind === 'expiry_delete' && ($payload['system_cleanup'] ?? false) === true && self::validSystemCleanup($job, $payload); $vpcSystem = CloudVpcPortService::isSystemJob($job, $payload); if (($policyKind && !$systemPolicy) || ($job->kind === 'expiry_delete' && !$systemCleanup)) { throw new CloudError('系统策略任务与实例状态不匹配', 409); } $actor = ['user_id' => (int) $job->user_id, 'is_admin' => false, 'is_super_admin' => false, 'can_manage' => false, 'session_id' => '', 'operation_id' => $job->operation_id]; $user = User::find($job->user_id); if (!$systemPolicy && !$systemCleanup && !$vpcSystem && ($user === null || (int) $user->enable !== 1)) { $operation = CloudOperationRecoveryPolicy::isCreateLike( (string) $job->kind ) ? self::createNotDispatchedFailure() : ['status' => 'failed']; $operation['effect_may_have_applied'] = false; $operation['safe_to_resubmit'] = true; $this->finish($job, $operation, $payload); return true; } if ($job->kind === 'create') { self::assertCreateTrafficPoolSnapshot($job); $actor['can_provision'] = true; $actor['can_manage'] = true; } elseif ($vpcSystem) { // Host-route projections are PHP-derived control-plane work. // The Agent keeps its host-global VPC endpoint behind the // super-administrator boundary, while the job remains owned // by the tenant whose accepted port mutation it completes. $actor['is_admin'] = true; $actor['is_super_admin'] = true; $actor['can_manage'] = true; } elseif ($systemPolicy || $systemCleanup) { $actor['can_manage'] = true; if ($systemCleanup) { $actor['system_cleanup'] = true; } } elseif (($payload['can_manage'] ?? false) === true && $user !== null && (bool) $user->is_admin && in_array((int) $user->id, array_map('intval', explode(',', (string) \App\Services\Config::get('super_admin'))), true)) { $actor['is_admin'] = $actor['is_super_admin'] = $actor['can_manage'] = true; } elseif ((int) ($payload['owner_id'] ?? 0) === (int) $job->user_id) { $actor['can_manage'] = true; } if ($job->kind !== 'create' && !$actor['can_manage']) { $this->finish($job, ['status' => 'failed', 'effect_may_have_applied' => false, 'safe_to_resubmit' => true], $payload); return true; } if (!CloudFirewallService::assertJobRunnable($job, $payload)) { // A confirmed instance deletion dominates an overlapping // tenant firewall request. Close the obsolete job without // touching the deleted policy or contacting the Agent. $this->finish($job, [ 'operation_id' => (string) $job->operation_id, 'status' => 'failed', 'error_code' => 'firewall_apply_superseded', 'error' => '实例已删除,防火墙请求不再适用', ], $payload); return true; } CloudVpcPortService::assertJobRunnable($job, $payload); if ($systemPolicy && $job->instance_id !== null) { $instance = CloudInstance::find($job->instance_id); $policyMarker = in_array((string) $job->kind, ['network_suspend', 'network_resume'], true) ? CloudInstanceNetworkPolicy::payloadMarker( $payload, (int) $job->instance_id, (string) $job->operation_id ) : null; $desired = $policyMarker === null ? self::policyDesired($instance, (string) $job->kind) : CloudInstanceNetworkPolicy::jobStillDesired( (int) $job->instance_id, (string) $job->kind, (string) $job->operation_id ); if (!$desired) { $this->finish($job, ['status' => 'succeeded', 'result' => ['skipped' => true]], $payload); return true; } } if (!$systemPolicy && !$systemCleanup && !$vpcSystem && $job->kind !== 'create' && $job->instance_id !== null) { $instance = CloudInstance::find($job->instance_id); $policyTransition = $instance !== null && (in_array((string) $instance->service_status, ['suspending_expired', 'resume_pending', 'resuming', 'suspend_failed', 'resume_failed', 'expired_grace', 'purge_stopping', 'purge_stop_failed', 'purge_force_required', 'purge_deleting', 'purge_failed'], true) || in_array((string) $instance->network_status, ['blocking', 'blocking_expired', 'blocked_expired', 'restore_pending', 'restoring', 'block_failed', 'restore_failed'], true)); $globalOverride = $actor['is_super_admin'] && $actor['can_manage']; if ($instance === null || ($job->kind !== 'delete' && !$globalOverride && ($policyTransition || !$instance->serviceAvailable()))) { $this->finish($job, ['status' => 'failed', 'error' => '实例服务已到期或正在执行策略,不能执行该操作'], $payload); return true; } } if ((string) $job->kind === 'network_change' && $job->public_network_transition_id === null) { throw new CloudError('旧公网网络写任务已停用', 409); } elseif ((string) $job->kind === 'backup_restore') { self::assertPortableRestoreDispatchReady($job, $payload); } elseif (str_starts_with( (string) $job->kind, 'instance_migration_' )) { // Canonical PeerMigration rows carry an exact recoverable // identity. Reject stale scalar-IP or torn ledger jobs before // the dispatch CAS so they can never contact an Agent. This // must validate the live request directly: expected() also // accepts a post-effect recovery escrow, which is never a // valid shape for an accepted, not-yet-dispatched job. CloudOperationRecoveryIdentity::expectedFromPayload( $job, $payload, false ); } elseif (in_array( (string) $job->kind, ['delete', 'delete_force', 'expiry_delete'], true ) && $job->instance_id !== null) { $deleteInstance = CloudInstance::find( (int) $job->instance_id ); if ($deleteInstance !== null) { CloudInstanceResourceRelease:: assertCanDispatchDelete($deleteInstance); } } $wirePayload = $payload; $dispatchPayload = $payload; $dispatchSealedPayload = null; if ((string) $job->kind === 'backup_restore') { [$dispatchPayload, $wirePayload] = self::portableRestoreDispatchPayload($job, $payload); $dispatchSealedPayload = CloudSecrets::encrypt( $dispatchPayload, 'job-payload' ); } $now = time(); $dispatchClaim = self::dispatchableBeforeDeadline( CloudJob::where('id', $job->id) ->where('lease_token', $job->lease_token) ->where('phase', 'accepted') ->where('dispatch_attempts', 0), $now ); $dispatchUpdate = [ 'phase' => 'dispatching', 'status' => 'running', 'attempts' => 1, 'dispatch_attempts' => 1, 'dispatch_started_at' => $now, 'dispatched_at' => $now, 'receipt_deadline_at' => $now + CloudJobService::receiptDeadlineSecondsForJob($job), 'updated_at' => $now, ]; if ($dispatchSealedPayload !== null) { // Persist only the fingerprint of the exact one-shot wire // request. The presigned capability itself exists solely in // $wirePayload and can never become durable job state. $dispatchUpdate['sealed_payload'] = $dispatchSealedPayload; } $updated = $dispatchClaim->update($dispatchUpdate); if ($updated !== 1) { return true; } // Keep the claimed model aligned with the successful CAS. The // synchronous response is finalized through this same object; a // stale dispatch_attempts=0 value would incorrectly classify a // real Agent response as a local pre-dispatch failure and omit its // immutable identity from the terminal payload. $job->phase = 'dispatching'; $job->status = 'running'; $job->attempts = 1; $job->dispatch_attempts = 1; $job->dispatch_started_at = $now; $job->dispatched_at = $now; $job->receipt_deadline_at = $now + CloudJobService::receiptDeadlineSecondsForJob($job); if ($dispatchSealedPayload !== null) { $job->sealed_payload = $dispatchSealedPayload; $payload = $dispatchPayload; } CloudFirewallService::markJobRunning($job); CloudVpcPortService::markJobRunning($job); CloudPortableRestoreFinalizer::markDispatched($job); if ((string) $job->kind === 'network_change' && $job->public_network_transition_id !== null) { CloudPublicNetworkTransitionWorkflow::markDispatched($job); } $dispatched = true; $operation = self::operationForJob( $job, $this->agentForJob($job)->request( $payload['method'], $payload['request_target'], $actor, self::agentRequestBody((string) $job->kind, $wirePayload['body']) ) ); $operation = self::normalizeAgentOperation($operation); $agentStatus = strtolower((string) ($operation['status'] ?? '')); if (in_array($agentStatus, ['succeeded', 'failed'], true)) { $operation['effect_may_have_applied'] = $operation['effect_may_have_applied'] ?? true; $operation['safe_to_resubmit'] = $operation['safe_to_resubmit'] ?? false; $this->finish($job, $operation, $payload); } elseif (in_array($agentStatus, ['accepted', 'reserved', 'dispatching', 'pending', 'running'], true)) { CloudJob::where('id', $job->id) ->where('lease_token', $job->lease_token) ->where('phase', 'dispatching')->update([ 'phase' => 'awaiting_receipt', 'agent_accepted_at' => time(), 'lease_token' => null, 'lease_until' => 0, 'updated_at' => time(), ]); $fresh = CloudJob::find((int) $job->id); $event = $fresh === null ? null : CloudOperationCompletionService::pendingForJob($fresh); if ($event !== null) { $this->settleCompletionReceipt((int) $job->id, (string) $event->event_id); } } else { throw new CloudError('Agent 返回了无效的任务状态', 502); } } catch (\Throwable $error) { // Diagnostics deliberately contain no raw payloads, credentials or upstream tracebacks. $this->observedErrors[] = 'worker_tick_failed'; $diagnostic = self::exceptionDiagnostic($error, $dispatched, (string) $job->kind, $job->order_id !== null, self::payloadAction($payload)); $log = 'cloud worker job ' . $job->id . ' failed: ' . get_class($error); if ($error instanceof CloudError) { $log .= ' status=' . $error->httpStatus . ' message=' . $diagnostic; } error_log($log); $safe = !$dispatched || ($error instanceof CloudAgentError && ($error->safeValidationFailure || $error->safePreDispatchFailure)); $operation = $safe && CloudOperationRecoveryPolicy::isCreateLike( (string) $job->kind ) ? self::createNotDispatchedFailure() : ['status' => 'failed']; $operation['operation_id'] = (string) $job->operation_id; $operation['error_code'] = $error instanceof CloudAgentError ? $error->agentCode : 'dispatch_failed'; $operation['effect_may_have_applied'] = !$safe; $operation['safe_to_resubmit'] = $safe; try { $this->finish($job, $operation, $payload); } catch (\Throwable $finishError) { self::forceTerminalFailure((int) $job->id, (string) $job->lease_token, !$safe, $safe, 'local_finalization_failed', $safe); } } return true; } /** * Keep both the queue claim and the accepted-to-dispatching CAS behind the * same absolute deadline. The second check closes the boundary race where * an old storage task was scanned just before created_at+60 but would * otherwise cross dispatch using its former persisted 300-second value. */ private static function dispatchableBeforeDeadline($query, int $now) { $storageWindow = CloudJobService::dispatchDeadlineSeconds( 'host_storage_configure' ); return $query->where(static function ($safe): void { $safe->where(static function ($payment): void { $payment->where('execution_mode', 'local') ->where('kind', 'payment_create'); })->orWhere(static function ($agent): void { $agent->where('execution_mode', 'agent') ->whereNotExists(static function ($events): void { $events->selectRaw('1') ->from( 'cloud_agent_completion_events as dispatch_events' )->whereColumn( 'dispatch_events.host_id', 'cloud_jobs.host_id' )->whereColumn( 'dispatch_events.operation_id', 'cloud_jobs.operation_id' ); }); }); })->where('dispatch_deadline_at', '>', $now) ->where(static function ($eligible) use ( $now, $storageWindow ): void { $eligible->where('kind', '!=', 'host_storage_configure') ->orWhere('created_at', '>', $now - $storageWindow); }); } private function runPaymentCreate(CloudJob $job, array $payload, bool &$dispatched): void { $input = $payload['input'] ?? null; if ($job->invoice_id === null || !is_array($input) || !is_string($input['idempotency_key'] ?? null) || !is_string($input['method'] ?? null)) { $this->finishPaymentCreate($job, false, null, false, true, 'payment_request_invalid'); return; } $now = time(); $dispatchClaim = self::dispatchableBeforeDeadline( CloudJob::where('id', (int) $job->id) ->where('lease_token', (string) $job->lease_token) ->where('phase', 'accepted') ->where('dispatch_attempts', 0), $now ); $claimed = $dispatchClaim->update([ 'phase' => 'dispatching', 'status' => 'running', 'attempts' => 1, 'dispatch_attempts' => 1, 'dispatch_started_at' => $now, 'dispatched_at' => $now, 'receipt_deadline_at' => $now + CloudJobService::receiptDeadlineSeconds( (string) $job->kind ), 'updated_at' => $now, ]); if ($claimed !== 1) { return; } $dispatched = true; try { $invoice = CloudInvoice::where('id', (int) $job->invoice_id) ->where('user_id', (int) $job->user_id)->first(); $user = User::find((int) $job->user_id); if ($invoice === null || $user === null) { throw new CloudError('账单或账户不存在', 404); } $payment = CloudInvoicePaymentService::create( $invoice, $user, $input, (string) ($payload['client_ip'] ?? ''), (string) ($payload['user_agent'] ?? '') ); $this->finishPaymentCreate($job, true, $payment, false, false, ''); } catch (\Throwable $error) { $attempt = \App\Models\CloudInvoicePayment::where( 'user_id', (int) $job->user_id )->where('idempotency_key', (string) $input['idempotency_key']) ->first(); if ($error instanceof PaymentOrderCreationException) { $effect = $error->effectMayHaveApplied; $safe = $error->safeToResubmit; $code = $error->failureCode; } elseif ((string) $input['method'] === 'usdt') { // USDT order creation is one local database transaction; it // does not submit a non-idempotent provider mutation here. $effect = false; $safe = true; $code = 'payment_creation_failed'; } else { $status = (string) ($attempt?->status ?? ''); $effect = $attempt !== null && $status !== 'not_created'; $safe = !$effect; $code = $effect ? 'payment_creation_unknown' : 'payment_creation_failed'; } $this->finishPaymentCreate($job, false, null, $effect, $safe, $code); } } private function finishPaymentCreate(CloudJob $job, bool $success, ?array $result, bool $effect, bool $safe, string $code): void { CloudJob::where('id', (int) $job->id) ->where('lease_token', (string) $job->lease_token) ->where('phase', '<>', 'terminal')->update([ 'phase' => 'terminal', 'outcome' => $success ? 'succeeded' : 'failed', 'status' => $success ? 'succeeded' : 'failed', 'terminal_at' => time(), 'effect_may_have_applied' => $effect ? 1 : 0, 'safe_to_resubmit' => $safe ? 1 : 0, 'effect_resolution' => $success ? 'applied' : ($effect ? 'unknown' : 'not_applied'), 'error_code' => $success ? null : $code, 'result_json' => $result === null ? null : json_encode(['payment_id' => (string) ($result['id'] ?? '')], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE), 'sealed_payload' => CloudSecrets::encrypt([], 'job-payload'), 'lease_token' => null, 'lease_until' => 0, 'available_at' => 0, 'updated_at' => time(), ]); if (!$success) { $this->observedErrors[] = 'task_failed'; } } /** Expire accepted or dispatched work into a local terminal failure. */ public static function expireDueJobs(int $limit = 100): int { return self::expireDueJobsMatching(null, null, $limit); } /** * Close only one owner's public tasks when the worker daemon is absent. * This path consumes durable receipts and performs local finalization; it * never opens an Agent connection or dispatches a mutation. */ public static function expireDueTasksForOwner(int $userId, ?int $taskId = null, int $limit = 100): int { if ($userId < 1 || ($taskId !== null && $taskId < 1)) { return 0; } return self::expireDueJobsMatching($userId, $taskId, $limit); } /** * Close one already-due task from an administrator read path. * This never contacts an Agent and never dispatches a mutation. */ public static function expireDueTaskForAdmin(int $taskId): int { if ($taskId < 1) { return 0; } return self::expireDueJobsMatching(null, $taskId, 1); } private static function expireDueJobsMatching(?int $userId, ?int $taskId, int $limit): int { $now = time(); $storageDispatchWindow = CloudJobService::dispatchDeadlineSeconds( 'host_storage_configure' ); $storageWindow = CloudJobService::receiptDeadlineSeconds( 'host_storage_configure' ); $query = CloudJob::where(static function ($query) use ( $now, $storageDispatchWindow, $storageWindow ): void { $query->where(static function ($accepted) use ( $now, $storageDispatchWindow ): void { $accepted->where('phase', 'accepted') ->where(static function ($deadline) use ( $now, $storageDispatchWindow ): void { $deadline->where('dispatch_deadline_at', '<=', $now) ->orWhereNull('dispatch_deadline_at') ->orWhere(static function ($storage) use ( $now, $storageDispatchWindow ): void { $storage->where( 'kind', 'host_storage_configure' )->where('created_at', '<=', $now - $storageDispatchWindow); }); }); })->orWhere(static function ($sent) use ( $now, $storageWindow ): void { $sent->whereIn('phase', ['dispatching', 'awaiting_receipt']) ->where(static function ($deadline) use ( $now, $storageWindow ): void { $deadline->where(static function ($due) use ( $now ): void { $due->whereNotNull('receipt_deadline_at') ->where('receipt_deadline_at', '<=', $now); })->orWhereNull('receipt_deadline_at') ->orWhere(static function ($storage) use ( $now, $storageWindow ): void { $storage->where( 'kind', 'host_storage_configure' )->whereNotNull('dispatched_at') ->where('dispatched_at', '<=', $now - $storageWindow); }); }); }); })->where(static function ($claimable): void { // A VPC batch parent is a PHP-local coordination ledger. It never // crosses an Agent dispatch deadline; only its durable child // finalizers may complete it. $claimable->whereNotIn('kind', [ 'vpc_membership_batch', 'vpc_batch_instance_fence', 'public_ip_pool_replacement_batch', ]) ->orWhere('execution_mode', '<>', 'local'); }); if ($userId !== null) { // Child jobs are not publicly listed, but the owner-scoped // daemon-loss fallback must still terminalize them before their // aggregate parent can close. $query->where('user_id', $userId); } if ($taskId !== null) { $query->where('id', $taskId); } $rows = $query ->orderByRaw('CASE WHEN parent_job_id IS NULL THEN 1 ELSE 0 END') ->orderBy('id') ->limit(max(1, min(100, $limit)))->get(); $expired = 0; foreach ($rows as $row) { $dispatchConflict = CloudDispatchLedger::isInvalid($row); $effect = CloudDispatchLedger::mayHaveDispatched($row); $failureCode = $dispatchConflict ? CloudDispatchLedger::CONFLICT_CODE : ($effect ? 'completion_deadline_exceeded' : 'dispatch_deadline_exceeded'); $token = bin2hex(random_bytes(16)); $claim = CloudJob::where('id', (int) $row->id) ->where('phase', (string) $row->phase) ->where('dispatch_attempts', (int) $row->dispatch_attempts); if ($row->lease_token === null) { $claim->whereNull('lease_token'); } else { $claim->where('lease_token', (string) $row->lease_token); } // An absolute deadline wins over a stale worker lease. The CAS on // phase/attempt/token prevents two local expiry paths from owning // the same finalization. $updated = $claim->update([ 'lease_token' => $token, 'lease_until' => $now + self::LEASE_SECONDS, 'updated_at' => $now, ]); if ($updated !== 1) { continue; } $expired++; $claimed = CloudJob::find((int) $row->id); try { if ($claimed !== null && CloudDispatchLedger::isInvalid($claimed)) { self::forceTerminalFailure( (int) $row->id, $token, true, false, CloudDispatchLedger::CONFLICT_CODE ); continue; } if ($claimed !== null && $claimed->host_id !== null && CloudDispatchLedger::isDispatched($claimed)) { $event = CloudOperationCompletionService::pendingForJob( $claimed ); if ($event !== null) { (new self())->settleCompletionReceipt( (int) $claimed->id, (string) $event->event_id ); continue; } } if ($claimed !== null && (string) $claimed->execution_mode === 'local' && (string) $claimed->kind === 'payment_create') { $payload = CloudSecrets::decrypt( (string) $claimed->sealed_payload, 'job-payload' ); [$success, $result, $effect, $safe, $code] = self::expiredPaymentResult($claimed, $payload); (new self())->finishPaymentCreate( $claimed, $success, $result, $effect, $safe, $code ); continue; } if ($claimed !== null && $claimed->host_id === null && $claimed->parent_job_id === null && (string) $claimed->execution_mode === 'aggregate') { self::expireAggregate($claimed, $token); continue; } $payload = CloudSecrets::decrypt( (string) $claimed->sealed_payload, 'job-payload' ); $operation = !$effect && CloudOperationRecoveryPolicy::isCreateLike( (string) $claimed->kind ) ? self::createNotDispatchedFailure() : ['status' => 'failed']; $operation['operation_id'] = (string) $claimed->operation_id; $operation['error_code'] = $failureCode; $operation['effect_may_have_applied'] = $effect; $operation['safe_to_resubmit'] = !$effect; (new self())->finish($claimed, $operation, $payload); } catch (\Throwable $error) { self::forceTerminalFailure((int) $row->id, $token, $effect, !$effect, $failureCode); } } return $expired; } /** * Derive a local payment task's deadline result from its durable attempt. * Claiming the worker is not proof that the provider POST was sent. * This read never creates or retransmits a provider order. * * @return array{0:bool,1:?array,2:bool,3:bool,4:string} */ private static function expiredPaymentResult(CloudJob $job, array $payload): array { $key = $payload['input']['idempotency_key'] ?? null; if (!is_string($key) || preg_match('/^[A-Za-z0-9_.-]{16,128}$/D', $key) !== 1) { return [false, null, true, false, 'payment_creation_unknown']; } $attempt = \App\Models\CloudInvoicePayment::where( 'user_id', (int) $job->user_id )->where('invoice_id', (int) $job->invoice_id) ->where('idempotency_key', $key)->first(); if ($attempt === null) { return [false, null, false, true, 'payment_order_not_dispatched']; } $status = (string) $attempt->status; if (in_array($status, [ 'pending', 'failed', 'expired', 'paid_invoice', 'credited_balance', ], true)) { return [true, ['id' => (string) $attempt->id], false, false, '']; } if ($status === 'not_created') { return [false, null, false, true, 'payment_order_rejected']; } return [false, null, true, false, 'payment_creation_unknown']; } public function settleCompletionReceipt(int $jobId, string $eventId): CloudJob { $event = \App\Models\CloudAgentCompletionEvent::where( 'event_id', $eventId )->first(); if ($event === null) { throw new CloudError('完成回执不存在', 404); } $source = CloudJob::find($jobId); if ($source === null) { throw new CloudError('任务不存在', 404); } if ((string) $source->execution_mode !== 'agent' || $source->host_id === null || (int) $event->host_id !== (int) $source->host_id || !hash_equals((string) $source->operation_id, (string) $event->operation_id)) { CloudOperationCompletionService::mark($event, 'rejected'); throw new CloudError('完成回执与任务不匹配', 409); } if (CloudDispatchLedger::isInvalid($source)) { CloudOperationCompletionService::deferDispatchLedgerConflict( $source ); return CloudJob::find($jobId) ?? $source; } if ((string) $source->phase === 'terminal') { if (CloudOperationCompletionService::settledTaskWithEvent( $source, $event )) { CloudOperationCompletionService::mark($event, 'applied'); } else { CloudOperationRecoveryService::settleLateReceipt( $source, $event ); } return CloudJob::find($jobId) ?? $source; } if ((string) $event->disposition !== 'received') { throw new CloudError('完成回执已经处理', 409); } $receiptDeadline = CloudJobService::effectiveReceiptDeadline($source); if ($receiptDeadline < 1 || (int) $event->received_at >= $receiptDeadline) { // The durable receive timestamp, rather than PHP processing time, // decides the boundary. A receipt persisted before the absolute // deadline can still settle later; one received after it is audit // evidence only and can never revive or succeed the generation. $token = bin2hex(random_bytes(16)); $claim = CloudJob::where('id', $jobId) ->whereIn('phase', ['dispatching', 'awaiting_receipt']) ->where('dispatch_attempts', 1); if ($source->lease_token === null) { $claim->whereNull('lease_token'); } else { $claim->where('lease_token', (string) $source->lease_token); } $claimed = $claim->update([ 'lease_token' => $token, 'lease_until' => time() + self::LEASE_SECONDS, 'updated_at' => time(), ]); if ($claimed === 1) { self::forceTerminalFailure( $jobId, $token, true, false, 'completion_deadline_exceeded' ); } $current = CloudJob::find($jobId) ?? $source; if ((string) $current->phase === 'terminal') { CloudOperationRecoveryService::settleLateReceipt( $current, $event ); } return $current; } $operation = CloudOperationCompletionService::operation($event); $payload = CloudSecrets::decrypt((string) $source->sealed_payload, 'job-payload'); $token = bin2hex(random_bytes(16)); $claim = CloudJob::where('id', $jobId) ->whereIn('phase', ['dispatching', 'awaiting_receipt']) ->where('dispatch_attempts', 1); if ($source->lease_token === null) { $claim->whereNull('lease_token'); } else { $claim->where('lease_token', (string) $source->lease_token); } $claimed = $claim->update([ 'lease_token' => $token, 'lease_until' => time() + self::LEASE_SECONDS, 'completion_event_id' => $eventId, 'updated_at' => time(), ]); if ($claimed !== 1) { $current = CloudJob::find($jobId) ?? $source; if ((string) $current->phase === 'terminal') { if (CloudOperationCompletionService::settledTaskWithEvent( $current, $event )) { CloudOperationCompletionService::mark($event, 'applied'); } else { CloudOperationRecoveryService::settleLateReceipt( $current, $event ); } } return $current; } $job = CloudJob::find($jobId); $normalized = self::normalizeAgentOperation( self::operationForJob($job, $operation) ); try { $this->finish($job, $normalized, $payload); } catch (\Throwable $error) { $createNotApplied = CloudOperationRecoveryPolicy::isCreateLike( (string) $job->kind ) && strtolower((string) ($normalized['status'] ?? '')) === 'failed' && self::createFailureConfirmedAbsent($normalized); $safeToResubmit = $createNotApplied && ($normalized['safe_to_resubmit'] ?? false) === true; self::forceTerminalFailure( $jobId, $token, !$createNotApplied, $safeToResubmit, 'completion_finalization_failed' ); } $current = CloudJob::find($jobId) ?? $source; if (CloudOperationCompletionService::settledTaskWithEvent( $current, $event )) { CloudOperationCompletionService::mark($event, 'applied'); } elseif ((string) $current->phase === 'terminal') { CloudOperationRecoveryService::settleLateReceipt( $current, $event ); } return $current; } private static function forceTerminalFailure(int $jobId, string $token, bool $effect, bool $safe, string $code, bool $confirmedNotDispatched = false): void { $claimed = CloudJob::where('id', $jobId) ->where('lease_token', $token)->where('phase', '<>', 'terminal') ->first(); $recoverableFinalizer = $claimed !== null && (CloudOperationRecoveryPolicy::supports( (string) $claimed->kind ) || CloudOperationRecoveryPolicy::isCreateLike( (string) $claimed->kind )); $preserveRecoveryEnvelope = $recoverableFinalizer || ($claimed !== null && CloudOperationRecoveryPolicy::isCreateLike( (string) $claimed->kind )); // Preserve the opaque escrow before attempting decryption. A temporary // key/configuration failure is itself recoverable; replacing the only // encrypted request with an empty string would make the PHP-only // finalizer impossible to resume after the environment is repaired. $recoveryEnvelope = $preserveRecoveryEnvelope ? (string) $claimed->sealed_payload : ''; $incidentCode = $code; $dispatchConflict = $claimed !== null && CloudDispatchLedger::isInvalid($claimed); if ($dispatchConflict) { // A corrupt counter can never prove non-dispatch. Do not invoke a // normal success/not-applied finalizer or consume Agent evidence; // preserve the encrypted escrow so repair can resume this exact // operation without replaying it. $effect = true; $safe = false; $code = CloudDispatchLedger::CONFLICT_CODE; $recoveryEnvelope = (string) $claimed->sealed_payload; } elseif ($claimed !== null) { try { if ((string) $claimed->execution_mode === 'aggregate') { if (!self::forceAggregateFailure($claimed, $token)) { return; } } elseif ((string) $claimed->kind === 'payment_create') { [$success, $result, $paymentEffect, $paymentSafe, $paymentCode] = self::expiredPaymentResult( $claimed, ['input' => ['idempotency_key' => (string) $claimed->request_idempotency_key]] ); (new self())->finishPaymentCreate( $claimed, $success, $result, $paymentEffect, $paymentSafe, $paymentCode ); } else { try { $payload = CloudSecrets::decrypt( (string) $claimed->sealed_payload, 'job-payload' ); } catch (\Throwable $error) { throw new CloudRecoveryPayloadUnavailable( '任务加密恢复数据暂时无法读取', 409 ); } // Every explicitly registered recovery kind retains its // encrypted request until the exact Agent operation is // reconciled. The recovery worker never re-dispatches // this payload; it is needed only by PHP's existing // kind-specific ledger finalizers. if (CloudOperationRecoveryPolicy::supports( (string) $claimed->kind ) || CloudOperationRecoveryPolicy::isCreateLike( (string) $claimed->kind )) { // An exact create success receipt can finish the paid // order only while the encrypted wire request and // credential escrow remain available. Keep it across // a local finalizer crash just like policy recovery // payloads; successful convergence scrubs it. $recoveryEnvelope = (string) $claimed->sealed_payload; } else { $safePayload = CloudInput::safeResult($payload); if (CloudDispatchLedger::isDispatched($claimed) && CloudOperationRecoveryIdentity::supports( (string) $claimed->kind )) { $identity = CloudOperationRecoveryIdentity:: expectedFromPayload($claimed, $payload, false); $safePayload[ CloudOperationRecoveryIdentity::payloadKey() ] = CloudOperationRecoveryIdentity::persisted( $identity ); } $recoveryEnvelope = CloudSecrets::encrypt( $safePayload, 'job-payload' ); } (new self())->finish($claimed, [ 'operation_id' => (string) $claimed->operation_id, 'status' => 'failed', 'error_code' => $code, 'effect_may_have_applied' => $effect, 'safe_to_resubmit' => $safe, ], $payload); } $terminal = CloudJob::where('id', $jobId) ->where('phase', 'terminal')->exists(); if ($terminal) { return; } } catch (\Throwable $finalizationError) { $incidentCode = self::localFinalizationIncidentCode( $finalizationError ); error_log(sprintf( 'cloud task ledger finalization failed job_id=%d kind=%s error=%s', $jobId, (string) $claimed->kind, get_class($finalizationError) )); } } if ($claimed !== null && (string) $claimed->execution_mode === 'aggregate' && self::releaseAggregateParentWhenChildrenActive( (int) $claimed->id, $token )) { // A child can win the lease CAS between our snapshot and claim. // Never hide still-active child work behind a terminal parent; // a later bounded deadline pass will finish children first. return; } // An order-backed create must never fall through to a job-only write: // its wallet, invoice, stock and placement ledgers are one atomic // finalization unit. Ambiguous effects retain a durable placement // fence; proven no-effect failures refund and release exactly once. if ($claimed !== null && (string) $claimed->kind === 'create' && $claimed->order_id !== null) { try { CloudCreateFailureFinalizer::forceTerminalFailure( $jobId, $token, $effect, $safe, $code ); if (CloudJob::where('id', $jobId) ->where('phase', 'terminal')->exists()) { return; } } catch (\Throwable $finalizationError) { error_log(sprintf( 'cloud create ledger finalization incident job_id=%d error=%s', $jobId, get_class($finalizationError) )); } } if ($claimed !== null) { // Best-effort resource latches make the incident visible in the // affected catalog. Their own failure must never keep the public // task active; the durable finalization_incident below remains the // authoritative operator alert. try { CloudSnapshotService::markJobUncertain($claimed); } catch (\Throwable $ignored) { } try { CloudBackupService::markJobUncertain($claimed); } catch (\Throwable $ignored) { } try { CloudFirewallService::markJobUncertain($claimed); } catch (\Throwable $ignored) { } try { CloudVpcPortService::markJobUncertain($claimed); } catch (\Throwable $ignored) { } if ((string) $claimed->kind === 'network_change' && $claimed->public_network_transition_id !== null) { try { CloudPublicNetworkTransitionWorkflow::markJobUncertain( $claimed ); } catch (\Throwable $ignored) { } } } // Last-resort terminalization is deliberately business-conservative: // never refund, release or claim success after a ledger finalizer // failed. Retain a validated encrypted request only when it can support // later ledger recovery, retain every resource fence, and publish an // explicit durable incident for operators. // Registered policy kinds and an order-backed create have PHP-only // retry paths that can safely publish a known not-applied effect for // generation zero or for one exact transport-level // dispatch_state=not_dispatched proof. The immutable counter remains // one in the latter case. Aggregate/other special finalizers retain // the conservative unknown effect if their atomic fallback fails. $neverDispatched = $recoverableFinalizer && CloudDispatchLedger::isNeverDispatched($claimed); $transportNotDispatched = $recoverableFinalizer && $confirmedNotDispatched && !$effect && $safe && CloudDispatchLedger::isDispatched($claimed) && ($claimed->agent_accepted_at ?? null) === null && ($claimed->completion_event_id ?? null) === null; $notDispatched = $neverDispatched || $transportNotDispatched; $terminalUpdate = [ 'phase' => 'terminal', 'outcome' => 'failed', 'status' => 'failed', 'terminal_at' => time(), 'effect_may_have_applied' => $notDispatched ? 0 : 1, 'safe_to_resubmit' => 0, 'effect_resolution' => $notDispatched ? 'not_applied' : 'unknown', 'error_code' => $dispatchConflict ? CloudDispatchLedger::CONFLICT_CODE : 'local_finalization_failed', 'finalization_incident' => 1, 'finalization_error_code' => substr( $dispatchConflict ? CloudDispatchLedger::CONFLICT_CODE : $incidentCode, 0, 128 ), 'sealed_payload' => $recoveryEnvelope, 'lease_token' => null, 'lease_until' => 0, 'available_at' => ($dispatchConflict || $recoverableFinalizer) ? time() + 300 : 0, 'updated_at' => time(), ]; if ($transportNotDispatched) { $terminalUpdate['effect_resolution_reason'] = CloudDispatchLedger::TRANSPORT_NOT_DISPATCHED_REASON; } CloudJob::where('id', $jobId)->where('lease_token', $token) ->where('phase', '<>', 'terminal')->update($terminalUpdate); } private static function localFinalizationIncidentCode( \Throwable $error ): string { return match (true) { $error instanceof CloudRecoveryDispatchLedgerConflict => CloudDispatchLedger::CONFLICT_CODE, $error instanceof CloudRecoveryAggregateStateConflict => 'recovery_aggregate_state_conflict', $error instanceof CloudRecoveryPayloadUnavailable => 'recovery_payload_unavailable', $error instanceof CloudRecoveryOperationIdentityConflict => 'recovery_operation_identity_conflict', $error instanceof CloudRecoveryReceiptInvalid => 'recovery_receipt_invalid', $error instanceof CloudRecoveryInstanceStateConflict => 'recovery_instance_state_conflict', $error instanceof CloudError => 'recovery_ledger_conflict', default => 'recovery_internal_error', }; } /** Close aggregate children first, then atomically converge its catalog. */ private static function forceAggregateFailure(CloudJob $parent, string $token): bool { $children = CloudJob::where('parent_job_id', (int) $parent->id) ->where('phase', '<>', 'terminal')->orderBy('id')->get(); foreach ($children as $child) { $childToken = bin2hex(random_bytes(16)); $claim = CloudJob::where('id', (int) $child->id) ->where('phase', (string) $child->phase) ->where('dispatch_attempts', (int) $child->dispatch_attempts); if ($child->lease_token === null) { $claim->whereNull('lease_token'); } else { $claim->where('lease_token', (string) $child->lease_token); } if ($claim->update([ 'lease_token' => $childToken, 'lease_until' => time() + self::LEASE_SECONDS, 'updated_at' => time(), ]) === 1) { $crossed = CloudDispatchLedger::mayHaveDispatched($child); $failureCode = CloudDispatchLedger::isInvalid($child) ? CloudDispatchLedger::CONFLICT_CODE : 'aggregate_finalization_failed'; self::forceTerminalFailure( (int) $child->id, $childToken, $crossed, !$crossed, $failureCode ); } } if (self::releaseAggregateParentWhenChildrenActive( (int) $parent->id, $token )) { return false; } (new CloudJob())->getConnection()->transaction( static function () use ($parent, $token): void { $fresh = CloudJob::where('id', (int) $parent->id) ->where('lease_token', $token) ->lockForUpdate()->first(); if ($fresh === null || (string) $fresh->phase === 'terminal') { return; } CloudUserIsoService::expireAggregateLocked($fresh); CloudTemplateCatalogService::expireAggregateLocked($fresh); $fresh = CloudJob::where('id', (int) $parent->id) ->where('lease_token', $token)->lockForUpdate()->first(); if ($fresh !== null && (string) $fresh->phase !== 'terminal') { CloudJobService::completeAggregateLocked( $fresh, 'failed', true, false, 'aggregate_finalization_failed' ); } }, 3 ); return true; } /** Release the parent lease while any child can still execute. */ private static function releaseAggregateParentWhenChildrenActive( int $parentId, string $token ): bool { if (!CloudJob::where('parent_job_id', $parentId) ->where('phase', '<>', 'terminal')->exists()) { return false; } CloudJob::where('id', $parentId) ->where('lease_token', $token) ->where('phase', '<>', 'terminal')->update([ 'lease_token' => null, 'lease_until' => 0, 'updated_at' => time(), ]); return true; } /** * A parent deadline is an absolute local boundary. Close every remaining * child without contacting the Agent, let resource finalizers consume * those failures, then close any aggregate that still has no result. */ private static function expireAggregate(CloudJob $parent, string $token): void { $children = CloudJob::where('parent_job_id', (int) $parent->id) ->where('phase', '<>', 'terminal')->orderBy('id')->get(); foreach ($children as $child) { if ($child->host_id !== null && CloudDispatchLedger::isDispatched($child)) { $event = CloudOperationCompletionService::pendingForJob( $child ); if ($event !== null) { try { (new self())->settleCompletionReceipt( (int) $child->id, (string) $event->event_id ); } catch (\Throwable $ignored) { // A malformed/unapplicable receipt cannot keep the // public parent active past its absolute deadline. } $child = CloudJob::find((int) $child->id) ?? $child; if ((string) $child->phase === 'terminal') { continue; } } } $childToken = bin2hex(random_bytes(16)); $claim = CloudJob::where('id', (int) $child->id) ->where('phase', (string) $child->phase) ->where('dispatch_attempts', (int) $child->dispatch_attempts); if ($child->lease_token === null) { $claim->whereNull('lease_token'); } else { $claim->where('lease_token', (string) $child->lease_token); } $claimed = $claim->update([ 'lease_token' => $childToken, 'lease_until' => time() + self::LEASE_SECONDS, 'updated_at' => time(), ]); if ($claimed !== 1) { continue; } $crossed = CloudDispatchLedger::mayHaveDispatched($child); $failureCode = CloudDispatchLedger::isInvalid($child) ? CloudDispatchLedger::CONFLICT_CODE : 'aggregate_deadline_exceeded'; self::forceTerminalFailure((int) $child->id, $childToken, $crossed, !$crossed, $failureCode); } if (CloudJob::where('parent_job_id', (int) $parent->id) ->where('phase', '<>', 'terminal')->exists()) { // A child changed concurrently after our snapshot. Keep the parent // active and release this local lease; a later bounded fallback // pass will close the child first. Never hide active child work // behind a terminal parent. CloudJob::where('id', (int) $parent->id) ->where('lease_token', $token) ->where('phase', '<>', 'terminal')->update([ 'lease_token' => null, 'lease_until' => 0, 'updated_at' => time(), ]); return; } (new CloudJob())->getConnection()->transaction( static function () use ($parent, $token): void { $fresh = CloudJob::where('id', (int) $parent->id) ->where('lease_token', $token) ->lockForUpdate()->first(); if ($fresh === null || (string) $fresh->phase === 'terminal') { return; } CloudUserIsoService::expireAggregateLocked($fresh); CloudTemplateCatalogService::expireAggregateLocked($fresh); $fresh = CloudJob::where('id', (int) $parent->id) ->where('lease_token', $token)->lockForUpdate()->first(); if ($fresh !== null && (string) $fresh->phase !== 'terminal') { CloudJobService::completeAggregateLocked( $fresh, 'failed', true, false, 'aggregate_deadline_exceeded' ); } }, 3 ); } private static function policyDesired(?CloudInstance $instance, string $kind): bool { if ($instance === null) { return false; } $status = (string) $instance->status; return match ($kind) { 'service_suspend' => in_array($status, ['active', 'deleting'], true) && (string) $instance->service_status === 'purge_stopping', 'network_suspend' => $status === 'active' && in_array((string) $instance->network_status, ['blocking', 'blocking_expired'], true), 'network_resume' => $status === 'active' && in_array((string) $instance->network_status, ['restore_pending', 'restoring'], true), default => false, }; } private static function validPreparedRecoveryMarker(string $marker): bool { if (strlen($marker) > 500) { return false; } return preg_match( '/^prepared:[a-z][a-z0-9_]{2,63}:[A-Za-z0-9_.-]{1,160}:[a-f0-9]{64}$/D', $marker ) === 1 || preg_match( '/^automatic_recovery:v1:[1-9][0-9]{0,10}:[1-9][0-9]{0,9}:prepared:(?:automatic_(?:applied|not_applied)_(?:succeeded|failed)_(?:fence|nofence):[a-f0-9]{64}|late_completion_receipt:[A-Za-z0-9_.-]{1,160}):[a-f0-9]{64}$/D', $marker ) === 1; } /** * Return the immutable prepared-evidence binding without the mutable * automatic retry counter. Older prepared envelopes stored the complete * automatic marker, while new envelopes store only this stable suffix. */ private static function preparedRecoveryBinding(string $marker): ?string { if (preg_match( '/^(?:automatic_recovery:v1:[1-9][0-9]{0,10}:' . '[1-9][0-9]{0,9}:)?' . '(prepared:[a-z][a-z0-9_]{2,63}:' . '[A-Za-z0-9_.-]{1,160}:[a-f0-9]{64})$/D', $marker, $matches ) !== 1) { return null; } return $matches[1]; } /** * Capability URLs are needed only for the one Agent dispatch. Once exact * operation identity and recovery evidence have been validated, retaining * hundreds of expired multipart URLs would make the prepared recovery * envelope unnecessarily large. The persisted request fingerprint still * binds the complete original wire request. */ private static function compactPreparedRecoveryPayload( string $kind, array $payload ): array { $identity = $payload[ CloudOperationRecoveryIdentity::payloadKey() ] ?? null; if ($kind === 'backup_create') { return [ CloudOperationRecoveryIdentity::payloadKey() => $identity, 'owner_id' => $payload['owner_id'] ?? null, 'can_manage' => $payload['can_manage'] ?? null, 'backup_key' => $payload['backup_key'] ?? null, 'maximum_size_bytes' => $payload['maximum_size_bytes'] ?? null, ]; } if ($kind === 'backup_restore') { return CloudPortableRestoreJobEnvelope::compactRecoveryPayload( $payload ); } if ($kind === 'instance_migration_prepare') { $body = is_array($payload['body'] ?? null) ? $payload['body'] : []; return [ CloudOperationRecoveryIdentity::payloadKey() => $identity, 'owner_id' => $payload['owner_id'] ?? null, 'can_manage' => $payload['can_manage'] ?? null, 'maintenance_migration_id' => $payload['maintenance_migration_id'] ?? null, 'body' => [ 'ticket' => $body['ticket'] ?? null, ], ]; } if ($kind === 'instance_migration_transfer') { $body = is_array($payload['body'] ?? null) ? $payload['body'] : []; return [ CloudOperationRecoveryIdentity::payloadKey() => $identity, 'owner_id' => $payload['owner_id'] ?? null, 'can_manage' => $payload['can_manage'] ?? null, 'reservation_id' => $payload['reservation_id'] ?? null, 'maintenance_migration_id' => $payload['maintenance_migration_id'] ?? null, 'body' => [ 'vcpu' => $body['vcpu'] ?? null, 'memory_mib' => $body['memory_mib'] ?? null, 'disk_gib' => $body['disk_gib'] ?? null, 'bandwidth_mbps' => $body['bandwidth_mbps'] ?? null, 'public_network' => $body['public_network'] ?? null, 'vpc_port' => $body['vpc_port'] ?? null, ], ]; } if ($kind === 'backup_recover') { $body = is_array($payload['body'] ?? null) ? $payload['body'] : []; return [ CloudOperationRecoveryIdentity::payloadKey() => $identity, 'owner_id' => $payload['owner_id'] ?? null, 'can_manage' => $payload['can_manage'] ?? null, 'backup_key' => $payload['backup_key'] ?? null, 'recovery_key' => $payload['recovery_key'] ?? null, 'body' => [ 'backup_key' => $body['backup_key'] ?? null, 'source_size_bytes' => $body['source_size_bytes'] ?? null, 'source_sha256' => $body['source_sha256'] ?? null, ], ]; } return $payload; } private static function lateReceiptRecoveryMarker(string $marker): bool { return preg_match( '/^(?:automatic_recovery:v1:[1-9][0-9]{0,10}:[1-9][0-9]{0,9}:)?prepared:late_completion_receipt:[A-Za-z0-9_.-]{1,160}:[a-f0-9]{64}$/D', $marker ) === 1; } private static function preparedRecoveryTransitionAllowed( string $current, string $prepared, array $operation ): bool { if ($current === '' || hash_equals($prepared, $current)) { return true; } $legacyOperationHash = CloudOperationRecoveryService:: legacyCompletionOperationHash($operation); $currentLate = []; $preparedLate = []; if ($legacyOperationHash !== null && preg_match( '/^(?:automatic_recovery:v1:[1-9][0-9]{0,10}:' . '[1-9][0-9]{0,9}:)?' . 'prepared:late_completion_receipt:' . '([A-Za-z0-9_.-]{1,160}):([a-f0-9]{64})$/D', $current, $currentLate ) === 1 && preg_match( '/^prepared:late_completion_receipt:' . '([A-Za-z0-9_.-]{1,160}):([a-f0-9]{64})$/D', $prepared, $preparedLate ) === 1 && hash_equals($currentLate[1], $preparedLate[1]) && hash_equals($currentLate[2], $legacyOperationHash) && hash_equals( $preparedLate[2], CloudOperationRecoveryService::preparedOperationHash( $operation ) )) { return true; } if (preg_match( '/^automatic_recovery:v1:[1-9][0-9]{0,10}:[1-9][0-9]{0,9}:pending$/D', $current ) === 1) { if (preg_match( '/^automatic_recovery:v1:[1-9][0-9]{0,10}:1:prepared:automatic_(applied|not_applied)_(?:succeeded|failed)_(?:fence|nofence):[a-f0-9]{64}:[a-f0-9]{64}$/D', $prepared, $next ) === 1) { return ($next[1] === 'applied' && ($operation['status'] ?? null) === 'succeeded') || ($next[1] === 'not_applied' && ($operation['status'] ?? null) === 'failed'); } return preg_match( '/^prepared:late_completion_receipt:[A-Za-z0-9_.-]{1,160}:[a-f0-9]{64}$/D', $prepared ) === 1; } if (preg_match( '/^automatic_recovery:v1:[1-9][0-9]{0,10}:' . '[1-9][0-9]{0,9}:prepared:automatic_' . '(applied|not_applied)_(succeeded|failed)' . '_(fence|nofence):[a-f0-9]{64}:([a-f0-9]{64})$/D', $current, $existing ) !== 1) { return false; } if (preg_match( '/^prepared:late_completion_receipt:[A-Za-z0-9_.-]{1,160}:([a-f0-9]{64})$/D', $prepared, $late ) !== 1) { return false; } $operationHashMatches = hash_equals($existing[4], $late[1]); if (!$operationHashMatches && $legacyOperationHash !== null) { $operationHashMatches = hash_equals( $existing[4], $legacyOperationHash ); } return $existing[3] === 'nofence' && $operationHashMatches && (($existing[1] === 'applied' && $existing[2] === 'succeeded' && ($operation['status'] ?? null) === 'succeeded') || ($existing[1] === 'not_applied' && $existing[2] === 'failed' && ($operation['status'] ?? null) === 'failed')); } private static function operationErrorCode(array $operation): string { $code = is_string($operation['error_code'] ?? null) ? strtolower(trim($operation['error_code'])) : ''; if ($code === '' && is_array($operation['error'] ?? null) && is_string($operation['error']['code'] ?? null)) { $code = strtolower(trim($operation['error']['code'])); } return preg_match('/^[a-z][a-z0-9_]{0,63}$/D', $code) === 1 ? $code : ''; } /** * Validate the common immutable job envelope used by every automatic * recovery strategy. Kind-specific business validation remains in * finish(), which is the one authoritative ledger finalizer. * * @return array{payload:array,instance:?CloudInstance} */ public static function recoveryContext(CloudJob $job): array { CloudDispatchLedger::assertValid($job); if (!CloudOperationRecoveryPolicy::supportsLedgerFinalizer( (string) $job->kind ) || (string) $job->execution_mode !== 'agent' || (string) $job->phase !== 'terminal' || (string) $job->outcome !== 'failed' || (string) $job->status !== 'failed' || (int) $job->dispatch_attempts !== 1 || !(bool) $job->effect_may_have_applied || (bool) $job->safe_to_resubmit || (string) $job->effect_resolution !== 'unknown') { throw new CloudRecoveryInstanceStateConflict( '任务不处于可自动核对状态', 409 ); } if (!is_string($job->operation_id) || preg_match('/^[a-f0-9]{32}$/D', $job->operation_id) !== 1) { throw new CloudRecoveryOperationIdentityConflict( '任务缺少合法的 Agent 操作标识', 409 ); } if ((int) $job->host_id < 1 || $job->dispatched_at === null || (int) $job->dispatched_at < 1) { throw new CloudRecoveryInstanceStateConflict( '任务缺少宿主机或派发时间', 409 ); } try { $payload = CloudSecrets::decrypt( (string) $job->sealed_payload, 'job-payload' ); } catch (\Throwable $error) { throw new CloudRecoveryPayloadUnavailable( '任务恢复数据无法解密', 409 ); } if (!is_array($payload) || array_is_list($payload)) { throw new CloudRecoveryPayloadUnavailable( '任务恢复数据结构不完整', 409 ); } $instance = null; if ($job->instance_id !== null) { $instance = CloudInstance::find((int) $job->instance_id); if ($instance === null || (int) $instance->host_id !== (int) $job->host_id || !is_string($instance->uuid) || preg_match( '/^[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/D', strtolower((string) $instance->uuid) ) !== 1) { throw new CloudRecoveryInstanceStateConflict( '任务与实例账本不匹配', 409 ); } } return compact('payload', 'instance'); } /** * Return the exact normalized operation durably stored with a prepared * recovery. This is encrypted at rest and bound to both kind and marker. */ public static function preparedRecoveryOperation(CloudJob $job): ?array { $context = self::recoveryContext($job); $envelope = $context['payload']['_operation_recovery'] ?? null; if ($envelope === null) { return null; } $operation = is_array($envelope) ? ($envelope['operation'] ?? null) : null; $storedMarker = is_array($envelope) && is_string($envelope['prepared_marker'] ?? null) ? self::preparedRecoveryBinding($envelope['prepared_marker']) : null; $currentMarker = self::preparedRecoveryBinding( (string) $job->effect_resolution_reason ); if (!is_array($envelope) || array_is_list($envelope) || array_diff(array_keys($envelope), [ 'version', 'kind', 'prepared_marker', 'operation', ]) !== [] || ($envelope['version'] ?? null) !== 1 || ($envelope['kind'] ?? null) !== (string) $job->kind || $storedMarker === null || $currentMarker === null || !hash_equals($currentMarker, $storedMarker) || !is_array($operation) || array_is_list($operation) || ($operation['operation_id'] ?? null) !== (string) $job->operation_id || !in_array($operation['status'] ?? null, ['succeeded', 'failed'], true)) { throw new CloudRecoveryPayloadUnavailable( '任务预提交恢复数据不完整', 409 ); } return $operation; } /** * Validate PHP's encrypted settings intent before recovery evidence can * affect ledgers. Both live inspection and immutable receipt recovery use * this exact contract so neither path can silently finalize an empty or * damaged escrow. * * @return array{instance:CloudInstance,payload:array,body:array,fields:array} */ public static function settingsRecoveryContext(CloudJob $job): array { if ((string) $job->kind !== 'instance_settings') { throw new CloudError('任务不处于可自动核对状态', 409); } $context = self::recoveryContext($job); $instance = $context['instance']; if ($instance === null || (string) $instance->status === 'deleted' ) { throw new CloudRecoveryInstanceStateConflict( '任务与实例账本不匹配', 409 ); } $payload = $context['payload']; $body = $payload['body'] ?? null; $allowedBody = [ 'hostname', 'root_password', 'ssh_public_key', 'confirm_restart', ]; if (!is_array($body) || array_is_list($body) || array_diff(array_keys($body), $allowedBody) !== [] || ($body['confirm_restart'] ?? null) !== true || !in_array(strtoupper((string) ($payload['method'] ?? '')), ['POST', 'PATCH'], true) || ($payload['request_target'] ?? null) !== '/v1/vms/' . rawurlencode((string) $instance->uuid) . '/settings' || (int) ($payload['owner_id'] ?? 0) !== (int) $instance->user_id || ((int) $job->user_id !== (int) $instance->user_id && ($payload['can_manage'] ?? null) !== true)) { throw new CloudError('实例设置恢复请求记录无效', 409); } $fields = []; foreach ([ 'hostname' => 'hostname', 'root_password' => 'password', 'ssh_public_key' => 'ssh_key', ] as $bodyField => $code) { if (array_key_exists($bodyField, $body)) { $fields[] = $code; } } if ($fields === [] || (array_key_exists('settings_fields', $payload) && $payload['settings_fields'] !== $fields)) { throw new CloudError('实例设置恢复字段记录无效', 409); } if (array_key_exists('hostname', $body) && CloudInput::hostname($body['hostname']) !== $body['hostname']) { throw new CloudError('实例设置恢复主机名记录无效', 409); } if (array_key_exists('root_password', $body) && (!is_string($body['root_password']) || CloudInput::password($body['root_password']) !== $body['root_password'])) { throw new CloudError('实例设置恢复密码记录无效', 409); } if (array_key_exists('ssh_public_key', $body)) { if (!is_string($body['ssh_public_key'])) { throw new CloudError('实例设置恢复 SSH 公钥记录无效', 409); } $parsedKey = CloudSSHKeys::parse($body['ssh_public_key']); if (($parsedKey['public_key'] ?? null) !== $body['ssh_public_key']) { throw new CloudError('实例设置恢复 SSH 公钥记录无效', 409); } } $credential = $payload['credential'] ?? null; if ($credential !== null) { if (!is_array($credential) || array_is_list($credential) || array_diff( array_keys($credential), ['password', 'login_name', 'auth_mode'] ) !== [] || ($credential['login_name'] ?? null) !== 'root' || !in_array( $credential['auth_mode'] ?? null, [ 'password', 'password_and_key', 'ssh_key', 'ssh_key_password', ], true )) { throw new CloudError('实例设置恢复凭据记录无效', 409); } $savedPassword = $credential['password'] ?? null; if ($savedPassword !== null && (!is_string($savedPassword) || CloudInput::password($savedPassword) !== $savedPassword)) { throw new CloudError('实例设置恢复凭据记录无效', 409); } $passwordMode = in_array( $credential['auth_mode'], ['password', 'password_and_key', 'ssh_key_password'], true ); if (($passwordMode && $savedPassword === null) || (!$passwordMode && $savedPassword !== null)) { throw new CloudError('实例设置恢复凭据记录无效', 409); } } $changesCredential = array_key_exists('root_password', $body) || array_key_exists('ssh_public_key', $body); if (!$changesCredential && $credential !== null) { throw new CloudError('实例设置恢复凭据与请求字段不匹配', 409); } if (array_key_exists('ssh_public_key', $body) && is_array($credential) && !in_array( $credential['auth_mode'], ['password_and_key', 'ssh_key', 'ssh_key_password'], true )) { throw new CloudError('实例 SSH 公钥恢复凭据模式不匹配', 409); } if (array_key_exists('root_password', $body)) { if (!is_array($credential) || ($credential['password'] ?? null) !== $body['root_password']) { throw new CloudError('实例密码恢复凭据不匹配', 409); } } return compact('instance', 'payload', 'body', 'fields'); } /** * Apply an already-validated recovery result to PHP-owned ledgers only. * The original Agent mutation is never dispatched from this path. */ public function prepareRecoveredEffect( int $jobId, array $operation, string $preparedMarker ): CloudJob { if (!self::validPreparedRecoveryMarker($preparedMarker)) { throw new CloudError('任务效果恢复标记无效', 409); } $job = CloudJob::find($jobId); if ($job === null) { throw new CloudError('任务不存在', 404); } if (!CloudOperationRecoveryPolicy::supportsLedgerFinalizer( (string) $job->kind )) { throw new CloudError('任务类型不支持自动效果恢复', 409); } $context = (string) $job->kind === 'instance_settings' ? self::settingsRecoveryContext($job) : self::recoveryContext($job); $payload = $context['payload']; $normalized = self::normalizeAgentOperation( self::operationForJob($job, $operation) ); try { $this->finish($job, $normalized, $payload, $preparedMarker); } catch (CloudRecoveryPayloadUnavailable | CloudRecoveryEvidenceConflict | CloudRecoveryReceiptInvalid | CloudRecoveryOperationIdentityConflict | CloudRecoveryAggregateStateConflict | CloudRecoveryInstanceStateConflict $error) { throw $error; } catch (CloudError $error) { if ($job->instance_id !== null) { throw new CloudRecoveryInstanceStateConflict( '实例或关联资源账本与原任务证据不一致', 409 ); } throw $error; } return CloudJob::find($jobId) ?? $job; } /** * Materialize one paid create from its exact immutable success receipt. * * This is deliberately not part of the generic recovery policy: it never * inspects or replays an Agent mutation. The original failed task remains * terminal for audit, while its create ledgers and receipt ACK commit as * one local database transaction. */ public function settleRecoveredCreateCompletion( int $jobId, \App\Models\CloudAgentCompletionEvent $event ): CloudJob { $job = CloudJob::find($jobId); if ($job === null) { throw new CloudError('任务不存在', 404); } if ((string) $job->kind !== 'create') { throw new CloudError('任务类型不支持开通回执恢复', 409); } try { $payload = CloudSecrets::decrypt( (string) $job->sealed_payload, 'job-payload' ); } catch (\Throwable $error) { throw new CloudRecoveryPayloadUnavailable( '任务加密恢复数据暂时无法读取', 409 ); } $operation = self::normalizeAgentOperation( self::operationForJob( $job, CloudOperationCompletionService::operation($event) ) ); $this->finish( $job, $operation, $payload, null, (string) $event->event_id ); return CloudJob::find($jobId) ?? $job; } /** * Retry only the PHP ledger finalizer for an operation proven not sent. * The terminal task receipt remains immutable and no Agent request occurs. */ public function retryUndispatchedFinalization( int $jobId, string $leaseToken ): CloudJob { $job = CloudJob::find($jobId); if ($job === null) { throw new CloudError('任务不存在', 404); } if ((string) $job->kind === 'create') { CloudCreateFailureFinalizer::retryUndispatchedFailure( $jobId, $leaseToken ); return CloudJob::find($jobId) ?? $job; } try { $payload = CloudSecrets::decrypt( (string) $job->sealed_payload, 'job-payload' ); } catch (\Throwable $error) { throw new CloudRecoveryPayloadUnavailable( '任务恢复数据无法解密', 409 ); } if (!is_array($payload) || array_is_list($payload)) { throw new CloudRecoveryPayloadUnavailable( '任务恢复数据结构不完整', 409 ); } $operation = CloudOperationRecoveryPolicy::isCreateLike( (string) $job->kind ) ? self::createNotDispatchedFailure() : ['status' => 'failed']; $operation['operation_id'] = (string) $job->operation_id; $operation['error_code'] = 'dispatch_not_sent'; $operation['effect_may_have_applied'] = false; $operation['safe_to_resubmit'] = true; $this->finish( $job, $operation, $payload, null, null, $leaseToken ); return CloudJob::find($jobId) ?? $job; } private function finish( CloudJob $claimed, array $operation, array $payload, ?string $preparedRecoveryMarker = null, ?string $recoveredCreateEventId = null, ?string $localFinalizerToken = null ): void { if ((string) $claimed->execution_mode === 'agent') { CloudDispatchLedger::assertValid($claimed); } $operation['operation_id'] = $operation['operation_id'] ?? (string) $claimed->operation_id; $status = self::disposition( $operation, CloudOperationRecoveryPolicy::isCreateLike( (string) $claimed->kind ), (string) $claimed->kind); if (!in_array($status, ['succeeded', 'failed'], true)) { throw new CloudError('任务尚未取得终态回执', 409); } $operationIdentity = null; // The Agent triple exists only after the immutable mutation crossed // the dispatch boundary. A locally rejected, never-dispatched task is // already proven not-applied and must still reach its kind finalizer; // requiring an Agent identity there would turn damaged local input // into an unnecessary unknown-effect fence. if (CloudDispatchLedger::isDispatched($claimed) && CloudOperationRecoveryIdentity::supports( (string) $claimed->kind )) { $operationIdentity = self::operationIdentityForFinalization( $claimed, $payload, $status === 'succeeded', $preparedRecoveryMarker !== null ); $payload[CloudOperationRecoveryIdentity::payloadKey()] = CloudOperationRecoveryIdentity::persisted( $operationIdentity ); } if ($status === 'succeeded' && in_array((string) $claimed->kind, ['delete', 'expiry_delete', 'delete_force'], true) && !self::deleteSuccessConfirmed($operation, (string) $claimed->kind)) { throw new CloudError('Agent 删除回执缺少资源已清理证据', 502); } if ((string) $claimed->kind === 'reinstall' && self::reinstallNeedsReview($operation)) { throw new CloudError('Agent 重装回执未确认清理完成', 502); } if ($status === 'failed') { $this->observedErrors[] = 'task_failed'; } $forceRequired = $status === 'failed' && in_array((string) $claimed->kind, ['delete', 'expiry_delete', 'service_suspend'], true) && self::operationErrorCode($operation) === 'force_required'; $effectMayHaveApplied = $status === 'failed' && ($operation['effect_may_have_applied'] ?? false) === true; $safeToResubmit = $status === 'failed' && ($operation['safe_to_resubmit'] ?? false) === true; if ($forceRequired) { $effectMayHaveApplied = false; $safeToResubmit = false; } if ($effectMayHaveApplied) { $safeToResubmit = false; } if ($status === 'failed' && CloudOperationRecoveryPolicy::isCreateLike( (string) $claimed->kind )) { if (self::createFailureConfirmedAbsent($operation)) { // Exact absence evidence is the authority for financial and // placement rollback. safe_to_resubmit is informational only; // this generation is never dispatched again. $effectMayHaveApplied = false; } else { // Boolean transport hints alone cannot release a paid order. // Keep every resource fenced until a fresh read-only host // inspection proves both the guest and artifacts are absent. $effectMayHaveApplied = true; $safeToResubmit = false; } } // Downstream error classification must use the validated PHP safety // decision, never the Agent's original booleans. In particular an // ambiguous create must not be labelled as refunded. $operation['effect_may_have_applied'] = $effectMayHaveApplied; $operation['safe_to_resubmit'] = $safeToResubmit; $resourceStatus = $status === 'failed' && $effectMayHaveApplied ? 'uncertain' : $status; $result = $operation['result'] ?? null; CloudPortableRestoreFinalizer::recordEvidence( $claimed, $resourceStatus, $result, $payload ); if ((string) $claimed->kind === 'network_change' && $claimed->public_network_transition_id !== null) { CloudPublicNetworkTransitionWorkflow::recordEvidence( $claimed, $resourceStatus, $result, $payload, $operation, $preparedRecoveryMarker !== null ); } $aggregateParentId = (int) ($claimed->parent_job_id ?? 0); (new CloudJob())->getConnection()->transaction(function () use ($claimed, $payload, $status, $resourceStatus, $effectMayHaveApplied, $safeToResubmit, $forceRequired, $result, $operation, $preparedRecoveryMarker, $operationIdentity, $recoveredCreateEventId, $localFinalizerToken, $aggregateParentId) { // All financial transactions acquire the wallet first, avoiding wallet/order deadlocks. $user = User::where('id', $claimed->user_id)->lockForUpdate()->first(); $order = $claimed->order_id === null ? null : CloudOrder::where('id', $claimed->order_id)->lockForUpdate()->first(); // Aggregate recovery and the legacy sweep both use the same // parent -> children lock order. Never lock a leaf first and then // wait for its parent: concurrent sibling completions otherwise // form a deadlock cycle with the parent-led sweep. $aggregateParent = null; if ($aggregateParentId > 0) { $aggregateParent = CloudJob::where('id', $aggregateParentId) ->lockForUpdate()->first(); if ($aggregateParent === null) { throw new CloudRecoveryAggregateStateConflict( '任务的汇总父任务不存在', 409 ); } CloudTemplateCatalogService::lockAggregateCatalogLocked( $aggregateParent ); CloudUserIsoService::lockAggregateCatalogLocked( $aggregateParent ); } $job = CloudJob::where('id', $claimed->id)->lockForUpdate()->first(); if ($job === null) { throw new CloudError('任务不存在', 404); } if ((string) $job->execution_mode === 'agent') { CloudDispatchLedger::assertValid($job); if (CloudDispatchLedger::attempts($job) !== CloudDispatchLedger::attempts($claimed)) { throw new CloudRecoveryDispatchLedgerConflict( '任务单次派发代次在收敛期间发生变化', 409 ); } } if ((int) ($job->parent_job_id ?? 0) !== $aggregateParentId) { throw new CloudRecoveryAggregateStateConflict( '任务的汇总父级关系在收敛期间发生变化', 409 ); } $localFinalizerRecovery = $localFinalizerToken !== null; if ($localFinalizerRecovery && (!CloudOperationRecoveryPolicy::supportsLedgerFinalizer( (string) $job->kind ) || (string) $job->execution_mode !== 'agent' || (string) $job->phase !== 'terminal' || (string) $job->outcome !== 'failed' || (string) $job->status !== 'failed' || !CloudDispatchLedger:: isLocallyFinalizableNotDispatched($job) || !(bool) $job->finalization_incident || !in_array((string) $job->effect_resolution, ['unknown', 'not_applied'], true) || !is_string($job->lease_token) || !hash_equals( (string) $job->lease_token, $localFinalizerToken ))) { throw new CloudError( '任务不处于可恢复的本地收尾状态', 409 ); } if ($localFinalizerRecovery) { if (CloudDispatchLedger::hasRepairIncident($job)) { throw new CloudRecoveryInstanceStateConflict( '任务仍处于派发账本修复状态', 409 ); } CloudOperationCompletionService:: assertLocalNotDispatchedRecoveryState($job); } $recoveredCreateEvent = null; if ($recoveredCreateEventId !== null) { $recoveredCreateEvent = \App\Models\CloudAgentCompletionEvent::where( 'event_id', $recoveredCreateEventId )->lockForUpdate()->first(); if ($preparedRecoveryMarker !== null || (string) $job->kind !== 'create' || (string) $job->execution_mode !== 'agent' || (string) $job->phase !== 'terminal' || (string) $job->outcome !== 'failed' || (string) $job->status !== 'failed' || (int) $job->dispatch_attempts !== 1 || !(bool) $job->effect_may_have_applied || (bool) $job->safe_to_resubmit || (string) $job->effect_resolution !== 'unknown' || $job->order_id === null || $order === null || $operationIdentity === null || $recoveredCreateEvent === null || (string) $recoveredCreateEvent->disposition !== 'received' || (int) $recoveredCreateEvent->host_id !== (int) $job->host_id || !hash_equals( (string) $job->operation_id, (string) $recoveredCreateEvent->operation_id ) || (string) $recoveredCreateEvent->outcome !== 'succeeded' || (bool) $recoveredCreateEvent->effect_may_have_applied || (bool) $recoveredCreateEvent->safe_to_resubmit || !CloudOperationCompletionService::isExactStoredEvent( $recoveredCreateEvent )) { throw new CloudError( '开通完成回执不处于可安全收敛状态', 409 ); } $lockedIdentity = CloudOperationRecoveryIdentity:: expectedFromPayload($job, $payload, true); CloudOperationRecoveryIdentity::assertMatches( $lockedIdentity, CloudOperationCompletionService::identity( $recoveredCreateEvent ) ); } elseif ($preparedRecoveryMarker !== null) { CloudOperationRecoveryService:: assertRecoveryIncidentAllowsPrepare($job); if ((string) $job->phase !== 'terminal' || (string) $job->outcome !== 'failed' || (string) $job->status !== 'failed' || !(bool) $job->effect_may_have_applied || (bool) $job->safe_to_resubmit || (string) $job->effect_resolution !== 'unknown') { throw new CloudError('任务已不再等待效果恢复', 409); } if (!self::preparedRecoveryTransitionAllowed( (string) $job->effect_resolution_reason, $preparedRecoveryMarker, $operation )) { if (self::lateReceiptRecoveryMarker( $preparedRecoveryMarker )) { throw new CloudRecoveryEvidenceConflict( '任务已有冲突的效果恢复证据', 409 ); } throw new CloudError('任务已有不同的效果恢复证据', 409); } if (hash_equals( $preparedRecoveryMarker, (string) $job->effect_resolution_reason )) { if (self::lateReceiptRecoveryMarker( $preparedRecoveryMarker ) && in_array( (string) ($job->finalization_error_code ?? ''), [ 'recovery_limit_reached', 'recovery_state_invalid', ], true )) { $job->finalization_incident = 0; $job->finalization_error_code = null; $job->updated_at = time(); $job->save(); } return; } } elseif (!$localFinalizerRecovery && ($job->lease_token !== $claimed->lease_token || (string) $job->phase === 'terminal')) { return; } self::assertRescueResultMatchesRequest( $job, $status, $result, $payload ); $safe = is_array($result) ? CloudOperationErrorCatalog::workerResult($result) : null; $causeCode = CloudOperationErrorCatalog::agentCauseCode( $operation['cause_code'] ?? null ); if ($causeCode !== null) { $durableResult = is_array($result) && !array_is_list($result) ? $result : []; // Keep the stable cause beside the sanitized operation // evidence. It is never copied into the public error_code or // ordinary user DTOs. $durableResult['cause_code'] = $causeCode; $safe = CloudOperationErrorCatalog::workerResult($durableResult); } $catalogStatus = $resourceStatus; // Catalog/configuration mutations use the public terminal status, // never the legacy resource-level "uncertain" label. Their // replica rows and aggregate parent are settled in this same // transaction as the child job. CloudTemplateCatalogService::finishJobLocked( $job, $status, $result, $payload, $operation, $preparedRecoveryMarker !== null || $localFinalizerRecovery ); CloudUserIsoService::finishJobLocked( $job, $status, $result, $payload, $operation, $preparedRecoveryMarker !== null || $localFinalizerRecovery ); CloudRescueTemplateReplicaCoordinator::finishJobLocked( $job, $status, $result, $payload, $operation, $preparedRecoveryMarker !== null || $localFinalizerRecovery ); CloudRescueTemplateRetirementService::finishJobLocked( $job, $status, $result, $payload, $operation, $preparedRecoveryMarker !== null || $localFinalizerRecovery ); CloudHostStorageConfigurationService::finishJobLocked( $job, $status, $result, $operation ); CloudSnapshotService::finishJobLocked( $job, $catalogStatus, $result, $payload ); CloudBackupService::finishJobLocked( $job, $catalogStatus, $result, $payload, $preparedRecoveryMarker !== null ); CloudSnapshotService::finishInstanceDeleteLocked( $job, $catalogStatus ); CloudFirewallService::finishJobLocked( $job, $catalogStatus, $result, $payload, $operation ); CloudVpcPortService::finishJobLocked( $job, $catalogStatus, $result, $payload, $operation ); CloudInstanceMigrationService::finishJobLocked( $job, $catalogStatus, $result, $payload ); CloudFirewallService::finishInstanceDeleteLocked( $job, $catalogStatus ); if ($job->kind === 'instance_upgrade' && in_array($catalogStatus, ['succeeded', 'failed', 'uncertain'], true)) { CloudPlanUpgradeService::finishJobLocked( $job, $catalogStatus, $result, $payload, $user ); } elseif ($job->kind === 'network_change' && $job->public_network_transition_id !== null && in_array($catalogStatus, ['succeeded', 'failed', 'uncertain'], true)) { CloudPublicNetworkTransitionWorkflow::finishJobLocked( $job, $catalogStatus, $payload, $user, $preparedRecoveryMarker !== null, $aggregateParent ); } elseif ($status === 'succeeded' && $job->kind === 'create') { $vm = is_array($result) ? ($result['vm'] ?? $result) : []; $uuid = $vm['uuid'] ?? ''; if ($order === null || !is_string($uuid) || $operationIdentity === null || !hash_equals( (string) $operationIdentity['resource_id'], strtolower($uuid) ) || preg_match('/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/iD', $uuid) !== 1) { throw new CloudError('Agent 成功结果缺少虚拟机标识', 502); } $orderSpec = json_decode((string) $order->spec_json, true); try { $frozenPublicNetwork = CloudPublicNetworkSnapshot::fromArray( is_array($orderSpec) ? ($orderSpec['public_network'] ?? null) : null ); $requestedPublicNetwork = CloudPublicNetworkSnapshot::fromArray( is_array($payload['body'] ?? null) ? ($payload['body']['public_network'] ?? null) : null ); $confirmedPublicNetwork = CloudPublicNetworkSnapshot::fromArray( $vm['public_network'] ?? null ); } catch (\InvalidArgumentException) { throw new CloudError( 'Agent 成功结果缺少有效的双栈公网快照,必须人工核查', 502 ); } if (!hash_equals( $frozenPublicNetwork->fingerprint(), $requestedPublicNetwork->fingerprint() ) || !hash_equals( $requestedPublicNetwork->fingerprint(), $confirmedPublicNetwork->fingerprint() )) { // A completion result may describe a VM whose network was // changed out of band after dispatch. Never commit either // original address in that case; retain the whole bundle. throw new CloudError( 'Agent 成功结果与预留双栈公网不一致,必须人工核查', 502 ); } $provisionInvoice = CloudInvoice::where( 'order_id', (int) $order->id )->lockForUpdate()->first(); if (!in_array((string) $order->status, ['queued', 'failed', 'uncertain'], true) || $order->refunded_at !== null || !self::createStockLedgerIsConsistent($order) || $order->instance_id !== null || $provisionInvoice === null || (string) $provisionInvoice->purpose !== 'provision' || (string) $provisionInvoice->status !== 'paid' || $provisionInvoice->paid_at === null || (int) $provisionInvoice->order_id !== (int) $order->id || (int) $provisionInvoice->user_id !== (int) $order->user_id || (int) $provisionInvoice->plan_id !== (int) $order->plan_id || (int) $provisionInvoice->offer_id !== (int) $order->offer_id || strtoupper((string) $provisionInvoice->currency) !== strtoupper((string) $order->currency) || CloudMoney::cents((string) $provisionInvoice->amount) !== CloudMoney::cents((string) $order->amount)) { throw new CloudRecoveryInstanceStateConflict( '开通成功回执与资金、库存或资源预留账本不一致', 409 ); } $instance = CloudInstance::where('order_id', $order->id)->first(); $newInstance = $instance === null; $instance ??= new CloudInstance(); $instance->uuid = $uuid; $instance->host_id = $order->host_id; $instance->user_id = $order->user_id; $instance->order_id = $order->id; $instance->offer_id = $order->offer_id; $instance->resource_pool_id = $order->resource_pool_id; $instance->reservation_id = $order->reservation_id; $spec = $orderSpec; $templateMetadata = self::trustedTemplateMetadata($payload); $spec = array_replace($spec, $templateMetadata); $now = time(); $days = filter_var($spec['validity_days'] ?? null, FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]); $traffic = filter_var($spec['traffic_quota_bytes'] ?? null, FILTER_VALIDATE_INT, ['options' => ['min_range' => 0]]); if ($days === false || $traffic === false) { throw new CloudError('订单缺少不可变的计费周期快照', 409); } $trafficPoolBinding = CloudTrafficPoolPeriodBinding::orderSnapshot( $spec, $traffic ); $instancePrice = CloudMoney::cents( $spec['instance_price'] ?? null ); $instance->plan_id = $order->plan_id; // A duplicate completion result must not overwrite a display // name the owner changed after the first success committed. if ($newInstance) { $instance->name = $payload['body']['name']; } $instance->status = 'active'; $instance->service_status = 'active'; $instance->network_status = 'active'; $instance->expires_at = $now + $days * 86400; // Optional services are renewed from their own subscription // ledgers. Keep this field as the base compute price only. $instance->renewal_price = CloudMoney::decimal( $instancePrice ); $instance->renewal_days = $days; $instance->renewal_traffic_bytes = $traffic; $instance->traffic_quota_bytes = $traffic; $instance->traffic_download_bytes = 0; $instance->traffic_upload_bytes = 0; $instance->lifetime_download_bytes = 0; $instance->lifetime_upload_bytes = 0; $instance->traffic_synced_at = 0; // Persist only the Agent's slow-changing public metadata. // SSR pages can then render IP, operating system, login name // and actual capacity without issuing a host read // on every request. Billing/order fields remain authoritative // because the allowlist cannot overwrite them. $instanceSpec = CloudInstanceMetadata::merge($spec, $vm); if ($templateMetadata !== null) { // Agent observations cannot replace PHP's catalog // snapshot, even if an older Agent happens to return // similarly named fields. $instanceSpec = array_replace( $instanceSpec, $templateMetadata ); unset($instanceSpec['install_iso']); } // A VM is panel-managed only after the Agent's exact create // operation has returned the validated success above. $instanceSpec['public_network'] = $confirmedPublicNetwork->toArray(); $instanceSpec['managed'] = true; $instance->spec_json = json_encode( $instanceSpec, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE ); if (isset($payload['backup_metadata'])) { // The restored system disk is authoritative. The user may // have changed the guest password after provisioning, so // copying the panel's historical envelope would expose a // password that is no longer valid. $instance->sealed_credential = null; } else { $instance->sealed_credential = isset($payload['credential']) ? CloudSecrets::encrypt( $payload['credential'], 'instance-credential' ) : null; } // The paid service begins when provisioning actually succeeds, // not while an opening invoice is waiting for payment. A // duplicate success must retain the original value. if (!$instance->exists) { $instance->created_at = $now; } $instance->updated_at = time(); $instance->save(); CloudPlacementService::commit( $order, (int) $instance->id, $confirmedPublicNetwork->toArray() ); // Seed the independent runtime observation immediately after // provisioning. If the Agent returned a trusted power // state, promote it now; otherwise the background host batch // will provide the first observation without showing "未知". self::recordRuntime($instance, $vm, 'create', []); $period = CloudServicePeriod::where('order_id', $order->id)->first() ?? new CloudServicePeriod(); $period->instance_id = $instance->id; $period->user_id = $order->user_id; $period->plan_id = $order->plan_id; $period->order_id = $order->id; $period->starts_at = $now; $period->ends_at = $instance->expires_at; $period->traffic_quota_bytes = $traffic; CloudTrafficPoolPeriodBinding::bindPeriod( $period, $trafficPoolBinding['traffic_pool_group_id'], $traffic ); $period->download_base_bytes = 0; $period->upload_base_bytes = 0; $period->download_used_bytes = 0; $period->upload_used_bytes = 0; $period->status = 'active'; $period->activated_at = $now; $period->created_at = $period->created_at ?: $now; $period->updated_at = $now; $period->save(); $instance->current_period_id = $period->id; $instance->save(); $backupSubscription = CloudAutomaticBackupService::activatePurchased($instance); CloudAutomaticBackupBillingService::grantProvisionEntitlementLocked( $instance, $period, $provisionInvoice, $backupSubscription, $now ); $job->instance_id = $order->instance_id = $instance->id; $provisionInvoice->instance_id = $instance->id; $provisionInvoice->updated_at = time(); $provisionInvoice->save(); CloudInvoiceService::queueProvisionSucceeded( $provisionInvoice, $now ); $safe = $safe ?? []; $safe['id'] = (string) $instance->id; $safe['uuid'] = $uuid; } elseif ($job->kind === 'reinstall' && $job->instance_id !== null && $status === 'succeeded') { $instance = CloudInstance::where('id', $job->instance_id) ->lockForUpdate()->first(); if ($instance === null) { throw new CloudRecoveryInstanceStateConflict( '重装任务关联实例已缺失', 409 ); } $instance->sealed_credential = CloudSecrets::encrypt( $payload['credential'], 'instance-credential' ); // Once the Agent proves the replacement guest committed, // its credential and template snapshot supersede the old // guest even if deletion of the old volume still requires // operator review. The task still ends as failed. $decoded = json_decode((string) $instance->spec_json, true); if (!is_array($decoded) || array_is_list($decoded)) { throw new CloudRecoveryInstanceStateConflict( '重装任务关联实例配置账本损坏', 409 ); } $templateMetadata = self::trustedTemplateMetadata($payload); $templateId = is_array($payload['body'] ?? null) ? ($payload['body']['template_id'] ?? null) : null; if (is_string($templateId) && preg_match('/^[A-Za-z0-9][A-Za-z0-9_.-]{0,79}$/D', $templateId) === 1) { $decoded['template_id'] = $templateId; } $decoded = array_replace($decoded, $templateMetadata); unset($decoded['install_iso']); $decoded['managed'] = true; // Force one fresh capacity/IP observation while the trusted // PHP presentation snapshot remains usable. unset($decoded['agent_metadata_version']); $instance->spec_json = json_encode($decoded, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE); self::recordRuntime($instance, is_array($result) ? $result : [], (string) $job->kind, $payload); $instance->updated_at = time(); $instance->save(); } elseif ($status === 'failed' && !$effectMayHaveApplied && $job->kind === 'create' && $order !== null) { if ($user === null) { throw new CloudError('退款账户不存在', 409); } CloudCreateFailureFinalizer::refundAndReleaseLocked( $job, $user, $order, 'create_failed' ); } elseif ($forceRequired && $job->instance_id !== null) { $instance = CloudInstance::where('id', $job->instance_id) ->lockForUpdate()->first(); if ($instance === null) { throw new CloudRecoveryInstanceStateConflict( '删除任务关联实例已缺失', 409 ); } if ((string) $job->kind === 'delete' && (string) $instance->status === 'active') { // A tenant graceful-delete request deliberately leaves the // live instance unchanged when the Agent requires a new, // separately authorized force-delete generation. } elseif ((string) $instance->status === 'deleting') { $instance->service_status = 'purge_force_required'; $instance->updated_at = time(); $instance->save(); } elseif ((string) $instance->service_status !== 'purge_force_required') { throw new CloudRecoveryInstanceStateConflict( '删除任务的实例状态与强制阶段矛盾', 409 ); } } elseif ($status === 'failed' && $job->kind === 'expiry_delete' && $job->instance_id !== null) { $instance = CloudInstance::where('id', $job->instance_id) ->lockForUpdate()->first(); if ($instance !== null && (string) $instance->status !== 'deleted') { $instance->status = 'delete_failed'; $instance->service_status = 'purge_failed'; $instance->updated_at = time(); $instance->save(); } } elseif ($status === 'failed' && $job->instance_id !== null && in_array($job->kind, ['service_suspend', 'network_suspend', 'network_resume'], true)) { // Latch a visible policy failure. This generation is never // queued again automatically. $instance = CloudInstance::where('id', $job->instance_id)->lockForUpdate()->first(); if ($instance === null) { throw new CloudRecoveryInstanceStateConflict( '系统策略任务关联实例已缺失', 409 ); } $networkPolicyMarker = in_array((string) $job->kind, ['network_suspend', 'network_resume'], true) ? CloudInstanceNetworkPolicy::payloadMarker( $payload, (int) $job->instance_id, (string) $job->operation_id ) : null; if ($networkPolicyMarker !== null) { CloudInstanceNetworkPolicy::finishJobLocked( $instance, (string) $job->kind, (string) $job->operation_id, 'failed', false, $effectMayHaveApplied, time() ); } else { $expected = [ 'service_suspend' => ['suspending_expired', 'purge_stopping'], 'network_suspend' => ['blocking', 'blocking_expired'], 'network_resume' => ['restore_pending', 'restoring'], ][$job->kind]; $already = match ((string) $job->kind) { 'service_suspend' => in_array( (string) $instance->service_status, ['suspend_failed', 'purge_stop_failed'], true ), 'network_suspend' => (string) $instance->network_status === 'block_failed', 'network_resume' => (string) $instance->network_status === 'restore_failed', default => false, }; $transitioned = false; if ($job->kind === 'service_suspend' && in_array((string) $instance->service_status, $expected, true)) { if ((string) $instance->status === 'deleting' && (string) $instance->service_status === 'purge_stopping') { // Preserve the irreversible deletion latch, but do // not enqueue this failed generation again. $instance->service_status = 'purge_stop_failed'; } else { $instance->service_status = 'suspend_failed'; } $transitioned = true; } elseif ($job->kind === 'network_suspend' && in_array((string) $instance->network_status, $expected, true)) { $instance->network_status = 'block_failed'; $transitioned = true; } elseif ($job->kind === 'network_resume' && in_array((string) $instance->network_status, $expected, true)) { $instance->network_status = 'restore_failed'; $transitioned = true; } if (!$transitioned && !$already) { throw new CloudRecoveryInstanceStateConflict( '系统策略失败结论与实例当前状态矛盾', 409 ); } } $instance->updated_at = time(); $instance->save(); } elseif ($status === 'succeeded' && $job->instance_id !== null) { $instance = CloudInstance::where('id', $job->instance_id)->lockForUpdate()->first(); if ($instance === null) { throw new CloudRecoveryInstanceStateConflict( '任务成功结论的关联实例已缺失', 409 ); } $networkPolicyMarker = in_array((string) $job->kind, ['network_suspend', 'network_resume'], true) ? CloudInstanceNetworkPolicy::payloadMarker( $payload, (int) $job->instance_id, (string) $job->operation_id ) : null; if ($networkPolicyMarker !== null) { CloudInstanceNetworkPolicy::finishJobLocked( $instance, (string) $job->kind, (string) $job->operation_id, 'succeeded', is_array($result) && ($result['skipped'] ?? false) === true, false, time() ); } elseif (in_array((string) $job->kind, ['delete', 'expiry_delete', 'delete_force'], true)) { CloudVpcPortService::releaseAfterInstanceDeletedLocked( $instance ); CloudInstanceNetworkPolicy:: releaseAfterInstanceDeletedLocked($instance); $instance->status = 'deleted'; $instance->sealed_credential = null; CloudBackupSubscription::where( 'instance_id', (int) $instance->id )->update([ 'enabled' => 0, 'renew_next_period' => 0, 'updated_at' => time(), ]); CloudInstanceResourceRelease:: releaseAfterConfirmedDelete( $instance, 'instance_deleted' ); } elseif ($job->kind === 'instance_settings') { // The Agent must acknowledge the controlled seed // replacement before PHP changes any cached metadata. // Passwords and public keys are intentionally never // copied into spec_json or result_json. if (!is_array($result) || ($result['settings_changed'] ?? false) !== true) { self::throwInvalidSettingsResult( $preparedRecoveryMarker, 'Agent 成功结果缺少已确认的实例设置' ); } $requested = is_array($payload['body'] ?? null) ? $payload['body'] : []; if (array_key_exists('hostname', $requested)) { $confirmed = $result['hostname'] ?? null; if (!is_string($confirmed) || $confirmed !== $requested['hostname'] || preg_match('/^[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?(?:\.[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?)*$/D', $confirmed) !== 1 || strlen($confirmed) > 253) { self::throwInvalidSettingsResult( $preparedRecoveryMarker, 'Agent 成功结果缺少已确认的主机名' ); } } if (isset($payload['credential']) && is_array($payload['credential'])) { $instance->sealed_credential = CloudSecrets::encrypt( $payload['credential'], 'instance-credential' ); } $decoded = json_decode((string) $instance->spec_json, true); if (!is_array($decoded) || array_is_list($decoded)) { throw new CloudError('虚拟机配置记录损坏,无法保存实例设置', 409); } if (array_key_exists('hostname', $requested)) { $decoded['guest_hostname'] = $result['hostname']; } $instance->spec_json = json_encode($decoded, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE); } elseif (in_array((string) $job->kind, [ 'service_suspend', 'network_suspend', 'network_resume', ], true) && is_array($result) && ($result['skipped'] ?? false) === true) { // The desired state changed before dispatch. No remote // effect occurred, so retain the current projection and // let lifecycle derive the next generation if needed. } elseif ($job->kind === 'service_suspend' && in_array((string) $instance->service_status, array_merge( ['purge_stopping', 'purge_force_required'], $preparedRecoveryMarker === null ? [] : ['purge_stop_failed', 'suspend_failed'] ), true)) { if ((string) $instance->status === 'deleting' && in_array((string) $instance->service_status, [ 'purge_stopping', 'purge_force_required', 'purge_stop_failed', ], true)) { $instance->service_status = 'purge_deleting'; } elseif ((string) $instance->service_status === 'suspend_failed') { $instance->service_status = 'suspended_expired'; $instance->suspended_at = time(); } } elseif ($job->kind === 'service_suspend' && !in_array((string) $instance->service_status, ['suspended_expired', 'purge_deleting'], true)) { throw new CloudRecoveryInstanceStateConflict( '服务暂停成功结论与实例状态矛盾', 409 ); } elseif ($job->kind === 'network_suspend' && in_array((string) $instance->network_status, array_merge( ['blocking', 'blocking_expired'], $preparedRecoveryMarker === null ? [] : ['block_failed'] ), true)) { $instance->network_status = ($payload['suspend_reason'] ?? '') === 'expired' ? 'blocked_expired' : 'blocked_traffic'; $instance->network_suspended_at = time(); } elseif ($job->kind === 'network_suspend' && !in_array((string) $instance->network_status, ['blocked_expired', 'blocked_traffic'], true)) { throw new CloudRecoveryInstanceStateConflict( '网络暂停成功结论与实例状态矛盾', 409 ); } elseif ($job->kind === 'network_resume' && in_array((string) $instance->network_status, array_merge( [ 'restoring', 'restore_pending', 'blocking', 'blocking_expired', ], $preparedRecoveryMarker === null ? [] : ['restore_failed'] ), true)) { if (((string) $instance->service_status === 'expired_grace' && $instance->expires_at !== null && (int) $instance->expires_at <= time()) || (string) $instance->network_status === 'blocking_expired') { // Expiry may race with a previously valid resume. // The receipt proves the links are now up, so queue // a fresh suspend generation instead of publishing // a misleading active projection. $instance->network_status = 'blocking_expired'; } elseif ((string) $instance->network_status === 'blocking') { $instance->network_status = 'blocking'; } else { $instance->network_status = 'active'; } $instance->network_suspended_at = null; } elseif ($job->kind === 'network_resume' && (string) $instance->network_status !== 'active') { throw new CloudRecoveryInstanceStateConflict( '网络恢复成功结论与实例状态矛盾', 409 ); } // Successful power/reinstall/policy operations often // include the resulting VM state. Persist that bounded // observation so the next page render is immediate; // missing state is deliberately left for vm-states. self::recordRuntime($instance, is_array($result) ? $result : [], (string) $job->kind, $payload); $instance->updated_at = time(); $instance->save(); } if ($localFinalizerRecovery) { // The kind-specific projection above is the only work being // retried. Preserve the original terminal receipt audit // (phase/outcome/status/terminal_at/error/result/event) and // update only the independently recoverable effect ledger. $now = time(); $job->effect_may_have_applied = 0; $job->safe_to_resubmit = 1; $job->effect_resolution = 'not_applied'; $job->effect_resolved_at = $now; $job->effect_resolved_by = null; $job->effect_resolution_reason = 'automatic_local_not_dispatched'; $job->sealed_payload = CloudSecrets::encrypt( CloudInput::safeResult($payload), 'job-payload' ); $job->finalization_incident = 0; $job->finalization_error_code = null; $job->read_at = null; $job->lease_token = null; $job->lease_until = 0; $job->available_at = 0; $job->updated_at = $now; $job->save(); CloudJobService::reconcileAggregateParentEffectLocked( $job, null ); return; } if ($recoveredCreateEvent !== null) { $now = time(); $safePayload = CloudInput::safeResult($payload); $safePayload[CloudOperationRecoveryIdentity::payloadKey()] = CloudOperationRecoveryIdentity::persisted( $operationIdentity ); $job->completion_event_id = (string) $recoveredCreateEvent->event_id; $job->effect_resolution = 'applied'; $job->effect_resolved_at = $now; $job->effect_resolved_by = null; $job->effect_resolution_reason = 'late_completion_receipt:' . (string) $recoveredCreateEvent->event_id; $job->sealed_payload = CloudSecrets::encrypt( $safePayload, 'job-payload' ); $job->result_json = $safe === null ? null : json_encode( $safe, JSON_THROW_ON_ERROR ); $job->finalization_incident = 0; $job->finalization_error_code = null; $job->read_at = null; $job->lease_token = null; $job->lease_until = 0; $job->available_at = 0; $job->updated_at = $now; $job->save(); $order->status = 'succeeded'; $order->updated_at = $now; $order->save(); $recoveredCreateEvent->disposition = 'applied'; $recoveredCreateEvent->applied_at = $now; $recoveredCreateEvent->save(); return; } if ($preparedRecoveryMarker !== null) { // The resource projection and this marker commit atomically. // Keep the original encrypted request plus the exact validated // operation until commit runs after the matching Agent fence // has been released. This makes the prepare/fence crash window // resumable for every registered kind, not only settings. $preparedPayload = self::compactPreparedRecoveryPayload( (string) $job->kind, $payload ); $preparedBinding = self::preparedRecoveryBinding( $preparedRecoveryMarker ); if ($preparedBinding === null) { throw new CloudRecoveryPayloadUnavailable( '任务预提交恢复标记无效', 409 ); } $preparedPayload['_operation_recovery'] = [ 'version' => 1, 'kind' => (string) $job->kind, 'prepared_marker' => $preparedBinding, 'operation' => $operation, ]; $encodedPrepared = json_encode( $preparedPayload, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE ); if (strlen($encodedPrepared) > 196608) { throw new CloudRecoveryPayloadUnavailable( '任务预提交恢复数据过大', 409 ); } $job->effect_resolution_reason = $preparedRecoveryMarker; $job->sealed_payload = CloudSecrets::encrypt( $preparedPayload, 'job-payload' ); if ($forceRequired) { // This exact no-effect Agent conclusion authorizes a new, // separately audited force-delete generation. Keep the // concrete code on the immutable graceful task instead of // its earlier transport/deadline summary. $job->error_code = 'force_required'; } if (self::lateReceiptRecoveryMarker( $preparedRecoveryMarker ) && in_array( (string) ($job->finalization_error_code ?? ''), [ 'recovery_limit_reached', 'recovery_state_invalid', ], true )) { $job->finalization_incident = 0; $job->finalization_error_code = null; } $job->updated_at = time(); $job->save(); if ($order !== null) { $order->status = $status; $order->updated_at = time(); $order->save(); } return; } $job->status = $status; $job->phase = 'terminal'; $job->outcome = $status; $job->terminal_at = time(); $job->effect_may_have_applied = $effectMayHaveApplied ? 1 : 0; $job->safe_to_resubmit = $safeToResubmit ? 1 : 0; $job->effect_resolution = $status === 'succeeded' ? 'applied' : ($effectMayHaveApplied ? 'unknown' : 'not_applied'); $retainRecoveryEscrow = $status === 'failed' && $effectMayHaveApplied && (CloudOperationRecoveryPolicy::supports( (string) $job->kind ) || CloudOperationRecoveryPolicy::isCreateLike( (string) $job->kind )); if (!$retainRecoveryEscrow) { // A resolved terminal task never needs credentials again. // Unknown operations in the explicit recovery policy retain // encrypted escrow only until their effect is resolved. $safePayload = CloudInput::safeResult($payload); if ($operationIdentity !== null) { $safePayload[ CloudOperationRecoveryIdentity::payloadKey() ] = CloudOperationRecoveryIdentity::persisted( $operationIdentity ); } $job->sealed_payload = CloudSecrets::encrypt( $safePayload, 'job-payload' ); } $job->result_json = $safe === null ? null : json_encode($safe, JSON_THROW_ON_ERROR); $job->error_code = CloudOperationErrorCatalog::workerSummary( (string) $job->kind, $status, $operation, $order !== null, self::payloadAction($payload) ); $job->lease_token = null; $job->lease_until = 0; $job->available_at = 0; $job->updated_at = time(); $job->save(); // Aggregate catalog parents may be failed/unknown even though all // leaf effects are already known (for example one replica applied // and another failed before changing the host). Publish the parent // effect now as well as after a later recovery commit, otherwise a // fully-known partial aggregate would retain a permanent fence. CloudJobService::reconcileAggregateParentEffectLocked( $job, null ); if ($order !== null) { if ($job->kind === 'create' && $effectMayHaveApplied) { CloudPlacementService::markUncertain($order); } $order->status = $status; $order->updated_at = time(); $order->save(); } }, 3); if ((string) $claimed->kind === 'host_storage_configure' && $status === 'succeeded') { $this->scheduleTemplateBackfillAfterStorage( (int) $claimed->host_id ); $this->scheduleRescueBackfillAfterStorage( (int) $claimed->host_id ); } if ((string) $claimed->kind === 'rescue_catalog_validate_replica' && $status === 'succeeded') { $this->scheduleRescueDistributionAfterValidation( $claimed ); } } /** Run object-storage discovery only after the storage transaction commits. */ private function scheduleTemplateBackfillAfterStorage(int $hostId): void { try { $actor = CloudDaemonActor::resolve('cloud-template-backfill'); if ($actor === null) { throw new CloudError( '云镜像补齐缺少可用的系统管理员执行者', 503 ); } CloudTemplateHostBackfillService::scheduleReadyHost( $hostId, $actor, 'storage_ready' ); } catch (\Throwable $error) { error_log(sprintf( 'cloud template backfill host=%d trigger=storage_ready unavailable: %s', $hostId, get_class($error) )); } } /** Start all-host distribution only after first-host boot validation. */ private function scheduleRescueDistributionAfterValidation( CloudJob $job ): void { try { $replica = CloudRescueTemplateReplica::query() ->where('job_id', (int) $job->id) ->where('host_id', (int) $job->host_id)->first(); $templateId = $replica === null ? '' : (string) $replica->template_id; if (preg_match('/^[a-f0-9]{32}$/D', $templateId) !== 1) { throw new CloudError('救援 ISO 校验任务缺少目录标识', 409); } $actor = CloudDaemonActor::resolve( 'cloud-rescue-distribution' ); if ($actor === null) { throw new CloudError( '救援 ISO 分发缺少可用的系统管理员执行者', 503 ); } CloudRescueTemplateReplicaCoordinator::distribute( $templateId, $actor, 'validation_complete' ); } catch (\Throwable $error) { // The validation and immutable object promotion already committed. // The periodic reconciler resumes this idempotently. error_log(sprintf( 'cloud rescue distribution validation_job=%d deferred: %s', (int) $job->id, get_class($error) )); } } /** Rescue replicas share the same host-readiness boundary as templates. */ private function scheduleRescueBackfillAfterStorage(int $hostId): void { try { $actor = CloudDaemonActor::resolve('cloud-rescue-backfill'); if ($actor === null) { throw new CloudError( '救援 ISO 补齐缺少可用的系统管理员执行者', 503 ); } CloudRescueTemplateHostBackfillService::scheduleReadyHost( $hostId, $actor, 'storage_ready' ); } catch (\Throwable $error) { error_log(sprintf( 'cloud rescue backfill host=%d trigger=storage_ready unavailable: %s', $hostId, get_class($error) )); } } private static function throwInvalidSettingsResult( ?string $preparedRecoveryMarker, string $message ): never { if ($preparedRecoveryMarker !== null && self::lateReceiptRecoveryMarker( $preparedRecoveryMarker )) { throw new CloudRecoveryReceiptInvalid($message, 502); } throw new CloudError($message, 502); } /** Rescue success is accepted only for the exact requested generation. */ private static function assertRescueResultMatchesRequest( CloudJob $job, string $status, mixed $result, array $payload ): void { if ((string) $job->kind !== 'rescue' || $status !== 'succeeded') { return; } $target = (string) ($payload['request_target'] ?? ''); $body = is_array($payload['body'] ?? null) ? $payload['body'] : []; if (!is_array($result) || array_is_list($result) || !is_bool($result['active'] ?? null) || !is_bool($result['staged'] ?? null) || !is_bool($result['network_suspended'] ?? null)) { throw new CloudError('Agent 救援操作成功回执无效', 502); } if (str_ends_with($target, '/rescue/enter')) { if (!(($result['active'] ?? false) || ($result['staged'] ?? false)) || ($result['template_id'] ?? null) !== ($body['template_id'] ?? null) || ($result['revision'] ?? null) !== ($body['revision'] ?? null) || ($result['generation_token'] ?? null) !== ($body['generation_token'] ?? null)) { throw new CloudError( 'Agent 救援操作回执与请求版本不匹配', 502 ); } return; } if (str_ends_with($target, '/rescue/force-start')) { if (($result['active'] ?? false) !== true || !is_string($result['template_id'] ?? null) || preg_match('/^[a-f0-9]{32}$/D', $result['template_id']) !== 1 || !is_int($result['revision'] ?? null) || (int) $result['revision'] < 1 || !is_string($result['generation_token'] ?? null) || preg_match('/^[a-f0-9]{32}$/D', $result['generation_token']) !== 1) { throw new CloudError('Agent 救援强制启动回执无效', 502); } return; } if (!str_ends_with($target, '/rescue/exit') || ($result['active'] ?? true) !== false || ($result['staged'] ?? true) !== false || !in_array($result['template_id'] ?? '', ['', null], true) || !in_array($result['revision'] ?? 0, [0, null], true) || !in_array( $result['generation_token'] ?? '', ['', null], true )) { throw new CloudError('Agent 救援退出回执无效', 502); } } private static function validSystemCleanup(CloudJob $job, array $payload): bool { if ($job->instance_id === null || ($payload['body'] ?? null) !== ['remove_managed_disks' => true]) { return false; } $instance = CloudInstance::find($job->instance_id); if ($instance === null || (string) $instance->status !== 'deleting' || (string) $instance->service_status !== 'purge_deleting' || (int) $job->user_id !== (int) $instance->user_id || (int) $job->host_id !== (int) $instance->host_id || (int) ($payload['owner_id'] ?? 0) !== (int) $instance->user_id) { return false; } return ($payload['method'] ?? null) === 'DELETE' && ($payload['request_target'] ?? null) === '/v1/vms/' . rawurlencode((string) $instance->uuid); } private static function validSystemPolicy(CloudJob $job, array $payload): bool { if ($job->instance_id === null || !is_array($payload['body'] ?? null)) { return false; } $instance = CloudInstance::find($job->instance_id); if ($instance === null || (int) $job->user_id !== (int) $instance->user_id || (int) $job->host_id !== (int) $instance->host_id || (int) ($payload['owner_id'] ?? 0) !== (int) $instance->user_id) { return false; } $suffix = [ 'service_suspend' => 'service/suspend', 'network_suspend' => 'network/suspend', 'network_resume' => 'network/resume', ][$job->kind] ?? null; $expectedBody = match ((string) $job->kind) { 'service_suspend' => ['reason' => 'purge'], 'network_suspend' => ['reason' => (string) ( $payload['suspend_reason'] ?? '' )], 'network_resume' => [], default => null, }; return $suffix !== null && $expectedBody !== null && ($payload['method'] ?? null) === 'POST' && ($payload['body'] ?? null) === $expectedBody && ($payload['request_target'] ?? null) === '/v1/vms/' . rawurlencode((string) $instance->uuid) . '/' . $suffix; } private static function payloadAction(array $payload): ?string { $body = $payload['body'] ?? null; $action = is_array($body) && is_string($body['action'] ?? null) ? $body['action'] : null; return CloudOperationCatalog::isPowerAction($action ?? '') ? $action : null; } /** * Validate the immutable purchase-time stock mode before materialising an * exact Agent create success. Legacy finite orders retain their reservation * marker. A markerless legacy order is accepted as unlimited only while the * locked offer row still proves that it was unchanged before the order. */ private static function createStockLedgerIsConsistent( CloudOrder $order ): bool { $reserved = $order->stock_reserved_at !== null; $released = $order->stock_released_at !== null; $spec = json_decode((string) $order->spec_json, true); if (!is_array($spec) || array_is_list($spec)) { return false; } if (array_key_exists('stock_mode', $spec)) { if ($spec['stock_mode'] === 'finite') { return $reserved && !$released; } if ($spec['stock_mode'] === 'unlimited') { return !$reserved && !$released; } return false; } if ($reserved) { return !$released; } if ($released) { return false; } foreach ([ 'offer_id' => (int) $order->offer_id, 'plan_id' => (int) $order->plan_id, 'resource_pool_id' => (int) $order->resource_pool_id, ] as $key => $expected) { if (($spec[$key] ?? null) !== $expected) { return false; } } $offer = CloudOffer::where('id', (int) $order->offer_id) ->lockForUpdate()->first(); return $offer !== null && (int) $offer->plan_id === (int) $order->plan_id && (int) $offer->resource_pool_id === (int) $order->resource_pool_id && (int) $offer->stock === -1 && (int) $offer->created_at > 0 && (int) $offer->created_at <= (int) $offer->updated_at && (int) $offer->updated_at < (int) $order->created_at && ($spec['price_version'] ?? null) === (int) $offer->price_version; } /** * Validate the mandatory PHP-owned template snapshot without forwarding * it to the Agent. This is a clean-break schema: create/reinstall work * without the snapshot is invalid and finalizes as a local failure. */ private static function trustedTemplateMetadata(array $payload): array { if (array_key_exists('backup_metadata', $payload)) { $metadata = $payload['backup_metadata']; $body = $payload['body'] ?? null; if (!is_array($metadata) || array_is_list($metadata) || !is_array($body) || array_is_list($body) || filter_var( $metadata['backup_id'] ?? null, FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]] ) === false || preg_match('/^[a-f0-9]{32}$/D', (string) ($metadata['backup_key'] ?? '')) !== 1 || !hash_equals( (string) $metadata['backup_key'], (string) ($body['backup_key'] ?? '') ) || ($metadata['os_source'] ?? null) !== 'cloud_template' || !is_string($metadata['os_name'] ?? null) || trim($metadata['os_name']) === '' || ($metadata['login_name'] ?? null) !== 'root' || !is_bool($metadata['debian_12_network_fix'] ?? null) || ($body['debian_12_network_fix'] ?? null) !== $metadata['debian_12_network_fix']) { throw new CloudError( '任务中的云备份展示快照与 Agent 请求不匹配,必须人工核查', 409 ); } return [ 'os_source' => 'cloud_template', 'os_name' => $metadata['os_name'], 'login_name' => 'root', 'debian_12_network_fix' => $metadata['debian_12_network_fix'], 'restored_from_backup' => $metadata['backup_key'], ]; } if (!array_key_exists('template_metadata', $payload)) { throw new CloudError( '任务缺少云镜像展示快照,必须终止并人工核查', 409 ); } $metadata = $payload['template_metadata']; $body = $payload['body'] ?? null; if (!is_array($metadata) || array_is_list($metadata) || !is_array($body) || array_is_list($body) || !is_string($metadata['template_id'] ?? null) || !is_string($body['template_id'] ?? null) || preg_match('/^[A-Za-z0-9][A-Za-z0-9_.-]{0,79}$/D', $metadata['template_id']) !== 1 || !hash_equals($body['template_id'], $metadata['template_id'])) { throw new CloudError( '任务中的云镜像展示快照与 Agent 请求不匹配,必须人工核查', 409 ); } $public = CloudInstanceMetadata::publicSpec($metadata); if (($public['os_source'] ?? null) !== 'cloud_template' || !is_string($public['os_name'] ?? null) || ($public['login_name'] ?? null) !== 'root' || !is_bool($metadata['debian_12_network_fix'] ?? null)) { throw new CloudError('任务中的云镜像展示快照无效,必须人工核查', 409); } return [ 'template_id' => $metadata['template_id'], 'os_source' => 'cloud_template', 'os_name' => $public['os_name'], 'login_name' => 'root', 'debian_12_network_fix' => $metadata['debian_12_network_fix'], ]; } /** Reject retired restore jobs before the dispatch CAS can contact Agent. */ private static function assertPortableRestoreDispatchReady( CloudJob $job, array $payload ): void { $attemptKey = $payload['attempt_key'] ?? null; if (!CloudPortableRestoreFinalizer::owns($job) || !is_string($attemptKey) || preg_match('/^[a-f0-9]{32}$/D', $attemptKey) !== 1) { throw new CloudError('旧备份恢复任务已停用', 409); } } /** * Create the one transient restore request and persist only its immutable * Agent-journal identity. The URL is never returned in the stable payload, * so every terminal, prepared and unknown-effect database path remains * capability-free while still being able to reconcile the exact request. * * @return array{0:array,1:array} stable payload, wire payload */ private static function portableRestoreDispatchPayload( CloudJob $job, array $payload ): array { $envelope = CloudPortableRestoreJobEnvelope::fromJobForDispatch( $job, $payload ); $source = CloudBackupService::provisioningSource( $envelope->backup() ); $sourceUrl = $source['source_url'] ?? null; if (!is_string($sourceUrl)) { throw new CloudError('恢复任务临时下载能力缺失', 409); } $wire = $envelope->wirePayload($sourceUrl); $identity = CloudOperationRecoveryIdentity::expectedFromPayload( $job, $wire ); $stable = $envelope->payload(); $stable[CloudOperationRecoveryIdentity::payloadKey()] = CloudOperationRecoveryIdentity::persisted($identity); return [$stable, $wire]; } /** * @return array{action:string,resource_id:string,request_fingerprint:string} */ private static function operationIdentityForFinalization( CloudJob $job, array $payload, bool $forAppliedEffect, bool $preparedRecovery ): array { $portableAttempt = (string) $job->kind === 'backup_restore' && is_string($payload['attempt_key'] ?? null) && isset($payload[ CloudOperationRecoveryIdentity::payloadKey() ]); if ($preparedRecovery || $portableAttempt) { return CloudOperationRecoveryIdentity::expected( $job, $forAppliedEffect ); } return CloudOperationRecoveryIdentity::expectedFromPayload( $job, $payload, $forAppliedEffect ); } /** * Translate panel vocabulary only at the Agent wire boundary. The PHP job * keeps "poweroff" so its audit label and user-facing history remain * stable, while the Go protocol uses libvirt's explicit "destroy" action. */ private static function agentRequestBody(string $kind, array $body): array { return CloudOperationRecoveryIdentity::wireBody($kind, $body); } /** * Bind every journal-backed Agent response to the claimed PHP job before * its status or result can influence durable panel state. Missing IDs and * cross-operation responses are ambiguous, never legacy success shortcuts. */ private static function operationForJob(CloudJob $job, array $operation): array { $expected = (string) $job->operation_id; $actual = $operation['operation_id'] ?? null; if (!is_string($actual) || $expected === '' || !hash_equals($expected, $actual)) { throw new CloudError( 'Agent 操作标识与当前任务不匹配,必须人工核查', 502 ); } if (CloudOperationRecoveryIdentity::supports((string) $job->kind)) { $expectedIdentity = CloudOperationRecoveryIdentity::expected( $job, strtolower((string) ($operation['status'] ?? '')) === 'succeeded' ); $fingerprint = $operation['fingerprint'] ?? $operation['request_fingerprint'] ?? null; CloudOperationRecoveryIdentity::assertMatches( $expectedIdentity, [ 'action' => $operation['action'] ?? null, 'resource_id' => $operation['resource_id'] ?? null, 'request_fingerprint' => $fingerprint, ] ); } return $operation; } private static function normalizeAgentOperation(array $operation): array { $causeCode = CloudOperationErrorCatalog::agentCauseCode( $operation['cause_code'] ?? null ); if ($causeCode === null) { unset($operation['cause_code']); } else { $operation['cause_code'] = $causeCode; } $error = $operation['error'] ?? null; if (is_array($error) && is_string($error['code'] ?? null) && preg_match('/^[a-z][a-z0-9_]{0,63}$/D', $error['code']) === 1) { $operation['error_code'] = $error['code']; } // Never persist or expose free-form Agent error messages. The local // catalog maps the bounded code and operation kind to reviewed text. if (is_array($error)) { unset($operation['error']); } return $operation; } /** * Persist only a successful, allow-listed power observation. Runtime * cache failures are isolated from the durable job outcome: the Agent * result and billing state remain authoritative even during a rolling * schema upgrade where the cache table may not exist yet. */ private static function recordRuntime(CloudInstance $instance, array $result, string $kind, array $payload): void { try { if (in_array($kind, ['delete', 'expiry_delete', 'delete_force'], true)) { CloudInstanceRuntimeService::forget((int) $instance->id); return; } $source = is_array($result['vm'] ?? null) ? $result['vm'] : $result; $state = $source['status'] ?? ($source['state'] ?? ($source['power_state'] ?? null)); $action = $kind === 'action' ? self::payloadAction($payload) : null; $normalized = CloudInstanceRuntimeService::normalizeState($state); if ($kind === 'action') { // A successful graceful-shutdown job only confirms that the // request was accepted. libvirt may still report running (or // omit the state) until the guest completes ACPI shutdown. // Preserve that distinction instead of publishing either a // new fresh-running observation or a fabricated stopped one. $state = CloudInstanceRuntimeService::stateAfterSuccessfulAction( $state, $action ); } $instanceId = (int) $instance->id; $hostId = (int) $instance->host_id; $uuid = (string) $instance->uuid; if ($kind === 'create') { CloudInstanceRuntimeService::seed($instanceId, $hostId, $uuid); } if (CloudInstanceRuntimeService::normalizeState($state) !== 'unknown') { CloudInstanceRuntimeService::markObserved( $instanceId, $hostId, $uuid, $state ); } } catch (\Throwable $error) { error_log('cloud runtime observation write unavailable: ' . get_class($error)); } } /** Fail a malformed clean-break order before any Agent create mutation. */ private static function assertCreateTrafficPoolSnapshot(CloudJob $job): void { if ($job->order_id === null) { throw new CloudError('开通任务缺少订单', 409); } $order = CloudOrder::find((int) $job->order_id); $spec = $order === null ? null : json_decode((string) $order->spec_json, true); if (!is_array($spec) || array_is_list($spec)) { throw new CloudError('订单计费快照损坏', 409); } $quota = filter_var( $spec['traffic_quota_bytes'] ?? null, FILTER_VALIDATE_INT, ['options' => ['min_range' => 0]] ); if ($quota === false) { throw new CloudError('订单缺少不可变的计费周期快照', 409); } CloudTrafficPoolPeriodBinding::orderSnapshot($spec, (int) $quota); } private static function exceptionDiagnostic(\Throwable $error, bool $dispatched, ?string $kind, bool $hasOrder, ?string $action): string { $operation = []; if ($error instanceof CloudError) { // CloudError text is Agent-/host-controlled too. The catalog // keeps only exact reviewed details and otherwise returns a // stable generic code for the durable job row. $operation['error'] = $error->getMessage(); } elseif ($error instanceof \GuzzleHttp\Exception\TransferException) { $operation['error'] = 'Agent 连接中断或操作超时'; } $status = 'failed'; return CloudOperationErrorCatalog::workerSummary($kind, $status, $operation, $hasOrder, $action) ?? CloudOperationErrorCatalog::userMessage($kind, $status, null, $hasOrder, $action) ?? '虚拟机操作失败,请稍后重试'; } }