observedErrors)); } public function __construct(private ?CloudEngineClient $engine = null, bool $policyOnly = false) { $this->policyOnly = $policyOnly; // A cloud-image clone can legitimately take minutes on remote storage. // Keep the worker lease longer than the HTTP timeout so another worker // cannot claim the same job while the first request is still active. $this->engine ??= new CloudEngineClient(null, $policyOnly ? self::POLICY_ENGINE_TIMEOUT_SECONDS : self::ENGINE_TIMEOUT_SECONDS); } public static function disposition(array $operation, bool $creating, ?string $kind = null): string { $status = is_string($operation['status'] ?? null) ? strtolower(trim($operation['status'])) : 'uncertain'; if ($kind === 'reinstall' && in_array($status, ['succeeded', 'completed', 'done', 'failed'], true) && self::reinstallNeedsReview($operation)) { // A legacy/internal response can carry a post-commit marker while // its top-level status is still failed. Keep it uncertain so the // replacement credential/spec can be reconciled without allowing // a destructive reinstall replay. return 'uncertain'; } if (in_array($status, ['succeeded', 'completed', 'done'], true)) { return 'succeeded'; } if ($status === 'failed') { if (!$creating || ($operation['safe_to_refund'] ?? false) === true || ($operation['result']['creation_absent'] ?? false) === true) { return 'failed'; } return 'uncertain'; } return in_array($status, ['pending', 'running'], true) ? 'running' : 'uncertain'; } /** * A reinstall can commit the replacement guest before old-volume cleanup * is confirmed. Preserve the new credential/spec while keeping the job * visible as uncertain; legacy engines may only provide a warning, so * that diagnostic is accepted as the compatibility signal here. */ private static function reinstallNeedsReview(array $operation): bool { $code = is_string($operation['error_code'] ?? null) ? strtolower(trim($operation['error_code'])) : ''; if (in_array($code, self::REINSTALL_REVIEW_CODES, true)) { return true; } $result = $operation['result'] ?? null; return is_array($result) && (($result['reinstall_committed'] ?? false) === true || ($result['cleanup_pending'] ?? false) === true || (is_string($result['warning'] ?? null) && trim($result['warning']) !== '')); } public function tick(): bool { $this->observedErrors = []; $job = (new CloudJob())->getConnection()->transaction(function () { $query = CloudJob::whereIn('status', ['queued', 'running', 'uncertain']) ->where('available_at', '<=', time())->where('lease_until', '<=', time()); if ($this->policyOnly) { $query->whereIn('kind', self::POLICY_KINDS); } else { // The regular worker remains a fail-safe policy consumer when // the dedicated workers are not installed, but always claims // target-state policy ahead of slow image/provisioning jobs. $quoted = "'" . implode("','", self::POLICY_KINDS) . "'"; $query->orderByRaw("CASE WHEN kind IN (" . $quoted . ") THEN 0 WHEN kind = 'expiry_delete' THEN 1 ELSE 2 END"); } $job = $query->orderBy('id')->lockForUpdate()->first(); if ($job === null) { return null; } $job->lease_token = bin2hex(random_bytes(16)); $job->lease_until = time() + (in_array($job->kind, self::POLICY_KINDS, true) ? 150 : self::LEASE_SECONDS); $job->attempts++; $job->save(); return $job; }, 3); if ($job === null) { return false; } $payload = []; $dispatched = $job->dispatched_at !== null; try { $payload = CloudSecrets::decrypt($job->payload_cipher); $policyKind = in_array($job->kind, self::POLICY_KINDS, true); $systemPolicy = ($payload['system_policy'] ?? false) === true && $policyKind && self::validSystemPolicy($job, $payload); // Expiry deletion is authorized by the server-side lifecycle // daemon, but remains a destructive one-shot mutation. Keep it // separate from replayable target-state policy so a lost response // can never cause PHP to submit DELETE a second time. $systemCleanup = $job->kind === 'expiry_delete' && ($payload['system_cleanup'] ?? false) === true && self::validSystemCleanup($job, $payload); if (($policyKind && !$systemPolicy) || ($job->kind === 'expiry_delete' && !$systemCleanup)) { throw new CloudError('系统策略任务与实例状态不匹配', 409); } $actor = ['user_id' => (int) $job->user_id, 'is_admin' => false, 'is_super_admin' => false, 'can_manage' => false, 'session_id' => '', 'operation_id' => $job->operation_id]; $user = User::find($job->user_id); if (!$systemPolicy && !$systemCleanup && ($user === null || (int) $user->enable !== 1)) { if ($job->dispatched_at === null) { $this->finish($job, ['status' => 'failed', 'safe_to_refund' => true], $payload); return true; } } if ($job->kind === 'create') { $actor['can_provision'] = true; $actor['can_manage'] = true; } elseif ($systemPolicy || $systemCleanup) { $actor['can_manage'] = true; if ($systemCleanup) { $actor['system_cleanup'] = true; } } elseif (($payload['can_manage'] ?? false) === true && $user !== null && (bool) $user->is_admin && in_array((int) $user->id, array_map('intval', explode(',', (string) \App\Services\Config::get('super_admin'))), true)) { $actor['is_admin'] = $actor['is_super_admin'] = $actor['can_manage'] = true; } elseif ((int) ($payload['owner_id'] ?? 0) === (int) $job->user_id) { $actor['can_manage'] = true; } if ($job->kind !== 'create' && !$actor['can_manage'] && $job->dispatched_at === null) { $this->finish($job, ['status' => 'failed'], $payload); return true; } if (!CloudFirewallService::assertJobRunnable($job, $payload)) { // A confirmed instance deletion dominates an overlapping // tenant firewall request. Close the obsolete job without // touching the deleted policy or contacting the engine. $this->finish($job, [ 'operation_id' => (string) $job->operation_id, 'status' => 'failed', 'error_code' => 'firewall_apply_superseded', 'error' => '实例已删除,防火墙请求不再适用', ], $payload); return true; } if ($systemPolicy && $job->dispatched_at === null && $job->instance_id !== null) { $instance = CloudInstance::find($job->instance_id); $desired = self::policyDesired($instance, (string) $job->kind); if (!$desired) { $this->finish($job, ['status' => 'succeeded', 'result' => ['skipped' => true]], $payload); return true; } } if (!$systemPolicy && !$systemCleanup && $job->kind !== 'create' && $job->dispatched_at === null && $job->instance_id !== null) { $instance = CloudInstance::find($job->instance_id); $policyTransition = $instance !== null && (in_array((string) $instance->service_status, ['suspending_expired', 'resume_pending', 'resuming', 'suspend_failed', 'resume_failed', 'expired_grace', 'purge_stopping', 'purge_stop_failed', 'purge_deleting', 'purge_failed'], true) || in_array((string) $instance->network_status, ['blocking', 'blocking_expired', 'blocked_expired', 'restore_pending', 'restoring', 'block_failed', 'restore_failed'], true)); $globalOverride = $actor['is_super_admin'] && $actor['can_manage']; if ($instance === null || ($job->kind !== 'delete' && !$globalOverride && ($policyTransition || !$instance->serviceAvailable()))) { $this->finish($job, ['status' => 'failed', 'error' => '实例服务已到期或正在执行策略,不能执行该操作'], $payload); return true; } } if ($job->dispatched_at !== null) { // A crash after dispatch is ambiguous. Destructive/user jobs // are never replayed. Policy lookups are explicitly read-only: // the engine's ordinary GET reconciles target state and could // otherwise apply a stale suspend before PHP rechecks that a // new paid period now desires resume. if ($systemPolicy) { try { $operation = $this->engine->request('GET', '/internal/v1/operations/' . $job->operation_id . '?reconcile=false', $actor, null, $job->operation_id); } catch (CloudError $error) { if ($error->httpStatus !== 404) { throw $error; } $operation = null; } $terminal = is_array($operation) && in_array( (string) ($operation['status'] ?? ''), ['succeeded', 'failed'], true); if (!$terminal) { // Do not close or reverse a stale policy while its // original engine executor can still touch libvirt. // Keeping this job uncertain also prevents lifecycle // from enqueueing the opposite target until the next // reconciliation pass observes a terminal operation. if (($operation['executor_active'] ?? false) === true) { throw new CloudError('引擎仍在执行当前实例策略,请稍后核对', 503); } // Re-read desired state after the read-only lookup. // Most recorded target states must converge before an // opposite transition is sent. The retired expiry // shutdown is different: grace now requires only link // down, so never replay it after an online upgrade. $current = $job->instance_id === null ? null : CloudInstance::find($job->instance_id); $desired = self::policyDesired($current, (string) $job->kind); $retiredExpirySuspend = $job->kind === 'service_suspend' && ($payload['suspend_reason'] ?? null) === 'expired' && !$desired; if (!$desired && ($operation === null || $retiredExpirySuspend)) { $result = ['skipped' => true]; if ($job->kind === 'service_suspend' && ($operation['service_was_running_hint'] ?? false) === true) { $result['service_was_running'] = true; } $this->finish($job, ['status' => 'succeeded', 'result' => $result], $payload); return true; } $operation = $this->engine->host((int) $job->host_id, $payload['method'], $payload['resource'], $actor, $payload['body'], $job->operation_id); } } elseif ($systemCleanup) { // Ask the engine to reconcile its durable operation. The // delete mutation itself is never replayed: ordinary // ambiguous deletes become terminal failures, while a VM // whose deletion was already proven may retry only the // persisted public-IP claim cleanup. Missing or otherwise // nonterminal records remain uncertain for inspection. try { $operation = $this->engine->request('GET', '/internal/v1/operations/' . $job->operation_id, $actor, null, $job->operation_id); } catch (CloudError $error) { if ($error->httpStatus === 404) { throw new CloudError( '引擎中缺少到期删除记录,必须人工核查宿主机', 503); } throw $error; } if (!in_array((string) ($operation['status'] ?? ''), ['succeeded', 'failed'], true)) { throw new CloudError( '到期删除结果不确定,必须人工核查宿主机', 503); } } else { try { $operation = $this->engine->request('GET', '/internal/v1/operations/' . $job->operation_id, $actor, null, $job->operation_id); } catch (CloudError $error) { if ($job->kind === 'create' || $error->httpStatus !== 404) { throw $error; } // Never replay destructive/user mutations. An // authoritative missing operation means there is no // engine registry entry left to reconcile; terminate // the local job visibly so it cannot block expiry. $this->finish($job, [ 'operation_id' => (string) $job->operation_id, 'status' => 'failed', 'error' => '引擎中已无此任务记录,请核查虚拟机状态后重试', ], $payload); return true; } } } else { $updated = CloudJob::where('id', $job->id)->where('lease_token', $job->lease_token) ->whereNull('dispatched_at')->update(['dispatched_at' => time(), 'status' => 'running', 'updated_at' => time()]); if ($updated !== 1) { return true; } CloudFirewallService::markJobRunning($job); $dispatched = true; $operation = $this->engine->host((int) $job->host_id, $payload['method'], $payload['resource'], $actor, $payload['body'], $job->operation_id); } if (isset($operation['uuid']) && !array_key_exists('result', $operation)) { $operation = ['status' => $operation['status'] ?? 'succeeded', 'result' => $operation]; } $this->finish($job, $operation, $payload); } catch (\Throwable $error) { // Diagnostics deliberately contain no raw payloads, credentials or upstream tracebacks. $this->observedErrors[] = 'worker_tick_failed'; $diagnostic = self::exceptionDiagnostic($error, $dispatched, (string) $job->kind, $job->order_id !== null, self::payloadAction($payload)); $log = 'cloud worker job ' . $job->id . ' failed: ' . get_class($error); if ($error instanceof CloudError) { $log .= ' status=' . $error->httpStatus . ' message=' . $diagnostic; } error_log($log); if (!$dispatched) { // Decryption/configuration/local validation failed before an // engine request was sent, so create did not happen and an // order refund is safe. try { $this->finish($job, ['status' => 'failed', 'safe_to_refund' => true, 'error' => $diagnostic], $payload); return true; } catch (\Throwable $finishError) { error_log('cloud worker job ' . $job->id . ' could not record pre-dispatch failure: ' . get_class($finishError)); } } $markedUncertain = CloudJob::where('id', $job->id) ->where('lease_token', $job->lease_token)->update([ 'status' => 'uncertain', 'error_code' => $diagnostic, 'lease_until' => 0, 'lease_token' => null, 'available_at' => time() + 30, 'updated_at' => time()]); // Losing the lease means another worker owns every subsequent // state transition. Never mutate its order, reservation, or IPs. if ($markedUncertain !== 1) { return true; } if (in_array((string) $job->kind, ['snapshot_create', 'snapshot_restore', 'snapshot_delete', 'delete', 'expiry_delete'], true)) { try { CloudSnapshotService::markJobUncertain($job); } catch (\Throwable $snapshotError) { error_log('cloud worker snapshot catalog could not be marked uncertain: ' . get_class($snapshotError)); } } if (in_array((string) $job->kind, ['firewall_apply', 'delete', 'expiry_delete'], true)) { try { CloudFirewallService::markJobUncertain($job); } catch (\Throwable $firewallError) { error_log('cloud worker firewall state could not be marked uncertain: ' . get_class($firewallError)); } } if ($job->kind === 'network_change') { try { (new CloudJob())->getConnection()->transaction( function () use ($job, $payload): void { $instance = $job->instance_id === null ? null : CloudInstance::where('id', $job->instance_id) ->lockForUpdate()->first(); $sourceIpId = filter_var( $payload['source_public_ip_id'] ?? null, FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]] ); $targetIpId = filter_var( $payload['target_public_ip_id'] ?? null, FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]] ); $networkAttachmentId = filter_var( $payload['network_attachment_id'] ?? null, FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]] ); $sourcePoolId = filter_var( $payload['source_public_ip_pool_id'] ?? null, FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]] ); $targetPoolId = filter_var( $payload['target_public_ip_pool_id'] ?? null, FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]] ); if ($instance === null || $sourceIpId === false || $targetIpId === false || $networkAttachmentId === false || $sourcePoolId === false || $targetPoolId === false) { throw new CloudError( '公网 IP 变更任务缺少账本关联', 409 ); } CloudPlacementService::finishNetworkChange( $instance, (string) $job->operation_id, (int) $sourceIpId, (int) $targetIpId, (int) $networkAttachmentId, (int) $sourcePoolId, (int) $targetPoolId, 'uncertain' ); }, 3 ); } catch (\Throwable $ledgerError) { error_log('cloud worker network ledger could not be marked uncertain: ' . get_class($ledgerError)); } } if ($job->order_id !== null) { CloudOrder::where('id', $job->order_id)->whereNotIn('status', ['succeeded', 'failed']) ->update(['status' => 'uncertain', 'updated_at' => time()]); try { (new CloudOrder())->getConnection()->transaction(function () use ($job): void { $order = CloudOrder::where('id', $job->order_id) ->lockForUpdate()->first(); if ($order !== null) { CloudPlacementService::markUncertain($order); } }, 3); } catch (\Throwable $reservationError) { error_log('cloud worker reservation could not be marked uncertain: ' . get_class($reservationError)); } } } return true; } private static function policyDesired(?CloudInstance $instance, string $kind): bool { if ($instance === null) { return false; } $status = (string) $instance->status; return match ($kind) { 'service_suspend' => in_array($status, ['active', 'deleting'], true) && (string) $instance->service_status === 'purge_stopping', 'service_resume' => $status === 'active' && in_array((string) $instance->service_status, ['resume_pending', 'resuming'], true), 'network_suspend' => $status === 'active' && in_array((string) $instance->network_status, ['blocking', 'blocking_expired'], true), 'network_resume' => $status === 'active' && in_array((string) $instance->network_status, ['restore_pending', 'restoring'], true), default => false, }; } private function finish(CloudJob $claimed, array $operation, array $payload): void { $status = self::disposition($operation, $claimed->kind === 'create', (string) $claimed->kind); if (in_array($status, ['failed', 'uncertain'], true)) { $this->observedErrors[] = $status === 'failed' ? 'task_failed' : 'task_uncertain'; } $result = $operation['result'] ?? null; (new CloudJob())->getConnection()->transaction(function () use ($claimed, $payload, $status, $result, $operation) { // All financial transactions acquire the wallet first, avoiding wallet/order deadlocks. $user = User::where('id', $claimed->user_id)->lockForUpdate()->first(); $order = $claimed->order_id === null ? null : CloudOrder::where('id', $claimed->order_id)->lockForUpdate()->first(); $job = CloudJob::where('id', $claimed->id)->lockForUpdate()->first(); if ($job === null || $job->lease_token !== $claimed->lease_token || in_array($job->status, ['succeeded', 'failed', 'cancelled'], true)) { return; } $safe = is_array($result) ? CloudOperationErrorCatalog::workerResult($result) : null; CloudSnapshotService::finishJobLocked($job, $status, $result, $payload); CloudSnapshotService::finishInstanceDeleteLocked($job, $status); CloudFirewallService::finishJobLocked( $job, $status, $result, $payload, $operation ); CloudFirewallService::finishInstanceDeleteLocked($job, $status); if ($job->kind === 'network_change' && in_array($status, ['succeeded', 'failed', 'uncertain'], true)) { $instance = $job->instance_id === null ? null : CloudInstance::where('id', $job->instance_id) ->lockForUpdate()->first(); $sourceIpId = filter_var($payload['source_public_ip_id'] ?? null, FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]); $targetIpId = filter_var($payload['target_public_ip_id'] ?? null, FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]); $networkAttachmentId = filter_var( $payload['network_attachment_id'] ?? null, FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]] ); $sourcePoolId = filter_var( $payload['source_public_ip_pool_id'] ?? null, FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]] ); $targetPoolId = filter_var( $payload['target_public_ip_pool_id'] ?? null, FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]] ); if ($instance === null || $sourceIpId === false || $targetIpId === false || $networkAttachmentId === false || $sourcePoolId === false || $targetPoolId === false) { throw new CloudError('公网 IP 变更任务缺少账本关联', 409); } if ($status === 'succeeded') { $requested = is_array($payload['body'] ?? null) ? ($payload['body']['public_ip_cidr'] ?? null) : null; $requestedPoolKey = is_array($payload['body'] ?? null) ? ($payload['body']['public_pool_key'] ?? null) : null; $confirmed = is_array($result) ? ($result['public_ip_cidr'] ?? null) : null; $confirmedPoolKey = is_array($result) ? ($result['public_pool_key'] ?? null) : null; if (!is_string($requested) || !is_string($confirmed) || !is_string($requestedPoolKey) || !is_string($confirmedPoolKey) || $requested !== $confirmed || !hash_equals($requestedPoolKey, $confirmedPoolKey)) { throw new CloudError( '引擎成功结果缺少已确认的公网 IP 或地址池', 502 ); } $decoded = json_decode((string) $instance->spec_json, true); if (!is_array($decoded) || array_is_list($decoded)) { throw new CloudError('虚拟机配置记录损坏,无法保存新的公网 IP', 409); } CloudPlacementService::finishNetworkChange($instance, (string) $job->operation_id, (int) $sourceIpId, (int) $targetIpId, (int) $networkAttachmentId, (int) $sourcePoolId, (int) $targetPoolId, 'succeeded'); $decoded['public_ip_cidr'] = $confirmed; $instance->spec_json = json_encode($decoded, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE); self::recordRuntime($instance, is_array($result) ? $result : [], (string) $job->kind, $payload); $instance->updated_at = time(); $instance->save(); } else { CloudPlacementService::finishNetworkChange($instance, (string) $job->operation_id, (int) $sourceIpId, (int) $targetIpId, (int) $networkAttachmentId, (int) $sourcePoolId, (int) $targetPoolId, $status); } } elseif ($status === 'succeeded' && $job->kind === 'create') { $vm = is_array($result) ? ($result['vm'] ?? $result) : []; $uuid = $vm['uuid'] ?? ''; if ($order === null || !is_string($uuid) || preg_match('/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/iD', $uuid) !== 1) { throw new CloudError('引擎成功结果缺少虚拟机标识', 502); } $requestedPublicIpCidr = is_array($payload['body'] ?? null) ? ($payload['body']['public_ip_cidr'] ?? null) : null; $requestedPublicPoolKey = is_array($payload['body'] ?? null) ? ($payload['body']['public_pool_key'] ?? null) : null; $confirmedPublicIpCidr = $vm['public_ip_cidr'] ?? null; $confirmedPublicPoolKey = $vm['public_pool_key'] ?? null; // Jobs queued before the multi-pool rollout have no key; the // engine's unique-IP inference remains their compatibility // path. Every newly queued placement includes a key and must // receive the same identity back before PHP commits it. $poolIdentityMatches = $requestedPublicPoolKey === null || (is_string($requestedPublicPoolKey) && is_string($confirmedPublicPoolKey) && hash_equals($requestedPublicPoolKey, $confirmedPublicPoolKey)); if (!is_string($requestedPublicIpCidr) || !is_string($confirmedPublicIpCidr) || !hash_equals($requestedPublicIpCidr, $confirmedPublicIpCidr) || !$poolIdentityMatches) { // A recovered create result may describe a VM whose // network was changed out of band after the original // response was lost. Never commit the original address in // that case: keep the order and reservation uncertain for // an administrator to reconcile. throw new CloudError( '引擎成功结果与预留公网 IP 不一致,必须人工核查', 502 ); } $instance = CloudInstance::where('order_id', $order->id)->first(); $newInstance = $instance === null; $instance ??= new CloudInstance(); $instance->uuid = $uuid; $instance->host_id = $order->host_id; $instance->user_id = $order->user_id; $instance->order_id = $order->id; $instance->offer_id = $order->offer_id; $instance->resource_pool_id = $order->resource_pool_id; $instance->reservation_id = $order->reservation_id; $spec = json_decode((string) $order->spec_json, true) ?: []; $now = time(); $billingSnapshot = array_key_exists('validity_days', $spec) && array_key_exists('traffic_quota_bytes', $spec) && (int) $spec['validity_days'] >= 1; $days = $billingSnapshot ? (int) $spec['validity_days'] : 0; $traffic = $billingSnapshot ? max(0, (int) $spec['traffic_quota_bytes']) : 0; $instance->plan_id = $order->plan_id; // A late/replayed create result must not overwrite a display // name the owner changed after the first success committed. if ($newInstance) { $instance->name = $payload['body']['name']; } $instance->status = 'active'; // Orders submitted by a pre-billing release do not contain an // immutable term/quota snapshot. Never invent a 30-day expiry // while such an order is in flight during upgrade; preserve // its historical unlimited/exempt behavior instead. $instance->service_status = $billingSnapshot ? 'active' : 'exempt'; $instance->network_status = 'active'; $instance->expires_at = $billingSnapshot ? $now + $days * 86400 : null; $instance->renewal_price = $billingSnapshot ? $order->amount : null; $instance->renewal_days = $days; $instance->renewal_traffic_bytes = $traffic; $instance->traffic_quota_bytes = $traffic; $instance->traffic_download_bytes = 0; $instance->traffic_upload_bytes = 0; $instance->lifetime_download_bytes = 0; $instance->lifetime_upload_bytes = 0; $instance->traffic_synced_at = 0; // Persist only the engine's slow-changing public metadata. // SSR pages can then render IP, operating system, login name // and actual capacity without issuing a full libvirt/XML read // on every request. Billing/order fields remain authoritative // because the allowlist cannot overwrite them. $instance->spec_json = json_encode( CloudInstanceMetadata::merge($spec, $vm), JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE ); $instance->credential_cipher = isset($payload['credential']) ? CloudSecrets::encrypt($payload['credential']) : null; // The paid service begins when provisioning actually succeeds, // not while an opening invoice is waiting for payment. A // reconciled/replayed success must retain the original value. if (!$instance->exists) { $instance->created_at = $now; } $instance->updated_at = time(); $instance->save(); CloudPlacementService::commit( $order, (int) $instance->id, $confirmedPublicIpCidr ); // Seed the independent runtime observation immediately after // provisioning. If the engine returned a trusted power // state, promote it now; otherwise the background host batch // will provide the first observation without showing "未知". self::recordRuntime($instance, $vm, 'create', []); if ($billingSnapshot) { $period = CloudServicePeriod::where('order_id', $order->id)->first() ?? new CloudServicePeriod(); $period->instance_id = $instance->id; $period->user_id = $order->user_id; $period->plan_id = $order->plan_id; $period->order_id = $order->id; $period->starts_at = $now; $period->ends_at = $instance->expires_at; $period->traffic_quota_bytes = $traffic; $period->download_base_bytes = 0; $period->upload_base_bytes = 0; $period->download_used_bytes = 0; $period->upload_used_bytes = 0; $period->status = 'active'; $period->activated_at = $now; $period->created_at = $period->created_at ?: $now; $period->updated_at = $now; $period->save(); $instance->current_period_id = $period->id; $instance->save(); } $job->instance_id = $order->instance_id = $instance->id; $provisionInvoice = CloudInvoice::where('order_id', $order->id) ->lockForUpdate()->first(); if ($provisionInvoice !== null && (string) $provisionInvoice->purpose === 'provision') { $provisionInvoice->instance_id = $instance->id; $provisionInvoice->updated_at = time(); $provisionInvoice->save(); CloudInvoiceService::queueProvisionSucceeded( $provisionInvoice, $now ); } $safe = $safe ?? []; $safe['id'] = (string) $instance->id; $safe['uuid'] = $uuid; } elseif ($status === 'failed' && $order !== null && $order->refunded_at === null) { if ($user === null) { throw new CloudError('退款账户不存在', 409); } $user->money = CloudMoney::decimal(CloudMoney::cents((string) $user->money) + CloudMoney::cents($order->amount)); $user->save(); $invoice = CloudInvoice::where('order_id', $order->id) ->lockForUpdate()->first(); // Payment placement locks offer before host. Keep the same // cross-resource order during refunds so a failed creation on // one order cannot deadlock a concurrent payment for the same // offer and host (offer -> host -> reservation -> IP). CloudOrderService::releaseStock($order); CloudPlacementService::release($order, 'create_failed'); if ($invoice !== null && (string) $invoice->purpose === 'provision') { $invoice->status = 'refunded'; $invoice->updated_at = time(); $invoice->save(); } $order->refunded_at = time(); } elseif ($status === 'failed' && $job->kind === 'expiry_delete' && $job->instance_id !== null) { $instance = CloudInstance::where('id', $job->instance_id) ->lockForUpdate()->first(); if ($instance !== null && (string) $instance->status !== 'deleted') { $instance->status = 'delete_failed'; $instance->service_status = 'purge_failed'; $instance->updated_at = time(); $instance->save(); } } elseif ($status === 'failed' && $job->instance_id !== null && in_array($job->kind, ['service_suspend', 'service_resume', 'network_suspend', 'network_resume'], true)) { // Record a visible policy error. The lifecycle reconciler // retries this desired state with bounded exponential backoff // after a temporary host/SSH failure is repaired. $instance = CloudInstance::where('id', $job->instance_id)->lockForUpdate()->first(); if ($instance !== null) { $expected = [ 'service_suspend' => ['suspending_expired', 'purge_stopping'], 'service_resume' => ['resume_pending', 'resuming'], 'network_suspend' => ['blocking', 'blocking_expired'], 'network_resume' => ['restore_pending', 'restoring'], ][$job->kind]; if ($job->kind === 'service_suspend' && in_array((string) $instance->service_status, $expected, true)) { if ((string) $instance->status === 'deleting' && (string) $instance->service_status === 'purge_stopping') { // Stopping is a replayable target state. Leave the // irreversible deletion latch in place while the // lifecycle daemon applies its bounded retry. $instance->service_status = 'purge_stop_failed'; } else { $instance->service_status = 'suspend_failed'; } } elseif ($job->kind === 'service_resume' && in_array((string) $instance->service_status, $expected, true)) { $instance->service_status = 'resume_failed'; } elseif ($job->kind === 'network_suspend' && in_array((string) $instance->network_status, $expected, true)) { $instance->network_status = 'block_failed'; } elseif ($job->kind === 'network_resume' && in_array((string) $instance->network_status, $expected, true)) { $instance->network_status = 'restore_failed'; } $instance->updated_at = time(); $instance->save(); } } elseif ($job->instance_id !== null && ($status === 'succeeded' || ($status === 'uncertain' && $job->kind === 'reinstall' && self::reinstallNeedsReview($operation)))) { $instance = CloudInstance::where('id', $job->instance_id)->lockForUpdate()->first(); if ($instance !== null) { if (in_array((string) $job->kind, ['delete', 'expiry_delete'], true)) { $instance->status = 'deleted'; $instance->credential_cipher = null; if ($instance->order_id !== null) { $provisionOrder = CloudOrder::where('id', $instance->order_id) ->lockForUpdate()->first(); if ($provisionOrder !== null) { CloudPlacementService::release( $provisionOrder, 'instance_deleted', true ); } } } elseif ($job->kind === 'reinstall') { if (isset($payload['credential']) && is_array($payload['credential'])) { $instance->credential_cipher = CloudSecrets::encrypt( $payload['credential'] ); } // The replacement template changes the static OS // metadata even though IP and disk capacity remain // stable. Invalidate only those cached fields so the // next detail render performs one bounded engine // metadata read and persists the new system label. $decoded = json_decode((string) $instance->spec_json, true); if (is_array($decoded) && !array_is_list($decoded)) { $templateId = is_array($payload['body'] ?? null) ? ($payload['body']['template_id'] ?? null) : null; if (is_string($templateId) && preg_match('/^[A-Za-z0-9][A-Za-z0-9_.-]{0,79}$/D', $templateId) === 1) { $decoded['template_id'] = $templateId; } unset($decoded['os_source'], $decoded['os_name'], $decoded['install_iso'], $decoded['login_name'], $decoded['engine_metadata_version']); $instance->spec_json = json_encode($decoded, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE); } } elseif ($job->kind === 'instance_settings') { // The engine must acknowledge the controlled seed // replacement before PHP changes any cached metadata. // Passwords and public keys are intentionally never // copied into spec_json or result_json. if (!is_array($result) || ($result['settings_changed'] ?? false) !== true) { throw new CloudError('引擎成功结果缺少已确认的实例设置', 502); } if (isset($payload['credential']) && is_array($payload['credential'])) { $instance->credential_cipher = CloudSecrets::encrypt( $payload['credential'] ); } $requested = is_array($payload['body'] ?? null) ? $payload['body'] : []; $decoded = json_decode((string) $instance->spec_json, true); if (!is_array($decoded) || array_is_list($decoded)) { throw new CloudError('虚拟机配置记录损坏,无法保存实例设置', 409); } if (array_key_exists('hostname', $requested)) { $confirmed = $result['hostname'] ?? null; if (!is_string($confirmed) || $confirmed !== $requested['hostname'] || preg_match('/^[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?(?:\.[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?)*$/D', $confirmed) !== 1 || strlen($confirmed) > 253) { throw new CloudError('引擎成功结果缺少已确认的主机名', 502); } $decoded['guest_hostname'] = $confirmed; } $instance->spec_json = json_encode($decoded, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE); } elseif ($job->kind === 'service_suspend' && in_array((string) $instance->service_status, ['suspending_expired', 'resume_pending', 'resuming', 'expired_grace', 'purge_stopping'], true)) { // Preserve proof that the retired expiry policy // touched a previously running guest. Network must be // blocked before the lifecycle daemon safely restarts // it; a tenant-stopped guest has no marker and stays // stopped. if (($result['service_was_running'] ?? false) === true && (string) $instance->service_status !== 'purge_stopping') { $instance->suspended_at = time(); } if ((string) $instance->status === 'deleting' && (string) $instance->service_status === 'purge_stopping') { $instance->service_status = 'purge_deleting'; } elseif (($payload['suspend_reason'] ?? null) === 'expired') { $expired = $instance->expires_at !== null && (int) $instance->expires_at <= time(); $instance->service_status = $instance->suspended_at !== null ? 'resume_pending' : ($expired ? 'expired_grace' : 'active'); } elseif ($instance->service_status === 'suspending_expired') { $instance->service_status = 'suspended_expired'; } } elseif ($job->kind === 'service_resume' && in_array((string) $instance->service_status, ['resuming', 'resume_pending'], true)) { $instance->service_status = $instance->expires_at !== null && (int) $instance->expires_at <= time() ? 'expired_grace' : 'active'; $instance->suspended_at = null; } elseif ($job->kind === 'network_suspend' && in_array((string) $instance->network_status, ['blocking', 'blocking_expired'], true)) { $instance->network_status = ($payload['suspend_reason'] ?? '') === 'expired' ? 'blocked_expired' : 'blocked_traffic'; $instance->network_suspended_at = time(); } elseif ($job->kind === 'network_resume' && in_array((string) $instance->network_status, ['restoring', 'restore_pending'], true)) { $instance->network_status = 'active'; $instance->network_suspended_at = null; } // Successful power/reinstall/policy operations often // include the resulting VM state. Persist that bounded // observation so the next page render is immediate; // missing state is deliberately left for vm-states. self::recordRuntime($instance, is_array($result) ? $result : [], (string) $job->kind, $payload); $instance->updated_at = time(); $instance->save(); } } $job->status = $status; if (in_array($status, ['succeeded', 'failed'], true)) { // Terminal jobs never need credentials again. Preserve only // a sanitized audit payload so an old create/reinstall row // cannot remain a second recoverable copy of the root password. $job->payload_cipher = CloudSecrets::encrypt( CloudInput::safeResult($payload) ); } $job->result_json = $safe === null ? null : json_encode($safe, JSON_THROW_ON_ERROR); $job->error_code = CloudOperationErrorCatalog::workerSummary( (string) $job->kind, $status, $operation, $order !== null, self::payloadAction($payload) ); $job->lease_token = null; $job->lease_until = 0; $job->available_at = time() + 15; $job->updated_at = time(); $job->save(); if ($order !== null) { if ($job->kind === 'create' && $status === 'uncertain') { CloudPlacementService::markUncertain($order); } $order->status = $status; $order->updated_at = time(); $order->save(); } }, 3); } private static function validSystemCleanup(CloudJob $job, array $payload): bool { if ($job->instance_id === null || ($payload['body'] ?? null) !== []) { return false; } $instance = CloudInstance::find($job->instance_id); if ($instance === null || (string) $instance->status !== 'deleting' || (string) $instance->service_status !== 'purge_deleting' || (int) $job->user_id !== (int) $instance->user_id || (int) $job->host_id !== (int) $instance->host_id || (int) ($payload['owner_id'] ?? 0) !== (int) $instance->user_id) { return false; } return ($payload['method'] ?? null) === 'DELETE' && ($payload['resource'] ?? null) === 'vms/' . rawurlencode((string) $instance->uuid) . '?delete_disks=true'; } private static function validSystemPolicy(CloudJob $job, array $payload): bool { if ($job->instance_id === null || !is_array($payload['body'] ?? null)) { return false; } $instance = CloudInstance::find($job->instance_id); if ($instance === null || (int) $job->user_id !== (int) $instance->user_id || (int) $job->host_id !== (int) $instance->host_id || (int) ($payload['owner_id'] ?? 0) !== (int) $instance->user_id) { return false; } $suffix = [ 'service_suspend' => 'service-suspend', 'service_resume' => 'service-resume', 'network_suspend' => 'network-suspend', 'network_resume' => 'network-resume', ][$job->kind] ?? null; return $suffix !== null && ($payload['method'] ?? null) === 'POST' && ($payload['resource'] ?? null) === 'vms/' . rawurlencode((string) $instance->uuid) . '/' . $suffix; } private static function payloadAction(array $payload): ?string { $body = $payload['body'] ?? null; $action = is_array($body) && is_string($body['action'] ?? null) ? $body['action'] : null; return CloudOperationCatalog::isPowerAction($action ?? '') ? $action : null; } /** * Persist only a successful, allow-listed power observation. Runtime * cache failures are isolated from the durable job outcome: the engine * result and billing state remain authoritative even during a rolling * schema upgrade where the cache table may not exist yet. */ private static function recordRuntime(CloudInstance $instance, array $result, string $kind, array $payload): void { try { if (in_array($kind, ['delete', 'expiry_delete'], true)) { CloudInstanceRuntimeService::forget((int) $instance->id); return; } $source = is_array($result['vm'] ?? null) ? $result['vm'] : $result; $state = $source['state'] ?? ($source['power_state'] ?? null); $action = $kind === 'action' ? self::payloadAction($payload) : null; $normalized = CloudInstanceRuntimeService::normalizeState($state); if ($kind === 'action') { // A successful graceful-shutdown job only confirms that the // request was accepted. libvirt may still report running (or // omit the state) until the guest completes ACPI shutdown. // Preserve that distinction instead of publishing either a // new fresh-running observation or a fabricated stopped one. $state = CloudInstanceRuntimeService::stateAfterSuccessfulAction( $state, $action ); } elseif ($normalized === 'unknown' && $kind === 'reinstall') { // The engine starts the replacement definition before // returning a successful reinstall result. Older engine // responses may omit the nested vm.state, so preserve // the same successful postcondition explicitly. $state = 'running'; } $instanceId = (int) $instance->id; $hostId = (int) $instance->host_id; $uuid = (string) $instance->uuid; if ($kind === 'create') { CloudInstanceRuntimeService::seed($instanceId, $hostId, $uuid); } if (CloudInstanceRuntimeService::normalizeState($state) !== 'unknown') { CloudInstanceRuntimeService::markObserved( $instanceId, $hostId, $uuid, $state ); } } catch (\Throwable $error) { error_log('cloud runtime observation write unavailable: ' . get_class($error)); } } private static function exceptionDiagnostic(\Throwable $error, bool $dispatched, ?string $kind, bool $hasOrder, ?string $action): string { $operation = []; if ($error instanceof CloudError) { // CloudError text is engine-/host-controlled too. The catalog // keeps only exact reviewed details and otherwise returns a // stable generic code for the durable job row. $operation['error'] = $error->getMessage(); } elseif ($error instanceof \GuzzleHttp\Exception\TransferException) { $operation['error'] = '引擎连接中断或操作超时,正在核对结果'; } $status = $dispatched ? 'uncertain' : 'failed'; return CloudOperationErrorCatalog::workerSummary($kind, $status, $operation, $hasOrder, $action) ?? CloudOperationErrorCatalog::userMessage($kind, $status, null, $hasOrder, $action) ?? '虚拟机操作失败,请稍后重试'; } }