clock = $clock === null ? static fn (): int => time() : \Closure::fromCallable($clock); } /** @return array|null */ public function claimOne(?int $batchId = null): ?array { $this->assertBatchId($batchId); $this->recoverExpired($batchId); for ($scan = 0; $scan < 8; $scan++) { $now = $this->now(); $query = CloudServiceCompensationItem::whereIn( 'state', ['pending', 'deferred'] )->where('available_at', '<=', $now) ->where('attempts', '<', self::MAX_ATTEMPTS) ->whereIn('batch_id', static function ($batches): void { $batches->select('id') ->from('cloud_service_compensations') ->whereIn('state', ['pending', 'applying']); }); if ($batchId !== null) { $query->where('batch_id', $batchId); } $candidate = $query->orderBy('available_at')->orderBy('id') ->first(['id', 'batch_id']); if ($candidate === null) { return null; } $claimed = DB::connection('default')->transaction( function () use ($candidate, $batchId, $now): ?array { $batch = CloudServiceCompensation::where( 'id', (int) $candidate->batch_id )->lockForUpdate()->first(); if ($batch === null || ($batchId !== null && (int) $batch->id !== $batchId) || in_array((string) $batch->state, ['completed', 'partial', 'attention'], true)) { return null; } $item = CloudServiceCompensationItem::where( 'id', (int) $candidate->id )->lockForUpdate()->first(); if ($item === null || (int) $item->batch_id !== (int) $batch->id || !in_array((string) $item->state, ['pending', 'deferred'], true) || (int) $item->available_at > $now || (int) $item->attempts >= self::MAX_ATTEMPTS) { return null; } $token = bin2hex(random_bytes(16)); $item->state = 'applying'; $item->attempts = (int) $item->attempts + 1; $item->lease_token = $token; $item->lease_until = $now + self::LEASE_SECONDS; $item->last_error_code = ''; $item->last_error = ''; $item->updated_at = $now; $item->save(); $batch->state = 'applying'; $batch->last_error_code = ''; $batch->updated_at = $now; $batch->save(); return [ 'item_id' => (int) $item->id, 'item_key' => (string) $item->item_key, 'batch_id' => (int) $batch->id, 'token' => $token, 'lease_until' => (int) $item->lease_until, ]; }, 3 ); if ($claimed !== null) { return $claimed; } } return null; } /** Move abandoned claims back to a bounded retry or terminal attention. */ public function recoverExpired(?int $batchId = null): int { $this->assertBatchId($batchId); $now = $this->now(); $query = CloudServiceCompensationItem::where('state', 'applying') ->where('lease_until', '>', 0) ->where('lease_until', '<=', $now); if ($batchId !== null) { $query->where('batch_id', $batchId); } $ids = $query->orderBy('id')->pluck('id')->map( static fn ($id): int => (int) $id )->all(); $recovered = 0; foreach ($ids as $itemId) { $changed = DB::connection('default')->transaction( function () use ($itemId, $batchId, $now): bool { $hint = CloudServiceCompensationItem::find($itemId); if ($hint === null) { return false; } $batch = CloudServiceCompensation::where( 'id', (int) $hint->batch_id )->lockForUpdate()->first(); $item = CloudServiceCompensationItem::where('id', $itemId) ->lockForUpdate()->first(); if ($batch === null || $item === null || ($batchId !== null && (int) $batch->id !== $batchId) || (int) $item->batch_id !== (int) $batch->id || (string) $item->state !== 'applying' || (int) $item->lease_until < 1 || (int) $item->lease_until > $now) { return false; } $this->finishLocked( $batch, $item, (int) $item->attempts >= self::MAX_ATTEMPTS ? 'attention' : 'deferred', (int) $item->attempts >= self::MAX_ATTEMPTS ? 'retry_exhausted' : 'worker_lease_expired', (int) $item->attempts >= self::MAX_ATTEMPTS ? '补偿任务多次失去执行租约,需要管理员核对' : '补偿执行租约已过期,系统将安全重试', $now ); return true; }, 3 ); $recovered += $changed ? 1 : 0; } return $recovered; } /** @return array */ public function applyClaim(int $itemId, string $token): array { if ($itemId < 1 || preg_match('/^[a-f0-9]{32}$/D', $token) !== 1) { throw new CloudError('实例补偿执行凭据无效', 422); } $hint = CloudServiceCompensationItem::find($itemId); if ($hint === null) { throw new CloudError('实例补偿明细不存在', 404); } if ((string) $hint->state === 'applied') { return $this->result($hint); } if ((string) $hint->state !== 'applying' || !is_string($hint->lease_token) || !hash_equals((string) $hint->lease_token, $token)) { throw new CloudError('实例补偿执行租约已失效', 409); } if ((int) $hint->lease_until <= $this->now()) { $this->recoverExpired((int) $hint->batch_id); return $this->result( CloudServiceCompensationItem::findOrFail($itemId) ); } $periodHint = CloudServicePeriod::where( 'instance_id', (int) $hint->instance_id )->orderByDesc('ends_at')->orderByDesc('id')->first(); if ($periodHint !== null) { try { CloudTrafficPoolAccounting::reconcileServicePeriod( (int) $periodHint->id, 'compensation-preflight:' . (string) $hint->item_key, $this->now() ); } catch (CloudError $error) { return $this->transitionClaim( $itemId, $token, 'attention', 'traffic_reconciliation_failed', '共享流量权益需要先完成核对' ); } catch (\Throwable $error) { return $this->transitionClaim( $itemId, $token, 'deferred', 'traffic_reconciliation_unavailable', '共享流量权益暂时无法核对,系统将重试' ); } } $preflightPeriodId = $periodHint === null ? null : (int) $periodHint->id; try { return DB::connection('default')->transaction( function () use ( $hint, $itemId, $token, $preflightPeriodId ): array { $now = $this->now(); // Canonical compensation lock order: owner -> instance -> // batch -> item -> commercial projections. $owner = User::where('id', (int) $hint->user_id) ->lockForUpdate()->first(); $instance = CloudInstance::where( 'id', (int) $hint->instance_id )->lockForUpdate()->first(); $batch = CloudServiceCompensation::where( 'id', (int) $hint->batch_id )->lockForUpdate()->first(); $item = CloudServiceCompensationItem::where('id', $itemId) ->lockForUpdate()->first(); if ($batch === null || $item === null || (int) $item->batch_id !== (int) $batch->id) { throw new CloudError('实例补偿账本不完整', 409); } if ((string) $item->state === 'applied') { return $this->result($item); } if ((string) $item->state !== 'applying' || !is_string($item->lease_token) || !hash_equals((string) $item->lease_token, $token)) { throw new CloudError('实例补偿执行租约已失效', 409); } if ((int) $item->lease_until <= $now) { return $this->finishLocked( $batch, $item, 'deferred', 'worker_lease_expired', '补偿执行租约已过期,系统将安全重试', $now ); } if ($owner === null || $instance === null || (int) $item->user_id !== (int) ($owner->id ?? 0) || (int) $item->instance_id !== (int) ($instance->id ?? 0) || (int) $instance->user_id !== (int) $item->user_id) { return $this->finishLocked( $batch, $item, 'attention', 'instance_identity_conflict', '实例或用户身份账本不一致', $now ); } $admission = $this->admissionLocked($instance, $now); if ($admission !== null) { return $this->finishLocked( $batch, $item, $admission['state'], $admission['code'], $admission['message'], $now, $admission['consume_retry_budget'] ); } $period = CloudServicePeriod::where( 'instance_id', (int) $instance->id )->orderByDesc('ends_at')->orderByDesc('id') ->first(); if ($period === null) { return $this->finishLocked( $batch, $item, 'attention', 'service_period_missing', '实例缺少当前服务周期', $now ); } if ($preflightPeriodId !== null && (int) $period->id !== $preflightPeriodId) { return $this->finishLocked( $batch, $item, 'deferred', 'service_period_changed', '实例服务周期已变化,系统将重新核对', $now, false ); } $oldExpiry = (int) $instance->expires_at; $seconds = (int) $item->requested_seconds; if (!in_array((string) $period->status, ['active', 'exhausted', 'scheduled'], true) || (int) $period->starts_at >= (int) $period->ends_at || (int) $period->ends_at !== $oldExpiry) { return $this->finishLocked( $batch, $item, 'attention', 'service_period_expiry_mismatch', '实例到期时间与当前服务周期不一致', $now ); } if ($seconds < 1 || $oldExpiry < 1 || $oldExpiry > PHP_INT_MAX - $seconds) { return $this->finishLocked( $batch, $item, 'attention', 'expiry_overflow', '实例补偿后的到期时间超出有效范围', $now ); } $newExpiry = $oldExpiry + $seconds; $financial = $this->financialAdmissionLocked( $instance, $oldExpiry ); if ($financial !== null) { return $this->finishLocked( $batch, $item, $financial['state'], $financial['code'], $financial['message'], $now, $financial['consume_retry_budget'] ); } $operationId = substr(hash( 'sha256', 'service-compensation-operation:' . (string) $item->item_key ), 0, 32); if (CloudJob::where('operation_id', $operationId) ->lockForUpdate()->exists()) { return $this->finishLocked( $batch, $item, 'attention', 'audit_job_conflict', '补偿审计任务与明细状态不一致', $now ); } $before = $this->snapshot($instance, $period); $instance->expires_at = $newExpiry; $invoiceResult = CloudInvoiceService:: reconcileRenewalAfterCompensationLocked( $instance, $oldExpiry, $newExpiry, $now ); $traffic = CloudTrafficPoolAccounting:: extendEntitlementWindowForCompensationLocked( (int) $period->id, $oldExpiry, $newExpiry, (int) $item->id, $now ); $backup = CloudAutomaticBackupBillingService:: extendEntitlementWindowForCompensationLocked( (int) $period->id, $oldExpiry, $newExpiry, (int) $item->id, $now ); // The traffic helper owns pool -> period lock order. Only // now re-read that retained row lock before changing it. $lockedPeriod = CloudServicePeriod::where( 'id', (int) $period->id )->lockForUpdate()->first(); if ($lockedPeriod === null || (int) $lockedPeriod->instance_id !== (int) $instance->id || (int) $lockedPeriod->ends_at !== $oldExpiry) { throw new CloudError( '实例服务周期在补偿事务中发生变化', 409 ); } $lockedPeriod->ends_at = $newExpiry; $lockedPeriod->updated_at = $now; $lockedPeriod->save(); $instance->updated_at = $now; $instance->save(); $job = $this->auditJob( $instance, $item, $batch, $operationId, $oldExpiry, $newExpiry, $now ); $after = $this->snapshot($instance, $lockedPeriod); $item->service_period_id = (int) $lockedPeriod->id; $item->traffic_entitlement_id = $traffic === null ? null : (int) $traffic->id; $item->backup_entitlement_id = $backup === null ? null : (int) $backup->id; $item->source_invoice_id = $invoiceResult[ 'source_invoice_id' ]; $replacement = $invoiceResult['replacement_invoice']; $item->replacement_invoice_id = $replacement === null ? null : (int) $replacement->id; $item->audit_job_id = (int) $job->id; $item->before_expires_at = $oldExpiry; $item->after_expires_at = $newExpiry; $item->before_period_ends_at = $oldExpiry; $item->after_period_ends_at = $newExpiry; $item->before_traffic_ends_at = $traffic === null ? null : $oldExpiry; $item->after_traffic_ends_at = $traffic === null ? null : $newExpiry; $item->before_backup_ends_at = $backup === null ? null : $oldExpiry; $item->after_backup_ends_at = $backup === null ? null : $newExpiry; // The reviewed snapshot is immutable evidence. Actual // application-time projections live only in the after // evidence because a renewal may have advanced the tail. $item->after_snapshot_json = self::json([ 'applied_from' => $before, 'applied_to' => $after, ]); $item->state = 'applied'; $item->applied_at = $now; $item->terminal_at = $now; $item->lease_token = null; $item->lease_until = 0; $item->available_at = $now; $item->last_error_code = ''; $item->last_error = ''; $item->updated_at = $now; $item->save(); (new CloudServiceCompensationService()) ->recalculateLocked($batch, $now); return $this->result($item); }, 3 ); } catch (CloudError $error) { return $this->recoverAfterApplyError( $itemId, $token, 'projection_conflict', '补偿账本投影发生冲突,需要管理员核对', true ); } catch (\Throwable $error) { return $this->recoverAfterApplyError( $itemId, $token, 'apply_failed', '补偿执行遇到暂时故障,系统将安全重试', false ); } } /** Claim and execute at most one due item. */ public function tick(?int $batchId = null): bool { $claim = $this->claimOne($batchId); if ($claim === null) { return false; } $this->applyClaim((int) $claim['item_id'], (string) $claim['token']); return true; } /** * @return array{state:string,code:string,message:string, * consume_retry_budget:bool}|null */ private function admissionLocked(CloudInstance $instance, int $now): ?array { if ((string) $instance->status === 'deleted') { return $this->decision( 'rejected', 'instance_deleted', '实例已经删除' ); } if (CloudInvoiceService::instanceDeleting($instance)) { return $this->decision( 'deferred', 'instance_deleting', '实例删除流程尚未收敛', false ); } if ((string) $instance->service_status !== 'active') { return $this->decision( 'rejected', 'service_not_active', '实例服务当前不是有效状态' ); } if ($instance->expires_at === null || (int) $instance->expires_at <= $now) { return $this->decision( 'rejected', 'service_expired', '实例服务已经到期' ); } if (CloudInstanceMigrationService::hasOpenMigration( (int) $instance->id )) { return $this->decision( 'deferred', 'migration_open', '实例维护迁移尚未收敛', false ); } $jobs = CloudJobService::blockingQueryForInstance( (int) $instance->id )->orderBy('id')->lockForUpdate()->get([ 'id', 'phase', 'effect_resolution', 'finalization_incident', ]); if ($jobs->contains(static fn ($job): bool => (string) $job->phase !== 'terminal')) { return $this->decision( 'deferred', 'instance_operation_open', '实例仍有操作正在执行', false ); } if (!$jobs->isEmpty()) { return $this->decision( 'attention', 'instance_operation_uncertain', '实例存在效果未知或最终化异常的操作' ); } return null; } /** * @return array{state:string,code:string,message:string, * consume_retry_budget:bool}|null */ private function financialAdmissionLocked( CloudInstance $instance, int $oldExpiry ): ?array { $invoices = CloudInvoice::where('instance_id', (int) $instance->id) ->where('purpose', 'renewal') ->where('renews_expires_at', $oldExpiry) ->where('status', '<>', 'void') ->orderBy('id')->lockForUpdate()->get(); if ($invoices->count() > 1) { return $this->decision( 'attention', 'renewal_invoice_duplicate', '旧到期时间存在重复续费账单' ); } $invoice = $invoices->first(); if ($invoice === null) { return null; } if ((int) $invoice->user_id !== (int) $instance->user_id || $invoice->renewal_id !== null || $invoice->paid_at !== null || (string) $invoice->status === 'paid') { return $this->decision( 'attention', 'renewal_invoice_settled', '旧到期时间的续费账单已经结算' ); } if (!in_array((string) $invoice->status, ['open', 'overdue', 'expired'], true)) { return $this->decision( 'attention', 'renewal_invoice_state_invalid', '旧到期时间的续费账单状态异常' ); } // Match payment-effect recovery's job -> payment lock order. $jobs = CloudJobService::blockingQueryForInvoice((int) $invoice->id) ->orderBy('id')->lockForUpdate()->get([ 'id', 'phase', 'effect_resolution', 'finalization_incident', ]); if ($jobs->contains(static fn ($job): bool => (string) $job->phase !== 'terminal')) { return $this->decision( 'deferred', 'renewal_payment_job_open', '旧续费账单的支付任务仍在执行', false ); } if (!$jobs->isEmpty()) { return $this->decision( 'attention', 'renewal_payment_job_uncertain', '旧续费账单的支付任务效果未知' ); } $payments = CloudInvoicePayment::where( 'invoice_id', (int) $invoice->id )->orderBy('id')->lockForUpdate()->get(); if ($payments->contains(static fn ($payment): bool => in_array( (string) $payment->status, ['creating', 'pending'], true ))) { return $this->decision( 'deferred', 'renewal_payment_open', '旧续费账单仍有支付正在处理', false ); } if ($payments->contains(static fn ($payment): bool => !in_array( (string) $payment->status, ['failed', 'expired', 'not_created'], true ))) { return $this->decision( 'attention', 'renewal_payment_uncertain', '旧续费账单存在已支付或效果未知的支付记录' ); } if (!CloudInvoiceService::fenceRenewalMailForCompensationLocked( $invoice, $this->now(), false )) { return $this->decision( 'deferred', 'renewal_mail_sending', '旧续费账单提醒邮件仍在投递', false ); } return null; } private function auditJob( CloudInstance $instance, CloudServiceCompensationItem $item, CloudServiceCompensation $batch, string $operationId, int $oldExpiry, int $newExpiry, int $now ): CloudJob { $fingerprint = hash('sha256', self::json([ 'batch_key' => (string) $batch->compensation_key, 'item_key' => (string) $item->item_key, 'incident_key' => (string) $batch->incident_key, 'instance_id' => (int) $instance->id, 'requested_seconds' => (int) $item->requested_seconds, 'previous_expires_at' => $oldExpiry, 'new_expires_at' => $newExpiry, ])); $job = new CloudJob(); $job->operation_id = $operationId; $job->request_idempotency_key = 'service-compensation-' . (string) $item->item_key; $job->request_fingerprint = $fingerprint; $job->instance_id = (int) $instance->id; $job->user_id = (int) $instance->user_id; $job->host_id = null; $job->kind = 'service_compensation'; $job->execution_mode = 'local'; $job->sealed_payload = CloudSecrets::encrypt([ 'compensation_key' => (string) $batch->compensation_key, 'item_key' => (string) $item->item_key, 'incident_key' => (string) $batch->incident_key, ], 'job-payload'); $job->dispatch_deadline_at = $now + CloudJob::DISPATCH_DEADLINE_SECONDS; $job->available_at = $now; $job->created_at = $job->updated_at = $now; $job->save(); CloudJobService::completeWithoutDispatchLocked($job, [ 'compensation_key' => (string) $batch->compensation_key, 'item_key' => (string) $item->item_key, 'incident_key' => (string) $batch->incident_key, 'seconds_added' => (int) $item->requested_seconds, 'previous_expires_at' => $oldExpiry, 'new_expires_at' => $newExpiry, ]); return $job; } /** @return array */ private function finishLocked( CloudServiceCompensation $batch, CloudServiceCompensationItem $item, string $state, string $code, string $message, int $now, bool $consumeRetryBudget = true ): array { if ($state === 'deferred' && !$consumeRetryBudget) { // Claiming increments attempts before the external admission facts // are locked. Undo only that claim so long-running migrations, // deletes and payments do not consume the technical failure // budget. Prior real failures remain counted. $item->attempts = max(0, (int) $item->attempts - 1); } if ($state === 'deferred' && (int) $item->attempts >= self::MAX_ATTEMPTS) { $state = 'attention'; $code = 'retry_exhausted'; $message = '补偿任务重试次数已耗尽,需要管理员核对'; } if (!in_array($state, ['deferred', 'rejected', 'attention'], true)) { throw new \LogicException('Invalid compensation transition.'); } $item->state = $state; $item->lease_token = null; $item->lease_until = 0; $item->available_at = $state === 'deferred' ? $now + ($consumeRetryBudget ? $this->backoff((int) $item->attempts) : self::EXTERNAL_WAIT_SECONDS) : $now; $item->terminal_at = $state === 'deferred' ? null : $now; $item->last_error_code = substr($code, 0, 64); $item->last_error = mb_substr($message, 0, 500); $item->updated_at = $now; $item->save(); (new CloudServiceCompensationService()) ->recalculateLocked($batch, $now); return $this->result($item); } /** @return array */ private function transitionClaim( int $itemId, string $token, string $state, string $code, string $message ): array { return DB::connection('default')->transaction( function () use ($itemId, $token, $state, $code, $message): array { $hint = CloudServiceCompensationItem::find($itemId); if ($hint === null) { throw new CloudError('实例补偿明细不存在', 404); } $batch = CloudServiceCompensation::where( 'id', (int) $hint->batch_id )->lockForUpdate()->first(); $item = CloudServiceCompensationItem::where('id', $itemId) ->lockForUpdate()->first(); if ($batch === null || $item === null) { throw new CloudError('实例补偿账本不完整', 409); } if ((string) $item->state === 'applied' || (string) $item->state !== 'applying' || !is_string($item->lease_token) || !hash_equals((string) $item->lease_token, $token)) { return $this->result($item); } return $this->finishLocked( $batch, $item, $state, $code, $message, $this->now() ); }, 3 ); } /** @return array */ private function recoverAfterApplyError( int $itemId, string $token, string $code, string $message, bool $attention ): array { $latest = CloudServiceCompensationItem::find($itemId); if ($latest !== null && (string) $latest->state === 'applied') { return $this->result($latest); } return $this->transitionClaim( $itemId, $token, $attention ? 'attention' : 'deferred', $code, $message ); } /** * @return array{state:string,code:string,message:string, * consume_retry_budget:bool} */ private function decision( string $state, string $code, string $message, bool $consumeRetryBudget = true ): array { return [ 'state' => $state, 'code' => $code, 'message' => $message, 'consume_retry_budget' => $consumeRetryBudget, ]; } private function backoff(int $attempts): int { return min(3600, 30 * (2 ** max(0, min(7, $attempts - 1)))); } /** @return array */ private function snapshot( CloudInstance $instance, CloudServicePeriod $period ): array { return [ 'instance' => [ 'id' => (int) $instance->id, 'user_id' => (int) $instance->user_id, 'host_id' => (int) $instance->host_id, 'name' => (string) $instance->name, 'status' => (string) $instance->status, 'service_status' => (string) $instance->service_status, 'expires_at' => (int) $instance->expires_at, 'plan_id' => (int) $instance->plan_id, 'offer_id' => (int) $instance->offer_id, 'renewal_price' => (string) $instance->renewal_price, 'renewal_days' => (int) $instance->renewal_days, 'renewal_traffic_bytes' => (int) $instance->renewal_traffic_bytes, ], 'service_period' => [ 'id' => (int) $period->id, 'starts_at' => (int) $period->starts_at, 'ends_at' => (int) $period->ends_at, 'status' => (string) $period->status, 'traffic_quota_mode' => (string) $period->traffic_quota_mode, 'traffic_quota_bytes' => (int) $period->traffic_quota_bytes, ], ]; } /** @return array */ private function result(CloudServiceCompensationItem $item): array { return [ 'item_id' => (int) $item->id, 'item_key' => (string) $item->item_key, 'batch_id' => (int) $item->batch_id, 'instance_id' => (int) $item->instance_id, 'state' => (string) $item->state, 'attempts' => (int) $item->attempts, 'audit_job_id' => $item->audit_job_id === null ? null : (int) $item->audit_job_id, 'before_expires_at' => $item->before_expires_at === null ? null : (int) $item->before_expires_at, 'after_expires_at' => $item->after_expires_at === null ? null : (int) $item->after_expires_at, 'available_at' => (int) $item->available_at, 'last_error_code' => (string) $item->last_error_code, ]; } private function assertBatchId(?int $batchId): void { if ($batchId !== null && $batchId < 1) { throw new \InvalidArgumentException('Compensation batch is invalid.'); } } private function now(): int { return max(1, (int) ($this->clock)()); } private static function json(array $value): string { return json_encode($value, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE); } }