You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
914 lines
37 KiB
PHP
914 lines
37 KiB
PHP
<?php
|
|
|
|
namespace App\Services\Cloud;
|
|
|
|
use App\Models\CloudInstance;
|
|
use App\Models\CloudInvoice;
|
|
use App\Models\CloudInvoicePayment;
|
|
use App\Models\CloudJob;
|
|
use App\Models\CloudServiceCompensation;
|
|
use App\Models\CloudServiceCompensationItem;
|
|
use App\Models\CloudServicePeriod;
|
|
use App\Models\User;
|
|
use Illuminate\Database\Capsule\Manager as DB;
|
|
|
|
/**
|
|
* Claims and applies one durable service-compensation item at a time.
|
|
*
|
|
* The instance, tail period, commercial projections, audit job and terminal
|
|
* item state commit together. A lost lease can never hide an applied change.
|
|
*/
|
|
final class CloudServiceCompensationExecutionService
|
|
{
|
|
public const LEASE_SECONDS = 120;
|
|
public const MAX_ATTEMPTS = 8;
|
|
private const EXTERNAL_WAIT_SECONDS = 300;
|
|
|
|
private readonly \Closure $clock;
|
|
|
|
public function __construct(?callable $clock = null)
|
|
{
|
|
$this->clock = $clock === null
|
|
? static fn (): int => time()
|
|
: \Closure::fromCallable($clock);
|
|
}
|
|
|
|
/** @return array<string, mixed>|null */
|
|
public function claimOne(?int $batchId = null): ?array
|
|
{
|
|
$this->assertBatchId($batchId);
|
|
$this->recoverExpired($batchId);
|
|
for ($scan = 0; $scan < 8; $scan++) {
|
|
$now = $this->now();
|
|
$query = CloudServiceCompensationItem::whereIn(
|
|
'state', ['pending', 'deferred']
|
|
)->where('available_at', '<=', $now)
|
|
->where('attempts', '<', self::MAX_ATTEMPTS)
|
|
->whereIn('batch_id', static function ($batches): void {
|
|
$batches->select('id')
|
|
->from('cloud_service_compensations')
|
|
->whereIn('state', ['pending', 'applying']);
|
|
});
|
|
if ($batchId !== null) {
|
|
$query->where('batch_id', $batchId);
|
|
}
|
|
$candidate = $query->orderBy('available_at')->orderBy('id')
|
|
->first(['id', 'batch_id']);
|
|
if ($candidate === null) {
|
|
return null;
|
|
}
|
|
$claimed = DB::connection('default')->transaction(
|
|
function () use ($candidate, $batchId, $now): ?array {
|
|
$batch = CloudServiceCompensation::where(
|
|
'id', (int) $candidate->batch_id
|
|
)->lockForUpdate()->first();
|
|
if ($batch === null
|
|
|| ($batchId !== null
|
|
&& (int) $batch->id !== $batchId)
|
|
|| in_array((string) $batch->state,
|
|
['completed', 'partial', 'attention'], true)) {
|
|
return null;
|
|
}
|
|
$item = CloudServiceCompensationItem::where(
|
|
'id', (int) $candidate->id
|
|
)->lockForUpdate()->first();
|
|
if ($item === null
|
|
|| (int) $item->batch_id !== (int) $batch->id
|
|
|| !in_array((string) $item->state,
|
|
['pending', 'deferred'], true)
|
|
|| (int) $item->available_at > $now
|
|
|| (int) $item->attempts >= self::MAX_ATTEMPTS) {
|
|
return null;
|
|
}
|
|
$token = bin2hex(random_bytes(16));
|
|
$item->state = 'applying';
|
|
$item->attempts = (int) $item->attempts + 1;
|
|
$item->lease_token = $token;
|
|
$item->lease_until = $now + self::LEASE_SECONDS;
|
|
$item->last_error_code = '';
|
|
$item->last_error = '';
|
|
$item->updated_at = $now;
|
|
$item->save();
|
|
$batch->state = 'applying';
|
|
$batch->last_error_code = '';
|
|
$batch->updated_at = $now;
|
|
$batch->save();
|
|
return [
|
|
'item_id' => (int) $item->id,
|
|
'item_key' => (string) $item->item_key,
|
|
'batch_id' => (int) $batch->id,
|
|
'token' => $token,
|
|
'lease_until' => (int) $item->lease_until,
|
|
];
|
|
},
|
|
3
|
|
);
|
|
if ($claimed !== null) {
|
|
return $claimed;
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
/** Move abandoned claims back to a bounded retry or terminal attention. */
|
|
public function recoverExpired(?int $batchId = null): int
|
|
{
|
|
$this->assertBatchId($batchId);
|
|
$now = $this->now();
|
|
$query = CloudServiceCompensationItem::where('state', 'applying')
|
|
->where('lease_until', '>', 0)
|
|
->where('lease_until', '<=', $now);
|
|
if ($batchId !== null) {
|
|
$query->where('batch_id', $batchId);
|
|
}
|
|
$ids = $query->orderBy('id')->pluck('id')->map(
|
|
static fn ($id): int => (int) $id
|
|
)->all();
|
|
$recovered = 0;
|
|
foreach ($ids as $itemId) {
|
|
$changed = DB::connection('default')->transaction(
|
|
function () use ($itemId, $batchId, $now): bool {
|
|
$hint = CloudServiceCompensationItem::find($itemId);
|
|
if ($hint === null) {
|
|
return false;
|
|
}
|
|
$batch = CloudServiceCompensation::where(
|
|
'id', (int) $hint->batch_id
|
|
)->lockForUpdate()->first();
|
|
$item = CloudServiceCompensationItem::where('id', $itemId)
|
|
->lockForUpdate()->first();
|
|
if ($batch === null || $item === null
|
|
|| ($batchId !== null
|
|
&& (int) $batch->id !== $batchId)
|
|
|| (int) $item->batch_id !== (int) $batch->id
|
|
|| (string) $item->state !== 'applying'
|
|
|| (int) $item->lease_until < 1
|
|
|| (int) $item->lease_until > $now) {
|
|
return false;
|
|
}
|
|
$this->finishLocked(
|
|
$batch,
|
|
$item,
|
|
(int) $item->attempts >= self::MAX_ATTEMPTS
|
|
? 'attention' : 'deferred',
|
|
(int) $item->attempts >= self::MAX_ATTEMPTS
|
|
? 'retry_exhausted' : 'worker_lease_expired',
|
|
(int) $item->attempts >= self::MAX_ATTEMPTS
|
|
? '补偿任务多次失去执行租约,需要管理员核对'
|
|
: '补偿执行租约已过期,系统将安全重试',
|
|
$now
|
|
);
|
|
return true;
|
|
},
|
|
3
|
|
);
|
|
$recovered += $changed ? 1 : 0;
|
|
}
|
|
return $recovered;
|
|
}
|
|
|
|
/** @return array<string, mixed> */
|
|
public function applyClaim(int $itemId, string $token): array
|
|
{
|
|
if ($itemId < 1 || preg_match('/^[a-f0-9]{32}$/D', $token) !== 1) {
|
|
throw new CloudError('实例补偿执行凭据无效', 422);
|
|
}
|
|
$hint = CloudServiceCompensationItem::find($itemId);
|
|
if ($hint === null) {
|
|
throw new CloudError('实例补偿明细不存在', 404);
|
|
}
|
|
if ((string) $hint->state === 'applied') {
|
|
return $this->result($hint);
|
|
}
|
|
if ((string) $hint->state !== 'applying'
|
|
|| !is_string($hint->lease_token)
|
|
|| !hash_equals((string) $hint->lease_token, $token)) {
|
|
throw new CloudError('实例补偿执行租约已失效', 409);
|
|
}
|
|
if ((int) $hint->lease_until <= $this->now()) {
|
|
$this->recoverExpired((int) $hint->batch_id);
|
|
return $this->result(
|
|
CloudServiceCompensationItem::findOrFail($itemId)
|
|
);
|
|
}
|
|
|
|
$periodHint = CloudServicePeriod::where(
|
|
'instance_id', (int) $hint->instance_id
|
|
)->orderByDesc('ends_at')->orderByDesc('id')->first();
|
|
if ($periodHint !== null) {
|
|
try {
|
|
CloudTrafficPoolAccounting::reconcileServicePeriod(
|
|
(int) $periodHint->id,
|
|
'compensation-preflight:' . (string) $hint->item_key,
|
|
$this->now()
|
|
);
|
|
} catch (CloudError $error) {
|
|
return $this->transitionClaim(
|
|
$itemId,
|
|
$token,
|
|
'attention',
|
|
'traffic_reconciliation_failed',
|
|
'共享流量权益需要先完成核对'
|
|
);
|
|
} catch (\Throwable $error) {
|
|
return $this->transitionClaim(
|
|
$itemId,
|
|
$token,
|
|
'deferred',
|
|
'traffic_reconciliation_unavailable',
|
|
'共享流量权益暂时无法核对,系统将重试'
|
|
);
|
|
}
|
|
}
|
|
$preflightPeriodId = $periodHint === null
|
|
? null : (int) $periodHint->id;
|
|
|
|
try {
|
|
return DB::connection('default')->transaction(
|
|
function () use (
|
|
$hint,
|
|
$itemId,
|
|
$token,
|
|
$preflightPeriodId
|
|
): array {
|
|
$now = $this->now();
|
|
// Canonical compensation lock order: owner -> instance ->
|
|
// batch -> item -> commercial projections.
|
|
$owner = User::where('id', (int) $hint->user_id)
|
|
->lockForUpdate()->first();
|
|
$instance = CloudInstance::where(
|
|
'id', (int) $hint->instance_id
|
|
)->lockForUpdate()->first();
|
|
$batch = CloudServiceCompensation::where(
|
|
'id', (int) $hint->batch_id
|
|
)->lockForUpdate()->first();
|
|
$item = CloudServiceCompensationItem::where('id', $itemId)
|
|
->lockForUpdate()->first();
|
|
if ($batch === null || $item === null
|
|
|| (int) $item->batch_id !== (int) $batch->id) {
|
|
throw new CloudError('实例补偿账本不完整', 409);
|
|
}
|
|
if ((string) $item->state === 'applied') {
|
|
return $this->result($item);
|
|
}
|
|
if ((string) $item->state !== 'applying'
|
|
|| !is_string($item->lease_token)
|
|
|| !hash_equals((string) $item->lease_token, $token)) {
|
|
throw new CloudError('实例补偿执行租约已失效', 409);
|
|
}
|
|
if ((int) $item->lease_until <= $now) {
|
|
return $this->finishLocked(
|
|
$batch,
|
|
$item,
|
|
'deferred',
|
|
'worker_lease_expired',
|
|
'补偿执行租约已过期,系统将安全重试',
|
|
$now
|
|
);
|
|
}
|
|
if ($owner === null || $instance === null
|
|
|| (int) $item->user_id !== (int) ($owner->id ?? 0)
|
|
|| (int) $item->instance_id
|
|
!== (int) ($instance->id ?? 0)
|
|
|| (int) $instance->user_id !== (int) $item->user_id) {
|
|
return $this->finishLocked(
|
|
$batch,
|
|
$item,
|
|
'attention',
|
|
'instance_identity_conflict',
|
|
'实例或用户身份账本不一致',
|
|
$now
|
|
);
|
|
}
|
|
$admission = $this->admissionLocked($instance, $now);
|
|
if ($admission !== null) {
|
|
return $this->finishLocked(
|
|
$batch,
|
|
$item,
|
|
$admission['state'],
|
|
$admission['code'],
|
|
$admission['message'],
|
|
$now,
|
|
$admission['consume_retry_budget']
|
|
);
|
|
}
|
|
|
|
$period = CloudServicePeriod::where(
|
|
'instance_id', (int) $instance->id
|
|
)->orderByDesc('ends_at')->orderByDesc('id')
|
|
->first();
|
|
if ($period === null) {
|
|
return $this->finishLocked(
|
|
$batch,
|
|
$item,
|
|
'attention',
|
|
'service_period_missing',
|
|
'实例缺少当前服务周期',
|
|
$now
|
|
);
|
|
}
|
|
if ($preflightPeriodId !== null
|
|
&& (int) $period->id !== $preflightPeriodId) {
|
|
return $this->finishLocked(
|
|
$batch,
|
|
$item,
|
|
'deferred',
|
|
'service_period_changed',
|
|
'实例服务周期已变化,系统将重新核对',
|
|
$now,
|
|
false
|
|
);
|
|
}
|
|
$oldExpiry = (int) $instance->expires_at;
|
|
$seconds = (int) $item->requested_seconds;
|
|
if (!in_array((string) $period->status,
|
|
['active', 'exhausted', 'scheduled'], true)
|
|
|| (int) $period->starts_at >= (int) $period->ends_at
|
|
|| (int) $period->ends_at !== $oldExpiry) {
|
|
return $this->finishLocked(
|
|
$batch,
|
|
$item,
|
|
'attention',
|
|
'service_period_expiry_mismatch',
|
|
'实例到期时间与当前服务周期不一致',
|
|
$now
|
|
);
|
|
}
|
|
if ($seconds < 1 || $oldExpiry < 1
|
|
|| $oldExpiry > PHP_INT_MAX - $seconds) {
|
|
return $this->finishLocked(
|
|
$batch,
|
|
$item,
|
|
'attention',
|
|
'expiry_overflow',
|
|
'实例补偿后的到期时间超出有效范围',
|
|
$now
|
|
);
|
|
}
|
|
$newExpiry = $oldExpiry + $seconds;
|
|
$financial = $this->financialAdmissionLocked(
|
|
$instance,
|
|
$oldExpiry
|
|
);
|
|
if ($financial !== null) {
|
|
return $this->finishLocked(
|
|
$batch,
|
|
$item,
|
|
$financial['state'],
|
|
$financial['code'],
|
|
$financial['message'],
|
|
$now,
|
|
$financial['consume_retry_budget']
|
|
);
|
|
}
|
|
$operationId = substr(hash(
|
|
'sha256',
|
|
'service-compensation-operation:'
|
|
. (string) $item->item_key
|
|
), 0, 32);
|
|
if (CloudJob::where('operation_id', $operationId)
|
|
->lockForUpdate()->exists()) {
|
|
return $this->finishLocked(
|
|
$batch,
|
|
$item,
|
|
'attention',
|
|
'audit_job_conflict',
|
|
'补偿审计任务与明细状态不一致',
|
|
$now
|
|
);
|
|
}
|
|
|
|
$before = $this->snapshot($instance, $period);
|
|
$instance->expires_at = $newExpiry;
|
|
$invoiceResult = CloudInvoiceService::
|
|
reconcileRenewalAfterCompensationLocked(
|
|
$instance,
|
|
$oldExpiry,
|
|
$newExpiry,
|
|
$now
|
|
);
|
|
$traffic = CloudTrafficPoolAccounting::
|
|
extendEntitlementWindowForCompensationLocked(
|
|
(int) $period->id,
|
|
$oldExpiry,
|
|
$newExpiry,
|
|
(int) $item->id,
|
|
$now
|
|
);
|
|
$backup = CloudAutomaticBackupBillingService::
|
|
extendEntitlementWindowForCompensationLocked(
|
|
(int) $period->id,
|
|
$oldExpiry,
|
|
$newExpiry,
|
|
(int) $item->id,
|
|
$now
|
|
);
|
|
// The traffic helper owns pool -> period lock order. Only
|
|
// now re-read that retained row lock before changing it.
|
|
$lockedPeriod = CloudServicePeriod::where(
|
|
'id', (int) $period->id
|
|
)->lockForUpdate()->first();
|
|
if ($lockedPeriod === null
|
|
|| (int) $lockedPeriod->instance_id
|
|
!== (int) $instance->id
|
|
|| (int) $lockedPeriod->ends_at !== $oldExpiry) {
|
|
throw new CloudError(
|
|
'实例服务周期在补偿事务中发生变化',
|
|
409
|
|
);
|
|
}
|
|
$lockedPeriod->ends_at = $newExpiry;
|
|
$lockedPeriod->updated_at = $now;
|
|
$lockedPeriod->save();
|
|
$instance->updated_at = $now;
|
|
$instance->save();
|
|
|
|
$job = $this->auditJob(
|
|
$instance,
|
|
$item,
|
|
$batch,
|
|
$operationId,
|
|
$oldExpiry,
|
|
$newExpiry,
|
|
$now
|
|
);
|
|
$after = $this->snapshot($instance, $lockedPeriod);
|
|
$item->service_period_id = (int) $lockedPeriod->id;
|
|
$item->traffic_entitlement_id = $traffic === null
|
|
? null : (int) $traffic->id;
|
|
$item->backup_entitlement_id = $backup === null
|
|
? null : (int) $backup->id;
|
|
$item->source_invoice_id = $invoiceResult[
|
|
'source_invoice_id'
|
|
];
|
|
$replacement = $invoiceResult['replacement_invoice'];
|
|
$item->replacement_invoice_id = $replacement === null
|
|
? null : (int) $replacement->id;
|
|
$item->audit_job_id = (int) $job->id;
|
|
$item->before_expires_at = $oldExpiry;
|
|
$item->after_expires_at = $newExpiry;
|
|
$item->before_period_ends_at = $oldExpiry;
|
|
$item->after_period_ends_at = $newExpiry;
|
|
$item->before_traffic_ends_at = $traffic === null
|
|
? null : $oldExpiry;
|
|
$item->after_traffic_ends_at = $traffic === null
|
|
? null : $newExpiry;
|
|
$item->before_backup_ends_at = $backup === null
|
|
? null : $oldExpiry;
|
|
$item->after_backup_ends_at = $backup === null
|
|
? null : $newExpiry;
|
|
// The reviewed snapshot is immutable evidence. Actual
|
|
// application-time projections live only in the after
|
|
// evidence because a renewal may have advanced the tail.
|
|
$item->after_snapshot_json = self::json([
|
|
'applied_from' => $before,
|
|
'applied_to' => $after,
|
|
]);
|
|
$item->state = 'applied';
|
|
$item->applied_at = $now;
|
|
$item->terminal_at = $now;
|
|
$item->lease_token = null;
|
|
$item->lease_until = 0;
|
|
$item->available_at = $now;
|
|
$item->last_error_code = '';
|
|
$item->last_error = '';
|
|
$item->updated_at = $now;
|
|
$item->save();
|
|
(new CloudServiceCompensationService())
|
|
->recalculateLocked($batch, $now);
|
|
return $this->result($item);
|
|
},
|
|
3
|
|
);
|
|
} catch (CloudError $error) {
|
|
return $this->recoverAfterApplyError(
|
|
$itemId,
|
|
$token,
|
|
'projection_conflict',
|
|
'补偿账本投影发生冲突,需要管理员核对',
|
|
true
|
|
);
|
|
} catch (\Throwable $error) {
|
|
return $this->recoverAfterApplyError(
|
|
$itemId,
|
|
$token,
|
|
'apply_failed',
|
|
'补偿执行遇到暂时故障,系统将安全重试',
|
|
false
|
|
);
|
|
}
|
|
}
|
|
|
|
/** Claim and execute at most one due item. */
|
|
public function tick(?int $batchId = null): bool
|
|
{
|
|
$claim = $this->claimOne($batchId);
|
|
if ($claim === null) {
|
|
return false;
|
|
}
|
|
$this->applyClaim((int) $claim['item_id'], (string) $claim['token']);
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* @return array{state:string,code:string,message:string,
|
|
* consume_retry_budget:bool}|null
|
|
*/
|
|
private function admissionLocked(CloudInstance $instance, int $now): ?array
|
|
{
|
|
if ((string) $instance->status === 'deleted') {
|
|
return $this->decision(
|
|
'rejected', 'instance_deleted', '实例已经删除'
|
|
);
|
|
}
|
|
if (CloudInvoiceService::instanceDeleting($instance)) {
|
|
return $this->decision(
|
|
'deferred', 'instance_deleting', '实例删除流程尚未收敛', false
|
|
);
|
|
}
|
|
if ((string) $instance->service_status !== 'active') {
|
|
return $this->decision(
|
|
'rejected', 'service_not_active', '实例服务当前不是有效状态'
|
|
);
|
|
}
|
|
if ($instance->expires_at === null
|
|
|| (int) $instance->expires_at <= $now) {
|
|
return $this->decision(
|
|
'rejected', 'service_expired', '实例服务已经到期'
|
|
);
|
|
}
|
|
if (CloudInstanceMigrationService::hasOpenMigration(
|
|
(int) $instance->id
|
|
)) {
|
|
return $this->decision(
|
|
'deferred', 'migration_open', '实例维护迁移尚未收敛', false
|
|
);
|
|
}
|
|
$jobs = CloudJobService::blockingQueryForInstance(
|
|
(int) $instance->id
|
|
)->orderBy('id')->lockForUpdate()->get([
|
|
'id', 'phase', 'effect_resolution', 'finalization_incident',
|
|
]);
|
|
if ($jobs->contains(static fn ($job): bool =>
|
|
(string) $job->phase !== 'terminal')) {
|
|
return $this->decision(
|
|
'deferred', 'instance_operation_open', '实例仍有操作正在执行', false
|
|
);
|
|
}
|
|
if (!$jobs->isEmpty()) {
|
|
return $this->decision(
|
|
'attention',
|
|
'instance_operation_uncertain',
|
|
'实例存在效果未知或最终化异常的操作'
|
|
);
|
|
}
|
|
return null;
|
|
}
|
|
|
|
/**
|
|
* @return array{state:string,code:string,message:string,
|
|
* consume_retry_budget:bool}|null
|
|
*/
|
|
private function financialAdmissionLocked(
|
|
CloudInstance $instance,
|
|
int $oldExpiry
|
|
): ?array {
|
|
$invoices = CloudInvoice::where('instance_id', (int) $instance->id)
|
|
->where('purpose', 'renewal')
|
|
->where('renews_expires_at', $oldExpiry)
|
|
->where('status', '<>', 'void')
|
|
->orderBy('id')->lockForUpdate()->get();
|
|
if ($invoices->count() > 1) {
|
|
return $this->decision(
|
|
'attention', 'renewal_invoice_duplicate',
|
|
'旧到期时间存在重复续费账单'
|
|
);
|
|
}
|
|
$invoice = $invoices->first();
|
|
if ($invoice === null) {
|
|
return null;
|
|
}
|
|
if ((int) $invoice->user_id !== (int) $instance->user_id
|
|
|| $invoice->renewal_id !== null
|
|
|| $invoice->paid_at !== null
|
|
|| (string) $invoice->status === 'paid') {
|
|
return $this->decision(
|
|
'attention', 'renewal_invoice_settled',
|
|
'旧到期时间的续费账单已经结算'
|
|
);
|
|
}
|
|
if (!in_array((string) $invoice->status,
|
|
['open', 'overdue', 'expired'], true)) {
|
|
return $this->decision(
|
|
'attention', 'renewal_invoice_state_invalid',
|
|
'旧到期时间的续费账单状态异常'
|
|
);
|
|
}
|
|
// Match payment-effect recovery's job -> payment lock order.
|
|
$jobs = CloudJobService::blockingQueryForInvoice((int) $invoice->id)
|
|
->orderBy('id')->lockForUpdate()->get([
|
|
'id', 'phase', 'effect_resolution', 'finalization_incident',
|
|
]);
|
|
if ($jobs->contains(static fn ($job): bool =>
|
|
(string) $job->phase !== 'terminal')) {
|
|
return $this->decision(
|
|
'deferred', 'renewal_payment_job_open',
|
|
'旧续费账单的支付任务仍在执行', false
|
|
);
|
|
}
|
|
if (!$jobs->isEmpty()) {
|
|
return $this->decision(
|
|
'attention', 'renewal_payment_job_uncertain',
|
|
'旧续费账单的支付任务效果未知'
|
|
);
|
|
}
|
|
$payments = CloudInvoicePayment::where(
|
|
'invoice_id', (int) $invoice->id
|
|
)->orderBy('id')->lockForUpdate()->get();
|
|
if ($payments->contains(static fn ($payment): bool => in_array(
|
|
(string) $payment->status,
|
|
['creating', 'pending'],
|
|
true
|
|
))) {
|
|
return $this->decision(
|
|
'deferred', 'renewal_payment_open',
|
|
'旧续费账单仍有支付正在处理', false
|
|
);
|
|
}
|
|
if ($payments->contains(static fn ($payment): bool => !in_array(
|
|
(string) $payment->status,
|
|
['failed', 'expired', 'not_created'],
|
|
true
|
|
))) {
|
|
return $this->decision(
|
|
'attention', 'renewal_payment_uncertain',
|
|
'旧续费账单存在已支付或效果未知的支付记录'
|
|
);
|
|
}
|
|
if (!CloudInvoiceService::fenceRenewalMailForCompensationLocked(
|
|
$invoice,
|
|
$this->now(),
|
|
false
|
|
)) {
|
|
return $this->decision(
|
|
'deferred',
|
|
'renewal_mail_sending',
|
|
'旧续费账单提醒邮件仍在投递',
|
|
false
|
|
);
|
|
}
|
|
return null;
|
|
}
|
|
|
|
private function auditJob(
|
|
CloudInstance $instance,
|
|
CloudServiceCompensationItem $item,
|
|
CloudServiceCompensation $batch,
|
|
string $operationId,
|
|
int $oldExpiry,
|
|
int $newExpiry,
|
|
int $now
|
|
): CloudJob {
|
|
$fingerprint = hash('sha256', self::json([
|
|
'batch_key' => (string) $batch->compensation_key,
|
|
'item_key' => (string) $item->item_key,
|
|
'incident_key' => (string) $batch->incident_key,
|
|
'instance_id' => (int) $instance->id,
|
|
'requested_seconds' => (int) $item->requested_seconds,
|
|
'previous_expires_at' => $oldExpiry,
|
|
'new_expires_at' => $newExpiry,
|
|
]));
|
|
$job = new CloudJob();
|
|
$job->operation_id = $operationId;
|
|
$job->request_idempotency_key = 'service-compensation-'
|
|
. (string) $item->item_key;
|
|
$job->request_fingerprint = $fingerprint;
|
|
$job->instance_id = (int) $instance->id;
|
|
$job->user_id = (int) $instance->user_id;
|
|
$job->host_id = null;
|
|
$job->kind = 'service_compensation';
|
|
$job->execution_mode = 'local';
|
|
$job->sealed_payload = CloudSecrets::encrypt([
|
|
'compensation_key' => (string) $batch->compensation_key,
|
|
'item_key' => (string) $item->item_key,
|
|
'incident_key' => (string) $batch->incident_key,
|
|
], 'job-payload');
|
|
$job->dispatch_deadline_at = $now + CloudJob::DISPATCH_DEADLINE_SECONDS;
|
|
$job->available_at = $now;
|
|
$job->created_at = $job->updated_at = $now;
|
|
$job->save();
|
|
CloudJobService::completeWithoutDispatchLocked($job, [
|
|
'compensation_key' => (string) $batch->compensation_key,
|
|
'item_key' => (string) $item->item_key,
|
|
'incident_key' => (string) $batch->incident_key,
|
|
'seconds_added' => (int) $item->requested_seconds,
|
|
'previous_expires_at' => $oldExpiry,
|
|
'new_expires_at' => $newExpiry,
|
|
]);
|
|
return $job;
|
|
}
|
|
|
|
/** @return array<string, mixed> */
|
|
private function finishLocked(
|
|
CloudServiceCompensation $batch,
|
|
CloudServiceCompensationItem $item,
|
|
string $state,
|
|
string $code,
|
|
string $message,
|
|
int $now,
|
|
bool $consumeRetryBudget = true
|
|
): array {
|
|
if ($state === 'deferred' && !$consumeRetryBudget) {
|
|
// Claiming increments attempts before the external admission facts
|
|
// are locked. Undo only that claim so long-running migrations,
|
|
// deletes and payments do not consume the technical failure
|
|
// budget. Prior real failures remain counted.
|
|
$item->attempts = max(0, (int) $item->attempts - 1);
|
|
}
|
|
if ($state === 'deferred'
|
|
&& (int) $item->attempts >= self::MAX_ATTEMPTS) {
|
|
$state = 'attention';
|
|
$code = 'retry_exhausted';
|
|
$message = '补偿任务重试次数已耗尽,需要管理员核对';
|
|
}
|
|
if (!in_array($state, ['deferred', 'rejected', 'attention'], true)) {
|
|
throw new \LogicException('Invalid compensation transition.');
|
|
}
|
|
$item->state = $state;
|
|
$item->lease_token = null;
|
|
$item->lease_until = 0;
|
|
$item->available_at = $state === 'deferred'
|
|
? $now + ($consumeRetryBudget
|
|
? $this->backoff((int) $item->attempts)
|
|
: self::EXTERNAL_WAIT_SECONDS)
|
|
: $now;
|
|
$item->terminal_at = $state === 'deferred' ? null : $now;
|
|
$item->last_error_code = substr($code, 0, 64);
|
|
$item->last_error = mb_substr($message, 0, 500);
|
|
$item->updated_at = $now;
|
|
$item->save();
|
|
(new CloudServiceCompensationService())
|
|
->recalculateLocked($batch, $now);
|
|
return $this->result($item);
|
|
}
|
|
|
|
/** @return array<string, mixed> */
|
|
private function transitionClaim(
|
|
int $itemId,
|
|
string $token,
|
|
string $state,
|
|
string $code,
|
|
string $message
|
|
): array {
|
|
return DB::connection('default')->transaction(
|
|
function () use ($itemId, $token, $state, $code, $message): array {
|
|
$hint = CloudServiceCompensationItem::find($itemId);
|
|
if ($hint === null) {
|
|
throw new CloudError('实例补偿明细不存在', 404);
|
|
}
|
|
$batch = CloudServiceCompensation::where(
|
|
'id', (int) $hint->batch_id
|
|
)->lockForUpdate()->first();
|
|
$item = CloudServiceCompensationItem::where('id', $itemId)
|
|
->lockForUpdate()->first();
|
|
if ($batch === null || $item === null) {
|
|
throw new CloudError('实例补偿账本不完整', 409);
|
|
}
|
|
if ((string) $item->state === 'applied'
|
|
|| (string) $item->state !== 'applying'
|
|
|| !is_string($item->lease_token)
|
|
|| !hash_equals((string) $item->lease_token, $token)) {
|
|
return $this->result($item);
|
|
}
|
|
return $this->finishLocked(
|
|
$batch,
|
|
$item,
|
|
$state,
|
|
$code,
|
|
$message,
|
|
$this->now()
|
|
);
|
|
},
|
|
3
|
|
);
|
|
}
|
|
|
|
/** @return array<string, mixed> */
|
|
private function recoverAfterApplyError(
|
|
int $itemId,
|
|
string $token,
|
|
string $code,
|
|
string $message,
|
|
bool $attention
|
|
): array {
|
|
$latest = CloudServiceCompensationItem::find($itemId);
|
|
if ($latest !== null && (string) $latest->state === 'applied') {
|
|
return $this->result($latest);
|
|
}
|
|
return $this->transitionClaim(
|
|
$itemId,
|
|
$token,
|
|
$attention ? 'attention' : 'deferred',
|
|
$code,
|
|
$message
|
|
);
|
|
}
|
|
|
|
/**
|
|
* @return array{state:string,code:string,message:string,
|
|
* consume_retry_budget:bool}
|
|
*/
|
|
private function decision(
|
|
string $state,
|
|
string $code,
|
|
string $message,
|
|
bool $consumeRetryBudget = true
|
|
): array {
|
|
return [
|
|
'state' => $state,
|
|
'code' => $code,
|
|
'message' => $message,
|
|
'consume_retry_budget' => $consumeRetryBudget,
|
|
];
|
|
}
|
|
|
|
private function backoff(int $attempts): int
|
|
{
|
|
return min(3600, 30 * (2 ** max(0, min(7, $attempts - 1))));
|
|
}
|
|
|
|
/** @return array<string, mixed> */
|
|
private function snapshot(
|
|
CloudInstance $instance,
|
|
CloudServicePeriod $period
|
|
): array {
|
|
return [
|
|
'instance' => [
|
|
'id' => (int) $instance->id,
|
|
'user_id' => (int) $instance->user_id,
|
|
'host_id' => (int) $instance->host_id,
|
|
'name' => (string) $instance->name,
|
|
'status' => (string) $instance->status,
|
|
'service_status' => (string) $instance->service_status,
|
|
'expires_at' => (int) $instance->expires_at,
|
|
'plan_id' => (int) $instance->plan_id,
|
|
'offer_id' => (int) $instance->offer_id,
|
|
'renewal_price' => (string) $instance->renewal_price,
|
|
'renewal_days' => (int) $instance->renewal_days,
|
|
'renewal_traffic_bytes' =>
|
|
(int) $instance->renewal_traffic_bytes,
|
|
],
|
|
'service_period' => [
|
|
'id' => (int) $period->id,
|
|
'starts_at' => (int) $period->starts_at,
|
|
'ends_at' => (int) $period->ends_at,
|
|
'status' => (string) $period->status,
|
|
'traffic_quota_mode' =>
|
|
(string) $period->traffic_quota_mode,
|
|
'traffic_quota_bytes' =>
|
|
(int) $period->traffic_quota_bytes,
|
|
],
|
|
];
|
|
}
|
|
|
|
/** @return array<string, mixed> */
|
|
private function result(CloudServiceCompensationItem $item): array
|
|
{
|
|
return [
|
|
'item_id' => (int) $item->id,
|
|
'item_key' => (string) $item->item_key,
|
|
'batch_id' => (int) $item->batch_id,
|
|
'instance_id' => (int) $item->instance_id,
|
|
'state' => (string) $item->state,
|
|
'attempts' => (int) $item->attempts,
|
|
'audit_job_id' => $item->audit_job_id === null
|
|
? null : (int) $item->audit_job_id,
|
|
'before_expires_at' => $item->before_expires_at === null
|
|
? null : (int) $item->before_expires_at,
|
|
'after_expires_at' => $item->after_expires_at === null
|
|
? null : (int) $item->after_expires_at,
|
|
'available_at' => (int) $item->available_at,
|
|
'last_error_code' => (string) $item->last_error_code,
|
|
];
|
|
}
|
|
|
|
private function assertBatchId(?int $batchId): void
|
|
{
|
|
if ($batchId !== null && $batchId < 1) {
|
|
throw new \InvalidArgumentException('Compensation batch is invalid.');
|
|
}
|
|
}
|
|
|
|
private function now(): int
|
|
{
|
|
return max(1, (int) ($this->clock)());
|
|
}
|
|
|
|
private static function json(array $value): string
|
|
{
|
|
return json_encode($value, JSON_THROW_ON_ERROR
|
|
| JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE);
|
|
}
|
|
|
|
}
|