feat(cloud): execute service compensation safely
parent
eff7605c0f
commit
1e025f86f8
@ -0,0 +1,871 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Cloud;
|
||||
|
||||
use App\Models\CloudInstance;
|
||||
use App\Models\CloudInvoice;
|
||||
use App\Models\CloudInvoicePayment;
|
||||
use App\Models\CloudJob;
|
||||
use App\Models\CloudServiceCompensation;
|
||||
use App\Models\CloudServiceCompensationItem;
|
||||
use App\Models\CloudServicePeriod;
|
||||
use App\Models\User;
|
||||
use Illuminate\Database\Capsule\Manager as DB;
|
||||
|
||||
/**
|
||||
* Claims and applies one durable service-compensation item at a time.
|
||||
*
|
||||
* The instance, tail period, commercial projections, audit job and terminal
|
||||
* item state commit together. A lost lease can never hide an applied change.
|
||||
*/
|
||||
final class CloudServiceCompensationExecutionService
|
||||
{
|
||||
public const LEASE_SECONDS = 120;
|
||||
public const MAX_ATTEMPTS = 8;
|
||||
|
||||
private readonly \Closure $clock;
|
||||
|
||||
public function __construct(?callable $clock = null)
|
||||
{
|
||||
$this->clock = $clock === null
|
||||
? static fn (): int => time()
|
||||
: \Closure::fromCallable($clock);
|
||||
}
|
||||
|
||||
/** @return array<string, mixed>|null */
|
||||
public function claimOne(?int $batchId = null): ?array
|
||||
{
|
||||
$this->assertBatchId($batchId);
|
||||
$this->recoverExpired($batchId);
|
||||
for ($scan = 0; $scan < 8; $scan++) {
|
||||
$now = $this->now();
|
||||
$query = CloudServiceCompensationItem::whereIn(
|
||||
'state', ['pending', 'deferred']
|
||||
)->where('available_at', '<=', $now)
|
||||
->where('attempts', '<', self::MAX_ATTEMPTS)
|
||||
->whereIn('batch_id', static function ($batches): void {
|
||||
$batches->select('id')
|
||||
->from('cloud_service_compensations')
|
||||
->whereIn('state', ['pending', 'applying']);
|
||||
});
|
||||
if ($batchId !== null) {
|
||||
$query->where('batch_id', $batchId);
|
||||
}
|
||||
$candidate = $query->orderBy('available_at')->orderBy('id')
|
||||
->first(['id', 'batch_id']);
|
||||
if ($candidate === null) {
|
||||
return null;
|
||||
}
|
||||
$claimed = DB::connection('default')->transaction(
|
||||
function () use ($candidate, $batchId, $now): ?array {
|
||||
$batch = CloudServiceCompensation::where(
|
||||
'id', (int) $candidate->batch_id
|
||||
)->lockForUpdate()->first();
|
||||
if ($batch === null
|
||||
|| ($batchId !== null
|
||||
&& (int) $batch->id !== $batchId)
|
||||
|| in_array((string) $batch->state,
|
||||
['completed', 'partial', 'attention'], true)) {
|
||||
return null;
|
||||
}
|
||||
$item = CloudServiceCompensationItem::where(
|
||||
'id', (int) $candidate->id
|
||||
)->lockForUpdate()->first();
|
||||
if ($item === null
|
||||
|| (int) $item->batch_id !== (int) $batch->id
|
||||
|| !in_array((string) $item->state,
|
||||
['pending', 'deferred'], true)
|
||||
|| (int) $item->available_at > $now
|
||||
|| (int) $item->attempts >= self::MAX_ATTEMPTS) {
|
||||
return null;
|
||||
}
|
||||
$token = bin2hex(random_bytes(16));
|
||||
$item->state = 'applying';
|
||||
$item->attempts = (int) $item->attempts + 1;
|
||||
$item->lease_token = $token;
|
||||
$item->lease_until = $now + self::LEASE_SECONDS;
|
||||
$item->last_error_code = '';
|
||||
$item->last_error = '';
|
||||
$item->updated_at = $now;
|
||||
$item->save();
|
||||
$batch->state = 'applying';
|
||||
$batch->last_error_code = '';
|
||||
$batch->updated_at = $now;
|
||||
$batch->save();
|
||||
return [
|
||||
'item_id' => (int) $item->id,
|
||||
'item_key' => (string) $item->item_key,
|
||||
'batch_id' => (int) $batch->id,
|
||||
'token' => $token,
|
||||
'lease_until' => (int) $item->lease_until,
|
||||
];
|
||||
},
|
||||
3
|
||||
);
|
||||
if ($claimed !== null) {
|
||||
return $claimed;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Move abandoned claims back to a bounded retry or terminal attention. */
|
||||
public function recoverExpired(?int $batchId = null): int
|
||||
{
|
||||
$this->assertBatchId($batchId);
|
||||
$now = $this->now();
|
||||
$query = CloudServiceCompensationItem::where('state', 'applying')
|
||||
->where('lease_until', '>', 0)
|
||||
->where('lease_until', '<=', $now);
|
||||
if ($batchId !== null) {
|
||||
$query->where('batch_id', $batchId);
|
||||
}
|
||||
$ids = $query->orderBy('id')->pluck('id')->map(
|
||||
static fn ($id): int => (int) $id
|
||||
)->all();
|
||||
$recovered = 0;
|
||||
foreach ($ids as $itemId) {
|
||||
$changed = DB::connection('default')->transaction(
|
||||
function () use ($itemId, $batchId, $now): bool {
|
||||
$hint = CloudServiceCompensationItem::find($itemId);
|
||||
if ($hint === null) {
|
||||
return false;
|
||||
}
|
||||
$batch = CloudServiceCompensation::where(
|
||||
'id', (int) $hint->batch_id
|
||||
)->lockForUpdate()->first();
|
||||
$item = CloudServiceCompensationItem::where('id', $itemId)
|
||||
->lockForUpdate()->first();
|
||||
if ($batch === null || $item === null
|
||||
|| ($batchId !== null
|
||||
&& (int) $batch->id !== $batchId)
|
||||
|| (int) $item->batch_id !== (int) $batch->id
|
||||
|| (string) $item->state !== 'applying'
|
||||
|| (int) $item->lease_until < 1
|
||||
|| (int) $item->lease_until > $now) {
|
||||
return false;
|
||||
}
|
||||
$this->finishLocked(
|
||||
$batch,
|
||||
$item,
|
||||
(int) $item->attempts >= self::MAX_ATTEMPTS
|
||||
? 'attention' : 'deferred',
|
||||
(int) $item->attempts >= self::MAX_ATTEMPTS
|
||||
? 'retry_exhausted' : 'worker_lease_expired',
|
||||
(int) $item->attempts >= self::MAX_ATTEMPTS
|
||||
? '补偿任务多次失去执行租约,需要管理员核对'
|
||||
: '补偿执行租约已过期,系统将安全重试',
|
||||
$now
|
||||
);
|
||||
return true;
|
||||
},
|
||||
3
|
||||
);
|
||||
$recovered += $changed ? 1 : 0;
|
||||
}
|
||||
return $recovered;
|
||||
}
|
||||
|
||||
/** @return array<string, mixed> */
|
||||
public function applyClaim(int $itemId, string $token): array
|
||||
{
|
||||
if ($itemId < 1 || preg_match('/^[a-f0-9]{32}$/D', $token) !== 1) {
|
||||
throw new CloudError('实例补偿执行凭据无效', 422);
|
||||
}
|
||||
$hint = CloudServiceCompensationItem::find($itemId);
|
||||
if ($hint === null) {
|
||||
throw new CloudError('实例补偿明细不存在', 404);
|
||||
}
|
||||
if ((string) $hint->state === 'applied') {
|
||||
return $this->result($hint);
|
||||
}
|
||||
if ((string) $hint->state !== 'applying'
|
||||
|| !is_string($hint->lease_token)
|
||||
|| !hash_equals((string) $hint->lease_token, $token)) {
|
||||
throw new CloudError('实例补偿执行租约已失效', 409);
|
||||
}
|
||||
if ((int) $hint->lease_until <= $this->now()) {
|
||||
$this->recoverExpired((int) $hint->batch_id);
|
||||
return $this->result(
|
||||
CloudServiceCompensationItem::findOrFail($itemId)
|
||||
);
|
||||
}
|
||||
|
||||
$periodHint = CloudServicePeriod::where(
|
||||
'instance_id', (int) $hint->instance_id
|
||||
)->orderByDesc('ends_at')->orderByDesc('id')->first();
|
||||
if ($periodHint !== null) {
|
||||
try {
|
||||
CloudTrafficPoolAccounting::reconcileServicePeriod(
|
||||
(int) $periodHint->id,
|
||||
'compensation-preflight:' . (string) $hint->item_key,
|
||||
$this->now()
|
||||
);
|
||||
} catch (CloudError $error) {
|
||||
return $this->transitionClaim(
|
||||
$itemId,
|
||||
$token,
|
||||
'attention',
|
||||
'traffic_reconciliation_failed',
|
||||
'共享流量权益需要先完成核对'
|
||||
);
|
||||
} catch (\Throwable $error) {
|
||||
return $this->transitionClaim(
|
||||
$itemId,
|
||||
$token,
|
||||
'deferred',
|
||||
'traffic_reconciliation_unavailable',
|
||||
'共享流量权益暂时无法核对,系统将重试'
|
||||
);
|
||||
}
|
||||
}
|
||||
$preflightPeriodId = $periodHint === null
|
||||
? null : (int) $periodHint->id;
|
||||
|
||||
try {
|
||||
return DB::connection('default')->transaction(
|
||||
function () use (
|
||||
$hint,
|
||||
$itemId,
|
||||
$token,
|
||||
$preflightPeriodId
|
||||
): array {
|
||||
$now = $this->now();
|
||||
// Canonical compensation lock order: owner -> instance ->
|
||||
// batch -> item -> commercial projections.
|
||||
$owner = User::where('id', (int) $hint->user_id)
|
||||
->lockForUpdate()->first();
|
||||
$instance = CloudInstance::where(
|
||||
'id', (int) $hint->instance_id
|
||||
)->lockForUpdate()->first();
|
||||
$batch = CloudServiceCompensation::where(
|
||||
'id', (int) $hint->batch_id
|
||||
)->lockForUpdate()->first();
|
||||
$item = CloudServiceCompensationItem::where('id', $itemId)
|
||||
->lockForUpdate()->first();
|
||||
if ($batch === null || $item === null
|
||||
|| (int) $item->batch_id !== (int) $batch->id) {
|
||||
throw new CloudError('实例补偿账本不完整', 409);
|
||||
}
|
||||
if ((string) $item->state === 'applied') {
|
||||
return $this->result($item);
|
||||
}
|
||||
if ((string) $item->state !== 'applying'
|
||||
|| !is_string($item->lease_token)
|
||||
|| !hash_equals((string) $item->lease_token, $token)) {
|
||||
throw new CloudError('实例补偿执行租约已失效', 409);
|
||||
}
|
||||
if ((int) $item->lease_until <= $now) {
|
||||
return $this->finishLocked(
|
||||
$batch,
|
||||
$item,
|
||||
'deferred',
|
||||
'worker_lease_expired',
|
||||
'补偿执行租约已过期,系统将安全重试',
|
||||
$now
|
||||
);
|
||||
}
|
||||
if ($owner === null || $instance === null
|
||||
|| (int) $item->user_id !== (int) ($owner->id ?? 0)
|
||||
|| (int) $item->instance_id
|
||||
!== (int) ($instance->id ?? 0)
|
||||
|| (int) $instance->user_id !== (int) $item->user_id) {
|
||||
return $this->finishLocked(
|
||||
$batch,
|
||||
$item,
|
||||
'attention',
|
||||
'instance_identity_conflict',
|
||||
'实例或用户身份账本不一致',
|
||||
$now
|
||||
);
|
||||
}
|
||||
$admission = $this->admissionLocked($instance, $now);
|
||||
if ($admission !== null) {
|
||||
return $this->finishLocked(
|
||||
$batch,
|
||||
$item,
|
||||
$admission['state'],
|
||||
$admission['code'],
|
||||
$admission['message'],
|
||||
$now
|
||||
);
|
||||
}
|
||||
|
||||
$period = CloudServicePeriod::where(
|
||||
'instance_id', (int) $instance->id
|
||||
)->orderByDesc('ends_at')->orderByDesc('id')
|
||||
->first();
|
||||
if ($period === null) {
|
||||
return $this->finishLocked(
|
||||
$batch,
|
||||
$item,
|
||||
'attention',
|
||||
'service_period_missing',
|
||||
'实例缺少当前服务周期',
|
||||
$now
|
||||
);
|
||||
}
|
||||
if ($preflightPeriodId !== null
|
||||
&& (int) $period->id !== $preflightPeriodId) {
|
||||
return $this->finishLocked(
|
||||
$batch,
|
||||
$item,
|
||||
'deferred',
|
||||
'service_period_changed',
|
||||
'实例服务周期已变化,系统将重新核对',
|
||||
$now
|
||||
);
|
||||
}
|
||||
$oldExpiry = (int) $instance->expires_at;
|
||||
$seconds = (int) $item->requested_seconds;
|
||||
if (!in_array((string) $period->status,
|
||||
['active', 'exhausted', 'scheduled'], true)
|
||||
|| (int) $period->starts_at >= (int) $period->ends_at
|
||||
|| (int) $period->ends_at !== $oldExpiry) {
|
||||
return $this->finishLocked(
|
||||
$batch,
|
||||
$item,
|
||||
'attention',
|
||||
'service_period_expiry_mismatch',
|
||||
'实例到期时间与当前服务周期不一致',
|
||||
$now
|
||||
);
|
||||
}
|
||||
if ($seconds < 1 || $oldExpiry < 1
|
||||
|| $oldExpiry > PHP_INT_MAX - $seconds) {
|
||||
return $this->finishLocked(
|
||||
$batch,
|
||||
$item,
|
||||
'attention',
|
||||
'expiry_overflow',
|
||||
'实例补偿后的到期时间超出有效范围',
|
||||
$now
|
||||
);
|
||||
}
|
||||
$newExpiry = $oldExpiry + $seconds;
|
||||
$financial = $this->financialAdmissionLocked(
|
||||
$instance,
|
||||
$oldExpiry
|
||||
);
|
||||
if ($financial !== null) {
|
||||
return $this->finishLocked(
|
||||
$batch,
|
||||
$item,
|
||||
$financial['state'],
|
||||
$financial['code'],
|
||||
$financial['message'],
|
||||
$now
|
||||
);
|
||||
}
|
||||
$operationId = substr(hash(
|
||||
'sha256',
|
||||
'service-compensation-operation:'
|
||||
. (string) $item->item_key
|
||||
), 0, 32);
|
||||
if (CloudJob::where('operation_id', $operationId)
|
||||
->lockForUpdate()->exists()) {
|
||||
return $this->finishLocked(
|
||||
$batch,
|
||||
$item,
|
||||
'attention',
|
||||
'audit_job_conflict',
|
||||
'补偿审计任务与明细状态不一致',
|
||||
$now
|
||||
);
|
||||
}
|
||||
|
||||
$before = $this->snapshot($instance, $period);
|
||||
$instance->expires_at = $newExpiry;
|
||||
$invoiceResult = CloudInvoiceService::
|
||||
reconcileRenewalAfterCompensationLocked(
|
||||
$instance,
|
||||
$oldExpiry,
|
||||
$newExpiry,
|
||||
$now
|
||||
);
|
||||
$traffic = CloudTrafficPoolAccounting::
|
||||
extendEntitlementWindowForCompensationLocked(
|
||||
(int) $period->id,
|
||||
$oldExpiry,
|
||||
$newExpiry,
|
||||
(int) $item->id,
|
||||
$now
|
||||
);
|
||||
$backup = CloudAutomaticBackupBillingService::
|
||||
extendEntitlementWindowForCompensationLocked(
|
||||
(int) $period->id,
|
||||
$oldExpiry,
|
||||
$newExpiry,
|
||||
(int) $item->id,
|
||||
$now
|
||||
);
|
||||
// The traffic helper owns pool -> period lock order. Only
|
||||
// now re-read that retained row lock before changing it.
|
||||
$lockedPeriod = CloudServicePeriod::where(
|
||||
'id', (int) $period->id
|
||||
)->lockForUpdate()->first();
|
||||
if ($lockedPeriod === null
|
||||
|| (int) $lockedPeriod->instance_id
|
||||
!== (int) $instance->id
|
||||
|| (int) $lockedPeriod->ends_at !== $oldExpiry) {
|
||||
throw new CloudError(
|
||||
'实例服务周期在补偿事务中发生变化',
|
||||
409
|
||||
);
|
||||
}
|
||||
$lockedPeriod->ends_at = $newExpiry;
|
||||
$lockedPeriod->updated_at = $now;
|
||||
$lockedPeriod->save();
|
||||
$instance->updated_at = $now;
|
||||
$instance->save();
|
||||
|
||||
$job = $this->auditJob(
|
||||
$instance,
|
||||
$item,
|
||||
$batch,
|
||||
$operationId,
|
||||
$oldExpiry,
|
||||
$newExpiry,
|
||||
$now
|
||||
);
|
||||
$after = $this->snapshot($instance, $lockedPeriod);
|
||||
$item->service_period_id = (int) $lockedPeriod->id;
|
||||
$item->traffic_entitlement_id = $traffic === null
|
||||
? null : (int) $traffic->id;
|
||||
$item->backup_entitlement_id = $backup === null
|
||||
? null : (int) $backup->id;
|
||||
$item->source_invoice_id = $invoiceResult[
|
||||
'source_invoice_id'
|
||||
];
|
||||
$replacement = $invoiceResult['replacement_invoice'];
|
||||
$item->replacement_invoice_id = $replacement === null
|
||||
? null : (int) $replacement->id;
|
||||
$item->audit_job_id = (int) $job->id;
|
||||
$item->before_expires_at = $oldExpiry;
|
||||
$item->after_expires_at = $newExpiry;
|
||||
$item->before_period_ends_at = $oldExpiry;
|
||||
$item->after_period_ends_at = $newExpiry;
|
||||
$item->before_traffic_ends_at = $traffic === null
|
||||
? null : $oldExpiry;
|
||||
$item->after_traffic_ends_at = $traffic === null
|
||||
? null : $newExpiry;
|
||||
$item->before_backup_ends_at = $backup === null
|
||||
? null : $oldExpiry;
|
||||
$item->after_backup_ends_at = $backup === null
|
||||
? null : $newExpiry;
|
||||
// The reviewed snapshot is immutable evidence. Actual
|
||||
// application-time projections live only in the after
|
||||
// evidence because a renewal may have advanced the tail.
|
||||
$item->after_snapshot_json = self::json([
|
||||
'applied_from' => $before,
|
||||
'applied_to' => $after,
|
||||
]);
|
||||
$item->state = 'applied';
|
||||
$item->applied_at = $now;
|
||||
$item->terminal_at = $now;
|
||||
$item->lease_token = null;
|
||||
$item->lease_until = 0;
|
||||
$item->available_at = $now;
|
||||
$item->last_error_code = '';
|
||||
$item->last_error = '';
|
||||
$item->updated_at = $now;
|
||||
$item->save();
|
||||
(new CloudServiceCompensationService())
|
||||
->recalculateLocked($batch, $now);
|
||||
return $this->result($item);
|
||||
},
|
||||
3
|
||||
);
|
||||
} catch (CloudError $error) {
|
||||
return $this->recoverAfterApplyError(
|
||||
$itemId,
|
||||
$token,
|
||||
'projection_conflict',
|
||||
'补偿账本投影发生冲突,需要管理员核对',
|
||||
true
|
||||
);
|
||||
} catch (\Throwable $error) {
|
||||
return $this->recoverAfterApplyError(
|
||||
$itemId,
|
||||
$token,
|
||||
'apply_failed',
|
||||
'补偿执行遇到暂时故障,系统将安全重试',
|
||||
false
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/** Claim and execute at most one due item. */
|
||||
public function tick(?int $batchId = null): bool
|
||||
{
|
||||
$claim = $this->claimOne($batchId);
|
||||
if ($claim === null) {
|
||||
return false;
|
||||
}
|
||||
$this->applyClaim((int) $claim['item_id'], (string) $claim['token']);
|
||||
return true;
|
||||
}
|
||||
|
||||
/** @return array{state:string,code:string,message:string}|null */
|
||||
private function admissionLocked(CloudInstance $instance, int $now): ?array
|
||||
{
|
||||
if ((string) $instance->status === 'deleted') {
|
||||
return $this->decision(
|
||||
'rejected', 'instance_deleted', '实例已经删除'
|
||||
);
|
||||
}
|
||||
if (CloudInvoiceService::instanceDeleting($instance)) {
|
||||
return $this->decision(
|
||||
'deferred', 'instance_deleting', '实例删除流程尚未收敛'
|
||||
);
|
||||
}
|
||||
if ((string) $instance->service_status !== 'active') {
|
||||
return $this->decision(
|
||||
'rejected', 'service_not_active', '实例服务当前不是有效状态'
|
||||
);
|
||||
}
|
||||
if ($instance->expires_at === null
|
||||
|| (int) $instance->expires_at <= $now) {
|
||||
return $this->decision(
|
||||
'rejected', 'service_expired', '实例服务已经到期'
|
||||
);
|
||||
}
|
||||
if (CloudInstanceMigrationService::hasOpenMigration(
|
||||
(int) $instance->id
|
||||
)) {
|
||||
return $this->decision(
|
||||
'deferred', 'migration_open', '实例维护迁移尚未收敛'
|
||||
);
|
||||
}
|
||||
$jobs = CloudJobService::blockingQueryForInstance(
|
||||
(int) $instance->id
|
||||
)->orderBy('id')->lockForUpdate()->get([
|
||||
'id', 'phase', 'effect_resolution', 'finalization_incident',
|
||||
]);
|
||||
if ($jobs->contains(static fn ($job): bool =>
|
||||
(string) $job->phase !== 'terminal')) {
|
||||
return $this->decision(
|
||||
'deferred', 'instance_operation_open', '实例仍有操作正在执行'
|
||||
);
|
||||
}
|
||||
if (!$jobs->isEmpty()) {
|
||||
return $this->decision(
|
||||
'attention',
|
||||
'instance_operation_uncertain',
|
||||
'实例存在效果未知或最终化异常的操作'
|
||||
);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** @return array{state:string,code:string,message:string}|null */
|
||||
private function financialAdmissionLocked(
|
||||
CloudInstance $instance,
|
||||
int $oldExpiry
|
||||
): ?array {
|
||||
$invoices = CloudInvoice::where('instance_id', (int) $instance->id)
|
||||
->where('purpose', 'renewal')
|
||||
->where('renews_expires_at', $oldExpiry)
|
||||
->where('status', '<>', 'void')
|
||||
->orderBy('id')->lockForUpdate()->get();
|
||||
if ($invoices->count() > 1) {
|
||||
return $this->decision(
|
||||
'attention', 'renewal_invoice_duplicate',
|
||||
'旧到期时间存在重复续费账单'
|
||||
);
|
||||
}
|
||||
$invoice = $invoices->first();
|
||||
if ($invoice === null) {
|
||||
return null;
|
||||
}
|
||||
if ((int) $invoice->user_id !== (int) $instance->user_id
|
||||
|| $invoice->renewal_id !== null
|
||||
|| $invoice->paid_at !== null
|
||||
|| (string) $invoice->status === 'paid') {
|
||||
return $this->decision(
|
||||
'attention', 'renewal_invoice_settled',
|
||||
'旧到期时间的续费账单已经结算'
|
||||
);
|
||||
}
|
||||
if (!in_array((string) $invoice->status,
|
||||
['open', 'overdue', 'expired'], true)) {
|
||||
return $this->decision(
|
||||
'attention', 'renewal_invoice_state_invalid',
|
||||
'旧到期时间的续费账单状态异常'
|
||||
);
|
||||
}
|
||||
// Match payment-effect recovery's job -> payment lock order.
|
||||
$jobs = CloudJobService::blockingQueryForInvoice((int) $invoice->id)
|
||||
->orderBy('id')->lockForUpdate()->get([
|
||||
'id', 'phase', 'effect_resolution', 'finalization_incident',
|
||||
]);
|
||||
if ($jobs->contains(static fn ($job): bool =>
|
||||
(string) $job->phase !== 'terminal')) {
|
||||
return $this->decision(
|
||||
'deferred', 'renewal_payment_job_open',
|
||||
'旧续费账单的支付任务仍在执行'
|
||||
);
|
||||
}
|
||||
if (!$jobs->isEmpty()) {
|
||||
return $this->decision(
|
||||
'attention', 'renewal_payment_job_uncertain',
|
||||
'旧续费账单的支付任务效果未知'
|
||||
);
|
||||
}
|
||||
$payments = CloudInvoicePayment::where(
|
||||
'invoice_id', (int) $invoice->id
|
||||
)->orderBy('id')->lockForUpdate()->get();
|
||||
if ($payments->contains(static fn ($payment): bool => in_array(
|
||||
(string) $payment->status,
|
||||
['creating', 'pending'],
|
||||
true
|
||||
))) {
|
||||
return $this->decision(
|
||||
'deferred', 'renewal_payment_open',
|
||||
'旧续费账单仍有支付正在处理'
|
||||
);
|
||||
}
|
||||
if ($payments->contains(static fn ($payment): bool => !in_array(
|
||||
(string) $payment->status,
|
||||
['failed', 'expired', 'not_created'],
|
||||
true
|
||||
))) {
|
||||
return $this->decision(
|
||||
'attention', 'renewal_payment_uncertain',
|
||||
'旧续费账单存在已支付或效果未知的支付记录'
|
||||
);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private function auditJob(
|
||||
CloudInstance $instance,
|
||||
CloudServiceCompensationItem $item,
|
||||
CloudServiceCompensation $batch,
|
||||
string $operationId,
|
||||
int $oldExpiry,
|
||||
int $newExpiry,
|
||||
int $now
|
||||
): CloudJob {
|
||||
$fingerprint = hash('sha256', self::json([
|
||||
'batch_key' => (string) $batch->compensation_key,
|
||||
'item_key' => (string) $item->item_key,
|
||||
'incident_key' => (string) $batch->incident_key,
|
||||
'instance_id' => (int) $instance->id,
|
||||
'requested_seconds' => (int) $item->requested_seconds,
|
||||
'previous_expires_at' => $oldExpiry,
|
||||
'new_expires_at' => $newExpiry,
|
||||
]));
|
||||
$job = new CloudJob();
|
||||
$job->operation_id = $operationId;
|
||||
$job->request_idempotency_key = 'service-compensation-'
|
||||
. (string) $item->item_key;
|
||||
$job->request_fingerprint = $fingerprint;
|
||||
$job->instance_id = (int) $instance->id;
|
||||
$job->user_id = (int) $instance->user_id;
|
||||
$job->host_id = null;
|
||||
$job->kind = 'service_compensation';
|
||||
$job->execution_mode = 'local';
|
||||
$job->sealed_payload = CloudSecrets::encrypt([
|
||||
'compensation_key' => (string) $batch->compensation_key,
|
||||
'item_key' => (string) $item->item_key,
|
||||
'incident_key' => (string) $batch->incident_key,
|
||||
], 'job-payload');
|
||||
$job->dispatch_deadline_at = $now + CloudJob::DISPATCH_DEADLINE_SECONDS;
|
||||
$job->available_at = $now;
|
||||
$job->created_at = $job->updated_at = $now;
|
||||
$job->save();
|
||||
CloudJobService::completeWithoutDispatchLocked($job, [
|
||||
'compensation_key' => (string) $batch->compensation_key,
|
||||
'item_key' => (string) $item->item_key,
|
||||
'incident_key' => (string) $batch->incident_key,
|
||||
'seconds_added' => (int) $item->requested_seconds,
|
||||
'previous_expires_at' => $oldExpiry,
|
||||
'new_expires_at' => $newExpiry,
|
||||
]);
|
||||
return $job;
|
||||
}
|
||||
|
||||
/** @return array<string, mixed> */
|
||||
private function finishLocked(
|
||||
CloudServiceCompensation $batch,
|
||||
CloudServiceCompensationItem $item,
|
||||
string $state,
|
||||
string $code,
|
||||
string $message,
|
||||
int $now
|
||||
): array {
|
||||
if ($state === 'deferred'
|
||||
&& (int) $item->attempts >= self::MAX_ATTEMPTS) {
|
||||
$state = 'attention';
|
||||
$code = 'retry_exhausted';
|
||||
$message = '补偿任务重试次数已耗尽,需要管理员核对';
|
||||
}
|
||||
if (!in_array($state, ['deferred', 'rejected', 'attention'], true)) {
|
||||
throw new \LogicException('Invalid compensation transition.');
|
||||
}
|
||||
$item->state = $state;
|
||||
$item->lease_token = null;
|
||||
$item->lease_until = 0;
|
||||
$item->available_at = $state === 'deferred'
|
||||
? $now + $this->backoff((int) $item->attempts) : $now;
|
||||
$item->terminal_at = $state === 'deferred' ? null : $now;
|
||||
$item->last_error_code = substr($code, 0, 64);
|
||||
$item->last_error = mb_substr($message, 0, 500);
|
||||
$item->updated_at = $now;
|
||||
$item->save();
|
||||
(new CloudServiceCompensationService())
|
||||
->recalculateLocked($batch, $now);
|
||||
return $this->result($item);
|
||||
}
|
||||
|
||||
/** @return array<string, mixed> */
|
||||
private function transitionClaim(
|
||||
int $itemId,
|
||||
string $token,
|
||||
string $state,
|
||||
string $code,
|
||||
string $message
|
||||
): array {
|
||||
return DB::connection('default')->transaction(
|
||||
function () use ($itemId, $token, $state, $code, $message): array {
|
||||
$hint = CloudServiceCompensationItem::find($itemId);
|
||||
if ($hint === null) {
|
||||
throw new CloudError('实例补偿明细不存在', 404);
|
||||
}
|
||||
$batch = CloudServiceCompensation::where(
|
||||
'id', (int) $hint->batch_id
|
||||
)->lockForUpdate()->first();
|
||||
$item = CloudServiceCompensationItem::where('id', $itemId)
|
||||
->lockForUpdate()->first();
|
||||
if ($batch === null || $item === null) {
|
||||
throw new CloudError('实例补偿账本不完整', 409);
|
||||
}
|
||||
if ((string) $item->state === 'applied'
|
||||
|| (string) $item->state !== 'applying'
|
||||
|| !is_string($item->lease_token)
|
||||
|| !hash_equals((string) $item->lease_token, $token)) {
|
||||
return $this->result($item);
|
||||
}
|
||||
return $this->finishLocked(
|
||||
$batch,
|
||||
$item,
|
||||
$state,
|
||||
$code,
|
||||
$message,
|
||||
$this->now()
|
||||
);
|
||||
},
|
||||
3
|
||||
);
|
||||
}
|
||||
|
||||
/** @return array<string, mixed> */
|
||||
private function recoverAfterApplyError(
|
||||
int $itemId,
|
||||
string $token,
|
||||
string $code,
|
||||
string $message,
|
||||
bool $attention
|
||||
): array {
|
||||
$latest = CloudServiceCompensationItem::find($itemId);
|
||||
if ($latest !== null && (string) $latest->state === 'applied') {
|
||||
return $this->result($latest);
|
||||
}
|
||||
return $this->transitionClaim(
|
||||
$itemId,
|
||||
$token,
|
||||
$attention ? 'attention' : 'deferred',
|
||||
$code,
|
||||
$message
|
||||
);
|
||||
}
|
||||
|
||||
/** @return array{state:string,code:string,message:string} */
|
||||
private function decision(
|
||||
string $state,
|
||||
string $code,
|
||||
string $message
|
||||
): array {
|
||||
return compact('state', 'code', 'message');
|
||||
}
|
||||
|
||||
private function backoff(int $attempts): int
|
||||
{
|
||||
return min(3600, 30 * (2 ** max(0, min(7, $attempts - 1))));
|
||||
}
|
||||
|
||||
/** @return array<string, mixed> */
|
||||
private function snapshot(
|
||||
CloudInstance $instance,
|
||||
CloudServicePeriod $period
|
||||
): array {
|
||||
return [
|
||||
'instance' => [
|
||||
'id' => (int) $instance->id,
|
||||
'user_id' => (int) $instance->user_id,
|
||||
'host_id' => (int) $instance->host_id,
|
||||
'name' => (string) $instance->name,
|
||||
'status' => (string) $instance->status,
|
||||
'service_status' => (string) $instance->service_status,
|
||||
'expires_at' => (int) $instance->expires_at,
|
||||
'plan_id' => (int) $instance->plan_id,
|
||||
'offer_id' => (int) $instance->offer_id,
|
||||
'renewal_price' => (string) $instance->renewal_price,
|
||||
'renewal_days' => (int) $instance->renewal_days,
|
||||
'renewal_traffic_bytes' =>
|
||||
(int) $instance->renewal_traffic_bytes,
|
||||
],
|
||||
'service_period' => [
|
||||
'id' => (int) $period->id,
|
||||
'starts_at' => (int) $period->starts_at,
|
||||
'ends_at' => (int) $period->ends_at,
|
||||
'status' => (string) $period->status,
|
||||
'traffic_quota_mode' =>
|
||||
(string) $period->traffic_quota_mode,
|
||||
'traffic_quota_bytes' =>
|
||||
(int) $period->traffic_quota_bytes,
|
||||
],
|
||||
];
|
||||
}
|
||||
|
||||
/** @return array<string, mixed> */
|
||||
private function result(CloudServiceCompensationItem $item): array
|
||||
{
|
||||
return [
|
||||
'item_id' => (int) $item->id,
|
||||
'item_key' => (string) $item->item_key,
|
||||
'batch_id' => (int) $item->batch_id,
|
||||
'instance_id' => (int) $item->instance_id,
|
||||
'state' => (string) $item->state,
|
||||
'attempts' => (int) $item->attempts,
|
||||
'audit_job_id' => $item->audit_job_id === null
|
||||
? null : (int) $item->audit_job_id,
|
||||
'before_expires_at' => $item->before_expires_at === null
|
||||
? null : (int) $item->before_expires_at,
|
||||
'after_expires_at' => $item->after_expires_at === null
|
||||
? null : (int) $item->after_expires_at,
|
||||
'available_at' => (int) $item->available_at,
|
||||
'last_error_code' => (string) $item->last_error_code,
|
||||
];
|
||||
}
|
||||
|
||||
private function assertBatchId(?int $batchId): void
|
||||
{
|
||||
if ($batchId !== null && $batchId < 1) {
|
||||
throw new \InvalidArgumentException('Compensation batch is invalid.');
|
||||
}
|
||||
}
|
||||
|
||||
private function now(): int
|
||||
{
|
||||
return max(1, (int) ($this->clock)());
|
||||
}
|
||||
|
||||
private static function json(array $value): string
|
||||
{
|
||||
return json_encode($value, JSON_THROW_ON_ERROR
|
||||
| JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE);
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,152 @@
|
||||
<?php
|
||||
declare(strict_types=1);
|
||||
|
||||
$root = dirname(__DIR__);
|
||||
$source = file_get_contents(
|
||||
$root . '/app/Services/Cloud/CloudServiceCompensationExecutionService.php'
|
||||
);
|
||||
$assert = static function (bool $condition, string $message): void {
|
||||
if (!$condition) {
|
||||
throw new RuntimeException($message);
|
||||
}
|
||||
};
|
||||
$section = static function (
|
||||
string $source,
|
||||
string $start,
|
||||
string $next
|
||||
): string {
|
||||
$offset = strpos($source, $start);
|
||||
$end = $offset === false ? false : strpos($source, $next, $offset + 1);
|
||||
if ($offset === false || $end === false || $end <= $offset) {
|
||||
throw new RuntimeException('Unable to isolate execution contract section.');
|
||||
}
|
||||
return substr($source, $offset, $end - $offset);
|
||||
};
|
||||
|
||||
$assert($source !== false, 'Compensation execution source could not be read.');
|
||||
|
||||
$apply = $section(
|
||||
$source,
|
||||
'public function applyClaim(',
|
||||
'public function tick('
|
||||
);
|
||||
$ownerLock = strpos($apply, 'User::where(');
|
||||
$instanceLock = strpos($apply, 'CloudInstance::where(');
|
||||
$batchLock = strpos($apply, 'CloudServiceCompensation::where(');
|
||||
$itemLock = strpos($apply, 'CloudServiceCompensationItem::where(');
|
||||
$assert($ownerLock !== false && $instanceLock !== false
|
||||
&& $batchLock !== false && $itemLock !== false
|
||||
&& $ownerLock < $instanceLock && $instanceLock < $batchLock
|
||||
&& $batchLock < $itemLock,
|
||||
'Apply lost the canonical owner-instance-batch-item lock order.');
|
||||
$assert(str_contains($apply,
|
||||
'// Canonical compensation lock order: owner -> instance ->')
|
||||
&& str_contains($apply, '->lockForUpdate()->first();'),
|
||||
'The canonical compensation lock order is no longer explicit.');
|
||||
|
||||
$assert(substr_count($apply, "->orderByDesc('ends_at')") >= 2
|
||||
&& substr_count($apply, "->orderByDesc('id')") >= 2
|
||||
&& str_contains($apply, "'service_period_changed'"),
|
||||
'Compensation no longer revalidates the latest tail service period.');
|
||||
|
||||
foreach ([
|
||||
'CloudInstanceMigrationService::hasOpenMigration(',
|
||||
'CloudJobService::blockingQueryForInstance(',
|
||||
"'finalization_incident'",
|
||||
"'instance_operation_open'",
|
||||
"'instance_operation_uncertain'",
|
||||
'CloudJobService::blockingQueryForInvoice(',
|
||||
"['creating', 'pending']",
|
||||
"['failed', 'expired', 'not_created']",
|
||||
"'renewal_payment_open'",
|
||||
"'renewal_payment_uncertain'",
|
||||
] as $fence) {
|
||||
$assert(str_contains($source, $fence),
|
||||
'Compensation execution lost fence: ' . $fence);
|
||||
}
|
||||
|
||||
$invoiceProjection = strpos($apply,
|
||||
'reconcileRenewalAfterCompensationLocked(');
|
||||
$trafficProjection = strpos($apply,
|
||||
'extendEntitlementWindowForCompensationLocked(',
|
||||
(int) $invoiceProjection);
|
||||
$backupProjection = $trafficProjection === false ? false : strpos(
|
||||
$apply,
|
||||
'extendEntitlementWindowForCompensationLocked(',
|
||||
$trafficProjection + 1
|
||||
);
|
||||
$periodSave = strpos($apply, '$lockedPeriod->save();');
|
||||
$instanceSave = strpos($apply, '$instance->save();');
|
||||
$audit = strpos($apply, '$job = $this->auditJob(');
|
||||
$itemSave = strpos($apply, '$item->save();', (int) $audit);
|
||||
$recalculate = strpos($apply, '->recalculateLocked($batch, $now);');
|
||||
$assert($invoiceProjection !== false && $trafficProjection !== false
|
||||
&& $backupProjection !== false && $periodSave !== false
|
||||
&& $instanceSave !== false && $audit !== false
|
||||
&& $itemSave !== false && $recalculate !== false
|
||||
&& $invoiceProjection < $trafficProjection
|
||||
&& $trafficProjection < $backupProjection
|
||||
&& $backupProjection < $periodSave
|
||||
&& $periodSave < $instanceSave
|
||||
&& $instanceSave < $audit && $audit < $itemSave
|
||||
&& $itemSave < $recalculate,
|
||||
'Commercial projections, core rows and audit do not converge in order.');
|
||||
$assert(str_contains($apply, "DB::connection('default')->transaction(")
|
||||
&& str_contains($apply, "\$item->state = 'applied'")
|
||||
&& str_contains($apply, '$item->audit_job_id = (int) $job->id'),
|
||||
'The atomic compensation transaction lost its applied audit linkage.');
|
||||
|
||||
$auditMethod = $section(
|
||||
$source,
|
||||
'private function auditJob(',
|
||||
'private function finishLocked('
|
||||
);
|
||||
foreach ([
|
||||
"\$job->kind = 'service_compensation'",
|
||||
"\$job->execution_mode = 'local'",
|
||||
"\$job->host_id = null",
|
||||
'CloudJobService::completeWithoutDispatchLocked($job, [',
|
||||
"'seconds_added' => (int) \$item->requested_seconds",
|
||||
"'previous_expires_at' => \$oldExpiry",
|
||||
"'new_expires_at' => \$newExpiry",
|
||||
] as $auditContract) {
|
||||
$assert(str_contains($auditMethod, $auditContract),
|
||||
'Local audit contract is missing: ' . $auditContract);
|
||||
}
|
||||
|
||||
$recovery = $section(
|
||||
$source,
|
||||
'private function recoverAfterApplyError(',
|
||||
'private function decision('
|
||||
);
|
||||
$latest = strpos($recovery,
|
||||
'$latest = CloudServiceCompensationItem::find($itemId);');
|
||||
$applied = strpos($recovery,
|
||||
"(string) \$latest->state === 'applied'");
|
||||
$transition = strpos($recovery, 'return $this->transitionClaim(');
|
||||
$assert($latest !== false && $applied !== false && $transition !== false
|
||||
&& $latest < $applied && $applied < $transition,
|
||||
'Error recovery may replay before checking for an already committed item.');
|
||||
|
||||
foreach ([
|
||||
'public const MAX_ATTEMPTS = 8;',
|
||||
'public const LEASE_SECONDS = 120;',
|
||||
"->where('attempts', '<', self::MAX_ATTEMPTS)",
|
||||
"? 'retry_exhausted' : 'worker_lease_expired'",
|
||||
"'deferred', 'rejected', 'attention'",
|
||||
'min(3600, 30 * (2 ** max(0, min(7, $attempts - 1))))',
|
||||
] as $leaseContract) {
|
||||
$assert(str_contains($source, $leaseContract),
|
||||
'Bounded retry contract is missing: ' . $leaseContract);
|
||||
}
|
||||
|
||||
$assert(!str_contains($source, 'new CloudTrafficEntitlement')
|
||||
&& !str_contains($source, 'new CloudBackupPeriodEntitlement')
|
||||
&& !str_contains($source, 'new CloudBackupSubscription')
|
||||
&& !str_contains($source, 'granted_bytes =')
|
||||
&& !str_contains($source, 'traffic_quota_bytes =')
|
||||
&& !str_contains($source, '->money =')
|
||||
&& !str_contains($source, 'Wallet'),
|
||||
'Compensation execution may grant traffic/backup cover or mutate wallet.');
|
||||
|
||||
echo "Service compensation execution contracts passed.\n";
|
||||
@ -0,0 +1,206 @@
|
||||
<?php
|
||||
declare(strict_types=1);
|
||||
|
||||
require dirname(__DIR__) . '/vendor/autoload.php';
|
||||
|
||||
use App\Services\Cloud\CloudServiceCompensationExecutionService;
|
||||
use Illuminate\Database\Capsule\Manager as Capsule;
|
||||
use Illuminate\Database\Schema\Blueprint;
|
||||
|
||||
$assert = static function (bool $condition, string $message): void {
|
||||
if (!$condition) {
|
||||
throw new RuntimeException($message);
|
||||
}
|
||||
};
|
||||
|
||||
$capsule = new Capsule();
|
||||
$capsule->addConnection([
|
||||
'driver' => 'sqlite',
|
||||
'database' => ':memory:',
|
||||
], 'default');
|
||||
$capsule->setAsGlobal();
|
||||
$capsule->bootEloquent();
|
||||
$schema = $capsule->schema('default');
|
||||
|
||||
$schema->create(
|
||||
'cloud_service_compensations',
|
||||
static function (Blueprint $table): void {
|
||||
$table->increments('id');
|
||||
$table->integer('instance_count');
|
||||
$table->integer('applied_count')->default(0);
|
||||
$table->integer('deferred_count')->default(0);
|
||||
$table->integer('rejected_count')->default(0);
|
||||
$table->integer('attention_count')->default(0);
|
||||
$table->string('state');
|
||||
$table->string('last_error_code')->default('');
|
||||
$table->integer('terminal_at')->nullable();
|
||||
$table->integer('updated_at');
|
||||
}
|
||||
);
|
||||
$schema->create(
|
||||
'cloud_service_compensation_items',
|
||||
static function (Blueprint $table): void {
|
||||
$table->increments('id');
|
||||
$table->string('item_key');
|
||||
$table->integer('batch_id');
|
||||
$table->integer('instance_id');
|
||||
$table->string('state');
|
||||
$table->integer('attempts')->default(0);
|
||||
$table->integer('available_at');
|
||||
$table->string('lease_token')->nullable();
|
||||
$table->integer('lease_until')->default(0);
|
||||
$table->integer('audit_job_id')->nullable();
|
||||
$table->integer('before_expires_at')->nullable();
|
||||
$table->integer('after_expires_at')->nullable();
|
||||
$table->integer('applied_at')->nullable();
|
||||
$table->integer('terminal_at')->nullable();
|
||||
$table->string('last_error_code')->default('');
|
||||
$table->string('last_error')->default('');
|
||||
$table->integer('updated_at');
|
||||
}
|
||||
);
|
||||
|
||||
$now = 2_100_000_000;
|
||||
$service = new CloudServiceCompensationExecutionService(
|
||||
static function () use (&$now): int {
|
||||
return $now;
|
||||
}
|
||||
);
|
||||
$batch = static function (int $id, int $at): array {
|
||||
return [
|
||||
'id' => $id,
|
||||
'instance_count' => 1,
|
||||
'applied_count' => 0,
|
||||
'deferred_count' => 0,
|
||||
'rejected_count' => 0,
|
||||
'attention_count' => 0,
|
||||
'state' => 'pending',
|
||||
'last_error_code' => '',
|
||||
'terminal_at' => null,
|
||||
'updated_at' => $at,
|
||||
];
|
||||
};
|
||||
$item = static function (
|
||||
int $id,
|
||||
int $batchId,
|
||||
int $attempts,
|
||||
int $at
|
||||
): array {
|
||||
return [
|
||||
'id' => $id,
|
||||
'item_key' => str_pad((string) $id, 32, 'a'),
|
||||
'batch_id' => $batchId,
|
||||
'instance_id' => 100 + $id,
|
||||
'state' => 'pending',
|
||||
'attempts' => $attempts,
|
||||
'available_at' => $at,
|
||||
'lease_token' => null,
|
||||
'lease_until' => 0,
|
||||
'audit_job_id' => null,
|
||||
'before_expires_at' => null,
|
||||
'after_expires_at' => null,
|
||||
'applied_at' => null,
|
||||
'terminal_at' => null,
|
||||
'last_error_code' => '',
|
||||
'last_error' => '',
|
||||
'updated_at' => $at,
|
||||
];
|
||||
};
|
||||
|
||||
Capsule::table('cloud_service_compensations')->insert($batch(1, $now));
|
||||
Capsule::table('cloud_service_compensation_items')->insert(
|
||||
$item(1, 1, 0, $now)
|
||||
);
|
||||
|
||||
$claim = $service->claimOne(1);
|
||||
$assert(is_array($claim), 'due item should be claimed');
|
||||
$assert((int) $claim['item_id'] === 1, 'claim should identify the item');
|
||||
$assert((int) $claim['lease_until'] === $now + 120,
|
||||
'claim should create exactly a 120 second lease');
|
||||
$assert(preg_match('/^[a-f0-9]{32}$/D', (string) $claim['token']) === 1,
|
||||
'claim should return a strong lease token');
|
||||
$claimed = Capsule::table('cloud_service_compensation_items')
|
||||
->where('id', 1)->first();
|
||||
$assert((string) $claimed->state === 'applying'
|
||||
&& (int) $claimed->attempts === 1,
|
||||
'claim should enter applying and increment attempts once');
|
||||
$assert((string) Capsule::table('cloud_service_compensations')
|
||||
->where('id', 1)->value('state') === 'applying',
|
||||
'claim should project an applying batch');
|
||||
|
||||
$now += 119;
|
||||
$assert($service->recoverExpired(1) === 0,
|
||||
'a live lease must not be recovered early');
|
||||
$now += 1;
|
||||
$assert($service->recoverExpired(1) === 1,
|
||||
'an expired lease should be recovered once');
|
||||
$recovered = Capsule::table('cloud_service_compensation_items')
|
||||
->where('id', 1)->first();
|
||||
$assert((string) $recovered->state === 'deferred',
|
||||
'first lost lease should defer safely');
|
||||
$assert((int) $recovered->available_at === $now + 30,
|
||||
'first retry should use a 30 second backoff');
|
||||
$assert($recovered->lease_token === null
|
||||
&& (int) $recovered->lease_until === 0,
|
||||
'recovery should clear the abandoned lease');
|
||||
$assert((string) $recovered->last_error_code === 'worker_lease_expired',
|
||||
'recovery should preserve a stable diagnostic code');
|
||||
$batchOne = Capsule::table('cloud_service_compensations')
|
||||
->where('id', 1)->first();
|
||||
$assert((string) $batchOne->state === 'applying'
|
||||
&& (int) $batchOne->deferred_count === 1
|
||||
&& $batchOne->terminal_at === null,
|
||||
'a deferred item should keep the batch nonterminal and counted');
|
||||
$assert($service->claimOne(1) === null,
|
||||
'a deferred item must remain unavailable during backoff');
|
||||
|
||||
Capsule::table('cloud_service_compensations')->insert($batch(2, $now));
|
||||
Capsule::table('cloud_service_compensation_items')->insert(
|
||||
$item(2, 2, 7, $now)
|
||||
);
|
||||
$lastClaim = $service->claimOne(2);
|
||||
$assert(is_array($lastClaim), 'seventh-attempt item should receive final claim');
|
||||
$attemptEight = Capsule::table('cloud_service_compensation_items')
|
||||
->where('id', 2)->first();
|
||||
$assert((int) $attemptEight->attempts === 8
|
||||
&& (string) $attemptEight->state === 'applying',
|
||||
'final claim should be attempt eight');
|
||||
$now += 120;
|
||||
$assert($service->recoverExpired(2) === 1,
|
||||
'the final abandoned lease should converge');
|
||||
$terminal = Capsule::table('cloud_service_compensation_items')
|
||||
->where('id', 2)->first();
|
||||
$assert((string) $terminal->state === 'attention'
|
||||
&& (string) $terminal->last_error_code === 'retry_exhausted'
|
||||
&& (int) $terminal->terminal_at === $now,
|
||||
'attempt eight should become terminal attention');
|
||||
$batchTwo = Capsule::table('cloud_service_compensations')
|
||||
->where('id', 2)->first();
|
||||
$assert((string) $batchTwo->state === 'attention'
|
||||
&& (int) $batchTwo->attention_count === 1
|
||||
&& (int) $batchTwo->terminal_at === $now,
|
||||
'terminal item should project batch attention');
|
||||
$assert($service->recoverExpired(2) === 0,
|
||||
'terminal attention must not be recovered again');
|
||||
|
||||
// A corrupt due item under a terminal batch must not starve later healthy
|
||||
// batches in the global queue.
|
||||
Capsule::table('cloud_service_compensation_items')->delete();
|
||||
Capsule::table('cloud_service_compensations')->delete();
|
||||
$terminalBatch = $batch(3, $now);
|
||||
$terminalBatch['state'] = 'attention';
|
||||
$terminalBatch['attention_count'] = 1;
|
||||
$terminalBatch['terminal_at'] = $now;
|
||||
Capsule::table('cloud_service_compensations')->insert($terminalBatch);
|
||||
Capsule::table('cloud_service_compensation_items')->insert(
|
||||
$item(3, 3, 0, $now)
|
||||
);
|
||||
Capsule::table('cloud_service_compensations')->insert($batch(4, $now));
|
||||
Capsule::table('cloud_service_compensation_items')->insert(
|
||||
$item(4, 4, 0, $now)
|
||||
);
|
||||
$healthyClaim = $service->claimOne();
|
||||
$assert(is_array($healthyClaim) && (int) $healthyClaim['item_id'] === 4,
|
||||
'a terminal parent batch starved a later healthy queue item');
|
||||
|
||||
echo "Service compensation execution lease tests passed.\n";
|
||||
@ -0,0 +1,423 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Cloud {
|
||||
/** Keep this focused transaction test independent from live key files. */
|
||||
final class CloudSecrets
|
||||
{
|
||||
public static function encrypt(array $payload, string $purpose): string
|
||||
{
|
||||
return json_encode([
|
||||
'purpose' => $purpose,
|
||||
'payload' => $payload,
|
||||
], JSON_THROW_ON_ERROR);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
namespace {
|
||||
require dirname(__DIR__) . '/vendor/autoload.php';
|
||||
|
||||
use App\Models\CloudInstance;
|
||||
use App\Models\CloudJob;
|
||||
use App\Models\CloudServiceCompensation;
|
||||
use App\Models\CloudServiceCompensationItem;
|
||||
use App\Models\CloudServicePeriod;
|
||||
use App\Services\Cloud\CloudServiceCompensationExecutionService;
|
||||
use Illuminate\Database\Capsule\Manager as Capsule;
|
||||
use Illuminate\Database\Schema\Blueprint;
|
||||
|
||||
$assert = static function (bool $condition, string $message): void {
|
||||
if (!$condition) {
|
||||
throw new RuntimeException($message);
|
||||
}
|
||||
};
|
||||
|
||||
$capsule = new Capsule();
|
||||
$capsule->addConnection([
|
||||
'driver' => 'sqlite', 'database' => ':memory:',
|
||||
], 'default');
|
||||
$capsule->setAsGlobal();
|
||||
$capsule->bootEloquent();
|
||||
$schema = $capsule->schema('default');
|
||||
|
||||
$schema->create('user', static function (Blueprint $table): void {
|
||||
$table->increments('id');
|
||||
$table->string('money');
|
||||
$table->string('email')->default('');
|
||||
$table->integer('enable')->default(1);
|
||||
$table->boolean('is_admin')->default(false);
|
||||
});
|
||||
$schema->create('cloud_instances', static function (Blueprint $table): void {
|
||||
$table->increments('id');
|
||||
$table->integer('host_id');
|
||||
$table->integer('user_id');
|
||||
$table->integer('order_id')->nullable();
|
||||
$table->integer('plan_id')->nullable();
|
||||
$table->integer('offer_id')->nullable();
|
||||
$table->integer('current_period_id')->nullable();
|
||||
$table->string('uuid');
|
||||
$table->string('name');
|
||||
$table->string('status');
|
||||
$table->string('service_status');
|
||||
$table->string('network_status');
|
||||
$table->integer('expires_at')->nullable();
|
||||
$table->string('renewal_price')->nullable();
|
||||
$table->integer('renewal_days')->default(0);
|
||||
$table->integer('renewal_traffic_bytes')->default(0);
|
||||
$table->integer('traffic_quota_bytes')->default(0);
|
||||
$table->text('spec_json');
|
||||
$table->integer('created_at');
|
||||
$table->integer('updated_at');
|
||||
});
|
||||
$schema->create('cloud_service_periods', static function (Blueprint $table): void {
|
||||
$table->increments('id');
|
||||
$table->integer('instance_id');
|
||||
$table->integer('user_id');
|
||||
$table->integer('plan_id')->nullable();
|
||||
$table->integer('order_id')->nullable();
|
||||
$table->integer('renewal_id')->nullable();
|
||||
$table->integer('traffic_pool_group_id')->nullable();
|
||||
$table->string('traffic_quota_mode')->default('finite');
|
||||
$table->integer('starts_at');
|
||||
$table->integer('ends_at');
|
||||
$table->integer('traffic_quota_bytes')->default(0);
|
||||
$table->integer('download_base_bytes')->nullable();
|
||||
$table->integer('upload_base_bytes')->nullable();
|
||||
$table->integer('download_used_bytes')->default(0);
|
||||
$table->integer('upload_used_bytes')->default(0);
|
||||
$table->string('status');
|
||||
$table->integer('activated_at')->nullable();
|
||||
$table->integer('created_at');
|
||||
$table->integer('updated_at');
|
||||
});
|
||||
$schema->create('cloud_invoices', static function (Blueprint $table): void {
|
||||
$table->increments('id');
|
||||
$table->string('invoice_no')->unique();
|
||||
$table->string('purpose');
|
||||
$table->integer('instance_id')->nullable();
|
||||
$table->integer('order_id')->nullable();
|
||||
$table->integer('user_id');
|
||||
$table->integer('plan_id')->nullable();
|
||||
$table->integer('renewal_id')->nullable();
|
||||
$table->integer('renews_expires_at')->nullable();
|
||||
$table->integer('due_at');
|
||||
$table->string('amount');
|
||||
$table->string('currency')->default('USD');
|
||||
$table->integer('validity_days');
|
||||
$table->integer('traffic_quota_bytes');
|
||||
$table->string('status');
|
||||
$table->string('paid_via')->nullable();
|
||||
$table->string('payment_reference')->nullable();
|
||||
$table->integer('paid_at')->nullable();
|
||||
$table->integer('created_at');
|
||||
$table->integer('updated_at');
|
||||
});
|
||||
$schema->create('cloud_invoice_payments', static function (Blueprint $table): void {
|
||||
$table->increments('id');
|
||||
$table->integer('invoice_id');
|
||||
$table->integer('user_id');
|
||||
$table->string('provider');
|
||||
$table->string('provider_order_no');
|
||||
$table->string('idempotency_key');
|
||||
$table->string('amount');
|
||||
$table->string('status');
|
||||
$table->integer('created_at');
|
||||
$table->integer('updated_at');
|
||||
});
|
||||
$schema->create('cloud_jobs', static function (Blueprint $table): void {
|
||||
$table->increments('id');
|
||||
$table->string('operation_id')->nullable()->unique();
|
||||
$table->string('request_idempotency_key')->nullable();
|
||||
$table->string('request_fingerprint')->nullable();
|
||||
$table->integer('parent_job_id')->nullable();
|
||||
$table->integer('supersedes_job_id')->nullable();
|
||||
$table->integer('vpc_repair_root_job_id')->nullable();
|
||||
$table->integer('order_id')->nullable();
|
||||
$table->integer('invoice_id')->nullable();
|
||||
$table->integer('instance_id')->nullable();
|
||||
$table->integer('snapshot_id')->nullable();
|
||||
$table->integer('backup_id')->nullable();
|
||||
$table->integer('backup_restore_id')->nullable();
|
||||
$table->integer('backup_recovery_id')->nullable();
|
||||
$table->integer('public_network_transition_id')->nullable();
|
||||
$table->integer('vpc_id')->nullable();
|
||||
$table->integer('vpc_port_id')->nullable();
|
||||
$table->integer('vpc_batch_id')->nullable();
|
||||
$table->integer('user_id');
|
||||
$table->integer('host_id')->nullable();
|
||||
$table->string('kind');
|
||||
$table->string('execution_mode');
|
||||
$table->string('phase')->default('accepted');
|
||||
$table->string('outcome')->nullable();
|
||||
$table->string('status')->default('queued');
|
||||
$table->text('sealed_payload');
|
||||
$table->integer('attempts')->default(0);
|
||||
$table->integer('dispatch_attempts')->default(0);
|
||||
$table->integer('dispatch_deadline_at');
|
||||
$table->integer('dispatch_started_at')->nullable();
|
||||
$table->integer('agent_accepted_at')->nullable();
|
||||
$table->integer('receipt_deadline_at')->nullable();
|
||||
$table->integer('terminal_at')->nullable();
|
||||
$table->boolean('effect_may_have_applied')->nullable();
|
||||
$table->boolean('safe_to_resubmit')->nullable();
|
||||
$table->string('effect_resolution')->default('not_applicable');
|
||||
$table->integer('effect_resolved_at')->nullable();
|
||||
$table->integer('effect_resolved_by')->nullable();
|
||||
$table->text('effect_resolution_reason')->default('');
|
||||
$table->boolean('finalization_incident')->default(false);
|
||||
$table->string('finalization_error_code')->nullable();
|
||||
$table->string('completion_event_id')->nullable();
|
||||
$table->integer('read_at')->nullable();
|
||||
$table->integer('dispatched_at')->nullable();
|
||||
$table->string('lease_token')->nullable();
|
||||
$table->integer('lease_until')->default(0);
|
||||
$table->integer('available_at');
|
||||
$table->string('error_code')->nullable();
|
||||
$table->text('result_json')->nullable();
|
||||
$table->integer('created_at');
|
||||
$table->integer('updated_at');
|
||||
});
|
||||
$schema->create('cloud_traffic_entitlements', static function (Blueprint $table): void {
|
||||
$table->increments('id');
|
||||
$table->integer('service_period_id');
|
||||
});
|
||||
$schema->create('cloud_backup_subscriptions', static function (Blueprint $table): void {
|
||||
$table->increments('id');
|
||||
$table->integer('instance_id')->unique();
|
||||
$table->integer('user_id');
|
||||
});
|
||||
$schema->create('cloud_backup_period_entitlements', static function (Blueprint $table): void {
|
||||
$table->increments('id');
|
||||
$table->integer('subscription_id');
|
||||
$table->integer('instance_id');
|
||||
$table->integer('service_period_id')->unique();
|
||||
$table->integer('user_id');
|
||||
});
|
||||
$schema->create('cloud_service_compensations', static function (Blueprint $table): void {
|
||||
$table->increments('id');
|
||||
$table->string('compensation_key')->unique();
|
||||
$table->string('incident_key');
|
||||
$table->integer('requested_by_user_id');
|
||||
$table->string('request_idempotency_key');
|
||||
$table->string('request_fingerprint');
|
||||
$table->string('scope_type');
|
||||
$table->integer('scope_id');
|
||||
$table->text('scope_snapshot_json');
|
||||
$table->string('reason_text');
|
||||
$table->integer('incident_starts_at')->nullable();
|
||||
$table->integer('incident_ends_at')->nullable();
|
||||
$table->integer('compensation_seconds');
|
||||
$table->boolean('notify_users');
|
||||
$table->string('audience_digest');
|
||||
$table->integer('instance_count');
|
||||
$table->integer('recipient_count')->default(0);
|
||||
$table->integer('applied_count')->default(0);
|
||||
$table->integer('deferred_count')->default(0);
|
||||
$table->integer('rejected_count')->default(0);
|
||||
$table->integer('attention_count')->default(0);
|
||||
$table->string('state');
|
||||
$table->string('last_error_code')->default('');
|
||||
$table->integer('terminal_at')->nullable();
|
||||
$table->integer('created_at');
|
||||
$table->integer('updated_at');
|
||||
});
|
||||
$schema->create('cloud_service_compensation_items', static function (Blueprint $table): void {
|
||||
$table->increments('id');
|
||||
$table->string('item_key')->unique();
|
||||
$table->integer('batch_id');
|
||||
$table->string('incident_key');
|
||||
$table->integer('instance_id');
|
||||
$table->integer('user_id');
|
||||
$table->integer('host_id');
|
||||
$table->string('instance_name');
|
||||
$table->integer('requested_seconds');
|
||||
$table->integer('service_period_id')->nullable();
|
||||
$table->integer('traffic_entitlement_id')->nullable();
|
||||
$table->integer('backup_entitlement_id')->nullable();
|
||||
$table->integer('source_invoice_id')->nullable();
|
||||
$table->integer('replacement_invoice_id')->nullable();
|
||||
$table->integer('audit_job_id')->nullable();
|
||||
$table->integer('before_expires_at')->nullable();
|
||||
$table->integer('after_expires_at')->nullable();
|
||||
$table->integer('before_period_ends_at')->nullable();
|
||||
$table->integer('after_period_ends_at')->nullable();
|
||||
$table->integer('before_traffic_ends_at')->nullable();
|
||||
$table->integer('after_traffic_ends_at')->nullable();
|
||||
$table->integer('before_backup_ends_at')->nullable();
|
||||
$table->integer('after_backup_ends_at')->nullable();
|
||||
$table->text('before_snapshot_json');
|
||||
$table->text('after_snapshot_json')->nullable();
|
||||
$table->string('state');
|
||||
$table->integer('attempts')->default(0);
|
||||
$table->integer('available_at');
|
||||
$table->string('lease_token')->nullable();
|
||||
$table->integer('lease_until')->default(0);
|
||||
$table->integer('applied_at')->nullable();
|
||||
$table->integer('terminal_at')->nullable();
|
||||
$table->string('last_error_code')->default('');
|
||||
$table->string('last_error')->default('');
|
||||
$table->integer('created_at');
|
||||
$table->integer('updated_at');
|
||||
});
|
||||
|
||||
$now = 2_100_000_000;
|
||||
Capsule::table('user')->insert([
|
||||
'id' => 1, 'money' => '50.00', 'email' => 'owner@example.test',
|
||||
'enable' => 1, 'is_admin' => 0,
|
||||
]);
|
||||
|
||||
$makeInstance = static function (int $id, int $expiry, string $name) use (
|
||||
$now
|
||||
): void {
|
||||
Capsule::table('cloud_instances')->insert([
|
||||
'id' => $id, 'host_id' => 7, 'user_id' => 1,
|
||||
'order_id' => null, 'plan_id' => 5, 'offer_id' => 9,
|
||||
'current_period_id' => $id, 'uuid' => sprintf(
|
||||
'00000000-0000-4000-8000-%012d', $id
|
||||
), 'name' => $name, 'status' => 'active',
|
||||
'service_status' => 'active', 'network_status' => 'active',
|
||||
'expires_at' => $expiry, 'renewal_price' => '12.00',
|
||||
'renewal_days' => 30, 'renewal_traffic_bytes' => 2048,
|
||||
'traffic_quota_bytes' => 2048,
|
||||
'spec_json' => json_encode(['vcpu' => 2], JSON_THROW_ON_ERROR),
|
||||
'created_at' => $now - 1000, 'updated_at' => $now - 100,
|
||||
]);
|
||||
Capsule::table('cloud_service_periods')->insert([
|
||||
'id' => $id, 'instance_id' => $id, 'user_id' => 1,
|
||||
'plan_id' => 5, 'order_id' => null, 'renewal_id' => null,
|
||||
'traffic_pool_group_id' => null,
|
||||
'traffic_quota_mode' => 'finite', 'starts_at' => $now - 1000,
|
||||
'ends_at' => $expiry, 'traffic_quota_bytes' => 2048,
|
||||
'download_base_bytes' => 0, 'upload_base_bytes' => 0,
|
||||
'download_used_bytes' => 0, 'upload_used_bytes' => 0,
|
||||
'status' => 'active', 'activated_at' => $now - 1000,
|
||||
'created_at' => $now - 1000, 'updated_at' => $now - 100,
|
||||
]);
|
||||
};
|
||||
$makeBatch = static function (int $id, int $instanceId, string $suffix) use (
|
||||
$now
|
||||
): void {
|
||||
Capsule::table('cloud_service_compensations')->insert([
|
||||
'id' => $id, 'compensation_key' => str_repeat($suffix, 32),
|
||||
'incident_key' => 'incident-' . $suffix,
|
||||
'requested_by_user_id' => 99,
|
||||
'request_idempotency_key' => 'request-key-' . str_repeat($suffix, 16),
|
||||
'request_fingerprint' => str_repeat($suffix, 64),
|
||||
'scope_type' => 'instance', 'scope_id' => $instanceId,
|
||||
'scope_snapshot_json' => '{}', 'reason_text' => 'test outage',
|
||||
'incident_starts_at' => null, 'incident_ends_at' => null,
|
||||
'compensation_seconds' => 3600, 'notify_users' => 0,
|
||||
'audience_digest' => str_repeat($suffix, 64),
|
||||
'instance_count' => 1, 'recipient_count' => 0,
|
||||
'applied_count' => 0, 'deferred_count' => 0,
|
||||
'rejected_count' => 0, 'attention_count' => 0,
|
||||
'state' => 'pending', 'last_error_code' => '',
|
||||
'terminal_at' => null, 'created_at' => $now,
|
||||
'updated_at' => $now,
|
||||
]);
|
||||
Capsule::table('cloud_service_compensation_items')->insert([
|
||||
'id' => $id, 'item_key' => str_repeat($suffix, 32),
|
||||
'batch_id' => $id, 'incident_key' => 'incident-' . $suffix,
|
||||
'instance_id' => $instanceId, 'user_id' => 1, 'host_id' => 7,
|
||||
'instance_name' => 'vm-' . $suffix, 'requested_seconds' => 3600,
|
||||
'before_snapshot_json' => '{}', 'state' => 'pending',
|
||||
'attempts' => 0, 'available_at' => $now,
|
||||
'lease_token' => null, 'lease_until' => 0,
|
||||
'last_error_code' => '', 'last_error' => '',
|
||||
'created_at' => $now, 'updated_at' => $now,
|
||||
]);
|
||||
};
|
||||
|
||||
$oldExpiry = $now + 10 * 86400;
|
||||
$makeInstance(1, $oldExpiry, 'vm-success');
|
||||
$makeBatch(1, 1, 'a');
|
||||
$execution = new CloudServiceCompensationExecutionService(
|
||||
static fn (): int => $now
|
||||
);
|
||||
$claim = $execution->claimOne(1);
|
||||
$assert(is_array($claim), 'Success item was not claimed.');
|
||||
$result = $execution->applyClaim(
|
||||
(int) $claim['item_id'], (string) $claim['token']
|
||||
);
|
||||
$assert(($result['state'] ?? '') === 'applied',
|
||||
'Success item did not reach applied.');
|
||||
$instance = CloudInstance::findOrFail(1);
|
||||
$period = CloudServicePeriod::findOrFail(1);
|
||||
$item = CloudServiceCompensationItem::findOrFail(1);
|
||||
$batch = CloudServiceCompensation::findOrFail(1);
|
||||
$assert((int) $instance->expires_at === $oldExpiry + 3600,
|
||||
'Instance expiry was not extended exactly once.');
|
||||
$assert((int) $period->ends_at === $oldExpiry + 3600,
|
||||
'Tail period was not extended exactly once.');
|
||||
$assert((string) Capsule::table('user')->where('id', 1)->value('money')
|
||||
=== '50.00', 'Compensation changed wallet balance.');
|
||||
$assert(Capsule::table('cloud_traffic_entitlements')->count() === 0
|
||||
&& Capsule::table('cloud_backup_period_entitlements')->count() === 0,
|
||||
'Compensation created a new paid entitlement.');
|
||||
$assert((string) $item->state === 'applied'
|
||||
&& (int) $item->audit_job_id > 0
|
||||
&& (int) $item->available_at === $now
|
||||
&& (string) $batch->state === 'completed'
|
||||
&& (int) $batch->applied_count === 1,
|
||||
'Applied item or batch projection is incomplete.');
|
||||
$afterEvidence = json_decode(
|
||||
(string) $item->after_snapshot_json,
|
||||
true,
|
||||
64,
|
||||
JSON_THROW_ON_ERROR
|
||||
);
|
||||
$assert((string) $item->before_snapshot_json === '{}'
|
||||
&& (int) ($afterEvidence['applied_from']['instance']['expires_at'] ?? 0)
|
||||
=== $oldExpiry
|
||||
&& (int) ($afterEvidence['applied_to']['instance']['expires_at'] ?? 0)
|
||||
=== $oldExpiry + 3600,
|
||||
'Reviewed evidence changed or application evidence is incomplete.');
|
||||
$job = CloudJob::findOrFail((int) $item->audit_job_id);
|
||||
$assert((string) $job->kind === 'service_compensation'
|
||||
&& (string) $job->execution_mode === 'local'
|
||||
&& (string) $job->phase === 'terminal'
|
||||
&& (string) $job->outcome === 'succeeded'
|
||||
&& (string) $job->effect_resolution === 'applied',
|
||||
'Local compensation audit job is not terminal success.');
|
||||
$execution->applyClaim((int) $claim['item_id'], (string) $claim['token']);
|
||||
$assert((int) CloudInstance::findOrFail(1)->expires_at
|
||||
=== $oldExpiry + 3600
|
||||
&& CloudJob::where('instance_id', 1)->count() === 1,
|
||||
'Idempotent replay extended time or duplicated the audit job.');
|
||||
|
||||
$pendingExpiry = $oldExpiry + 100;
|
||||
$makeInstance(2, $pendingExpiry, 'vm-payment-open');
|
||||
$makeBatch(2, 2, 'b');
|
||||
Capsule::table('cloud_invoices')->insert([
|
||||
'id' => 2, 'invoice_no' => 'INV-PENDING', 'purpose' => 'renewal',
|
||||
'instance_id' => 2, 'order_id' => null, 'user_id' => 1,
|
||||
'plan_id' => 5, 'renewal_id' => null,
|
||||
'renews_expires_at' => $pendingExpiry, 'due_at' => $pendingExpiry,
|
||||
'amount' => '12.00', 'currency' => 'USD', 'validity_days' => 30,
|
||||
'traffic_quota_bytes' => 2048, 'status' => 'open',
|
||||
'paid_via' => null, 'payment_reference' => null, 'paid_at' => null,
|
||||
'created_at' => $now, 'updated_at' => $now,
|
||||
]);
|
||||
Capsule::table('cloud_invoice_payments')->insert([
|
||||
'invoice_id' => 2, 'user_id' => 1, 'provider' => 'stripe_card',
|
||||
'provider_order_no' => 'pending-order',
|
||||
'idempotency_key' => 'pending-payment-key', 'amount' => '12.00',
|
||||
'status' => 'pending', 'created_at' => $now, 'updated_at' => $now,
|
||||
]);
|
||||
$claim = $execution->claimOne(2);
|
||||
$assert(is_array($claim), 'Payment-fenced item was not claimed.');
|
||||
$deferred = $execution->applyClaim(
|
||||
(int) $claim['item_id'], (string) $claim['token']
|
||||
);
|
||||
$assert(($deferred['state'] ?? '') === 'deferred'
|
||||
&& ($deferred['last_error_code'] ?? '') === 'renewal_payment_open',
|
||||
'Pending payment did not defer compensation.');
|
||||
$assert((int) CloudInstance::findOrFail(2)->expires_at === $pendingExpiry
|
||||
&& (int) CloudServicePeriod::findOrFail(2)->ends_at === $pendingExpiry
|
||||
&& CloudJob::where('instance_id', 2)->count() === 0
|
||||
&& (string) Capsule::table('cloud_invoices')
|
||||
->where('id', 2)->value('status') === 'open',
|
||||
'Deferred payment fence changed business state.');
|
||||
|
||||
echo "Service compensation execution tests passed.\n";
|
||||
}
|
||||
Loading…
Reference in New Issue