From 836835ff41d7b3beb3d3f6d2a29bdb94c77c3dac Mon Sep 17 00:00:00 2001 From: codex Date: Fri, 6 Nov 2026 03:48:26 +0800 Subject: [PATCH] fix(cloud): keep compensation waits recoverable --- docs/SERVICE_COMPENSATION.md | 2 +- ...oudServiceCompensationExecutionService.php | 60 +++++++++++++----- ...rviceCompensationExecutionContractTest.php | 25 ++++++++ .../CloudServiceCompensationExecutionTest.php | 61 ++++++++++++++++++- 4 files changed, 131 insertions(+), 17 deletions(-) diff --git a/docs/SERVICE_COMPENSATION.md b/docs/SERVICE_COMPENSATION.md index e95cff9d..4980d774 100644 --- a/docs/SERVICE_COMPENSATION.md +++ b/docs/SERVICE_COMPENSATION.md @@ -28,7 +28,7 @@ ## 执行、租约和审计 -普通 `cloudWorker` 每轮最多领取一个补偿明细。明细使用 120 秒租约、有界尝试和指数退避;领取、业务投影、审计任务和明细终态遵循同一数据库事务边界。租约或数据库故障不会调用 Agent,也不会产生宿主机未知副作用。 +普通 `cloudWorker` 每轮最多领取一个补偿明细。明细使用 120 秒租约;租约丢失、数据库或投影暂时故障使用有界尝试和指数退避。已确认补偿还未发生、只是在等待删除、迁移、实例任务或支付收敛时,则每 300 秒重新核对原状态且不消耗技术失败预算;外部状态结束后会继续原明细。领取、业务投影、审计任务和明细终态遵循同一数据库事务边界。租约或数据库故障不会调用 Agent,也不会产生宿主机未知副作用。 批次公开状态为 `pending`、`applying`、`completed`、`partial` 或 `attention`。管理员应从批次详情查看每个实例的前后到期时间、审计任务和错误代码: diff --git a/web/app/Services/Cloud/CloudServiceCompensationExecutionService.php b/web/app/Services/Cloud/CloudServiceCompensationExecutionService.php index 619f7cf8..f2c7c380 100644 --- a/web/app/Services/Cloud/CloudServiceCompensationExecutionService.php +++ b/web/app/Services/Cloud/CloudServiceCompensationExecutionService.php @@ -22,6 +22,7 @@ final class CloudServiceCompensationExecutionService { public const LEASE_SECONDS = 120; public const MAX_ATTEMPTS = 8; + private const EXTERNAL_WAIT_SECONDS = 300; private readonly \Closure $clock; @@ -287,7 +288,8 @@ final class CloudServiceCompensationExecutionService $admission['state'], $admission['code'], $admission['message'], - $now + $now, + $admission['consume_retry_budget'] ); } @@ -313,7 +315,8 @@ final class CloudServiceCompensationExecutionService 'deferred', 'service_period_changed', '实例服务周期已变化,系统将重新核对', - $now + $now, + false ); } $oldExpiry = (int) $instance->expires_at; @@ -354,7 +357,8 @@ final class CloudServiceCompensationExecutionService $financial['state'], $financial['code'], $financial['message'], - $now + $now, + $financial['consume_retry_budget'] ); } $operationId = substr(hash( @@ -506,7 +510,10 @@ final class CloudServiceCompensationExecutionService return true; } - /** @return array{state:string,code:string,message:string}|null */ + /** + * @return array{state:string,code:string,message:string, + * consume_retry_budget:bool}|null + */ private function admissionLocked(CloudInstance $instance, int $now): ?array { if ((string) $instance->status === 'deleted') { @@ -516,7 +523,7 @@ final class CloudServiceCompensationExecutionService } if (CloudInvoiceService::instanceDeleting($instance)) { return $this->decision( - 'deferred', 'instance_deleting', '实例删除流程尚未收敛' + 'deferred', 'instance_deleting', '实例删除流程尚未收敛', false ); } if ((string) $instance->service_status !== 'active') { @@ -534,7 +541,7 @@ final class CloudServiceCompensationExecutionService (int) $instance->id )) { return $this->decision( - 'deferred', 'migration_open', '实例维护迁移尚未收敛' + 'deferred', 'migration_open', '实例维护迁移尚未收敛', false ); } $jobs = CloudJobService::blockingQueryForInstance( @@ -545,7 +552,7 @@ final class CloudServiceCompensationExecutionService if ($jobs->contains(static fn ($job): bool => (string) $job->phase !== 'terminal')) { return $this->decision( - 'deferred', 'instance_operation_open', '实例仍有操作正在执行' + 'deferred', 'instance_operation_open', '实例仍有操作正在执行', false ); } if (!$jobs->isEmpty()) { @@ -558,7 +565,10 @@ final class CloudServiceCompensationExecutionService return null; } - /** @return array{state:string,code:string,message:string}|null */ + /** + * @return array{state:string,code:string,message:string, + * consume_retry_budget:bool}|null + */ private function financialAdmissionLocked( CloudInstance $instance, int $oldExpiry @@ -603,7 +613,7 @@ final class CloudServiceCompensationExecutionService (string) $job->phase !== 'terminal')) { return $this->decision( 'deferred', 'renewal_payment_job_open', - '旧续费账单的支付任务仍在执行' + '旧续费账单的支付任务仍在执行', false ); } if (!$jobs->isEmpty()) { @@ -622,7 +632,7 @@ final class CloudServiceCompensationExecutionService ))) { return $this->decision( 'deferred', 'renewal_payment_open', - '旧续费账单仍有支付正在处理' + '旧续费账单仍有支付正在处理', false ); } if ($payments->contains(static fn ($payment): bool => !in_array( @@ -693,8 +703,16 @@ final class CloudServiceCompensationExecutionService string $state, string $code, string $message, - int $now + int $now, + bool $consumeRetryBudget = true ): array { + if ($state === 'deferred' && !$consumeRetryBudget) { + // Claiming increments attempts before the external admission facts + // are locked. Undo only that claim so long-running migrations, + // deletes and payments do not consume the technical failure + // budget. Prior real failures remain counted. + $item->attempts = max(0, (int) $item->attempts - 1); + } if ($state === 'deferred' && (int) $item->attempts >= self::MAX_ATTEMPTS) { $state = 'attention'; @@ -708,7 +726,10 @@ final class CloudServiceCompensationExecutionService $item->lease_token = null; $item->lease_until = 0; $item->available_at = $state === 'deferred' - ? $now + $this->backoff((int) $item->attempts) : $now; + ? $now + ($consumeRetryBudget + ? $this->backoff((int) $item->attempts) + : self::EXTERNAL_WAIT_SECONDS) + : $now; $item->terminal_at = $state === 'deferred' ? null : $now; $item->last_error_code = substr($code, 0, 64); $item->last_error = mb_substr($message, 0, 500); @@ -781,13 +802,22 @@ final class CloudServiceCompensationExecutionService ); } - /** @return array{state:string,code:string,message:string} */ + /** + * @return array{state:string,code:string,message:string, + * consume_retry_budget:bool} + */ private function decision( string $state, string $code, - string $message + string $message, + bool $consumeRetryBudget = true ): array { - return compact('state', 'code', 'message'); + return [ + 'state' => $state, + 'code' => $code, + 'message' => $message, + 'consume_retry_budget' => $consumeRetryBudget, + ]; } private function backoff(int $attempts): int diff --git a/web/tests/CloudServiceCompensationExecutionContractTest.php b/web/tests/CloudServiceCompensationExecutionContractTest.php index 0b617a79..f9d59a55 100644 --- a/web/tests/CloudServiceCompensationExecutionContractTest.php +++ b/web/tests/CloudServiceCompensationExecutionContractTest.php @@ -65,6 +65,31 @@ foreach ([ 'Compensation execution lost fence: ' . $fence); } +foreach ([ + 'instance_deleting', + 'migration_open', + 'instance_operation_open', + 'renewal_payment_job_open', + 'renewal_payment_open', +] as $externalWaitCode) { + $assert(preg_match( + "/'deferred',\\s*'" . preg_quote($externalWaitCode, '/') + . "'.*?false\\s*\\)/s", + $source + ) === 1, + 'External wait consumes technical retry budget: ' . $externalWaitCode); +} +$assert(preg_match( + '/\'service_period_changed\'.*?\$now,\s*false\s*\)/s', + $apply + ) === 1 + && str_contains($source, 'private const EXTERNAL_WAIT_SECONDS = 300;') + && str_contains( + $source, + '$item->attempts = max(0, (int) $item->attempts - 1);' + ), + 'External service-period wait lost its bounded non-consuming transition.'); + $invoiceProjection = strpos($apply, 'reconcileRenewalAfterCompensationLocked('); $trafficProjection = strpos($apply, diff --git a/web/tests/CloudServiceCompensationExecutionTest.php b/web/tests/CloudServiceCompensationExecutionTest.php index 4f2e9ee4..273f4261 100644 --- a/web/tests/CloudServiceCompensationExecutionTest.php +++ b/web/tests/CloudServiceCompensationExecutionTest.php @@ -124,6 +124,20 @@ namespace { $table->integer('created_at'); $table->integer('updated_at'); }); + $schema->create('cloud_mail_outbox', static function (Blueprint $table): void { + $table->increments('id'); + $table->integer('invoice_id'); + $table->string('kind'); + $table->string('status'); + $table->integer('attempts')->default(0); + $table->integer('available_at'); + $table->string('lease_token')->nullable(); + $table->integer('lease_until')->default(0); + $table->integer('sent_at')->nullable(); + $table->string('last_error')->default(''); + $table->integer('created_at'); + $table->integer('updated_at'); + }); $schema->create('cloud_jobs', static function (Blueprint $table): void { $table->increments('id'); $table->string('operation_id')->nullable()->unique(); @@ -331,8 +345,11 @@ namespace { $oldExpiry = $now + 10 * 86400; $makeInstance(1, $oldExpiry, 'vm-success'); $makeBatch(1, 1, 'a'); + $clockNow = $now; $execution = new CloudServiceCompensationExecutionService( - static fn (): int => $now + static function () use (&$clockNow): int { + return $clockNow; + } ); $claim = $execution->claimOne(1); $assert(is_array($claim), 'Success item was not claimed.'); @@ -412,6 +429,10 @@ namespace { $assert(($deferred['state'] ?? '') === 'deferred' && ($deferred['last_error_code'] ?? '') === 'renewal_payment_open', 'Pending payment did not defer compensation.'); + $firstWait = CloudServiceCompensationItem::findOrFail(2); + $assert((int) $firstWait->attempts === 0 + && (int) $firstWait->available_at === $clockNow + 300, + 'External payment wait consumed retry budget or used unbounded polling.'); $assert((int) CloudInstance::findOrFail(2)->expires_at === $pendingExpiry && (int) CloudServicePeriod::findOrFail(2)->ends_at === $pendingExpiry && CloudJob::where('instance_id', 2)->count() === 0 @@ -419,5 +440,43 @@ namespace { ->where('id', 2)->value('status') === 'open', 'Deferred payment fence changed business state.'); + // An external payment can legitimately remain pending longer than the + // worker's technical retry budget. Rechecking that unchanged fact must + // not turn a still-recoverable compensation into terminal attention. + for ($wait = 1; $wait < 10; $wait++) { + $waiting = CloudServiceCompensationItem::findOrFail(2); + if ((string) $waiting->state !== 'deferred') { + break; + } + $clockNow = (int) $waiting->available_at; + $waitClaim = $execution->claimOne(2); + if ($waitClaim === null) { + break; + } + $execution->applyClaim( + (int) $waitClaim['item_id'], + (string) $waitClaim['token'] + ); + } + $waiting = CloudServiceCompensationItem::findOrFail(2); + $assert((string) $waiting->state === 'deferred' + && (int) $waiting->attempts === 0 + && (string) $waiting->last_error_code === 'renewal_payment_open', + 'External payment wait exhausted the technical retry budget.'); + + Capsule::table('cloud_invoice_payments')->where('invoice_id', 2)->delete(); + $clockNow = (int) $waiting->available_at; + $resumedClaim = $execution->claimOne(2); + $assert(is_array($resumedClaim), + 'Resolved payment wait did not make compensation claimable.'); + $resumed = $execution->applyClaim( + (int) $resumedClaim['item_id'], + (string) $resumedClaim['token'] + ); + $assert(($resumed['state'] ?? '') === 'applied' + && (int) CloudInstance::findOrFail(2)->expires_at + === $pendingExpiry + 3600, + 'Resolved payment wait did not resume compensation exactly once.'); + echo "Service compensation execution tests passed.\n"; }