fix(cloud): keep compensation waits recoverable

go
codex 2 weeks ago
parent 6f4978bdf4
commit 836835ff41

@ -28,7 +28,7 @@
## 执行、租约和审计
普通 `cloudWorker` 每轮最多领取一个补偿明细。明细使用 120 秒租约、有界尝试和指数退避;领取、业务投影、审计任务和明细终态遵循同一数据库事务边界。租约或数据库故障不会调用 Agent,也不会产生宿主机未知副作用。
普通 `cloudWorker` 每轮最多领取一个补偿明细。明细使用 120 秒租约;租约丢失、数据库或投影暂时故障使用有界尝试和指数退避。已确认补偿还未发生、只是在等待删除、迁移、实例任务或支付收敛时,则每 300 秒重新核对原状态且不消耗技术失败预算;外部状态结束后会继续原明细。领取、业务投影、审计任务和明细终态遵循同一数据库事务边界。租约或数据库故障不会调用 Agent,也不会产生宿主机未知副作用。
批次公开状态为 `pending`、`applying`、`completed`、`partial` 或 `attention`。管理员应从批次详情查看每个实例的前后到期时间、审计任务和错误代码:

@ -22,6 +22,7 @@ final class CloudServiceCompensationExecutionService
{
public const LEASE_SECONDS = 120;
public const MAX_ATTEMPTS = 8;
private const EXTERNAL_WAIT_SECONDS = 300;
private readonly \Closure $clock;
@ -287,7 +288,8 @@ final class CloudServiceCompensationExecutionService
$admission['state'],
$admission['code'],
$admission['message'],
$now
$now,
$admission['consume_retry_budget']
);
}
@ -313,7 +315,8 @@ final class CloudServiceCompensationExecutionService
'deferred',
'service_period_changed',
'实例服务周期已变化,系统将重新核对',
$now
$now,
false
);
}
$oldExpiry = (int) $instance->expires_at;
@ -354,7 +357,8 @@ final class CloudServiceCompensationExecutionService
$financial['state'],
$financial['code'],
$financial['message'],
$now
$now,
$financial['consume_retry_budget']
);
}
$operationId = substr(hash(
@ -506,7 +510,10 @@ final class CloudServiceCompensationExecutionService
return true;
}
/** @return array{state:string,code:string,message:string}|null */
/**
* @return array{state:string,code:string,message:string,
* consume_retry_budget:bool}|null
*/
private function admissionLocked(CloudInstance $instance, int $now): ?array
{
if ((string) $instance->status === 'deleted') {
@ -516,7 +523,7 @@ final class CloudServiceCompensationExecutionService
}
if (CloudInvoiceService::instanceDeleting($instance)) {
return $this->decision(
'deferred', 'instance_deleting', '实例删除流程尚未收敛'
'deferred', 'instance_deleting', '实例删除流程尚未收敛', false
);
}
if ((string) $instance->service_status !== 'active') {
@ -534,7 +541,7 @@ final class CloudServiceCompensationExecutionService
(int) $instance->id
)) {
return $this->decision(
'deferred', 'migration_open', '实例维护迁移尚未收敛'
'deferred', 'migration_open', '实例维护迁移尚未收敛', false
);
}
$jobs = CloudJobService::blockingQueryForInstance(
@ -545,7 +552,7 @@ final class CloudServiceCompensationExecutionService
if ($jobs->contains(static fn ($job): bool =>
(string) $job->phase !== 'terminal')) {
return $this->decision(
'deferred', 'instance_operation_open', '实例仍有操作正在执行'
'deferred', 'instance_operation_open', '实例仍有操作正在执行', false
);
}
if (!$jobs->isEmpty()) {
@ -558,7 +565,10 @@ final class CloudServiceCompensationExecutionService
return null;
}
/** @return array{state:string,code:string,message:string}|null */
/**
* @return array{state:string,code:string,message:string,
* consume_retry_budget:bool}|null
*/
private function financialAdmissionLocked(
CloudInstance $instance,
int $oldExpiry
@ -603,7 +613,7 @@ final class CloudServiceCompensationExecutionService
(string) $job->phase !== 'terminal')) {
return $this->decision(
'deferred', 'renewal_payment_job_open',
'旧续费账单的支付任务仍在执行'
'旧续费账单的支付任务仍在执行', false
);
}
if (!$jobs->isEmpty()) {
@ -622,7 +632,7 @@ final class CloudServiceCompensationExecutionService
))) {
return $this->decision(
'deferred', 'renewal_payment_open',
'旧续费账单仍有支付正在处理'
'旧续费账单仍有支付正在处理', false
);
}
if ($payments->contains(static fn ($payment): bool => !in_array(
@ -693,8 +703,16 @@ final class CloudServiceCompensationExecutionService
string $state,
string $code,
string $message,
int $now
int $now,
bool $consumeRetryBudget = true
): array {
if ($state === 'deferred' && !$consumeRetryBudget) {
// Claiming increments attempts before the external admission facts
// are locked. Undo only that claim so long-running migrations,
// deletes and payments do not consume the technical failure
// budget. Prior real failures remain counted.
$item->attempts = max(0, (int) $item->attempts - 1);
}
if ($state === 'deferred'
&& (int) $item->attempts >= self::MAX_ATTEMPTS) {
$state = 'attention';
@ -708,7 +726,10 @@ final class CloudServiceCompensationExecutionService
$item->lease_token = null;
$item->lease_until = 0;
$item->available_at = $state === 'deferred'
? $now + $this->backoff((int) $item->attempts) : $now;
? $now + ($consumeRetryBudget
? $this->backoff((int) $item->attempts)
: self::EXTERNAL_WAIT_SECONDS)
: $now;
$item->terminal_at = $state === 'deferred' ? null : $now;
$item->last_error_code = substr($code, 0, 64);
$item->last_error = mb_substr($message, 0, 500);
@ -781,13 +802,22 @@ final class CloudServiceCompensationExecutionService
);
}
/** @return array{state:string,code:string,message:string} */
/**
* @return array{state:string,code:string,message:string,
* consume_retry_budget:bool}
*/
private function decision(
string $state,
string $code,
string $message
string $message,
bool $consumeRetryBudget = true
): array {
return compact('state', 'code', 'message');
return [
'state' => $state,
'code' => $code,
'message' => $message,
'consume_retry_budget' => $consumeRetryBudget,
];
}
private function backoff(int $attempts): int

@ -65,6 +65,31 @@ foreach ([
'Compensation execution lost fence: ' . $fence);
}
foreach ([
'instance_deleting',
'migration_open',
'instance_operation_open',
'renewal_payment_job_open',
'renewal_payment_open',
] as $externalWaitCode) {
$assert(preg_match(
"/'deferred',\\s*'" . preg_quote($externalWaitCode, '/')
. "'.*?false\\s*\\)/s",
$source
) === 1,
'External wait consumes technical retry budget: ' . $externalWaitCode);
}
$assert(preg_match(
'/\'service_period_changed\'.*?\$now,\s*false\s*\)/s',
$apply
) === 1
&& str_contains($source, 'private const EXTERNAL_WAIT_SECONDS = 300;')
&& str_contains(
$source,
'$item->attempts = max(0, (int) $item->attempts - 1);'
),
'External service-period wait lost its bounded non-consuming transition.');
$invoiceProjection = strpos($apply,
'reconcileRenewalAfterCompensationLocked(');
$trafficProjection = strpos($apply,

@ -124,6 +124,20 @@ namespace {
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_mail_outbox', static function (Blueprint $table): void {
$table->increments('id');
$table->integer('invoice_id');
$table->string('kind');
$table->string('status');
$table->integer('attempts')->default(0);
$table->integer('available_at');
$table->string('lease_token')->nullable();
$table->integer('lease_until')->default(0);
$table->integer('sent_at')->nullable();
$table->string('last_error')->default('');
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_jobs', static function (Blueprint $table): void {
$table->increments('id');
$table->string('operation_id')->nullable()->unique();
@ -331,8 +345,11 @@ namespace {
$oldExpiry = $now + 10 * 86400;
$makeInstance(1, $oldExpiry, 'vm-success');
$makeBatch(1, 1, 'a');
$clockNow = $now;
$execution = new CloudServiceCompensationExecutionService(
static fn (): int => $now
static function () use (&$clockNow): int {
return $clockNow;
}
);
$claim = $execution->claimOne(1);
$assert(is_array($claim), 'Success item was not claimed.');
@ -412,6 +429,10 @@ namespace {
$assert(($deferred['state'] ?? '') === 'deferred'
&& ($deferred['last_error_code'] ?? '') === 'renewal_payment_open',
'Pending payment did not defer compensation.');
$firstWait = CloudServiceCompensationItem::findOrFail(2);
$assert((int) $firstWait->attempts === 0
&& (int) $firstWait->available_at === $clockNow + 300,
'External payment wait consumed retry budget or used unbounded polling.');
$assert((int) CloudInstance::findOrFail(2)->expires_at === $pendingExpiry
&& (int) CloudServicePeriod::findOrFail(2)->ends_at === $pendingExpiry
&& CloudJob::where('instance_id', 2)->count() === 0
@ -419,5 +440,43 @@ namespace {
->where('id', 2)->value('status') === 'open',
'Deferred payment fence changed business state.');
// An external payment can legitimately remain pending longer than the
// worker's technical retry budget. Rechecking that unchanged fact must
// not turn a still-recoverable compensation into terminal attention.
for ($wait = 1; $wait < 10; $wait++) {
$waiting = CloudServiceCompensationItem::findOrFail(2);
if ((string) $waiting->state !== 'deferred') {
break;
}
$clockNow = (int) $waiting->available_at;
$waitClaim = $execution->claimOne(2);
if ($waitClaim === null) {
break;
}
$execution->applyClaim(
(int) $waitClaim['item_id'],
(string) $waitClaim['token']
);
}
$waiting = CloudServiceCompensationItem::findOrFail(2);
$assert((string) $waiting->state === 'deferred'
&& (int) $waiting->attempts === 0
&& (string) $waiting->last_error_code === 'renewal_payment_open',
'External payment wait exhausted the technical retry budget.');
Capsule::table('cloud_invoice_payments')->where('invoice_id', 2)->delete();
$clockNow = (int) $waiting->available_at;
$resumedClaim = $execution->claimOne(2);
$assert(is_array($resumedClaim),
'Resolved payment wait did not make compensation claimable.');
$resumed = $execution->applyClaim(
(int) $resumedClaim['item_id'],
(string) $resumedClaim['token']
);
$assert(($resumed['state'] ?? '') === 'applied'
&& (int) CloudInstance::findOrFail(2)->expires_at
=== $pendingExpiry + 3600,
'Resolved payment wait did not resume compensation exactly once.');
echo "Service compensation execution tests passed.\n";
}

Loading…
Cancel
Save