From 4640b8d8699d2aae6bc3b9228013b0ca8477fb78 Mon Sep 17 00:00:00 2001 From: codex Date: Thu, 29 Oct 2026 17:03:57 +0800 Subject: [PATCH] fix(vpc): add fail-closed batch rollback exit --- web/app/Controllers/CloudVpcController.php | 15 + .../Cloud/CloudVpcMembershipBatchService.php | 268 +++++++++++++++++- .../Services/Cloud/CloudVpcPortService.php | 59 +++- web/config/routes_cloud.php | 4 + web/public/cloud/vpcs.js | 48 +++- web/tests/CloudVpcPortWorkflowTest.php | 188 ++++++++++++ web/tests/CloudVpcUserContractTest.php | 5 + 7 files changed, 573 insertions(+), 14 deletions(-) diff --git a/web/app/Controllers/CloudVpcController.php b/web/app/Controllers/CloudVpcController.php index f1058b89..29974c62 100644 --- a/web/app/Controllers/CloudVpcController.php +++ b/web/app/Controllers/CloudVpcController.php @@ -100,4 +100,19 @@ final class CloudVpcController return $response->withJson($result, $status) ->withHeader('Cache-Control', 'private, no-store'); } + + /** Continue a fail-closed attach rollback without replaying uncertainty. */ + public function cancelMembershipBatch($request, $response, $args) + { + $actor = $this->actor($request); + $result = CloudVpcMembershipBatchService::cancelAttach( + $actor, + (string) ($args['vpc_key'] ?? ''), + (string) ($args['batch_key'] ?? '') + ); + return $response->withJson([ + 'task_id' => (string) $result['job']->id, + 'status_url' => '/api/v1/tasks/' . (int) $result['job']->id, + ], 202)->withHeader('Cache-Control', 'private, no-store'); + } } diff --git a/web/app/Services/Cloud/CloudVpcMembershipBatchService.php b/web/app/Services/Cloud/CloudVpcMembershipBatchService.php index 663dfd47..da4033af 100644 --- a/web/app/Services/Cloud/CloudVpcMembershipBatchService.php +++ b/web/app/Services/Cloud/CloudVpcMembershipBatchService.php @@ -158,6 +158,95 @@ final class CloudVpcMembershipBatchService ->all(); } + /** + * Continue only an attach rollback whose previous cleanup attempts are + * all proven not applied. This never replays an unknown Agent mutation. + */ + public static function cancelAttach(array $actor, string $vpcKey, + string $batchKey): array + { + if (preg_match('/^[a-f0-9]{32}$/D', $vpcKey) !== 1 + || preg_match('/^[a-f0-9]{32}$/D', $batchKey) !== 1) { + throw new CloudError('VPC 批次标识无效', 404); + } + $userId = (int) ($actor['user_id'] ?? 0); + return (new CloudVpcMembershipBatch())->getConnection()->transaction( + static function () use ($userId, $vpcKey, $batchKey): array { + $vpc = CloudVpc::query()->where('vpc_key', $vpcKey) + ->lockForUpdate()->first(); + if ($vpc === null || (int) $vpc->user_id !== $userId) { + throw new CloudError('VPC 不存在或当前不可用', 404); + } + $batch = CloudVpcMembershipBatch::query() + ->where('batch_key', $batchKey) + ->where('vpc_id', (int) $vpc->id) + ->where('user_id', $userId) + ->lockForUpdate()->first(); + if ($batch === null || (string) $batch->kind !== 'attach') { + throw new CloudError('VPC 加入批次不存在', 404); + } + $parent = CloudJob::query() + ->where('id', (int) $batch->parent_job_id) + ->lockForUpdate()->first(); + if ($parent === null + || (string) $parent->kind !== self::PARENT_KIND) { + throw new CloudError('VPC 批次父任务账本不一致', 409); + } + if (in_array((string) $batch->state, + self::TERMINAL_STATES, true)) { + return ['job' => $parent, 'batch' => self::dto($batch), + 'unchanged' => true]; + } + if ((string) $batch->state === 'cleanup') { + return ['job' => $parent, 'batch' => self::dto($batch), + 'unchanged' => true]; + } + $items = self::itemsLocked($batch); + if ((string) $batch->state !== 'blocked' + || !self::canRetryAttachCleanup($batch, $items, true)) { + throw new CloudError( + '撤销结果尚未确认,不能重复清理;请等待任务核对', + 409 + ); + } + $focus = CloudVpcPort::query() + ->where('id', (int) $items->first()->vpc_port_id) + ->where('active_claim', 1)->lockForUpdate()->first(); + if ($focus === null) { + throw new CloudError('VPC 撤销端口账本缺失', 409); + } + $revision = (int) $vpc->desired_revision + 1; + CloudVpcPort::query() + ->whereIn('id', $items->pluck('vpc_port_id')->all()) + ->where('active_claim', 1)->update([ + 'desired_revision' => $revision, + 'updated_at' => time(), + ]); + $batch->state = 'cleanup'; + $batch->desired_revision = $revision; + $batch->last_error_code = ''; + $batch->updated_at = time(); + $batch->save(); + $vpc->desired_revision = $revision; + $vpc->state = 'provisioning'; + $vpc->updated_at = time(); + $vpc->save(); + $jobs = CloudVpcPortService::queueBatchProjectionLocked( + $vpc, + $focus, + $revision, + (int) $batch->id + ); + if ($jobs === []) { + self::finishAttachCleanupLocked($batch, $parent, $vpc); + } + return ['job' => $parent, 'batch' => self::dto($batch), + 'unchanged' => false]; + }, + 3 + ); + } + /** Called inside the worker's child-finalization transaction. */ public static function finishChildLocked(CloudJob $job, string $status, $result, array $operation, CloudVpc $vpc, CloudVpcPort $port, @@ -393,15 +482,21 @@ final class CloudVpcMembershipBatchService return; } if ($outcomes['failed'] > 0) { + if ((string) $batch->kind === 'attach' + && (string) $batch->state === 'cleanup') { + self::blockLocked($batch, $vpc, + $code !== '' ? $code : 'vpc_attach_cleanup_failed'); + return; + } if ((string) $batch->kind === 'attach') { - CloudVpcMembershipBatchItem::query() - ->where('batch_id', (int) $batch->id) - ->where('state', 'waiting_route')->update([ - 'state' => 'failed', - 'last_error_code' => $code !== '' - ? $code : 'vpc_host_apply_failed', - 'updated_at' => time(), - ]); + self::beginAttachCleanupLocked( + $batch, + $parent, + $vpc, + $code !== '' ? $code : 'vpc_host_apply_failed', + (string) $current->operation_id + ); + return; } $vpc->state = 'degraded'; $vpc->updated_at = time(); @@ -413,12 +508,117 @@ final class CloudVpcMembershipBatchService return; } if ((string) $batch->kind === 'attach') { + if ((string) $batch->state === 'cleanup') { + self::finishAttachCleanupLocked($batch, $parent, $vpc); + return; + } self::releaseAttachPortJobsLocked($batch, $vpc); return; } self::finishDetachCleanupLocked($batch, $parent, $vpc); } + private static function beginAttachCleanupLocked( + CloudVpcMembershipBatch $batch, + CloudJob $parent, + CloudVpc $vpc, + string $code, + string $provenNotAppliedOperationId + ): void { + $items = self::itemsLocked($batch); + if ($items->isEmpty() + || $items->contains(static fn ($item): bool => + (string) $item->state !== 'waiting_route')) { + throw new CloudError('VPC 加入撤销明细状态不一致', 409); + } + $now = time(); + $revision = (int) $vpc->desired_revision + 1; + foreach ($items as $item) { + $port = CloudVpcPort::query() + ->where('id', (int) $item->vpc_port_id) + ->where('active_claim', 1)->lockForUpdate()->first(); + if ($port === null || (string) $port->state !== 'preparing') { + throw new CloudError('VPC 加入撤销端口状态不一致', 409); + } + $port->state = 'release_pending'; + $port->desired_revision = $revision; + $port->last_error_code = substr($code, 0, 64); + $port->updated_at = $now; + $port->save(); + $item->state = 'failed'; + $item->last_error_code = substr($code, 0, 64); + $item->updated_at = $now; + $item->save(); + } + $vpc->desired_revision = $revision; + $vpc->state = 'provisioning'; + $vpc->updated_at = $now; + $vpc->save(); + $batch->state = 'cleanup'; + $batch->desired_revision = $revision; + $batch->last_error_code = substr($code, 0, 64); + $batch->updated_at = $now; + $batch->save(); + $focus = CloudVpcPort::query() + ->where('id', (int) $items->first()->vpc_port_id) + ->where('active_claim', 1)->lockForUpdate()->first(); + if ($focus === null) { + throw new CloudError('VPC 加入撤销缺少焦点端口', 409); + } + $jobs = CloudVpcPortService::queueBatchProjectionLocked( + $vpc, + $focus, + $revision, + (int) $batch->id, + $provenNotAppliedOperationId + ); + if ($jobs === []) { + self::finishAttachCleanupLocked($batch, $parent, $vpc); + } + } + + private static function finishAttachCleanupLocked( + CloudVpcMembershipBatch $batch, + CloudJob $parent, + CloudVpc $vpc + ): void { + $items = self::itemsLocked($batch); + $now = time(); + $failureCode = 'vpc_batch_attach_route_failed'; + foreach ($items as $item) { + if ((string) $item->last_error_code !== '') { + $failureCode = (string) $item->last_error_code; + break; + } + } + foreach ($items as $item) { + $port = CloudVpcPort::query() + ->where('id', (int) $item->vpc_port_id) + ->where('active_claim', 1)->lockForUpdate()->first(); + if ($port === null || (string) $port->state !== 'release_pending') { + throw new CloudError('VPC 加入撤销释放账本不一致', 409); + } + $port->state = 'released'; + $port->active_claim = null; + $port->last_operation_id = null; + $port->last_error_code = ''; + $port->released_at = $now; + $port->updated_at = $now; + $port->save(); + } + $vpc->state = 'active'; + $vpc->updated_at = $now; + $vpc->save(); + self::finishLocked( + $batch, + $parent, + false, + false, + true, + $failureCode + ); + } + private static function releaseAttachPortJobsLocked( CloudVpcMembershipBatch $batch, CloudVpc $vpc @@ -630,6 +830,11 @@ final class CloudVpcMembershipBatchService 'unknown' => 0]; $jobs = CloudJob::query()->where('vpc_batch_id', (int) $batch->id) ->whereIn('kind', ['vpc_host_apply', 'vpc_host_remove']) + ->where('request_idempotency_key', 'like', + CloudVpcPortService::batchProjectionRequestPrefix( + (int) $batch->id, + (int) $batch->desired_revision + ) . '%') ->orderBy('id')->get(); if ($jobs->isEmpty()) { throw new CloudError('VPC 批次宿主机子任务账本缺失', 409); @@ -656,6 +861,51 @@ final class CloudVpcMembershipBatchService return $counts; } + private static function canRetryAttachCleanup( + CloudVpcMembershipBatch $batch, + $items, + bool $lock + ): bool { + if ((string) $batch->kind !== 'attach' || $items->isEmpty()) { + return false; + } + $portQuery = CloudVpcPort::query() + ->whereIn('id', $items->pluck('vpc_port_id')->all()) + ->where('active_claim', 1)->orderBy('id'); + $ports = $lock ? $portQuery->lockForUpdate()->get() + : $portQuery->get(); + if ($ports->count() !== $items->count() + || $ports->contains(static fn ($port): bool => + (string) $port->state !== 'release_pending')) { + return false; + } + $stateQuery = CloudVpcHostState::query() + ->where('vpc_id', (int) $batch->vpc_id)->orderBy('host_id'); + $states = $lock ? $stateQuery->lockForUpdate()->get() + : $stateQuery->get(); + foreach ($states as $state) { + if ((string) $state->sync_state === 'ready') { + continue; + } + if ((string) $state->sync_state !== 'failed' + || !is_string($state->last_operation_id)) { + return false; + } + $jobQuery = CloudJob::query() + ->where('operation_id', (string) $state->last_operation_id) + ->where('vpc_batch_id', (int) $batch->id); + $job = $lock ? $jobQuery->lockForUpdate()->first() + : $jobQuery->first(); + if ($job === null || (string) $job->phase !== 'terminal' + || (string) $job->outcome !== 'failed' + || (string) $job->effect_resolution !== 'not_applied' + || (int) $job->effect_may_have_applied !== 0) { + return false; + } + } + return true; + } + private static function newParentJob(int $userId, string $requestKey, string $fingerprint, CloudVpc $vpc, string $action, array $instanceIds): CloudJob @@ -807,6 +1057,8 @@ final class CloudVpcMembershipBatchService 'succeeded' => $items->where('state', 'succeeded')->count(), 'failed' => $items->where('state', 'failed')->count(), 'unknown' => $items->where('state', 'unknown')->count(), + 'can_cancel' => (string) $batch->state === 'blocked' + && self::canRetryAttachCleanup($batch, $items, false), 'task_id' => (string) $batch->parent_job_id, 'status_url' => '/api/v1/tasks/' . (int) $batch->parent_job_id, 'created_at' => (int) $batch->created_at, diff --git a/web/app/Services/Cloud/CloudVpcPortService.php b/web/app/Services/Cloud/CloudVpcPortService.php index 7989a26b..8ab51143 100644 --- a/web/app/Services/Cloud/CloudVpcPortService.php +++ b/web/app/Services/Cloud/CloudVpcPortService.php @@ -1241,7 +1241,8 @@ final class CloudVpcPortService /** @return list */ private static function queueProjectionLocked(CloudVpc $vpc, CloudVpcPort $focusPort, int $revision, - array $jobContext = []): array + array $jobContext = [], + ?string $provenNotAppliedOperationId = null): array { [$projectionPorts, $hostIds] = self::projectionRowsLocked($vpc); $transports = self::readyTransports($hostIds); @@ -1268,7 +1269,10 @@ final class CloudVpcPortService $state->last_failure_at = 0; $state->created_at = $now; } elseif (!self::hostReady($state) - && self::knownNotAppliedHostFailureLocked($state) === null) { + && !self::isKnownNotAppliedHostState( + $state, + $provenNotAppliedOperationId + )) { throw new CloudError( '旧 VPC 宿主机投影尚未收敛,不能生成新一代', 409 ); @@ -1288,7 +1292,10 @@ final class CloudVpcPortService } foreach ($states as $state) { if (!self::hostReady($state)) { - if (self::knownNotAppliedHostFailureLocked($state) === null) { + if (!self::isKnownNotAppliedHostState( + $state, + $provenNotAppliedOperationId + )) { throw new CloudError( '旧 VPC 宿主机投影尚未收敛,不能清理', 409 ); @@ -1453,6 +1460,17 @@ final class CloudVpcPortService $job = self::baseJob( $kind, $vpc, $port, (int) $state->host_id, $batchId ); + if ($batchId !== null && isset($jobContext['vpc_batch_revision'])) { + $job->request_idempotency_key = substr( + self::batchProjectionRequestPrefix( + $batchId, + (int) $jobContext['vpc_batch_revision'] + ) . $kind . '-' . (int) $state->host_id . '-' + . (string) $job->operation_id, + 0, + 128 + ); + } $body = $kind === 'vpc_host_apply' ? [ 'operation_id' => (string) $job->operation_id, 'desired_digest' => (string) $state->desired_digest, @@ -1607,11 +1625,42 @@ final class CloudVpcPortService CloudVpc $vpc, CloudVpcPort $focusPort, int $revision, - int $batchId + int $batchId, + ?string $provenNotAppliedOperationId = null ): array { return self::queueProjectionLocked($vpc, $focusPort, $revision, [ 'vpc_batch_id' => $batchId, - ]); + 'vpc_batch_revision' => $revision, + ], $provenNotAppliedOperationId); + } + + public static function batchProjectionRequestPrefix( + int $batchId, + int $revision + ): string { + if ($batchId < 1 || $revision < 1) { + throw new \InvalidArgumentException( + 'VPC batch projection identity must be positive' + ); + } + return 'vpc-batch-route-' . $batchId . '-' . $revision . '-'; + } + + private static function isKnownNotAppliedHostState( + CloudVpcHostState $state, + ?string $provenNotAppliedOperationId + ): bool { + if ((string) $state->sync_state === 'failed' + && $provenNotAppliedOperationId !== null + && preg_match('/^[a-f0-9]{32}$/D', + $provenNotAppliedOperationId) === 1 + && hash_equals( + (string) $state->last_operation_id, + $provenNotAppliedOperationId + )) { + return true; + } + return self::knownNotAppliedHostFailureLocked($state) !== null; } public static function queueBatchPortJobLocked( diff --git a/web/config/routes_cloud.php b/web/config/routes_cloud.php index 2afb0f63..fe520dfe 100644 --- a/web/config/routes_cloud.php +++ b/web/config/routes_cloud.php @@ -32,6 +32,10 @@ $app->group('/api/v1', function () { 'App\Controllers\CloudVpcController:delete'); $this->post('/vpcs/{vpc_key:[0-9a-f]{32}}/ports/batch', 'App\Controllers\CloudVpcController:membershipBatch'); + $this->post( + '/vpcs/{vpc_key:[0-9a-f]{32}}/ports/batches/{batch_key:[0-9a-f]{32}}/cancel', + 'App\Controllers\CloudVpcController:cancelMembershipBatch' + ); $this->map(['GET', 'POST'], '/cloud-snapshots', 'App\Controllers\CloudApiController:cloudSnapshots'); $this->delete('/cloud-snapshots/{snapshot_key:[0-9a-f]{32}}', diff --git a/web/public/cloud/vpcs.js b/web/public/cloud/vpcs.js index 009d81b4..3986197f 100644 --- a/web/public/cloud/vpcs.js +++ b/web/public/cloud/vpcs.js @@ -277,7 +277,9 @@ function renderBatchStatus(vpc) { routing: '正在一次同步所需路由;完成后会同时释放全部实例任务。', ports: `全部实例任务已释放,由工作进程并发执行关机、网卡变更和冷启动。`, cleanup: '实例网卡已处理,正在统一清理并确认宿主机路由。', - blocked: '有子任务结果无法确认,系统已停止重放并保留资源锁,请在任务中心交由管理员核对。', + blocked: batch.can_cancel === true + ? '加入操作已撤销,但宿主机清理未完成;可继续执行已确认未生效的安全清理。' + : '有子任务结果无法确认,系统已停止重放并保留资源锁,请在任务中心交由管理员核对。', }[batchState] || '服务端正在收敛批次状态。'; const progress = failed > 0 ? `${succeeded} 成功,${failed} 失败,共 ${total} 台` @@ -286,6 +288,14 @@ function renderBatchStatus(vpc) { const message = phase; content.append(element('strong', '', title), element('small', '', message)); box.append(content); + if (batch.can_cancel === true) { + const cancel = element('button', 'button button-outline small danger', + '继续撤销并清理'); + cancel.type = 'button'; + cancel.disabled = state.mutatingBatches.has(vpcKey(vpc)); + cancel.addEventListener('click', () => void cancelMembershipBatch(vpc, batch)); + box.append(cancel); + } return box; } @@ -732,6 +742,42 @@ async function submitMembershipBatch(vpc, action, instances) { } } +async function cancelMembershipBatch(vpc, batch) { + const key = vpcKey(vpc); + const batchKey = String(batch?.batch_key || ''); + if (!key || !/^[a-f0-9]{32}$/.test(batchKey) + || batch?.can_cancel !== true || state.mutatingBatches.has(key)) return; + const confirmed = await requestUserConfirmation({ + title: '继续撤销 VPC 加入操作', + message: '系统只会重试已确认未生效的宿主机清理,不会重放结果未知的操作。清理完成后将释放预留私网地址和实例操作锁。', + confirmText: '继续安全清理', + tone: 'danger', + }); + if (!confirmed || state.mutatingBatches.has(key)) return; + state.mutatingBatches.add(key); + setPageStatus('正在继续撤销 VPC 加入操作…'); + render(); + try { + const receipt = await api( + `/api/v1/vpcs/${encodeURIComponent(key)}/ports/batches/${encodeURIComponent(batchKey)}/cancel`, + { method: 'POST', expectedStatus: 202, requireTaskReceipt: true } + ); + const next = receipt?.batch || { ...batch, state: 'cleanup', can_cancel: false }; + state.membershipBatches = state.membershipBatches + .filter(item => String(item?.batch_key || '') !== batchKey); + state.membershipBatches.push(next); + observeBatch(next, false); + setPageStatus('已继续安全清理;完成前实例仍保持操作锁。'); + await load(true); + } catch (error) { + await load(true); + setPageStatus(error?.message || '继续撤销 VPC 加入操作失败。', 'error'); + } finally { + state.mutatingBatches.delete(key); + render(); + } +} + async function mutatePort(instance, vpc, requestedAction, options = {}) { const id = instanceId(instance); if (!id || state.mutatingInstances.has(id)) return { status: 'ignored' }; diff --git a/web/tests/CloudVpcPortWorkflowTest.php b/web/tests/CloudVpcPortWorkflowTest.php index aebe07af..788655a4 100644 --- a/web/tests/CloudVpcPortWorkflowTest.php +++ b/web/tests/CloudVpcPortWorkflowTest.php @@ -952,6 +952,194 @@ namespace { && CloudVpcHostState::where('vpc_id', 1)->count() === 0, 'Batch detach did not converge route cleanup before releasing ports.'); + // A known route-stage failure automatically creates a new projection that + // excludes every reserved port. Instance fences remain active until that + // rollback converges, then the IPs and fences are released together. + $rollbackInstances = [$newInstance(20, 1), $newInstance(21, 2)]; + $rollbackBatch = CloudVpcMembershipBatchService::start([ + 'user_id' => 7, + 'request_idempotency_key' => 'vpc-batch-attach-rollback-safe-0001', + ], (string) $vpc->vpc_key, [ + 'action' => 'attach', + 'instance_ids' => [20, 21], + 'confirm_restart' => true, + ]); + $rollbackBatchId = (int) CloudVpcMembershipBatch::where( + 'parent_job_id', (int) $rollbackBatch['job']->id + )->value('id'); + $rollbackRouteJobs = CloudJob::where('vpc_batch_id', $rollbackBatchId) + ->where('kind', 'vpc_host_apply')->orderBy('id')->get(); + foreach ($rollbackRouteJobs as $index => $routeJob) { + $payload = CloudSecrets::decrypt( + (string) $routeJob->sealed_payload, 'job-payload' + ); + if ($index === 0) { + $hostState = CloudVpcHostState::where('vpc_id', 1) + ->where('host_id', (int) $routeJob->host_id)->first(); + CloudVpcPortService::finishJobLocked($routeJob, 'succeeded', [ + 'vpc_key' => (string) $vpc->vpc_key, + 'desired_revision' => (int) $hostState->desired_revision, + 'desired_digest' => (string) $hostState->desired_digest, + 'bridge' => CloudVpcProjection::bridgeName( + (string) $vpc->vpc_key + ), + 'applied' => true, + ], $payload, [ + 'operation_id' => (string) $routeJob->operation_id, + ]); + $terminalize($routeJob); + } else { + CloudVpcPortService::finishJobLocked( + $routeJob, + 'failed', + null, + $payload, + [ + 'operation_id' => (string) $routeJob->operation_id, + 'error_code' => 'vpc_host_apply_failed', + ] + ); + $terminalizeFailed($routeJob); + } + } + $rollbackBatchRow = CloudVpcMembershipBatch::find($rollbackBatchId); + $rollbackCleanupJobs = CloudJob::where( + 'vpc_batch_id', $rollbackBatchId + )->whereIn('kind', ['vpc_host_apply', 'vpc_host_remove']) + ->where('phase', 'accepted')->orderBy('id')->get(); + $assert((string) $rollbackBatchRow->state === 'cleanup' + && $rollbackCleanupJobs->count() === 1 + && CloudVpcPort::whereIn('instance_id', [20, 21]) + ->where('state', 'release_pending')->count() === 2 + && CloudJob::where('vpc_batch_id', $rollbackBatchId) + ->where('kind', 'vpc_batch_instance_fence') + ->where('phase', '<>', 'terminal')->count() === 2, + 'Known route failure did not enter a fenced rollback generation.'); + foreach ($rollbackCleanupJobs as $cleanupJob) { + $payload = CloudSecrets::decrypt( + (string) $cleanupJob->sealed_payload, 'job-payload' + ); + $hostState = CloudVpcHostState::where('vpc_id', 1) + ->where('host_id', (int) $cleanupJob->host_id)->first(); + CloudVpcPortService::finishJobLocked($cleanupJob, 'succeeded', [ + 'vpc_key' => (string) $vpc->vpc_key, + 'desired_revision' => (int) $hostState->applied_revision, + 'desired_digest' => (string) $hostState->applied_digest, + 'removed' => true, + ], $payload, [ + 'operation_id' => (string) $cleanupJob->operation_id, + ]); + $terminalize($cleanupJob); + } + $assert((string) $rollbackBatch['job']->fresh()->outcome === 'failed' + && (int) $rollbackBatch['job']->fresh()->safe_to_resubmit === 1 + && CloudVpcPort::whereIn('instance_id', [20, 21]) + ->whereNotNull('active_claim')->count() === 0 + && CloudJob::where('vpc_batch_id', $rollbackBatchId) + ->where('kind', 'vpc_batch_instance_fence') + ->where('phase', '<>', 'terminal')->count() === 0, + 'Successful attach rollback did not release ports and instance fences.'); + + // A cleanup failure remains fenced. Once its exact Agent job is terminal + // and proven not applied, the user can continue the same rollback parent; + // an unknown result would keep can_cancel false and be rejected. + $newInstance(30, 1); $newInstance(31, 2); + $retryBatch = CloudVpcMembershipBatchService::start([ + 'user_id' => 7, + 'request_idempotency_key' => 'vpc-batch-attach-rollback-retry-0001', + ], (string) $vpc->vpc_key, [ + 'action' => 'attach', 'instance_ids' => [30, 31], + 'confirm_restart' => true, + ]); + $retryBatchId = (int) CloudVpcMembershipBatch::where( + 'parent_job_id', (int) $retryBatch['job']->id + )->value('id'); + $initialJobs = CloudJob::where('vpc_batch_id', $retryBatchId) + ->where('kind', 'vpc_host_apply')->orderBy('id')->get(); + foreach ($initialJobs as $index => $routeJob) { + $payload = CloudSecrets::decrypt( + (string) $routeJob->sealed_payload, 'job-payload' + ); + if ($index === 0) { + $hostState = CloudVpcHostState::where('vpc_id', 1) + ->where('host_id', (int) $routeJob->host_id)->first(); + CloudVpcPortService::finishJobLocked($routeJob, 'succeeded', [ + 'vpc_key' => (string) $vpc->vpc_key, + 'desired_revision' => (int) $hostState->desired_revision, + 'desired_digest' => (string) $hostState->desired_digest, + 'bridge' => CloudVpcProjection::bridgeName( + (string) $vpc->vpc_key + ), + 'applied' => true, + ], $payload, [ + 'operation_id' => (string) $routeJob->operation_id, + ]); + $terminalize($routeJob); + } else { + CloudVpcPortService::finishJobLocked($routeJob, 'failed', null, + $payload, [ + 'operation_id' => (string) $routeJob->operation_id, + 'error_code' => 'vpc_host_apply_failed', + ]); + $terminalizeFailed($routeJob); + } + } + $failedCleanup = CloudJob::where('vpc_batch_id', $retryBatchId) + ->where('kind', 'vpc_host_remove')->where('phase', 'accepted') + ->first(); + $failedCleanupPayload = CloudSecrets::decrypt( + (string) $failedCleanup->sealed_payload, 'job-payload' + ); + CloudVpcPortService::finishJobLocked($failedCleanup, 'failed', null, + $failedCleanupPayload, [ + 'operation_id' => (string) $failedCleanup->operation_id, + 'error_code' => 'vpc_host_remove_failed', + ]); + $terminalizeFailed($failedCleanup); + $retryBatchRow = CloudVpcMembershipBatch::find($retryBatchId); + $retryDto = collect( + CloudVpcMembershipBatchService::activeForOwner(7) + )->firstWhere('batch_key', (string) $retryBatchRow->batch_key); + $assert((string) $retryBatchRow->state === 'blocked' + && ($retryDto['can_cancel'] ?? false) === true + && collect([30, 31])->every(static fn (int $id): bool => + CloudJobService::blockingQueryForInstance($id)->exists()), + 'Known-not-applied rollback failure did not expose a safe cancel exit.'); + $retryResult = CloudVpcMembershipBatchService::cancelAttach( + ['user_id' => 7], + (string) $vpc->vpc_key, + (string) $retryBatchRow->batch_key + ); + $assert((int) $retryResult['job']->id === (int) $retryBatch['job']->id + && (string) CloudVpcMembershipBatch::find($retryBatchId)->state + === 'cleanup', + 'Safe rollback retry did not continue the original parent task.'); + $retriedCleanup = CloudJob::where('vpc_batch_id', $retryBatchId) + ->where('kind', 'vpc_host_remove')->where('phase', 'accepted') + ->first(); + $retriedPayload = CloudSecrets::decrypt( + (string) $retriedCleanup->sealed_payload, 'job-payload' + ); + $retryHostState = CloudVpcHostState::where('vpc_id', 1) + ->where('host_id', (int) $retriedCleanup->host_id)->first(); + CloudVpcPortService::finishJobLocked($retriedCleanup, 'succeeded', [ + 'vpc_key' => (string) $vpc->vpc_key, + 'desired_revision' => (int) $retryHostState->applied_revision, + 'desired_digest' => (string) $retryHostState->applied_digest, + 'removed' => true, + ], $retriedPayload, [ + 'operation_id' => (string) $retriedCleanup->operation_id, + ]); + $terminalize($retriedCleanup); + $assert((string) $retryBatch['job']->fresh()->outcome === 'failed', + 'Retried rollback did not terminalize its original parent.'); + $assert(CloudVpcPort::whereIn('instance_id', [30, 31]) + ->whereNotNull('active_claim')->count() === 0, + 'Retried rollback did not release every reserved private IP.'); + $assert(collect([30, 31])->every(static fn (int $id): bool => + !CloudJobService::blockingQueryForInstance($id)->exists()), + 'Retried rollback left an instance operation fence behind.'); + $worker = file_get_contents( dirname(__DIR__) . '/app/Services/Cloud/CloudWorkerService.php' ); diff --git a/web/tests/CloudVpcUserContractTest.php b/web/tests/CloudVpcUserContractTest.php index b581cfc3..3a2ae491 100644 --- a/web/tests/CloudVpcUserContractTest.php +++ b/web/tests/CloudVpcUserContractTest.php @@ -49,11 +49,14 @@ $assert(str_contains($apiRoutes, "'/vpcs'") && str_contains($apiRoutes, "'/vpcs/{vpc_key:") && str_contains($apiRoutes, "'/vms/{id:[0-9]+}/vpc'") && str_contains($apiRoutes, "'/vpcs/{vpc_key:[0-9a-f]{32}}/ports/batch'") + && str_contains($apiRoutes, + "'/vpcs/{vpc_key:[0-9a-f]{32}}/ports/batches/{batch_key:[0-9a-f]{32}}/cancel'") && str_contains($controller, "getHeaderLine('Idempotency-Key')") && str_contains($controller, 'CloudVpcPortService::listing') && str_contains($controller, 'CloudVpcPortService::attach') && str_contains($controller, 'CloudVpcPortService::detach') && str_contains($controller, 'CloudVpcMembershipBatchService::start') + && str_contains($controller, 'CloudVpcMembershipBatchService::cancelAttach') && str_contains($controller, "'Cache-Control', 'private, no-store'"), 'VPC user API lacks port lifecycle, idempotency or private-cache boundaries.'); $assert(str_contains($sidebar, "panel_cloud_active_key == 'vpcs'") @@ -89,6 +92,8 @@ $assert(str_contains($template, 'class="cloud-vpcs-stack"') && str_contains($javascript, 'function renderInstanceManager(') && str_contains($javascript, "element('details', 'cloud-vpc-card cloud-vpc-disclosure')") && str_contains($javascript, 'async function submitMembershipBatch(') + && str_contains($javascript, 'async function cancelMembershipBatch(') + && str_contains($javascript, 'batch?.can_cancel !== true') && str_contains($javascript, "action, instance_ids: ids") && str_contains($javascript, 'function observeActiveBatches(') && !str_contains($javascript, 'async function advanceBatchPlan(')