fix(vpc): add fail-closed batch rollback exit

go
codex 3 weeks ago
parent d4c69f4d77
commit 4640b8d869

@ -100,4 +100,19 @@ final class CloudVpcController
return $response->withJson($result, $status)
->withHeader('Cache-Control', 'private, no-store');
}
/** Continue a fail-closed attach rollback without replaying uncertainty. */
public function cancelMembershipBatch($request, $response, $args)
{
$actor = $this->actor($request);
$result = CloudVpcMembershipBatchService::cancelAttach(
$actor,
(string) ($args['vpc_key'] ?? ''),
(string) ($args['batch_key'] ?? '')
);
return $response->withJson([
'task_id' => (string) $result['job']->id,
'status_url' => '/api/v1/tasks/' . (int) $result['job']->id,
], 202)->withHeader('Cache-Control', 'private, no-store');
}
}

@ -158,6 +158,95 @@ final class CloudVpcMembershipBatchService
->all();
}
/**
* Continue only an attach rollback whose previous cleanup attempts are
* all proven not applied. This never replays an unknown Agent mutation.
*/
public static function cancelAttach(array $actor, string $vpcKey,
string $batchKey): array
{
if (preg_match('/^[a-f0-9]{32}$/D', $vpcKey) !== 1
|| preg_match('/^[a-f0-9]{32}$/D', $batchKey) !== 1) {
throw new CloudError('VPC 批次标识无效', 404);
}
$userId = (int) ($actor['user_id'] ?? 0);
return (new CloudVpcMembershipBatch())->getConnection()->transaction(
static function () use ($userId, $vpcKey, $batchKey): array {
$vpc = CloudVpc::query()->where('vpc_key', $vpcKey)
->lockForUpdate()->first();
if ($vpc === null || (int) $vpc->user_id !== $userId) {
throw new CloudError('VPC 不存在或当前不可用', 404);
}
$batch = CloudVpcMembershipBatch::query()
->where('batch_key', $batchKey)
->where('vpc_id', (int) $vpc->id)
->where('user_id', $userId)
->lockForUpdate()->first();
if ($batch === null || (string) $batch->kind !== 'attach') {
throw new CloudError('VPC 加入批次不存在', 404);
}
$parent = CloudJob::query()
->where('id', (int) $batch->parent_job_id)
->lockForUpdate()->first();
if ($parent === null
|| (string) $parent->kind !== self::PARENT_KIND) {
throw new CloudError('VPC 批次父任务账本不一致', 409);
}
if (in_array((string) $batch->state,
self::TERMINAL_STATES, true)) {
return ['job' => $parent, 'batch' => self::dto($batch),
'unchanged' => true];
}
if ((string) $batch->state === 'cleanup') {
return ['job' => $parent, 'batch' => self::dto($batch),
'unchanged' => true];
}
$items = self::itemsLocked($batch);
if ((string) $batch->state !== 'blocked'
|| !self::canRetryAttachCleanup($batch, $items, true)) {
throw new CloudError(
'撤销结果尚未确认,不能重复清理;请等待任务核对',
409
);
}
$focus = CloudVpcPort::query()
->where('id', (int) $items->first()->vpc_port_id)
->where('active_claim', 1)->lockForUpdate()->first();
if ($focus === null) {
throw new CloudError('VPC 撤销端口账本缺失', 409);
}
$revision = (int) $vpc->desired_revision + 1;
CloudVpcPort::query()
->whereIn('id', $items->pluck('vpc_port_id')->all())
->where('active_claim', 1)->update([
'desired_revision' => $revision,
'updated_at' => time(),
]);
$batch->state = 'cleanup';
$batch->desired_revision = $revision;
$batch->last_error_code = '';
$batch->updated_at = time();
$batch->save();
$vpc->desired_revision = $revision;
$vpc->state = 'provisioning';
$vpc->updated_at = time();
$vpc->save();
$jobs = CloudVpcPortService::queueBatchProjectionLocked(
$vpc,
$focus,
$revision,
(int) $batch->id
);
if ($jobs === []) {
self::finishAttachCleanupLocked($batch, $parent, $vpc);
}
return ['job' => $parent, 'batch' => self::dto($batch),
'unchanged' => false];
},
3
);
}
/** Called inside the worker's child-finalization transaction. */
public static function finishChildLocked(CloudJob $job, string $status,
$result, array $operation, CloudVpc $vpc, CloudVpcPort $port,
@ -393,15 +482,21 @@ final class CloudVpcMembershipBatchService
return;
}
if ($outcomes['failed'] > 0) {
if ((string) $batch->kind === 'attach'
&& (string) $batch->state === 'cleanup') {
self::blockLocked($batch, $vpc,
$code !== '' ? $code : 'vpc_attach_cleanup_failed');
return;
}
if ((string) $batch->kind === 'attach') {
CloudVpcMembershipBatchItem::query()
->where('batch_id', (int) $batch->id)
->where('state', 'waiting_route')->update([
'state' => 'failed',
'last_error_code' => $code !== ''
? $code : 'vpc_host_apply_failed',
'updated_at' => time(),
]);
self::beginAttachCleanupLocked(
$batch,
$parent,
$vpc,
$code !== '' ? $code : 'vpc_host_apply_failed',
(string) $current->operation_id
);
return;
}
$vpc->state = 'degraded';
$vpc->updated_at = time();
@ -413,12 +508,117 @@ final class CloudVpcMembershipBatchService
return;
}
if ((string) $batch->kind === 'attach') {
if ((string) $batch->state === 'cleanup') {
self::finishAttachCleanupLocked($batch, $parent, $vpc);
return;
}
self::releaseAttachPortJobsLocked($batch, $vpc);
return;
}
self::finishDetachCleanupLocked($batch, $parent, $vpc);
}
private static function beginAttachCleanupLocked(
CloudVpcMembershipBatch $batch,
CloudJob $parent,
CloudVpc $vpc,
string $code,
string $provenNotAppliedOperationId
): void {
$items = self::itemsLocked($batch);
if ($items->isEmpty()
|| $items->contains(static fn ($item): bool =>
(string) $item->state !== 'waiting_route')) {
throw new CloudError('VPC 加入撤销明细状态不一致', 409);
}
$now = time();
$revision = (int) $vpc->desired_revision + 1;
foreach ($items as $item) {
$port = CloudVpcPort::query()
->where('id', (int) $item->vpc_port_id)
->where('active_claim', 1)->lockForUpdate()->first();
if ($port === null || (string) $port->state !== 'preparing') {
throw new CloudError('VPC 加入撤销端口状态不一致', 409);
}
$port->state = 'release_pending';
$port->desired_revision = $revision;
$port->last_error_code = substr($code, 0, 64);
$port->updated_at = $now;
$port->save();
$item->state = 'failed';
$item->last_error_code = substr($code, 0, 64);
$item->updated_at = $now;
$item->save();
}
$vpc->desired_revision = $revision;
$vpc->state = 'provisioning';
$vpc->updated_at = $now;
$vpc->save();
$batch->state = 'cleanup';
$batch->desired_revision = $revision;
$batch->last_error_code = substr($code, 0, 64);
$batch->updated_at = $now;
$batch->save();
$focus = CloudVpcPort::query()
->where('id', (int) $items->first()->vpc_port_id)
->where('active_claim', 1)->lockForUpdate()->first();
if ($focus === null) {
throw new CloudError('VPC 加入撤销缺少焦点端口', 409);
}
$jobs = CloudVpcPortService::queueBatchProjectionLocked(
$vpc,
$focus,
$revision,
(int) $batch->id,
$provenNotAppliedOperationId
);
if ($jobs === []) {
self::finishAttachCleanupLocked($batch, $parent, $vpc);
}
}
private static function finishAttachCleanupLocked(
CloudVpcMembershipBatch $batch,
CloudJob $parent,
CloudVpc $vpc
): void {
$items = self::itemsLocked($batch);
$now = time();
$failureCode = 'vpc_batch_attach_route_failed';
foreach ($items as $item) {
if ((string) $item->last_error_code !== '') {
$failureCode = (string) $item->last_error_code;
break;
}
}
foreach ($items as $item) {
$port = CloudVpcPort::query()
->where('id', (int) $item->vpc_port_id)
->where('active_claim', 1)->lockForUpdate()->first();
if ($port === null || (string) $port->state !== 'release_pending') {
throw new CloudError('VPC 加入撤销释放账本不一致', 409);
}
$port->state = 'released';
$port->active_claim = null;
$port->last_operation_id = null;
$port->last_error_code = '';
$port->released_at = $now;
$port->updated_at = $now;
$port->save();
}
$vpc->state = 'active';
$vpc->updated_at = $now;
$vpc->save();
self::finishLocked(
$batch,
$parent,
false,
false,
true,
$failureCode
);
}
private static function releaseAttachPortJobsLocked(
CloudVpcMembershipBatch $batch,
CloudVpc $vpc
@ -630,6 +830,11 @@ final class CloudVpcMembershipBatchService
'unknown' => 0];
$jobs = CloudJob::query()->where('vpc_batch_id', (int) $batch->id)
->whereIn('kind', ['vpc_host_apply', 'vpc_host_remove'])
->where('request_idempotency_key', 'like',
CloudVpcPortService::batchProjectionRequestPrefix(
(int) $batch->id,
(int) $batch->desired_revision
) . '%')
->orderBy('id')->get();
if ($jobs->isEmpty()) {
throw new CloudError('VPC 批次宿主机子任务账本缺失', 409);
@ -656,6 +861,51 @@ final class CloudVpcMembershipBatchService
return $counts;
}
private static function canRetryAttachCleanup(
CloudVpcMembershipBatch $batch,
$items,
bool $lock
): bool {
if ((string) $batch->kind !== 'attach' || $items->isEmpty()) {
return false;
}
$portQuery = CloudVpcPort::query()
->whereIn('id', $items->pluck('vpc_port_id')->all())
->where('active_claim', 1)->orderBy('id');
$ports = $lock ? $portQuery->lockForUpdate()->get()
: $portQuery->get();
if ($ports->count() !== $items->count()
|| $ports->contains(static fn ($port): bool =>
(string) $port->state !== 'release_pending')) {
return false;
}
$stateQuery = CloudVpcHostState::query()
->where('vpc_id', (int) $batch->vpc_id)->orderBy('host_id');
$states = $lock ? $stateQuery->lockForUpdate()->get()
: $stateQuery->get();
foreach ($states as $state) {
if ((string) $state->sync_state === 'ready') {
continue;
}
if ((string) $state->sync_state !== 'failed'
|| !is_string($state->last_operation_id)) {
return false;
}
$jobQuery = CloudJob::query()
->where('operation_id', (string) $state->last_operation_id)
->where('vpc_batch_id', (int) $batch->id);
$job = $lock ? $jobQuery->lockForUpdate()->first()
: $jobQuery->first();
if ($job === null || (string) $job->phase !== 'terminal'
|| (string) $job->outcome !== 'failed'
|| (string) $job->effect_resolution !== 'not_applied'
|| (int) $job->effect_may_have_applied !== 0) {
return false;
}
}
return true;
}
private static function newParentJob(int $userId, string $requestKey,
string $fingerprint, CloudVpc $vpc, string $action,
array $instanceIds): CloudJob
@ -807,6 +1057,8 @@ final class CloudVpcMembershipBatchService
'succeeded' => $items->where('state', 'succeeded')->count(),
'failed' => $items->where('state', 'failed')->count(),
'unknown' => $items->where('state', 'unknown')->count(),
'can_cancel' => (string) $batch->state === 'blocked'
&& self::canRetryAttachCleanup($batch, $items, false),
'task_id' => (string) $batch->parent_job_id,
'status_url' => '/api/v1/tasks/' . (int) $batch->parent_job_id,
'created_at' => (int) $batch->created_at,

@ -1241,7 +1241,8 @@ final class CloudVpcPortService
/** @return list<CloudJob> */
private static function queueProjectionLocked(CloudVpc $vpc,
CloudVpcPort $focusPort, int $revision,
array $jobContext = []): array
array $jobContext = [],
?string $provenNotAppliedOperationId = null): array
{
[$projectionPorts, $hostIds] = self::projectionRowsLocked($vpc);
$transports = self::readyTransports($hostIds);
@ -1268,7 +1269,10 @@ final class CloudVpcPortService
$state->last_failure_at = 0;
$state->created_at = $now;
} elseif (!self::hostReady($state)
&& self::knownNotAppliedHostFailureLocked($state) === null) {
&& !self::isKnownNotAppliedHostState(
$state,
$provenNotAppliedOperationId
)) {
throw new CloudError(
'旧 VPC 宿主机投影尚未收敛,不能生成新一代', 409
);
@ -1288,7 +1292,10 @@ final class CloudVpcPortService
}
foreach ($states as $state) {
if (!self::hostReady($state)) {
if (self::knownNotAppliedHostFailureLocked($state) === null) {
if (!self::isKnownNotAppliedHostState(
$state,
$provenNotAppliedOperationId
)) {
throw new CloudError(
'旧 VPC 宿主机投影尚未收敛,不能清理', 409
);
@ -1453,6 +1460,17 @@ final class CloudVpcPortService
$job = self::baseJob(
$kind, $vpc, $port, (int) $state->host_id, $batchId
);
if ($batchId !== null && isset($jobContext['vpc_batch_revision'])) {
$job->request_idempotency_key = substr(
self::batchProjectionRequestPrefix(
$batchId,
(int) $jobContext['vpc_batch_revision']
) . $kind . '-' . (int) $state->host_id . '-'
. (string) $job->operation_id,
0,
128
);
}
$body = $kind === 'vpc_host_apply' ? [
'operation_id' => (string) $job->operation_id,
'desired_digest' => (string) $state->desired_digest,
@ -1607,11 +1625,42 @@ final class CloudVpcPortService
CloudVpc $vpc,
CloudVpcPort $focusPort,
int $revision,
int $batchId
int $batchId,
?string $provenNotAppliedOperationId = null
): array {
return self::queueProjectionLocked($vpc, $focusPort, $revision, [
'vpc_batch_id' => $batchId,
]);
'vpc_batch_revision' => $revision,
], $provenNotAppliedOperationId);
}
public static function batchProjectionRequestPrefix(
int $batchId,
int $revision
): string {
if ($batchId < 1 || $revision < 1) {
throw new \InvalidArgumentException(
'VPC batch projection identity must be positive'
);
}
return 'vpc-batch-route-' . $batchId . '-' . $revision . '-';
}
private static function isKnownNotAppliedHostState(
CloudVpcHostState $state,
?string $provenNotAppliedOperationId
): bool {
if ((string) $state->sync_state === 'failed'
&& $provenNotAppliedOperationId !== null
&& preg_match('/^[a-f0-9]{32}$/D',
$provenNotAppliedOperationId) === 1
&& hash_equals(
(string) $state->last_operation_id,
$provenNotAppliedOperationId
)) {
return true;
}
return self::knownNotAppliedHostFailureLocked($state) !== null;
}
public static function queueBatchPortJobLocked(

@ -32,6 +32,10 @@ $app->group('/api/v1', function () {
'App\Controllers\CloudVpcController:delete');
$this->post('/vpcs/{vpc_key:[0-9a-f]{32}}/ports/batch',
'App\Controllers\CloudVpcController:membershipBatch');
$this->post(
'/vpcs/{vpc_key:[0-9a-f]{32}}/ports/batches/{batch_key:[0-9a-f]{32}}/cancel',
'App\Controllers\CloudVpcController:cancelMembershipBatch'
);
$this->map(['GET', 'POST'], '/cloud-snapshots',
'App\Controllers\CloudApiController:cloudSnapshots');
$this->delete('/cloud-snapshots/{snapshot_key:[0-9a-f]{32}}',

@ -277,7 +277,9 @@ function renderBatchStatus(vpc) {
routing: '正在一次同步所需路由;完成后会同时释放全部实例任务。',
ports: `全部实例任务已释放,由工作进程并发执行关机、网卡变更和冷启动。`,
cleanup: '实例网卡已处理,正在统一清理并确认宿主机路由。',
blocked: '有子任务结果无法确认,系统已停止重放并保留资源锁,请在任务中心交由管理员核对。',
blocked: batch.can_cancel === true
? '加入操作已撤销,但宿主机清理未完成;可继续执行已确认未生效的安全清理。'
: '有子任务结果无法确认,系统已停止重放并保留资源锁,请在任务中心交由管理员核对。',
}[batchState] || '服务端正在收敛批次状态。';
const progress = failed > 0
? `${succeeded} 成功,${failed} 失败,共 ${total} 台`
@ -286,6 +288,14 @@ function renderBatchStatus(vpc) {
const message = phase;
content.append(element('strong', '', title), element('small', '', message));
box.append(content);
if (batch.can_cancel === true) {
const cancel = element('button', 'button button-outline small danger',
'继续撤销并清理');
cancel.type = 'button';
cancel.disabled = state.mutatingBatches.has(vpcKey(vpc));
cancel.addEventListener('click', () => void cancelMembershipBatch(vpc, batch));
box.append(cancel);
}
return box;
}
@ -732,6 +742,42 @@ async function submitMembershipBatch(vpc, action, instances) {
}
}
async function cancelMembershipBatch(vpc, batch) {
const key = vpcKey(vpc);
const batchKey = String(batch?.batch_key || '');
if (!key || !/^[a-f0-9]{32}$/.test(batchKey)
|| batch?.can_cancel !== true || state.mutatingBatches.has(key)) return;
const confirmed = await requestUserConfirmation({
title: '继续撤销 VPC 加入操作',
message: '系统只会重试已确认未生效的宿主机清理,不会重放结果未知的操作。清理完成后将释放预留私网地址和实例操作锁。',
confirmText: '继续安全清理',
tone: 'danger',
});
if (!confirmed || state.mutatingBatches.has(key)) return;
state.mutatingBatches.add(key);
setPageStatus('正在继续撤销 VPC 加入操作…');
render();
try {
const receipt = await api(
`/api/v1/vpcs/${encodeURIComponent(key)}/ports/batches/${encodeURIComponent(batchKey)}/cancel`,
{ method: 'POST', expectedStatus: 202, requireTaskReceipt: true }
);
const next = receipt?.batch || { ...batch, state: 'cleanup', can_cancel: false };
state.membershipBatches = state.membershipBatches
.filter(item => String(item?.batch_key || '') !== batchKey);
state.membershipBatches.push(next);
observeBatch(next, false);
setPageStatus('已继续安全清理;完成前实例仍保持操作锁。');
await load(true);
} catch (error) {
await load(true);
setPageStatus(error?.message || '继续撤销 VPC 加入操作失败。', 'error');
} finally {
state.mutatingBatches.delete(key);
render();
}
}
async function mutatePort(instance, vpc, requestedAction, options = {}) {
const id = instanceId(instance);
if (!id || state.mutatingInstances.has(id)) return { status: 'ignored' };

@ -952,6 +952,194 @@ namespace {
&& CloudVpcHostState::where('vpc_id', 1)->count() === 0,
'Batch detach did not converge route cleanup before releasing ports.');
// A known route-stage failure automatically creates a new projection that
// excludes every reserved port. Instance fences remain active until that
// rollback converges, then the IPs and fences are released together.
$rollbackInstances = [$newInstance(20, 1), $newInstance(21, 2)];
$rollbackBatch = CloudVpcMembershipBatchService::start([
'user_id' => 7,
'request_idempotency_key' => 'vpc-batch-attach-rollback-safe-0001',
], (string) $vpc->vpc_key, [
'action' => 'attach',
'instance_ids' => [20, 21],
'confirm_restart' => true,
]);
$rollbackBatchId = (int) CloudVpcMembershipBatch::where(
'parent_job_id', (int) $rollbackBatch['job']->id
)->value('id');
$rollbackRouteJobs = CloudJob::where('vpc_batch_id', $rollbackBatchId)
->where('kind', 'vpc_host_apply')->orderBy('id')->get();
foreach ($rollbackRouteJobs as $index => $routeJob) {
$payload = CloudSecrets::decrypt(
(string) $routeJob->sealed_payload, 'job-payload'
);
if ($index === 0) {
$hostState = CloudVpcHostState::where('vpc_id', 1)
->where('host_id', (int) $routeJob->host_id)->first();
CloudVpcPortService::finishJobLocked($routeJob, 'succeeded', [
'vpc_key' => (string) $vpc->vpc_key,
'desired_revision' => (int) $hostState->desired_revision,
'desired_digest' => (string) $hostState->desired_digest,
'bridge' => CloudVpcProjection::bridgeName(
(string) $vpc->vpc_key
),
'applied' => true,
], $payload, [
'operation_id' => (string) $routeJob->operation_id,
]);
$terminalize($routeJob);
} else {
CloudVpcPortService::finishJobLocked(
$routeJob,
'failed',
null,
$payload,
[
'operation_id' => (string) $routeJob->operation_id,
'error_code' => 'vpc_host_apply_failed',
]
);
$terminalizeFailed($routeJob);
}
}
$rollbackBatchRow = CloudVpcMembershipBatch::find($rollbackBatchId);
$rollbackCleanupJobs = CloudJob::where(
'vpc_batch_id', $rollbackBatchId
)->whereIn('kind', ['vpc_host_apply', 'vpc_host_remove'])
->where('phase', 'accepted')->orderBy('id')->get();
$assert((string) $rollbackBatchRow->state === 'cleanup'
&& $rollbackCleanupJobs->count() === 1
&& CloudVpcPort::whereIn('instance_id', [20, 21])
->where('state', 'release_pending')->count() === 2
&& CloudJob::where('vpc_batch_id', $rollbackBatchId)
->where('kind', 'vpc_batch_instance_fence')
->where('phase', '<>', 'terminal')->count() === 2,
'Known route failure did not enter a fenced rollback generation.');
foreach ($rollbackCleanupJobs as $cleanupJob) {
$payload = CloudSecrets::decrypt(
(string) $cleanupJob->sealed_payload, 'job-payload'
);
$hostState = CloudVpcHostState::where('vpc_id', 1)
->where('host_id', (int) $cleanupJob->host_id)->first();
CloudVpcPortService::finishJobLocked($cleanupJob, 'succeeded', [
'vpc_key' => (string) $vpc->vpc_key,
'desired_revision' => (int) $hostState->applied_revision,
'desired_digest' => (string) $hostState->applied_digest,
'removed' => true,
], $payload, [
'operation_id' => (string) $cleanupJob->operation_id,
]);
$terminalize($cleanupJob);
}
$assert((string) $rollbackBatch['job']->fresh()->outcome === 'failed'
&& (int) $rollbackBatch['job']->fresh()->safe_to_resubmit === 1
&& CloudVpcPort::whereIn('instance_id', [20, 21])
->whereNotNull('active_claim')->count() === 0
&& CloudJob::where('vpc_batch_id', $rollbackBatchId)
->where('kind', 'vpc_batch_instance_fence')
->where('phase', '<>', 'terminal')->count() === 0,
'Successful attach rollback did not release ports and instance fences.');
// A cleanup failure remains fenced. Once its exact Agent job is terminal
// and proven not applied, the user can continue the same rollback parent;
// an unknown result would keep can_cancel false and be rejected.
$newInstance(30, 1); $newInstance(31, 2);
$retryBatch = CloudVpcMembershipBatchService::start([
'user_id' => 7,
'request_idempotency_key' => 'vpc-batch-attach-rollback-retry-0001',
], (string) $vpc->vpc_key, [
'action' => 'attach', 'instance_ids' => [30, 31],
'confirm_restart' => true,
]);
$retryBatchId = (int) CloudVpcMembershipBatch::where(
'parent_job_id', (int) $retryBatch['job']->id
)->value('id');
$initialJobs = CloudJob::where('vpc_batch_id', $retryBatchId)
->where('kind', 'vpc_host_apply')->orderBy('id')->get();
foreach ($initialJobs as $index => $routeJob) {
$payload = CloudSecrets::decrypt(
(string) $routeJob->sealed_payload, 'job-payload'
);
if ($index === 0) {
$hostState = CloudVpcHostState::where('vpc_id', 1)
->where('host_id', (int) $routeJob->host_id)->first();
CloudVpcPortService::finishJobLocked($routeJob, 'succeeded', [
'vpc_key' => (string) $vpc->vpc_key,
'desired_revision' => (int) $hostState->desired_revision,
'desired_digest' => (string) $hostState->desired_digest,
'bridge' => CloudVpcProjection::bridgeName(
(string) $vpc->vpc_key
),
'applied' => true,
], $payload, [
'operation_id' => (string) $routeJob->operation_id,
]);
$terminalize($routeJob);
} else {
CloudVpcPortService::finishJobLocked($routeJob, 'failed', null,
$payload, [
'operation_id' => (string) $routeJob->operation_id,
'error_code' => 'vpc_host_apply_failed',
]);
$terminalizeFailed($routeJob);
}
}
$failedCleanup = CloudJob::where('vpc_batch_id', $retryBatchId)
->where('kind', 'vpc_host_remove')->where('phase', 'accepted')
->first();
$failedCleanupPayload = CloudSecrets::decrypt(
(string) $failedCleanup->sealed_payload, 'job-payload'
);
CloudVpcPortService::finishJobLocked($failedCleanup, 'failed', null,
$failedCleanupPayload, [
'operation_id' => (string) $failedCleanup->operation_id,
'error_code' => 'vpc_host_remove_failed',
]);
$terminalizeFailed($failedCleanup);
$retryBatchRow = CloudVpcMembershipBatch::find($retryBatchId);
$retryDto = collect(
CloudVpcMembershipBatchService::activeForOwner(7)
)->firstWhere('batch_key', (string) $retryBatchRow->batch_key);
$assert((string) $retryBatchRow->state === 'blocked'
&& ($retryDto['can_cancel'] ?? false) === true
&& collect([30, 31])->every(static fn (int $id): bool =>
CloudJobService::blockingQueryForInstance($id)->exists()),
'Known-not-applied rollback failure did not expose a safe cancel exit.');
$retryResult = CloudVpcMembershipBatchService::cancelAttach(
['user_id' => 7],
(string) $vpc->vpc_key,
(string) $retryBatchRow->batch_key
);
$assert((int) $retryResult['job']->id === (int) $retryBatch['job']->id
&& (string) CloudVpcMembershipBatch::find($retryBatchId)->state
=== 'cleanup',
'Safe rollback retry did not continue the original parent task.');
$retriedCleanup = CloudJob::where('vpc_batch_id', $retryBatchId)
->where('kind', 'vpc_host_remove')->where('phase', 'accepted')
->first();
$retriedPayload = CloudSecrets::decrypt(
(string) $retriedCleanup->sealed_payload, 'job-payload'
);
$retryHostState = CloudVpcHostState::where('vpc_id', 1)
->where('host_id', (int) $retriedCleanup->host_id)->first();
CloudVpcPortService::finishJobLocked($retriedCleanup, 'succeeded', [
'vpc_key' => (string) $vpc->vpc_key,
'desired_revision' => (int) $retryHostState->applied_revision,
'desired_digest' => (string) $retryHostState->applied_digest,
'removed' => true,
], $retriedPayload, [
'operation_id' => (string) $retriedCleanup->operation_id,
]);
$terminalize($retriedCleanup);
$assert((string) $retryBatch['job']->fresh()->outcome === 'failed',
'Retried rollback did not terminalize its original parent.');
$assert(CloudVpcPort::whereIn('instance_id', [30, 31])
->whereNotNull('active_claim')->count() === 0,
'Retried rollback did not release every reserved private IP.');
$assert(collect([30, 31])->every(static fn (int $id): bool =>
!CloudJobService::blockingQueryForInstance($id)->exists()),
'Retried rollback left an instance operation fence behind.');
$worker = file_get_contents(
dirname(__DIR__) . '/app/Services/Cloud/CloudWorkerService.php'
);

@ -49,11 +49,14 @@ $assert(str_contains($apiRoutes, "'/vpcs'")
&& str_contains($apiRoutes, "'/vpcs/{vpc_key:")
&& str_contains($apiRoutes, "'/vms/{id:[0-9]+}/vpc'")
&& str_contains($apiRoutes, "'/vpcs/{vpc_key:[0-9a-f]{32}}/ports/batch'")
&& str_contains($apiRoutes,
"'/vpcs/{vpc_key:[0-9a-f]{32}}/ports/batches/{batch_key:[0-9a-f]{32}}/cancel'")
&& str_contains($controller, "getHeaderLine('Idempotency-Key')")
&& str_contains($controller, 'CloudVpcPortService::listing')
&& str_contains($controller, 'CloudVpcPortService::attach')
&& str_contains($controller, 'CloudVpcPortService::detach')
&& str_contains($controller, 'CloudVpcMembershipBatchService::start')
&& str_contains($controller, 'CloudVpcMembershipBatchService::cancelAttach')
&& str_contains($controller, "'Cache-Control', 'private, no-store'"),
'VPC user API lacks port lifecycle, idempotency or private-cache boundaries.');
$assert(str_contains($sidebar, "panel_cloud_active_key == 'vpcs'")
@ -89,6 +92,8 @@ $assert(str_contains($template, 'class="cloud-vpcs-stack"')
&& str_contains($javascript, 'function renderInstanceManager(')
&& str_contains($javascript, "element('details', 'cloud-vpc-card cloud-vpc-disclosure')")
&& str_contains($javascript, 'async function submitMembershipBatch(')
&& str_contains($javascript, 'async function cancelMembershipBatch(')
&& str_contains($javascript, 'batch?.can_cancel !== true')
&& str_contains($javascript, "action, instance_ids: ids")
&& str_contains($javascript, 'function observeActiveBatches(')
&& !str_contains($javascript, 'async function advanceBatchPlan(')

Loading…
Cancel
Save