|
|
|
|
@ -158,6 +158,95 @@ final class CloudVpcMembershipBatchService
|
|
|
|
|
->all();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Continue only an attach rollback whose previous cleanup attempts are
|
|
|
|
|
* all proven not applied. This never replays an unknown Agent mutation.
|
|
|
|
|
*/
|
|
|
|
|
public static function cancelAttach(array $actor, string $vpcKey,
|
|
|
|
|
string $batchKey): array
|
|
|
|
|
{
|
|
|
|
|
if (preg_match('/^[a-f0-9]{32}$/D', $vpcKey) !== 1
|
|
|
|
|
|| preg_match('/^[a-f0-9]{32}$/D', $batchKey) !== 1) {
|
|
|
|
|
throw new CloudError('VPC 批次标识无效', 404);
|
|
|
|
|
}
|
|
|
|
|
$userId = (int) ($actor['user_id'] ?? 0);
|
|
|
|
|
return (new CloudVpcMembershipBatch())->getConnection()->transaction(
|
|
|
|
|
static function () use ($userId, $vpcKey, $batchKey): array {
|
|
|
|
|
$vpc = CloudVpc::query()->where('vpc_key', $vpcKey)
|
|
|
|
|
->lockForUpdate()->first();
|
|
|
|
|
if ($vpc === null || (int) $vpc->user_id !== $userId) {
|
|
|
|
|
throw new CloudError('VPC 不存在或当前不可用', 404);
|
|
|
|
|
}
|
|
|
|
|
$batch = CloudVpcMembershipBatch::query()
|
|
|
|
|
->where('batch_key', $batchKey)
|
|
|
|
|
->where('vpc_id', (int) $vpc->id)
|
|
|
|
|
->where('user_id', $userId)
|
|
|
|
|
->lockForUpdate()->first();
|
|
|
|
|
if ($batch === null || (string) $batch->kind !== 'attach') {
|
|
|
|
|
throw new CloudError('VPC 加入批次不存在', 404);
|
|
|
|
|
}
|
|
|
|
|
$parent = CloudJob::query()
|
|
|
|
|
->where('id', (int) $batch->parent_job_id)
|
|
|
|
|
->lockForUpdate()->first();
|
|
|
|
|
if ($parent === null
|
|
|
|
|
|| (string) $parent->kind !== self::PARENT_KIND) {
|
|
|
|
|
throw new CloudError('VPC 批次父任务账本不一致', 409);
|
|
|
|
|
}
|
|
|
|
|
if (in_array((string) $batch->state,
|
|
|
|
|
self::TERMINAL_STATES, true)) {
|
|
|
|
|
return ['job' => $parent, 'batch' => self::dto($batch),
|
|
|
|
|
'unchanged' => true];
|
|
|
|
|
}
|
|
|
|
|
if ((string) $batch->state === 'cleanup') {
|
|
|
|
|
return ['job' => $parent, 'batch' => self::dto($batch),
|
|
|
|
|
'unchanged' => true];
|
|
|
|
|
}
|
|
|
|
|
$items = self::itemsLocked($batch);
|
|
|
|
|
if ((string) $batch->state !== 'blocked'
|
|
|
|
|
|| !self::canRetryAttachCleanup($batch, $items, true)) {
|
|
|
|
|
throw new CloudError(
|
|
|
|
|
'撤销结果尚未确认,不能重复清理;请等待任务核对',
|
|
|
|
|
409
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
$focus = CloudVpcPort::query()
|
|
|
|
|
->where('id', (int) $items->first()->vpc_port_id)
|
|
|
|
|
->where('active_claim', 1)->lockForUpdate()->first();
|
|
|
|
|
if ($focus === null) {
|
|
|
|
|
throw new CloudError('VPC 撤销端口账本缺失', 409);
|
|
|
|
|
}
|
|
|
|
|
$revision = (int) $vpc->desired_revision + 1;
|
|
|
|
|
CloudVpcPort::query()
|
|
|
|
|
->whereIn('id', $items->pluck('vpc_port_id')->all())
|
|
|
|
|
->where('active_claim', 1)->update([
|
|
|
|
|
'desired_revision' => $revision,
|
|
|
|
|
'updated_at' => time(),
|
|
|
|
|
]);
|
|
|
|
|
$batch->state = 'cleanup';
|
|
|
|
|
$batch->desired_revision = $revision;
|
|
|
|
|
$batch->last_error_code = '';
|
|
|
|
|
$batch->updated_at = time();
|
|
|
|
|
$batch->save();
|
|
|
|
|
$vpc->desired_revision = $revision;
|
|
|
|
|
$vpc->state = 'provisioning';
|
|
|
|
|
$vpc->updated_at = time();
|
|
|
|
|
$vpc->save();
|
|
|
|
|
$jobs = CloudVpcPortService::queueBatchProjectionLocked(
|
|
|
|
|
$vpc,
|
|
|
|
|
$focus,
|
|
|
|
|
$revision,
|
|
|
|
|
(int) $batch->id
|
|
|
|
|
);
|
|
|
|
|
if ($jobs === []) {
|
|
|
|
|
self::finishAttachCleanupLocked($batch, $parent, $vpc);
|
|
|
|
|
}
|
|
|
|
|
return ['job' => $parent, 'batch' => self::dto($batch),
|
|
|
|
|
'unchanged' => false];
|
|
|
|
|
},
|
|
|
|
|
3
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** Called inside the worker's child-finalization transaction. */
|
|
|
|
|
public static function finishChildLocked(CloudJob $job, string $status,
|
|
|
|
|
$result, array $operation, CloudVpc $vpc, CloudVpcPort $port,
|
|
|
|
|
@ -393,15 +482,21 @@ final class CloudVpcMembershipBatchService
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
if ($outcomes['failed'] > 0) {
|
|
|
|
|
if ((string) $batch->kind === 'attach'
|
|
|
|
|
&& (string) $batch->state === 'cleanup') {
|
|
|
|
|
self::blockLocked($batch, $vpc,
|
|
|
|
|
$code !== '' ? $code : 'vpc_attach_cleanup_failed');
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
if ((string) $batch->kind === 'attach') {
|
|
|
|
|
CloudVpcMembershipBatchItem::query()
|
|
|
|
|
->where('batch_id', (int) $batch->id)
|
|
|
|
|
->where('state', 'waiting_route')->update([
|
|
|
|
|
'state' => 'failed',
|
|
|
|
|
'last_error_code' => $code !== ''
|
|
|
|
|
? $code : 'vpc_host_apply_failed',
|
|
|
|
|
'updated_at' => time(),
|
|
|
|
|
]);
|
|
|
|
|
self::beginAttachCleanupLocked(
|
|
|
|
|
$batch,
|
|
|
|
|
$parent,
|
|
|
|
|
$vpc,
|
|
|
|
|
$code !== '' ? $code : 'vpc_host_apply_failed',
|
|
|
|
|
(string) $current->operation_id
|
|
|
|
|
);
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
$vpc->state = 'degraded';
|
|
|
|
|
$vpc->updated_at = time();
|
|
|
|
|
@ -413,12 +508,117 @@ final class CloudVpcMembershipBatchService
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
if ((string) $batch->kind === 'attach') {
|
|
|
|
|
if ((string) $batch->state === 'cleanup') {
|
|
|
|
|
self::finishAttachCleanupLocked($batch, $parent, $vpc);
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
self::releaseAttachPortJobsLocked($batch, $vpc);
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
self::finishDetachCleanupLocked($batch, $parent, $vpc);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static function beginAttachCleanupLocked(
|
|
|
|
|
CloudVpcMembershipBatch $batch,
|
|
|
|
|
CloudJob $parent,
|
|
|
|
|
CloudVpc $vpc,
|
|
|
|
|
string $code,
|
|
|
|
|
string $provenNotAppliedOperationId
|
|
|
|
|
): void {
|
|
|
|
|
$items = self::itemsLocked($batch);
|
|
|
|
|
if ($items->isEmpty()
|
|
|
|
|
|| $items->contains(static fn ($item): bool =>
|
|
|
|
|
(string) $item->state !== 'waiting_route')) {
|
|
|
|
|
throw new CloudError('VPC 加入撤销明细状态不一致', 409);
|
|
|
|
|
}
|
|
|
|
|
$now = time();
|
|
|
|
|
$revision = (int) $vpc->desired_revision + 1;
|
|
|
|
|
foreach ($items as $item) {
|
|
|
|
|
$port = CloudVpcPort::query()
|
|
|
|
|
->where('id', (int) $item->vpc_port_id)
|
|
|
|
|
->where('active_claim', 1)->lockForUpdate()->first();
|
|
|
|
|
if ($port === null || (string) $port->state !== 'preparing') {
|
|
|
|
|
throw new CloudError('VPC 加入撤销端口状态不一致', 409);
|
|
|
|
|
}
|
|
|
|
|
$port->state = 'release_pending';
|
|
|
|
|
$port->desired_revision = $revision;
|
|
|
|
|
$port->last_error_code = substr($code, 0, 64);
|
|
|
|
|
$port->updated_at = $now;
|
|
|
|
|
$port->save();
|
|
|
|
|
$item->state = 'failed';
|
|
|
|
|
$item->last_error_code = substr($code, 0, 64);
|
|
|
|
|
$item->updated_at = $now;
|
|
|
|
|
$item->save();
|
|
|
|
|
}
|
|
|
|
|
$vpc->desired_revision = $revision;
|
|
|
|
|
$vpc->state = 'provisioning';
|
|
|
|
|
$vpc->updated_at = $now;
|
|
|
|
|
$vpc->save();
|
|
|
|
|
$batch->state = 'cleanup';
|
|
|
|
|
$batch->desired_revision = $revision;
|
|
|
|
|
$batch->last_error_code = substr($code, 0, 64);
|
|
|
|
|
$batch->updated_at = $now;
|
|
|
|
|
$batch->save();
|
|
|
|
|
$focus = CloudVpcPort::query()
|
|
|
|
|
->where('id', (int) $items->first()->vpc_port_id)
|
|
|
|
|
->where('active_claim', 1)->lockForUpdate()->first();
|
|
|
|
|
if ($focus === null) {
|
|
|
|
|
throw new CloudError('VPC 加入撤销缺少焦点端口', 409);
|
|
|
|
|
}
|
|
|
|
|
$jobs = CloudVpcPortService::queueBatchProjectionLocked(
|
|
|
|
|
$vpc,
|
|
|
|
|
$focus,
|
|
|
|
|
$revision,
|
|
|
|
|
(int) $batch->id,
|
|
|
|
|
$provenNotAppliedOperationId
|
|
|
|
|
);
|
|
|
|
|
if ($jobs === []) {
|
|
|
|
|
self::finishAttachCleanupLocked($batch, $parent, $vpc);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static function finishAttachCleanupLocked(
|
|
|
|
|
CloudVpcMembershipBatch $batch,
|
|
|
|
|
CloudJob $parent,
|
|
|
|
|
CloudVpc $vpc
|
|
|
|
|
): void {
|
|
|
|
|
$items = self::itemsLocked($batch);
|
|
|
|
|
$now = time();
|
|
|
|
|
$failureCode = 'vpc_batch_attach_route_failed';
|
|
|
|
|
foreach ($items as $item) {
|
|
|
|
|
if ((string) $item->last_error_code !== '') {
|
|
|
|
|
$failureCode = (string) $item->last_error_code;
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
foreach ($items as $item) {
|
|
|
|
|
$port = CloudVpcPort::query()
|
|
|
|
|
->where('id', (int) $item->vpc_port_id)
|
|
|
|
|
->where('active_claim', 1)->lockForUpdate()->first();
|
|
|
|
|
if ($port === null || (string) $port->state !== 'release_pending') {
|
|
|
|
|
throw new CloudError('VPC 加入撤销释放账本不一致', 409);
|
|
|
|
|
}
|
|
|
|
|
$port->state = 'released';
|
|
|
|
|
$port->active_claim = null;
|
|
|
|
|
$port->last_operation_id = null;
|
|
|
|
|
$port->last_error_code = '';
|
|
|
|
|
$port->released_at = $now;
|
|
|
|
|
$port->updated_at = $now;
|
|
|
|
|
$port->save();
|
|
|
|
|
}
|
|
|
|
|
$vpc->state = 'active';
|
|
|
|
|
$vpc->updated_at = $now;
|
|
|
|
|
$vpc->save();
|
|
|
|
|
self::finishLocked(
|
|
|
|
|
$batch,
|
|
|
|
|
$parent,
|
|
|
|
|
false,
|
|
|
|
|
false,
|
|
|
|
|
true,
|
|
|
|
|
$failureCode
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static function releaseAttachPortJobsLocked(
|
|
|
|
|
CloudVpcMembershipBatch $batch,
|
|
|
|
|
CloudVpc $vpc
|
|
|
|
|
@ -630,6 +830,11 @@ final class CloudVpcMembershipBatchService
|
|
|
|
|
'unknown' => 0];
|
|
|
|
|
$jobs = CloudJob::query()->where('vpc_batch_id', (int) $batch->id)
|
|
|
|
|
->whereIn('kind', ['vpc_host_apply', 'vpc_host_remove'])
|
|
|
|
|
->where('request_idempotency_key', 'like',
|
|
|
|
|
CloudVpcPortService::batchProjectionRequestPrefix(
|
|
|
|
|
(int) $batch->id,
|
|
|
|
|
(int) $batch->desired_revision
|
|
|
|
|
) . '%')
|
|
|
|
|
->orderBy('id')->get();
|
|
|
|
|
if ($jobs->isEmpty()) {
|
|
|
|
|
throw new CloudError('VPC 批次宿主机子任务账本缺失', 409);
|
|
|
|
|
@ -656,6 +861,51 @@ final class CloudVpcMembershipBatchService
|
|
|
|
|
return $counts;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static function canRetryAttachCleanup(
|
|
|
|
|
CloudVpcMembershipBatch $batch,
|
|
|
|
|
$items,
|
|
|
|
|
bool $lock
|
|
|
|
|
): bool {
|
|
|
|
|
if ((string) $batch->kind !== 'attach' || $items->isEmpty()) {
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
$portQuery = CloudVpcPort::query()
|
|
|
|
|
->whereIn('id', $items->pluck('vpc_port_id')->all())
|
|
|
|
|
->where('active_claim', 1)->orderBy('id');
|
|
|
|
|
$ports = $lock ? $portQuery->lockForUpdate()->get()
|
|
|
|
|
: $portQuery->get();
|
|
|
|
|
if ($ports->count() !== $items->count()
|
|
|
|
|
|| $ports->contains(static fn ($port): bool =>
|
|
|
|
|
(string) $port->state !== 'release_pending')) {
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
$stateQuery = CloudVpcHostState::query()
|
|
|
|
|
->where('vpc_id', (int) $batch->vpc_id)->orderBy('host_id');
|
|
|
|
|
$states = $lock ? $stateQuery->lockForUpdate()->get()
|
|
|
|
|
: $stateQuery->get();
|
|
|
|
|
foreach ($states as $state) {
|
|
|
|
|
if ((string) $state->sync_state === 'ready') {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
if ((string) $state->sync_state !== 'failed'
|
|
|
|
|
|| !is_string($state->last_operation_id)) {
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
$jobQuery = CloudJob::query()
|
|
|
|
|
->where('operation_id', (string) $state->last_operation_id)
|
|
|
|
|
->where('vpc_batch_id', (int) $batch->id);
|
|
|
|
|
$job = $lock ? $jobQuery->lockForUpdate()->first()
|
|
|
|
|
: $jobQuery->first();
|
|
|
|
|
if ($job === null || (string) $job->phase !== 'terminal'
|
|
|
|
|
|| (string) $job->outcome !== 'failed'
|
|
|
|
|
|| (string) $job->effect_resolution !== 'not_applied'
|
|
|
|
|
|| (int) $job->effect_may_have_applied !== 0) {
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static function newParentJob(int $userId, string $requestKey,
|
|
|
|
|
string $fingerprint, CloudVpc $vpc, string $action,
|
|
|
|
|
array $instanceIds): CloudJob
|
|
|
|
|
@ -807,6 +1057,8 @@ final class CloudVpcMembershipBatchService
|
|
|
|
|
'succeeded' => $items->where('state', 'succeeded')->count(),
|
|
|
|
|
'failed' => $items->where('state', 'failed')->count(),
|
|
|
|
|
'unknown' => $items->where('state', 'unknown')->count(),
|
|
|
|
|
'can_cancel' => (string) $batch->state === 'blocked'
|
|
|
|
|
&& self::canRetryAttachCleanup($batch, $items, false),
|
|
|
|
|
'task_id' => (string) $batch->parent_job_id,
|
|
|
|
|
'status_url' => '/api/v1/tasks/' . (int) $batch->parent_job_id,
|
|
|
|
|
'created_at' => (int) $batch->created_at,
|
|
|
|
|
|